139
140 template <typename T>
141 static inline void store_at(oop base, ptrdiff_t offset, T value) {
142 verify_primitive_decorators<store_mo_decorators>();
143 AccessInternal::store_at<decorators>(base, offset, value);
144 }
145
146 template <typename T>
147 static inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
148 verify_primitive_decorators<atomic_cmpxchg_mo_decorators>();
149 return AccessInternal::atomic_cmpxchg_at<decorators>(new_value, base, offset, compare_value);
150 }
151
152 template <typename T>
153 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
154 verify_primitive_decorators<atomic_xchg_mo_decorators>();
155 return AccessInternal::atomic_xchg_at<decorators>(new_value, base, offset);
156 }
157
158 template <typename T>
159 static inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) {
160 verify_decorators<ARRAYCOPY_DECORATOR_MASK | IN_HEAP |
161 AS_DECORATOR_MASK>();
162 AccessInternal::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
163 }
164
165 // Oop heap accesses
166 static inline AccessInternal::OopLoadAtProxy<decorators> oop_load_at(oop base, ptrdiff_t offset) {
167 verify_heap_oop_decorators<load_mo_decorators>();
168 return AccessInternal::OopLoadAtProxy<decorators>(base, offset);
169 }
170
171 template <typename T>
172 static inline void oop_store_at(oop base, ptrdiff_t offset, T value) {
173 verify_heap_oop_decorators<store_mo_decorators>();
174 typedef typename AccessInternal::OopOrNarrowOop<T>::type OopType;
175 OopType oop_value = value;
176 AccessInternal::store_at<decorators | INTERNAL_VALUE_IS_OOP>(base, offset, oop_value);
177 }
178
179 template <typename T>
180 static inline T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
181 verify_heap_oop_decorators<atomic_cmpxchg_mo_decorators>();
182 typedef typename AccessInternal::OopOrNarrowOop<T>::type OopType;
183 OopType new_oop_value = new_value;
184 OopType compare_oop_value = compare_value;
185 return AccessInternal::atomic_cmpxchg_at<decorators | INTERNAL_VALUE_IS_OOP>(new_oop_value, base, offset, compare_oop_value);
186 }
187
188 template <typename T>
189 static inline T oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
190 verify_heap_oop_decorators<atomic_xchg_mo_decorators>();
191 typedef typename AccessInternal::OopOrNarrowOop<T>::type OopType;
192 OopType new_oop_value = new_value;
193 return AccessInternal::atomic_xchg_at<decorators | INTERNAL_VALUE_IS_OOP>(new_oop_value, base, offset);
194 }
195
196 template <typename T>
197 static inline bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) {
198 verify_decorators<ARRAYCOPY_DECORATOR_MASK | IN_HEAP | AS_DECORATOR_MASK>();
199 return AccessInternal::arraycopy<decorators | INTERNAL_VALUE_IS_OOP>(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
200 }
201
202 // Clone an object from src to dst
203 static inline void clone(oop src, oop dst, size_t size) {
204 verify_decorators<IN_HEAP>();
205 AccessInternal::clone<decorators>(src, dst, size);
206 }
207
208 // Primitive accesses
209 template <typename P>
210 static inline P load(P* addr) {
211 verify_primitive_decorators<load_mo_decorators>();
212 return AccessInternal::load<decorators, P, P>(addr);
213 }
214
215 template <typename P, typename T>
216 static inline void store(P* addr, T value) {
217 verify_primitive_decorators<store_mo_decorators>();
218 AccessInternal::store<decorators>(addr, value);
219 }
276 // Helper for performing raw accesses (knows only of memory ordering
277 // atomicity decorators as well as compressed oops)
278 template <DecoratorSet decorators = INTERNAL_EMPTY>
279 class RawAccess: public Access<AS_RAW | decorators> {};
280
281 // Helper for performing normal accesses on the heap. These accesses
282 // may resolve an accessor on a GC barrier set
283 template <DecoratorSet decorators = INTERNAL_EMPTY>
284 class HeapAccess: public Access<IN_HEAP | decorators> {};
285
286 // Helper for performing normal accesses in roots. These accesses
287 // may resolve an accessor on a GC barrier set
288 template <DecoratorSet decorators = INTERNAL_EMPTY>
289 class RootAccess: public Access<IN_ROOT | decorators> {};
290
291 // Helper for array access.
292 template <DecoratorSet decorators = INTERNAL_EMPTY>
293 class ArrayAccess: public HeapAccess<IN_HEAP_ARRAY | decorators> {
294 public:
295 template <typename T>
296 static inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, arrayOop dst_obj, size_t dst_offset_in_bytes, size_t length) {
297 HeapAccess<decorators>::arraycopy(src_obj, src_offset_in_bytes, (const T*) NULL, dst_obj, dst_offset_in_bytes, (T*) NULL, length);
298 }
299
300 template <typename T>
301 static inline void arraycopy_to_native(arrayOop src_obj, size_t src_offset_in_bytes, T* dst, size_t length) {
302 HeapAccess<decorators>::arraycopy(src_obj, src_offset_in_bytes, (const T*) NULL, NULL, 0, dst, length);
303 }
304
305 template <typename T>
306 static inline void arraycopy_from_native(const T* src, arrayOop dst_obj, size_t dst_offset_in_bytes, size_t length) {
307 HeapAccess<decorators>::arraycopy(NULL, 0, src, dst_obj, dst_offset_in_bytes, (T*) NULL, length);
308 }
309
310 template <typename T>
311 static inline bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, arrayOop dst_obj, size_t dst_offset_in_bytes, size_t length) {
312 return HeapAccess<decorators>::oop_arraycopy(src_obj, src_offset_in_bytes, (const T*) NULL, dst_obj, dst_offset_in_bytes, (T*) NULL, length);
313 }
314
315 template <typename T>
316 static inline bool oop_arraycopy_raw(T* src, T* dst, size_t length) {
317 return HeapAccess<decorators>::oop_arraycopy(NULL, 0, src, NULL, 0, dst, length);
318 }
319
320 };
321
322 template <DecoratorSet decorators>
323 template <DecoratorSet expected_decorators>
324 void Access<decorators>::verify_decorators() {
325 STATIC_ASSERT((~expected_decorators & decorators) == 0); // unexpected decorator used
326 const DecoratorSet barrier_strength_decorators = decorators & AS_DECORATOR_MASK;
327 STATIC_ASSERT(barrier_strength_decorators == 0 || ( // make sure barrier strength decorators are disjoint if set
328 (barrier_strength_decorators ^ AS_NO_KEEPALIVE) == 0 ||
329 (barrier_strength_decorators ^ AS_DEST_NOT_INITIALIZED) == 0 ||
330 (barrier_strength_decorators ^ AS_RAW) == 0 ||
331 (barrier_strength_decorators ^ AS_NORMAL) == 0
332 ));
333 const DecoratorSet ref_strength_decorators = decorators & ON_DECORATOR_MASK;
334 STATIC_ASSERT(ref_strength_decorators == 0 || ( // make sure ref strength decorators are disjoint if set
335 (ref_strength_decorators ^ ON_STRONG_OOP_REF) == 0 ||
336 (ref_strength_decorators ^ ON_WEAK_OOP_REF) == 0 ||
337 (ref_strength_decorators ^ ON_PHANTOM_OOP_REF) == 0 ||
|
139
140 template <typename T>
141 static inline void store_at(oop base, ptrdiff_t offset, T value) {
142 verify_primitive_decorators<store_mo_decorators>();
143 AccessInternal::store_at<decorators>(base, offset, value);
144 }
145
146 template <typename T>
147 static inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
148 verify_primitive_decorators<atomic_cmpxchg_mo_decorators>();
149 return AccessInternal::atomic_cmpxchg_at<decorators>(new_value, base, offset, compare_value);
150 }
151
152 template <typename T>
153 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
154 verify_primitive_decorators<atomic_xchg_mo_decorators>();
155 return AccessInternal::atomic_xchg_at<decorators>(new_value, base, offset);
156 }
157
158 template <typename T>
159 static inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
160 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
161 size_t length) {
162 verify_decorators<ARRAYCOPY_DECORATOR_MASK | IN_HEAP | IN_HEAP_ARRAY |
163 AS_DECORATOR_MASK>();
164 AccessInternal::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
165 dst_obj, dst_offset_in_bytes, dst_raw,
166 length);
167 }
168
169 // Oop heap accesses
170 static inline AccessInternal::OopLoadAtProxy<decorators> oop_load_at(oop base, ptrdiff_t offset) {
171 verify_heap_oop_decorators<load_mo_decorators>();
172 return AccessInternal::OopLoadAtProxy<decorators>(base, offset);
173 }
174
175 template <typename T>
176 static inline void oop_store_at(oop base, ptrdiff_t offset, T value) {
177 verify_heap_oop_decorators<store_mo_decorators>();
178 typedef typename AccessInternal::OopOrNarrowOop<T>::type OopType;
179 OopType oop_value = value;
180 AccessInternal::store_at<decorators | INTERNAL_VALUE_IS_OOP>(base, offset, oop_value);
181 }
182
183 template <typename T>
184 static inline T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
185 verify_heap_oop_decorators<atomic_cmpxchg_mo_decorators>();
186 typedef typename AccessInternal::OopOrNarrowOop<T>::type OopType;
187 OopType new_oop_value = new_value;
188 OopType compare_oop_value = compare_value;
189 return AccessInternal::atomic_cmpxchg_at<decorators | INTERNAL_VALUE_IS_OOP>(new_oop_value, base, offset, compare_oop_value);
190 }
191
192 template <typename T>
193 static inline T oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
194 verify_heap_oop_decorators<atomic_xchg_mo_decorators>();
195 typedef typename AccessInternal::OopOrNarrowOop<T>::type OopType;
196 OopType new_oop_value = new_value;
197 return AccessInternal::atomic_xchg_at<decorators | INTERNAL_VALUE_IS_OOP>(new_oop_value, base, offset);
198 }
199
200 template <typename T>
201 static inline bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
202 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
203 size_t length) {
204 verify_decorators<ARRAYCOPY_DECORATOR_MASK | IN_HEAP | IN_HEAP_ARRAY |
205 AS_DECORATOR_MASK>();
206 return AccessInternal::arraycopy<decorators | INTERNAL_VALUE_IS_OOP>(src_obj, src_offset_in_bytes, src_raw,
207 dst_obj, dst_offset_in_bytes, dst_raw,
208 length);
209 }
210
211 // Clone an object from src to dst
212 static inline void clone(oop src, oop dst, size_t size) {
213 verify_decorators<IN_HEAP>();
214 AccessInternal::clone<decorators>(src, dst, size);
215 }
216
217 // Primitive accesses
218 template <typename P>
219 static inline P load(P* addr) {
220 verify_primitive_decorators<load_mo_decorators>();
221 return AccessInternal::load<decorators, P, P>(addr);
222 }
223
224 template <typename P, typename T>
225 static inline void store(P* addr, T value) {
226 verify_primitive_decorators<store_mo_decorators>();
227 AccessInternal::store<decorators>(addr, value);
228 }
285 // Helper for performing raw accesses (knows only of memory ordering
286 // atomicity decorators as well as compressed oops)
287 template <DecoratorSet decorators = INTERNAL_EMPTY>
288 class RawAccess: public Access<AS_RAW | decorators> {};
289
290 // Helper for performing normal accesses on the heap. These accesses
291 // may resolve an accessor on a GC barrier set
292 template <DecoratorSet decorators = INTERNAL_EMPTY>
293 class HeapAccess: public Access<IN_HEAP | decorators> {};
294
295 // Helper for performing normal accesses in roots. These accesses
296 // may resolve an accessor on a GC barrier set
297 template <DecoratorSet decorators = INTERNAL_EMPTY>
298 class RootAccess: public Access<IN_ROOT | decorators> {};
299
300 // Helper for array access.
301 template <DecoratorSet decorators = INTERNAL_EMPTY>
302 class ArrayAccess: public HeapAccess<IN_HEAP_ARRAY | decorators> {
303 public:
304 template <typename T>
305 static inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes,
306 arrayOop dst_obj, size_t dst_offset_in_bytes,
307 size_t length) {
308 HeapAccess<decorators | IN_HEAP_ARRAY>::arraycopy(src_obj, src_offset_in_bytes, (const T*) NULL,
309 dst_obj, dst_offset_in_bytes, (T*) NULL,
310 length);
311 }
312
313 template <typename T>
314 static inline void arraycopy_to_native(arrayOop src_obj, size_t src_offset_in_bytes, T* dst, size_t length) {
315 HeapAccess<decorators | IN_HEAP_ARRAY>::arraycopy(src_obj, src_offset_in_bytes, (const T*) NULL,
316 NULL, 0, dst,
317 length);
318 }
319
320 template <typename T>
321 static inline void arraycopy_from_native(const T* src, arrayOop dst_obj, size_t dst_offset_in_bytes, size_t length) {
322 HeapAccess<decorators | IN_HEAP_ARRAY>::arraycopy(NULL, 0, src, dst_obj, dst_offset_in_bytes, (T*) NULL, length);
323 }
324
325 template <typename T>
326 static inline bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes,
327 arrayOop dst_obj, size_t dst_offset_in_bytes,
328 size_t length) {
329 return HeapAccess<decorators | IN_HEAP_ARRAY>::oop_arraycopy(src_obj, src_offset_in_bytes, (const T*) NULL,
330 dst_obj, dst_offset_in_bytes, (T*) NULL, length);
331 }
332
333 template <typename T>
334 static inline bool oop_arraycopy_raw(T* src, T* dst, size_t length) {
335 return HeapAccess<decorators | IN_HEAP_ARRAY>::oop_arraycopy(NULL, 0, src, NULL, 0, dst, length);
336 }
337
338 };
339
340 template <DecoratorSet decorators>
341 template <DecoratorSet expected_decorators>
342 void Access<decorators>::verify_decorators() {
343 STATIC_ASSERT((~expected_decorators & decorators) == 0); // unexpected decorator used
344 const DecoratorSet barrier_strength_decorators = decorators & AS_DECORATOR_MASK;
345 STATIC_ASSERT(barrier_strength_decorators == 0 || ( // make sure barrier strength decorators are disjoint if set
346 (barrier_strength_decorators ^ AS_NO_KEEPALIVE) == 0 ||
347 (barrier_strength_decorators ^ AS_DEST_NOT_INITIALIZED) == 0 ||
348 (barrier_strength_decorators ^ AS_RAW) == 0 ||
349 (barrier_strength_decorators ^ AS_NORMAL) == 0
350 ));
351 const DecoratorSet ref_strength_decorators = decorators & ON_DECORATOR_MASK;
352 STATIC_ASSERT(ref_strength_decorators == 0 || ( // make sure ref strength decorators are disjoint if set
353 (ref_strength_decorators ^ ON_STRONG_OOP_REF) == 0 ||
354 (ref_strength_decorators ^ ON_WEAK_OOP_REF) == 0 ||
355 (ref_strength_decorators ^ ON_PHANTOM_OOP_REF) == 0 ||
|