92 // locking to support wide atomics or not.
93 template <typename T>
94 #ifdef SUPPORTS_NATIVE_CX8
95 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
96 #else
97 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
98 #endif
99
100 template <DecoratorSet decorators, typename T>
101 struct AccessFunctionTypes {
102 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
103 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
104 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
105 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
106
107 typedef T (*load_func_t)(void* addr);
108 typedef void (*store_func_t)(void* addr, T value);
109 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
110 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
111
112 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
113 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
114 size_t length);
115 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
116 typedef oop (*resolve_func_t)(oop obj);
117 typedef bool (*equals_func_t)(oop o1, oop o2);
118 };
119
120 template <DecoratorSet decorators>
121 struct AccessFunctionTypes<decorators, void> {
122 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, const void* src,
123 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
124 size_t length);
125 };
126
127 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
128
129 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
130 template <DecoratorSet decorators, typename T> \
131 struct AccessFunction<decorators, T, bt>: AllStatic{ \
132 typedef typename AccessFunctionTypes<decorators, T>::func type; \
133 }
134 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
|
92 // locking to support wide atomics or not.
93 template <typename T>
94 #ifdef SUPPORTS_NATIVE_CX8
95 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
96 #else
97 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
98 #endif
99
100 template <DecoratorSet decorators, typename T>
101 struct AccessFunctionTypes {
102 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
103 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
104 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
105 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
106
107 typedef T (*load_func_t)(void* addr);
108 typedef void (*store_func_t)(void* addr, T value);
109 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
110 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
111
112 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
113 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
114 size_t length);
115 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
116 typedef oop (*resolve_func_t)(oop obj);
117 typedef bool (*equals_func_t)(oop o1, oop o2);
118 };
119
120 template <DecoratorSet decorators>
121 struct AccessFunctionTypes<decorators, void> {
122 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
123 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
124 size_t length);
125 };
126
127 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
128
129 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
130 template <DecoratorSet decorators, typename T> \
131 struct AccessFunction<decorators, T, bt>: AllStatic{ \
132 typedef typename AccessFunctionTypes<decorators, T>::func type; \
133 }
134 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
|
242 static typename EnableIf<
243 HasDecorator<ds, MO_ACQUIRE>::value, T>::type
244 load_internal(void* addr);
245
246 template <DecoratorSet ds, typename T>
247 static typename EnableIf<
248 HasDecorator<ds, MO_RELAXED>::value, T>::type
249 load_internal(void* addr);
250
251 template <DecoratorSet ds, typename T>
252 static inline typename EnableIf<
253 HasDecorator<ds, MO_VOLATILE>::value, T>::type
254 load_internal(void* addr) {
255 return *reinterpret_cast<const volatile T*>(addr);
256 }
257
258 template <DecoratorSet ds, typename T>
259 static inline typename EnableIf<
260 HasDecorator<ds, MO_UNORDERED>::value, T>::type
261 load_internal(void* addr) {
262 return *reinterpret_cast<const T*>(addr);
263 }
264
265 template <DecoratorSet ds, typename T>
266 static typename EnableIf<
267 HasDecorator<ds, MO_SEQ_CST>::value>::type
268 store_internal(void* addr, T value);
269
270 template <DecoratorSet ds, typename T>
271 static typename EnableIf<
272 HasDecorator<ds, MO_RELEASE>::value>::type
273 store_internal(void* addr, T value);
274
275 template <DecoratorSet ds, typename T>
276 static typename EnableIf<
277 HasDecorator<ds, MO_RELAXED>::value>::type
278 store_internal(void* addr, T value);
279
280 template <DecoratorSet ds, typename T>
281 static inline typename EnableIf<
|
242 static typename EnableIf<
243 HasDecorator<ds, MO_ACQUIRE>::value, T>::type
244 load_internal(void* addr);
245
246 template <DecoratorSet ds, typename T>
247 static typename EnableIf<
248 HasDecorator<ds, MO_RELAXED>::value, T>::type
249 load_internal(void* addr);
250
251 template <DecoratorSet ds, typename T>
252 static inline typename EnableIf<
253 HasDecorator<ds, MO_VOLATILE>::value, T>::type
254 load_internal(void* addr) {
255 return *reinterpret_cast<const volatile T*>(addr);
256 }
257
258 template <DecoratorSet ds, typename T>
259 static inline typename EnableIf<
260 HasDecorator<ds, MO_UNORDERED>::value, T>::type
261 load_internal(void* addr) {
262 return *reinterpret_cast<T*>(addr);
263 }
264
265 template <DecoratorSet ds, typename T>
266 static typename EnableIf<
267 HasDecorator<ds, MO_SEQ_CST>::value>::type
268 store_internal(void* addr, T value);
269
270 template <DecoratorSet ds, typename T>
271 static typename EnableIf<
272 HasDecorator<ds, MO_RELEASE>::value>::type
273 store_internal(void* addr, T value);
274
275 template <DecoratorSet ds, typename T>
276 static typename EnableIf<
277 HasDecorator<ds, MO_RELAXED>::value>::type
278 store_internal(void* addr, T value);
279
280 template <DecoratorSet ds, typename T>
281 static inline typename EnableIf<
|
339 static inline void store(void* addr, T value) {
340 store_internal<decorators>(addr, value);
341 }
342
343 template <typename T>
344 static inline T load(void* addr) {
345 return load_internal<decorators, T>(addr);
346 }
347
348 template <typename T>
349 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
350 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
351 }
352
353 template <typename T>
354 static inline T atomic_xchg(T new_value, void* addr) {
355 return atomic_xchg_maybe_locked<decorators>(new_value, addr);
356 }
357
358 template <typename T>
359 static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
360 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
361 size_t length);
362
363 template <typename T>
364 static void oop_store(void* addr, T value);
365 template <typename T>
366 static void oop_store_at(oop base, ptrdiff_t offset, T value);
367
368 template <typename T>
369 static T oop_load(void* addr);
370 template <typename T>
371 static T oop_load_at(oop base, ptrdiff_t offset);
372
373 template <typename T>
374 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
375 template <typename T>
376 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
377
378 template <typename T>
|
339 static inline void store(void* addr, T value) {
340 store_internal<decorators>(addr, value);
341 }
342
343 template <typename T>
344 static inline T load(void* addr) {
345 return load_internal<decorators, T>(addr);
346 }
347
348 template <typename T>
349 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
350 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
351 }
352
353 template <typename T>
354 static inline T atomic_xchg(T new_value, void* addr) {
355 return atomic_xchg_maybe_locked<decorators>(new_value, addr);
356 }
357
358 template <typename T>
359 static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
360 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
361 size_t length);
362
363 template <typename T>
364 static void oop_store(void* addr, T value);
365 template <typename T>
366 static void oop_store_at(oop base, ptrdiff_t offset, T value);
367
368 template <typename T>
369 static T oop_load(void* addr);
370 template <typename T>
371 static T oop_load_at(oop base, ptrdiff_t offset);
372
373 template <typename T>
374 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
375 template <typename T>
376 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
377
378 template <typename T>
|
384 static void store_at(oop base, ptrdiff_t offset, T value) {
385 store(field_addr(base, offset), value);
386 }
387
388 template <typename T>
389 static T load_at(oop base, ptrdiff_t offset) {
390 return load<T>(field_addr(base, offset));
391 }
392
393 template <typename T>
394 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
395 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
396 }
397
398 template <typename T>
399 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
400 return atomic_xchg(new_value, field_addr(base, offset));
401 }
402
403 template <typename T>
404 static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
405 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
406 size_t length);
407
408 static void clone(oop src, oop dst, size_t size);
409
410 static oop resolve(oop obj) { return obj; }
411
412 static bool equals(oop o1, oop o2) { return o1 == o2; }
413 };
414
415 // Below is the implementation of the first 4 steps of the template pipeline:
416 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
417 // and sets default decorators to sensible values.
418 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
419 // multiple types. The P type of the address and T type of the value must
420 // match.
421 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
422 // avoided, and in that case avoids it (calling raw accesses or
423 // primitive accesses in a build that does not require primitive GC barriers)
|
384 static void store_at(oop base, ptrdiff_t offset, T value) {
385 store(field_addr(base, offset), value);
386 }
387
388 template <typename T>
389 static T load_at(oop base, ptrdiff_t offset) {
390 return load<T>(field_addr(base, offset));
391 }
392
393 template <typename T>
394 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
395 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
396 }
397
398 template <typename T>
399 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
400 return atomic_xchg(new_value, field_addr(base, offset));
401 }
402
403 template <typename T>
404 static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
405 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
406 size_t length);
407
408 static void clone(oop src, oop dst, size_t size);
409
410 static oop resolve(oop obj) { return obj; }
411
412 static bool equals(oop o1, oop o2) { return o1 == o2; }
413 };
414
415 // Below is the implementation of the first 4 steps of the template pipeline:
416 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
417 // and sets default decorators to sensible values.
418 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
419 // multiple types. The P type of the address and T type of the value must
420 // match.
421 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
422 // avoided, and in that case avoids it (calling raw accesses or
423 // primitive accesses in a build that does not require primitive GC barriers)
|
549 }
550 };
551
552 template <DecoratorSet decorators, typename T>
553 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
554 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
555 static func_t _atomic_xchg_at_func;
556
557 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
558
559 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
560 return _atomic_xchg_at_func(new_value, base, offset);
561 }
562 };
563
564 template <DecoratorSet decorators, typename T>
565 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
566 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
567 static func_t _arraycopy_func;
568
569 static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
570 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
571 size_t length);
572
573 static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
574 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
575 size_t length) {
576 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
577 dst_obj, dst_offset_in_bytes, dst_raw,
578 length);
579 }
580 };
581
582 template <DecoratorSet decorators, typename T>
583 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
584 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
585 static func_t _clone_func;
586
587 static void clone_init(oop src, oop dst, size_t size);
588
589 static inline void clone(oop src, oop dst, size_t size) {
590 _clone_func(src, dst, size);
591 }
592 };
|
549 }
550 };
551
552 template <DecoratorSet decorators, typename T>
553 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
554 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
555 static func_t _atomic_xchg_at_func;
556
557 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
558
559 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
560 return _atomic_xchg_at_func(new_value, base, offset);
561 }
562 };
563
564 template <DecoratorSet decorators, typename T>
565 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
566 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
567 static func_t _arraycopy_func;
568
569 static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
570 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
571 size_t length);
572
573 static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
574 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
575 size_t length) {
576 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
577 dst_obj, dst_offset_in_bytes, dst_raw,
578 length);
579 }
580 };
581
582 template <DecoratorSet decorators, typename T>
583 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
584 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
585 static func_t _clone_func;
586
587 static void clone_init(oop src, oop dst, size_t size);
588
589 static inline void clone(oop src, oop dst, size_t size) {
590 _clone_func(src, dst, size);
591 }
592 };
|
896 HasDecorator<decorators, AS_RAW>::value, T>::type
897 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
898 return atomic_xchg<decorators>(new_value, field_addr(base, offset));
899 }
900
901 template <DecoratorSet decorators, typename T>
902 inline static typename EnableIf<
903 !HasDecorator<decorators, AS_RAW>::value, T>::type
904 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
905 if (is_hardwired_primitive<decorators>()) {
906 const DecoratorSet expanded_decorators = decorators | AS_RAW;
907 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
908 } else {
909 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
910 }
911 }
912
913 template <DecoratorSet decorators, typename T>
914 inline static typename EnableIf<
915 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
916 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
917 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
918 size_t length) {
919 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
920 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
921 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
922 dst_obj, dst_offset_in_bytes, dst_raw,
923 length);
924 } else {
925 return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
926 dst_obj, dst_offset_in_bytes, dst_raw,
927 length);
928 }
929 }
930
931 template <DecoratorSet decorators, typename T>
932 inline static typename EnableIf<
933 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
934 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
935 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
936 size_t length) {
937 if (UseCompressedOops) {
938 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
939 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
940 dst_obj, dst_offset_in_bytes, dst_raw,
941 length);
942 } else {
943 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
944 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
945 dst_obj, dst_offset_in_bytes, dst_raw,
946 length);
947 }
948 }
949
950 template <DecoratorSet decorators, typename T>
951 inline static typename EnableIf<
952 !HasDecorator<decorators, AS_RAW>::value, bool>::type
953 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
954 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
955 size_t length) {
956 if (is_hardwired_primitive<decorators>()) {
957 const DecoratorSet expanded_decorators = decorators | AS_RAW;
958 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
959 dst_obj, dst_offset_in_bytes, dst_raw,
960 length);
961 } else {
962 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
963 dst_obj, dst_offset_in_bytes, dst_raw,
964 length);
965 }
966 }
967
968 template <DecoratorSet decorators>
969 inline static typename EnableIf<
970 HasDecorator<decorators, AS_RAW>::value>::type
971 clone(oop src, oop dst, size_t size) {
972 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
|
896 HasDecorator<decorators, AS_RAW>::value, T>::type
897 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
898 return atomic_xchg<decorators>(new_value, field_addr(base, offset));
899 }
900
901 template <DecoratorSet decorators, typename T>
902 inline static typename EnableIf<
903 !HasDecorator<decorators, AS_RAW>::value, T>::type
904 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
905 if (is_hardwired_primitive<decorators>()) {
906 const DecoratorSet expanded_decorators = decorators | AS_RAW;
907 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
908 } else {
909 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
910 }
911 }
912
913 template <DecoratorSet decorators, typename T>
914 inline static typename EnableIf<
915 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
916 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
917 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
918 size_t length) {
919 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
920 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
921 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
922 dst_obj, dst_offset_in_bytes, dst_raw,
923 length);
924 } else {
925 return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
926 dst_obj, dst_offset_in_bytes, dst_raw,
927 length);
928 }
929 }
930
931 template <DecoratorSet decorators, typename T>
932 inline static typename EnableIf<
933 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
934 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
935 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
936 size_t length) {
937 if (UseCompressedOops) {
938 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
939 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
940 dst_obj, dst_offset_in_bytes, dst_raw,
941 length);
942 } else {
943 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
944 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
945 dst_obj, dst_offset_in_bytes, dst_raw,
946 length);
947 }
948 }
949
950 template <DecoratorSet decorators, typename T>
951 inline static typename EnableIf<
952 !HasDecorator<decorators, AS_RAW>::value, bool>::type
953 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
954 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
955 size_t length) {
956 if (is_hardwired_primitive<decorators>()) {
957 const DecoratorSet expanded_decorators = decorators | AS_RAW;
958 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
959 dst_obj, dst_offset_in_bytes, dst_raw,
960 length);
961 } else {
962 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
963 dst_obj, dst_offset_in_bytes, dst_raw,
964 length);
965 }
966 }
967
968 template <DecoratorSet decorators>
969 inline static typename EnableIf<
970 HasDecorator<decorators, AS_RAW>::value>::type
971 clone(oop src, oop dst, size_t size) {
972 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
|
1106
1107 template <DecoratorSet decorators, typename T>
1108 inline T load_reduce_types(T* addr) {
1109 return PreRuntimeDispatch::load<decorators, T>(addr);
1110 }
1111
1112 template <DecoratorSet decorators, typename T>
1113 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1114 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1115 INTERNAL_RT_USE_COMPRESSED_OOPS;
1116 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1117 }
1118
1119 template <DecoratorSet decorators, typename T>
1120 inline oop load_reduce_types(HeapWord* addr) {
1121 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1122 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1123 }
1124
1125 template <DecoratorSet decorators, typename T>
1126 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1127 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1128 size_t length) {
1129 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1130 dst_obj, dst_offset_in_bytes, dst_raw,
1131 length);
1132 }
1133
1134 template <DecoratorSet decorators>
1135 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, const HeapWord* src_raw,
1136 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1137 size_t length) {
1138 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1139 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1140 dst_obj, dst_offset_in_bytes, dst_raw,
1141 length);
1142 }
1143
1144 template <DecoratorSet decorators>
1145 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, const narrowOop* src_raw,
1146 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1147 size_t length) {
1148 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1149 INTERNAL_RT_USE_COMPRESSED_OOPS;
1150 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1151 dst_obj, dst_offset_in_bytes, dst_raw,
1152 length);
1153 }
1154
1155 // Step 1: Set default decorators. This step remembers if a type was volatile
1156 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1157 // memory ordering is set for the access, and the implied decorator rules
1158 // are applied to select sensible defaults for decorators that have not been
1159 // explicitly set. For example, default object referent strength is set to strong.
1160 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1161 // and references from the types). This step also perform some type verification
1162 // that the passed in types make sense.
1163
1164 template <DecoratorSet decorators, typename T>
|
1106
1107 template <DecoratorSet decorators, typename T>
1108 inline T load_reduce_types(T* addr) {
1109 return PreRuntimeDispatch::load<decorators, T>(addr);
1110 }
1111
1112 template <DecoratorSet decorators, typename T>
1113 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1114 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1115 INTERNAL_RT_USE_COMPRESSED_OOPS;
1116 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1117 }
1118
1119 template <DecoratorSet decorators, typename T>
1120 inline oop load_reduce_types(HeapWord* addr) {
1121 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1122 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1123 }
1124
1125 template <DecoratorSet decorators, typename T>
1126 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1127 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1128 size_t length) {
1129 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1130 dst_obj, dst_offset_in_bytes, dst_raw,
1131 length);
1132 }
1133
1134 template <DecoratorSet decorators>
1135 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1136 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1137 size_t length) {
1138 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1139 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1140 dst_obj, dst_offset_in_bytes, dst_raw,
1141 length);
1142 }
1143
1144 template <DecoratorSet decorators>
1145 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1146 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1147 size_t length) {
1148 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1149 INTERNAL_RT_USE_COMPRESSED_OOPS;
1150 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1151 dst_obj, dst_offset_in_bytes, dst_raw,
1152 length);
1153 }
1154
1155 // Step 1: Set default decorators. This step remembers if a type was volatile
1156 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1157 // memory ordering is set for the access, and the implied decorator rules
1158 // are applied to select sensible defaults for decorators that have not been
1159 // explicitly set. For example, default object referent strength is set to strong.
1160 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1161 // and references from the types). This step also perform some type verification
1162 // that the passed in types make sense.
1163
1164 template <DecoratorSet decorators, typename T>
|
1273 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1274 verify_types<decorators, T>();
1275 typedef typename Decay<T>::type DecayedT;
1276 DecayedT new_decayed_value = new_value;
1277 // atomic_xchg is only available in SEQ_CST flavour.
1278 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1279 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1280 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value;
1281 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1282 }
1283
1284 template <DecoratorSet decorators, typename T>
1285 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1286 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1287 size_t length) {
1288 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1289 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1290 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1291 typedef typename Decay<T>::type DecayedT;
1292 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IN_HEAP_ARRAY | IN_HEAP>::value;
1293 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<const DecayedT*>(src_raw),
1294 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1295 length);
1296 }
1297
1298 template <DecoratorSet decorators>
1299 inline void clone(oop src, oop dst, size_t size) {
1300 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1301 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1302 }
1303
1304 template <DecoratorSet decorators>
1305 inline oop resolve(oop obj) {
1306 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1307 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1308 }
1309
1310 template <DecoratorSet decorators>
1311 inline bool equals(oop o1, oop o2) {
1312 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
|
1273 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1274 verify_types<decorators, T>();
1275 typedef typename Decay<T>::type DecayedT;
1276 DecayedT new_decayed_value = new_value;
1277 // atomic_xchg is only available in SEQ_CST flavour.
1278 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1279 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1280 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value;
1281 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1282 }
1283
1284 template <DecoratorSet decorators, typename T>
1285 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1286 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1287 size_t length) {
1288 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1289 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1290 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1291 typedef typename Decay<T>::type DecayedT;
1292 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IN_HEAP_ARRAY | IN_HEAP>::value;
1293 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1294 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1295 length);
1296 }
1297
1298 template <DecoratorSet decorators>
1299 inline void clone(oop src, oop dst, size_t size) {
1300 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1301 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1302 }
1303
1304 template <DecoratorSet decorators>
1305 inline oop resolve(oop obj) {
1306 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1307 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1308 }
1309
1310 template <DecoratorSet decorators>
1311 inline bool equals(oop o1, oop o2) {
1312 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
|