93 // locking to support wide atomics or not.
94 template <typename T>
95 #ifdef SUPPORTS_NATIVE_CX8
96 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
97 #else
98 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
99 #endif
100
101 template <DecoratorSet decorators, typename T>
102 struct AccessFunctionTypes {
103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
106 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
107
108 typedef T (*load_func_t)(void* addr);
109 typedef void (*store_func_t)(void* addr, T value);
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
112
113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length);
114 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
115 typedef oop (*resolve_func_t)(oop obj);
116 typedef bool (*equals_func_t)(oop o1, oop o2);
117 };
118
119 template <DecoratorSet decorators>
120 struct AccessFunctionTypes<decorators, void> {
121 typedef bool (*arraycopy_func_t)(arrayOop src_obj, arrayOop dst_obj, void* src, void* dst, size_t length);
122 };
123
124 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
125
126 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
127 template <DecoratorSet decorators, typename T> \
128 struct AccessFunction<decorators, T, bt>: AllStatic{ \
129 typedef typename AccessFunctionTypes<decorators, T>::func type; \
130 }
131 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
132 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
133 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
134 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_RESOLVE, resolve_func_t);
336 static inline void store(void* addr, T value) {
337 store_internal<decorators>(addr, value);
338 }
339
340 template <typename T>
341 static inline T load(void* addr) {
342 return load_internal<decorators, T>(addr);
343 }
344
345 template <typename T>
346 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
347 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
348 }
349
350 template <typename T>
351 static inline T atomic_xchg(T new_value, void* addr) {
352 return atomic_xchg_maybe_locked<decorators>(new_value, addr);
353 }
354
355 template <typename T>
356 static bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length);
357
358 template <typename T>
359 static void oop_store(void* addr, T value);
360 template <typename T>
361 static void oop_store_at(oop base, ptrdiff_t offset, T value);
362
363 template <typename T>
364 static T oop_load(void* addr);
365 template <typename T>
366 static T oop_load_at(oop base, ptrdiff_t offset);
367
368 template <typename T>
369 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
370 template <typename T>
371 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
372
373 template <typename T>
374 static T oop_atomic_xchg(T new_value, void* addr);
375 template <typename T>
376 static T oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset);
379 static void store_at(oop base, ptrdiff_t offset, T value) {
380 store(field_addr(base, offset), value);
381 }
382
383 template <typename T>
384 static T load_at(oop base, ptrdiff_t offset) {
385 return load<T>(field_addr(base, offset));
386 }
387
388 template <typename T>
389 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
390 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
391 }
392
393 template <typename T>
394 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
395 return atomic_xchg(new_value, field_addr(base, offset));
396 }
397
398 template <typename T>
399 static bool oop_arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length);
400
401 static void clone(oop src, oop dst, size_t size);
402
403 static oop resolve(oop obj) { return obj; }
404
405 static bool equals(oop o1, oop o2) { return o1 == o2; }
406 };
407
408 // Below is the implementation of the first 4 steps of the template pipeline:
409 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
410 // and sets default decorators to sensible values.
411 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
412 // multiple types. The P type of the address and T type of the value must
413 // match.
414 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
415 // avoided, and in that case avoids it (calling raw accesses or
416 // primitive accesses in a build that does not require primitive GC barriers)
417 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
418 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers
419 // to the access.
542 }
543 };
544
545 template <DecoratorSet decorators, typename T>
546 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
547 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
548 static func_t _atomic_xchg_at_func;
549
550 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
551
552 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
553 return _atomic_xchg_at_func(new_value, base, offset);
554 }
555 };
556
557 template <DecoratorSet decorators, typename T>
558 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
559 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
560 static func_t _arraycopy_func;
561
562 static bool arraycopy_init(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length);
563
564 static inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
565 return _arraycopy_func(src_obj, dst_obj, src, dst, length);
566 }
567 };
568
569 template <DecoratorSet decorators, typename T>
570 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
571 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
572 static func_t _clone_func;
573
574 static void clone_init(oop src, oop dst, size_t size);
575
576 static inline void clone(oop src, oop dst, size_t size) {
577 _clone_func(src, dst, size);
578 }
579 };
580
581 template <DecoratorSet decorators, typename T>
582 struct RuntimeDispatch<decorators, T, BARRIER_RESOLVE>: AllStatic {
583 typedef typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type func_t;
584 static func_t _resolve_func;
585
883 HasDecorator<decorators, AS_RAW>::value, T>::type
884 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
885 return atomic_xchg<decorators>(new_value, field_addr(base, offset));
886 }
887
888 template <DecoratorSet decorators, typename T>
889 inline static typename EnableIf<
890 !HasDecorator<decorators, AS_RAW>::value, T>::type
891 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
892 if (is_hardwired_primitive<decorators>()) {
893 const DecoratorSet expanded_decorators = decorators | AS_RAW;
894 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
895 } else {
896 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
897 }
898 }
899
900 template <DecoratorSet decorators, typename T>
901 inline static typename EnableIf<
902 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
903 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
904 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
905 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
906 return Raw::oop_arraycopy(src_obj, dst_obj, src, dst, length);
907 } else {
908 return Raw::arraycopy(src_obj, dst_obj, src, dst, length);
909 }
910 }
911
912 template <DecoratorSet decorators, typename T>
913 inline static typename EnableIf<
914 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
915 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
916 if (UseCompressedOops) {
917 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
918 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length);
919 } else {
920 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
921 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length);
922 }
923 }
924
925 template <DecoratorSet decorators, typename T>
926 inline static typename EnableIf<
927 !HasDecorator<decorators, AS_RAW>::value, bool>::type
928 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
929 if (is_hardwired_primitive<decorators>()) {
930 const DecoratorSet expanded_decorators = decorators | AS_RAW;
931 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length);
932 } else {
933 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, dst_obj, src, dst, length);
934 }
935 }
936
937 template <DecoratorSet decorators>
938 inline static typename EnableIf<
939 HasDecorator<decorators, AS_RAW>::value>::type
940 clone(oop src, oop dst, size_t size) {
941 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
942 Raw::clone(src, dst, size);
943 }
944
945 template <DecoratorSet decorators>
946 inline static typename EnableIf<
947 !HasDecorator<decorators, AS_RAW>::value>::type
948 clone(oop src, oop dst, size_t size) {
949 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
950 }
951
952 template <DecoratorSet decorators>
953 inline static typename EnableIf<
1100
1101 template <DecoratorSet decorators, typename T>
1102 inline T load_reduce_types(T* addr) {
1103 return PreRuntimeDispatch::load<decorators, T>(addr);
1104 }
1105
1106 template <DecoratorSet decorators, typename T>
1107 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1108 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1109 INTERNAL_RT_USE_COMPRESSED_OOPS;
1110 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1111 }
1112
1113 template <DecoratorSet decorators, typename T>
1114 inline oop load_reduce_types(HeapWord* addr) {
1115 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1116 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1117 }
1118
1119 template <DecoratorSet decorators, typename T>
1120 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
1121 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, dst_obj, src, dst, length);
1122 }
1123
1124 template <DecoratorSet decorators>
1125 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, HeapWord* src, HeapWord* dst, size_t length) {
1126 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1127 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length);
1128 }
1129
1130 template <DecoratorSet decorators>
1131 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, narrowOop* src, narrowOop* dst, size_t length) {
1132 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1133 INTERNAL_RT_USE_COMPRESSED_OOPS;
1134 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length);
1135 }
1136
1137 // Step 1: Set default decorators. This step remembers if a type was volatile
1138 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1139 // memory ordering is set for the access, and the implied decorator rules
1140 // are applied to select sensible defaults for decorators that have not been
1141 // explicitly set. For example, default object referent strength is set to strong.
1142 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1143 // and references from the types). This step also perform some type verification
1144 // that the passed in types make sense.
1145
1146 template <DecoratorSet decorators, typename T>
1147 static void verify_types(){
1148 // If this fails to compile, then you have sent in something that is
1149 // not recognized as a valid primitive type to a primitive Access function.
1150 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1151 (IsPointer<T>::value || IsIntegral<T>::value) ||
1152 IsFloatingPoint<T>::value)); // not allowed primitive type
1153 }
1154
1247 DecayedT new_decayed_value = new_value;
1248 // atomic_xchg is only available in SEQ_CST flavour.
1249 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1250 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1251 const_cast<DecayedP*>(addr));
1252 }
1253
1254 template <DecoratorSet decorators, typename T>
1255 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1256 verify_types<decorators, T>();
1257 typedef typename Decay<T>::type DecayedT;
1258 DecayedT new_decayed_value = new_value;
1259 // atomic_xchg is only available in SEQ_CST flavour.
1260 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1261 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1262 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value;
1263 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1264 }
1265
1266 template <DecoratorSet decorators, typename T>
1267 inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
1268 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1269 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1270 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1271 typedef typename Decay<T>::type DecayedT;
1272 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IN_HEAP_ARRAY | IN_HEAP>::value;
1273 return arraycopy_reduce_types<expanded_decorators>(src_obj, dst_obj,
1274 const_cast<DecayedT*>(src),
1275 const_cast<DecayedT*>(dst),
1276 length);
1277 }
1278
1279 template <DecoratorSet decorators>
1280 inline void clone(oop src, oop dst, size_t size) {
1281 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1282 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1283 }
1284
1285 template <DecoratorSet decorators>
1286 inline oop resolve(oop obj) {
1287 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1288 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1289 }
1290
1291 template <DecoratorSet decorators>
1292 inline bool equals(oop o1, oop o2) {
1293 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1294 return PreRuntimeDispatch::equals<expanded_decorators>(o1, o2);
1295 }
|
93 // locking to support wide atomics or not.
94 template <typename T>
95 #ifdef SUPPORTS_NATIVE_CX8
96 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
97 #else
98 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
99 #endif
100
101 template <DecoratorSet decorators, typename T>
102 struct AccessFunctionTypes {
103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
106 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
107
108 typedef T (*load_func_t)(void* addr);
109 typedef void (*store_func_t)(void* addr, T value);
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
112
113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length);
114 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
115 typedef oop (*resolve_func_t)(oop obj);
116 typedef bool (*equals_func_t)(oop o1, oop o2);
117 };
118
119 template <DecoratorSet decorators>
120 struct AccessFunctionTypes<decorators, void> {
121 typedef bool (*arraycopy_func_t)(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const void* src, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, void* dst, size_t length);
122 };
123
124 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
125
126 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
127 template <DecoratorSet decorators, typename T> \
128 struct AccessFunction<decorators, T, bt>: AllStatic{ \
129 typedef typename AccessFunctionTypes<decorators, T>::func type; \
130 }
131 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
132 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
133 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
134 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_RESOLVE, resolve_func_t);
336 static inline void store(void* addr, T value) {
337 store_internal<decorators>(addr, value);
338 }
339
340 template <typename T>
341 static inline T load(void* addr) {
342 return load_internal<decorators, T>(addr);
343 }
344
345 template <typename T>
346 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
347 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
348 }
349
350 template <typename T>
351 static inline T atomic_xchg(T new_value, void* addr) {
352 return atomic_xchg_maybe_locked<decorators>(new_value, addr);
353 }
354
355 template <typename T>
356 static bool arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length);
357
358 template <typename T>
359 static void oop_store(void* addr, T value);
360 template <typename T>
361 static void oop_store_at(oop base, ptrdiff_t offset, T value);
362
363 template <typename T>
364 static T oop_load(void* addr);
365 template <typename T>
366 static T oop_load_at(oop base, ptrdiff_t offset);
367
368 template <typename T>
369 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
370 template <typename T>
371 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
372
373 template <typename T>
374 static T oop_atomic_xchg(T new_value, void* addr);
375 template <typename T>
376 static T oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset);
379 static void store_at(oop base, ptrdiff_t offset, T value) {
380 store(field_addr(base, offset), value);
381 }
382
383 template <typename T>
384 static T load_at(oop base, ptrdiff_t offset) {
385 return load<T>(field_addr(base, offset));
386 }
387
388 template <typename T>
389 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
390 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
391 }
392
393 template <typename T>
394 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
395 return atomic_xchg(new_value, field_addr(base, offset));
396 }
397
398 template <typename T>
399 static bool oop_arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length);
400
401 static void clone(oop src, oop dst, size_t size);
402
403 static oop resolve(oop obj) { return obj; }
404
405 static bool equals(oop o1, oop o2) { return o1 == o2; }
406 };
407
408 // Below is the implementation of the first 4 steps of the template pipeline:
409 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
410 // and sets default decorators to sensible values.
411 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
412 // multiple types. The P type of the address and T type of the value must
413 // match.
414 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
415 // avoided, and in that case avoids it (calling raw accesses or
416 // primitive accesses in a build that does not require primitive GC barriers)
417 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
418 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers
419 // to the access.
542 }
543 };
544
545 template <DecoratorSet decorators, typename T>
546 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
547 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
548 static func_t _atomic_xchg_at_func;
549
550 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
551
552 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
553 return _atomic_xchg_at_func(new_value, base, offset);
554 }
555 };
556
557 template <DecoratorSet decorators, typename T>
558 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
559 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
560 static func_t _arraycopy_func;
561
562 static bool arraycopy_init(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length);
563
564 static inline bool arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length) {
565 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
566 }
567 };
568
569 template <DecoratorSet decorators, typename T>
570 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
571 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
572 static func_t _clone_func;
573
574 static void clone_init(oop src, oop dst, size_t size);
575
576 static inline void clone(oop src, oop dst, size_t size) {
577 _clone_func(src, dst, size);
578 }
579 };
580
581 template <DecoratorSet decorators, typename T>
582 struct RuntimeDispatch<decorators, T, BARRIER_RESOLVE>: AllStatic {
583 typedef typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type func_t;
584 static func_t _resolve_func;
585
883 HasDecorator<decorators, AS_RAW>::value, T>::type
884 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
885 return atomic_xchg<decorators>(new_value, field_addr(base, offset));
886 }
887
888 template <DecoratorSet decorators, typename T>
889 inline static typename EnableIf<
890 !HasDecorator<decorators, AS_RAW>::value, T>::type
891 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
892 if (is_hardwired_primitive<decorators>()) {
893 const DecoratorSet expanded_decorators = decorators | AS_RAW;
894 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
895 } else {
896 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
897 }
898 }
899
900 template <DecoratorSet decorators, typename T>
901 inline static typename EnableIf<
902 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
903 arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length) {
904 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
905 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
906 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
907 } else {
908 return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
909 }
910 }
911
912 template <DecoratorSet decorators, typename T>
913 inline static typename EnableIf<
914 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
915 arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length) {
916 if (UseCompressedOops) {
917 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
918 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
919 } else {
920 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
921 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
922 }
923 }
924
925 template <DecoratorSet decorators, typename T>
926 inline static typename EnableIf<
927 !HasDecorator<decorators, AS_RAW>::value, bool>::type
928 arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length) {
929 if (is_hardwired_primitive<decorators>()) {
930 const DecoratorSet expanded_decorators = decorators | AS_RAW;
931 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
932 } else {
933 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
934 }
935 }
936
937 template <DecoratorSet decorators>
938 inline static typename EnableIf<
939 HasDecorator<decorators, AS_RAW>::value>::type
940 clone(oop src, oop dst, size_t size) {
941 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
942 Raw::clone(src, dst, size);
943 }
944
945 template <DecoratorSet decorators>
946 inline static typename EnableIf<
947 !HasDecorator<decorators, AS_RAW>::value>::type
948 clone(oop src, oop dst, size_t size) {
949 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
950 }
951
952 template <DecoratorSet decorators>
953 inline static typename EnableIf<
1100
1101 template <DecoratorSet decorators, typename T>
1102 inline T load_reduce_types(T* addr) {
1103 return PreRuntimeDispatch::load<decorators, T>(addr);
1104 }
1105
1106 template <DecoratorSet decorators, typename T>
1107 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1108 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1109 INTERNAL_RT_USE_COMPRESSED_OOPS;
1110 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1111 }
1112
1113 template <DecoratorSet decorators, typename T>
1114 inline oop load_reduce_types(HeapWord* addr) {
1115 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1116 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1117 }
1118
1119 template <DecoratorSet decorators, typename T>
1120 inline bool arraycopy_reduce_types(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length) {
1121 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
1122 }
1123
1124 template <DecoratorSet decorators>
1125 inline bool arraycopy_reduce_types(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const HeapWord* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, HeapWord* dst_raw, size_t length) {
1126 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1127 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
1128 }
1129
1130 template <DecoratorSet decorators>
1131 inline bool arraycopy_reduce_types(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const narrowOop* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, narrowOop* dst_raw, size_t length) {
1132 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1133 INTERNAL_RT_USE_COMPRESSED_OOPS;
1134 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
1135 }
1136
1137 // Step 1: Set default decorators. This step remembers if a type was volatile
1138 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1139 // memory ordering is set for the access, and the implied decorator rules
1140 // are applied to select sensible defaults for decorators that have not been
1141 // explicitly set. For example, default object referent strength is set to strong.
1142 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1143 // and references from the types). This step also perform some type verification
1144 // that the passed in types make sense.
1145
1146 template <DecoratorSet decorators, typename T>
1147 static void verify_types(){
1148 // If this fails to compile, then you have sent in something that is
1149 // not recognized as a valid primitive type to a primitive Access function.
1150 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1151 (IsPointer<T>::value || IsIntegral<T>::value) ||
1152 IsFloatingPoint<T>::value)); // not allowed primitive type
1153 }
1154
1247 DecayedT new_decayed_value = new_value;
1248 // atomic_xchg is only available in SEQ_CST flavour.
1249 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1250 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1251 const_cast<DecayedP*>(addr));
1252 }
1253
1254 template <DecoratorSet decorators, typename T>
1255 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1256 verify_types<decorators, T>();
1257 typedef typename Decay<T>::type DecayedT;
1258 DecayedT new_decayed_value = new_value;
1259 // atomic_xchg is only available in SEQ_CST flavour.
1260 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1261 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1262 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value;
1263 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1264 }
1265
1266 template <DecoratorSet decorators, typename T>
1267 inline bool arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length) {
1268 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1269 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1270 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1271 typedef typename Decay<T>::type DecayedT;
1272 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IN_HEAP_ARRAY | IN_HEAP>::value;
1273 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<const DecayedT*>(src_raw),
1274 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1275 length);
1276 }
1277
1278 template <DecoratorSet decorators>
1279 inline void clone(oop src, oop dst, size_t size) {
1280 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1281 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1282 }
1283
1284 template <DecoratorSet decorators>
1285 inline oop resolve(oop obj) {
1286 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1287 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1288 }
1289
1290 template <DecoratorSet decorators>
1291 inline bool equals(oop o1, oop o2) {
1292 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1293 return PreRuntimeDispatch::equals<expanded_decorators>(o1, o2);
1294 }
|