93 // locking to support wide atomics or not.
94 template <typename T>
95 #ifdef SUPPORTS_NATIVE_CX8
96 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
97 #else
98 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
99 #endif
100
101 template <DecoratorSet decorators, typename T>
102 struct AccessFunctionTypes {
103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
106 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
107
108 typedef T (*load_func_t)(void* addr);
109 typedef void (*store_func_t)(void* addr, T value);
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
112
113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length);
114 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
115 typedef oop (*resolve_func_t)(oop obj);
116 typedef bool (*equals_func_t)(oop o1, oop o2);
117 };
118
119 template <DecoratorSet decorators>
120 struct AccessFunctionTypes<decorators, void> {
121 typedef bool (*arraycopy_func_t)(arrayOop src_obj, arrayOop dst_obj, void* src, void* dst, size_t length);
122 };
123
124 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
125
126 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
127 template <DecoratorSet decorators, typename T> \
128 struct AccessFunction<decorators, T, bt>: AllStatic{ \
129 typedef typename AccessFunctionTypes<decorators, T>::func type; \
130 }
131 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
132 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
133 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
134 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_RESOLVE, resolve_func_t);
336 static inline void store(void* addr, T value) {
337 store_internal<decorators>(addr, value);
338 }
339
340 template <typename T>
341 static inline T load(void* addr) {
342 return load_internal<decorators, T>(addr);
343 }
344
345 template <typename T>
346 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
347 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
348 }
349
350 template <typename T>
351 static inline T atomic_xchg(T new_value, void* addr) {
352 return atomic_xchg_maybe_locked<decorators>(new_value, addr);
353 }
354
355 template <typename T>
356 static bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length);
357
358 template <typename T>
359 static void oop_store(void* addr, T value);
360 template <typename T>
361 static void oop_store_at(oop base, ptrdiff_t offset, T value);
362
363 template <typename T>
364 static T oop_load(void* addr);
365 template <typename T>
366 static T oop_load_at(oop base, ptrdiff_t offset);
367
368 template <typename T>
369 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
370 template <typename T>
371 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
372
373 template <typename T>
374 static T oop_atomic_xchg(T new_value, void* addr);
375 template <typename T>
376 static T oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset);
379 static void store_at(oop base, ptrdiff_t offset, T value) {
380 store(field_addr(base, offset), value);
381 }
382
383 template <typename T>
384 static T load_at(oop base, ptrdiff_t offset) {
385 return load<T>(field_addr(base, offset));
386 }
387
388 template <typename T>
389 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
390 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
391 }
392
393 template <typename T>
394 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
395 return atomic_xchg(new_value, field_addr(base, offset));
396 }
397
398 template <typename T>
399 static bool oop_arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length);
400
401 static void clone(oop src, oop dst, size_t size);
402
403 static oop resolve(oop obj) { return obj; }
404
405 static bool equals(oop o1, oop o2) { return o1 == o2; }
406 };
407
408 // Below is the implementation of the first 4 steps of the template pipeline:
409 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
410 // and sets default decorators to sensible values.
411 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
412 // multiple types. The P type of the address and T type of the value must
413 // match.
414 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
415 // avoided, and in that case avoids it (calling raw accesses or
416 // primitive accesses in a build that does not require primitive GC barriers)
417 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
418 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers
419 // to the access.
542 }
543 };
544
545 template <DecoratorSet decorators, typename T>
546 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
547 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
548 static func_t _atomic_xchg_at_func;
549
550 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
551
552 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
553 return _atomic_xchg_at_func(new_value, base, offset);
554 }
555 };
556
557 template <DecoratorSet decorators, typename T>
558 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
559 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
560 static func_t _arraycopy_func;
561
562 static bool arraycopy_init(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length);
563
564 static inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
565 return _arraycopy_func(src_obj, dst_obj, src, dst, length);
566 }
567 };
568
569 template <DecoratorSet decorators, typename T>
570 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
571 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
572 static func_t _clone_func;
573
574 static void clone_init(oop src, oop dst, size_t size);
575
576 static inline void clone(oop src, oop dst, size_t size) {
577 _clone_func(src, dst, size);
578 }
579 };
580
581 template <DecoratorSet decorators, typename T>
582 struct RuntimeDispatch<decorators, T, BARRIER_RESOLVE>: AllStatic {
583 typedef typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type func_t;
584 static func_t _resolve_func;
585
883 HasDecorator<decorators, AS_RAW>::value, T>::type
884 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
885 return atomic_xchg<decorators>(new_value, field_addr(base, offset));
886 }
887
888 template <DecoratorSet decorators, typename T>
889 inline static typename EnableIf<
890 !HasDecorator<decorators, AS_RAW>::value, T>::type
891 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
892 if (is_hardwired_primitive<decorators>()) {
893 const DecoratorSet expanded_decorators = decorators | AS_RAW;
894 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
895 } else {
896 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
897 }
898 }
899
900 template <DecoratorSet decorators, typename T>
901 inline static typename EnableIf<
902 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
903 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
904 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
905 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
906 return Raw::oop_arraycopy(src_obj, dst_obj, src, dst, length);
907 } else {
908 return Raw::arraycopy(src_obj, dst_obj, src, dst, length);
909 }
910 }
911
912 template <DecoratorSet decorators, typename T>
913 inline static typename EnableIf<
914 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
915 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
916 if (UseCompressedOops) {
917 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
918 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length);
919 } else {
920 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
921 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length);
922 }
923 }
924
925 template <DecoratorSet decorators, typename T>
926 inline static typename EnableIf<
927 !HasDecorator<decorators, AS_RAW>::value, bool>::type
928 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
929 if (is_hardwired_primitive<decorators>()) {
930 const DecoratorSet expanded_decorators = decorators | AS_RAW;
931 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length);
932 } else {
933 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, dst_obj, src, dst, length);
934 }
935 }
936
937 template <DecoratorSet decorators>
938 inline static typename EnableIf<
939 HasDecorator<decorators, AS_RAW>::value>::type
940 clone(oop src, oop dst, size_t size) {
941 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
942 Raw::clone(src, dst, size);
943 }
944
945 template <DecoratorSet decorators>
946 inline static typename EnableIf<
947 !HasDecorator<decorators, AS_RAW>::value>::type
948 clone(oop src, oop dst, size_t size) {
949 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
950 }
951
952 template <DecoratorSet decorators>
953 inline static typename EnableIf<
1075
1076 template <DecoratorSet decorators, typename T>
1077 inline T load_reduce_types(T* addr) {
1078 return PreRuntimeDispatch::load<decorators, T>(addr);
1079 }
1080
1081 template <DecoratorSet decorators, typename T>
1082 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1083 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1084 INTERNAL_RT_USE_COMPRESSED_OOPS;
1085 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1086 }
1087
1088 template <DecoratorSet decorators, typename T>
1089 inline oop load_reduce_types(HeapWord* addr) {
1090 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1091 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1092 }
1093
1094 template <DecoratorSet decorators, typename T>
1095 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
1096 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, dst_obj, src, dst, length);
1097 }
1098
1099 template <DecoratorSet decorators>
1100 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, HeapWord* src, HeapWord* dst, size_t length) {
1101 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1102 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length);
1103 }
1104
1105 template <DecoratorSet decorators>
1106 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, narrowOop* src, narrowOop* dst, size_t length) {
1107 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1108 INTERNAL_RT_USE_COMPRESSED_OOPS;
1109 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length);
1110 }
1111
1112 // Step 1: Set default decorators. This step remembers if a type was volatile
1113 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1114 // memory ordering is set for the access, and the implied decorator rules
1115 // are applied to select sensible defaults for decorators that have not been
1116 // explicitly set. For example, default object referent strength is set to strong.
1117 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1118 // and references from the types). This step also perform some type verification
1119 // that the passed in types make sense.
1120
1121 template <DecoratorSet decorators, typename T>
1122 static void verify_types(){
1123 // If this fails to compile, then you have sent in something that is
1124 // not recognized as a valid primitive type to a primitive Access function.
1125 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1126 (IsPointer<T>::value || IsIntegral<T>::value) ||
1127 IsFloatingPoint<T>::value)); // not allowed primitive type
1128 }
1129
1222 DecayedT new_decayed_value = new_value;
1223 // atomic_xchg is only available in SEQ_CST flavour.
1224 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1225 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1226 const_cast<DecayedP*>(addr));
1227 }
1228
1229 template <DecoratorSet decorators, typename T>
1230 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1231 verify_types<decorators, T>();
1232 typedef typename Decay<T>::type DecayedT;
1233 DecayedT new_decayed_value = new_value;
1234 // atomic_xchg is only available in SEQ_CST flavour.
1235 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1236 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1237 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value;
1238 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1239 }
1240
1241 template <DecoratorSet decorators, typename T>
1242 inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
1243 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1244 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1245 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1246 typedef typename Decay<T>::type DecayedT;
1247 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IN_HEAP_ARRAY | IN_HEAP>::value;
1248 return arraycopy_reduce_types<expanded_decorators>(src_obj, dst_obj,
1249 const_cast<DecayedT*>(src),
1250 const_cast<DecayedT*>(dst),
1251 length);
1252 }
1253
1254 template <DecoratorSet decorators>
1255 inline void clone(oop src, oop dst, size_t size) {
1256 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1257 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1258 }
1259
1260 template <DecoratorSet decorators>
1261 inline oop resolve(oop obj) {
1262 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1263 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1264 }
1265
1266 template <DecoratorSet decorators>
1267 inline bool equals(oop o1, oop o2) {
1268 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1269 return PreRuntimeDispatch::equals<expanded_decorators>(o1, o2);
1270 }
|
93 // locking to support wide atomics or not.
94 template <typename T>
95 #ifdef SUPPORTS_NATIVE_CX8
96 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
97 #else
98 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
99 #endif
100
101 template <DecoratorSet decorators, typename T>
102 struct AccessFunctionTypes {
103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
106 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
107
108 typedef T (*load_func_t)(void* addr);
109 typedef void (*store_func_t)(void* addr, T value);
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
112
113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
114 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
115 size_t length);
116 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
117 typedef oop (*resolve_func_t)(oop obj);
118 typedef bool (*equals_func_t)(oop o1, oop o2);
119 };
120
121 template <DecoratorSet decorators>
122 struct AccessFunctionTypes<decorators, void> {
123 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, const void* src,
124 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
125 size_t length);
126 };
127
128 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
129
130 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
131 template <DecoratorSet decorators, typename T> \
132 struct AccessFunction<decorators, T, bt>: AllStatic{ \
133 typedef typename AccessFunctionTypes<decorators, T>::func type; \
134 }
135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
142 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
143 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
144 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
145 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_RESOLVE, resolve_func_t);
340 static inline void store(void* addr, T value) {
341 store_internal<decorators>(addr, value);
342 }
343
344 template <typename T>
345 static inline T load(void* addr) {
346 return load_internal<decorators, T>(addr);
347 }
348
349 template <typename T>
350 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
351 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
352 }
353
354 template <typename T>
355 static inline T atomic_xchg(T new_value, void* addr) {
356 return atomic_xchg_maybe_locked<decorators>(new_value, addr);
357 }
358
359 template <typename T>
360 static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
361 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
362 size_t length);
363
364 template <typename T>
365 static void oop_store(void* addr, T value);
366 template <typename T>
367 static void oop_store_at(oop base, ptrdiff_t offset, T value);
368
369 template <typename T>
370 static T oop_load(void* addr);
371 template <typename T>
372 static T oop_load_at(oop base, ptrdiff_t offset);
373
374 template <typename T>
375 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
376 template <typename T>
377 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
378
379 template <typename T>
380 static T oop_atomic_xchg(T new_value, void* addr);
381 template <typename T>
382 static T oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset);
385 static void store_at(oop base, ptrdiff_t offset, T value) {
386 store(field_addr(base, offset), value);
387 }
388
389 template <typename T>
390 static T load_at(oop base, ptrdiff_t offset) {
391 return load<T>(field_addr(base, offset));
392 }
393
394 template <typename T>
395 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
396 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
397 }
398
399 template <typename T>
400 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
401 return atomic_xchg(new_value, field_addr(base, offset));
402 }
403
404 template <typename T>
405 static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
406 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
407 size_t length);
408
409 static void clone(oop src, oop dst, size_t size);
410
411 static oop resolve(oop obj) { return obj; }
412
413 static bool equals(oop o1, oop o2) { return o1 == o2; }
414 };
415
416 // Below is the implementation of the first 4 steps of the template pipeline:
417 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
418 // and sets default decorators to sensible values.
419 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
420 // multiple types. The P type of the address and T type of the value must
421 // match.
422 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
423 // avoided, and in that case avoids it (calling raw accesses or
424 // primitive accesses in a build that does not require primitive GC barriers)
425 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
426 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers
427 // to the access.
550 }
551 };
552
553 template <DecoratorSet decorators, typename T>
554 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
555 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
556 static func_t _atomic_xchg_at_func;
557
558 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
559
560 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
561 return _atomic_xchg_at_func(new_value, base, offset);
562 }
563 };
564
565 template <DecoratorSet decorators, typename T>
566 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
567 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
568 static func_t _arraycopy_func;
569
570 static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
571 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
572 size_t length);
573
574 static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
575 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
576 size_t length) {
577 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
578 dst_obj, dst_offset_in_bytes, dst_raw,
579 length);
580 }
581 };
582
583 template <DecoratorSet decorators, typename T>
584 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
585 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
586 static func_t _clone_func;
587
588 static void clone_init(oop src, oop dst, size_t size);
589
590 static inline void clone(oop src, oop dst, size_t size) {
591 _clone_func(src, dst, size);
592 }
593 };
594
595 template <DecoratorSet decorators, typename T>
596 struct RuntimeDispatch<decorators, T, BARRIER_RESOLVE>: AllStatic {
597 typedef typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type func_t;
598 static func_t _resolve_func;
599
897 HasDecorator<decorators, AS_RAW>::value, T>::type
898 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
899 return atomic_xchg<decorators>(new_value, field_addr(base, offset));
900 }
901
902 template <DecoratorSet decorators, typename T>
903 inline static typename EnableIf<
904 !HasDecorator<decorators, AS_RAW>::value, T>::type
905 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
906 if (is_hardwired_primitive<decorators>()) {
907 const DecoratorSet expanded_decorators = decorators | AS_RAW;
908 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
909 } else {
910 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
911 }
912 }
913
914 template <DecoratorSet decorators, typename T>
915 inline static typename EnableIf<
916 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
917 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
918 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
919 size_t length) {
920 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
921 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
922 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
923 dst_obj, dst_offset_in_bytes, dst_raw,
924 length);
925 } else {
926 return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
927 dst_obj, dst_offset_in_bytes, dst_raw,
928 length);
929 }
930 }
931
932 template <DecoratorSet decorators, typename T>
933 inline static typename EnableIf<
934 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
935 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
936 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
937 size_t length) {
938 if (UseCompressedOops) {
939 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
940 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
941 dst_obj, dst_offset_in_bytes, dst_raw,
942 length);
943 } else {
944 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
945 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
946 dst_obj, dst_offset_in_bytes, dst_raw,
947 length);
948 }
949 }
950
951 template <DecoratorSet decorators, typename T>
952 inline static typename EnableIf<
953 !HasDecorator<decorators, AS_RAW>::value, bool>::type
954 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
955 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
956 size_t length) {
957 if (is_hardwired_primitive<decorators>()) {
958 const DecoratorSet expanded_decorators = decorators | AS_RAW;
959 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
960 dst_obj, dst_offset_in_bytes, dst_raw,
961 length);
962 } else {
963 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
964 dst_obj, dst_offset_in_bytes, dst_raw,
965 length);
966 }
967 }
968
969 template <DecoratorSet decorators>
970 inline static typename EnableIf<
971 HasDecorator<decorators, AS_RAW>::value>::type
972 clone(oop src, oop dst, size_t size) {
973 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
974 Raw::clone(src, dst, size);
975 }
976
977 template <DecoratorSet decorators>
978 inline static typename EnableIf<
979 !HasDecorator<decorators, AS_RAW>::value>::type
980 clone(oop src, oop dst, size_t size) {
981 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
982 }
983
984 template <DecoratorSet decorators>
985 inline static typename EnableIf<
1107
1108 template <DecoratorSet decorators, typename T>
1109 inline T load_reduce_types(T* addr) {
1110 return PreRuntimeDispatch::load<decorators, T>(addr);
1111 }
1112
1113 template <DecoratorSet decorators, typename T>
1114 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1115 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1116 INTERNAL_RT_USE_COMPRESSED_OOPS;
1117 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1118 }
1119
1120 template <DecoratorSet decorators, typename T>
1121 inline oop load_reduce_types(HeapWord* addr) {
1122 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1123 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1124 }
1125
1126 template <DecoratorSet decorators, typename T>
1127 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1128 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1129 size_t length) {
1130 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1131 dst_obj, dst_offset_in_bytes, dst_raw,
1132 length);
1133 }
1134
1135 template <DecoratorSet decorators>
1136 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, const HeapWord* src_raw,
1137 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1138 size_t length) {
1139 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1140 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1141 dst_obj, dst_offset_in_bytes, dst_raw,
1142 length);
1143 }
1144
1145 template <DecoratorSet decorators>
1146 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, const narrowOop* src_raw,
1147 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1148 size_t length) {
1149 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1150 INTERNAL_RT_USE_COMPRESSED_OOPS;
1151 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1152 dst_obj, dst_offset_in_bytes, dst_raw,
1153 length);
1154 }
1155
1156 // Step 1: Set default decorators. This step remembers if a type was volatile
1157 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1158 // memory ordering is set for the access, and the implied decorator rules
1159 // are applied to select sensible defaults for decorators that have not been
1160 // explicitly set. For example, default object referent strength is set to strong.
1161 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1162 // and references from the types). This step also perform some type verification
1163 // that the passed in types make sense.
1164
1165 template <DecoratorSet decorators, typename T>
1166 static void verify_types(){
1167 // If this fails to compile, then you have sent in something that is
1168 // not recognized as a valid primitive type to a primitive Access function.
1169 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1170 (IsPointer<T>::value || IsIntegral<T>::value) ||
1171 IsFloatingPoint<T>::value)); // not allowed primitive type
1172 }
1173
1266 DecayedT new_decayed_value = new_value;
1267 // atomic_xchg is only available in SEQ_CST flavour.
1268 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1269 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1270 const_cast<DecayedP*>(addr));
1271 }
1272
1273 template <DecoratorSet decorators, typename T>
1274 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1275 verify_types<decorators, T>();
1276 typedef typename Decay<T>::type DecayedT;
1277 DecayedT new_decayed_value = new_value;
1278 // atomic_xchg is only available in SEQ_CST flavour.
1279 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1280 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1281 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value;
1282 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1283 }
1284
1285 template <DecoratorSet decorators, typename T>
1286 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1287 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1288 size_t length) {
1289 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1290 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1291 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1292 typedef typename Decay<T>::type DecayedT;
1293 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IN_HEAP_ARRAY | IN_HEAP>::value;
1294 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<const DecayedT*>(src_raw),
1295 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1296 length);
1297 }
1298
1299 template <DecoratorSet decorators>
1300 inline void clone(oop src, oop dst, size_t size) {
1301 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1302 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1303 }
1304
1305 template <DecoratorSet decorators>
1306 inline oop resolve(oop obj) {
1307 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1308 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1309 }
1310
1311 template <DecoratorSet decorators>
1312 inline bool equals(oop o1, oop o2) {
1313 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1314 return PreRuntimeDispatch::equals<expanded_decorators>(o1, o2);
1315 }
|