93 // locking to support wide atomics or not.
94 template <typename T>
95 #ifdef SUPPORTS_NATIVE_CX8
96 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
97 #else
98 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
99 #endif
100
101 template <DecoratorSet decorators, typename T>
102 struct AccessFunctionTypes {
103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
106 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
107
108 typedef T (*load_func_t)(void* addr);
109 typedef void (*store_func_t)(void* addr, T value);
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
112
113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
114 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
115 size_t length);
116 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
117 typedef oop (*resolve_func_t)(oop obj);
118 typedef bool (*equals_func_t)(oop o1, oop o2);
119 };
120
121 template <DecoratorSet decorators>
122 struct AccessFunctionTypes<decorators, void> {
123 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
124 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
125 size_t length);
126 };
127
128 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
129
130 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
131 template <DecoratorSet decorators, typename T> \
132 struct AccessFunction<decorators, T, bt>: AllStatic{ \
133 typedef typename AccessFunctionTypes<decorators, T>::func type; \
134 }
135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
142 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
143 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
340 static inline void store(void* addr, T value) {
341 store_internal<decorators>(addr, value);
342 }
343
344 template <typename T>
345 static inline T load(void* addr) {
346 return load_internal<decorators, T>(addr);
347 }
348
349 template <typename T>
350 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
351 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
352 }
353
354 template <typename T>
355 static inline T atomic_xchg(T new_value, void* addr) {
356 return atomic_xchg_maybe_locked<decorators>(new_value, addr);
357 }
358
359 template <typename T>
360 static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
361 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
362 size_t length);
363
364 template <typename T>
365 static void oop_store(void* addr, T value);
366 template <typename T>
367 static void oop_store_at(oop base, ptrdiff_t offset, T value);
368
369 template <typename T>
370 static T oop_load(void* addr);
371 template <typename T>
372 static T oop_load_at(oop base, ptrdiff_t offset);
373
374 template <typename T>
375 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
376 template <typename T>
377 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
378
379 template <typename T>
380 static T oop_atomic_xchg(T new_value, void* addr);
385 static void store_at(oop base, ptrdiff_t offset, T value) {
386 store(field_addr(base, offset), value);
387 }
388
389 template <typename T>
390 static T load_at(oop base, ptrdiff_t offset) {
391 return load<T>(field_addr(base, offset));
392 }
393
394 template <typename T>
395 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
396 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
397 }
398
399 template <typename T>
400 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
401 return atomic_xchg(new_value, field_addr(base, offset));
402 }
403
404 template <typename T>
405 static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
406 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
407 size_t length);
408
409 static void clone(oop src, oop dst, size_t size);
410
411 static oop resolve(oop obj) { return obj; }
412
413 static bool equals(oop o1, oop o2) { return (void*)o1 == (void*)o2; }
414 };
415
416 // Below is the implementation of the first 4 steps of the template pipeline:
417 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
418 // and sets default decorators to sensible values.
419 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
420 // multiple types. The P type of the address and T type of the value must
421 // match.
422 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
423 // avoided, and in that case avoids it (calling raw accesses or
424 // primitive accesses in a build that does not require primitive GC barriers)
425 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
550 }
551 };
552
553 template <DecoratorSet decorators, typename T>
554 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
555 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
556 static func_t _atomic_xchg_at_func;
557
558 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
559
560 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
561 return _atomic_xchg_at_func(new_value, base, offset);
562 }
563 };
564
565 template <DecoratorSet decorators, typename T>
566 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
567 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
568 static func_t _arraycopy_func;
569
570 static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
571 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
572 size_t length);
573
574 static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
575 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
576 size_t length) {
577 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
578 dst_obj, dst_offset_in_bytes, dst_raw,
579 length);
580 }
581 };
582
583 template <DecoratorSet decorators, typename T>
584 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
585 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
586 static func_t _clone_func;
587
588 static void clone_init(oop src, oop dst, size_t size);
589
590 static inline void clone(oop src, oop dst, size_t size) {
591 _clone_func(src, dst, size);
592 }
593 };
594
896 inline static typename EnableIf<
897 HasDecorator<decorators, AS_RAW>::value, T>::type
898 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
899 return atomic_xchg<decorators>(new_value, field_addr(base, offset));
900 }
901
902 template <DecoratorSet decorators, typename T>
903 inline static typename EnableIf<
904 !HasDecorator<decorators, AS_RAW>::value, T>::type
905 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
906 if (is_hardwired_primitive<decorators>()) {
907 const DecoratorSet expanded_decorators = decorators | AS_RAW;
908 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
909 } else {
910 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
911 }
912 }
913
914 template <DecoratorSet decorators, typename T>
915 inline static typename EnableIf<
916 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
917 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
918 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
919 size_t length) {
920 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
921 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
922 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
923 dst_obj, dst_offset_in_bytes, dst_raw,
924 length);
925 } else {
926 return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
927 dst_obj, dst_offset_in_bytes, dst_raw,
928 length);
929 }
930 }
931
932 template <DecoratorSet decorators, typename T>
933 inline static typename EnableIf<
934 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
935 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
936 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
937 size_t length) {
938 if (UseCompressedOops) {
939 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
940 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
941 dst_obj, dst_offset_in_bytes, dst_raw,
942 length);
943 } else {
944 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
945 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
946 dst_obj, dst_offset_in_bytes, dst_raw,
947 length);
948 }
949 }
950
951 template <DecoratorSet decorators, typename T>
952 inline static typename EnableIf<
953 !HasDecorator<decorators, AS_RAW>::value, bool>::type
954 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
955 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
956 size_t length) {
957 if (is_hardwired_primitive<decorators>()) {
958 const DecoratorSet expanded_decorators = decorators | AS_RAW;
959 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
960 dst_obj, dst_offset_in_bytes, dst_raw,
961 length);
962 } else {
963 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
964 dst_obj, dst_offset_in_bytes, dst_raw,
965 length);
966 }
967 }
968
969 template <DecoratorSet decorators>
970 inline static typename EnableIf<
971 HasDecorator<decorators, AS_RAW>::value>::type
972 clone(oop src, oop dst, size_t size) {
973 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
974 Raw::clone(src, dst, size);
975 }
976
977 template <DecoratorSet decorators>
978 inline static typename EnableIf<
979 !HasDecorator<decorators, AS_RAW>::value>::type
980 clone(oop src, oop dst, size_t size) {
981 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
982 }
983
1107
1108 template <DecoratorSet decorators, typename T>
1109 inline T load_reduce_types(T* addr) {
1110 return PreRuntimeDispatch::load<decorators, T>(addr);
1111 }
1112
1113 template <DecoratorSet decorators, typename T>
1114 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1115 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1116 INTERNAL_RT_USE_COMPRESSED_OOPS;
1117 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1118 }
1119
1120 template <DecoratorSet decorators, typename T>
1121 inline oop load_reduce_types(HeapWord* addr) {
1122 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1123 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1124 }
1125
1126 template <DecoratorSet decorators, typename T>
1127 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1128 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1129 size_t length) {
1130 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1131 dst_obj, dst_offset_in_bytes, dst_raw,
1132 length);
1133 }
1134
1135 template <DecoratorSet decorators>
1136 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1137 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1138 size_t length) {
1139 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1140 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1141 dst_obj, dst_offset_in_bytes, dst_raw,
1142 length);
1143 }
1144
1145 template <DecoratorSet decorators>
1146 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1147 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1148 size_t length) {
1149 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1150 INTERNAL_RT_USE_COMPRESSED_OOPS;
1151 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1152 dst_obj, dst_offset_in_bytes, dst_raw,
1153 length);
1154 }
1155
1156 // Step 1: Set default decorators. This step remembers if a type was volatile
1157 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1158 // memory ordering is set for the access, and the implied decorator rules
1159 // are applied to select sensible defaults for decorators that have not been
1160 // explicitly set. For example, default object referent strength is set to strong.
1161 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1162 // and references from the types). This step also perform some type verification
1163 // that the passed in types make sense.
1164
1165 template <DecoratorSet decorators, typename T>
1166 static void verify_types(){
1167 // If this fails to compile, then you have sent in something that is
1168 // not recognized as a valid primitive type to a primitive Access function.
1169 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1170 (IsPointer<T>::value || IsIntegral<T>::value) ||
1171 IsFloatingPoint<T>::value)); // not allowed primitive type
1266 DecayedT new_decayed_value = new_value;
1267 // atomic_xchg is only available in SEQ_CST flavour.
1268 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1269 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1270 const_cast<DecayedP*>(addr));
1271 }
1272
1273 template <DecoratorSet decorators, typename T>
1274 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1275 verify_types<decorators, T>();
1276 typedef typename Decay<T>::type DecayedT;
1277 DecayedT new_decayed_value = new_value;
1278 // atomic_xchg is only available in SEQ_CST flavour.
1279 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1280 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1281 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1282 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1283 }
1284
1285 template <DecoratorSet decorators, typename T>
1286 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1287 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1288 size_t length) {
1289 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1290 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1291 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1292 typedef typename Decay<T>::type DecayedT;
1293 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1294 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1295 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1296 length);
1297 }
1298
1299 template <DecoratorSet decorators>
1300 inline void clone(oop src, oop dst, size_t size) {
1301 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1302 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1303 }
1304
1305 template <DecoratorSet decorators>
1306 inline oop resolve(oop obj) {
1307 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1308 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1309 }
1310
1311 template <DecoratorSet decorators>
1312 inline bool equals(oop o1, oop o2) {
1313 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1314 return PreRuntimeDispatch::equals<expanded_decorators>(o1, o2);
|
93 // locking to support wide atomics or not.
94 template <typename T>
95 #ifdef SUPPORTS_NATIVE_CX8
96 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
97 #else
98 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
99 #endif
100
101 template <DecoratorSet decorators, typename T>
102 struct AccessFunctionTypes {
103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
106 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
107
108 typedef T (*load_func_t)(void* addr);
109 typedef void (*store_func_t)(void* addr, T value);
110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
112
113 typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
114 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
115 size_t length);
116 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
117 typedef oop (*resolve_func_t)(oop obj);
118 typedef bool (*equals_func_t)(oop o1, oop o2);
119 };
120
121 template <DecoratorSet decorators>
122 struct AccessFunctionTypes<decorators, void> {
123 typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
124 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
125 size_t length);
126 };
127
128 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
129
130 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
131 template <DecoratorSet decorators, typename T> \
132 struct AccessFunction<decorators, T, bt>: AllStatic{ \
133 typedef typename AccessFunctionTypes<decorators, T>::func type; \
134 }
135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
142 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
143 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
340 static inline void store(void* addr, T value) {
341 store_internal<decorators>(addr, value);
342 }
343
344 template <typename T>
345 static inline T load(void* addr) {
346 return load_internal<decorators, T>(addr);
347 }
348
349 template <typename T>
350 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
351 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
352 }
353
354 template <typename T>
355 static inline T atomic_xchg(T new_value, void* addr) {
356 return atomic_xchg_maybe_locked<decorators>(new_value, addr);
357 }
358
359 template <typename T>
360 static void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
361 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
362 size_t length);
363
364 template <typename T>
365 static void oop_store(void* addr, T value);
366 template <typename T>
367 static void oop_store_at(oop base, ptrdiff_t offset, T value);
368
369 template <typename T>
370 static T oop_load(void* addr);
371 template <typename T>
372 static T oop_load_at(oop base, ptrdiff_t offset);
373
374 template <typename T>
375 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
376 template <typename T>
377 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
378
379 template <typename T>
380 static T oop_atomic_xchg(T new_value, void* addr);
385 static void store_at(oop base, ptrdiff_t offset, T value) {
386 store(field_addr(base, offset), value);
387 }
388
389 template <typename T>
390 static T load_at(oop base, ptrdiff_t offset) {
391 return load<T>(field_addr(base, offset));
392 }
393
394 template <typename T>
395 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
396 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
397 }
398
399 template <typename T>
400 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
401 return atomic_xchg(new_value, field_addr(base, offset));
402 }
403
404 template <typename T>
405 static void oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
406 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
407 size_t length);
408
409 static void clone(oop src, oop dst, size_t size);
410
411 static oop resolve(oop obj) { return obj; }
412
413 static bool equals(oop o1, oop o2) { return (void*)o1 == (void*)o2; }
414 };
415
416 // Below is the implementation of the first 4 steps of the template pipeline:
417 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
418 // and sets default decorators to sensible values.
419 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
420 // multiple types. The P type of the address and T type of the value must
421 // match.
422 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
423 // avoided, and in that case avoids it (calling raw accesses or
424 // primitive accesses in a build that does not require primitive GC barriers)
425 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
550 }
551 };
552
553 template <DecoratorSet decorators, typename T>
554 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
555 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
556 static func_t _atomic_xchg_at_func;
557
558 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
559
560 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
561 return _atomic_xchg_at_func(new_value, base, offset);
562 }
563 };
564
565 template <DecoratorSet decorators, typename T>
566 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
567 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
568 static func_t _arraycopy_func;
569
570 static void arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
571 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
572 size_t length);
573
574 static inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
575 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
576 size_t length) {
577 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
578 dst_obj, dst_offset_in_bytes, dst_raw,
579 length);
580 }
581 };
582
583 template <DecoratorSet decorators, typename T>
584 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
585 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
586 static func_t _clone_func;
587
588 static void clone_init(oop src, oop dst, size_t size);
589
590 static inline void clone(oop src, oop dst, size_t size) {
591 _clone_func(src, dst, size);
592 }
593 };
594
896 inline static typename EnableIf<
897 HasDecorator<decorators, AS_RAW>::value, T>::type
898 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
899 return atomic_xchg<decorators>(new_value, field_addr(base, offset));
900 }
901
902 template <DecoratorSet decorators, typename T>
903 inline static typename EnableIf<
904 !HasDecorator<decorators, AS_RAW>::value, T>::type
905 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
906 if (is_hardwired_primitive<decorators>()) {
907 const DecoratorSet expanded_decorators = decorators | AS_RAW;
908 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
909 } else {
910 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
911 }
912 }
913
914 template <DecoratorSet decorators, typename T>
915 inline static typename EnableIf<
916 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, void>::type
917 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
918 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
919 size_t length) {
920 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
921 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
922 Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
923 dst_obj, dst_offset_in_bytes, dst_raw,
924 length);
925 } else {
926 Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
927 dst_obj, dst_offset_in_bytes, dst_raw,
928 length);
929 }
930 }
931
932 template <DecoratorSet decorators, typename T>
933 inline static typename EnableIf<
934 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, void>::type
935 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
936 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
937 size_t length) {
938 if (UseCompressedOops) {
939 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
940 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
941 dst_obj, dst_offset_in_bytes, dst_raw,
942 length);
943 } else {
944 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
945 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
946 dst_obj, dst_offset_in_bytes, dst_raw,
947 length);
948 }
949 }
950
951 template <DecoratorSet decorators, typename T>
952 inline static typename EnableIf<
953 !HasDecorator<decorators, AS_RAW>::value, void>::type
954 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
955 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
956 size_t length) {
957 if (is_hardwired_primitive<decorators>()) {
958 const DecoratorSet expanded_decorators = decorators | AS_RAW;
959 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
960 dst_obj, dst_offset_in_bytes, dst_raw,
961 length);
962 } else {
963 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
964 dst_obj, dst_offset_in_bytes, dst_raw,
965 length);
966 }
967 }
968
969 template <DecoratorSet decorators>
970 inline static typename EnableIf<
971 HasDecorator<decorators, AS_RAW>::value>::type
972 clone(oop src, oop dst, size_t size) {
973 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
974 Raw::clone(src, dst, size);
975 }
976
977 template <DecoratorSet decorators>
978 inline static typename EnableIf<
979 !HasDecorator<decorators, AS_RAW>::value>::type
980 clone(oop src, oop dst, size_t size) {
981 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
982 }
983
1107
1108 template <DecoratorSet decorators, typename T>
1109 inline T load_reduce_types(T* addr) {
1110 return PreRuntimeDispatch::load<decorators, T>(addr);
1111 }
1112
1113 template <DecoratorSet decorators, typename T>
1114 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1115 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1116 INTERNAL_RT_USE_COMPRESSED_OOPS;
1117 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1118 }
1119
1120 template <DecoratorSet decorators, typename T>
1121 inline oop load_reduce_types(HeapWord* addr) {
1122 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1123 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1124 }
1125
1126 template <DecoratorSet decorators, typename T>
1127 inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1128 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1129 size_t length) {
1130 PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1131 dst_obj, dst_offset_in_bytes, dst_raw,
1132 length);
1133 }
1134
1135 template <DecoratorSet decorators>
1136 inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1137 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1138 size_t length) {
1139 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1140 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1141 dst_obj, dst_offset_in_bytes, dst_raw,
1142 length);
1143 }
1144
1145 template <DecoratorSet decorators>
1146 inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1147 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1148 size_t length) {
1149 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1150 INTERNAL_RT_USE_COMPRESSED_OOPS;
1151 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1152 dst_obj, dst_offset_in_bytes, dst_raw,
1153 length);
1154 }
1155
1156 // Step 1: Set default decorators. This step remembers if a type was volatile
1157 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1158 // memory ordering is set for the access, and the implied decorator rules
1159 // are applied to select sensible defaults for decorators that have not been
1160 // explicitly set. For example, default object referent strength is set to strong.
1161 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1162 // and references from the types). This step also perform some type verification
1163 // that the passed in types make sense.
1164
1165 template <DecoratorSet decorators, typename T>
1166 static void verify_types(){
1167 // If this fails to compile, then you have sent in something that is
1168 // not recognized as a valid primitive type to a primitive Access function.
1169 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1170 (IsPointer<T>::value || IsIntegral<T>::value) ||
1171 IsFloatingPoint<T>::value)); // not allowed primitive type
1266 DecayedT new_decayed_value = new_value;
1267 // atomic_xchg is only available in SEQ_CST flavour.
1268 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1269 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1270 const_cast<DecayedP*>(addr));
1271 }
1272
1273 template <DecoratorSet decorators, typename T>
1274 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1275 verify_types<decorators, T>();
1276 typedef typename Decay<T>::type DecayedT;
1277 DecayedT new_decayed_value = new_value;
1278 // atomic_xchg is only available in SEQ_CST flavour.
1279 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1280 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1281 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1282 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1283 }
1284
1285 template <DecoratorSet decorators, typename T>
1286 inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1287 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1288 size_t length) {
1289 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1290 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1291 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1292 typedef typename Decay<T>::type DecayedT;
1293 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1294 arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1295 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1296 length);
1297 }
1298
1299 template <DecoratorSet decorators>
1300 inline void clone(oop src, oop dst, size_t size) {
1301 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1302 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1303 }
1304
1305 template <DecoratorSet decorators>
1306 inline oop resolve(oop obj) {
1307 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1308 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1309 }
1310
1311 template <DecoratorSet decorators>
1312 inline bool equals(oop o1, oop o2) {
1313 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1314 return PreRuntimeDispatch::equals<expanded_decorators>(o1, o2);
|