101 return oop_atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); 102 } 103 104 template <DecoratorSet decorators> 105 template <typename T> 106 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(T new_value, void* addr) { 107 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 108 Encoded encoded_new = encode(new_value); 109 Encoded encoded_result = atomic_xchg(encoded_new, reinterpret_cast<Encoded*>(addr)); 110 return decode<T>(encoded_result); 111 } 112 113 template <DecoratorSet decorators> 114 template <typename T> 115 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 116 return oop_atomic_xchg(new_value, field_addr(base, offset)); 117 } 118 119 template <DecoratorSet decorators> 120 template <typename T> 121 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 122 return arraycopy(src_obj, dst_obj, src, dst, length); 123 } 124 125 template <DecoratorSet decorators> 126 template <DecoratorSet ds, typename T> 127 inline typename EnableIf< 128 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 129 RawAccessBarrier<decorators>::load_internal(void* addr) { 130 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 131 OrderAccess::fence(); 132 } 133 return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr)); 134 } 135 136 template <DecoratorSet decorators> 137 template <DecoratorSet ds, typename T> 138 inline typename EnableIf< 139 HasDecorator<ds, MO_ACQUIRE>::value, T>::type 140 RawAccessBarrier<decorators>::load_internal(void* addr) { 141 return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr)); 142 } 230 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 231 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value) { 232 if (!AccessInternal::wide_atomic_needs_locking()) { 233 return atomic_cmpxchg_internal<ds>(new_value, addr, compare_value); 234 } else { 235 AccessInternal::AccessLocker access_lock; 236 volatile T* p = reinterpret_cast<volatile T*>(addr); 237 T old_val = RawAccess<>::load(p); 238 if (old_val == compare_value) { 239 RawAccess<>::store(p, new_value); 240 } 241 return old_val; 242 } 243 } 244 245 class RawAccessBarrierArrayCopy: public AllStatic { 246 public: 247 template <DecoratorSet decorators, typename T> 248 static inline typename EnableIf< 249 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 250 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 251 // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic. 252 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) { 253 AccessInternal::arraycopy_arrayof_conjoint_oops(src, dst, length); 254 } else { 255 typedef typename HeapOopType<decorators>::type OopType; 256 AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src), 257 reinterpret_cast<OopType*>(dst), length); 258 } 259 } 260 261 template <DecoratorSet decorators, typename T> 262 static inline typename EnableIf< 263 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 264 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 265 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) { 266 AccessInternal::arraycopy_arrayof_conjoint(src, dst, length); 267 } else if (HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && sizeof(T) == HeapWordSize) { 268 // There is only a disjoint optimization for word granularity copying 269 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) { 270 AccessInternal::arraycopy_disjoint_words_atomic(src, dst, length); 271 } else { 272 AccessInternal::arraycopy_disjoint_words(src, dst, length); 273 } 274 } else { 275 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) { 276 AccessInternal::arraycopy_conjoint_atomic(src, dst, length); 277 } else { 278 AccessInternal::arraycopy_conjoint(src, dst, length); 279 } 280 } 281 } 282 283 template <DecoratorSet decorators> 284 static inline typename EnableIf< 285 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 286 arraycopy(arrayOop src_obj, arrayOop dst_obj, void* src, void* dst, size_t length) { 287 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) { 288 AccessInternal::arraycopy_conjoint_atomic(src, dst, length); 289 } else { 290 AccessInternal::arraycopy_conjoint(src, dst, length); 291 } 292 } 293 }; 294 295 template <DecoratorSet decorators> 296 template <typename T> 297 inline bool RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 298 RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, dst_obj, src, dst, length); 299 return true; 300 } 301 302 template <DecoratorSet decorators> 303 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) { 304 // 4839641 (4840070): We must do an oop-atomic copy, because if another thread 305 // is modifying a reference field in the clonee, a non-oop-atomic copy might 306 // be suspended in the middle of copying the pointer and end up with parts 307 // of two different pointers in the field. Subsequent dereferences will crash. 308 // 4846409: an oop-copy of objects with long or double fields or arrays of same 309 // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead 310 // of oops. We know objects are aligned on a minimum of an jlong boundary. 311 // The same is true of StubRoutines::object_copy and the various oop_copy 312 // variants, and of the code generated by the inline_native_clone intrinsic. 313 314 assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned"); 315 AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src), 316 reinterpret_cast<jlong*>((oopDesc*)dst), 317 align_object_size(size) / HeapWordsPerLong); 318 // Clear the header | 101 return oop_atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); 102 } 103 104 template <DecoratorSet decorators> 105 template <typename T> 106 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(T new_value, void* addr) { 107 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 108 Encoded encoded_new = encode(new_value); 109 Encoded encoded_result = atomic_xchg(encoded_new, reinterpret_cast<Encoded*>(addr)); 110 return decode<T>(encoded_result); 111 } 112 113 template <DecoratorSet decorators> 114 template <typename T> 115 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 116 return oop_atomic_xchg(new_value, field_addr(base, offset)); 117 } 118 119 template <DecoratorSet decorators> 120 template <typename T> 121 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length) { 122 return arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); 123 } 124 125 template <DecoratorSet decorators> 126 template <DecoratorSet ds, typename T> 127 inline typename EnableIf< 128 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 129 RawAccessBarrier<decorators>::load_internal(void* addr) { 130 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 131 OrderAccess::fence(); 132 } 133 return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr)); 134 } 135 136 template <DecoratorSet decorators> 137 template <DecoratorSet ds, typename T> 138 inline typename EnableIf< 139 HasDecorator<ds, MO_ACQUIRE>::value, T>::type 140 RawAccessBarrier<decorators>::load_internal(void* addr) { 141 return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr)); 142 } 230 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 231 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value) { 232 if (!AccessInternal::wide_atomic_needs_locking()) { 233 return atomic_cmpxchg_internal<ds>(new_value, addr, compare_value); 234 } else { 235 AccessInternal::AccessLocker access_lock; 236 volatile T* p = reinterpret_cast<volatile T*>(addr); 237 T old_val = RawAccess<>::load(p); 238 if (old_val == compare_value) { 239 RawAccess<>::store(p, new_value); 240 } 241 return old_val; 242 } 243 } 244 245 class RawAccessBarrierArrayCopy: public AllStatic { 246 public: 247 template <DecoratorSet decorators, typename T> 248 static inline typename EnableIf< 249 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 250 arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length) { 251 252 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); 253 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); 254 255 // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic. 256 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) { 257 AccessInternal::arraycopy_arrayof_conjoint_oops(const_cast<T*>(src_raw), dst_raw, length); 258 } else { 259 typedef typename HeapOopType<decorators>::type OopType; 260 AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(const_cast<T*>(src_raw)), 261 reinterpret_cast<OopType*>(dst_raw), length); 262 } 263 } 264 265 template <DecoratorSet decorators, typename T> 266 static inline typename EnableIf< 267 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 268 arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length) { 269 270 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); 271 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); 272 273 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) { 274 AccessInternal::arraycopy_arrayof_conjoint(const_cast<T*>(src_raw), dst_raw, length); 275 } else if (HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && sizeof(T) == HeapWordSize) { 276 // There is only a disjoint optimization for word granularity copying 277 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) { 278 AccessInternal::arraycopy_disjoint_words_atomic(const_cast<T*>(src_raw), dst_raw, length); 279 } else { 280 AccessInternal::arraycopy_disjoint_words(const_cast<T*>(src_raw), dst_raw, length); 281 } 282 } else { 283 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) { 284 AccessInternal::arraycopy_conjoint_atomic(const_cast<T*>(src_raw), dst_raw, length); 285 } else { 286 AccessInternal::arraycopy_conjoint(const_cast<T*>(src_raw), dst_raw, length); 287 } 288 } 289 } 290 291 template <DecoratorSet decorators> 292 static inline typename EnableIf< 293 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 294 arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const void* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, void* dst_raw, size_t length) { 295 296 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); 297 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); 298 299 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) { 300 AccessInternal::arraycopy_conjoint_atomic(const_cast<void*>(src_raw), dst_raw, length); 301 } else { 302 AccessInternal::arraycopy_conjoint(const_cast<void*>(src_raw), dst_raw, length); 303 } 304 } 305 }; 306 307 template <DecoratorSet decorators> 308 template <typename T> 309 inline bool RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, ptrdiff_t src_offset_in_bytes, const T* src_raw, arrayOop dst_obj, ptrdiff_t dst_offset_in_bytes, T* dst_raw, size_t length) { 310 RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); 311 return true; 312 } 313 314 template <DecoratorSet decorators> 315 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) { 316 // 4839641 (4840070): We must do an oop-atomic copy, because if another thread 317 // is modifying a reference field in the clonee, a non-oop-atomic copy might 318 // be suspended in the middle of copying the pointer and end up with parts 319 // of two different pointers in the field. Subsequent dereferences will crash. 320 // 4846409: an oop-copy of objects with long or double fields or arrays of same 321 // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead 322 // of oops. We know objects are aligned on a minimum of an jlong boundary. 323 // The same is true of StubRoutines::object_copy and the various oop_copy 324 // variants, and of the code generated by the inline_native_clone intrinsic. 325 326 assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned"); 327 AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src), 328 reinterpret_cast<jlong*>((oopDesc*)dst), 329 align_object_size(size) / HeapWordsPerLong); 330 // Clear the header |