68 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) { 69 oop_store(field_addr(base, offset), value); 70 } 71 72 template <DecoratorSet decorators> 73 template <typename T> 74 inline T RawAccessBarrier<decorators>::oop_load(void* addr) { 75 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 76 Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr)); 77 return decode<T>(encoded); 78 } 79 80 template <DecoratorSet decorators> 81 template <typename T> 82 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) { 83 return oop_load<T>(field_addr(base, offset)); 84 } 85 86 template <DecoratorSet decorators> 87 template <typename T> 88 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(T new_value, void* addr, T compare_value) { 89 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 90 Encoded encoded_new = encode(new_value); 91 Encoded encoded_compare = encode(compare_value); 92 Encoded encoded_result = atomic_cmpxchg(encoded_new, 93 reinterpret_cast<Encoded*>(addr), 94 encoded_compare); 95 return decode<T>(encoded_result); 96 } 97 98 template <DecoratorSet decorators> 99 template <typename T> 100 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 101 return oop_atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); 102 } 103 104 template <DecoratorSet decorators> 105 template <typename T> 106 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(T new_value, void* addr) { 107 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 108 Encoded encoded_new = encode(new_value); 109 Encoded encoded_result = atomic_xchg(encoded_new, reinterpret_cast<Encoded*>(addr)); 110 return decode<T>(encoded_result); 111 } 112 113 template <DecoratorSet decorators> 114 template <typename T> 115 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 116 return oop_atomic_xchg(new_value, field_addr(base, offset)); 117 } 118 119 template <DecoratorSet decorators> 120 template <typename T> 121 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 122 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 123 size_t length) { 124 return arraycopy(src_obj, src_offset_in_bytes, src_raw, 125 dst_obj, dst_offset_in_bytes, dst_raw, 126 length); 127 } 128 129 template <DecoratorSet decorators> 130 template <DecoratorSet ds, typename T> 131 inline typename EnableIf< 132 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 133 RawAccessBarrier<decorators>::load_internal(void* addr) { 134 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 135 OrderAccess::fence(); 136 } 157 template <DecoratorSet ds, typename T> 158 inline typename EnableIf< 159 HasDecorator<ds, MO_SEQ_CST>::value>::type 160 RawAccessBarrier<decorators>::store_internal(void* addr, T value) { 161 Atomic::release_store_fence(reinterpret_cast<volatile T*>(addr), value); 162 } 163 164 template <DecoratorSet decorators> 165 template <DecoratorSet ds, typename T> 166 inline typename EnableIf< 167 HasDecorator<ds, MO_RELEASE>::value>::type 168 RawAccessBarrier<decorators>::store_internal(void* addr, T value) { 169 Atomic::release_store(reinterpret_cast<volatile T*>(addr), value); 170 } 171 172 template <DecoratorSet decorators> 173 template <DecoratorSet ds, typename T> 174 inline typename EnableIf< 175 HasDecorator<ds, MO_RELAXED>::value>::type 176 RawAccessBarrier<decorators>::store_internal(void* addr, T value) { 177 Atomic::store(value, reinterpret_cast<volatile T*>(addr)); 178 } 179 180 template <DecoratorSet decorators> 181 template <DecoratorSet ds, typename T> 182 inline typename EnableIf< 183 HasDecorator<ds, MO_RELAXED>::value, T>::type 184 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(T new_value, void* addr, T compare_value) { 185 return Atomic::cmpxchg(new_value, 186 reinterpret_cast<volatile T*>(addr), 187 compare_value, 188 memory_order_relaxed); 189 } 190 191 template <DecoratorSet decorators> 192 template <DecoratorSet ds, typename T> 193 inline typename EnableIf< 194 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 195 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(T new_value, void* addr, T compare_value) { 196 return Atomic::cmpxchg(new_value, 197 reinterpret_cast<volatile T*>(addr), 198 compare_value, 199 memory_order_conservative); 200 } 201 202 template <DecoratorSet decorators> 203 template <DecoratorSet ds, typename T> 204 inline typename EnableIf< 205 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 206 RawAccessBarrier<decorators>::atomic_xchg_internal(T new_value, void* addr) { 207 return Atomic::xchg(new_value, 208 reinterpret_cast<volatile T*>(addr)); 209 } 210 211 // For platforms that do not have native support for wide atomics, 212 // we can emulate the atomicity using a lock. So here we check 213 // whether that is necessary or not. 214 215 template <DecoratorSet ds> 216 template <DecoratorSet decorators, typename T> 217 inline typename EnableIf< 218 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 219 RawAccessBarrier<ds>::atomic_xchg_maybe_locked(T new_value, void* addr) { 220 if (!AccessInternal::wide_atomic_needs_locking()) { 221 return atomic_xchg_internal<ds>(new_value, addr); 222 } else { 223 AccessInternal::AccessLocker access_lock; 224 volatile T* p = reinterpret_cast<volatile T*>(addr); 225 T old_val = RawAccess<>::load(p); 226 RawAccess<>::store(p, new_value); 227 return old_val; 228 } 229 } 230 231 template <DecoratorSet ds> 232 template <DecoratorSet decorators, typename T> 233 inline typename EnableIf< 234 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 235 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value) { 236 if (!AccessInternal::wide_atomic_needs_locking()) { 237 return atomic_cmpxchg_internal<ds>(new_value, addr, compare_value); 238 } else { 239 AccessInternal::AccessLocker access_lock; 240 volatile T* p = reinterpret_cast<volatile T*>(addr); 241 T old_val = RawAccess<>::load(p); 242 if (old_val == compare_value) { 243 RawAccess<>::store(p, new_value); 244 } 245 return old_val; 246 } 247 } 248 249 class RawAccessBarrierArrayCopy: public AllStatic { 250 template<typename T> struct IsHeapWordSized: public IntegralConstant<bool, sizeof(T) == HeapWordSize> { }; 251 public: 252 template <DecoratorSet decorators, typename T> 253 static inline typename EnableIf< 254 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 255 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 256 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 257 size_t length) { | 68 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) { 69 oop_store(field_addr(base, offset), value); 70 } 71 72 template <DecoratorSet decorators> 73 template <typename T> 74 inline T RawAccessBarrier<decorators>::oop_load(void* addr) { 75 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 76 Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr)); 77 return decode<T>(encoded); 78 } 79 80 template <DecoratorSet decorators> 81 template <typename T> 82 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) { 83 return oop_load<T>(field_addr(base, offset)); 84 } 85 86 template <DecoratorSet decorators> 87 template <typename T> 88 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(void* addr, T compare_value, T new_value) { 89 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 90 Encoded encoded_new = encode(new_value); 91 Encoded encoded_compare = encode(compare_value); 92 Encoded encoded_result = atomic_cmpxchg(reinterpret_cast<Encoded*>(addr), 93 encoded_compare, 94 encoded_new); 95 return decode<T>(encoded_result); 96 } 97 98 template <DecoratorSet decorators> 99 template <typename T> 100 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) { 101 return oop_atomic_cmpxchg(field_addr(base, offset), compare_value, new_value); 102 } 103 104 template <DecoratorSet decorators> 105 template <typename T> 106 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(void* addr, T new_value) { 107 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 108 Encoded encoded_new = encode(new_value); 109 Encoded encoded_result = atomic_xchg(reinterpret_cast<Encoded*>(addr), encoded_new); 110 return decode<T>(encoded_result); 111 } 112 113 template <DecoratorSet decorators> 114 template <typename T> 115 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { 116 return oop_atomic_xchg(field_addr(base, offset), new_value); 117 } 118 119 template <DecoratorSet decorators> 120 template <typename T> 121 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 122 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 123 size_t length) { 124 return arraycopy(src_obj, src_offset_in_bytes, src_raw, 125 dst_obj, dst_offset_in_bytes, dst_raw, 126 length); 127 } 128 129 template <DecoratorSet decorators> 130 template <DecoratorSet ds, typename T> 131 inline typename EnableIf< 132 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 133 RawAccessBarrier<decorators>::load_internal(void* addr) { 134 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 135 OrderAccess::fence(); 136 } 157 template <DecoratorSet ds, typename T> 158 inline typename EnableIf< 159 HasDecorator<ds, MO_SEQ_CST>::value>::type 160 RawAccessBarrier<decorators>::store_internal(void* addr, T value) { 161 Atomic::release_store_fence(reinterpret_cast<volatile T*>(addr), value); 162 } 163 164 template <DecoratorSet decorators> 165 template <DecoratorSet ds, typename T> 166 inline typename EnableIf< 167 HasDecorator<ds, MO_RELEASE>::value>::type 168 RawAccessBarrier<decorators>::store_internal(void* addr, T value) { 169 Atomic::release_store(reinterpret_cast<volatile T*>(addr), value); 170 } 171 172 template <DecoratorSet decorators> 173 template <DecoratorSet ds, typename T> 174 inline typename EnableIf< 175 HasDecorator<ds, MO_RELAXED>::value>::type 176 RawAccessBarrier<decorators>::store_internal(void* addr, T value) { 177 Atomic::store(reinterpret_cast<volatile T*>(addr), value); 178 } 179 180 template <DecoratorSet decorators> 181 template <DecoratorSet ds, typename T> 182 inline typename EnableIf< 183 HasDecorator<ds, MO_RELAXED>::value, T>::type 184 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) { 185 return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr), 186 compare_value, 187 new_value, 188 memory_order_relaxed); 189 } 190 191 template <DecoratorSet decorators> 192 template <DecoratorSet ds, typename T> 193 inline typename EnableIf< 194 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 195 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) { 196 return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr), 197 compare_value, 198 new_value, 199 memory_order_conservative); 200 } 201 202 template <DecoratorSet decorators> 203 template <DecoratorSet ds, typename T> 204 inline typename EnableIf< 205 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 206 RawAccessBarrier<decorators>::atomic_xchg_internal(void* addr, T new_value) { 207 return Atomic::xchg(reinterpret_cast<volatile T*>(addr), 208 new_value); 209 } 210 211 // For platforms that do not have native support for wide atomics, 212 // we can emulate the atomicity using a lock. So here we check 213 // whether that is necessary or not. 214 215 template <DecoratorSet ds> 216 template <DecoratorSet decorators, typename T> 217 inline typename EnableIf< 218 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 219 RawAccessBarrier<ds>::atomic_xchg_maybe_locked(void* addr, T new_value) { 220 if (!AccessInternal::wide_atomic_needs_locking()) { 221 return atomic_xchg_internal<ds>(addr, new_value); 222 } else { 223 AccessInternal::AccessLocker access_lock; 224 volatile T* p = reinterpret_cast<volatile T*>(addr); 225 T old_val = RawAccess<>::load(p); 226 RawAccess<>::store(p, new_value); 227 return old_val; 228 } 229 } 230 231 template <DecoratorSet ds> 232 template <DecoratorSet decorators, typename T> 233 inline typename EnableIf< 234 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 235 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(void* addr, T compare_value, T new_value) { 236 if (!AccessInternal::wide_atomic_needs_locking()) { 237 return atomic_cmpxchg_internal<ds>(addr, compare_value, new_value); 238 } else { 239 AccessInternal::AccessLocker access_lock; 240 volatile T* p = reinterpret_cast<volatile T*>(addr); 241 T old_val = RawAccess<>::load(p); 242 if (old_val == compare_value) { 243 RawAccess<>::store(p, new_value); 244 } 245 return old_val; 246 } 247 } 248 249 class RawAccessBarrierArrayCopy: public AllStatic { 250 template<typename T> struct IsHeapWordSized: public IntegralConstant<bool, sizeof(T) == HeapWordSize> { }; 251 public: 252 template <DecoratorSet decorators, typename T> 253 static inline typename EnableIf< 254 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 255 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 256 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 257 size_t length) { |