1 /* 2 * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP 26 #define SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP 27 28 #include "oops/access.hpp" 29 #include "oops/accessBackend.hpp" 30 #include "oops/oop.inline.hpp" 31 32 template <DecoratorSet decorators> 33 template <DecoratorSet idecorators, typename T> 34 inline typename EnableIf< 35 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type 36 RawAccessBarrier<decorators>::decode_internal(typename HeapOopType<idecorators>::type value) { 37 if (HasDecorator<decorators, OOP_NOT_NULL>::value) { 38 return oopDesc::decode_heap_oop_not_null(value); 39 } else { 40 return oopDesc::decode_heap_oop(value); 41 } 42 } 43 44 template <DecoratorSet decorators> 45 template <DecoratorSet idecorators, typename T> 46 inline typename EnableIf< 47 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, 48 typename HeapOopType<idecorators>::type>::type 49 RawAccessBarrier<decorators>::encode_internal(T value) { 50 if (HasDecorator<decorators, OOP_NOT_NULL>::value) { 51 return oopDesc::encode_heap_oop_not_null(value); 52 } else { 53 return oopDesc::encode_heap_oop(value); 54 } 55 } 56 57 template <DecoratorSet decorators> 58 template <typename T> 59 inline void RawAccessBarrier<decorators>::oop_store(void* addr, T value) { 60 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 61 Encoded encoded = encode(value); 62 store(reinterpret_cast<Encoded*>(addr), encoded); 63 } 64 65 template <DecoratorSet decorators> 66 template <typename T> 67 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) { 68 oop_store(field_addr(base, offset), value); 69 } 70 71 template <DecoratorSet decorators> 72 template <typename T> 73 inline T RawAccessBarrier<decorators>::oop_load(void* addr) { 74 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 75 Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr)); 76 return decode<T>(encoded); 77 } 78 79 template <DecoratorSet decorators> 80 template <typename T> 81 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) { 82 return oop_load<T>(field_addr(base, offset)); 83 } 84 85 template <DecoratorSet decorators> 86 template <typename T> 87 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(T new_value, void* addr, T compare_value) { 88 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 89 Encoded encoded_new = encode(new_value); 90 Encoded encoded_compare = encode(compare_value); 91 Encoded encoded_result = atomic_cmpxchg(encoded_new, 92 reinterpret_cast<Encoded*>(addr), 93 encoded_compare); 94 return decode<T>(encoded_result); 95 } 96 97 template <DecoratorSet decorators> 98 template <typename T> 99 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 100 return oop_atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); 101 } 102 103 template <DecoratorSet decorators> 104 template <typename T> 105 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(T new_value, void* addr) { 106 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded; 107 Encoded encoded_new = encode(new_value); 108 Encoded encoded_result = atomic_xchg(encoded_new, reinterpret_cast<Encoded*>(addr)); 109 return decode<T>(encoded_result); 110 } 111 112 template <DecoratorSet decorators> 113 template <typename T> 114 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 115 return oop_atomic_xchg(new_value, field_addr(base, offset)); 116 } 117 118 template <DecoratorSet decorators> 119 template <typename T> 120 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 121 return arraycopy(src_obj, dst_obj, src, dst, length); 122 } 123 124 template <DecoratorSet decorators> 125 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, arrayOop dst_obj, HeapWord* src, HeapWord* dst, size_t length) { 126 bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value && 127 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value; 128 if (needs_oop_compress) { 129 return arraycopy(reinterpret_cast<narrowOop*>(src), reinterpret_cast<narrowOop*>(dst), length); 130 } else { 131 return arraycopy(reinterpret_cast<oop*>(src), reinterpret_cast<oop*>(dst), length); 132 } 133 } 134 135 template <DecoratorSet decorators> 136 template <DecoratorSet ds, typename T> 137 inline typename EnableIf< 138 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 139 RawAccessBarrier<decorators>::load_internal(void* addr) { 140 if (support_IRIW_for_not_multiple_copy_atomic_cpu) { 141 OrderAccess::fence(); 142 } 143 return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr)); 144 } 145 146 template <DecoratorSet decorators> 147 template <DecoratorSet ds, typename T> 148 inline typename EnableIf< 149 HasDecorator<ds, MO_ACQUIRE>::value, T>::type 150 RawAccessBarrier<decorators>::load_internal(void* addr) { 151 return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr)); 152 } 153 154 template <DecoratorSet decorators> 155 template <DecoratorSet ds, typename T> 156 inline typename EnableIf< 157 HasDecorator<ds, MO_RELAXED>::value, T>::type 158 RawAccessBarrier<decorators>::load_internal(void* addr) { 159 return Atomic::load(reinterpret_cast<const volatile T*>(addr)); 160 } 161 162 template <DecoratorSet decorators> 163 template <DecoratorSet ds, typename T> 164 inline typename EnableIf< 165 HasDecorator<ds, MO_SEQ_CST>::value>::type 166 RawAccessBarrier<decorators>::store_internal(void* addr, T value) { 167 OrderAccess::release_store_fence(reinterpret_cast<volatile T*>(addr), value); 168 } 169 170 template <DecoratorSet decorators> 171 template <DecoratorSet ds, typename T> 172 inline typename EnableIf< 173 HasDecorator<ds, MO_RELEASE>::value>::type 174 RawAccessBarrier<decorators>::store_internal(void* addr, T value) { 175 OrderAccess::release_store(reinterpret_cast<volatile T*>(addr), value); 176 } 177 178 template <DecoratorSet decorators> 179 template <DecoratorSet ds, typename T> 180 inline typename EnableIf< 181 HasDecorator<ds, MO_RELAXED>::value>::type 182 RawAccessBarrier<decorators>::store_internal(void* addr, T value) { 183 Atomic::store(value, reinterpret_cast<volatile T*>(addr)); 184 } 185 186 template <DecoratorSet decorators> 187 template <DecoratorSet ds, typename T> 188 inline typename EnableIf< 189 HasDecorator<ds, MO_RELAXED>::value, T>::type 190 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(T new_value, void* addr, T compare_value) { 191 return Atomic::cmpxchg(new_value, 192 reinterpret_cast<volatile T*>(addr), 193 compare_value, 194 memory_order_relaxed); 195 } 196 197 template <DecoratorSet decorators> 198 template <DecoratorSet ds, typename T> 199 inline typename EnableIf< 200 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 201 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(T new_value, void* addr, T compare_value) { 202 return Atomic::cmpxchg(new_value, 203 reinterpret_cast<volatile T*>(addr), 204 compare_value, 205 memory_order_conservative); 206 } 207 208 template <DecoratorSet decorators> 209 template <DecoratorSet ds, typename T> 210 inline typename EnableIf< 211 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 212 RawAccessBarrier<decorators>::atomic_xchg_internal(T new_value, void* addr) { 213 return Atomic::xchg(new_value, 214 reinterpret_cast<volatile T*>(addr)); 215 } 216 217 // For platforms that do not have native support for wide atomics, 218 // we can emulate the atomicity using a lock. So here we check 219 // whether that is necessary or not. 220 221 template <DecoratorSet ds> 222 template <DecoratorSet decorators, typename T> 223 inline typename EnableIf< 224 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 225 RawAccessBarrier<ds>::atomic_xchg_maybe_locked(T new_value, void* addr) { 226 if (!AccessInternal::wide_atomic_needs_locking()) { 227 return atomic_xchg_internal<ds>(new_value, addr); 228 } else { 229 AccessInternal::AccessLocker access_lock; 230 volatile T* p = reinterpret_cast<volatile T*>(addr); 231 T old_val = RawAccess<>::load(p); 232 RawAccess<>::store(p, new_value); 233 return old_val; 234 } 235 } 236 237 template <DecoratorSet ds> 238 template <DecoratorSet decorators, typename T> 239 inline typename EnableIf< 240 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 241 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value) { 242 if (!AccessInternal::wide_atomic_needs_locking()) { 243 return atomic_cmpxchg_internal<ds>(new_value, addr, compare_value); 244 } else { 245 AccessInternal::AccessLocker access_lock; 246 volatile T* p = reinterpret_cast<volatile T*>(addr); 247 T old_val = RawAccess<>::load(p); 248 if (old_val == compare_value) { 249 RawAccess<>::store(p, new_value); 250 } 251 return old_val; 252 } 253 } 254 255 class RawAccessBarrierArrayCopy: public AllStatic { 256 public: 257 template <DecoratorSet decorators, typename T> 258 static inline typename EnableIf< 259 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 260 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 261 // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic. 262 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) { 263 AccessInternal::arraycopy_arrayof_conjoint_oops(src, dst, length); 264 } else { 265 typedef typename HeapOopType<decorators>::type OopType; 266 AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src), 267 reinterpret_cast<OopType*>(dst), length); 268 } 269 } 270 271 template <DecoratorSet decorators, typename T> 272 static inline typename EnableIf< 273 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 274 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 275 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) { 276 AccessInternal::arraycopy_arrayof_conjoint(src, dst, length); 277 } else if (HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && sizeof(T) == HeapWordSize) { 278 // There is only a disjoint optimization for word granularity copying 279 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) { 280 AccessInternal::arraycopy_disjoint_words_atomic(src, dst, length); 281 } else { 282 AccessInternal::arraycopy_disjoint_words(src, dst, length); 283 } 284 } else { 285 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) { 286 AccessInternal::arraycopy_conjoint_atomic(src, dst, length); 287 } else { 288 AccessInternal::arraycopy_conjoint(src, dst, length); 289 } 290 } 291 } 292 293 template <DecoratorSet decorators> 294 static inline typename EnableIf< 295 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type 296 arraycopy(arrayOop src_obj, arrayOop dst_obj, void* src, void* dst, size_t length) { 297 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) { 298 AccessInternal::arraycopy_conjoint_atomic(src, dst, length); 299 } else { 300 AccessInternal::arraycopy_conjoint(src, dst, length); 301 } 302 } 303 }; 304 305 template <DecoratorSet decorators> 306 template <typename T> 307 inline bool RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 308 RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, dst_obj, src, dst, length); 309 return true; 310 } 311 312 template <DecoratorSet decorators> 313 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) { 314 // 4839641 (4840070): We must do an oop-atomic copy, because if another thread 315 // is modifying a reference field in the clonee, a non-oop-atomic copy might 316 // be suspended in the middle of copying the pointer and end up with parts 317 // of two different pointers in the field. Subsequent dereferences will crash. 318 // 4846409: an oop-copy of objects with long or double fields or arrays of same 319 // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead 320 // of oops. We know objects are aligned on a minimum of an jlong boundary. 321 // The same is true of StubRoutines::object_copy and the various oop_copy 322 // variants, and of the code generated by the inline_native_clone intrinsic. 323 324 assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned"); 325 AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src), 326 reinterpret_cast<jlong*>((oopDesc*)dst), 327 align_object_size(size) / HeapWordsPerLong); 328 // Clear the header 329 dst->init_mark(); 330 } 331 332 #endif // SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP