1 /*
   2  * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP
  26 #define SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP
  27 
  28 #include "oops/access.hpp"
  29 #include "oops/accessBackend.hpp"
  30 #include "oops/oop.inline.hpp"
  31 
  32 template <DecoratorSet decorators>
  33 template <DecoratorSet idecorators, typename T>
  34 inline typename EnableIf<
  35   AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
  36 RawAccessBarrier<decorators>::decode_internal(typename HeapOopType<idecorators>::type value) {
  37   if (HasDecorator<decorators, OOP_NOT_NULL>::value) {
  38     return oopDesc::decode_heap_oop_not_null(value);
  39   } else {
  40     return oopDesc::decode_heap_oop(value);
  41   }
  42 }
  43 
  44 template <DecoratorSet decorators>
  45 template <DecoratorSet idecorators, typename T>
  46 inline typename EnableIf<
  47   AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
  48   typename HeapOopType<idecorators>::type>::type
  49 RawAccessBarrier<decorators>::encode_internal(T value) {
  50   if (HasDecorator<decorators, OOP_NOT_NULL>::value) {
  51     return oopDesc::encode_heap_oop_not_null(value);
  52   } else {
  53     return oopDesc::encode_heap_oop(value);
  54   }
  55 }
  56 
  57 template <DecoratorSet decorators>
  58 template <typename T>
  59 inline void RawAccessBarrier<decorators>::oop_store(void* addr, T value) {
  60   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
  61   Encoded encoded = encode(value);
  62   store(reinterpret_cast<Encoded*>(addr), encoded);
  63 }
  64 
  65 template <DecoratorSet decorators>
  66 template <typename T>
  67 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) {
  68   oop_store(field_addr(base, offset), value);
  69 }
  70 
  71 template <DecoratorSet decorators>
  72 template <typename T>
  73 inline T RawAccessBarrier<decorators>::oop_load(void* addr) {
  74   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
  75   Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr));
  76   return decode<T>(encoded);
  77 }
  78 
  79 template <DecoratorSet decorators>
  80 template <typename T>
  81 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) {
  82   return oop_load<T>(field_addr(base, offset));
  83 }
  84 
  85 template <DecoratorSet decorators>
  86 template <typename T>
  87 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(T new_value, void* addr, T compare_value) {
  88   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
  89   Encoded encoded_new = encode(new_value);
  90   Encoded encoded_compare = encode(compare_value);
  91   Encoded encoded_result = atomic_cmpxchg(encoded_new,
  92                                           reinterpret_cast<Encoded*>(addr),
  93                                           encoded_compare);
  94   return decode<T>(encoded_result);
  95 }
  96 
  97 template <DecoratorSet decorators>
  98 template <typename T>
  99 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
 100   return oop_atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
 101 }
 102 
 103 template <DecoratorSet decorators>
 104 template <typename T>
 105 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(T new_value, void* addr) {
 106   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
 107   Encoded encoded_new = encode(new_value);
 108   Encoded encoded_result = atomic_xchg(encoded_new, reinterpret_cast<Encoded*>(addr));
 109   return decode<T>(encoded_result);
 110 }
 111 
 112 template <DecoratorSet decorators>
 113 template <typename T>
 114 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
 115   return oop_atomic_xchg(new_value, field_addr(base, offset));
 116 }
 117 
 118 template <DecoratorSet decorators>
 119 template <typename T>
 120 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
 121   return arraycopy(src, dst, length);
 122 }
 123 
 124 template <DecoratorSet decorators>
 125 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, arrayOop dst_obj, HeapWord* src, HeapWord* dst, size_t length) {
 126   bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
 127                             HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
 128   if (needs_oop_compress) {
 129     return arraycopy(reinterpret_cast<narrowOop*>(src), reinterpret_cast<narrowOop*>(dst), length);
 130   } else {
 131     return arraycopy(reinterpret_cast<oop*>(src), reinterpret_cast<oop*>(dst), length);
 132   }
 133 }
 134 
 135 template <DecoratorSet decorators>
 136 template <DecoratorSet ds, typename T>
 137 inline typename EnableIf<
 138   HasDecorator<ds, MO_SEQ_CST>::value, T>::type
 139 RawAccessBarrier<decorators>::load_internal(void* addr) {
 140   if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
 141     OrderAccess::fence();
 142   }
 143   return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr));
 144 }
 145 
 146 template <DecoratorSet decorators>
 147 template <DecoratorSet ds, typename T>
 148 inline typename EnableIf<
 149   HasDecorator<ds, MO_ACQUIRE>::value, T>::type
 150 RawAccessBarrier<decorators>::load_internal(void* addr) {
 151   return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr));
 152 }
 153 
 154 template <DecoratorSet decorators>
 155 template <DecoratorSet ds, typename T>
 156 inline typename EnableIf<
 157   HasDecorator<ds, MO_RELAXED>::value, T>::type
 158 RawAccessBarrier<decorators>::load_internal(void* addr) {
 159   return Atomic::load(reinterpret_cast<const volatile T*>(addr));
 160 }
 161 
 162 template <DecoratorSet decorators>
 163 template <DecoratorSet ds, typename T>
 164 inline typename EnableIf<
 165   HasDecorator<ds, MO_SEQ_CST>::value>::type
 166 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
 167   OrderAccess::release_store_fence(reinterpret_cast<volatile T*>(addr), value);
 168 }
 169 
 170 template <DecoratorSet decorators>
 171 template <DecoratorSet ds, typename T>
 172 inline typename EnableIf<
 173   HasDecorator<ds, MO_RELEASE>::value>::type
 174 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
 175   OrderAccess::release_store(reinterpret_cast<volatile T*>(addr), value);
 176 }
 177 
 178 template <DecoratorSet decorators>
 179 template <DecoratorSet ds, typename T>
 180 inline typename EnableIf<
 181   HasDecorator<ds, MO_RELAXED>::value>::type
 182 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
 183   Atomic::store(value, reinterpret_cast<volatile T*>(addr));
 184 }
 185 
 186 template <DecoratorSet decorators>
 187 template <DecoratorSet ds, typename T>
 188 inline typename EnableIf<
 189   HasDecorator<ds, MO_RELAXED>::value, T>::type
 190 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(T new_value, void* addr, T compare_value) {
 191   return Atomic::cmpxchg(new_value,
 192                          reinterpret_cast<volatile T*>(addr),
 193                          compare_value,
 194                          memory_order_relaxed);
 195 }
 196 
 197 template <DecoratorSet decorators>
 198 template <DecoratorSet ds, typename T>
 199 inline typename EnableIf<
 200   HasDecorator<ds, MO_SEQ_CST>::value, T>::type
 201 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(T new_value, void* addr, T compare_value) {
 202   return Atomic::cmpxchg(new_value,
 203                          reinterpret_cast<volatile T*>(addr),
 204                          compare_value,
 205                          memory_order_conservative);
 206 }
 207 
 208 template <DecoratorSet decorators>
 209 template <DecoratorSet ds, typename T>
 210 inline typename EnableIf<
 211   HasDecorator<ds, MO_SEQ_CST>::value, T>::type
 212 RawAccessBarrier<decorators>::atomic_xchg_internal(T new_value, void* addr) {
 213   return Atomic::xchg(new_value,
 214                       reinterpret_cast<volatile T*>(addr));
 215 }
 216 
 217 // For platforms that do not have native support for wide atomics,
 218 // we can emulate the atomicity using a lock. So here we check
 219 // whether that is necessary or not.
 220 
 221 template <DecoratorSet ds>
 222 template <DecoratorSet decorators, typename T>
 223 inline typename EnableIf<
 224   AccessInternal::PossiblyLockedAccess<T>::value, T>::type
 225 RawAccessBarrier<ds>::atomic_xchg_maybe_locked(T new_value, void* addr) {
 226   if (!AccessInternal::wide_atomic_needs_locking()) {
 227     return atomic_xchg_internal<ds>(new_value, addr);
 228   } else {
 229     AccessInternal::AccessLocker access_lock;
 230     volatile T* p = reinterpret_cast<volatile T*>(addr);
 231     T old_val = RawAccess<>::load(p);
 232     RawAccess<>::store(p, new_value);
 233     return old_val;
 234   }
 235 }
 236 
 237 template <DecoratorSet ds>
 238 template <DecoratorSet decorators, typename T>
 239 inline typename EnableIf<
 240   AccessInternal::PossiblyLockedAccess<T>::value, T>::type
 241 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value) {
 242   if (!AccessInternal::wide_atomic_needs_locking()) {
 243     return atomic_cmpxchg_internal<ds>(new_value, addr, compare_value);
 244   } else {
 245     AccessInternal::AccessLocker access_lock;
 246     volatile T* p = reinterpret_cast<volatile T*>(addr);
 247     T old_val = RawAccess<>::load(p);
 248     if (old_val == compare_value) {
 249       RawAccess<>::store(p, new_value);
 250     }
 251     return old_val;
 252   }
 253 }
 254 
 255 class RawAccessBarrierArrayCopy: public AllStatic {
 256 public:
 257   template <DecoratorSet decorators, typename T>
 258   static inline typename EnableIf<
 259   HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
 260   arraycopy(T* src, T* dst, size_t length) {
 261     // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic.
 262     if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
 263       AccessInternal::arraycopy_arrayof_conjoint_oops(src, dst, length);
 264     } else {
 265       typedef typename HeapOopType<decorators>::type OopType;
 266       AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src),
 267                                               reinterpret_cast<OopType*>(dst), length);
 268     }
 269   }
 270 
 271   template <DecoratorSet decorators, typename T>
 272   static inline typename EnableIf<
 273     !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
 274   arraycopy(T* src, T* dst, size_t length) {
 275     if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
 276       AccessInternal::arraycopy_arrayof_conjoint(src, dst, length);
 277     } else if (HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && sizeof(T) == HeapWordSize) {
 278       // There is only a disjoint optimization for word granularity copying
 279       if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
 280         AccessInternal::arraycopy_disjoint_words_atomic(src, dst, length);
 281       } else {
 282         AccessInternal::arraycopy_disjoint_words(src, dst, length);
 283       }
 284     } else {
 285       if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
 286         AccessInternal::arraycopy_conjoint_atomic(src, dst, length);
 287       } else {
 288         AccessInternal::arraycopy_conjoint(src, dst, length);
 289       }
 290     }
 291   }
 292 };
 293 
 294 template <DecoratorSet decorators>
 295 template <typename T>
 296 inline bool RawAccessBarrier<decorators>::arraycopy(T* src, T* dst, size_t length) {
 297   RawAccessBarrierArrayCopy::arraycopy<decorators>(src, dst, length);
 298   return true;
 299 }
 300 
 301 template <DecoratorSet decorators>
 302 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
 303   // 4839641 (4840070): We must do an oop-atomic copy, because if another thread
 304   // is modifying a reference field in the clonee, a non-oop-atomic copy might
 305   // be suspended in the middle of copying the pointer and end up with parts
 306   // of two different pointers in the field.  Subsequent dereferences will crash.
 307   // 4846409: an oop-copy of objects with long or double fields or arrays of same
 308   // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead
 309   // of oops.  We know objects are aligned on a minimum of an jlong boundary.
 310   // The same is true of StubRoutines::object_copy and the various oop_copy
 311   // variants, and of the code generated by the inline_native_clone intrinsic.
 312 
 313   assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
 314   AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src),
 315                                             reinterpret_cast<jlong*>((oopDesc*)dst),
 316                                             align_object_size(size) / HeapWordsPerLong);
 317   // Clear the header
 318   dst->init_mark();
 319 }
 320 
 321 #endif // SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP