1 /*
2 * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
26 #define SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
27
28 #include "oops/access.hpp"
29 #include "oops/accessBackend.hpp"
30 #include "oops/compressedOops.inline.hpp"
31 #include "oops/oopsHierarchy.hpp"
32 #include "runtime/atomic.hpp"
33 #include "runtime/orderAccess.hpp"
34
35 template <DecoratorSet decorators>
36 template <DecoratorSet idecorators, typename T>
37 inline typename EnableIf<
38 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
39 RawAccessBarrier<decorators>::decode_internal(typename HeapOopType<idecorators>::type value) {
40 if (HasDecorator<decorators, IS_NOT_NULL>::value) {
41 return CompressedOops::decode_not_null(value);
42 } else {
43 return CompressedOops::decode(value);
44 }
45 }
46
47 template <DecoratorSet decorators>
48 template <DecoratorSet idecorators, typename T>
49 inline typename EnableIf<
50 AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
51 typename HeapOopType<idecorators>::type>::type
52 RawAccessBarrier<decorators>::encode_internal(T value) {
53 if (HasDecorator<decorators, IS_NOT_NULL>::value) {
54 return CompressedOops::encode_not_null(value);
55 } else {
56 return CompressedOops::encode(value);
57 }
58 }
59
60 template <DecoratorSet decorators>
61 template <typename T>
62 inline void RawAccessBarrier<decorators>::oop_store(void* addr, T value) {
63 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
64 Encoded encoded = encode(value);
65 store(reinterpret_cast<Encoded*>(addr), encoded);
66 }
67
68 template <DecoratorSet decorators>
69 template <typename T>
70 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) {
71 oop_store(field_addr(base, offset), value);
72 }
73
74 template <DecoratorSet decorators>
75 template <typename T>
76 inline T RawAccessBarrier<decorators>::oop_load(void* addr) {
77 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
78 Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr));
79 return decode<T>(encoded);
80 }
81
82 template <DecoratorSet decorators>
83 template <typename T>
84 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) {
85 return oop_load<T>(field_addr(base, offset));
86 }
87
88 template <DecoratorSet decorators>
89 template <typename T>
90 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(void* addr, T compare_value, T new_value) {
91 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
92 Encoded encoded_new = encode(new_value);
93 Encoded encoded_compare = encode(compare_value);
94 Encoded encoded_result = atomic_cmpxchg(reinterpret_cast<Encoded*>(addr),
95 encoded_compare,
96 encoded_new);
97 return decode<T>(encoded_result);
98 }
99
100 template <DecoratorSet decorators>
101 template <typename T>
102 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
103 return oop_atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
104 }
105
106 template <DecoratorSet decorators>
107 template <typename T>
108 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(void* addr, T new_value) {
109 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
110 Encoded encoded_new = encode(new_value);
111 Encoded encoded_result = atomic_xchg(reinterpret_cast<Encoded*>(addr), encoded_new);
112 return decode<T>(encoded_result);
113 }
114
115 template <DecoratorSet decorators>
116 template <typename T>
117 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
118 return oop_atomic_xchg(field_addr(base, offset), new_value);
119 }
120
121 template <DecoratorSet decorators>
122 template <typename T>
123 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
124 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
125 size_t length) {
126 return arraycopy(src_obj, src_offset_in_bytes, src_raw,
127 dst_obj, dst_offset_in_bytes, dst_raw,
128 length);
129 }
130
131 template <DecoratorSet decorators>
132 template <DecoratorSet ds, typename T>
133 inline typename EnableIf<
134 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
135 RawAccessBarrier<decorators>::load_internal(void* addr) {
136 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
137 OrderAccess::fence();
138 }
139 return Atomic::load_acquire(reinterpret_cast<const volatile T*>(addr));
140 }
141
142 template <DecoratorSet decorators>
143 template <DecoratorSet ds, typename T>
144 inline typename EnableIf<
145 HasDecorator<ds, MO_ACQUIRE>::value, T>::type
146 RawAccessBarrier<decorators>::load_internal(void* addr) {
147 return Atomic::load_acquire(reinterpret_cast<const volatile T*>(addr));
148 }
149
150 template <DecoratorSet decorators>
151 template <DecoratorSet ds, typename T>
152 inline typename EnableIf<
153 HasDecorator<ds, MO_RELAXED>::value, T>::type
154 RawAccessBarrier<decorators>::load_internal(void* addr) {
155 return Atomic::load(reinterpret_cast<const volatile T*>(addr));
156 }
157
158 template <DecoratorSet decorators>
159 template <DecoratorSet ds, typename T>
160 inline typename EnableIf<
161 HasDecorator<ds, MO_SEQ_CST>::value>::type
162 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
163 Atomic::release_store_fence(reinterpret_cast<volatile T*>(addr), value);
164 }
165
166 template <DecoratorSet decorators>
167 template <DecoratorSet ds, typename T>
168 inline typename EnableIf<
169 HasDecorator<ds, MO_RELEASE>::value>::type
170 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
171 Atomic::release_store(reinterpret_cast<volatile T*>(addr), value);
172 }
173
174 template <DecoratorSet decorators>
175 template <DecoratorSet ds, typename T>
176 inline typename EnableIf<
177 HasDecorator<ds, MO_RELAXED>::value>::type
178 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
179 Atomic::store(reinterpret_cast<volatile T*>(addr), value);
180 }
181
182 template <DecoratorSet decorators>
183 template <DecoratorSet ds, typename T>
184 inline typename EnableIf<
185 HasDecorator<ds, MO_RELAXED>::value, T>::type
186 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
187 return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr),
188 compare_value,
189 new_value,
190 memory_order_relaxed);
191 }
192
193 template <DecoratorSet decorators>
194 template <DecoratorSet ds, typename T>
195 inline typename EnableIf<
196 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
197 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
198 return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr),
199 compare_value,
200 new_value,
201 memory_order_conservative);
202 }
203
204 template <DecoratorSet decorators>
205 template <DecoratorSet ds, typename T>
206 inline typename EnableIf<
207 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
208 RawAccessBarrier<decorators>::atomic_xchg_internal(void* addr, T new_value) {
209 return Atomic::xchg(reinterpret_cast<volatile T*>(addr),
210 new_value);
211 }
212
213 // For platforms that do not have native support for wide atomics,
214 // we can emulate the atomicity using a lock. So here we check
215 // whether that is necessary or not.
216
217 template <DecoratorSet ds>
218 template <DecoratorSet decorators, typename T>
219 inline typename EnableIf<
220 AccessInternal::PossiblyLockedAccess<T>::value, T>::type
221 RawAccessBarrier<ds>::atomic_xchg_maybe_locked(void* addr, T new_value) {
222 if (!AccessInternal::wide_atomic_needs_locking()) {
223 return atomic_xchg_internal<ds>(addr, new_value);
224 } else {
225 AccessInternal::AccessLocker access_lock;
226 volatile T* p = reinterpret_cast<volatile T*>(addr);
227 T old_val = RawAccess<>::load(p);
228 RawAccess<>::store(p, new_value);
229 return old_val;
230 }
231 }
232
233 template <DecoratorSet ds>
234 template <DecoratorSet decorators, typename T>
235 inline typename EnableIf<
236 AccessInternal::PossiblyLockedAccess<T>::value, T>::type
237 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(void* addr, T compare_value, T new_value) {
238 if (!AccessInternal::wide_atomic_needs_locking()) {
239 return atomic_cmpxchg_internal<ds>(addr, compare_value, new_value);
240 } else {
241 AccessInternal::AccessLocker access_lock;
242 volatile T* p = reinterpret_cast<volatile T*>(addr);
243 T old_val = RawAccess<>::load(p);
244 if (old_val == compare_value) {
245 RawAccess<>::store(p, new_value);
246 }
247 return old_val;
248 }
249 }
250
251 class RawAccessBarrierArrayCopy: public AllStatic {
252 template<typename T> struct IsHeapWordSized: public IntegralConstant<bool, sizeof(T) == HeapWordSize> { };
253 public:
254 template <DecoratorSet decorators, typename T>
255 static inline typename EnableIf<
256 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
257 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
258 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
259 size_t length) {
260 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
261 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
262
263 // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic.
264 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
265 AccessInternal::arraycopy_arrayof_conjoint_oops(src_raw, dst_raw, length);
266 } else {
267 typedef typename HeapOopType<decorators>::type OopType;
268 AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src_raw),
269 reinterpret_cast<OopType*>(dst_raw), length);
270 }
271 }
272
273 template <DecoratorSet decorators, typename T>
274 static inline typename EnableIf<
275 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
276 HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value>::type
277 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
278 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
279 size_t length) {
280 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
281 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
282
283 AccessInternal::arraycopy_arrayof_conjoint(src_raw, dst_raw, length);
284 }
285
286 template <DecoratorSet decorators, typename T>
287 static inline typename EnableIf<
288 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
289 HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value>::type
290 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
291 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
292 size_t length) {
293 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
294 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
295
296 // There is only a disjoint optimization for word granularity copying
297 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
298 AccessInternal::arraycopy_disjoint_words_atomic(src_raw, dst_raw, length);
299 } else {
300 AccessInternal::arraycopy_disjoint_words(src_raw, dst_raw, length);
301 }
302 }
303
304 template <DecoratorSet decorators, typename T>
305 static inline typename EnableIf<
306 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
307 !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
308 !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
309 !HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
310 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
311 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
312 size_t length) {
313 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
314 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
315
316 AccessInternal::arraycopy_conjoint(src_raw, dst_raw, length);
317 }
318
319 template <DecoratorSet decorators, typename T>
320 static inline typename EnableIf<
321 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
322 !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
323 !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
324 HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
325 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
326 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
327 size_t length) {
328 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
329 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
330
331 AccessInternal::arraycopy_conjoint_atomic(src_raw, dst_raw, length);
332 }
333 };
334
335 template<> struct RawAccessBarrierArrayCopy::IsHeapWordSized<void>: public IntegralConstant<bool, false> { };
336
337 template <DecoratorSet decorators>
338 template <typename T>
339 inline bool RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
340 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
341 size_t length) {
342 RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
343 dst_obj, dst_offset_in_bytes, dst_raw,
344 length);
345 return true;
346 }
347
348 template <DecoratorSet decorators>
349 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
350 // 4839641 (4840070): We must do an oop-atomic copy, because if another thread
351 // is modifying a reference field in the clonee, a non-oop-atomic copy might
352 // be suspended in the middle of copying the pointer and end up with parts
353 // of two different pointers in the field. Subsequent dereferences will crash.
354 // 4846409: an oop-copy of objects with long or double fields or arrays of same
355 // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead
356 // of oops. We know objects are aligned on a minimum of an jlong boundary.
357 // The same is true of StubRoutines::object_copy and the various oop_copy
358 // variants, and of the code generated by the inline_native_clone intrinsic.
359
360 assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
361 AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src),
362 reinterpret_cast<jlong*>((oopDesc*)dst),
363 align_object_size(size) / HeapWordsPerLong);
364 // Clear the header
365 dst->init_mark_raw();
366 }
367
368 #endif // SHARE_OOPS_ACCESSBACKEND_INLINE_HPP