1 /*
2 * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
26 #define SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
27
28 #include "oops/access.hpp"
29 #include "oops/accessBackend.hpp"
30 #include "oops/compressedOops.inline.hpp"
31 #include "oops/oopsHierarchy.hpp"
32 #include "oops/valueKlass.hpp"
33
34 template <DecoratorSet decorators>
35 template <DecoratorSet idecorators, typename T>
36 inline typename EnableIf<
37 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
38 RawAccessBarrier<decorators>::decode_internal(typename HeapOopType<idecorators>::type value) {
39 if (HasDecorator<decorators, IS_NOT_NULL>::value) {
40 return CompressedOops::decode_not_null(value);
41 } else {
42 return CompressedOops::decode(value);
43 }
44 }
45
46 template <DecoratorSet decorators>
47 template <DecoratorSet idecorators, typename T>
48 inline typename EnableIf<
49 AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
50 typename HeapOopType<idecorators>::type>::type
51 RawAccessBarrier<decorators>::encode_internal(T value) {
52 if (HasDecorator<decorators, IS_NOT_NULL>::value) {
53 return CompressedOops::encode_not_null(value);
54 } else {
55 return CompressedOops::encode(value);
56 }
57 }
58
59 template <DecoratorSet decorators>
60 template <typename T>
61 inline void RawAccessBarrier<decorators>::oop_store(void* addr, T value) {
62 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
63 Encoded encoded = encode(value);
64 store(reinterpret_cast<Encoded*>(addr), encoded);
65 }
66
67 template <DecoratorSet decorators>
68 template <typename T>
69 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) {
70 oop_store(field_addr(base, offset), value);
71 }
72
73 template <DecoratorSet decorators>
74 template <typename T>
75 inline T RawAccessBarrier<decorators>::oop_load(void* addr) {
76 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
77 Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr));
78 return decode<T>(encoded);
79 }
80
81 template <DecoratorSet decorators>
82 template <typename T>
83 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) {
84 return oop_load<T>(field_addr(base, offset));
85 }
86
87 template <DecoratorSet decorators>
88 template <typename T>
89 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(T new_value, void* addr, T compare_value) {
90 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
91 Encoded encoded_new = encode(new_value);
92 Encoded encoded_compare = encode(compare_value);
93 Encoded encoded_result = atomic_cmpxchg(encoded_new,
94 reinterpret_cast<Encoded*>(addr),
95 encoded_compare);
96 return decode<T>(encoded_result);
97 }
98
99 template <DecoratorSet decorators>
100 template <typename T>
101 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
102 return oop_atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
103 }
104
105 template <DecoratorSet decorators>
106 template <typename T>
107 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(T new_value, void* addr) {
108 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
109 Encoded encoded_new = encode(new_value);
110 Encoded encoded_result = atomic_xchg(encoded_new, reinterpret_cast<Encoded*>(addr));
111 return decode<T>(encoded_result);
112 }
113
114 template <DecoratorSet decorators>
115 template <typename T>
116 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
117 return oop_atomic_xchg(new_value, field_addr(base, offset));
118 }
119
120 template <DecoratorSet decorators>
121 template <typename T>
122 inline void RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
123 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
124 size_t length) {
125 arraycopy(src_obj, src_offset_in_bytes, src_raw,
126 dst_obj, dst_offset_in_bytes, dst_raw,
127 length);
128 }
129
130 template <DecoratorSet decorators>
131 template <DecoratorSet ds, typename T>
132 inline typename EnableIf<
133 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
134 RawAccessBarrier<decorators>::load_internal(void* addr) {
135 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
136 OrderAccess::fence();
137 }
138 return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr));
139 }
140
141 template <DecoratorSet decorators>
142 template <DecoratorSet ds, typename T>
143 inline typename EnableIf<
144 HasDecorator<ds, MO_ACQUIRE>::value, T>::type
145 RawAccessBarrier<decorators>::load_internal(void* addr) {
146 return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr));
147 }
148
149 template <DecoratorSet decorators>
150 template <DecoratorSet ds, typename T>
151 inline typename EnableIf<
152 HasDecorator<ds, MO_RELAXED>::value, T>::type
153 RawAccessBarrier<decorators>::load_internal(void* addr) {
154 return Atomic::load(reinterpret_cast<const volatile T*>(addr));
155 }
156
157 template <DecoratorSet decorators>
158 template <DecoratorSet ds, typename T>
159 inline typename EnableIf<
160 HasDecorator<ds, MO_SEQ_CST>::value>::type
161 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
162 OrderAccess::release_store_fence(reinterpret_cast<volatile T*>(addr), value);
163 }
164
165 template <DecoratorSet decorators>
166 template <DecoratorSet ds, typename T>
167 inline typename EnableIf<
168 HasDecorator<ds, MO_RELEASE>::value>::type
169 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
170 OrderAccess::release_store(reinterpret_cast<volatile T*>(addr), value);
171 }
172
173 template <DecoratorSet decorators>
174 template <DecoratorSet ds, typename T>
175 inline typename EnableIf<
176 HasDecorator<ds, MO_RELAXED>::value>::type
177 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
178 Atomic::store(value, reinterpret_cast<volatile T*>(addr));
179 }
180
181 template <DecoratorSet decorators>
182 template <DecoratorSet ds, typename T>
183 inline typename EnableIf<
184 HasDecorator<ds, MO_RELAXED>::value, T>::type
185 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(T new_value, void* addr, T compare_value) {
186 return Atomic::cmpxchg(new_value,
187 reinterpret_cast<volatile T*>(addr),
188 compare_value,
189 memory_order_relaxed);
190 }
191
192 template <DecoratorSet decorators>
193 template <DecoratorSet ds, typename T>
194 inline typename EnableIf<
195 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
196 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(T new_value, void* addr, T compare_value) {
197 return Atomic::cmpxchg(new_value,
198 reinterpret_cast<volatile T*>(addr),
199 compare_value,
200 memory_order_conservative);
201 }
202
203 template <DecoratorSet decorators>
204 template <DecoratorSet ds, typename T>
205 inline typename EnableIf<
206 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
207 RawAccessBarrier<decorators>::atomic_xchg_internal(T new_value, void* addr) {
208 return Atomic::xchg(new_value,
209 reinterpret_cast<volatile T*>(addr));
210 }
211
212 // For platforms that do not have native support for wide atomics,
213 // we can emulate the atomicity using a lock. So here we check
214 // whether that is necessary or not.
215
216 template <DecoratorSet ds>
217 template <DecoratorSet decorators, typename T>
218 inline typename EnableIf<
219 AccessInternal::PossiblyLockedAccess<T>::value, T>::type
220 RawAccessBarrier<ds>::atomic_xchg_maybe_locked(T new_value, void* addr) {
221 if (!AccessInternal::wide_atomic_needs_locking()) {
222 return atomic_xchg_internal<ds>(new_value, addr);
223 } else {
224 AccessInternal::AccessLocker access_lock;
225 volatile T* p = reinterpret_cast<volatile T*>(addr);
226 T old_val = RawAccess<>::load(p);
227 RawAccess<>::store(p, new_value);
228 return old_val;
229 }
230 }
231
232 template <DecoratorSet ds>
233 template <DecoratorSet decorators, typename T>
234 inline typename EnableIf<
235 AccessInternal::PossiblyLockedAccess<T>::value, T>::type
236 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value) {
237 if (!AccessInternal::wide_atomic_needs_locking()) {
238 return atomic_cmpxchg_internal<ds>(new_value, addr, compare_value);
239 } else {
240 AccessInternal::AccessLocker access_lock;
241 volatile T* p = reinterpret_cast<volatile T*>(addr);
242 T old_val = RawAccess<>::load(p);
243 if (old_val == compare_value) {
244 RawAccess<>::store(p, new_value);
245 }
246 return old_val;
247 }
248 }
249
250 class RawAccessBarrierArrayCopy: public AllStatic {
251 template<typename T> struct IsHeapWordSized: public IntegralConstant<bool, sizeof(T) == HeapWordSize> { };
252 public:
253 template <DecoratorSet decorators, typename T>
254 static inline typename EnableIf<
255 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
256 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
257 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
258 size_t length) {
259 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
260 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
261
262 // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic.
263 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
264 AccessInternal::arraycopy_arrayof_conjoint_oops(src_raw, dst_raw, length);
265 } else {
266 typedef typename HeapOopType<decorators>::type OopType;
267 AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src_raw),
268 reinterpret_cast<OopType*>(dst_raw), length);
269 }
270 }
271
272 template <DecoratorSet decorators, typename T>
273 static inline typename EnableIf<
274 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
275 HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value>::type
276 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
277 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
278 size_t length) {
279 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
280 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
281
282 AccessInternal::arraycopy_arrayof_conjoint(src_raw, dst_raw, length);
283 }
284
285 template <DecoratorSet decorators, typename T>
286 static inline typename EnableIf<
287 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
288 HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value>::type
289 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
290 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
291 size_t length) {
292 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
293 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
294
295 // There is only a disjoint optimization for word granularity copying
296 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
297 AccessInternal::arraycopy_disjoint_words_atomic(src_raw, dst_raw, length);
298 } else {
299 AccessInternal::arraycopy_disjoint_words(src_raw, dst_raw, length);
300 }
301 }
302
303 template <DecoratorSet decorators, typename T>
304 static inline typename EnableIf<
305 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
306 !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
307 !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
308 !HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
309 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
310 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
311 size_t length) {
312 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
313 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
314
315 AccessInternal::arraycopy_conjoint(src_raw, dst_raw, length);
316 }
317
318 template <DecoratorSet decorators, typename T>
319 static inline typename EnableIf<
320 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
321 !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
322 !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
323 HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
324 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
325 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
326 size_t length) {
327 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
328 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
329
330 AccessInternal::arraycopy_conjoint_atomic(src_raw, dst_raw, length);
331 }
332 };
333
334 template<> struct RawAccessBarrierArrayCopy::IsHeapWordSized<void>: public IntegralConstant<bool, false> { };
335
336 template <DecoratorSet decorators>
337 template <typename T>
338 inline void RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
339 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
340 size_t length) {
341 RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
342 dst_obj, dst_offset_in_bytes, dst_raw,
343 length);
344 }
345
346 template <DecoratorSet decorators>
347 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
348 // 4839641 (4840070): We must do an oop-atomic copy, because if another thread
349 // is modifying a reference field in the clonee, a non-oop-atomic copy might
350 // be suspended in the middle of copying the pointer and end up with parts
351 // of two different pointers in the field. Subsequent dereferences will crash.
352 // 4846409: an oop-copy of objects with long or double fields or arrays of same
353 // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead
354 // of oops. We know objects are aligned on a minimum of an jlong boundary.
355 // The same is true of StubRoutines::object_copy and the various oop_copy
356 // variants, and of the code generated by the inline_native_clone intrinsic.
357
358 assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
359 AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src),
360 reinterpret_cast<jlong*>((oopDesc*)dst),
361 align_object_size(size) / HeapWordsPerLong);
362 // Clear the header
363 dst->init_mark_raw();
364 }
365
366 template <DecoratorSet decorators>
367 inline void RawAccessBarrier<decorators>::value_copy(void* src, void* dst, ValueKlass* md) {
368 assert(is_aligned(src, md->get_alignment()) && is_aligned(dst, md->get_alignment()), "Unalign value_copy");
369 AccessInternal::arraycopy_conjoint_atomic(src, dst, static_cast<size_t>(md->get_exact_size_in_bytes()));
370 }
371 #endif // SHARE_OOPS_ACCESSBACKEND_INLINE_HPP