1 /*
2 * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP
26 #define SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP
27
28 #include "oops/access.hpp"
29 #include "oops/accessBackend.hpp"
30 #include "oops/oop.inline.hpp"
31
32 template <DecoratorSet decorators>
33 template <DecoratorSet idecorators, typename T>
34 inline typename EnableIf<
35 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
36 RawAccessBarrier<decorators>::decode_internal(typename HeapOopType<idecorators>::type value) {
37 if (HasDecorator<decorators, OOP_NOT_NULL>::value) {
38 return oopDesc::decode_heap_oop_not_null(value);
39 } else {
40 return oopDesc::decode_heap_oop(value);
41 }
42 }
43
44 template <DecoratorSet decorators>
45 template <DecoratorSet idecorators, typename T>
46 inline typename EnableIf<
47 AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
48 typename HeapOopType<idecorators>::type>::type
49 RawAccessBarrier<decorators>::encode_internal(T value) {
50 if (HasDecorator<decorators, OOP_NOT_NULL>::value) {
51 return oopDesc::encode_heap_oop_not_null(value);
52 } else {
53 return oopDesc::encode_heap_oop(value);
54 }
55 }
56
57 template <DecoratorSet decorators>
58 template <typename T>
59 inline void RawAccessBarrier<decorators>::oop_store(void* addr, T value) {
60 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
61 Encoded encoded = encode(value);
62 store(reinterpret_cast<Encoded*>(addr), encoded);
63 }
64
65 template <DecoratorSet decorators>
66 template <typename T>
67 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) {
68 oop_store(field_addr(base, offset), value);
69 }
70
71 template <DecoratorSet decorators>
72 template <typename T>
73 inline T RawAccessBarrier<decorators>::oop_load(void* addr) {
74 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
75 Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr));
76 return decode<T>(encoded);
77 }
78
79 template <DecoratorSet decorators>
80 template <typename T>
81 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) {
82 return oop_load<T>(field_addr(base, offset));
83 }
84
85 template <DecoratorSet decorators>
86 template <typename T>
87 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(T new_value, void* addr, T compare_value) {
88 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
89 Encoded encoded_new = encode(new_value);
90 Encoded encoded_compare = encode(compare_value);
91 Encoded encoded_result = atomic_cmpxchg(encoded_new,
92 reinterpret_cast<Encoded*>(addr),
93 encoded_compare);
94 return decode<T>(encoded_result);
95 }
96
97 template <DecoratorSet decorators>
98 template <typename T>
99 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
100 return oop_atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
101 }
102
103 template <DecoratorSet decorators>
104 template <typename T>
105 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(T new_value, void* addr) {
106 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
107 Encoded encoded_new = encode(new_value);
108 Encoded encoded_result = atomic_xchg(encoded_new, reinterpret_cast<Encoded*>(addr));
109 return decode<T>(encoded_result);
110 }
111
112 template <DecoratorSet decorators>
113 template <typename T>
114 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
115 return oop_atomic_xchg(new_value, field_addr(base, offset));
116 }
117
118 template <DecoratorSet decorators>
119 template <typename T>
120 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
121 return arraycopy(src_obj, dst_obj, src, dst, length);
122 }
123
124 template <DecoratorSet decorators>
125 template <DecoratorSet ds, typename T>
126 inline typename EnableIf<
127 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
128 RawAccessBarrier<decorators>::load_internal(void* addr) {
129 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
130 OrderAccess::fence();
131 }
132 return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr));
133 }
134
135 template <DecoratorSet decorators>
136 template <DecoratorSet ds, typename T>
137 inline typename EnableIf<
138 HasDecorator<ds, MO_ACQUIRE>::value, T>::type
139 RawAccessBarrier<decorators>::load_internal(void* addr) {
140 return OrderAccess::load_acquire(reinterpret_cast<const volatile T*>(addr));
141 }
142
143 template <DecoratorSet decorators>
144 template <DecoratorSet ds, typename T>
145 inline typename EnableIf<
146 HasDecorator<ds, MO_RELAXED>::value, T>::type
147 RawAccessBarrier<decorators>::load_internal(void* addr) {
148 return Atomic::load(reinterpret_cast<const volatile T*>(addr));
149 }
150
151 template <DecoratorSet decorators>
152 template <DecoratorSet ds, typename T>
153 inline typename EnableIf<
154 HasDecorator<ds, MO_SEQ_CST>::value>::type
155 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
156 OrderAccess::release_store_fence(reinterpret_cast<volatile T*>(addr), value);
157 }
158
159 template <DecoratorSet decorators>
160 template <DecoratorSet ds, typename T>
161 inline typename EnableIf<
162 HasDecorator<ds, MO_RELEASE>::value>::type
163 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
164 OrderAccess::release_store(reinterpret_cast<volatile T*>(addr), value);
165 }
166
167 template <DecoratorSet decorators>
168 template <DecoratorSet ds, typename T>
169 inline typename EnableIf<
170 HasDecorator<ds, MO_RELAXED>::value>::type
171 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
172 Atomic::store(value, reinterpret_cast<volatile T*>(addr));
173 }
174
175 template <DecoratorSet decorators>
176 template <DecoratorSet ds, typename T>
177 inline typename EnableIf<
178 HasDecorator<ds, MO_RELAXED>::value, T>::type
179 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(T new_value, void* addr, T compare_value) {
180 return Atomic::cmpxchg(new_value,
181 reinterpret_cast<volatile T*>(addr),
182 compare_value,
183 memory_order_relaxed);
184 }
185
186 template <DecoratorSet decorators>
187 template <DecoratorSet ds, typename T>
188 inline typename EnableIf<
189 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
190 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(T new_value, void* addr, T compare_value) {
191 return Atomic::cmpxchg(new_value,
192 reinterpret_cast<volatile T*>(addr),
193 compare_value,
194 memory_order_conservative);
195 }
196
197 template <DecoratorSet decorators>
198 template <DecoratorSet ds, typename T>
199 inline typename EnableIf<
200 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
201 RawAccessBarrier<decorators>::atomic_xchg_internal(T new_value, void* addr) {
202 return Atomic::xchg(new_value,
203 reinterpret_cast<volatile T*>(addr));
204 }
205
206 // For platforms that do not have native support for wide atomics,
207 // we can emulate the atomicity using a lock. So here we check
208 // whether that is necessary or not.
209
210 template <DecoratorSet ds>
211 template <DecoratorSet decorators, typename T>
212 inline typename EnableIf<
213 AccessInternal::PossiblyLockedAccess<T>::value, T>::type
214 RawAccessBarrier<ds>::atomic_xchg_maybe_locked(T new_value, void* addr) {
215 if (!AccessInternal::wide_atomic_needs_locking()) {
216 return atomic_xchg_internal<ds>(new_value, addr);
217 } else {
218 AccessInternal::AccessLocker access_lock;
219 volatile T* p = reinterpret_cast<volatile T*>(addr);
220 T old_val = RawAccess<>::load(p);
221 RawAccess<>::store(p, new_value);
222 return old_val;
223 }
224 }
225
226 template <DecoratorSet ds>
227 template <DecoratorSet decorators, typename T>
228 inline typename EnableIf<
229 AccessInternal::PossiblyLockedAccess<T>::value, T>::type
230 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value) {
231 if (!AccessInternal::wide_atomic_needs_locking()) {
232 return atomic_cmpxchg_internal<ds>(new_value, addr, compare_value);
233 } else {
234 AccessInternal::AccessLocker access_lock;
235 volatile T* p = reinterpret_cast<volatile T*>(addr);
236 T old_val = RawAccess<>::load(p);
237 if (old_val == compare_value) {
238 RawAccess<>::store(p, new_value);
239 }
240 return old_val;
241 }
242 }
243
244 class RawAccessBarrierArrayCopy: public AllStatic {
245 public:
246 template <DecoratorSet decorators, typename T>
247 static inline typename EnableIf<
248 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
249 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
250 // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic.
251 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
252 AccessInternal::arraycopy_arrayof_conjoint_oops(src, dst, length);
253 } else {
254 typedef typename HeapOopType<decorators>::type OopType;
255 AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src),
256 reinterpret_cast<OopType*>(dst), length);
257 }
258 }
259
260 template <DecoratorSet decorators, typename T>
261 static inline typename EnableIf<
262 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
263 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
264 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
265 AccessInternal::arraycopy_arrayof_conjoint(src, dst, length);
266 } else if (HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && sizeof(T) == HeapWordSize) {
267 // There is only a disjoint optimization for word granularity copying
268 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
269 AccessInternal::arraycopy_disjoint_words_atomic(src, dst, length);
270 } else {
271 AccessInternal::arraycopy_disjoint_words(src, dst, length);
272 }
273 } else {
274 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
275 AccessInternal::arraycopy_conjoint_atomic(src, dst, length);
276 } else {
277 AccessInternal::arraycopy_conjoint(src, dst, length);
278 }
279 }
280 }
281
282 template <DecoratorSet decorators>
283 static inline typename EnableIf<
284 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
285 arraycopy(arrayOop src_obj, arrayOop dst_obj, void* src, void* dst, size_t length) {
286 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
287 AccessInternal::arraycopy_conjoint_atomic(src, dst, length);
288 } else {
289 AccessInternal::arraycopy_conjoint(src, dst, length);
290 }
291 }
292 };
293
294 template <DecoratorSet decorators>
295 template <typename T>
296 inline bool RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
297 RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, dst_obj, src, dst, length);
298 return true;
299 }
300
301 template <DecoratorSet decorators>
302 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
303 // 4839641 (4840070): We must do an oop-atomic copy, because if another thread
304 // is modifying a reference field in the clonee, a non-oop-atomic copy might
305 // be suspended in the middle of copying the pointer and end up with parts
306 // of two different pointers in the field. Subsequent dereferences will crash.
307 // 4846409: an oop-copy of objects with long or double fields or arrays of same
308 // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead
309 // of oops. We know objects are aligned on a minimum of an jlong boundary.
310 // The same is true of StubRoutines::object_copy and the various oop_copy
311 // variants, and of the code generated by the inline_native_clone intrinsic.
312
313 assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
314 AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src),
315 reinterpret_cast<jlong*>((oopDesc*)dst),
316 align_object_size(size) / HeapWordsPerLong);
317 // Clear the header
318 dst->init_mark_raw();
319 }
320
321 #endif // SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP