1 /*
2 * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_OOPS_ACCESSBACKEND_HPP
26 #define SHARE_OOPS_ACCESSBACKEND_HPP
27
28 #include "gc/shared/barrierSetConfig.hpp"
29 #include "memory/allocation.hpp"
30 #include "metaprogramming/conditional.hpp"
31 #include "metaprogramming/decay.hpp"
32 #include "metaprogramming/enableIf.hpp"
33 #include "metaprogramming/integralConstant.hpp"
34 #include "metaprogramming/isFloatingPoint.hpp"
35 #include "metaprogramming/isIntegral.hpp"
36 #include "metaprogramming/isPointer.hpp"
37 #include "metaprogramming/isSame.hpp"
38 #include "metaprogramming/isVolatile.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/oopsHierarchy.hpp"
41 #include "runtime/globals.hpp"
42 #include "utilities/debug.hpp"
43 #include "utilities/globalDefinitions.hpp"
44
45
46 // This metafunction returns either oop or narrowOop depending on whether
47 // an access needs to use compressed oops or not.
48 template <DecoratorSet decorators>
49 struct HeapOopType: AllStatic {
50 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
51 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
52 typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type;
53 };
54
55 // This meta-function returns either oop or narrowOop depending on whether
56 // a back-end needs to consider compressed oops types or not.
57 template <DecoratorSet decorators>
58 struct ValueOopType: AllStatic {
59 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
60 typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type;
61 };
62
63 namespace AccessInternal {
64 enum BarrierType {
65 BARRIER_STORE,
66 BARRIER_STORE_AT,
67 BARRIER_LOAD,
68 BARRIER_LOAD_AT,
69 BARRIER_ATOMIC_CMPXCHG,
70 BARRIER_ATOMIC_CMPXCHG_AT,
71 BARRIER_ATOMIC_XCHG,
72 BARRIER_ATOMIC_XCHG_AT,
73 BARRIER_ARRAYCOPY,
74 BARRIER_CLONE,
75 BARRIER_VALUE_COPY,
76 BARRIER_RESOLVE
77 };
78
79 template <DecoratorSet decorators, typename T>
80 struct MustConvertCompressedOop: public IntegralConstant<bool,
81 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
82 IsSame<typename HeapOopType<decorators>::type, narrowOop>::value &&
83 IsSame<T, oop>::value> {};
84
85 // This metafunction returns an appropriate oop type if the value is oop-like
86 // and otherwise returns the same type T.
87 template <DecoratorSet decorators, typename T>
88 struct EncodedType: AllStatic {
89 typedef typename Conditional<
90 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
91 typename HeapOopType<decorators>::type, T>::type type;
92 };
93
94 template <DecoratorSet decorators>
95 inline typename HeapOopType<decorators>::type*
96 oop_field_addr(oop base, ptrdiff_t byte_offset) {
97 return reinterpret_cast<typename HeapOopType<decorators>::type*>(
98 reinterpret_cast<intptr_t>((void*)base) + byte_offset);
99 }
100
101 // This metafunction returns whether it is possible for a type T to require
102 // locking to support wide atomics or not.
103 template <typename T>
104 #ifdef SUPPORTS_NATIVE_CX8
105 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
106 #else
107 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
108 #endif
109
110 template <DecoratorSet decorators, typename T>
111 struct AccessFunctionTypes {
112 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
113 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
114 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
115 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
116
117 typedef T (*load_func_t)(void* addr);
118 typedef void (*store_func_t)(void* addr, T value);
119 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
120 typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
121
122 typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
123 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
124 size_t length);
125 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
126 typedef void (*value_copy_func_t)(void* src, void* dst, ValueKlass* md);
127 typedef oop (*resolve_func_t)(oop obj);
128 };
129
130 template <DecoratorSet decorators>
131 struct AccessFunctionTypes<decorators, void> {
132 typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
133 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
134 size_t length);
135 };
136
137 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
138
139 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
140 template <DecoratorSet decorators, typename T> \
141 struct AccessFunction<decorators, T, bt>: AllStatic{ \
142 typedef typename AccessFunctionTypes<decorators, T>::func type; \
143 }
144 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
145 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
146 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
147 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
148 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
149 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
150 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
151 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
152 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
153 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
154 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_VALUE_COPY, value_copy_func_t);
155 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_RESOLVE, resolve_func_t);
156 #undef ACCESS_GENERATE_ACCESS_FUNCTION
157
158 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
159 typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier();
160
161 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
162 typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier();
163
164 class AccessLocker {
165 public:
166 AccessLocker();
167 ~AccessLocker();
168 };
169 bool wide_atomic_needs_locking();
170
171 void* field_addr(oop base, ptrdiff_t offset);
172
173 // Forward calls to Copy:: in the cpp file to reduce dependencies and allow
174 // faster build times, given how frequently included access is.
175 void arraycopy_arrayof_conjoint_oops(void* src, void* dst, size_t length);
176 void arraycopy_conjoint_oops(oop* src, oop* dst, size_t length);
177 void arraycopy_conjoint_oops(narrowOop* src, narrowOop* dst, size_t length);
178
179 void arraycopy_disjoint_words(void* src, void* dst, size_t length);
180 void arraycopy_disjoint_words_atomic(void* src, void* dst, size_t length);
181
182 template<typename T>
183 void arraycopy_conjoint(T* src, T* dst, size_t length);
184 template<typename T>
185 void arraycopy_arrayof_conjoint(T* src, T* dst, size_t length);
186 template<typename T>
187 void arraycopy_conjoint_atomic(T* src, T* dst, size_t length);
188 }
189
190 // This mask specifies what decorators are relevant for raw accesses. When passing
191 // accesses to the raw layer, irrelevant decorators are removed.
192 const DecoratorSet RAW_DECORATOR_MASK = INTERNAL_DECORATOR_MASK | MO_DECORATOR_MASK |
193 ARRAYCOPY_DECORATOR_MASK | IS_NOT_NULL;
194
195 // The RawAccessBarrier performs raw accesses with additional knowledge of
196 // memory ordering, so that OrderAccess/Atomic is called when necessary.
197 // It additionally handles compressed oops, and hence is not completely "raw"
198 // strictly speaking.
199 template <DecoratorSet decorators>
200 class RawAccessBarrier: public AllStatic {
201 protected:
202 static inline void* field_addr(oop base, ptrdiff_t byte_offset) {
203 return AccessInternal::field_addr(base, byte_offset);
204 }
205
206 protected:
207 // Only encode if INTERNAL_VALUE_IS_OOP
208 template <DecoratorSet idecorators, typename T>
209 static inline typename EnableIf<
210 AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
211 typename HeapOopType<idecorators>::type>::type
212 encode_internal(T value);
213
214 template <DecoratorSet idecorators, typename T>
215 static inline typename EnableIf<
216 !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
217 encode_internal(T value) {
218 return value;
219 }
220
221 template <typename T>
222 static inline typename AccessInternal::EncodedType<decorators, T>::type
223 encode(T value) {
224 return encode_internal<decorators, T>(value);
225 }
226
227 // Only decode if INTERNAL_VALUE_IS_OOP
228 template <DecoratorSet idecorators, typename T>
229 static inline typename EnableIf<
230 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
231 decode_internal(typename HeapOopType<idecorators>::type value);
232
233 template <DecoratorSet idecorators, typename T>
234 static inline typename EnableIf<
235 !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
236 decode_internal(T value) {
237 return value;
238 }
239
240 template <typename T>
241 static inline T decode(typename AccessInternal::EncodedType<decorators, T>::type value) {
242 return decode_internal<decorators, T>(value);
243 }
244
245 protected:
246 template <DecoratorSet ds, typename T>
247 static typename EnableIf<
248 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
249 load_internal(void* addr);
250
251 template <DecoratorSet ds, typename T>
252 static typename EnableIf<
253 HasDecorator<ds, MO_ACQUIRE>::value, T>::type
254 load_internal(void* addr);
255
256 template <DecoratorSet ds, typename T>
257 static typename EnableIf<
258 HasDecorator<ds, MO_RELAXED>::value, T>::type
259 load_internal(void* addr);
260
261 template <DecoratorSet ds, typename T>
262 static inline typename EnableIf<
263 HasDecorator<ds, MO_VOLATILE>::value, T>::type
264 load_internal(void* addr) {
265 return *reinterpret_cast<const volatile T*>(addr);
266 }
267
268 template <DecoratorSet ds, typename T>
269 static inline typename EnableIf<
270 HasDecorator<ds, MO_UNORDERED>::value, T>::type
271 load_internal(void* addr) {
272 return *reinterpret_cast<T*>(addr);
273 }
274
275 template <DecoratorSet ds, typename T>
276 static typename EnableIf<
277 HasDecorator<ds, MO_SEQ_CST>::value>::type
278 store_internal(void* addr, T value);
279
280 template <DecoratorSet ds, typename T>
281 static typename EnableIf<
282 HasDecorator<ds, MO_RELEASE>::value>::type
283 store_internal(void* addr, T value);
284
285 template <DecoratorSet ds, typename T>
286 static typename EnableIf<
287 HasDecorator<ds, MO_RELAXED>::value>::type
288 store_internal(void* addr, T value);
289
290 template <DecoratorSet ds, typename T>
291 static inline typename EnableIf<
292 HasDecorator<ds, MO_VOLATILE>::value>::type
293 store_internal(void* addr, T value) {
294 (void)const_cast<T&>(*reinterpret_cast<volatile T*>(addr) = value);
295 }
296
297 template <DecoratorSet ds, typename T>
298 static inline typename EnableIf<
299 HasDecorator<ds, MO_UNORDERED>::value>::type
300 store_internal(void* addr, T value) {
301 *reinterpret_cast<T*>(addr) = value;
302 }
303
304 template <DecoratorSet ds, typename T>
305 static typename EnableIf<
306 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
307 atomic_cmpxchg_internal(T new_value, void* addr, T compare_value);
308
309 template <DecoratorSet ds, typename T>
310 static typename EnableIf<
311 HasDecorator<ds, MO_RELAXED>::value, T>::type
312 atomic_cmpxchg_internal(T new_value, void* addr, T compare_value);
313
314 template <DecoratorSet ds, typename T>
315 static typename EnableIf<
316 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
317 atomic_xchg_internal(T new_value, void* addr);
318
319 // The following *_locked mechanisms serve the purpose of handling atomic operations
320 // that are larger than a machine can handle, and then possibly opt for using
321 // a slower path using a mutex to perform the operation.
322
323 template <DecoratorSet ds, typename T>
324 static inline typename EnableIf<
325 !AccessInternal::PossiblyLockedAccess<T>::value, T>::type
326 atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value) {
327 return atomic_cmpxchg_internal<ds>(new_value, addr, compare_value);
328 }
329
330 template <DecoratorSet ds, typename T>
331 static typename EnableIf<
332 AccessInternal::PossiblyLockedAccess<T>::value, T>::type
333 atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value);
334
335 template <DecoratorSet ds, typename T>
336 static inline typename EnableIf<
337 !AccessInternal::PossiblyLockedAccess<T>::value, T>::type
338 atomic_xchg_maybe_locked(T new_value, void* addr) {
339 return atomic_xchg_internal<ds>(new_value, addr);
340 }
341
342 template <DecoratorSet ds, typename T>
343 static typename EnableIf<
344 AccessInternal::PossiblyLockedAccess<T>::value, T>::type
345 atomic_xchg_maybe_locked(T new_value, void* addr);
346
347 public:
348 template <typename T>
349 static inline void store(void* addr, T value) {
350 store_internal<decorators>(addr, value);
351 }
352
353 template <typename T>
354 static inline T load(void* addr) {
355 return load_internal<decorators, T>(addr);
356 }
357
358 template <typename T>
359 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
360 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
361 }
362
363 template <typename T>
364 static inline T atomic_xchg(T new_value, void* addr) {
365 return atomic_xchg_maybe_locked<decorators>(new_value, addr);
366 }
367
368 template <typename T>
369 static void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
370 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
371 size_t length);
372
373 template <typename T>
374 static void oop_store(void* addr, T value);
375 template <typename T>
376 static void oop_store_at(oop base, ptrdiff_t offset, T value);
377
378 template <typename T>
379 static T oop_load(void* addr);
380 template <typename T>
381 static T oop_load_at(oop base, ptrdiff_t offset);
382
383 template <typename T>
384 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
385 template <typename T>
386 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
387
388 template <typename T>
389 static T oop_atomic_xchg(T new_value, void* addr);
390 template <typename T>
391 static T oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset);
392
393 template <typename T>
394 static void store_at(oop base, ptrdiff_t offset, T value) {
395 store(field_addr(base, offset), value);
396 }
397
398 template <typename T>
399 static T load_at(oop base, ptrdiff_t offset) {
400 return load<T>(field_addr(base, offset));
401 }
402
403 template <typename T>
404 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
405 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
406 }
407
408 template <typename T>
409 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
410 return atomic_xchg(new_value, field_addr(base, offset));
411 }
412
413 template <typename T>
414 static void oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
415 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
416 size_t length);
417
418 static void clone(oop src, oop dst, size_t size);
419
420 static void value_copy(void* src, void* dst, ValueKlass* md);
421
422 static oop resolve(oop obj) { return obj; }
423 };
424
425 // Below is the implementation of the first 4 steps of the template pipeline:
426 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
427 // and sets default decorators to sensible values.
428 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
429 // multiple types. The P type of the address and T type of the value must
430 // match.
431 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
432 // avoided, and in that case avoids it (calling raw accesses or
433 // primitive accesses in a build that does not require primitive GC barriers)
434 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
435 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers
436 // to the access.
437
438 namespace AccessInternal {
439 template <typename T>
440 struct OopOrNarrowOopInternal: AllStatic {
441 typedef oop type;
442 };
443
444 template <>
445 struct OopOrNarrowOopInternal<narrowOop>: AllStatic {
446 typedef narrowOop type;
447 };
448
449 // This metafunction returns a canonicalized oop/narrowOop type for a passed
450 // in oop-like types passed in from oop_* overloads where the user has sworn
451 // that the passed in values should be oop-like (e.g. oop, oopDesc*, arrayOop,
452 // narrowOoop, instanceOopDesc*, and random other things).
453 // In the oop_* overloads, it must hold that if the passed in type T is not
454 // narrowOop, then it by contract has to be one of many oop-like types implicitly
455 // convertible to oop, and hence returns oop as the canonical oop type.
456 // If it turns out it was not, then the implicit conversion to oop will fail
457 // to compile, as desired.
458 template <typename T>
459 struct OopOrNarrowOop: AllStatic {
460 typedef typename OopOrNarrowOopInternal<typename Decay<T>::type>::type type;
461 };
462
463 inline void* field_addr(oop base, ptrdiff_t byte_offset) {
464 return reinterpret_cast<void*>(reinterpret_cast<intptr_t>((void*)base) + byte_offset);
465 }
466 // Step 4: Runtime dispatch
467 // The RuntimeDispatch class is responsible for performing a runtime dispatch of the
468 // accessor. This is required when the access either depends on whether compressed oops
469 // is being used, or it depends on which GC implementation was chosen (e.g. requires GC
470 // barriers). The way it works is that a function pointer initially pointing to an
471 // accessor resolution function gets called for each access. Upon first invocation,
472 // it resolves which accessor to be used in future invocations and patches the
473 // function pointer to this new accessor.
474
475 template <DecoratorSet decorators, typename T, BarrierType type>
476 struct RuntimeDispatch: AllStatic {};
477
478 template <DecoratorSet decorators, typename T>
479 struct RuntimeDispatch<decorators, T, BARRIER_STORE>: AllStatic {
480 typedef typename AccessFunction<decorators, T, BARRIER_STORE>::type func_t;
481 static func_t _store_func;
482
483 static void store_init(void* addr, T value);
484
485 static inline void store(void* addr, T value) {
486 _store_func(addr, value);
487 }
488 };
489
490 template <DecoratorSet decorators, typename T>
491 struct RuntimeDispatch<decorators, T, BARRIER_STORE_AT>: AllStatic {
492 typedef typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type func_t;
493 static func_t _store_at_func;
494
495 static void store_at_init(oop base, ptrdiff_t offset, T value);
496
497 static inline void store_at(oop base, ptrdiff_t offset, T value) {
498 _store_at_func(base, offset, value);
499 }
500 };
501
502 template <DecoratorSet decorators, typename T>
503 struct RuntimeDispatch<decorators, T, BARRIER_LOAD>: AllStatic {
504 typedef typename AccessFunction<decorators, T, BARRIER_LOAD>::type func_t;
505 static func_t _load_func;
506
507 static T load_init(void* addr);
508
509 static inline T load(void* addr) {
510 return _load_func(addr);
511 }
512 };
513
514 template <DecoratorSet decorators, typename T>
515 struct RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>: AllStatic {
516 typedef typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type func_t;
517 static func_t _load_at_func;
518
519 static T load_at_init(oop base, ptrdiff_t offset);
520
521 static inline T load_at(oop base, ptrdiff_t offset) {
522 return _load_at_func(base, offset);
523 }
524 };
525
526 template <DecoratorSet decorators, typename T>
527 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>: AllStatic {
528 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type func_t;
529 static func_t _atomic_cmpxchg_func;
530
531 static T atomic_cmpxchg_init(T new_value, void* addr, T compare_value);
532
533 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
534 return _atomic_cmpxchg_func(new_value, addr, compare_value);
535 }
536 };
537
538 template <DecoratorSet decorators, typename T>
539 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>: AllStatic {
540 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type func_t;
541 static func_t _atomic_cmpxchg_at_func;
542
543 static T atomic_cmpxchg_at_init(T new_value, oop base, ptrdiff_t offset, T compare_value);
544
545 static inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
546 return _atomic_cmpxchg_at_func(new_value, base, offset, compare_value);
547 }
548 };
549
550 template <DecoratorSet decorators, typename T>
551 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>: AllStatic {
552 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type func_t;
553 static func_t _atomic_xchg_func;
554
555 static T atomic_xchg_init(T new_value, void* addr);
556
557 static inline T atomic_xchg(T new_value, void* addr) {
558 return _atomic_xchg_func(new_value, addr);
559 }
560 };
561
562 template <DecoratorSet decorators, typename T>
563 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
564 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
565 static func_t _atomic_xchg_at_func;
566
567 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
568
569 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
570 return _atomic_xchg_at_func(new_value, base, offset);
571 }
572 };
573
574 template <DecoratorSet decorators, typename T>
575 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
576 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
577 static func_t _arraycopy_func;
578
579 static void arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
580 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
581 size_t length);
582
583 static inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
584 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
585 size_t length) {
586 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
587 dst_obj, dst_offset_in_bytes, dst_raw,
588 length);
589 }
590 };
591
592 template <DecoratorSet decorators, typename T>
593 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
594 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
595 static func_t _clone_func;
596
597 static void clone_init(oop src, oop dst, size_t size);
598
599 static inline void clone(oop src, oop dst, size_t size) {
600 _clone_func(src, dst, size);
601 }
602 };
603
604 template <DecoratorSet decorators, typename T>
605 struct RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>: AllStatic {
606 typedef typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type func_t;
607 static func_t _value_copy_func;
608
609 static void value_copy_init(void* src, void* dst, ValueKlass* md);
610
611 static inline void value_copy(void* src, void* dst, ValueKlass* md) {
612 _value_copy_func(src, dst, md);
613 }
614 };
615
616 template <DecoratorSet decorators, typename T>
617 struct RuntimeDispatch<decorators, T, BARRIER_RESOLVE>: AllStatic {
618 typedef typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type func_t;
619 static func_t _resolve_func;
620
621 static oop resolve_init(oop obj);
622
623 static inline oop resolve(oop obj) {
624 return _resolve_func(obj);
625 }
626 };
627
628 // Initialize the function pointers to point to the resolving function.
629 template <DecoratorSet decorators, typename T>
630 typename AccessFunction<decorators, T, BARRIER_STORE>::type
631 RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init;
632
633 template <DecoratorSet decorators, typename T>
634 typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type
635 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init;
636
637 template <DecoratorSet decorators, typename T>
638 typename AccessFunction<decorators, T, BARRIER_LOAD>::type
639 RuntimeDispatch<decorators, T, BARRIER_LOAD>::_load_func = &load_init;
640
641 template <DecoratorSet decorators, typename T>
642 typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type
643 RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::_load_at_func = &load_at_init;
644
645 template <DecoratorSet decorators, typename T>
646 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type
647 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::_atomic_cmpxchg_func = &atomic_cmpxchg_init;
648
649 template <DecoratorSet decorators, typename T>
650 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type
651 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init;
652
653 template <DecoratorSet decorators, typename T>
654 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type
655 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init;
656
657 template <DecoratorSet decorators, typename T>
658 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type
659 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init;
660
661 template <DecoratorSet decorators, typename T>
662 typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type
663 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init;
664
665 template <DecoratorSet decorators, typename T>
666 typename AccessFunction<decorators, T, BARRIER_CLONE>::type
667 RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
668
669 template <DecoratorSet decorators, typename T>
670 typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type
671 RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>::_value_copy_func = &value_copy_init;
672
673 template <DecoratorSet decorators, typename T>
674 typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type
675 RuntimeDispatch<decorators, T, BARRIER_RESOLVE>::_resolve_func = &resolve_init;
676
677 // Step 3: Pre-runtime dispatching.
678 // The PreRuntimeDispatch class is responsible for filtering the barrier strength
679 // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime
680 // dispatch point. Otherwise it goes through a runtime check if hardwiring was
681 // not possible.
682 struct PreRuntimeDispatch: AllStatic {
683 template<DecoratorSet decorators>
684 struct CanHardwireRaw: public IntegralConstant<
685 bool,
686 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access
687 !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address)
688 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address)
689 {};
690
691 static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP;
692
693 template<DecoratorSet decorators>
694 static bool is_hardwired_primitive() {
695 return !HasDecorator<decorators, INTERNAL_BT_BARRIER_ON_PRIMITIVES>::value &&
696 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value;
697 }
698
699 template <DecoratorSet decorators, typename T>
700 inline static typename EnableIf<
701 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value>::type
702 store(void* addr, T value) {
703 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
704 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
705 Raw::oop_store(addr, value);
706 } else {
707 Raw::store(addr, value);
708 }
709 }
710
711 template <DecoratorSet decorators, typename T>
712 inline static typename EnableIf<
713 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value>::type
714 store(void* addr, T value) {
715 if (UseCompressedOops) {
716 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
717 PreRuntimeDispatch::store<expanded_decorators>(addr, value);
718 } else {
719 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
720 PreRuntimeDispatch::store<expanded_decorators>(addr, value);
721 }
722 }
723
724 template <DecoratorSet decorators, typename T>
725 inline static typename EnableIf<
726 !HasDecorator<decorators, AS_RAW>::value>::type
727 store(void* addr, T value) {
728 if (is_hardwired_primitive<decorators>()) {
729 const DecoratorSet expanded_decorators = decorators | AS_RAW;
730 PreRuntimeDispatch::store<expanded_decorators>(addr, value);
731 } else {
732 RuntimeDispatch<decorators, T, BARRIER_STORE>::store(addr, value);
733 }
734 }
735
736 template <DecoratorSet decorators, typename T>
737 inline static typename EnableIf<
738 HasDecorator<decorators, AS_RAW>::value>::type
739 store_at(oop base, ptrdiff_t offset, T value) {
740 store<decorators>(field_addr(base, offset), value);
741 }
742
743 template <DecoratorSet decorators, typename T>
744 inline static typename EnableIf<
745 !HasDecorator<decorators, AS_RAW>::value>::type
746 store_at(oop base, ptrdiff_t offset, T value) {
747 if (is_hardwired_primitive<decorators>()) {
748 const DecoratorSet expanded_decorators = decorators | AS_RAW;
749 PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, value);
750 } else {
751 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::store_at(base, offset, value);
752 }
753 }
754
755 template <DecoratorSet decorators, typename T>
756 inline static typename EnableIf<
757 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type
758 load(void* addr) {
759 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
760 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
761 return Raw::template oop_load<T>(addr);
762 } else {
763 return Raw::template load<T>(addr);
764 }
765 }
766
767 template <DecoratorSet decorators, typename T>
768 inline static typename EnableIf<
769 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type
770 load(void* addr) {
771 if (UseCompressedOops) {
772 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
773 return PreRuntimeDispatch::load<expanded_decorators, T>(addr);
774 } else {
775 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
776 return PreRuntimeDispatch::load<expanded_decorators, T>(addr);
777 }
778 }
779
780 template <DecoratorSet decorators, typename T>
781 inline static typename EnableIf<
782 !HasDecorator<decorators, AS_RAW>::value, T>::type
783 load(void* addr) {
784 if (is_hardwired_primitive<decorators>()) {
785 const DecoratorSet expanded_decorators = decorators | AS_RAW;
786 return PreRuntimeDispatch::load<expanded_decorators, T>(addr);
787 } else {
788 return RuntimeDispatch<decorators, T, BARRIER_LOAD>::load(addr);
789 }
790 }
791
792 template <DecoratorSet decorators, typename T>
793 inline static typename EnableIf<
794 HasDecorator<decorators, AS_RAW>::value, T>::type
795 load_at(oop base, ptrdiff_t offset) {
796 return load<decorators, T>(field_addr(base, offset));
797 }
798
799 template <DecoratorSet decorators, typename T>
800 inline static typename EnableIf<
801 !HasDecorator<decorators, AS_RAW>::value, T>::type
802 load_at(oop base, ptrdiff_t offset) {
803 if (is_hardwired_primitive<decorators>()) {
804 const DecoratorSet expanded_decorators = decorators | AS_RAW;
805 return PreRuntimeDispatch::load_at<expanded_decorators, T>(base, offset);
806 } else {
807 return RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::load_at(base, offset);
808 }
809 }
810
811 template <DecoratorSet decorators, typename T>
812 inline static typename EnableIf<
813 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type
814 atomic_cmpxchg(T new_value, void* addr, T compare_value) {
815 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
816 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
817 return Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);
818 } else {
819 return Raw::atomic_cmpxchg(new_value, addr, compare_value);
820 }
821 }
822
823 template <DecoratorSet decorators, typename T>
824 inline static typename EnableIf<
825 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type
826 atomic_cmpxchg(T new_value, void* addr, T compare_value) {
827 if (UseCompressedOops) {
828 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
829 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value);
830 } else {
831 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
832 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value);
833 }
834 }
835
836 template <DecoratorSet decorators, typename T>
837 inline static typename EnableIf<
838 !HasDecorator<decorators, AS_RAW>::value, T>::type
839 atomic_cmpxchg(T new_value, void* addr, T compare_value) {
840 if (is_hardwired_primitive<decorators>()) {
841 const DecoratorSet expanded_decorators = decorators | AS_RAW;
842 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value);
843 } else {
844 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::atomic_cmpxchg(new_value, addr, compare_value);
845 }
846 }
847
848 template <DecoratorSet decorators, typename T>
849 inline static typename EnableIf<
850 HasDecorator<decorators, AS_RAW>::value, T>::type
851 atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
852 return atomic_cmpxchg<decorators>(new_value, field_addr(base, offset), compare_value);
853 }
854
855 template <DecoratorSet decorators, typename T>
856 inline static typename EnableIf<
857 !HasDecorator<decorators, AS_RAW>::value, T>::type
858 atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
859 if (is_hardwired_primitive<decorators>()) {
860 const DecoratorSet expanded_decorators = decorators | AS_RAW;
861 return PreRuntimeDispatch::atomic_cmpxchg_at<expanded_decorators>(new_value, base, offset, compare_value);
862 } else {
863 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::atomic_cmpxchg_at(new_value, base, offset, compare_value);
864 }
865 }
866
867 template <DecoratorSet decorators, typename T>
868 inline static typename EnableIf<
869 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type
870 atomic_xchg(T new_value, void* addr) {
871 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
872 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
873 return Raw::oop_atomic_xchg(new_value, addr);
874 } else {
875 return Raw::atomic_xchg(new_value, addr);
876 }
877 }
878
879 template <DecoratorSet decorators, typename T>
880 inline static typename EnableIf<
881 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type
882 atomic_xchg(T new_value, void* addr) {
883 if (UseCompressedOops) {
884 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
885 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr);
886 } else {
887 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
888 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr);
889 }
890 }
891
892 template <DecoratorSet decorators, typename T>
893 inline static typename EnableIf<
894 !HasDecorator<decorators, AS_RAW>::value, T>::type
895 atomic_xchg(T new_value, void* addr) {
896 if (is_hardwired_primitive<decorators>()) {
897 const DecoratorSet expanded_decorators = decorators | AS_RAW;
898 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr);
899 } else {
900 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::atomic_xchg(new_value, addr);
901 }
902 }
903
904 template <DecoratorSet decorators, typename T>
905 inline static typename EnableIf<
906 HasDecorator<decorators, AS_RAW>::value, T>::type
907 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
908 return atomic_xchg<decorators>(new_value, field_addr(base, offset));
909 }
910
911 template <DecoratorSet decorators, typename T>
912 inline static typename EnableIf<
913 !HasDecorator<decorators, AS_RAW>::value, T>::type
914 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
915 if (is_hardwired_primitive<decorators>()) {
916 const DecoratorSet expanded_decorators = decorators | AS_RAW;
917 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
918 } else {
919 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
920 }
921 }
922
923 template <DecoratorSet decorators, typename T>
924 inline static typename EnableIf<
925 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, void>::type
926 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
927 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
928 size_t length) {
929 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
930 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
931 Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
932 dst_obj, dst_offset_in_bytes, dst_raw,
933 length);
934 } else {
935 Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
936 dst_obj, dst_offset_in_bytes, dst_raw,
937 length);
938 }
939 }
940
941 template <DecoratorSet decorators, typename T>
942 inline static typename EnableIf<
943 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, void>::type
944 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
945 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
946 size_t length) {
947 if (UseCompressedOops) {
948 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
949 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
950 dst_obj, dst_offset_in_bytes, dst_raw,
951 length);
952 } else {
953 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
954 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
955 dst_obj, dst_offset_in_bytes, dst_raw,
956 length);
957 }
958 }
959
960 template <DecoratorSet decorators, typename T>
961 inline static typename EnableIf<
962 !HasDecorator<decorators, AS_RAW>::value, void>::type
963 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
964 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
965 size_t length) {
966 if (is_hardwired_primitive<decorators>()) {
967 const DecoratorSet expanded_decorators = decorators | AS_RAW;
968 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
969 dst_obj, dst_offset_in_bytes, dst_raw,
970 length);
971 } else {
972 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
973 dst_obj, dst_offset_in_bytes, dst_raw,
974 length);
975 }
976 }
977
978 template <DecoratorSet decorators>
979 inline static typename EnableIf<
980 HasDecorator<decorators, AS_RAW>::value>::type
981 clone(oop src, oop dst, size_t size) {
982 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
983 Raw::clone(src, dst, size);
984 }
985
986 template <DecoratorSet decorators>
987 inline static typename EnableIf<
988 !HasDecorator<decorators, AS_RAW>::value>::type
989 clone(oop src, oop dst, size_t size) {
990 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
991 }
992
993 template <DecoratorSet decorators>
994 inline static typename EnableIf<
995 HasDecorator<decorators, AS_RAW>::value>::type
996 value_copy(void* src, void* dst, ValueKlass* md) {
997 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
998 Raw::value_copy(src, dst, md);
999 }
1000
1001 template <DecoratorSet decorators>
1002 inline static typename EnableIf<
1003 !HasDecorator<decorators, AS_RAW>::value>::type
1004 value_copy(void* src, void* dst, ValueKlass* md) {
1005 const DecoratorSet expanded_decorators = decorators;
1006 RuntimeDispatch<expanded_decorators, void*, BARRIER_VALUE_COPY>::value_copy(src, dst, md);
1007 }
1008
1009
1010 template <DecoratorSet decorators>
1011 inline static typename EnableIf<
1012 HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type
1013 resolve(oop obj) {
1014 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
1015 return Raw::resolve(obj);
1016 }
1017
1018 template <DecoratorSet decorators>
1019 inline static typename EnableIf<
1020 !HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type
1021 resolve(oop obj) {
1022 return RuntimeDispatch<decorators, oop, BARRIER_RESOLVE>::resolve(obj);
1023 }
1024 };
1025
1026 // Step 2: Reduce types.
1027 // Enforce that for non-oop types, T and P have to be strictly the same.
1028 // P is the type of the address and T is the type of the values.
1029 // As for oop types, it is allow to send T in {narrowOop, oop} and
1030 // P in {narrowOop, oop, HeapWord*}. The following rules apply according to
1031 // the subsequent table. (columns are P, rows are T)
1032 // | | HeapWord | oop | narrowOop |
1033 // | oop | rt-comp | hw-none | hw-comp |
1034 // | narrowOop | x | x | hw-none |
1035 //
1036 // x means not allowed
1037 // rt-comp means it must be checked at runtime whether the oop is compressed.
1038 // hw-none means it is statically known the oop will not be compressed.
1039 // hw-comp means it is statically known the oop will be compressed.
1040
1041 template <DecoratorSet decorators, typename T>
1042 inline void store_reduce_types(T* addr, T value) {
1043 PreRuntimeDispatch::store<decorators>(addr, value);
1044 }
1045
1046 template <DecoratorSet decorators>
1047 inline void store_reduce_types(narrowOop* addr, oop value) {
1048 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1049 INTERNAL_RT_USE_COMPRESSED_OOPS;
1050 PreRuntimeDispatch::store<expanded_decorators>(addr, value);
1051 }
1052
1053 template <DecoratorSet decorators>
1054 inline void store_reduce_types(narrowOop* addr, narrowOop value) {
1055 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1056 INTERNAL_RT_USE_COMPRESSED_OOPS;
1057 PreRuntimeDispatch::store<expanded_decorators>(addr, value);
1058 }
1059
1060 template <DecoratorSet decorators>
1061 inline void store_reduce_types(HeapWord* addr, oop value) {
1062 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1063 PreRuntimeDispatch::store<expanded_decorators>(addr, value);
1064 }
1065
1066 template <DecoratorSet decorators, typename T>
1067 inline T atomic_cmpxchg_reduce_types(T new_value, T* addr, T compare_value) {
1068 return PreRuntimeDispatch::atomic_cmpxchg<decorators>(new_value, addr, compare_value);
1069 }
1070
1071 template <DecoratorSet decorators>
1072 inline oop atomic_cmpxchg_reduce_types(oop new_value, narrowOop* addr, oop compare_value) {
1073 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1074 INTERNAL_RT_USE_COMPRESSED_OOPS;
1075 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value);
1076 }
1077
1078 template <DecoratorSet decorators>
1079 inline narrowOop atomic_cmpxchg_reduce_types(narrowOop new_value, narrowOop* addr, narrowOop compare_value) {
1080 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1081 INTERNAL_RT_USE_COMPRESSED_OOPS;
1082 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value);
1083 }
1084
1085 template <DecoratorSet decorators>
1086 inline oop atomic_cmpxchg_reduce_types(oop new_value,
1087 HeapWord* addr,
1088 oop compare_value) {
1089 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1090 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value);
1091 }
1092
1093 template <DecoratorSet decorators, typename T>
1094 inline T atomic_xchg_reduce_types(T new_value, T* addr) {
1095 const DecoratorSet expanded_decorators = decorators;
1096 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr);
1097 }
1098
1099 template <DecoratorSet decorators>
1100 inline oop atomic_xchg_reduce_types(oop new_value, narrowOop* addr) {
1101 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1102 INTERNAL_RT_USE_COMPRESSED_OOPS;
1103 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr);
1104 }
1105
1106 template <DecoratorSet decorators>
1107 inline narrowOop atomic_xchg_reduce_types(narrowOop new_value, narrowOop* addr) {
1108 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1109 INTERNAL_RT_USE_COMPRESSED_OOPS;
1110 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr);
1111 }
1112
1113 template <DecoratorSet decorators>
1114 inline oop atomic_xchg_reduce_types(oop new_value, HeapWord* addr) {
1115 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1116 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr);
1117 }
1118
1119 template <DecoratorSet decorators, typename T>
1120 inline T load_reduce_types(T* addr) {
1121 return PreRuntimeDispatch::load<decorators, T>(addr);
1122 }
1123
1124 template <DecoratorSet decorators, typename T>
1125 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1126 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1127 INTERNAL_RT_USE_COMPRESSED_OOPS;
1128 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1129 }
1130
1131 template <DecoratorSet decorators, typename T>
1132 inline oop load_reduce_types(HeapWord* addr) {
1133 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1134 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1135 }
1136
1137 template <DecoratorSet decorators, typename T>
1138 inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1139 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1140 size_t length) {
1141 PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1142 dst_obj, dst_offset_in_bytes, dst_raw,
1143 length);
1144 }
1145
1146 template <DecoratorSet decorators>
1147 inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1148 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1149 size_t length) {
1150 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1151 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1152 dst_obj, dst_offset_in_bytes, dst_raw,
1153 length);
1154 }
1155
1156 template <DecoratorSet decorators>
1157 inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1158 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1159 size_t length) {
1160 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1161 INTERNAL_RT_USE_COMPRESSED_OOPS;
1162 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1163 dst_obj, dst_offset_in_bytes, dst_raw,
1164 length);
1165 }
1166
1167 // Step 1: Set default decorators. This step remembers if a type was volatile
1168 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1169 // memory ordering is set for the access, and the implied decorator rules
1170 // are applied to select sensible defaults for decorators that have not been
1171 // explicitly set. For example, default object referent strength is set to strong.
1172 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1173 // and references from the types). This step also perform some type verification
1174 // that the passed in types make sense.
1175
1176 template <DecoratorSet decorators, typename T>
1177 static void verify_types(){
1178 // If this fails to compile, then you have sent in something that is
1179 // not recognized as a valid primitive type to a primitive Access function.
1180 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1181 (IsPointer<T>::value || IsIntegral<T>::value) ||
1182 IsFloatingPoint<T>::value)); // not allowed primitive type
1183 }
1184
1185 template <DecoratorSet decorators, typename P, typename T>
1186 inline void store(P* addr, T value) {
1187 verify_types<decorators, T>();
1188 typedef typename Decay<P>::type DecayedP;
1189 typedef typename Decay<T>::type DecayedT;
1190 DecayedT decayed_value = value;
1191 // If a volatile address is passed in but no memory ordering decorator,
1192 // set the memory ordering to MO_VOLATILE by default.
1193 const DecoratorSet expanded_decorators = DecoratorFixup<
1194 (IsVolatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1195 (MO_VOLATILE | decorators) : decorators>::value;
1196 store_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), decayed_value);
1197 }
1198
1199 template <DecoratorSet decorators, typename T>
1200 inline void store_at(oop base, ptrdiff_t offset, T value) {
1201 verify_types<decorators, T>();
1202 typedef typename Decay<T>::type DecayedT;
1203 DecayedT decayed_value = value;
1204 const DecoratorSet expanded_decorators = DecoratorFixup<decorators |
1205 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1206 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1207 PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, decayed_value);
1208 }
1209
1210 template <DecoratorSet decorators, typename P, typename T>
1211 inline T load(P* addr) {
1212 verify_types<decorators, T>();
1213 typedef typename Decay<P>::type DecayedP;
1214 typedef typename Conditional<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
1215 typename OopOrNarrowOop<T>::type,
1216 typename Decay<T>::type>::type DecayedT;
1217 // If a volatile address is passed in but no memory ordering decorator,
1218 // set the memory ordering to MO_VOLATILE by default.
1219 const DecoratorSet expanded_decorators = DecoratorFixup<
1220 (IsVolatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1221 (MO_VOLATILE | decorators) : decorators>::value;
1222 return load_reduce_types<expanded_decorators, DecayedT>(const_cast<DecayedP*>(addr));
1223 }
1224
1225 template <DecoratorSet decorators, typename T>
1226 inline T load_at(oop base, ptrdiff_t offset) {
1227 verify_types<decorators, T>();
1228 typedef typename Conditional<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
1229 typename OopOrNarrowOop<T>::type,
1230 typename Decay<T>::type>::type DecayedT;
1231 // Expand the decorators (figure out sensible defaults)
1232 // Potentially remember if we need compressed oop awareness
1233 const DecoratorSet expanded_decorators = DecoratorFixup<decorators |
1234 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1235 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1236 return PreRuntimeDispatch::load_at<expanded_decorators, DecayedT>(base, offset);
1237 }
1238
1239 template <DecoratorSet decorators, typename P, typename T>
1240 inline T atomic_cmpxchg(T new_value, P* addr, T compare_value) {
1241 verify_types<decorators, T>();
1242 typedef typename Decay<P>::type DecayedP;
1243 typedef typename Decay<T>::type DecayedT;
1244 DecayedT new_decayed_value = new_value;
1245 DecayedT compare_decayed_value = compare_value;
1246 const DecoratorSet expanded_decorators = DecoratorFixup<
1247 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1248 (MO_SEQ_CST | decorators) : decorators>::value;
1249 return atomic_cmpxchg_reduce_types<expanded_decorators>(new_decayed_value,
1250 const_cast<DecayedP*>(addr),
1251 compare_decayed_value);
1252 }
1253
1254 template <DecoratorSet decorators, typename T>
1255 inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
1256 verify_types<decorators, T>();
1257 typedef typename Decay<T>::type DecayedT;
1258 DecayedT new_decayed_value = new_value;
1259 DecayedT compare_decayed_value = compare_value;
1260 // Determine default memory ordering
1261 const DecoratorSet expanded_decorators = DecoratorFixup<
1262 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1263 (MO_SEQ_CST | decorators) : decorators>::value;
1264 // Potentially remember that we need compressed oop awareness
1265 const DecoratorSet final_decorators = expanded_decorators |
1266 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1267 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE);
1268 return PreRuntimeDispatch::atomic_cmpxchg_at<final_decorators>(new_decayed_value, base,
1269 offset, compare_decayed_value);
1270 }
1271
1272 template <DecoratorSet decorators, typename P, typename T>
1273 inline T atomic_xchg(T new_value, P* addr) {
1274 verify_types<decorators, T>();
1275 typedef typename Decay<P>::type DecayedP;
1276 typedef typename Decay<T>::type DecayedT;
1277 DecayedT new_decayed_value = new_value;
1278 // atomic_xchg is only available in SEQ_CST flavour.
1279 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1280 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1281 const_cast<DecayedP*>(addr));
1282 }
1283
1284 template <DecoratorSet decorators, typename T>
1285 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1286 verify_types<decorators, T>();
1287 typedef typename Decay<T>::type DecayedT;
1288 DecayedT new_decayed_value = new_value;
1289 // atomic_xchg is only available in SEQ_CST flavour.
1290 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1291 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1292 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1293 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1294 }
1295
1296 template <DecoratorSet decorators, typename T>
1297 inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1298 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1299 size_t length) {
1300 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1301 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1302 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1303 typedef typename Decay<T>::type DecayedT;
1304 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1305 arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1306 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1307 length);
1308 }
1309
1310 template <DecoratorSet decorators>
1311 inline void clone(oop src, oop dst, size_t size) {
1312 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1313 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1314 }
1315
1316 template <DecoratorSet decorators>
1317 inline void value_copy(void* src, void* dst, ValueKlass* md) {
1318 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1319 PreRuntimeDispatch::value_copy<expanded_decorators>(src, dst, md);
1320 }
1321
1322 template <DecoratorSet decorators>
1323 inline oop resolve(oop obj) {
1324 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1325 return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1326 }
1327
1328 // Infer the type that should be returned from an Access::oop_load.
1329 template <typename P, DecoratorSet decorators>
1330 class OopLoadProxy: public StackObj {
1331 private:
1332 P *const _addr;
1333 public:
1334 OopLoadProxy(P* addr) : _addr(addr) {}
1335
1336 inline operator oop() {
1337 return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr);
1338 }
1339
1340 inline operator narrowOop() {
1341 return load<decorators | INTERNAL_VALUE_IS_OOP, P, narrowOop>(_addr);
1342 }
1343
1344 template <typename T>
1345 inline bool operator ==(const T& other) const {
1346 return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) == other;
1347 }
1348
1349 template <typename T>
1350 inline bool operator !=(const T& other) const {
1351 return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) != other;
1352 }
1353 };
1354
1355 // Infer the type that should be returned from an Access::load_at.
1356 template <DecoratorSet decorators>
1357 class LoadAtProxy: public StackObj {
1358 private:
1359 const oop _base;
1360 const ptrdiff_t _offset;
1361 public:
1362 LoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {}
1363
1364 template <typename T>
1365 inline operator T() const {
1366 return load_at<decorators, T>(_base, _offset);
1367 }
1368
1369 template <typename T>
1370 inline bool operator ==(const T& other) const { return load_at<decorators, T>(_base, _offset) == other; }
1371
1372 template <typename T>
1373 inline bool operator !=(const T& other) const { return load_at<decorators, T>(_base, _offset) != other; }
1374 };
1375
1376 // Infer the type that should be returned from an Access::oop_load_at.
1377 template <DecoratorSet decorators>
1378 class OopLoadAtProxy: public StackObj {
1379 private:
1380 const oop _base;
1381 const ptrdiff_t _offset;
1382 public:
1383 OopLoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {}
1384
1385 inline operator oop() const {
1386 return load_at<decorators | INTERNAL_VALUE_IS_OOP, oop>(_base, _offset);
1387 }
1388
1389 inline operator narrowOop() const {
1390 return load_at<decorators | INTERNAL_VALUE_IS_OOP, narrowOop>(_base, _offset);
1391 }
1392
1393 template <typename T>
1394 inline bool operator ==(const T& other) const {
1395 return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) == other;
1396 }
1397
1398 template <typename T>
1399 inline bool operator !=(const T& other) const {
1400 return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) != other;
1401 }
1402 };
1403 }
1404
1405 #endif // SHARE_OOPS_ACCESSBACKEND_HPP