1 /*
2 * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_VM_RUNTIME_ACCESS_INLINE_HPP
26 #define SHARE_VM_RUNTIME_ACCESS_INLINE_HPP
27
28 #include "gc/shared/barrierSet.inline.hpp"
29 #include "runtime/access.hpp"
30 #include "runtime/accessBackend.hpp"
31 #include "runtime/atomic.hpp"
32 #include "runtime/orderAccess.inline.hpp"
33
34 namespace AccessInternal {
35
36 template <DecoratorSet decorators, typename T, BarrierType type>
37 class AccessBarrierResolver: public AllStatic {
38 public:
39 static void* resolve_barrier();
40 };
41
42 template <DecoratorSet decorators>
43 class CloneAccessBarrierResolver: public AllStatic {
44 public:
45 static void* resolve_barrier();
46 };
47
48 template <DecoratorSet static_decorators, typename T, BarrierType type>
49 struct DispatcherFactory: public AllStatic {
50 static void* resolve_barrier_gc() {
51 return AccessBarrierResolver<static_decorators, T, type>::resolve_barrier();
52 }
53
54 static void* resolve_barrier_post_rt();
55
56 static void* resolve_barrier_rt() {
57 if (UseCompressedOops) {
58 return DispatcherFactory<static_decorators | RT_HAS_RUNTIME_DECORATOR | RT_USE_COMPRESSED_OOPS, T, type>::resolve_barrier_post_rt();
59 } else {
60 return DispatcherFactory<static_decorators | RT_HAS_RUNTIME_DECORATOR, T, type>::resolve_barrier_post_rt();
61 }
62 }
63
64 static void* resolve_barrier() {
65 return resolve_barrier_rt();
66 }
67 };
68
69 template <DecoratorSet static_decorators>
70 struct CloneDispatcherFactory: public AllStatic {
71 static void* resolve_barrier_gc() {
72 return CloneAccessBarrierResolver<static_decorators>::resolve_barrier();
73 }
74
75 static void* resolve_barrier_rt() {
76 if (UseCompressedOops) {
77 return CloneDispatcherFactory<static_decorators | RT_HAS_RUNTIME_DECORATOR | RT_USE_COMPRESSED_OOPS>::resolve_barrier_gc();
78 } else {
79 return CloneDispatcherFactory<static_decorators | RT_HAS_RUNTIME_DECORATOR>::resolve_barrier_gc();
80 }
81 }
82
83 static void* resolve_barrier() {
84 return resolve_barrier_rt();
85 }
86 };
87
88 template <DecoratorSet decorators>
89 struct CloneRuntimeDispatch {
90 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
91
92 static void clone_init(oop src, oop dst, size_t size) {
93 _clone_func = (clone_func_t)CloneDispatcherFactory<decorators>::resolve_barrier();
94 _clone_func(src, dst, size);
95 }
96
97 static clone_func_t _clone_func;
98
99 static inline void clone(oop src, oop dst, size_t size) {
100 return _clone_func(src, dst, size);
101 }
102 };
103
104 template <DecoratorSet decorators, typename T>
105 struct RuntimeDispatch {
106 static void store_init(void* addr, T value) {
107 _store_func = (typename AccessFunctionTypes<decorators, T>::store_func_t)DispatcherFactory<decorators, T, BARRIER_STORE>::resolve_barrier();
108 _store_func(addr, value);
109 }
110
111 static void store_at_init(typename BaseType<decorators>::type base, ptrdiff_t offset, T value) {
112 _store_at_func = (typename AccessFunctionTypes<decorators, T>::store_at_func_t)DispatcherFactory<decorators, T, BARRIER_STORE_AT>::resolve_barrier();
113 _store_at_func(base, offset, value);
114 }
115
116 static T load_init(void* addr) {
117 _load_func = (typename AccessFunctionTypes<decorators, T>::load_func_t)DispatcherFactory<decorators, T, BARRIER_LOAD>::resolve_barrier();
118 return _load_func(addr);
119 }
120
121 static T load_at_init(typename BaseType<decorators>::type base, ptrdiff_t offset) {
122 _load_at_func = (typename AccessFunctionTypes<decorators, T>::load_at_func_t)DispatcherFactory<decorators, T, BARRIER_LOAD_AT>::resolve_barrier();
123 return _load_at_func(base, offset);
124 }
125
126 static T cas_init(T new_value, void* addr, T compare_value) {
127 _cas_func = (typename AccessFunctionTypes<decorators, T>::cas_func_t)DispatcherFactory<decorators, T, BARRIER_CAS>::resolve_barrier();
128 return _cas_func(new_value, addr, compare_value);
129 }
130
131 static T cas_at_init(T new_value, typename BaseType<decorators>::type base, ptrdiff_t offset, T compare_value) {
132 _cas_at_func = (typename AccessFunctionTypes<decorators, T>::cas_at_func_t)DispatcherFactory<decorators, T, BARRIER_CAS_AT>::resolve_barrier();
133 return _cas_at_func(new_value, base, offset, compare_value);
134 }
135
136 static T swap_init(T new_value, void* addr) {
137 _swap_func = (typename AccessFunctionTypes<decorators, T>::swap_func_t)DispatcherFactory<decorators, T, BARRIER_SWAP>::resolve_barrier();
138 return _swap_func(new_value, addr);
139 }
140
141 static T swap_at_init(T new_value, typename BaseType<decorators>::type base, ptrdiff_t offset) {
142 _swap_at_func = (typename AccessFunctionTypes<decorators, T>::swap_at_func_t)DispatcherFactory<decorators, T, BARRIER_SWAP_AT>::resolve_barrier();
143 return _swap_at_func(new_value, base, offset);
144 }
145
146 static bool copy_init(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
147 _copy_func = (typename AccessFunctionTypes<decorators, T>::copy_func_t)DispatcherFactory<decorators, T, BARRIER_COPY>::resolve_barrier();
148 return _copy_func(src_obj, dst_obj, src, dst, length);
149 }
150
151 static typename AccessFunctionTypes<decorators, T>::store_func_t _store_func;
152 static typename AccessFunctionTypes<decorators, T>::store_at_func_t _store_at_func;
153 static typename AccessFunctionTypes<decorators, T>::load_func_t _load_func;
154 static typename AccessFunctionTypes<decorators, T>::load_at_func_t _load_at_func;
155 static typename AccessFunctionTypes<decorators, T>::cas_func_t _cas_func;
156 static typename AccessFunctionTypes<decorators, T>::cas_at_func_t _cas_at_func;
157 static typename AccessFunctionTypes<decorators, T>::swap_func_t _swap_func;
158 static typename AccessFunctionTypes<decorators, T>::swap_at_func_t _swap_at_func;
159 static typename AccessFunctionTypes<decorators, T>::copy_func_t _copy_func;
160
161 static inline void store(void* addr, T value) {
162 _store_func(addr, value);
163 }
164
165 static inline void store_at(void* base, ptrdiff_t offset, T value) {
166 _store_at_func((typename BaseType<decorators>::type)base, offset, value);
167 }
168
169 static inline T load(void* addr) {
170 return _load_func(addr);
171 }
172
173 static inline T load_at(void* base, ptrdiff_t offset) {
174 return _load_at_func((typename BaseType<decorators>::type)base, offset);
175 }
176
177 static inline T cas(T new_value, void* addr, T compare_value) {
178 return _cas_func(new_value, addr, compare_value);
179 }
180
181 static inline T cas_at(T new_value, void* base, ptrdiff_t offset, T compare_value) {
182 return _cas_at_func(new_value, (typename BaseType<decorators>::type)base, offset, compare_value);
183 }
184
185 static inline T swap(T new_value, void* addr) {
186 return _swap_func(new_value, addr);
187 }
188
189 static inline T swap_at(T new_value, void* base, ptrdiff_t offset) {
190 return _swap_at_func(new_value, (typename BaseType<decorators>::type)base, offset);
191 }
192
193 static inline bool copy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
194 return _copy_func(src_obj, dst_obj, src, dst, length);
195 }
196 };
197
198 template <DecoratorSet decorators>
199 struct PreRuntimeDispatch {
200 typedef RawAccessBarrier<DecoratorIntersection<decorators, RAW_DECORATOR_MASK>::value> Raw;
201 typedef BasicAccessBarrier<DecoratorIntersection<decorators, BASIC_DECORATOR_MASK>::value> Basic;
202
203 enum {
204 CAN_HARDWIRE_BASIC = DecoratorTest<decorators>::HAS_USE_COMPRESSED_OOPS == DecoratorTest<decorators>::HAS_GC_CONVERT_COMPRESSED_OOP
205 };
206
207 template <DecoratorSet idecorators, typename T>
208 inline static typename EnableIf<DecoratorTest<idecorators>::HAS_ACCESS_RAW || DecoratorTest<idecorators>::HAS_ACCESS_BASIC, void>::type store(void* addr, T value) {
209 if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
210 Raw::template store<T>(addr, value);
211 } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
212 if (!DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
213 Basic::template store<T>(addr, value);
214 } else if (CAN_HARDWIRE_BASIC) {
215 Basic::template oop_store<T>(addr, value);
216 } else if (UseCompressedOops) {
217 const DecoratorSet expanded_decorators = decorators | RT_USE_COMPRESSED_OOPS;
218 PreRuntimeDispatch<expanded_decorators>::template store<expanded_decorators, T>(addr, value);
219 } else {
220 const DecoratorSet expanded_decorators = decorators & ~GC_CONVERT_COMPRESSED_OOP;
221 PreRuntimeDispatch<expanded_decorators>::template store<expanded_decorators, T>(addr, value);
222 }
223 }
224 }
225
226 template <DecoratorSet idecorators, typename T>
227 inline static typename EnableIf<!DecoratorTest<idecorators>::HAS_ACCESS_RAW && !DecoratorTest<idecorators>::HAS_ACCESS_BASIC, void>::type store(void* addr, T value) {
228 if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
229 const DecoratorSet expanded_decorators = decorators | ACCESS_RAW;
230 PreRuntimeDispatch<expanded_decorators>::template store<expanded_decorators, T>(addr, value);
231 } else {
232 RuntimeDispatch<decorators, T>::store(addr, value);
233 }
234 }
235
236 template <DecoratorSet idecorators, typename T>
237 inline static typename EnableIf<DecoratorTest<idecorators>::HAS_ACCESS_RAW || DecoratorTest<idecorators>::HAS_ACCESS_BASIC, void>::type store_at(void* base, ptrdiff_t offset, T value) {
238 if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
239 Raw::template store<T>(field_addr(base, offset), value);
240 } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
241 store<idecorators, T>(field_addr(base, offset), value);
242 }
243 }
244
245 template <DecoratorSet idecorators, typename T>
246 inline static typename EnableIf<!DecoratorTest<idecorators>::HAS_ACCESS_RAW && !DecoratorTest<idecorators>::HAS_ACCESS_BASIC, void>::type store_at(void* base, ptrdiff_t offset, T value) {
247 if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
248 const DecoratorSet expanded_decorators = decorators | ACCESS_RAW;
249 PreRuntimeDispatch<expanded_decorators>::template store_at<expanded_decorators, T>(base, offset, value);
250 } else {
251 RuntimeDispatch<decorators, T>::store_at(base, offset, value);
252 }
253 }
254
255 template <DecoratorSet idecorators, typename T>
256 inline static typename EnableIf<DecoratorTest<idecorators>::HAS_ACCESS_RAW || DecoratorTest<idecorators>::HAS_ACCESS_BASIC, T>::type load(void* addr) {
257 if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
258 return Raw::template load<T>(addr);
259 } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
260 if (!DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
261 return Basic::template load<T>(addr);
262 } else if (CAN_HARDWIRE_BASIC) {
263 return Basic::template oop_load<T>(addr);
264 } else if (UseCompressedOops) {
265 const DecoratorSet expanded_decorators = decorators | RT_USE_COMPRESSED_OOPS;
266 return PreRuntimeDispatch<expanded_decorators>::template load<expanded_decorators, T>(addr);
267 } else {
268 const DecoratorSet expanded_decorators = decorators & ~GC_CONVERT_COMPRESSED_OOP;
269 return PreRuntimeDispatch<expanded_decorators>::template load<expanded_decorators, T>(addr);
270 }
271 }
272 }
273
274 template <DecoratorSet idecorators, typename T>
275 inline static typename EnableIf<!DecoratorTest<idecorators>::HAS_ACCESS_RAW && !DecoratorTest<idecorators>::HAS_ACCESS_BASIC, T>::type load(void* addr) {
276 if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
277 const DecoratorSet expanded_decorators = decorators | ACCESS_RAW;
278 return PreRuntimeDispatch<expanded_decorators>::template load<expanded_decorators, T>(addr);
279 } else {
280 return RuntimeDispatch<decorators, T>::load(addr);
281 }
282 }
283
284 template <DecoratorSet idecorators, typename T>
285 inline static typename EnableIf<DecoratorTest<idecorators>::HAS_ACCESS_RAW || DecoratorTest<idecorators>::HAS_ACCESS_BASIC, T>::type load_at(void* base, ptrdiff_t offset) {
286 if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
287 return Raw::template load<T>(field_addr(base, offset));
288 } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
289 return load<decorators, T>(field_addr(base, offset));
290 }
291 }
292
293 template <DecoratorSet idecorators, typename T>
294 inline static typename EnableIf<!DecoratorTest<idecorators>::HAS_ACCESS_RAW && !DecoratorTest<idecorators>::HAS_ACCESS_BASIC, T>::type load_at(void* base, ptrdiff_t offset) {
295 if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
296 const DecoratorSet expanded_decorators = decorators | ACCESS_RAW;
297 return PreRuntimeDispatch<expanded_decorators>::template load_at<expanded_decorators, T>(base, offset);
298 } else {
299 return RuntimeDispatch<decorators, T>::load_at(base, offset);
300 }
301 }
302
303 template <DecoratorSet idecorators, typename T>
304 inline static T cas(T new_value, void* addr, T compare_value) {
305 if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
306 return Raw::template cas<T>(new_value, addr, compare_value);
307 } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
308 if (!DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
309 return Basic::template cas<T>(new_value, addr, compare_value);
310 } else if (CAN_HARDWIRE_BASIC) {
311 return Basic::template oop_cas<T>(new_value, addr, compare_value);
312 } else if (UseCompressedOops) {
313 const DecoratorSet expanded_decorators = decorators | RT_USE_COMPRESSED_OOPS;
314 return PreRuntimeDispatch<expanded_decorators>::template cas<expanded_decorators, T>(new_value, addr, compare_value);
315 } else {
316 const DecoratorSet expanded_decorators = decorators & ~GC_CONVERT_COMPRESSED_OOP;
317 return PreRuntimeDispatch<expanded_decorators>::template cas<expanded_decorators, T>(new_value, addr, compare_value);
318 }
319 } else if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
320 return Raw::template cas<T>(new_value, addr, compare_value);
321 } else {
322 return RuntimeDispatch<decorators, T>::cas(new_value, addr, compare_value);
323 }
324 }
325
326 template <DecoratorSet idecorators, typename T>
327 inline static T cas_at(T new_value, void* base, ptrdiff_t offset, T compare_value) {
328 if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
329 return Raw::template cas<T>(new_value, field_addr(base, offset), compare_value);
330 } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
331 return cas<idecorators, T>(new_value, field_addr(base, offset), compare_value);
332 } else if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
333 return Raw::template cas<T>(new_value, field_addr(base, offset), compare_value);
334 } else {
335 return RuntimeDispatch<decorators, T>::cas_at(new_value, base, offset, compare_value);
336 }
337 }
338
339 template <DecoratorSet idecorators, typename T>
340 inline static T swap(T new_value, void* addr) {
341 if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
342 return Raw::template swap<T>(new_value, addr);
343 } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
344 if (!DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
345 return Basic::template swap<T>(new_value, addr);
346 } else if (CAN_HARDWIRE_BASIC) {
347 return Basic::template oop_swap<T>(new_value, addr);
348 } else if (UseCompressedOops) {
349 const DecoratorSet expanded_decorators = decorators | RT_USE_COMPRESSED_OOPS;
350 return PreRuntimeDispatch<expanded_decorators>::template swap<expanded_decorators, T>(new_value, addr);
351 } else {
352 const DecoratorSet expanded_decorators = decorators & ~GC_CONVERT_COMPRESSED_OOP;
353 return PreRuntimeDispatch<expanded_decorators>::template swap<expanded_decorators, T>(new_value, addr);
354 }
355 } else if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
356 return Raw::template swap<T>(new_value, addr);
357 } else {
358 return RuntimeDispatch<decorators, T>::swap(new_value, addr);
359 }
360 }
361
362 template <DecoratorSet idecorators, typename T>
363 inline static T swap_at(T new_value, void* base, ptrdiff_t offset) {
364 if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
365 return Raw::template swap<T>(new_value, field_addr(base, offset));
366 } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
367 return swap<idecorators, T>(new_value, field_addr(base, offset));
368 } else if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
369 return Raw::template swap<T>(new_value, field_addr(base, offset));
370 } else {
371 return RuntimeDispatch<decorators, T>::swap_at(new_value, base, offset);
372 }
373 }
374
375 template <DecoratorSet idecorators, typename T>
376 inline static bool copy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
377 if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
378 return Raw::template copy<T>(src, dst, length);
379 } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
380 if (!DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
381 return Basic::template copy<T>(src_obj, dst_obj, src, dst, length);
382 } else {
383 return Basic::template oop_copy<T>(src_obj, dst_obj, src, dst, length);
384 }
385 } else if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
386 return Raw::template copy<T>(src, dst, length);
387 } else {
388 return RuntimeDispatch<decorators, T>::copy(src_obj, dst_obj, src, dst, length);
389 }
390 }
391
392 template <DecoratorSet idecorators>
393 inline static void clone(oop src, oop dst, size_t size) {
394 if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
395 return Raw::clone(src, dst, size);
396 } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
397 return Basic::clone(src, dst, size);
398 } else {
399 return CloneRuntimeDispatch<decorators>::clone(src, dst, size);
400 }
401 }
402 };
403
404 template <DecoratorSet decorators>
405 struct BuildtimeDispatch {
406 template <typename T>
407 static void store(void* addr, T value);
408 template <typename T>
409 static void store_at(void* base, ptrdiff_t offset, T value);
410 template <typename T>
411 static T load(void* addr);
412 template <typename T>
413 static T load_at(void* base, ptrdiff_t offset);
414 template <typename T>
415 static T cas(T new_value, void* addr, T compare_value);
416 template <typename T>
417 static T cas_at(T new_value, void* base, ptrdiff_t offset, T compare_value);
418 template <typename T>
419 static T swap(T new_value, void* addr);
420 template <typename T>
421 static T swap_at(T new_value, void* base, ptrdiff_t offset);
422 template <typename T>
423 static bool copy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length);
424 static void clone(oop src, oop dst, size_t size);
425 };
426
427 /**
428 * This class adds implied properties that follow according to decorator rules
429 */
430 template <DecoratorSet input_decorators>
431 struct DecoratorFixup {
432 enum {
433 ref_strong_default = input_decorators | ((!DecoratorTest<input_decorators>::HAS_GC_ACCESS_ON_PHANTOM
434 && !DecoratorTest<input_decorators>::HAS_GC_ACCESS_ON_WEAK) ? GC_ACCESS_ON_STRONG : EMPTY_DECORATOR)
435 };
436 enum {
437 seqcst_release = ref_strong_default | (DecoratorTest<ref_strong_default>::HAS_MO_SEQ_CST ? MO_RELEASE : EMPTY_DECORATOR)
438 };
439 enum {
440 seqcst_acquire = seqcst_release | (DecoratorTest<seqcst_release>::HAS_MO_SEQ_CST ? MO_ACQUIRE : EMPTY_DECORATOR)
441 };
442 enum {
443 release_atomic = seqcst_acquire | (DecoratorTest<seqcst_acquire>::HAS_MO_RELEASE ? MO_ATOMIC : EMPTY_DECORATOR)
444 };
445 enum {
446 acquire_atomic = release_atomic | (DecoratorTest<release_atomic>::HAS_MO_ACQUIRE ? MO_ATOMIC : EMPTY_DECORATOR)
447 };
448 enum {
449 atomic_volatile = acquire_atomic | (DecoratorTest<acquire_atomic>::HAS_MO_ATOMIC ? MO_VOLATILE : EMPTY_DECORATOR)
450 };
451 enum {
452 novolatile_relaxed = atomic_volatile | (!DecoratorTest<atomic_volatile>::HAS_MO_VOLATILE ? MO_RELAXED : EMPTY_DECORATOR)
453 };
454 enum {
455 relaxed_root = novolatile_relaxed | ((DecoratorTest<novolatile_relaxed>::HAS_VALUE_IS_OOP &&
456 (DecoratorTest<novolatile_relaxed>::HAS_ACCESS_ON_NMETHOD ||
457 DecoratorTest<novolatile_relaxed>::HAS_ACCESS_ON_KLASS)) ?
458 ACCESS_ON_ROOT : EMPTY_DECORATOR)
459 };
460 enum {
461 value = relaxed_root
462 };
463 };
464
465 // Only let invocations where P and T are the same through
466 template <DecoratorSet decorators, typename P, typename T>
467 inline typename EnableIf<IsDerived<P, T>::value, void>::type
468 store_reduce_types(P* addr, T value) {
469 BuildtimeDispatch<decorators>::template store<T>((void*)addr, value);
470 }
471
472 template <DecoratorSet decorators, typename P, typename T>
473 inline typename EnableIf<!IsDerived<P, T>::value, void>::type
474 store_reduce_types(narrowOop* addr, oop value) {
475 const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP | RT_USE_COMPRESSED_OOPS;
476 BuildtimeDispatch<expanded_decorators>::template store<oop>((void*)addr, value);
477 }
478
479 template <DecoratorSet decorators, typename P, typename T>
480 inline typename EnableIf<!IsDerived<P, T>::value, void>::type
481 store_reduce_types(HeapWord* addr, oop value) {
482 const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP;
483 BuildtimeDispatch<expanded_decorators>::template store<oop>((void*)addr, value);
484 }
485
486 // If the decayed pointer and value types are the same, we decay their types and
487 // send them to the next stage in the pipeline
488 template <DecoratorSet decorators, typename P, typename T>
489 inline void store_decay_types(P* addr, T value) {
490 typedef typename Decay<P>::type DecayedP;
491 typedef typename Decay<T>::type DecayedT;
492 const DecoratorSet expanded_decorators = IsVolatile<P>::value ? DecoratorFixup<MO_VOLATILE | decorators>::value : decorators;
493 store_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedP*)addr, (DecayedT)value);
494 }
495
496 template <DecoratorSet decorators, typename P, typename T>
497 void store_at_reduce_types(P base, ptrdiff_t offset, T value);
498
499 template <DecoratorSet decorators, typename P, typename T>
500 inline void store_at_decay_types(P base, ptrdiff_t offset, T value) {
501 typedef typename Decay<P>::type DecayedP;
502 typedef typename Decay<T>::type DecayedT;
503 store_at_reduce_types<decorators, DecayedP, DecayedT>((DecayedP)base, offset, (DecayedT)value);
504 }
505
506 template <DecoratorSet decorators, typename P, typename T>
507 T cas_at_reduce_types(T new_value, P base, ptrdiff_t offset, T compare_value);
508
509 template <DecoratorSet decorators, typename P, typename T>
510 inline T cas_at_decay_types(T new_value, P base, ptrdiff_t offset, T compare_value) {
511 typedef typename Decay<P>::type DecayedP;
512 typedef typename Decay<T>::type DecayedT;
513 const DecoratorSet memory_ordering = !DecoratorTest<decorators>::HAS_MO_SEQ_CST ? MO_ATOMIC : MO_SEQ_CST;
514 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | memory_ordering>::value;
515 return cas_at_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedT)new_value, (DecayedP)base, offset, (DecayedT)compare_value);
516 }
517
518 template <DecoratorSet decorators, typename P, typename T>
519 inline typename EnableIf<IsDerived<P, T>::value, T>::type cas_reduce_types(T new_value, P* addr, T compare_value) {
520 return BuildtimeDispatch<decorators>::template cas<T>(new_value, (void*)addr, compare_value);
521 }
522
523 template <DecoratorSet decorators, typename P, typename T>
524 inline typename EnableIf<!IsDerived<P, T>::value, oop>::type cas_reduce_types(oop new_value, narrowOop* addr, oop compare_value) {
525 const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP | RT_USE_COMPRESSED_OOPS;
526 return BuildtimeDispatch<expanded_decorators>::template cas<oop>(new_value, (void*)addr, compare_value);
527 }
528
529 template <DecoratorSet decorators, typename P, typename T>
530 inline typename EnableIf<!IsDerived<P, T>::value, oop>::type cas_reduce_types(oop new_value, HeapWord* addr, oop compare_value) {
531 const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP;
532 return BuildtimeDispatch<expanded_decorators>::template cas<oop>(new_value, (void*)addr, compare_value);
533 }
534
535 template <DecoratorSet decorators, typename P, typename T>
536 inline T cas_decay_types(T new_value, P* addr, T compare_value) {
537 typedef typename Decay<P>::type DecayedP;
538 typedef typename Decay<T>::type DecayedT;
539 const DecoratorSet memory_ordering = !DecoratorTest<decorators>::HAS_MO_SEQ_CST ? MO_ATOMIC : MO_SEQ_CST;
540 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | memory_ordering>::value;
541 return (T)cas_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedT)new_value, (DecayedP*)addr, (DecayedT)compare_value);
542 }
543
544 template <DecoratorSet decorators, typename P, typename T>
545 inline T swap_at_reduce_types(T new_value, P base, ptrdiff_t offset);
546
547 template <DecoratorSet decorators, typename P, typename T>
548 inline T swap_at_decay_types(T new_value, P base, ptrdiff_t offset) {
549 typedef typename Decay<T>::type DecayedT;
550 typedef typename Decay<P>::type DecayedP;
551 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
552 return swap_at_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedT)new_value, (DecayedP)base, offset);
553 }
554
555 template <DecoratorSet decorators, typename P, typename T>
556 inline typename EnableIf<IsDerived<P, T>::value, T>::type swap_reduce_types(T new_value, P* addr) {
557 return BuildtimeDispatch<decorators>::template swap<T>(new_value, (void*)addr);
558 }
559
560 template <DecoratorSet decorators, typename P, typename T>
561 inline typename EnableIf<!IsDerived<P, T>::value, oop>::type swap_reduce_types(oop new_value, narrowOop* addr) {
562 const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP | RT_USE_COMPRESSED_OOPS;
563 return BuildtimeDispatch<expanded_decorators>::template swap<oop>(new_value, (void*)addr);
564 }
565
566 template <DecoratorSet decorators, typename P, typename T>
567 inline typename EnableIf<!IsDerived<P, T>::value, oop>::type
568 swap_reduce_types(oop new_value, HeapWord* addr) {
569 const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP;
570 return BuildtimeDispatch<expanded_decorators>::template swap<oop>(new_value, (void*)addr);
571 }
572
573 template <DecoratorSet decorators, typename P, typename T>
574 inline T swap_decay_types(T new_value, P* addr) {
575 typedef typename Decay<P>::type DecayedP;
576 typedef typename Decay<T>::type DecayedT;
577 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
578 return (T)swap_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedT)new_value, (DecayedP*)addr);
579 }
580
581 template <DecoratorSet decorators, typename T>
582 inline typename EnableIf<!DecoratorTest<decorators>::HAS_VALUE_IS_OOP, bool>::type copy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
583 return BuildtimeDispatch<decorators | ACCESS_ON_HEAP>::template copy<T>(src_obj, dst_obj, src, dst, length);
584 }
585
586 template <DecoratorSet decorators, typename T>
587 inline typename EnableIf<DecoratorTest<decorators>::HAS_VALUE_IS_OOP, bool>::type copy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
588 return BuildtimeDispatch<decorators | ACCESS_ON_HEAP | GC_CONVERT_COMPRESSED_OOP>::template copy<T>(src_obj, dst_obj, src, dst, length);
589 }
590
591 template <DecoratorSet decorators, typename T>
592 inline bool copy_decay_types(arrayOop src_obj, arrayOop dst_obj, T *src, T *dst, size_t length) {
593 typedef typename Decay<T>::type DecayedT;
594 return copy_reduce_types<DecoratorFixup<decorators>::value, DecayedT>(src_obj, dst_obj, (DecayedT*)src, (DecayedT*)dst, length);
595 }
596
597 template <DecoratorSet decorators, typename P, typename T>
598 inline typename EnableIf<IsDerived<P, T>::value, T>::type load_reduce_types(P* addr) {
599 return BuildtimeDispatch<decorators>::template load<T>((void*)addr);
600 }
601
602 template <DecoratorSet decorators, typename P, typename T>
603 inline typename EnableIf<!IsDerived<P, T>::value, oop>::type load_reduce_types(narrowOop* addr) {
604 const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP | RT_USE_COMPRESSED_OOPS;
605 return BuildtimeDispatch<expanded_decorators>::template load<oop>((void*)addr);
606 }
607
608 template <DecoratorSet decorators, typename P, typename T>
609 inline typename EnableIf<!IsDerived<P, T>::value, oop>::type load_reduce_types(HeapWord* addr) {
610 const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP;
611 return BuildtimeDispatch<expanded_decorators>::template load<oop>((void*)addr);
612 }
613
614 // If the decayed pointer and value types are the same, we decay their types and
615 // send them to the next stage in the pipeline
616 template <DecoratorSet decorators, typename P, typename T>
617 inline T load_decay_types(P* addr) {
618 typedef typename Decay<P>::type DecayedP;
619 typedef typename TypeIf<DecoratorTest<decorators>::HAS_VALUE_IS_OOP,
620 typename OopOrNarrowOop<T>::type, typename Decay<T>::type>::type DecayedT;
621 const DecoratorSet expanded_decorators = IsVolatile<P>::value ? DecoratorFixup<MO_VOLATILE | decorators>::value : decorators;
622 return (T)load_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedP*)addr);
623 }
624
625 template <DecoratorSet decorators, typename P, typename T>
626 class DispatchLoadAtReduceTypes {
627 public:
628 static T load_at(P base, ptrdiff_t offset);
629 };
630
631 template <DecoratorSet decorators, typename P, typename T>
632 inline T load_at_decay_types(P base, ptrdiff_t offset) {
633 typedef typename Decay<P>::type DecayedP;
634 typedef typename TypeIf<DecoratorTest<decorators>::HAS_VALUE_IS_OOP,
635 typename OopOrNarrowOop<T>::type, typename Decay<T>::type>::type DecayedT;
636 return (T)DispatchLoadAtReduceTypes<decorators, DecayedP, DecayedT>::load_at((DecayedP)base, offset);
637 }
638
639
640 template <DecoratorSet decorators, typename P, typename T>
641 inline void store(P* addr, T value) {
642 store_decay_types<DecoratorFixup<decorators>::value, P, T>(addr, value);
643 }
644
645 template <DecoratorSet decorators, typename P, typename T>
646 inline void store_at(P base, ptrdiff_t offset, T value) {
647 store_at_decay_types<DecoratorFixup<decorators>::value, P, T>(base, offset, value);
648 }
649
650 template <DecoratorSet decorators, typename P, typename T>
651 inline T load(P* addr) {
652 return load_decay_types<DecoratorFixup<decorators>::value, P, T>(addr);
653 }
654
655 template <DecoratorSet decorators, typename P, typename T>
656 inline T load_at(P base, ptrdiff_t offset) {
657 return load_at_decay_types<DecoratorFixup<decorators>::value, P, T>(base, offset);
658 }
659
660 template <DecoratorSet decorators, typename P, typename T>
661 inline T cas(T new_value, P* addr, T compare_value) {
662 return cas_decay_types<DecoratorFixup<decorators>::value, P, T>(new_value, addr, compare_value);
663 }
664
665 template <DecoratorSet decorators, typename P, typename T>
666 inline T cas_at(T new_value, P base, ptrdiff_t offset, T compare_value) {
667 return cas_at_decay_types<DecoratorFixup<decorators>::value, P, T>(new_value, base, offset, compare_value);
668 }
669
670 template <DecoratorSet decorators, typename P, typename T>
671 inline T swap(T new_value, P* addr) {
672 return swap_decay_types<DecoratorFixup<decorators>::value, P, T>(new_value, addr);
673 }
674
675 template <DecoratorSet decorators, typename P, typename T>
676 inline T swap_at(T new_value, P base, ptrdiff_t offset) {
677 return swap_at_decay_types<DecoratorFixup<decorators>::value, P, T>(new_value, base, offset);
678 }
679
680 template <DecoratorSet decorators, typename T>
681 inline bool copy(arrayOop src_obj, arrayOop dst_obj, T *src, T *dst, size_t length) {
682 return copy_decay_types<DecoratorFixup<decorators>::value, T>(src_obj, dst_obj, src, dst, length);
683 }
684
685 template <DecoratorSet decorators>
686 inline void clone(oop src, oop dst, size_t size) {
687 BuildtimeDispatch<DecoratorFixup<decorators>::value>::clone(src, dst, size);
688 }
689
690 template <DecoratorSet decorators, typename T>
691 typename AccessFunctionTypes<decorators, T>::store_func_t RuntimeDispatch<decorators, T>::_store_func = &store_init;
692
693 template <DecoratorSet decorators, typename T>
694 typename AccessFunctionTypes<decorators, T>::store_at_func_t RuntimeDispatch<decorators, T>::_store_at_func = &store_at_init;
695
696 template <DecoratorSet decorators, typename T>
697 typename AccessFunctionTypes<decorators, T>::load_func_t RuntimeDispatch<decorators, T>::_load_func = &load_init;
698
699 template <DecoratorSet decorators, typename T>
700 typename AccessFunctionTypes<decorators, T>::load_at_func_t RuntimeDispatch<decorators, T>::_load_at_func = &load_at_init;
701
702 template <DecoratorSet decorators, typename T>
703 typename AccessFunctionTypes<decorators, T>::cas_func_t RuntimeDispatch<decorators, T>::_cas_func = &cas_init;
704
705 template <DecoratorSet decorators, typename T>
706 typename AccessFunctionTypes<decorators, T>::cas_at_func_t RuntimeDispatch<decorators, T>::_cas_at_func = &cas_at_init;
707
708 template <DecoratorSet decorators, typename T>
709 typename AccessFunctionTypes<decorators, T>::swap_func_t RuntimeDispatch<decorators, T>::_swap_func = &swap_init;
710
711 template <DecoratorSet decorators, typename T>
712 typename AccessFunctionTypes<decorators, T>::swap_at_func_t RuntimeDispatch<decorators, T>::_swap_at_func = &swap_at_init;
713
714 template <DecoratorSet decorators, typename T>
715 typename AccessFunctionTypes<decorators, T>::copy_func_t RuntimeDispatch<decorators, T>::_copy_func = ©_init;
716
717 template <DecoratorSet decorators>
718 typename CloneRuntimeDispatch<decorators>::clone_func_t CloneRuntimeDispatch<decorators>::_clone_func = &clone_init;
719 }
720
721 template <DecoratorSet decorators>
722 template <DecoratorSet internal_decorators, typename T>
723 inline typename EnableIf<TestEncodable<internal_decorators, T>::value, T>::type
724 BasicAccessBarrier<decorators>::decode_internal(typename EncodedType<internal_decorators, T>::type value) {
725 if (DecoratorTest<decorators>::HAS_VALUE_NOT_NULL) {
726 return oopDesc::decode_heap_oop_not_null(value);
727 } else {
728 return oopDesc::decode_heap_oop(value);
729 }
730 }
731
732 template <DecoratorSet decorators>
733 template <DecoratorSet internal_decorators, typename T>
734 inline typename EnableIf<TestEncodable<internal_decorators, T>::value,
735 typename EncodedType<internal_decorators, T>::type>::type
736 BasicAccessBarrier<decorators>::encode_internal(T value) {
737 if (DecoratorTest<decorators>::HAS_VALUE_NOT_NULL) {
738 return oopDesc::encode_heap_oop_not_null(value);
739 } else {
740 return oopDesc::encode_heap_oop(value);
741 }
742 }
743
744 template <DecoratorSet decorators>
745 template <typename T>
746 inline void BasicAccessBarrier<decorators>::oop_store(void* addr, T value) {
747 typedef typename EncodedType<decorators, T>::type Encoded;
748 Encoded encoded = encode<T>(value);
749 Raw::template store<Encoded>((Encoded*)addr, encoded);
750 }
751
752 template <DecoratorSet decorators>
753 template <typename T>
754 inline void BasicAccessBarrier<decorators>::oop_store_at(void* base, ptrdiff_t offset, T value) {
755 oop_store(field_addr(base, offset), value);
756 }
757
758 template <DecoratorSet decorators>
759 template <typename T>
760 inline T BasicAccessBarrier<decorators>::oop_load(void* addr) {
761 typedef typename EncodedType<decorators, T>::type Encoded;
762 Encoded encoded = Raw::template load<Encoded>((Encoded*)addr);
763 return decode<T>(encoded);
764 }
765
766 template <DecoratorSet decorators>
767 template <typename T>
768 inline T BasicAccessBarrier<decorators>::oop_load_at(void* base, ptrdiff_t offset) {
769 return oop_load<T>(field_addr(base, offset));
770 }
771
772 template <DecoratorSet decorators>
773 template <typename T>
774 inline T BasicAccessBarrier<decorators>::oop_cas(T new_value, void* addr, T compare_value) {
775 typedef typename EncodedType<decorators, T>::type Encoded;
776 Encoded encoded_new = encode<T>(new_value);
777 Encoded encoded_compare = encode<T>(compare_value);
778 Encoded encoded_result = Raw::template cas<Encoded>(encoded_new, (Encoded*)addr, encoded_compare);
779 return decode<T>(encoded_result);
780 }
781
782 template <DecoratorSet decorators>
783 template <typename T>
784 inline T BasicAccessBarrier<decorators>::oop_cas_at(T new_value, void* base, ptrdiff_t offset, T compare_value) {
785 return oop_cas<T>(new_value, field_addr(base, offset), compare_value);
786 }
787
788 template <DecoratorSet decorators>
789 template <typename T>
790 inline T BasicAccessBarrier<decorators>::oop_swap(T new_value, void* addr) {
791 typedef typename EncodedType<decorators, T>::type Encoded;
792 Encoded encoded_new = encode<T>(new_value);
793 Encoded encoded_result = Raw::template swap<Encoded>(encoded_new, (Encoded*)addr);
794 return decode<T>(encoded_result);
795 }
796
797 template <DecoratorSet decorators>
798 template <typename T>
799 inline T BasicAccessBarrier<decorators>::oop_swap_at(T new_value, void* base, ptrdiff_t offset) {
800 return oop_swap<T>(new_value, field_addr(base, offset));
801 }
802
803 template <DecoratorSet decorators>
804 template <typename T>
805 inline bool BasicAccessBarrier<decorators>::oop_copy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
806 if (IsSame<T, HeapWord>::value) {
807 if (DecoratorTest<decorators>::NEEDS_OOP_COMPRESS) {
808 return Raw::template copy<narrowOop>((narrowOop*)src, (narrowOop*)dst, length);
809 } else {
810 return Raw::template copy<oop>((oop*)src, (oop*)dst, length);
811 }
812 } else {
813 return Raw::template copy<T>(src, dst, length);
814 }
815 }
816
817 template <DecoratorSet decorators>
818 inline void BasicAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
819 Raw::clone(src, dst, size);
820 }
821
822 template <DecoratorSet decorators>
823 template <typename T>
824 inline typename EnableIf<IsPointerSize<T>::value, void>::type
825 RawAccessBarrier<decorators>::atomic_store(void* addr, T value) {
826 Atomic::store_ptr((void*)value, (volatile void*)addr);
827 }
828
829 template <DecoratorSet decorators>
830 template <typename T>
831 inline typename EnableIf<!IsPointerSize<T>::value, void>::type
832 RawAccessBarrier<decorators>::atomic_store(void* addr, T value) {
833 Atomic::store(IntegerType<T>::cast_to_signed(value), (volatile typename IntegerType<T>::signed_type*)addr);
834 }
835
836 template <DecoratorSet decorators>
837 template <typename T>
838 inline typename EnableIf<IsPointerSize<T>::value, void>::type
839 RawAccessBarrier<decorators>::release_store(void* addr, T value) {
840 OrderAccess::release_store_ptr((volatile void*)addr, (void*)value);
841 }
842
843 template <DecoratorSet decorators>
844 template <typename T>
845 inline typename EnableIf<!IsPointerSize<T>::value, void>::type
846 RawAccessBarrier<decorators>::release_store(void* addr, T value) {
847 OrderAccess::release_store((volatile typename IntegerType<T>::signed_type*)addr, IntegerType<T>::cast_to_signed(value));
848 }
849
850 template <DecoratorSet decorators>
851 template <typename T>
852 typename EnableIf<IsPointerSize<T>::value, void>::type
853 RawAccessBarrier<decorators>::release_store_fence(void* addr, T value) {
854 OrderAccess::release_store_ptr_fence((volatile void*)addr, (void*)value);
855 }
856
857 template <DecoratorSet decorators>
858 template <typename T>
859 inline typename EnableIf<!IsPointerSize<T>::value, void>::type
860 RawAccessBarrier<decorators>::release_store_fence(void* addr, T value) {
861 OrderAccess::release_store_fence((volatile typename IntegerType<T>::signed_type*)addr, IntegerType<T>::cast_to_signed(value));
862 }
863
864 template <DecoratorSet decorators>
865 template <typename T>
866 inline typename EnableIf<IsPointerSize<T>::value, T>::type
867 RawAccessBarrier<decorators>::atomic_load(void* addr) {
868 return (T)Atomic::load_ptr((volatile void*)addr);
869 }
870
871 template <DecoratorSet decorators>
872 template <typename T>
873 inline typename EnableIf<!IsPointerSize<T>::value, T>::type
874 RawAccessBarrier<decorators>::atomic_load(void* addr) {
875 return (T)Atomic::load((volatile typename IntegerType<T>::signed_type*)addr);
876 }
877
878 template <DecoratorSet decorators>
879 template <typename T>
880 inline typename EnableIf<IsPointerSize<T>::value, T>::type
881 RawAccessBarrier<decorators>::load_acquire(void* addr) {
882 return (T)OrderAccess::load_ptr_acquire((volatile void*)addr);
883 }
884
885 template <DecoratorSet decorators>
886 template <typename T>
887 inline typename EnableIf<!IsPointerSize<T>::value, T>::type
888 RawAccessBarrier<decorators>::load_acquire(void* addr) {
889 return (T)OrderAccess::load_acquire((volatile typename IntegerType<T>::signed_type*)addr);
890 }
891
892 template <DecoratorSet decorators>
893 template <typename T>
894 inline typename EnableIf<IsPointerSize<T>::value, T>::type
895 RawAccessBarrier<decorators>::fence_load_acquire(void* addr) {
896 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
897 OrderAccess::fence();
898 }
899 return (T)OrderAccess::load_ptr_acquire((volatile void*)addr);
900 }
901
902 template <DecoratorSet decorators>
903 template <typename T>
904 inline typename EnableIf<!IsPointerSize<T>::value, T>::type
905 RawAccessBarrier<decorators>::fence_load_acquire(void* addr) {
906 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
907 OrderAccess::fence();
908 }
909 return (T)OrderAccess::load_acquire((volatile typename IntegerType<T>::signed_type*)addr);
910 }
911
912 template <DecoratorSet decorators>
913 template <typename T>
914 inline typename EnableIf<IsPointerSize<T>::value, T>::type
915 RawAccessBarrier<decorators>::cas_relaxed(T new_value, void* addr, T compare_value) {
916 return (T)Atomic::cmpxchg_ptr((void*)new_value,
917 (volatile void*)addr,
918 (void*)compare_value,
919 memory_order_relaxed);
920 }
921
922 template <DecoratorSet decorators>
923 template <typename T>
924 inline typename EnableIf<!IsPointerSize<T>::value, T>::type
925 RawAccessBarrier<decorators>::cas_relaxed(T new_value, void* addr, T compare_value) {
926 return (T)Atomic::cmpxchg((typename IntegerType<T>::signed_type)new_value,
927 (volatile typename IntegerType<T>::signed_type*)addr,
928 (typename IntegerType<T>::signed_type)compare_value,
929 memory_order_relaxed);
930 }
931
932 template <DecoratorSet decorators>
933 template <typename T>
934 inline typename EnableIf<IsPointerSize<T>::value, T>::type
935 RawAccessBarrier<decorators>::cas_seq_cst(T new_value, void* addr, T compare_value) {
936 return (T)Atomic::cmpxchg_ptr((void*)new_value,
937 (volatile void*)addr,
938 (void*)compare_value,
939 memory_order_conservative);
940 }
941
942 template <DecoratorSet decorators>
943 template <typename T>
944 inline typename EnableIf<!IsPointerSize<T>::value, T>::type
945 RawAccessBarrier<decorators>::cas_seq_cst(T new_value, void* addr, T compare_value) {
946 return (T)Atomic::cmpxchg((typename IntegerType<T>::signed_type)new_value,
947 (volatile typename IntegerType<T>::signed_type*)addr,
948 (typename IntegerType<T>::signed_type)compare_value,
949 memory_order_conservative);
950 }
951
952 template <DecoratorSet decorators>
953 template <typename T>
954 inline typename EnableIf<IsPointerSize<T>::value, T>::type
955 RawAccessBarrier<decorators>::swap_seq_cst(T new_value, void* addr) {
956 return (T)Atomic::xchg_ptr((void*)new_value,
957 (volatile void*)addr);
958 }
959
960 template <DecoratorSet decorators>
961 template <typename T>
962 inline typename EnableIf<!IsPointerSize<T>::value, T>::type
963 RawAccessBarrier<decorators>::swap_seq_cst(T new_value, void* addr) {
964 return (T)Atomic::xchg((typename IntegerType<T>::signed_type)new_value,
965 (volatile typename IntegerType<T>::signed_type*)addr);
966 }
967
968 template <DecoratorSet ds>
969 template <typename T, DecoratorSet decorators>
970 inline typename EnableIf<PossiblyLockedAccess<T, decorators>::value, T>::type
971 RawAccessBarrier<ds>::load_maybe_locked(void* addr) {
972 STATIC_ASSERT(DecoratorTest<ds>::HAS_ACCESS_ON_ANONYMOUS); // why else?
973 if (AccessInternal::wide_atomic_needs_locking()) {
974 return load_not_locked<T>(addr);
975 } else {
976 return (T)AccessInternal::load_locked(addr);
977 }
978 }
979
980 template <DecoratorSet ds>
981 template <typename T, DecoratorSet decorators>
982 inline typename EnableIf<PossiblyLockedAccess<T, decorators>::value, void>::type
983 RawAccessBarrier<ds>::store_maybe_locked(void* addr, T value) {
984 STATIC_ASSERT(DecoratorTest<ds>::HAS_ACCESS_ON_ANONYMOUS); // why else?
985 if (AccessInternal::wide_atomic_needs_locking()) {
986 store_not_locked<T>(addr, value);
987 } else {
988 AccessInternal::store_locked(addr, IntegerType<T>::cast_to_signed(value));
989 }
990 }
991
992 template <DecoratorSet ds>
993 template <typename T, DecoratorSet decorators>
994 inline typename EnableIf<PossiblyLockedAccess<T, decorators>::value, T>::type
995 RawAccessBarrier<ds>::swap_maybe_locked(T new_value, void* addr) {
996 STATIC_ASSERT(DecoratorTest<ds>::HAS_ACCESS_ON_ANONYMOUS); // why else?
997 if (AccessInternal::wide_atomic_needs_locking()) {
998 return swap_not_locked<T>(new_value, addr);
999 } else {
1000 return (T)AccessInternal::swap_locked(IntegerType<T>::cast_to_signed(new_value), addr);
1001 }
1002 }
1003
1004 template <DecoratorSet ds>
1005 template <typename T, DecoratorSet decorators>
1006 inline typename EnableIf<PossiblyLockedAccess<T, decorators>::value, T>::type
1007 RawAccessBarrier<ds>::cas_maybe_locked(T new_value, void* addr, T compare_value) {
1008 STATIC_ASSERT(DecoratorTest<ds>::HAS_ACCESS_ON_ANONYMOUS); // why else?
1009 if (AccessInternal::wide_atomic_needs_locking()) {
1010 return cas_not_locked<T>(new_value, addr, compare_value);
1011 } else {
1012 return (T)AccessInternal::cas_locked(IntegerType<T>::cast_to_signed(new_value), addr,
1013 IntegerType<T>::cast_to_signed(compare_value));
1014 }
1015 }
1016
1017 class RawAccessBarrierCopy: public AllStatic {
1018 public:
1019 template <DecoratorSet decorators, typename T>
1020 static inline typename EnableIf<DecoratorTest<decorators>::HAS_VALUE_IS_OOP &&
1021 !IsSame<T, HeapWord>::value, void>::type
1022 copy(T* src, T* dst, size_t length) {
1023 if (DecoratorTest<decorators>::HAS_COPY_ARRAYOF) {
1024 AccessInternal::copy_arrayof_conjoint_oops((void*)src, (void*)dst, length);
1025 } else {
1026 AccessInternal::copy_conjoint_oops((void*)src, (void*)dst, length);
1027 }
1028 }
1029
1030 template <DecoratorSet decorators, typename T>
1031 static inline typename EnableIf<!DecoratorTest<decorators>::HAS_VALUE_IS_OOP ||
1032 IsSame<T, HeapWord>::value, void>::type
1033 copy(T* src, T* dst, size_t length) {
1034 if (DecoratorTest<decorators>::HAS_DEST_CONJOINT) {
1035 if (DecoratorTest<decorators>::HAS_ACCESS_ATOMIC) {
1036 AccessInternal::copy_conjoint_memory_atomic((void*)src, (void*)dst, sizeof(T) * length);
1037 } else {
1038 AccessInternal::copy_conjoint_jbytes((void*)src, (void*)dst, sizeof(T) * length);
1039 }
1040 } else {
1041 if (DecoratorTest<decorators>::HAS_ACCESS_ATOMIC) {
1042 AccessInternal::copy_conjoint_memory_atomic((void*)src, (void*)dst, sizeof(T) * length);
1043 } else if (sizeof(T) == HeapWordSize) {
1044 AccessInternal::copy_disjoint_words((void*)src, (void*)dst, sizeof(T) * length);
1045 } else {
1046 AccessInternal::copy_conjoint_jbytes((void*)src, (void*)dst, sizeof(T) * length);
1047 }
1048 }
1049 }
1050 };
1051
1052 template <DecoratorSet decorators>
1053 template <typename T>
1054 inline typename EnableIf<!IsFloatLike<T>::value, bool>::type
1055 RawAccessBarrier<decorators>::copy(T* src, T* dst, size_t length) {
1056 RawAccessBarrierCopy::copy<decorators, T>(src, dst, length);
1057 return true;
1058 }
1059
1060 template <DecoratorSet decorators>
1061 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
1062 assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
1063 AccessInternal::copy_conjoint_jlongs_atomic((void*)src, (void*)dst,
1064 (size_t)align_object_size(size) / HeapWordsPerLong);
1065 // Clear the header
1066 dst->init_mark();
1067 }
1068
1069 namespace AccessInternal {
1070 // The first time a barrier needs to be resolved at runtime, it needs
1071 // to link it against the GC being used to patch a function pointer
1072 // directing subsequent memory accesses to the resolved barrier
1073 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
1074 void* AccessBarrierResolver<decorators, T, barrier_type>::resolve_barrier() {
1075 return BarrierSet::barrier_set()->resolve_barrier<decorators, T, barrier_type>();
1076 }
1077
1078 template <DecoratorSet decorators>
1079 void* CloneAccessBarrierResolver<decorators>::resolve_barrier() {
1080 return BarrierSet::barrier_set()->resolve_clone_barrier<decorators>();
1081 }
1082
1083 template <DecoratorSet static_decorators, typename T, BarrierType type>
1084 class PostRTDispatch: public AllStatic {
1085 enum {
1086 resolve_compressed_oops = type == BARRIER_COPY && DecoratorTest<static_decorators>::HAS_VALUE_IS_OOP &&
1087 IsSame<T, HeapWord>::value
1088 };
1089
1090 template <bool resolve_compressed_oops>
1091 static typename EnableIf<resolve_compressed_oops, void*>::type resolve_barrier_internal() {
1092 if (DecoratorTest<static_decorators>::NEEDS_OOP_COMPRESS) {
1093 return DispatcherFactory<static_decorators, narrowOop, type>::resolve_barrier_gc();
1094 } else {
1095 return DispatcherFactory<static_decorators, oop, type>::resolve_barrier_gc();
1096 }
1097 }
1098
1099 template <bool resolve_compressed_oops>
1100 static typename EnableIf<!resolve_compressed_oops, void*>::type resolve_barrier_internal() {
1101 return DispatcherFactory<static_decorators, T, type>::resolve_barrier_gc();
1102 }
1103 public:
1104 static void* resolve_barrier() {
1105 return resolve_barrier_internal<resolve_compressed_oops>();
1106 }
1107 };
1108
1109 template <DecoratorSet static_decorators, typename T, BarrierType type>
1110 inline void* DispatcherFactory<static_decorators, T, type>::resolve_barrier_post_rt() {
1111 return PostRTDispatch<static_decorators, T, type>::resolve_barrier();
1112 }
1113
1114 template <DecoratorSet decorators>
1115 template <typename T>
1116 inline void BuildtimeDispatch<decorators>::store(void* addr, T value) {
1117 const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
1118 PreRuntimeDispatch<expanded_decorators>::template store<expanded_decorators, T>(addr, value);
1119 }
1120
1121 template <DecoratorSet decorators>
1122 template <typename T>
1123 inline void BuildtimeDispatch<decorators>::store_at(void* base, ptrdiff_t offset, T value) {
1124 const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
1125 PreRuntimeDispatch<expanded_decorators>::template store_at<expanded_decorators, T>(base, offset, value);
1126 }
1127
1128 template <DecoratorSet decorators>
1129 template <typename T>
1130 inline T BuildtimeDispatch<decorators>::load(void* addr) {
1131 const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
1132 return PreRuntimeDispatch<expanded_decorators>::template load<expanded_decorators, T>(addr);
1133 }
1134
1135 template <DecoratorSet decorators>
1136 template <typename T>
1137 inline T BuildtimeDispatch<decorators>::load_at(void* base, ptrdiff_t offset) {
1138 const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
1139 return PreRuntimeDispatch<expanded_decorators>::template load_at<expanded_decorators, T>(base, offset);
1140 }
1141
1142 template <DecoratorSet decorators>
1143 template <typename T>
1144 inline T BuildtimeDispatch<decorators>::cas(T new_value, void* addr, T compare_value) {
1145 const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
1146 return PreRuntimeDispatch<expanded_decorators>::template cas<expanded_decorators, T>(new_value, addr, compare_value);
1147 }
1148
1149 template <DecoratorSet decorators>
1150 template <typename T>
1151 inline T BuildtimeDispatch<decorators>::cas_at(T new_value, void* base, ptrdiff_t offset, T compare_value) {
1152 const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
1153 return PreRuntimeDispatch<expanded_decorators>::template cas_at<expanded_decorators, T>(new_value, base, offset, compare_value);
1154 }
1155
1156 template <DecoratorSet decorators>
1157 template <typename T>
1158 inline T BuildtimeDispatch<decorators>::swap(T new_value, void* addr) {
1159 const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
1160 return PreRuntimeDispatch<expanded_decorators>::template swap<expanded_decorators, T>(new_value, addr);
1161 }
1162
1163 template <DecoratorSet decorators>
1164 template <typename T>
1165 inline T BuildtimeDispatch<decorators>::swap_at(T new_value, void* base, ptrdiff_t offset) {
1166 const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
1167 return PreRuntimeDispatch<expanded_decorators>::template swap_at<expanded_decorators, T>(new_value, base, offset);
1168 }
1169
1170 template <DecoratorSet decorators>
1171 template <typename T>
1172 inline bool BuildtimeDispatch<decorators>::copy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
1173 const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
1174 return PreRuntimeDispatch<expanded_decorators>::template copy<expanded_decorators, T>(src_obj, dst_obj, src, dst, length);
1175 }
1176
1177 template <DecoratorSet decorators>
1178 inline void BuildtimeDispatch<decorators>::clone(oop src, oop dst, size_t size) {
1179 const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
1180 PreRuntimeDispatch<expanded_decorators>::template clone<expanded_decorators>(src, dst, size);
1181 }
1182
1183 template <DecoratorSet decorators, typename P, typename T>
1184 inline void store_at_reduce_types(P base, ptrdiff_t offset, T value) {
1185 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1186 const DecoratorSet final_decorators = expanded_decorators |
1187 ((DecoratorTest<expanded_decorators>::HAS_VALUE_IS_OOP &&
1188 !DecoratorTest<expanded_decorators>::HAS_ACCESS_ON_ROOT) ?
1189 GC_CONVERT_COMPRESSED_OOP : EMPTY_DECORATOR);
1190 BuildtimeDispatch<final_decorators>::template store_at<T>((void*)base, offset, value);
1191 }
1192
1193 template <DecoratorSet decorators, typename P, typename T>
1194 inline T DispatchLoadAtReduceTypes<decorators, P, T>::load_at(P base, ptrdiff_t offset) {
1195 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1196 const DecoratorSet final_decorators = expanded_decorators |
1197 ((DecoratorTest<expanded_decorators>::HAS_VALUE_IS_OOP &&
1198 !DecoratorTest<expanded_decorators>::HAS_ACCESS_ON_ROOT) ?
1199 GC_CONVERT_COMPRESSED_OOP : EMPTY_DECORATOR);
1200 return BuildtimeDispatch<final_decorators>::template load_at<T>((void*)base, offset);
1201 }
1202
1203 template <DecoratorSet decorators, typename P, typename T>
1204 inline T cas_at_reduce_types(T new_value, P base, ptrdiff_t offset, T compare_value) {
1205 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1206 const DecoratorSet final_decorators = expanded_decorators |
1207 ((DecoratorTest<expanded_decorators>::HAS_VALUE_IS_OOP &&
1208 !DecoratorTest<expanded_decorators>::HAS_ACCESS_ON_ROOT) ?
1209 GC_CONVERT_COMPRESSED_OOP : EMPTY_DECORATOR);
1210 return BuildtimeDispatch<final_decorators>::template cas_at<T>(new_value, (void*)base, offset, compare_value);
1211 }
1212
1213 template <DecoratorSet decorators, typename P, typename T>
1214 inline T swap_at_reduce_types(T new_value, P base, ptrdiff_t offset) {
1215 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1216 const DecoratorSet final_decorators = expanded_decorators |
1217 ((DecoratorTest<expanded_decorators>::HAS_VALUE_IS_OOP &&
1218 !DecoratorTest<expanded_decorators>::HAS_ACCESS_ON_ROOT) ?
1219 GC_CONVERT_COMPRESSED_OOP : EMPTY_DECORATOR);
1220 return BuildtimeDispatch<final_decorators>::template swap_at<T>(new_value, (void*)base, offset);
1221 }
1222 }
1223
1224 #endif // SHARE_VM_RUNTIME_ACCESS_INLINE_HPP