1 /*
2 * Copyright (c) 1999, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_VM_RUNTIME_ATOMIC_HPP
26 #define SHARE_VM_RUNTIME_ATOMIC_HPP
27
28 #include "memory/allocation.hpp"
29 #include "metaprogramming/conditional.hpp"
30 #include "metaprogramming/enableIf.hpp"
31 #include "metaprogramming/isIntegral.hpp"
32 #include "metaprogramming/isPointer.hpp"
33 #include "metaprogramming/isSame.hpp"
34 #include "metaprogramming/primitiveConversions.hpp"
35 #include "metaprogramming/removeCV.hpp"
36 #include "metaprogramming/removePointer.hpp"
37 #include "utilities/align.hpp"
38 #include "utilities/macros.hpp"
39
40 enum cmpxchg_memory_order {
41 memory_order_relaxed,
42 // Use value which doesn't interfere with C++2011. We need to be more conservative.
43 memory_order_conservative = 8
44 };
45
46 class Atomic : AllStatic {
47 public:
48 // Atomic operations on jlong types are not available on all 32-bit
49 // platforms. If atomic ops on jlongs are defined here they must only
50 // be used from code that verifies they are available at runtime and
51 // can provide an alternative action if not - see supports_cx8() for
52 // a means to test availability.
53
54 // The memory operations that are mentioned with each of the atomic
55 // function families come from src/share/vm/runtime/orderAccess.hpp,
56 // e.g., <fence> is described in that file and is implemented by the
57 // OrderAccess::fence() function. See that file for the gory details
58 // on the Memory Access Ordering Model.
59
60 // All of the atomic operations that imply a read-modify-write action
61 // guarantee a two-way memory barrier across that operation. Historically
62 // these semantics reflect the strength of atomic operations that are
63 // provided on SPARC/X86. We assume that strength is necessary unless
64 // we can prove that a weaker form is sufficiently safe.
65
66 // Atomically store to a location
67 // The type T must be either a pointer type convertible to or equal
68 // to D, an integral/enum type equal to D, or a type equal to D that
69 // is primitive convertible using PrimitiveConversions.
70 template<typename T, typename D>
71 inline static void store(T store_value, volatile D* dest);
72
73 inline static void store_ptr(intptr_t store_value, volatile intptr_t* dest) {
74 Atomic::store(store_value, dest);
75 }
76
77 inline static void store_ptr(void* store_value, volatile void* dest) {
78 Atomic::store(store_value, reinterpret_cast<void* volatile*>(dest));
79 }
80
81 // Atomically load from a location
82 // The type T must be either a pointer type, an integral/enum type,
83 // or a type that is primitive convertible using PrimitiveConversions.
84 template<typename T>
85 inline static T load(const volatile T* dest);
86
87 // Atomically add to a location. Returns updated value. add*() provide:
88 // <fence> add-value-to-dest <membar StoreLoad|StoreStore>
89
90 template<typename I, typename D>
91 inline static D add(I add_value, D volatile* dest);
92
93 inline static intptr_t add_ptr(intptr_t add_value, volatile intptr_t* dest) {
94 return add(add_value, dest);
95 }
96
97 inline static void* add_ptr(intptr_t add_value, volatile void* dest) {
98 return add(add_value, reinterpret_cast<char* volatile*>(dest));
99 }
100
101 // Atomically increment location. inc() provide:
102 // <fence> increment-dest <membar StoreLoad|StoreStore>
103 // The type D may be either a pointer type, or an integral
104 // type. If it is a pointer type, then the increment is
105 // scaled to the size of the type pointed to by the pointer.
106 template<typename D>
107 inline static void inc(D volatile* dest);
108
109 // Atomically decrement a location. dec() provide:
110 // <fence> decrement-dest <membar StoreLoad|StoreStore>
111 // The type D may be either a pointer type, or an integral
112 // type. If it is a pointer type, then the decrement is
113 // scaled to the size of the type pointed to by the pointer.
114 template<typename D>
115 inline static void dec(D volatile* dest);
116
117 // Performs atomic exchange of *dest with exchange_value. Returns old
118 // prior value of *dest. xchg*() provide:
119 // <fence> exchange-value-with-dest <membar StoreLoad|StoreStore>
120 // The type T must be either a pointer type convertible to or equal
121 // to D, an integral/enum type equal to D, or a type equal to D that
122 // is primitive convertible using PrimitiveConversions.
123 template<typename T, typename D>
124 inline static D xchg(T exchange_value, volatile D* dest);
125
126 inline static intptr_t xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest) {
127 return xchg(exchange_value, dest);
128 }
129
130 inline static void* xchg_ptr(void* exchange_value, volatile void* dest) {
131 return xchg(exchange_value, reinterpret_cast<void* volatile*>(dest));
132 }
133
134 // Performs atomic compare of *dest and compare_value, and exchanges
135 // *dest with exchange_value if the comparison succeeded. Returns prior
136 // value of *dest. cmpxchg*() provide:
137 // <fence> compare-and-exchange <membar StoreLoad|StoreStore>
138
139 template<typename T, typename D, typename U>
140 inline static D cmpxchg(T exchange_value,
141 D volatile* dest,
142 U compare_value,
143 cmpxchg_memory_order order = memory_order_conservative);
144
145 // Performs atomic compare of *dest and NULL, and replaces *dest
146 // with exchange_value if the comparison succeeded. Returns true if
147 // the comparison succeeded and the exchange occurred. This is
148 // often used as part of lazy initialization, as a lock-free
149 // alternative to the Double-Checked Locking Pattern.
150 template<typename T, typename D>
151 inline static bool replace_if_null(T* value, D* volatile* dest,
152 cmpxchg_memory_order order = memory_order_conservative);
153
154 inline static intptr_t cmpxchg_ptr(intptr_t exchange_value,
155 volatile intptr_t* dest,
156 intptr_t compare_value,
157 cmpxchg_memory_order order = memory_order_conservative) {
158 return cmpxchg(exchange_value, dest, compare_value, order);
159 }
160
161 inline static void* cmpxchg_ptr(void* exchange_value,
162 volatile void* dest,
163 void* compare_value,
164 cmpxchg_memory_order order = memory_order_conservative) {
165 return cmpxchg(exchange_value,
166 reinterpret_cast<void* volatile*>(dest),
167 compare_value,
168 order);
169 }
170
171 private:
172 // Test whether From is implicitly convertible to To.
173 // From and To must be pointer types.
174 // Note: Provides the limited subset of C++11 std::is_convertible
175 // that is needed here.
176 template<typename From, typename To> struct IsPointerConvertible;
177
178 protected:
179 // Dispatch handler for store. Provides type-based validity
180 // checking and limited conversions around calls to the platform-
181 // specific implementation layer provided by PlatformOp.
182 template<typename T, typename D, typename PlatformOp, typename Enable = void>
183 struct StoreImpl;
184
185 // Platform-specific implementation of store. Support for sizes
186 // of 1, 2, 4, and (if different) pointer size bytes are required.
187 // The class is a function object that must be default constructable,
188 // with these requirements:
189 //
190 // either:
191 // - dest is of type D*, an integral, enum or pointer type.
192 // - new_value are of type T, an integral, enum or pointer type D or
193 // pointer type convertible to D.
194 // or:
195 // - T and D are the same and are primitive convertible using PrimitiveConversions
196 // and either way:
197 // - platform_store is an object of type PlatformStore<sizeof(T)>.
198 //
199 // Then
200 // platform_store(new_value, dest)
201 // must be a valid expression.
202 //
203 // The default implementation is a volatile store. If a platform
204 // requires more for e.g. 64 bit stores, a specialization is required
205 template<size_t byte_size> struct PlatformStore;
206
207 // Dispatch handler for load. Provides type-based validity
208 // checking and limited conversions around calls to the platform-
209 // specific implementation layer provided by PlatformOp.
210 template<typename T, typename PlatformOp, typename Enable = void>
211 struct LoadImpl;
212
213 // Platform-specific implementation of load. Support for sizes of
214 // 1, 2, 4 bytes and (if different) pointer size bytes are required.
215 // The class is a function object that must be default
216 // constructable, with these requirements:
217 //
218 // - dest is of type T*, an integral, enum or pointer type, or
219 // T is convertible to a primitive type using PrimitiveConversions
220 // - platform_load is an object of type PlatformLoad<sizeof(T)>.
221 //
222 // Then
223 // platform_load(src)
224 // must be a valid expression, returning a result convertible to T.
225 //
226 // The default implementation is a volatile load. If a platform
227 // requires more for e.g. 64 bit loads, a specialization is required
228 template<size_t byte_size> struct PlatformLoad;
229
230 private:
231 // Dispatch handler for add. Provides type-based validity checking
232 // and limited conversions around calls to the platform-specific
233 // implementation layer provided by PlatformAdd.
234 template<typename I, typename D, typename Enable = void>
235 struct AddImpl;
236
237 // Platform-specific implementation of add. Support for sizes of 4
238 // bytes and (if different) pointer size bytes are required. The
239 // class is a function object that must be default constructable,
240 // with these requirements:
241 //
242 // - dest is of type D*, an integral or pointer type.
243 // - add_value is of type I, an integral type.
244 // - sizeof(I) == sizeof(D).
245 // - if D is an integral type, I == D.
246 // - platform_add is an object of type PlatformAdd<sizeof(D)>.
247 //
248 // Then
249 // platform_add(add_value, dest)
250 // must be a valid expression, returning a result convertible to D.
251 //
252 // No definition is provided; all platforms must explicitly define
253 // this class and any needed specializations.
254 template<size_t byte_size> struct PlatformAdd;
255
256 // Helper base classes for defining PlatformAdd. To use, define
257 // PlatformAdd or a specialization that derives from one of these,
258 // and include in the PlatformAdd definition the support function
259 // (described below) required by the base class.
260 //
261 // These classes implement the required function object protocol for
262 // PlatformAdd, using a support function template provided by the
263 // derived class. Let add_value (of type I) and dest (of type D) be
264 // the arguments the object is called with. If D is a pointer type
265 // P*, then let addend (of type I) be add_value * sizeof(P);
266 // otherwise, addend is add_value.
267 //
268 // FetchAndAdd requires the derived class to provide
269 // fetch_and_add(addend, dest)
270 // atomically adding addend to the value of dest, and returning the
271 // old value.
272 //
273 // AddAndFetch requires the derived class to provide
274 // add_and_fetch(addend, dest)
275 // atomically adding addend to the value of dest, and returning the
276 // new value.
277 //
278 // When D is a pointer type P*, both fetch_and_add and add_and_fetch
279 // treat it as if it were a uintptr_t; they do not perform any
280 // scaling of the addend, as that has already been done by the
281 // caller.
282 public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11.
283 template<typename Derived> struct FetchAndAdd;
284 template<typename Derived> struct AddAndFetch;
285 private:
286
287 // Support for platforms that implement some variants of add using a
288 // (typically out of line) non-template helper function. The
289 // generic arguments passed to PlatformAdd need to be translated to
290 // the appropriate type for the helper function, the helper function
291 // invoked on the translated arguments, and the result translated
292 // back. Type is the parameter / return type of the helper
293 // function. No scaling of add_value is performed when D is a pointer
294 // type, so this function can be used to implement the support function
295 // required by AddAndFetch.
296 template<typename Type, typename Fn, typename I, typename D>
297 static D add_using_helper(Fn fn, I add_value, D volatile* dest);
298
299 // Dispatch handler for cmpxchg. Provides type-based validity
300 // checking and limited conversions around calls to the
301 // platform-specific implementation layer provided by
302 // PlatformCmpxchg.
303 template<typename T, typename D, typename U, typename Enable = void>
304 struct CmpxchgImpl;
305
306 // Platform-specific implementation of cmpxchg. Support for sizes
307 // of 1, 4, and 8 are required. The class is a function object that
308 // must be default constructable, with these requirements:
309 //
310 // - dest is of type T*.
311 // - exchange_value and compare_value are of type T.
312 // - order is of type cmpxchg_memory_order.
313 // - platform_cmpxchg is an object of type PlatformCmpxchg<sizeof(T)>.
314 //
315 // Then
316 // platform_cmpxchg(exchange_value, dest, compare_value, order)
317 // must be a valid expression, returning a result convertible to T.
318 //
319 // A default definition is provided, which declares a function template
320 // T operator()(T, T volatile*, T, cmpxchg_memory_order) const
321 //
322 // For each required size, a platform must either provide an
323 // appropriate definition of that function, or must entirely
324 // specialize the class template for that size.
325 template<size_t byte_size> struct PlatformCmpxchg;
326
327 // Support for platforms that implement some variants of cmpxchg
328 // using a (typically out of line) non-template helper function.
329 // The generic arguments passed to PlatformCmpxchg need to be
330 // translated to the appropriate type for the helper function, the
331 // helper invoked on the translated arguments, and the result
332 // translated back. Type is the parameter / return type of the
333 // helper function.
334 template<typename Type, typename Fn, typename T>
335 static T cmpxchg_using_helper(Fn fn,
336 T exchange_value,
337 T volatile* dest,
338 T compare_value);
339
340 // Support platforms that do not provide Read-Modify-Write
341 // byte-level atomic access. To use, derive PlatformCmpxchg<1> from
342 // this class.
343 public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11.
344 struct CmpxchgByteUsingInt;
345 private:
346
347 // Dispatch handler for xchg. Provides type-based validity
348 // checking and limited conversions around calls to the
349 // platform-specific implementation layer provided by
350 // PlatformXchg.
351 template<typename T, typename D, typename Enable = void>
352 struct XchgImpl;
353
354 // Platform-specific implementation of xchg. Support for sizes
355 // of 4, and sizeof(intptr_t) are required. The class is a function
356 // object that must be default constructable, with these requirements:
357 //
358 // - dest is of type T*.
359 // - exchange_value is of type T.
360 // - platform_xchg is an object of type PlatformXchg<sizeof(T)>.
361 //
362 // Then
363 // platform_xchg(exchange_value, dest)
364 // must be a valid expression, returning a result convertible to T.
365 //
366 // A default definition is provided, which declares a function template
367 // T operator()(T, T volatile*, T, cmpxchg_memory_order) const
368 //
369 // For each required size, a platform must either provide an
370 // appropriate definition of that function, or must entirely
371 // specialize the class template for that size.
372 template<size_t byte_size> struct PlatformXchg;
373
374 // Support for platforms that implement some variants of xchg
375 // using a (typically out of line) non-template helper function.
376 // The generic arguments passed to PlatformXchg need to be
377 // translated to the appropriate type for the helper function, the
378 // helper invoked on the translated arguments, and the result
379 // translated back. Type is the parameter / return type of the
380 // helper function.
381 template<typename Type, typename Fn, typename T>
382 static T xchg_using_helper(Fn fn,
383 T exchange_value,
384 T volatile* dest);
385 };
386
387 template<typename From, typename To>
388 struct Atomic::IsPointerConvertible<From*, To*> : AllStatic {
389 // Determine whether From* is implicitly convertible to To*, using
390 // the "sizeof trick".
391 typedef char yes;
392 typedef char (&no)[2];
393
394 static yes test(To*);
395 static no test(...);
396 static From* test_value;
397
398 static const bool value = (sizeof(yes) == sizeof(test(test_value)));
399 };
400
401 // Handle load for pointer, integral and enum types.
402 template<typename T, typename PlatformOp>
403 struct Atomic::LoadImpl<
404 T,
405 PlatformOp,
406 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value || IsPointer<T>::value>::type>
407 VALUE_OBJ_CLASS_SPEC
408 {
409 T operator()(T const volatile* dest) const {
410 // Forward to the platform handler for the size of T.
411 return PlatformOp()(dest);
412 }
413 };
414
415 // Handle load for types that have a translator.
416 //
417 // All the involved types must be identical.
418 //
419 // This translates the original call into a call on the decayed
420 // arguments, and returns the recovered result of that translated
421 // call.
422 template<typename T, typename PlatformOp>
423 struct Atomic::LoadImpl<
424 T,
425 PlatformOp,
426 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
427 VALUE_OBJ_CLASS_SPEC
428 {
429 T operator()(T const volatile* dest) const {
430 typedef PrimitiveConversions::Translate<T> Translator;
431 typedef typename Translator::Decayed Decayed;
432 STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
433 Decayed result = PlatformOp()(reinterpret_cast<Decayed const volatile*>(dest));
434 return Translator::recover(result);
435 }
436 };
437
438 // Default implementation of atomic load if a specific platform
439 // does not provide a specialization for a certain size class.
440 // For increased safety, the default implementation only allows
441 // load types that are pointer sized or smaller. If a platform still
442 // supports wide atomics, then it has to use specialization
443 // of Atomic::PlatformLoad for that wider size class.
444 template<size_t byte_size>
445 struct Atomic::PlatformLoad VALUE_OBJ_CLASS_SPEC {
446 template<typename T>
447 T operator()(T const volatile* dest) const {
448 STATIC_ASSERT(sizeof(T) <= sizeof(void*)); // wide atomics need specialization
449 return *dest;
450 }
451 };
452
453 // Handle store for integral and enum types.
454 //
455 // All the involved types must be identical.
456 template<typename T, typename PlatformOp>
457 struct Atomic::StoreImpl<
458 T, T,
459 PlatformOp,
460 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
461 VALUE_OBJ_CLASS_SPEC
462 {
463 void operator()(T new_value, T volatile* dest) const {
464 // Forward to the platform handler for the size of T.
465 PlatformOp()(new_value, dest);
466 }
467 };
468
469 // Handle store for pointer types.
470 //
471 // The new_value must be implicitly convertible to the
472 // destination's type; it must be type-correct to store the
473 // new_value in the destination.
474 template<typename T, typename D, typename PlatformOp>
475 struct Atomic::StoreImpl<
476 T*, D*,
477 PlatformOp,
478 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type>
479 VALUE_OBJ_CLASS_SPEC
480 {
481 void operator()(T* new_value, D* volatile* dest) const {
482 // Allow derived to base conversion, and adding cv-qualifiers.
483 D* value = new_value;
484 PlatformOp()(value, dest);
485 }
486 };
487
488 // Handle store for types that have a translator.
489 //
490 // All the involved types must be identical.
491 //
492 // This translates the original call into a call on the decayed
493 // arguments.
494 template<typename T, typename PlatformOp>
495 struct Atomic::StoreImpl<
496 T, T,
497 PlatformOp,
498 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
499 VALUE_OBJ_CLASS_SPEC
500 {
501 void operator()(T new_value, T volatile* dest) const {
502 typedef PrimitiveConversions::Translate<T> Translator;
503 typedef typename Translator::Decayed Decayed;
504 STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
505 PlatformOp()(Translator::decay(new_value),
506 reinterpret_cast<Decayed volatile*>(dest));
507 }
508 };
509
510 // Default implementation of atomic store if a specific platform
511 // does not provide a specialization for a certain size class.
512 // For increased safety, the default implementation only allows
513 // storing types that are pointer sized or smaller. If a platform still
514 // supports wide atomics, then it has to use specialization
515 // of Atomic::PlatformStore for that wider size class.
516 template<size_t byte_size>
517 struct Atomic::PlatformStore VALUE_OBJ_CLASS_SPEC {
518 template<typename T>
519 void operator()(T new_value,
520 T volatile* dest) const {
521 STATIC_ASSERT(sizeof(T) <= sizeof(void*)); // wide atomics need specialization
522 (void)const_cast<T&>(*dest = new_value);
523 }
524 };
525
526 // Define FetchAndAdd and AddAndFetch helper classes before including
527 // platform file, which may use these as base classes, requiring they
528 // be complete.
529
530 template<typename Derived>
531 struct Atomic::FetchAndAdd VALUE_OBJ_CLASS_SPEC {
532 template<typename I, typename D>
533 D operator()(I add_value, D volatile* dest) const;
534 };
535
536 template<typename Derived>
537 struct Atomic::AddAndFetch VALUE_OBJ_CLASS_SPEC {
538 template<typename I, typename D>
539 D operator()(I add_value, D volatile* dest) const;
540 };
541
542 template<typename D>
543 inline void Atomic::inc(D volatile* dest) {
544 STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
545 typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I;
546 Atomic::add(I(1), dest);
547 }
548
549 template<typename D>
550 inline void Atomic::dec(D volatile* dest) {
551 STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
552 typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I;
553 // Assumes two's complement integer representation.
554 #pragma warning(suppress: 4146)
555 Atomic::add(I(-1), dest);
556 }
557
558 // Define the class before including platform file, which may specialize
559 // the operator definition. No generic definition of specializations
560 // of the operator template are provided, nor are there any generic
561 // specializations of the class. The platform file is responsible for
562 // providing those.
563 template<size_t byte_size>
564 struct Atomic::PlatformCmpxchg VALUE_OBJ_CLASS_SPEC {
565 template<typename T>
566 T operator()(T exchange_value,
567 T volatile* dest,
568 T compare_value,
569 cmpxchg_memory_order order) const;
570 };
571
572 // Define the class before including platform file, which may use this
573 // as a base class, requiring it be complete. The definition is later
574 // in this file, near the other definitions related to cmpxchg.
575 struct Atomic::CmpxchgByteUsingInt VALUE_OBJ_CLASS_SPEC {
576 template<typename T>
577 T operator()(T exchange_value,
578 T volatile* dest,
579 T compare_value,
580 cmpxchg_memory_order order) const;
581 };
582
583 // Define the class before including platform file, which may specialize
584 // the operator definition. No generic definition of specializations
585 // of the operator template are provided, nor are there any generic
586 // specializations of the class. The platform file is responsible for
587 // providing those.
588 template<size_t byte_size>
589 struct Atomic::PlatformXchg VALUE_OBJ_CLASS_SPEC {
590 template<typename T>
591 T operator()(T exchange_value,
592 T volatile* dest) const;
593 };
594
595 // platform specific in-line definitions - must come before shared definitions
596
597 #include OS_CPU_HEADER(atomic)
598
599 // shared in-line definitions
600
601 // size_t casts...
602 #if (SIZE_MAX != UINTPTR_MAX)
603 #error size_t is not WORD_SIZE, interesting platform, but missing implementation here
604 #endif
605
606 template<typename T>
607 inline T Atomic::load(const volatile T* dest) {
608 return LoadImpl<T, PlatformLoad<sizeof(T)> >()(dest);
609 }
610
611 template<typename T, typename D>
612 inline void Atomic::store(T store_value, volatile D* dest) {
613 StoreImpl<T, D, PlatformStore<sizeof(D)> >()(store_value, dest);
614 }
615
616 template<typename I, typename D>
617 inline D Atomic::add(I add_value, D volatile* dest) {
618 return AddImpl<I, D>()(add_value, dest);
619 }
620
621 template<typename I, typename D>
622 struct Atomic::AddImpl<
623 I, D,
624 typename EnableIf<IsIntegral<I>::value &&
625 IsIntegral<D>::value &&
626 (sizeof(I) <= sizeof(D)) &&
627 (IsSigned<I>::value == IsSigned<D>::value)>::type>
628 VALUE_OBJ_CLASS_SPEC
629 {
630 D operator()(I add_value, D volatile* dest) const {
631 D addend = add_value;
632 return PlatformAdd<sizeof(D)>()(addend, dest);
633 }
634 };
635
636 template<typename I, typename P>
637 struct Atomic::AddImpl<
638 I, P*,
639 typename EnableIf<IsIntegral<I>::value && (sizeof(I) <= sizeof(P*))>::type>
640 VALUE_OBJ_CLASS_SPEC
641 {
642 P* operator()(I add_value, P* volatile* dest) const {
643 STATIC_ASSERT(sizeof(intptr_t) == sizeof(P*));
644 STATIC_ASSERT(sizeof(uintptr_t) == sizeof(P*));
645 typedef typename Conditional<IsSigned<I>::value,
646 intptr_t,
647 uintptr_t>::type CI;
648 CI addend = add_value;
649 return PlatformAdd<sizeof(P*)>()(addend, dest);
650 }
651 };
652
653 // Most platforms do not support atomic add on a 2-byte value. However,
654 // if the value occupies the most significant 16 bits of an aligned 32-bit
655 // word, then we can do this with an atomic add of (add_value << 16)
656 // to the 32-bit word.
657 //
658 // The least significant parts of this 32-bit word will never be affected, even
659 // in case of overflow/underflow.
660 //
661 // Use the ATOMIC_SHORT_PAIR macro (see macros.hpp) to get the desired alignment.
662 template<>
663 struct Atomic::AddImpl<jshort, jshort> VALUE_OBJ_CLASS_SPEC {
664 jshort operator()(jshort add_value, jshort volatile* dest) const {
665 #ifdef VM_LITTLE_ENDIAN
666 assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
667 jint new_value = Atomic::add(add_value << 16, (volatile jint*)(dest-1));
668 #else
669 assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
670 jint new_value = Atomic::add(add_value << 16, (volatile jint*)(dest));
671 #endif
672 return (jshort)(new_value >> 16); // preserves sign
673 }
674 };
675
676 template<typename Derived>
677 template<typename I, typename D>
678 inline D Atomic::FetchAndAdd<Derived>::operator()(I add_value, D volatile* dest) const {
679 I addend = add_value;
680 // If D is a pointer type P*, scale by sizeof(P).
681 if (IsPointer<D>::value) {
682 addend *= sizeof(typename RemovePointer<D>::type);
683 }
684 D old = static_cast<const Derived*>(this)->fetch_and_add(addend, dest);
685 return old + add_value;
686 }
687
688 template<typename Derived>
689 template<typename I, typename D>
690 inline D Atomic::AddAndFetch<Derived>::operator()(I add_value, D volatile* dest) const {
691 // If D is a pointer type P*, scale by sizeof(P).
692 if (IsPointer<D>::value) {
693 add_value *= sizeof(typename RemovePointer<D>::type);
694 }
695 return static_cast<const Derived*>(this)->add_and_fetch(add_value, dest);
696 }
697
698 template<typename Type, typename Fn, typename I, typename D>
699 inline D Atomic::add_using_helper(Fn fn, I add_value, D volatile* dest) {
700 return PrimitiveConversions::cast<D>(
701 fn(PrimitiveConversions::cast<Type>(add_value),
702 reinterpret_cast<Type volatile*>(dest)));
703 }
704
705 template<typename T, typename D, typename U>
706 inline D Atomic::cmpxchg(T exchange_value,
707 D volatile* dest,
708 U compare_value,
709 cmpxchg_memory_order order) {
710 return CmpxchgImpl<T, D, U>()(exchange_value, dest, compare_value, order);
711 }
712
713 template<typename T, typename D>
714 inline bool Atomic::replace_if_null(T* value, D* volatile* dest,
715 cmpxchg_memory_order order) {
716 // Presently using a trivial implementation in terms of cmpxchg.
717 // Consider adding platform support, to permit the use of compiler
718 // intrinsics like gcc's __sync_bool_compare_and_swap.
719 D* expected_null = NULL;
720 return expected_null == cmpxchg(value, dest, expected_null, order);
721 }
722
723 // Handle cmpxchg for integral and enum types.
724 //
725 // All the involved types must be identical.
726 template<typename T>
727 struct Atomic::CmpxchgImpl<
728 T, T, T,
729 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
730 VALUE_OBJ_CLASS_SPEC
731 {
732 T operator()(T exchange_value, T volatile* dest, T compare_value,
733 cmpxchg_memory_order order) const {
734 // Forward to the platform handler for the size of T.
735 return PlatformCmpxchg<sizeof(T)>()(exchange_value,
736 dest,
737 compare_value,
738 order);
739 }
740 };
741
742 // Handle cmpxchg for pointer types.
743 //
744 // The destination's type and the compare_value type must be the same,
745 // ignoring cv-qualifiers; we don't care about the cv-qualifiers of
746 // the compare_value.
747 //
748 // The exchange_value must be implicitly convertible to the
749 // destination's type; it must be type-correct to store the
750 // exchange_value in the destination.
751 template<typename T, typename D, typename U>
752 struct Atomic::CmpxchgImpl<
753 T*, D*, U*,
754 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value &&
755 IsSame<typename RemoveCV<D>::type,
756 typename RemoveCV<U>::type>::value>::type>
757 VALUE_OBJ_CLASS_SPEC
758 {
759 D* operator()(T* exchange_value, D* volatile* dest, U* compare_value,
760 cmpxchg_memory_order order) const {
761 // Allow derived to base conversion, and adding cv-qualifiers.
762 D* new_value = exchange_value;
763 // Don't care what the CV qualifiers for compare_value are,
764 // but we need to match D* when calling platform support.
765 D* old_value = const_cast<D*>(compare_value);
766 return PlatformCmpxchg<sizeof(D*)>()(new_value, dest, old_value, order);
767 }
768 };
769
770 // Handle cmpxchg for types that have a translator.
771 //
772 // All the involved types must be identical.
773 //
774 // This translates the original call into a call on the decayed
775 // arguments, and returns the recovered result of that translated
776 // call.
777 template<typename T>
778 struct Atomic::CmpxchgImpl<
779 T, T, T,
780 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
781 VALUE_OBJ_CLASS_SPEC
782 {
783 T operator()(T exchange_value, T volatile* dest, T compare_value,
784 cmpxchg_memory_order order) const {
785 typedef PrimitiveConversions::Translate<T> Translator;
786 typedef typename Translator::Decayed Decayed;
787 STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
788 return Translator::recover(
789 cmpxchg(Translator::decay(exchange_value),
790 reinterpret_cast<Decayed volatile*>(dest),
791 Translator::decay(compare_value),
792 order));
793 }
794 };
795
796 template<typename Type, typename Fn, typename T>
797 inline T Atomic::cmpxchg_using_helper(Fn fn,
798 T exchange_value,
799 T volatile* dest,
800 T compare_value) {
801 STATIC_ASSERT(sizeof(Type) == sizeof(T));
802 return PrimitiveConversions::cast<T>(
803 fn(PrimitiveConversions::cast<Type>(exchange_value),
804 reinterpret_cast<Type volatile*>(dest),
805 PrimitiveConversions::cast<Type>(compare_value)));
806 }
807
808 template<typename T>
809 inline T Atomic::CmpxchgByteUsingInt::operator()(T exchange_value,
810 T volatile* dest,
811 T compare_value,
812 cmpxchg_memory_order order) const {
813 STATIC_ASSERT(sizeof(T) == sizeof(uint8_t));
814 uint8_t canon_exchange_value = exchange_value;
815 uint8_t canon_compare_value = compare_value;
816 volatile uint32_t* aligned_dest
817 = reinterpret_cast<volatile uint32_t*>(align_down(dest, sizeof(uint32_t)));
818 size_t offset = pointer_delta(dest, aligned_dest, 1);
819 uint32_t cur = *aligned_dest;
820 uint8_t* cur_as_bytes = reinterpret_cast<uint8_t*>(&cur);
821
822 // current value may not be what we are looking for, so force it
823 // to that value so the initial cmpxchg will fail if it is different
824 cur_as_bytes[offset] = canon_compare_value;
825
826 // always execute a real cmpxchg so that we get the required memory
827 // barriers even on initial failure
828 do {
829 // value to swap in matches current value ...
830 uint32_t new_value = cur;
831 // ... except for the one jbyte we want to update
832 reinterpret_cast<uint8_t*>(&new_value)[offset] = canon_exchange_value;
833
834 uint32_t res = cmpxchg(new_value, aligned_dest, cur, order);
835 if (res == cur) break; // success
836
837 // at least one byte in the int changed value, so update
838 // our view of the current int
839 cur = res;
840 // if our byte is still as cur we loop and try again
841 } while (cur_as_bytes[offset] == canon_compare_value);
842
843 return PrimitiveConversions::cast<T>(cur_as_bytes[offset]);
844 }
845
846 // Handle xchg for integral and enum types.
847 //
848 // All the involved types must be identical.
849 template<typename T>
850 struct Atomic::XchgImpl<
851 T, T,
852 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
853 VALUE_OBJ_CLASS_SPEC
854 {
855 T operator()(T exchange_value, T volatile* dest) const {
856 // Forward to the platform handler for the size of T.
857 return PlatformXchg<sizeof(T)>()(exchange_value, dest);
858 }
859 };
860
861 // Handle xchg for pointer types.
862 //
863 // The exchange_value must be implicitly convertible to the
864 // destination's type; it must be type-correct to store the
865 // exchange_value in the destination.
866 template<typename T, typename D>
867 struct Atomic::XchgImpl<
868 T*, D*,
869 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type>
870 VALUE_OBJ_CLASS_SPEC
871 {
872 D* operator()(T* exchange_value, D* volatile* dest) const {
873 // Allow derived to base conversion, and adding cv-qualifiers.
874 D* new_value = exchange_value;
875 return PlatformXchg<sizeof(D*)>()(new_value, dest);
876 }
877 };
878
879 // Handle xchg for types that have a translator.
880 //
881 // All the involved types must be identical.
882 //
883 // This translates the original call into a call on the decayed
884 // arguments, and returns the recovered result of that translated
885 // call.
886 template<typename T>
887 struct Atomic::XchgImpl<
888 T, T,
889 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
890 VALUE_OBJ_CLASS_SPEC
891 {
892 T operator()(T exchange_value, T volatile* dest) const {
893 typedef PrimitiveConversions::Translate<T> Translator;
894 typedef typename Translator::Decayed Decayed;
895 STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
896 return Translator::recover(
897 xchg(Translator::decay(exchange_value),
898 reinterpret_cast<Decayed volatile*>(dest)));
899 }
900 };
901
902 template<typename Type, typename Fn, typename T>
903 inline T Atomic::xchg_using_helper(Fn fn,
904 T exchange_value,
905 T volatile* dest) {
906 STATIC_ASSERT(sizeof(Type) == sizeof(T));
907 return PrimitiveConversions::cast<T>(
908 fn(PrimitiveConversions::cast<Type>(exchange_value),
909 reinterpret_cast<Type volatile*>(dest)));
910 }
911
912 template<typename T, typename D>
913 inline D Atomic::xchg(T exchange_value, volatile D* dest) {
914 return XchgImpl<T, D>()(exchange_value, dest);
915 }
916
917 #endif // SHARE_VM_RUNTIME_ATOMIC_HPP