1 /*
2 * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_RUNTIME_ATOMIC_HPP
26 #define SHARE_RUNTIME_ATOMIC_HPP
27
28 #include "memory/allocation.hpp"
29 #include "metaprogramming/conditional.hpp"
30 #include "metaprogramming/enableIf.hpp"
31 #include "metaprogramming/isIntegral.hpp"
32 #include "metaprogramming/isPointer.hpp"
33 #include "metaprogramming/isSame.hpp"
34 #include "metaprogramming/primitiveConversions.hpp"
35 #include "metaprogramming/removeCV.hpp"
36 #include "metaprogramming/removePointer.hpp"
37 #include "runtime/orderAccess.hpp"
38 #include "utilities/align.hpp"
39 #include "utilities/macros.hpp"
40
41 enum atomic_memory_order {
42 // The modes that align with C++11 are intended to
43 // follow the same semantics.
44 memory_order_relaxed = 0,
45 memory_order_acquire = 2,
46 memory_order_release = 3,
47 memory_order_acq_rel = 4,
48 // Strong two-way memory barrier.
49 memory_order_conservative = 8
50 };
51
52 enum ScopedFenceType {
53 X_ACQUIRE
54 , RELEASE_X
55 , RELEASE_X_FENCE
56 };
57
58 class Atomic : AllStatic {
59 public:
60 // Atomic operations on int64 types are not available on all 32-bit
61 // platforms. If atomic ops on int64 are defined here they must only
62 // be used from code that verifies they are available at runtime and
63 // can provide an alternative action if not - see supports_cx8() for
64 // a means to test availability.
65
66 // The memory operations that are mentioned with each of the atomic
67 // function families come from src/share/vm/runtime/orderAccess.hpp,
68 // e.g., <fence> is described in that file and is implemented by the
69 // OrderAccess::fence() function. See that file for the gory details
70 // on the Memory Access Ordering Model.
71
72 // All of the atomic operations that imply a read-modify-write action
73 // guarantee a two-way memory barrier across that operation. Historically
74 // these semantics reflect the strength of atomic operations that are
75 // provided on SPARC/X86. We assume that strength is necessary unless
76 // we can prove that a weaker form is sufficiently safe.
77
78 // Atomically store to a location
79 // The type T must be either a pointer type convertible to or equal
80 // to D, an integral/enum type equal to D, or a type equal to D that
81 // is primitive convertible using PrimitiveConversions.
82 template<typename D, typename T>
83 inline static void store(volatile D* dest, T store_value);
84
85 template <typename D, typename T>
86 inline static void release_store(volatile D* dest, T store_value);
87
88 template <typename D, typename T>
89 inline static void release_store_fence(volatile D* dest, T store_value);
90
91 // Atomically load from a location
92 // The type T must be either a pointer type, an integral/enum type,
93 // or a type that is primitive convertible using PrimitiveConversions.
94 template<typename T>
95 inline static T load(const volatile T* dest);
96
97 template <typename T>
98 inline static T load_acquire(const volatile T* dest);
99
100 // Atomically add to a location. Returns updated value. add*() provide:
101 // <fence> add-value-to-dest <membar StoreLoad|StoreStore>
102
103 template<typename D, typename I>
104 inline static D add(D volatile* dest, I add_value,
105 atomic_memory_order order = memory_order_conservative);
106
107 template<typename D, typename I>
108 inline static D fetch_and_add(D volatile* dest, I add_value,
109 atomic_memory_order order = memory_order_conservative);
110
111 template<typename D, typename I>
112 inline static D sub(D volatile* dest, I sub_value,
113 atomic_memory_order order = memory_order_conservative);
114
115 // Atomically increment location. inc() provide:
116 // <fence> increment-dest <membar StoreLoad|StoreStore>
117 // The type D may be either a pointer type, or an integral
118 // type. If it is a pointer type, then the increment is
119 // scaled to the size of the type pointed to by the pointer.
120 template<typename D>
121 inline static void inc(D volatile* dest,
122 atomic_memory_order order = memory_order_conservative);
123
124 // Atomically decrement a location. dec() provide:
125 // <fence> decrement-dest <membar StoreLoad|StoreStore>
126 // The type D may be either a pointer type, or an integral
127 // type. If it is a pointer type, then the decrement is
128 // scaled to the size of the type pointed to by the pointer.
129 template<typename D>
130 inline static void dec(D volatile* dest,
131 atomic_memory_order order = memory_order_conservative);
132
133 // Performs atomic exchange of *dest with exchange_value. Returns old
134 // prior value of *dest. xchg*() provide:
135 // <fence> exchange-value-with-dest <membar StoreLoad|StoreStore>
136 // The type T must be either a pointer type convertible to or equal
137 // to D, an integral/enum type equal to D, or a type equal to D that
138 // is primitive convertible using PrimitiveConversions.
139 template<typename D, typename T>
140 inline static D xchg(volatile D* dest, T exchange_value,
141 atomic_memory_order order = memory_order_conservative);
142
143 // Performs atomic compare of *dest and compare_value, and exchanges
144 // *dest with exchange_value if the comparison succeeded. Returns prior
145 // value of *dest. cmpxchg*() provide:
146 // <fence> compare-and-exchange <membar StoreLoad|StoreStore>
147
148 template<typename D, typename U, typename T>
149 inline static D cmpxchg(D volatile* dest,
150 U compare_value,
151 T exchange_value,
152 atomic_memory_order order = memory_order_conservative);
153
154 // Performs atomic compare of *dest and NULL, and replaces *dest
155 // with exchange_value if the comparison succeeded. Returns true if
156 // the comparison succeeded and the exchange occurred. This is
157 // often used as part of lazy initialization, as a lock-free
158 // alternative to the Double-Checked Locking Pattern.
159 template<typename D, typename T>
160 inline static bool replace_if_null(D* volatile* dest, T* value,
161 atomic_memory_order order = memory_order_conservative);
162
163 private:
164 WINDOWS_ONLY(public:) // VS2017 warns (C2027) use of undefined type if IsPointerConvertible is declared private
165 // Test whether From is implicitly convertible to To.
166 // From and To must be pointer types.
167 // Note: Provides the limited subset of C++11 std::is_convertible
168 // that is needed here.
169 template<typename From, typename To> struct IsPointerConvertible;
170
171 protected:
172 // Dispatch handler for store. Provides type-based validity
173 // checking and limited conversions around calls to the platform-
174 // specific implementation layer provided by PlatformOp.
175 template<typename D, typename T, typename PlatformOp, typename Enable = void>
176 struct StoreImpl;
177
178 // Platform-specific implementation of store. Support for sizes
179 // of 1, 2, 4, and (if different) pointer size bytes are required.
180 // The class is a function object that must be default constructable,
181 // with these requirements:
182 //
183 // either:
184 // - dest is of type D*, an integral, enum or pointer type.
185 // - new_value are of type T, an integral, enum or pointer type D or
186 // pointer type convertible to D.
187 // or:
188 // - T and D are the same and are primitive convertible using PrimitiveConversions
189 // and either way:
190 // - platform_store is an object of type PlatformStore<sizeof(T)>.
191 //
192 // Then
193 // platform_store(new_value, dest)
194 // must be a valid expression.
195 //
196 // The default implementation is a volatile store. If a platform
197 // requires more for e.g. 64 bit stores, a specialization is required
198 template<size_t byte_size> struct PlatformStore;
199
200 // Dispatch handler for load. Provides type-based validity
201 // checking and limited conversions around calls to the platform-
202 // specific implementation layer provided by PlatformOp.
203 template<typename T, typename PlatformOp, typename Enable = void>
204 struct LoadImpl;
205
206 // Platform-specific implementation of load. Support for sizes of
207 // 1, 2, 4 bytes and (if different) pointer size bytes are required.
208 // The class is a function object that must be default
209 // constructable, with these requirements:
210 //
211 // - dest is of type T*, an integral, enum or pointer type, or
212 // T is convertible to a primitive type using PrimitiveConversions
213 // - platform_load is an object of type PlatformLoad<sizeof(T)>.
214 //
215 // Then
216 // platform_load(src)
217 // must be a valid expression, returning a result convertible to T.
218 //
219 // The default implementation is a volatile load. If a platform
220 // requires more for e.g. 64 bit loads, a specialization is required
221 template<size_t byte_size> struct PlatformLoad;
222
223 // Give platforms a variation point to specialize.
224 template<size_t byte_size, ScopedFenceType type> struct PlatformOrderedStore;
225 template<size_t byte_size, ScopedFenceType type> struct PlatformOrderedLoad;
226
227 private:
228 // Dispatch handler for add and fetch_and add. Provides type-based
229 // validity checking and limited conversions around calls to the
230 // platform-specific implementation layer provided by
231 // PlatformAddAndFetch and PlatformFetchAndAdd.
232 template<typename D, typename I, typename Enable = void>
233 struct AddImpl;
234
235 // Platform-specific implementation of add. Support for sizes of 4
236 // bytes and (if different) pointer size bytes are required. The
237 // class is a function object that must be default constructable,
238 // with these requirements:
239 //
240 // - dest is of type D*, an integral or pointer type.
241 // - add_value is of type I, an integral type.
242 // - sizeof(I) == sizeof(D).
243 // - if D is an integral type, I == D.
244 // - platform_add is an object of type PlatformAddAndFetch<sizeof(D)>.
245 //
246 // Then
247 // platform_add(dest, add_value)
248 // must be a valid expression, returning a result convertible to D.
249 //
250 // No definition is provided; all platforms must explicitly define
251 // this class and any needed specializations.
252 template<size_t byte_size> struct PlatformAddAndFetch;
253
254 // Platfom-specific implementation of fetch_and_add.
255 // See comment for PlatformAddAndFetch for further specification.
256 template<size_t byte_size> struct PlatformFetchAndAdd;
257
258 // Support for platforms that implement some variants of add using a
259 // (typically out of line) non-template helper function. The
260 // generic arguments passed to PlatformAdd need to be translated to
261 // the appropriate type for the helper function, the helper function
262 // invoked on the translated arguments, and the result translated
263 // back. Type is the parameter / return type of the helper
264 // function. No scaling of add_value is performed when D is a pointer
265 // type, so this function can be used to implement the support function
266 // required by AddAndFetch.
267 template<typename Type, typename Fn, typename D, typename I>
268 static D add_using_helper(Fn fn, D volatile* dest, I add_value);
269
270 // Dispatch handler for cmpxchg. Provides type-based validity
271 // checking and limited conversions around calls to the
272 // platform-specific implementation layer provided by
273 // PlatformCmpxchg.
274 template<typename D, typename U, typename T, typename Enable = void>
275 struct CmpxchgImpl;
276
277 // Platform-specific implementation of cmpxchg. Support for sizes
278 // of 1, 4, and 8 are required. The class is a function object that
279 // must be default constructable, with these requirements:
280 //
281 // - dest is of type T*.
282 // - exchange_value and compare_value are of type T.
283 // - order is of type atomic_memory_order.
284 // - platform_cmpxchg is an object of type PlatformCmpxchg<sizeof(T)>.
285 //
286 // Then
287 // platform_cmpxchg(dest, compare_value, exchange_value, order)
288 // must be a valid expression, returning a result convertible to T.
289 //
290 // A default definition is provided, which declares a function template
291 // T operator()(T volatile*, T, T, atomic_memory_order) const
292 //
293 // For each required size, a platform must either provide an
294 // appropriate definition of that function, or must entirely
295 // specialize the class template for that size.
296 template<size_t byte_size> struct PlatformCmpxchg;
297
298 // Support for platforms that implement some variants of cmpxchg
299 // using a (typically out of line) non-template helper function.
300 // The generic arguments passed to PlatformCmpxchg need to be
301 // translated to the appropriate type for the helper function, the
302 // helper invoked on the translated arguments, and the result
303 // translated back. Type is the parameter / return type of the
304 // helper function.
305 template<typename Type, typename Fn, typename T>
306 static T cmpxchg_using_helper(Fn fn,
307 T volatile* dest,
308 T compare_value,
309 T exchange_value);
310
311 // Support platforms that do not provide Read-Modify-Write
312 // byte-level atomic access. To use, derive PlatformCmpxchg<1> from
313 // this class.
314 public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11.
315 struct CmpxchgByteUsingInt;
316 private:
317
318 // Dispatch handler for xchg. Provides type-based validity
319 // checking and limited conversions around calls to the
320 // platform-specific implementation layer provided by
321 // PlatformXchg.
322 template<typename D, typename T, typename Enable = void>
323 struct XchgImpl;
324
325 // Platform-specific implementation of xchg. Support for sizes
326 // of 4, and sizeof(intptr_t) are required. The class is a function
327 // object that must be default constructable, with these requirements:
328 //
329 // - dest is of type T*.
330 // - exchange_value is of type T.
331 // - platform_xchg is an object of type PlatformXchg<sizeof(T)>.
332 //
333 // Then
334 // platform_xchg(dest, exchange_value)
335 // must be a valid expression, returning a result convertible to T.
336 //
337 // A default definition is provided, which declares a function template
338 // T operator()(T volatile*, T, atomic_memory_order) const
339 //
340 // For each required size, a platform must either provide an
341 // appropriate definition of that function, or must entirely
342 // specialize the class template for that size.
343 template<size_t byte_size> struct PlatformXchg;
344
345 // Support for platforms that implement some variants of xchg
346 // using a (typically out of line) non-template helper function.
347 // The generic arguments passed to PlatformXchg need to be
348 // translated to the appropriate type for the helper function, the
349 // helper invoked on the translated arguments, and the result
350 // translated back. Type is the parameter / return type of the
351 // helper function.
352 template<typename Type, typename Fn, typename T>
353 static T xchg_using_helper(Fn fn,
354 T volatile* dest,
355 T exchange_value);
356 };
357
358 template<typename From, typename To>
359 struct Atomic::IsPointerConvertible<From*, To*> : AllStatic {
360 // Determine whether From* is implicitly convertible to To*, using
361 // the "sizeof trick".
362 typedef char yes;
363 typedef char (&no)[2];
364
365 static yes test(To*);
366 static no test(...);
367 static From* test_value;
368
369 static const bool value = (sizeof(yes) == sizeof(test(test_value)));
370 };
371
372 // Handle load for pointer, integral and enum types.
373 template<typename T, typename PlatformOp>
374 struct Atomic::LoadImpl<
375 T,
376 PlatformOp,
377 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value || IsPointer<T>::value>::type>
378 {
379 T operator()(T const volatile* dest) const {
380 // Forward to the platform handler for the size of T.
381 return PlatformOp()(dest);
382 }
383 };
384
385 // Handle load for types that have a translator.
386 //
387 // All the involved types must be identical.
388 //
389 // This translates the original call into a call on the decayed
390 // arguments, and returns the recovered result of that translated
391 // call.
392 template<typename T, typename PlatformOp>
393 struct Atomic::LoadImpl<
394 T,
395 PlatformOp,
396 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
397 {
398 T operator()(T const volatile* dest) const {
399 typedef PrimitiveConversions::Translate<T> Translator;
400 typedef typename Translator::Decayed Decayed;
401 STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
402 Decayed result = PlatformOp()(reinterpret_cast<Decayed const volatile*>(dest));
403 return Translator::recover(result);
404 }
405 };
406
407 // Default implementation of atomic load if a specific platform
408 // does not provide a specialization for a certain size class.
409 // For increased safety, the default implementation only allows
410 // load types that are pointer sized or smaller. If a platform still
411 // supports wide atomics, then it has to use specialization
412 // of Atomic::PlatformLoad for that wider size class.
413 template<size_t byte_size>
414 struct Atomic::PlatformLoad {
415 template<typename T>
416 T operator()(T const volatile* dest) const {
417 STATIC_ASSERT(sizeof(T) <= sizeof(void*)); // wide atomics need specialization
418 return *dest;
419 }
420 };
421
422 // Handle store for integral and enum types.
423 //
424 // All the involved types must be identical.
425 template<typename T, typename PlatformOp>
426 struct Atomic::StoreImpl<
427 T, T,
428 PlatformOp,
429 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
430 {
431 void operator()(T volatile* dest, T new_value) const {
432 // Forward to the platform handler for the size of T.
433 PlatformOp()(dest, new_value);
434 }
435 };
436
437 // Handle store for pointer types.
438 //
439 // The new_value must be implicitly convertible to the
440 // destination's type; it must be type-correct to store the
441 // new_value in the destination.
442 template<typename D, typename T, typename PlatformOp>
443 struct Atomic::StoreImpl<
444 D*, T*,
445 PlatformOp,
446 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type>
447 {
448 void operator()(D* volatile* dest, T* new_value) const {
449 // Allow derived to base conversion, and adding cv-qualifiers.
450 D* value = new_value;
451 PlatformOp()(dest, value);
452 }
453 };
454
455 // Handle store for types that have a translator.
456 //
457 // All the involved types must be identical.
458 //
459 // This translates the original call into a call on the decayed
460 // arguments.
461 template<typename T, typename PlatformOp>
462 struct Atomic::StoreImpl<
463 T, T,
464 PlatformOp,
465 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
466 {
467 void operator()(T volatile* dest, T new_value) const {
468 typedef PrimitiveConversions::Translate<T> Translator;
469 typedef typename Translator::Decayed Decayed;
470 STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
471 PlatformOp()(reinterpret_cast<Decayed volatile*>(dest),
472 Translator::decay(new_value));
473 }
474 };
475
476 // Default implementation of atomic store if a specific platform
477 // does not provide a specialization for a certain size class.
478 // For increased safety, the default implementation only allows
479 // storing types that are pointer sized or smaller. If a platform still
480 // supports wide atomics, then it has to use specialization
481 // of Atomic::PlatformStore for that wider size class.
482 template<size_t byte_size>
483 struct Atomic::PlatformStore {
484 template<typename T>
485 void operator()(T volatile* dest,
486 T new_value) const {
487 STATIC_ASSERT(sizeof(T) <= sizeof(void*)); // wide atomics need specialization
488 (void)const_cast<T&>(*dest = new_value);
489 }
490 };
491
492 template<typename D>
493 inline void Atomic::inc(D volatile* dest, atomic_memory_order order) {
494 STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
495 typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I;
496 Atomic::add(dest, I(1), order);
497 }
498
499 template<typename D>
500 inline void Atomic::dec(D volatile* dest, atomic_memory_order order) {
501 STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
502 typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I;
503 // Assumes two's complement integer representation.
504 #pragma warning(suppress: 4146)
505 Atomic::add(dest, I(-1), order);
506 }
507
508 template<typename D, typename I>
509 inline D Atomic::sub(D volatile* dest, I sub_value, atomic_memory_order order) {
510 STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
511 STATIC_ASSERT(IsIntegral<I>::value);
512 // If D is a pointer type, use [u]intptr_t as the addend type,
513 // matching signedness of I. Otherwise, use D as the addend type.
514 typedef typename Conditional<IsSigned<I>::value, intptr_t, uintptr_t>::type PI;
515 typedef typename Conditional<IsPointer<D>::value, PI, D>::type AddendType;
516 // Only allow conversions that can't change the value.
517 STATIC_ASSERT(IsSigned<I>::value == IsSigned<AddendType>::value);
518 STATIC_ASSERT(sizeof(I) <= sizeof(AddendType));
519 AddendType addend = sub_value;
520 // Assumes two's complement integer representation.
521 #pragma warning(suppress: 4146) // In case AddendType is not signed.
522 return Atomic::add(dest, -addend, order);
523 }
524
525 // Define the class before including platform file, which may specialize
526 // the operator definition. No generic definition of specializations
527 // of the operator template are provided, nor are there any generic
528 // specializations of the class. The platform file is responsible for
529 // providing those.
530 template<size_t byte_size>
531 struct Atomic::PlatformCmpxchg {
532 template<typename T>
533 T operator()(T volatile* dest,
534 T compare_value,
535 T exchange_value,
536 atomic_memory_order order) const;
537 };
538
539 // Define the class before including platform file, which may use this
540 // as a base class, requiring it be complete. The definition is later
541 // in this file, near the other definitions related to cmpxchg.
542 struct Atomic::CmpxchgByteUsingInt {
543 template<typename T>
544 T operator()(T volatile* dest,
545 T compare_value,
546 T exchange_value,
547 atomic_memory_order order) const;
548 };
549
550 // Define the class before including platform file, which may specialize
551 // the operator definition. No generic definition of specializations
552 // of the operator template are provided, nor are there any generic
553 // specializations of the class. The platform file is responsible for
554 // providing those.
555 template<size_t byte_size>
556 struct Atomic::PlatformXchg {
557 template<typename T>
558 T operator()(T volatile* dest,
559 T exchange_value,
560 atomic_memory_order order) const;
561 };
562
563 template <ScopedFenceType T>
564 class ScopedFenceGeneral: public StackObj {
565 public:
566 void prefix() {}
567 void postfix() {}
568 };
569
570 // The following methods can be specialized using simple template specialization
571 // in the platform specific files for optimization purposes. Otherwise the
572 // generalized variant is used.
573
574 template<> inline void ScopedFenceGeneral<X_ACQUIRE>::postfix() { OrderAccess::acquire(); }
575 template<> inline void ScopedFenceGeneral<RELEASE_X>::prefix() { OrderAccess::release(); }
576 template<> inline void ScopedFenceGeneral<RELEASE_X_FENCE>::prefix() { OrderAccess::release(); }
577 template<> inline void ScopedFenceGeneral<RELEASE_X_FENCE>::postfix() { OrderAccess::fence(); }
578
579 template <ScopedFenceType T>
580 class ScopedFence : public ScopedFenceGeneral<T> {
581 void *const _field;
582 public:
583 ScopedFence(void *const field) : _field(field) { prefix(); }
584 ~ScopedFence() { postfix(); }
585 void prefix() { ScopedFenceGeneral<T>::prefix(); }
586 void postfix() { ScopedFenceGeneral<T>::postfix(); }
587 };
588
589 // platform specific in-line definitions - must come before shared definitions
590
591 #include OS_CPU_HEADER(atomic)
592
593 // shared in-line definitions
594
595 // size_t casts...
596 #if (SIZE_MAX != UINTPTR_MAX)
597 #error size_t is not WORD_SIZE, interesting platform, but missing implementation here
598 #endif
599
600 template<typename T>
601 inline T Atomic::load(const volatile T* dest) {
602 return LoadImpl<T, PlatformLoad<sizeof(T)> >()(dest);
603 }
604
605 template<size_t byte_size, ScopedFenceType type>
606 struct Atomic::PlatformOrderedLoad {
607 template <typename T>
608 T operator()(const volatile T* p) const {
609 ScopedFence<type> f((void*)p);
610 return Atomic::load(p);
611 }
612 };
613
614 template <typename T>
615 inline T Atomic::load_acquire(const volatile T* p) {
616 return LoadImpl<T, PlatformOrderedLoad<sizeof(T), X_ACQUIRE> >()(p);
617 }
618
619 template<typename D, typename T>
620 inline void Atomic::store(volatile D* dest, T store_value) {
621 StoreImpl<D, T, PlatformStore<sizeof(D)> >()(dest, store_value);
622 }
623
624 template<size_t byte_size, ScopedFenceType type>
625 struct Atomic::PlatformOrderedStore {
626 template <typename T>
627 void operator()(volatile T* p, T v) const {
628 ScopedFence<type> f((void*)p);
629 Atomic::store(p, v);
630 }
631 };
632
633 template <typename D, typename T>
634 inline void Atomic::release_store(volatile D* p, T v) {
635 StoreImpl<D, T, PlatformOrderedStore<sizeof(D), RELEASE_X> >()(p, v);
636 }
637
638 template <typename D, typename T>
639 inline void Atomic::release_store_fence(volatile D* p, T v) {
640 StoreImpl<D, T, PlatformOrderedStore<sizeof(D), RELEASE_X_FENCE> >()(p, v);
641 }
642
643 template<typename D, typename I>
644 inline D Atomic::add(D volatile* dest, I add_value,
645 atomic_memory_order order) {
646 return AddImpl<D, I>::add_and_fetch(dest, add_value, order);
647 }
648
649 template<typename D, typename I>
650 inline D Atomic::fetch_and_add(D volatile* dest, I add_value,
651 atomic_memory_order order) {
652 return AddImpl<D, I>::fetch_and_add(dest, add_value, order);
653 }
654
655 template<typename D, typename I>
656 struct Atomic::AddImpl<
657 D, I,
658 typename EnableIf<IsIntegral<I>::value &&
659 IsIntegral<D>::value &&
660 (sizeof(I) <= sizeof(D)) &&
661 (IsSigned<I>::value == IsSigned<D>::value)>::type>
662 {
663 static D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) {
664 D addend = add_value;
665 return PlatformAddAndFetch<sizeof(D)>()(dest, addend, order);
666 }
667 static D fetch_and_add(D volatile* dest, I add_value, atomic_memory_order order) {
668 D addend = add_value;
669 return PlatformFetchAndAdd<sizeof(D)>()(dest, addend, order);
670 }
671 };
672
673 template<typename P, typename I>
674 struct Atomic::AddImpl<
675 P*, I,
676 typename EnableIf<IsIntegral<I>::value && (sizeof(I) <= sizeof(P*))>::type>
677 {
678 STATIC_ASSERT(sizeof(intptr_t) == sizeof(P*));
679 STATIC_ASSERT(sizeof(uintptr_t) == sizeof(P*));
680 typedef typename Conditional<IsSigned<I>::value,
681 intptr_t,
682 uintptr_t>::type CI;
683
684 static I scale_addend(I add_value) {
685 return add_value * sizeof(P);
686 }
687
688 static P* add_and_fetch(P* volatile* dest, I add_value, atomic_memory_order order) {
689 CI addend = add_value;
690 return PlatformAddAndFetch<sizeof(P*)>()(dest, scale_addend(addend), order);
691 }
692 static P* fetch_and_add(P* volatile* dest, I add_value, atomic_memory_order order) {
693 CI addend = add_value;
694 return PlatformFetchAndAdd<sizeof(P*)>()(dest, scale_addend(addend), order);
695 }
696 };
697
698 template<typename Type, typename Fn, typename D, typename I>
699 inline D Atomic::add_using_helper(Fn fn, D volatile* dest, I add_value) {
700 return PrimitiveConversions::cast<D>(
701 fn(PrimitiveConversions::cast<Type>(add_value),
702 reinterpret_cast<Type volatile*>(dest)));
703 }
704
705 template<typename D, typename U, typename T>
706 inline D Atomic::cmpxchg(D volatile* dest,
707 U compare_value,
708 T exchange_value,
709 atomic_memory_order order) {
710 return CmpxchgImpl<D, U, T>()(dest, compare_value, exchange_value, order);
711 }
712
713 template<typename D, typename T>
714 inline bool Atomic::replace_if_null(D* volatile* dest, T* value,
715 atomic_memory_order order) {
716 // Presently using a trivial implementation in terms of cmpxchg.
717 // Consider adding platform support, to permit the use of compiler
718 // intrinsics like gcc's __sync_bool_compare_and_swap.
719 D* expected_null = NULL;
720 return expected_null == cmpxchg(dest, expected_null, value, order);
721 }
722
723 // Handle cmpxchg for integral and enum types.
724 //
725 // All the involved types must be identical.
726 template<typename T>
727 struct Atomic::CmpxchgImpl<
728 T, T, T,
729 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
730 {
731 T operator()(T volatile* dest, T compare_value, T exchange_value,
732 atomic_memory_order order) const {
733 // Forward to the platform handler for the size of T.
734 return PlatformCmpxchg<sizeof(T)>()(dest,
735 compare_value,
736 exchange_value,
737 order);
738 }
739 };
740
741 // Handle cmpxchg for pointer types.
742 //
743 // The destination's type and the compare_value type must be the same,
744 // ignoring cv-qualifiers; we don't care about the cv-qualifiers of
745 // the compare_value.
746 //
747 // The exchange_value must be implicitly convertible to the
748 // destination's type; it must be type-correct to store the
749 // exchange_value in the destination.
750 template<typename D, typename U, typename T>
751 struct Atomic::CmpxchgImpl<
752 D*, U*, T*,
753 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value &&
754 IsSame<typename RemoveCV<D>::type,
755 typename RemoveCV<U>::type>::value>::type>
756 {
757 D* operator()(D* volatile* dest, U* compare_value, T* exchange_value,
758 atomic_memory_order order) const {
759 // Allow derived to base conversion, and adding cv-qualifiers.
760 D* new_value = exchange_value;
761 // Don't care what the CV qualifiers for compare_value are,
762 // but we need to match D* when calling platform support.
763 D* old_value = const_cast<D*>(compare_value);
764 return PlatformCmpxchg<sizeof(D*)>()(dest, old_value, new_value, order);
765 }
766 };
767
768 // Handle cmpxchg for types that have a translator.
769 //
770 // All the involved types must be identical.
771 //
772 // This translates the original call into a call on the decayed
773 // arguments, and returns the recovered result of that translated
774 // call.
775 template<typename T>
776 struct Atomic::CmpxchgImpl<
777 T, T, T,
778 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
779 {
780 T operator()(T volatile* dest, T compare_value, T exchange_value,
781 atomic_memory_order order) const {
782 typedef PrimitiveConversions::Translate<T> Translator;
783 typedef typename Translator::Decayed Decayed;
784 STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
785 return Translator::recover(
786 cmpxchg(reinterpret_cast<Decayed volatile*>(dest),
787 Translator::decay(compare_value),
788 Translator::decay(exchange_value),
789 order));
790 }
791 };
792
793 template<typename Type, typename Fn, typename T>
794 inline T Atomic::cmpxchg_using_helper(Fn fn,
795 T volatile* dest,
796 T compare_value,
797 T exchange_value) {
798 STATIC_ASSERT(sizeof(Type) == sizeof(T));
799 return PrimitiveConversions::cast<T>(
800 fn(PrimitiveConversions::cast<Type>(exchange_value),
801 reinterpret_cast<Type volatile*>(dest),
802 PrimitiveConversions::cast<Type>(compare_value)));
803 }
804
805 template<typename T>
806 inline T Atomic::CmpxchgByteUsingInt::operator()(T volatile* dest,
807 T compare_value,
808 T exchange_value,
809 atomic_memory_order order) const {
810 STATIC_ASSERT(sizeof(T) == sizeof(uint8_t));
811 uint8_t canon_exchange_value = exchange_value;
812 uint8_t canon_compare_value = compare_value;
813 volatile uint32_t* aligned_dest
814 = reinterpret_cast<volatile uint32_t*>(align_down(dest, sizeof(uint32_t)));
815 size_t offset = pointer_delta(dest, aligned_dest, 1);
816 uint32_t cur = *aligned_dest;
817 uint8_t* cur_as_bytes = reinterpret_cast<uint8_t*>(&cur);
818
819 // current value may not be what we are looking for, so force it
820 // to that value so the initial cmpxchg will fail if it is different
821 cur_as_bytes[offset] = canon_compare_value;
822
823 // always execute a real cmpxchg so that we get the required memory
824 // barriers even on initial failure
825 do {
826 // value to swap in matches current value ...
827 uint32_t new_value = cur;
828 // ... except for the one byte we want to update
829 reinterpret_cast<uint8_t*>(&new_value)[offset] = canon_exchange_value;
830
831 uint32_t res = cmpxchg(aligned_dest, cur, new_value, order);
832 if (res == cur) break; // success
833
834 // at least one byte in the int changed value, so update
835 // our view of the current int
836 cur = res;
837 // if our byte is still as cur we loop and try again
838 } while (cur_as_bytes[offset] == canon_compare_value);
839
840 return PrimitiveConversions::cast<T>(cur_as_bytes[offset]);
841 }
842
843 // Handle xchg for integral and enum types.
844 //
845 // All the involved types must be identical.
846 template<typename T>
847 struct Atomic::XchgImpl<
848 T, T,
849 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
850 {
851 T operator()(T volatile* dest, T exchange_value, atomic_memory_order order) const {
852 // Forward to the platform handler for the size of T.
853 return PlatformXchg<sizeof(T)>()(dest, exchange_value, order);
854 }
855 };
856
857 // Handle xchg for pointer types.
858 //
859 // The exchange_value must be implicitly convertible to the
860 // destination's type; it must be type-correct to store the
861 // exchange_value in the destination.
862 template<typename D, typename T>
863 struct Atomic::XchgImpl<
864 D*, T*,
865 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type>
866 {
867 D* operator()(D* volatile* dest, T* exchange_value, atomic_memory_order order) const {
868 // Allow derived to base conversion, and adding cv-qualifiers.
869 D* new_value = exchange_value;
870 return PlatformXchg<sizeof(D*)>()(dest, new_value, order);
871 }
872 };
873
874 // Handle xchg for types that have a translator.
875 //
876 // All the involved types must be identical.
877 //
878 // This translates the original call into a call on the decayed
879 // arguments, and returns the recovered result of that translated
880 // call.
881 template<typename T>
882 struct Atomic::XchgImpl<
883 T, T,
884 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
885 {
886 T operator()(T volatile* dest, T exchange_value, atomic_memory_order order) const {
887 typedef PrimitiveConversions::Translate<T> Translator;
888 typedef typename Translator::Decayed Decayed;
889 STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
890 return Translator::recover(
891 xchg(reinterpret_cast<Decayed volatile*>(dest),
892 Translator::decay(exchange_value),
893 order));
894 }
895 };
896
897 template<typename Type, typename Fn, typename T>
898 inline T Atomic::xchg_using_helper(Fn fn,
899 T volatile* dest,
900 T exchange_value) {
901 STATIC_ASSERT(sizeof(Type) == sizeof(T));
902 // Notice the swapped order of arguments. Change when/if stubs are rewritten.
903 return PrimitiveConversions::cast<T>(
904 fn(PrimitiveConversions::cast<Type>(exchange_value),
905 reinterpret_cast<Type volatile*>(dest)));
906 }
907
908 template<typename D, typename T>
909 inline D Atomic::xchg(volatile D* dest, T exchange_value, atomic_memory_order order) {
910 return XchgImpl<D, T>()(dest, exchange_value, order);
911 }
912
913 #endif // SHARE_RUNTIME_ATOMIC_HPP