< prev index next >
src/hotspot/share/runtime/atomic.hpp
Print this page
rev 49986 : 8202080: Introduce ordering semantics for Atomic::add and other RMW atomics
Reviewed-by: lucy, rehn, dholmes
@@ -35,13 +35,18 @@
#include "metaprogramming/removeCV.hpp"
#include "metaprogramming/removePointer.hpp"
#include "utilities/align.hpp"
#include "utilities/macros.hpp"
-enum cmpxchg_memory_order {
- memory_order_relaxed,
- // Use value which doesn't interfere with C++2011. We need to be more conservative.
+enum atomic_memory_order {
+ // The modes that align with C++11 are intended to
+ // follow the same semantics.
+ memory_order_relaxed = 0,
+ memory_order_acquire = 2,
+ memory_order_release = 3,
+ memory_order_acq_rel = 4,
+ // Strong two-way memory barrier.
memory_order_conservative = 8
};
class Atomic : AllStatic {
public:
@@ -78,59 +83,64 @@
// Atomically add to a location. Returns updated value. add*() provide:
// <fence> add-value-to-dest <membar StoreLoad|StoreStore>
template<typename I, typename D>
- inline static D add(I add_value, D volatile* dest);
+ inline static D add(I add_value, D volatile* dest,
+ atomic_memory_order order = memory_order_conservative);
template<typename I, typename D>
- inline static D sub(I sub_value, D volatile* dest);
+ inline static D sub(I sub_value, D volatile* dest,
+ atomic_memory_order order = memory_order_conservative);
// Atomically increment location. inc() provide:
// <fence> increment-dest <membar StoreLoad|StoreStore>
// The type D may be either a pointer type, or an integral
// type. If it is a pointer type, then the increment is
// scaled to the size of the type pointed to by the pointer.
template<typename D>
- inline static void inc(D volatile* dest);
+ inline static void inc(D volatile* dest,
+ atomic_memory_order order = memory_order_conservative);
// Atomically decrement a location. dec() provide:
// <fence> decrement-dest <membar StoreLoad|StoreStore>
// The type D may be either a pointer type, or an integral
// type. If it is a pointer type, then the decrement is
// scaled to the size of the type pointed to by the pointer.
template<typename D>
- inline static void dec(D volatile* dest);
+ inline static void dec(D volatile* dest,
+ atomic_memory_order order = memory_order_conservative);
// Performs atomic exchange of *dest with exchange_value. Returns old
// prior value of *dest. xchg*() provide:
// <fence> exchange-value-with-dest <membar StoreLoad|StoreStore>
// The type T must be either a pointer type convertible to or equal
// to D, an integral/enum type equal to D, or a type equal to D that
// is primitive convertible using PrimitiveConversions.
template<typename T, typename D>
- inline static D xchg(T exchange_value, volatile D* dest);
+ inline static D xchg(T exchange_value, volatile D* dest,
+ atomic_memory_order order = memory_order_conservative);
// Performs atomic compare of *dest and compare_value, and exchanges
// *dest with exchange_value if the comparison succeeded. Returns prior
// value of *dest. cmpxchg*() provide:
// <fence> compare-and-exchange <membar StoreLoad|StoreStore>
template<typename T, typename D, typename U>
inline static D cmpxchg(T exchange_value,
D volatile* dest,
U compare_value,
- cmpxchg_memory_order order = memory_order_conservative);
+ atomic_memory_order order = memory_order_conservative);
// Performs atomic compare of *dest and NULL, and replaces *dest
// with exchange_value if the comparison succeeded. Returns true if
// the comparison succeeded and the exchange occurred. This is
// often used as part of lazy initialization, as a lock-free
// alternative to the Double-Checked Locking Pattern.
template<typename T, typename D>
inline static bool replace_if_null(T* value, D* volatile* dest,
- cmpxchg_memory_order order = memory_order_conservative);
+ atomic_memory_order order = memory_order_conservative);
private:
WINDOWS_ONLY(public:) // VS2017 warns (C2027) use of undefined type if IsPointerConvertible is declared private
// Test whether From is implicitly convertible to To.
// From and To must be pointer types.
@@ -270,19 +280,19 @@
// of 1, 4, and 8 are required. The class is a function object that
// must be default constructable, with these requirements:
//
// - dest is of type T*.
// - exchange_value and compare_value are of type T.
- // - order is of type cmpxchg_memory_order.
+ // - order is of type atomic_memory_order.
// - platform_cmpxchg is an object of type PlatformCmpxchg<sizeof(T)>.
//
// Then
// platform_cmpxchg(exchange_value, dest, compare_value, order)
// must be a valid expression, returning a result convertible to T.
//
// A default definition is provided, which declares a function template
- // T operator()(T, T volatile*, T, cmpxchg_memory_order) const
+ // T operator()(T, T volatile*, T, atomic_memory_order) const
//
// For each required size, a platform must either provide an
// appropriate definition of that function, or must entirely
// specialize the class template for that size.
template<size_t byte_size> struct PlatformCmpxchg;
@@ -325,11 +335,11 @@
// Then
// platform_xchg(exchange_value, dest)
// must be a valid expression, returning a result convertible to T.
//
// A default definition is provided, which declares a function template
- // T operator()(T, T volatile*, T, cmpxchg_memory_order) const
+ // T operator()(T, T volatile*, T, atomic_memory_order) const
//
// For each required size, a platform must either provide an
// appropriate definition of that function, or must entirely
// specialize the class template for that size.
template<size_t byte_size> struct PlatformXchg;
@@ -486,37 +496,37 @@
// be complete.
template<typename Derived>
struct Atomic::FetchAndAdd {
template<typename I, typename D>
- D operator()(I add_value, D volatile* dest) const;
+ D operator()(I add_value, D volatile* dest, atomic_memory_order order) const;
};
template<typename Derived>
struct Atomic::AddAndFetch {
template<typename I, typename D>
- D operator()(I add_value, D volatile* dest) const;
+ D operator()(I add_value, D volatile* dest, atomic_memory_order order) const;
};
template<typename D>
-inline void Atomic::inc(D volatile* dest) {
+inline void Atomic::inc(D volatile* dest, atomic_memory_order order) {
STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I;
- Atomic::add(I(1), dest);
+ Atomic::add(I(1), dest, order);
}
template<typename D>
-inline void Atomic::dec(D volatile* dest) {
+inline void Atomic::dec(D volatile* dest, atomic_memory_order order) {
STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I;
// Assumes two's complement integer representation.
#pragma warning(suppress: 4146)
- Atomic::add(I(-1), dest);
+ Atomic::add(I(-1), dest, order);
}
template<typename I, typename D>
-inline D Atomic::sub(I sub_value, D volatile* dest) {
+inline D Atomic::sub(I sub_value, D volatile* dest, atomic_memory_order order) {
STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
STATIC_ASSERT(IsIntegral<I>::value);
// If D is a pointer type, use [u]intptr_t as the addend type,
// matching signedness of I. Otherwise, use D as the addend type.
typedef typename Conditional<IsSigned<I>::value, intptr_t, uintptr_t>::type PI;
@@ -525,11 +535,11 @@
STATIC_ASSERT(IsSigned<I>::value == IsSigned<AddendType>::value);
STATIC_ASSERT(sizeof(I) <= sizeof(AddendType));
AddendType addend = sub_value;
// Assumes two's complement integer representation.
#pragma warning(suppress: 4146) // In case AddendType is not signed.
- return Atomic::add(-addend, dest);
+ return Atomic::add(-addend, dest, order);
}
// Define the class before including platform file, which may specialize
// the operator definition. No generic definition of specializations
// of the operator template are provided, nor are there any generic
@@ -539,22 +549,22 @@
struct Atomic::PlatformCmpxchg {
template<typename T>
T operator()(T exchange_value,
T volatile* dest,
T compare_value,
- cmpxchg_memory_order order) const;
+ atomic_memory_order order) const;
};
// Define the class before including platform file, which may use this
// as a base class, requiring it be complete. The definition is later
// in this file, near the other definitions related to cmpxchg.
struct Atomic::CmpxchgByteUsingInt {
template<typename T>
T operator()(T exchange_value,
T volatile* dest,
T compare_value,
- cmpxchg_memory_order order) const;
+ atomic_memory_order order) const;
};
// Define the class before including platform file, which may specialize
// the operator definition. No generic definition of specializations
// of the operator template are provided, nor are there any generic
@@ -562,11 +572,12 @@
// providing those.
template<size_t byte_size>
struct Atomic::PlatformXchg {
template<typename T>
T operator()(T exchange_value,
- T volatile* dest) const;
+ T volatile* dest,
+ atomic_memory_order order) const;
};
// platform specific in-line definitions - must come before shared definitions
#include OS_CPU_HEADER(atomic)
@@ -587,41 +598,42 @@
inline void Atomic::store(T store_value, volatile D* dest) {
StoreImpl<T, D, PlatformStore<sizeof(D)> >()(store_value, dest);
}
template<typename I, typename D>
-inline D Atomic::add(I add_value, D volatile* dest) {
- return AddImpl<I, D>()(add_value, dest);
+inline D Atomic::add(I add_value, D volatile* dest,
+ atomic_memory_order order) {
+ return AddImpl<I, D>()(add_value, dest, order);
}
template<typename I, typename D>
struct Atomic::AddImpl<
I, D,
typename EnableIf<IsIntegral<I>::value &&
IsIntegral<D>::value &&
(sizeof(I) <= sizeof(D)) &&
(IsSigned<I>::value == IsSigned<D>::value)>::type>
{
- D operator()(I add_value, D volatile* dest) const {
+ D operator()(I add_value, D volatile* dest, atomic_memory_order order) const {
D addend = add_value;
- return PlatformAdd<sizeof(D)>()(addend, dest);
+ return PlatformAdd<sizeof(D)>()(addend, dest, order);
}
};
template<typename I, typename P>
struct Atomic::AddImpl<
I, P*,
typename EnableIf<IsIntegral<I>::value && (sizeof(I) <= sizeof(P*))>::type>
{
- P* operator()(I add_value, P* volatile* dest) const {
+ P* operator()(I add_value, P* volatile* dest, atomic_memory_order order) const {
STATIC_ASSERT(sizeof(intptr_t) == sizeof(P*));
STATIC_ASSERT(sizeof(uintptr_t) == sizeof(P*));
typedef typename Conditional<IsSigned<I>::value,
intptr_t,
uintptr_t>::type CI;
CI addend = add_value;
- return PlatformAdd<sizeof(P*)>()(addend, dest);
+ return PlatformAdd<sizeof(P*)>()(addend, dest, order);
}
};
// Most platforms do not support atomic add on a 2-byte value. However,
// if the value occupies the most significant 16 bits of an aligned 32-bit
@@ -632,42 +644,44 @@
// in case of overflow/underflow.
//
// Use the ATOMIC_SHORT_PAIR macro (see macros.hpp) to get the desired alignment.
template<>
struct Atomic::AddImpl<short, short> {
- short operator()(short add_value, short volatile* dest) const {
+ short operator()(short add_value, short volatile* dest, atomic_memory_order order) const {
#ifdef VM_LITTLE_ENDIAN
assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
- int new_value = Atomic::add(add_value << 16, (volatile int*)(dest-1));
+ int new_value = Atomic::add(add_value << 16, (volatile int*)(dest-1), order);
#else
assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
- int new_value = Atomic::add(add_value << 16, (volatile int*)(dest));
+ int new_value = Atomic::add(add_value << 16, (volatile int*)(dest), order);
#endif
return (short)(new_value >> 16); // preserves sign
}
};
template<typename Derived>
template<typename I, typename D>
-inline D Atomic::FetchAndAdd<Derived>::operator()(I add_value, D volatile* dest) const {
+inline D Atomic::FetchAndAdd<Derived>::operator()(I add_value, D volatile* dest,
+ atomic_memory_order order) const {
I addend = add_value;
// If D is a pointer type P*, scale by sizeof(P).
if (IsPointer<D>::value) {
addend *= sizeof(typename RemovePointer<D>::type);
}
- D old = static_cast<const Derived*>(this)->fetch_and_add(addend, dest);
+ D old = static_cast<const Derived*>(this)->fetch_and_add(addend, dest, order);
return old + add_value;
}
template<typename Derived>
template<typename I, typename D>
-inline D Atomic::AddAndFetch<Derived>::operator()(I add_value, D volatile* dest) const {
+inline D Atomic::AddAndFetch<Derived>::operator()(I add_value, D volatile* dest,
+ atomic_memory_order order) const {
// If D is a pointer type P*, scale by sizeof(P).
if (IsPointer<D>::value) {
add_value *= sizeof(typename RemovePointer<D>::type);
}
- return static_cast<const Derived*>(this)->add_and_fetch(add_value, dest);
+ return static_cast<const Derived*>(this)->add_and_fetch(add_value, dest, order);
}
template<typename Type, typename Fn, typename I, typename D>
inline D Atomic::add_using_helper(Fn fn, I add_value, D volatile* dest) {
return PrimitiveConversions::cast<D>(
@@ -677,17 +691,17 @@
template<typename T, typename D, typename U>
inline D Atomic::cmpxchg(T exchange_value,
D volatile* dest,
U compare_value,
- cmpxchg_memory_order order) {
+ atomic_memory_order order) {
return CmpxchgImpl<T, D, U>()(exchange_value, dest, compare_value, order);
}
template<typename T, typename D>
inline bool Atomic::replace_if_null(T* value, D* volatile* dest,
- cmpxchg_memory_order order) {
+ atomic_memory_order order) {
// Presently using a trivial implementation in terms of cmpxchg.
// Consider adding platform support, to permit the use of compiler
// intrinsics like gcc's __sync_bool_compare_and_swap.
D* expected_null = NULL;
return expected_null == cmpxchg(value, dest, expected_null, order);
@@ -700,11 +714,11 @@
struct Atomic::CmpxchgImpl<
T, T, T,
typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
{
T operator()(T exchange_value, T volatile* dest, T compare_value,
- cmpxchg_memory_order order) const {
+ atomic_memory_order order) const {
// Forward to the platform handler for the size of T.
return PlatformCmpxchg<sizeof(T)>()(exchange_value,
dest,
compare_value,
order);
@@ -726,11 +740,11 @@
typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value &&
IsSame<typename RemoveCV<D>::type,
typename RemoveCV<U>::type>::value>::type>
{
D* operator()(T* exchange_value, D* volatile* dest, U* compare_value,
- cmpxchg_memory_order order) const {
+ atomic_memory_order order) const {
// Allow derived to base conversion, and adding cv-qualifiers.
D* new_value = exchange_value;
// Don't care what the CV qualifiers for compare_value are,
// but we need to match D* when calling platform support.
D* old_value = const_cast<D*>(compare_value);
@@ -749,11 +763,11 @@
struct Atomic::CmpxchgImpl<
T, T, T,
typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
{
T operator()(T exchange_value, T volatile* dest, T compare_value,
- cmpxchg_memory_order order) const {
+ atomic_memory_order order) const {
typedef PrimitiveConversions::Translate<T> Translator;
typedef typename Translator::Decayed Decayed;
STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
return Translator::recover(
cmpxchg(Translator::decay(exchange_value),
@@ -777,11 +791,11 @@
template<typename T>
inline T Atomic::CmpxchgByteUsingInt::operator()(T exchange_value,
T volatile* dest,
T compare_value,
- cmpxchg_memory_order order) const {
+ atomic_memory_order order) const {
STATIC_ASSERT(sizeof(T) == sizeof(uint8_t));
uint8_t canon_exchange_value = exchange_value;
uint8_t canon_compare_value = compare_value;
volatile uint32_t* aligned_dest
= reinterpret_cast<volatile uint32_t*>(align_down(dest, sizeof(uint32_t)));
@@ -819,13 +833,13 @@
template<typename T>
struct Atomic::XchgImpl<
T, T,
typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
{
- T operator()(T exchange_value, T volatile* dest) const {
+ T operator()(T exchange_value, T volatile* dest, atomic_memory_order order) const {
// Forward to the platform handler for the size of T.
- return PlatformXchg<sizeof(T)>()(exchange_value, dest);
+ return PlatformXchg<sizeof(T)>()(exchange_value, dest, order);
}
};
// Handle xchg for pointer types.
//
@@ -835,14 +849,14 @@
template<typename T, typename D>
struct Atomic::XchgImpl<
T*, D*,
typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type>
{
- D* operator()(T* exchange_value, D* volatile* dest) const {
+ D* operator()(T* exchange_value, D* volatile* dest, atomic_memory_order order) const {
// Allow derived to base conversion, and adding cv-qualifiers.
D* new_value = exchange_value;
- return PlatformXchg<sizeof(D*)>()(new_value, dest);
+ return PlatformXchg<sizeof(D*)>()(new_value, dest, order);
}
};
// Handle xchg for types that have a translator.
//
@@ -854,17 +868,18 @@
template<typename T>
struct Atomic::XchgImpl<
T, T,
typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
{
- T operator()(T exchange_value, T volatile* dest) const {
+ T operator()(T exchange_value, T volatile* dest, atomic_memory_order order) const {
typedef PrimitiveConversions::Translate<T> Translator;
typedef typename Translator::Decayed Decayed;
STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
return Translator::recover(
xchg(Translator::decay(exchange_value),
- reinterpret_cast<Decayed volatile*>(dest)));
+ reinterpret_cast<Decayed volatile*>(dest),
+ order));
}
};
template<typename Type, typename Fn, typename T>
inline T Atomic::xchg_using_helper(Fn fn,
@@ -875,10 +890,10 @@
fn(PrimitiveConversions::cast<Type>(exchange_value),
reinterpret_cast<Type volatile*>(dest)));
}
template<typename T, typename D>
-inline D Atomic::xchg(T exchange_value, volatile D* dest) {
- return XchgImpl<T, D>()(exchange_value, dest);
+inline D Atomic::xchg(T exchange_value, volatile D* dest, atomic_memory_order order) {
+ return XchgImpl<T, D>()(exchange_value, dest, order);
}
#endif // SHARE_VM_RUNTIME_ATOMIC_HPP
< prev index next >