< prev index next >
src/hotspot/share/runtime/atomic.hpp
Print this page
rev 49898 : 8202080: Introduce ordering semantics for Atomic::add and other RMW atomics
Reviewed-by:
*** 35,47 ****
#include "metaprogramming/removeCV.hpp"
#include "metaprogramming/removePointer.hpp"
#include "utilities/align.hpp"
#include "utilities/macros.hpp"
! enum cmpxchg_memory_order {
memory_order_relaxed,
! // Use value which doesn't interfere with C++2011. We need to be more conservative.
memory_order_conservative = 8
};
class Atomic : AllStatic {
public:
--- 35,52 ----
#include "metaprogramming/removeCV.hpp"
#include "metaprogramming/removePointer.hpp"
#include "utilities/align.hpp"
#include "utilities/macros.hpp"
! enum atomic_memory_order {
! // We support most semantics like in C++11.
memory_order_relaxed,
! memory_order_consume,
! memory_order_acquire,
! memory_order_release,
! memory_order_acq_rel,
! // We need to be more conservative than seq_cst on PPC64.
memory_order_conservative = 8
};
class Atomic : AllStatic {
public:
*** 78,136 ****
// Atomically add to a location. Returns updated value. add*() provide:
// <fence> add-value-to-dest <membar StoreLoad|StoreStore>
template<typename I, typename D>
! inline static D add(I add_value, D volatile* dest);
template<typename I, typename D>
! inline static D sub(I sub_value, D volatile* dest);
// Atomically increment location. inc() provide:
// <fence> increment-dest <membar StoreLoad|StoreStore>
// The type D may be either a pointer type, or an integral
// type. If it is a pointer type, then the increment is
// scaled to the size of the type pointed to by the pointer.
template<typename D>
! inline static void inc(D volatile* dest);
// Atomically decrement a location. dec() provide:
// <fence> decrement-dest <membar StoreLoad|StoreStore>
// The type D may be either a pointer type, or an integral
// type. If it is a pointer type, then the decrement is
// scaled to the size of the type pointed to by the pointer.
template<typename D>
! inline static void dec(D volatile* dest);
// Performs atomic exchange of *dest with exchange_value. Returns old
// prior value of *dest. xchg*() provide:
// <fence> exchange-value-with-dest <membar StoreLoad|StoreStore>
// The type T must be either a pointer type convertible to or equal
// to D, an integral/enum type equal to D, or a type equal to D that
// is primitive convertible using PrimitiveConversions.
template<typename T, typename D>
! inline static D xchg(T exchange_value, volatile D* dest);
// Performs atomic compare of *dest and compare_value, and exchanges
// *dest with exchange_value if the comparison succeeded. Returns prior
// value of *dest. cmpxchg*() provide:
// <fence> compare-and-exchange <membar StoreLoad|StoreStore>
template<typename T, typename D, typename U>
inline static D cmpxchg(T exchange_value,
D volatile* dest,
U compare_value,
! cmpxchg_memory_order order = memory_order_conservative);
// Performs atomic compare of *dest and NULL, and replaces *dest
// with exchange_value if the comparison succeeded. Returns true if
// the comparison succeeded and the exchange occurred. This is
// often used as part of lazy initialization, as a lock-free
// alternative to the Double-Checked Locking Pattern.
template<typename T, typename D>
inline static bool replace_if_null(T* value, D* volatile* dest,
! cmpxchg_memory_order order = memory_order_conservative);
private:
WINDOWS_ONLY(public:) // VS2017 warns (C2027) use of undefined type if IsPointerConvertible is declared private
// Test whether From is implicitly convertible to To.
// From and To must be pointer types.
--- 83,146 ----
// Atomically add to a location. Returns updated value. add*() provide:
// <fence> add-value-to-dest <membar StoreLoad|StoreStore>
template<typename I, typename D>
! inline static D add(I add_value, D volatile* dest,
! atomic_memory_order order = memory_order_conservative);
template<typename I, typename D>
! inline static D sub(I sub_value, D volatile* dest,
! atomic_memory_order order = memory_order_conservative);
// Atomically increment location. inc() provide:
// <fence> increment-dest <membar StoreLoad|StoreStore>
// The type D may be either a pointer type, or an integral
// type. If it is a pointer type, then the increment is
// scaled to the size of the type pointed to by the pointer.
template<typename D>
! inline static void inc(D volatile* dest,
! atomic_memory_order order = memory_order_conservative);
// Atomically decrement a location. dec() provide:
// <fence> decrement-dest <membar StoreLoad|StoreStore>
// The type D may be either a pointer type, or an integral
// type. If it is a pointer type, then the decrement is
// scaled to the size of the type pointed to by the pointer.
template<typename D>
! inline static void dec(D volatile* dest,
! atomic_memory_order order = memory_order_conservative);
// Performs atomic exchange of *dest with exchange_value. Returns old
// prior value of *dest. xchg*() provide:
// <fence> exchange-value-with-dest <membar StoreLoad|StoreStore>
// The type T must be either a pointer type convertible to or equal
// to D, an integral/enum type equal to D, or a type equal to D that
// is primitive convertible using PrimitiveConversions.
template<typename T, typename D>
! inline static D xchg(T exchange_value, volatile D* dest,
! atomic_memory_order order = memory_order_conservative);
// Performs atomic compare of *dest and compare_value, and exchanges
// *dest with exchange_value if the comparison succeeded. Returns prior
// value of *dest. cmpxchg*() provide:
// <fence> compare-and-exchange <membar StoreLoad|StoreStore>
template<typename T, typename D, typename U>
inline static D cmpxchg(T exchange_value,
D volatile* dest,
U compare_value,
! atomic_memory_order order = memory_order_conservative);
// Performs atomic compare of *dest and NULL, and replaces *dest
// with exchange_value if the comparison succeeded. Returns true if
// the comparison succeeded and the exchange occurred. This is
// often used as part of lazy initialization, as a lock-free
// alternative to the Double-Checked Locking Pattern.
template<typename T, typename D>
inline static bool replace_if_null(T* value, D* volatile* dest,
! atomic_memory_order order = memory_order_conservative);
private:
WINDOWS_ONLY(public:) // VS2017 warns (C2027) use of undefined type if IsPointerConvertible is declared private
// Test whether From is implicitly convertible to To.
// From and To must be pointer types.
*** 270,288 ****
// of 1, 4, and 8 are required. The class is a function object that
// must be default constructable, with these requirements:
//
// - dest is of type T*.
// - exchange_value and compare_value are of type T.
! // - order is of type cmpxchg_memory_order.
// - platform_cmpxchg is an object of type PlatformCmpxchg<sizeof(T)>.
//
// Then
// platform_cmpxchg(exchange_value, dest, compare_value, order)
// must be a valid expression, returning a result convertible to T.
//
// A default definition is provided, which declares a function template
! // T operator()(T, T volatile*, T, cmpxchg_memory_order) const
//
// For each required size, a platform must either provide an
// appropriate definition of that function, or must entirely
// specialize the class template for that size.
template<size_t byte_size> struct PlatformCmpxchg;
--- 280,298 ----
// of 1, 4, and 8 are required. The class is a function object that
// must be default constructable, with these requirements:
//
// - dest is of type T*.
// - exchange_value and compare_value are of type T.
! // - order is of type atomic_memory_order.
// - platform_cmpxchg is an object of type PlatformCmpxchg<sizeof(T)>.
//
// Then
// platform_cmpxchg(exchange_value, dest, compare_value, order)
// must be a valid expression, returning a result convertible to T.
//
// A default definition is provided, which declares a function template
! // T operator()(T, T volatile*, T, atomic_memory_order) const
//
// For each required size, a platform must either provide an
// appropriate definition of that function, or must entirely
// specialize the class template for that size.
template<size_t byte_size> struct PlatformCmpxchg;
*** 325,335 ****
// Then
// platform_xchg(exchange_value, dest)
// must be a valid expression, returning a result convertible to T.
//
// A default definition is provided, which declares a function template
! // T operator()(T, T volatile*, T, cmpxchg_memory_order) const
//
// For each required size, a platform must either provide an
// appropriate definition of that function, or must entirely
// specialize the class template for that size.
template<size_t byte_size> struct PlatformXchg;
--- 335,345 ----
// Then
// platform_xchg(exchange_value, dest)
// must be a valid expression, returning a result convertible to T.
//
// A default definition is provided, which declares a function template
! // T operator()(T, T volatile*, T, atomic_memory_order) const
//
// For each required size, a platform must either provide an
// appropriate definition of that function, or must entirely
// specialize the class template for that size.
template<size_t byte_size> struct PlatformXchg;
*** 486,522 ****
// be complete.
template<typename Derived>
struct Atomic::FetchAndAdd {
template<typename I, typename D>
! D operator()(I add_value, D volatile* dest) const;
};
template<typename Derived>
struct Atomic::AddAndFetch {
template<typename I, typename D>
! D operator()(I add_value, D volatile* dest) const;
};
template<typename D>
! inline void Atomic::inc(D volatile* dest) {
STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I;
! Atomic::add(I(1), dest);
}
template<typename D>
! inline void Atomic::dec(D volatile* dest) {
STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I;
// Assumes two's complement integer representation.
#pragma warning(suppress: 4146)
! Atomic::add(I(-1), dest);
}
template<typename I, typename D>
! inline D Atomic::sub(I sub_value, D volatile* dest) {
STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
STATIC_ASSERT(IsIntegral<I>::value);
// If D is a pointer type, use [u]intptr_t as the addend type,
// matching signedness of I. Otherwise, use D as the addend type.
typedef typename Conditional<IsSigned<I>::value, intptr_t, uintptr_t>::type PI;
--- 496,532 ----
// be complete.
template<typename Derived>
struct Atomic::FetchAndAdd {
template<typename I, typename D>
! D operator()(I add_value, D volatile* dest, atomic_memory_order order) const;
};
template<typename Derived>
struct Atomic::AddAndFetch {
template<typename I, typename D>
! D operator()(I add_value, D volatile* dest, atomic_memory_order order) const;
};
template<typename D>
! inline void Atomic::inc(D volatile* dest, atomic_memory_order order) {
STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I;
! Atomic::add(I(1), dest, order);
}
template<typename D>
! inline void Atomic::dec(D volatile* dest, atomic_memory_order order) {
STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I;
// Assumes two's complement integer representation.
#pragma warning(suppress: 4146)
! Atomic::add(I(-1), dest, order);
}
template<typename I, typename D>
! inline D Atomic::sub(I sub_value, D volatile* dest, atomic_memory_order order) {
STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value);
STATIC_ASSERT(IsIntegral<I>::value);
// If D is a pointer type, use [u]intptr_t as the addend type,
// matching signedness of I. Otherwise, use D as the addend type.
typedef typename Conditional<IsSigned<I>::value, intptr_t, uintptr_t>::type PI;
*** 525,535 ****
STATIC_ASSERT(IsSigned<I>::value == IsSigned<AddendType>::value);
STATIC_ASSERT(sizeof(I) <= sizeof(AddendType));
AddendType addend = sub_value;
// Assumes two's complement integer representation.
#pragma warning(suppress: 4146) // In case AddendType is not signed.
! return Atomic::add(-addend, dest);
}
// Define the class before including platform file, which may specialize
// the operator definition. No generic definition of specializations
// of the operator template are provided, nor are there any generic
--- 535,545 ----
STATIC_ASSERT(IsSigned<I>::value == IsSigned<AddendType>::value);
STATIC_ASSERT(sizeof(I) <= sizeof(AddendType));
AddendType addend = sub_value;
// Assumes two's complement integer representation.
#pragma warning(suppress: 4146) // In case AddendType is not signed.
! return Atomic::add(-addend, dest, order);
}
// Define the class before including platform file, which may specialize
// the operator definition. No generic definition of specializations
// of the operator template are provided, nor are there any generic
*** 539,560 ****
struct Atomic::PlatformCmpxchg {
template<typename T>
T operator()(T exchange_value,
T volatile* dest,
T compare_value,
! cmpxchg_memory_order order) const;
};
// Define the class before including platform file, which may use this
// as a base class, requiring it be complete. The definition is later
// in this file, near the other definitions related to cmpxchg.
struct Atomic::CmpxchgByteUsingInt {
template<typename T>
T operator()(T exchange_value,
T volatile* dest,
T compare_value,
! cmpxchg_memory_order order) const;
};
// Define the class before including platform file, which may specialize
// the operator definition. No generic definition of specializations
// of the operator template are provided, nor are there any generic
--- 549,570 ----
struct Atomic::PlatformCmpxchg {
template<typename T>
T operator()(T exchange_value,
T volatile* dest,
T compare_value,
! atomic_memory_order order) const;
};
// Define the class before including platform file, which may use this
// as a base class, requiring it be complete. The definition is later
// in this file, near the other definitions related to cmpxchg.
struct Atomic::CmpxchgByteUsingInt {
template<typename T>
T operator()(T exchange_value,
T volatile* dest,
T compare_value,
! atomic_memory_order order) const;
};
// Define the class before including platform file, which may specialize
// the operator definition. No generic definition of specializations
// of the operator template are provided, nor are there any generic
*** 562,572 ****
// providing those.
template<size_t byte_size>
struct Atomic::PlatformXchg {
template<typename T>
T operator()(T exchange_value,
! T volatile* dest) const;
};
// platform specific in-line definitions - must come before shared definitions
#include OS_CPU_HEADER(atomic)
--- 572,583 ----
// providing those.
template<size_t byte_size>
struct Atomic::PlatformXchg {
template<typename T>
T operator()(T exchange_value,
! T volatile* dest,
! atomic_memory_order order) const;
};
// platform specific in-line definitions - must come before shared definitions
#include OS_CPU_HEADER(atomic)
*** 587,627 ****
inline void Atomic::store(T store_value, volatile D* dest) {
StoreImpl<T, D, PlatformStore<sizeof(D)> >()(store_value, dest);
}
template<typename I, typename D>
! inline D Atomic::add(I add_value, D volatile* dest) {
! return AddImpl<I, D>()(add_value, dest);
}
template<typename I, typename D>
struct Atomic::AddImpl<
I, D,
typename EnableIf<IsIntegral<I>::value &&
IsIntegral<D>::value &&
(sizeof(I) <= sizeof(D)) &&
(IsSigned<I>::value == IsSigned<D>::value)>::type>
{
! D operator()(I add_value, D volatile* dest) const {
D addend = add_value;
! return PlatformAdd<sizeof(D)>()(addend, dest);
}
};
template<typename I, typename P>
struct Atomic::AddImpl<
I, P*,
typename EnableIf<IsIntegral<I>::value && (sizeof(I) <= sizeof(P*))>::type>
{
! P* operator()(I add_value, P* volatile* dest) const {
STATIC_ASSERT(sizeof(intptr_t) == sizeof(P*));
STATIC_ASSERT(sizeof(uintptr_t) == sizeof(P*));
typedef typename Conditional<IsSigned<I>::value,
intptr_t,
uintptr_t>::type CI;
CI addend = add_value;
! return PlatformAdd<sizeof(P*)>()(addend, dest);
}
};
// Most platforms do not support atomic add on a 2-byte value. However,
// if the value occupies the most significant 16 bits of an aligned 32-bit
--- 598,639 ----
inline void Atomic::store(T store_value, volatile D* dest) {
StoreImpl<T, D, PlatformStore<sizeof(D)> >()(store_value, dest);
}
template<typename I, typename D>
! inline D Atomic::add(I add_value, D volatile* dest,
! atomic_memory_order order) {
! return AddImpl<I, D>()(add_value, dest, order);
}
template<typename I, typename D>
struct Atomic::AddImpl<
I, D,
typename EnableIf<IsIntegral<I>::value &&
IsIntegral<D>::value &&
(sizeof(I) <= sizeof(D)) &&
(IsSigned<I>::value == IsSigned<D>::value)>::type>
{
! D operator()(I add_value, D volatile* dest, atomic_memory_order order) const {
D addend = add_value;
! return PlatformAdd<sizeof(D)>()(addend, dest, order);
}
};
template<typename I, typename P>
struct Atomic::AddImpl<
I, P*,
typename EnableIf<IsIntegral<I>::value && (sizeof(I) <= sizeof(P*))>::type>
{
! P* operator()(I add_value, P* volatile* dest, atomic_memory_order order) const {
STATIC_ASSERT(sizeof(intptr_t) == sizeof(P*));
STATIC_ASSERT(sizeof(uintptr_t) == sizeof(P*));
typedef typename Conditional<IsSigned<I>::value,
intptr_t,
uintptr_t>::type CI;
CI addend = add_value;
! return PlatformAdd<sizeof(P*)>()(addend, dest, order);
}
};
// Most platforms do not support atomic add on a 2-byte value. However,
// if the value occupies the most significant 16 bits of an aligned 32-bit
*** 632,673 ****
// in case of overflow/underflow.
//
// Use the ATOMIC_SHORT_PAIR macro (see macros.hpp) to get the desired alignment.
template<>
struct Atomic::AddImpl<short, short> {
! short operator()(short add_value, short volatile* dest) const {
#ifdef VM_LITTLE_ENDIAN
assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
! int new_value = Atomic::add(add_value << 16, (volatile int*)(dest-1));
#else
assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
! int new_value = Atomic::add(add_value << 16, (volatile int*)(dest));
#endif
return (short)(new_value >> 16); // preserves sign
}
};
template<typename Derived>
template<typename I, typename D>
! inline D Atomic::FetchAndAdd<Derived>::operator()(I add_value, D volatile* dest) const {
I addend = add_value;
// If D is a pointer type P*, scale by sizeof(P).
if (IsPointer<D>::value) {
addend *= sizeof(typename RemovePointer<D>::type);
}
! D old = static_cast<const Derived*>(this)->fetch_and_add(addend, dest);
return old + add_value;
}
template<typename Derived>
template<typename I, typename D>
! inline D Atomic::AddAndFetch<Derived>::operator()(I add_value, D volatile* dest) const {
// If D is a pointer type P*, scale by sizeof(P).
if (IsPointer<D>::value) {
add_value *= sizeof(typename RemovePointer<D>::type);
}
! return static_cast<const Derived*>(this)->add_and_fetch(add_value, dest);
}
template<typename Type, typename Fn, typename I, typename D>
inline D Atomic::add_using_helper(Fn fn, I add_value, D volatile* dest) {
return PrimitiveConversions::cast<D>(
--- 644,687 ----
// in case of overflow/underflow.
//
// Use the ATOMIC_SHORT_PAIR macro (see macros.hpp) to get the desired alignment.
template<>
struct Atomic::AddImpl<short, short> {
! short operator()(short add_value, short volatile* dest, atomic_memory_order order) const {
#ifdef VM_LITTLE_ENDIAN
assert((intx(dest) & 0x03) == 0x02, "wrong alignment");
! int new_value = Atomic::add(add_value << 16, (volatile int*)(dest-1), order);
#else
assert((intx(dest) & 0x03) == 0x00, "wrong alignment");
! int new_value = Atomic::add(add_value << 16, (volatile int*)(dest), order);
#endif
return (short)(new_value >> 16); // preserves sign
}
};
template<typename Derived>
template<typename I, typename D>
! inline D Atomic::FetchAndAdd<Derived>::operator()(I add_value, D volatile* dest,
! atomic_memory_order order) const {
I addend = add_value;
// If D is a pointer type P*, scale by sizeof(P).
if (IsPointer<D>::value) {
addend *= sizeof(typename RemovePointer<D>::type);
}
! D old = static_cast<const Derived*>(this)->fetch_and_add(addend, dest, order);
return old + add_value;
}
template<typename Derived>
template<typename I, typename D>
! inline D Atomic::AddAndFetch<Derived>::operator()(I add_value, D volatile* dest,
! atomic_memory_order order) const {
// If D is a pointer type P*, scale by sizeof(P).
if (IsPointer<D>::value) {
add_value *= sizeof(typename RemovePointer<D>::type);
}
! return static_cast<const Derived*>(this)->add_and_fetch(add_value, dest, order);
}
template<typename Type, typename Fn, typename I, typename D>
inline D Atomic::add_using_helper(Fn fn, I add_value, D volatile* dest) {
return PrimitiveConversions::cast<D>(
*** 677,693 ****
template<typename T, typename D, typename U>
inline D Atomic::cmpxchg(T exchange_value,
D volatile* dest,
U compare_value,
! cmpxchg_memory_order order) {
return CmpxchgImpl<T, D, U>()(exchange_value, dest, compare_value, order);
}
template<typename T, typename D>
inline bool Atomic::replace_if_null(T* value, D* volatile* dest,
! cmpxchg_memory_order order) {
// Presently using a trivial implementation in terms of cmpxchg.
// Consider adding platform support, to permit the use of compiler
// intrinsics like gcc's __sync_bool_compare_and_swap.
D* expected_null = NULL;
return expected_null == cmpxchg(value, dest, expected_null, order);
--- 691,707 ----
template<typename T, typename D, typename U>
inline D Atomic::cmpxchg(T exchange_value,
D volatile* dest,
U compare_value,
! atomic_memory_order order) {
return CmpxchgImpl<T, D, U>()(exchange_value, dest, compare_value, order);
}
template<typename T, typename D>
inline bool Atomic::replace_if_null(T* value, D* volatile* dest,
! atomic_memory_order order) {
// Presently using a trivial implementation in terms of cmpxchg.
// Consider adding platform support, to permit the use of compiler
// intrinsics like gcc's __sync_bool_compare_and_swap.
D* expected_null = NULL;
return expected_null == cmpxchg(value, dest, expected_null, order);
*** 700,710 ****
struct Atomic::CmpxchgImpl<
T, T, T,
typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
{
T operator()(T exchange_value, T volatile* dest, T compare_value,
! cmpxchg_memory_order order) const {
// Forward to the platform handler for the size of T.
return PlatformCmpxchg<sizeof(T)>()(exchange_value,
dest,
compare_value,
order);
--- 714,724 ----
struct Atomic::CmpxchgImpl<
T, T, T,
typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
{
T operator()(T exchange_value, T volatile* dest, T compare_value,
! atomic_memory_order order) const {
// Forward to the platform handler for the size of T.
return PlatformCmpxchg<sizeof(T)>()(exchange_value,
dest,
compare_value,
order);
*** 726,736 ****
typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value &&
IsSame<typename RemoveCV<D>::type,
typename RemoveCV<U>::type>::value>::type>
{
D* operator()(T* exchange_value, D* volatile* dest, U* compare_value,
! cmpxchg_memory_order order) const {
// Allow derived to base conversion, and adding cv-qualifiers.
D* new_value = exchange_value;
// Don't care what the CV qualifiers for compare_value are,
// but we need to match D* when calling platform support.
D* old_value = const_cast<D*>(compare_value);
--- 740,750 ----
typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value &&
IsSame<typename RemoveCV<D>::type,
typename RemoveCV<U>::type>::value>::type>
{
D* operator()(T* exchange_value, D* volatile* dest, U* compare_value,
! atomic_memory_order order) const {
// Allow derived to base conversion, and adding cv-qualifiers.
D* new_value = exchange_value;
// Don't care what the CV qualifiers for compare_value are,
// but we need to match D* when calling platform support.
D* old_value = const_cast<D*>(compare_value);
*** 749,759 ****
struct Atomic::CmpxchgImpl<
T, T, T,
typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
{
T operator()(T exchange_value, T volatile* dest, T compare_value,
! cmpxchg_memory_order order) const {
typedef PrimitiveConversions::Translate<T> Translator;
typedef typename Translator::Decayed Decayed;
STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
return Translator::recover(
cmpxchg(Translator::decay(exchange_value),
--- 763,773 ----
struct Atomic::CmpxchgImpl<
T, T, T,
typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
{
T operator()(T exchange_value, T volatile* dest, T compare_value,
! atomic_memory_order order) const {
typedef PrimitiveConversions::Translate<T> Translator;
typedef typename Translator::Decayed Decayed;
STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
return Translator::recover(
cmpxchg(Translator::decay(exchange_value),
*** 777,787 ****
template<typename T>
inline T Atomic::CmpxchgByteUsingInt::operator()(T exchange_value,
T volatile* dest,
T compare_value,
! cmpxchg_memory_order order) const {
STATIC_ASSERT(sizeof(T) == sizeof(uint8_t));
uint8_t canon_exchange_value = exchange_value;
uint8_t canon_compare_value = compare_value;
volatile uint32_t* aligned_dest
= reinterpret_cast<volatile uint32_t*>(align_down(dest, sizeof(uint32_t)));
--- 791,801 ----
template<typename T>
inline T Atomic::CmpxchgByteUsingInt::operator()(T exchange_value,
T volatile* dest,
T compare_value,
! atomic_memory_order order) const {
STATIC_ASSERT(sizeof(T) == sizeof(uint8_t));
uint8_t canon_exchange_value = exchange_value;
uint8_t canon_compare_value = compare_value;
volatile uint32_t* aligned_dest
= reinterpret_cast<volatile uint32_t*>(align_down(dest, sizeof(uint32_t)));
*** 819,831 ****
template<typename T>
struct Atomic::XchgImpl<
T, T,
typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
{
! T operator()(T exchange_value, T volatile* dest) const {
// Forward to the platform handler for the size of T.
! return PlatformXchg<sizeof(T)>()(exchange_value, dest);
}
};
// Handle xchg for pointer types.
//
--- 833,845 ----
template<typename T>
struct Atomic::XchgImpl<
T, T,
typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type>
{
! T operator()(T exchange_value, T volatile* dest, atomic_memory_order order) const {
// Forward to the platform handler for the size of T.
! return PlatformXchg<sizeof(T)>()(exchange_value, dest, order);
}
};
// Handle xchg for pointer types.
//
*** 835,848 ****
template<typename T, typename D>
struct Atomic::XchgImpl<
T*, D*,
typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type>
{
! D* operator()(T* exchange_value, D* volatile* dest) const {
// Allow derived to base conversion, and adding cv-qualifiers.
D* new_value = exchange_value;
! return PlatformXchg<sizeof(D*)>()(new_value, dest);
}
};
// Handle xchg for types that have a translator.
//
--- 849,862 ----
template<typename T, typename D>
struct Atomic::XchgImpl<
T*, D*,
typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type>
{
! D* operator()(T* exchange_value, D* volatile* dest, atomic_memory_order order) const {
// Allow derived to base conversion, and adding cv-qualifiers.
D* new_value = exchange_value;
! return PlatformXchg<sizeof(D*)>()(new_value, dest, order);
}
};
// Handle xchg for types that have a translator.
//
*** 854,870 ****
template<typename T>
struct Atomic::XchgImpl<
T, T,
typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
{
! T operator()(T exchange_value, T volatile* dest) const {
typedef PrimitiveConversions::Translate<T> Translator;
typedef typename Translator::Decayed Decayed;
STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
return Translator::recover(
xchg(Translator::decay(exchange_value),
! reinterpret_cast<Decayed volatile*>(dest)));
}
};
template<typename Type, typename Fn, typename T>
inline T Atomic::xchg_using_helper(Fn fn,
--- 868,885 ----
template<typename T>
struct Atomic::XchgImpl<
T, T,
typename EnableIf<PrimitiveConversions::Translate<T>::value>::type>
{
! T operator()(T exchange_value, T volatile* dest, atomic_memory_order order) const {
typedef PrimitiveConversions::Translate<T> Translator;
typedef typename Translator::Decayed Decayed;
STATIC_ASSERT(sizeof(T) == sizeof(Decayed));
return Translator::recover(
xchg(Translator::decay(exchange_value),
! reinterpret_cast<Decayed volatile*>(dest),
! order));
}
};
template<typename Type, typename Fn, typename T>
inline T Atomic::xchg_using_helper(Fn fn,
*** 875,884 ****
fn(PrimitiveConversions::cast<Type>(exchange_value),
reinterpret_cast<Type volatile*>(dest)));
}
template<typename T, typename D>
! inline D Atomic::xchg(T exchange_value, volatile D* dest) {
! return XchgImpl<T, D>()(exchange_value, dest);
}
#endif // SHARE_VM_RUNTIME_ATOMIC_HPP
--- 890,899 ----
fn(PrimitiveConversions::cast<Type>(exchange_value),
reinterpret_cast<Type volatile*>(dest)));
}
template<typename T, typename D>
! inline D Atomic::xchg(T exchange_value, volatile D* dest, atomic_memory_order order) {
! return XchgImpl<T, D>()(exchange_value, dest, order);
}
#endif // SHARE_VM_RUNTIME_ATOMIC_HPP
< prev index next >