diff --git a/src/hotspot/share/oops/accessBackend.hpp b/src/hotspot/share/oops/accessBackend.hpp index a705392..abcf59a 100644 --- a/src/hotspot/share/oops/accessBackend.hpp +++ b/src/hotspot/share/oops/accessBackend.hpp @@ -22,16 +22,26 @@ * */ -#ifndef SHARE_VM_RUNTIME_ACCESSBACKEND_HPP -#define SHARE_VM_RUNTIME_ACCESSBACKEND_HPP +#ifndef SHARE_OOPS_ACCESSBACKEND_HPP +#define SHARE_OOPS_ACCESSBACKEND_HPP +#include "gc/shared/barrierSetConfig.hpp" +#include "memory/allocation.hpp" #include "metaprogramming/conditional.hpp" +#include "metaprogramming/decay.hpp" #include "metaprogramming/enableIf.hpp" #include "metaprogramming/integralConstant.hpp" +#include "metaprogramming/isFloatingPoint.hpp" +#include "metaprogramming/isIntegral.hpp" +#include "metaprogramming/isPointer.hpp" #include "metaprogramming/isSame.hpp" +#include "metaprogramming/isVolatile.hpp" +#include "oops/accessDecorators.hpp" +#include "oops/oopsHierarchy.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" + // This metafunction returns either oop or narrowOop depending on whether // an access needs to use compressed oops or not. template @@ -395,4 +405,894 @@ public: static bool equals(oop o1, oop o2) { return o1 == o2; } }; -#endif // SHARE_VM_RUNTIME_ACCESSBACKEND_HPP +// Below is the implementation of the first 4 steps of the template pipeline: +// * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers +// and sets default decorators to sensible values. +// * Step 2: Reduce types. This step makes sure there is only a single T type and not +// multiple types. The P type of the address and T type of the value must +// match. +// * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be +// avoided, and in that case avoids it (calling raw accesses or +// primitive accesses in a build that does not require primitive GC barriers) +// * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding +// BarrierSet::AccessBarrier accessor that attaches GC-required barriers +// to the access. + +namespace AccessInternal { + template + struct OopOrNarrowOopInternal: AllStatic { + typedef oop type; + }; + + template <> + struct OopOrNarrowOopInternal: AllStatic { + typedef narrowOop type; + }; + + // This metafunction returns a canonicalized oop/narrowOop type for a passed + // in oop-like types passed in from oop_* overloads where the user has sworn + // that the passed in values should be oop-like (e.g. oop, oopDesc*, arrayOop, + // narrowOoop, instanceOopDesc*, and random other things). + // In the oop_* overloads, it must hold that if the passed in type T is not + // narrowOop, then it by contract has to be one of many oop-like types implicitly + // convertible to oop, and hence returns oop as the canonical oop type. + // If it turns out it was not, then the implicit conversion to oop will fail + // to compile, as desired. + template + struct OopOrNarrowOop: AllStatic { + typedef typename OopOrNarrowOopInternal::type>::type type; + }; + + inline void* field_addr(oop base, ptrdiff_t byte_offset) { + return reinterpret_cast(reinterpret_cast((void*)base) + byte_offset); + } + // Step 4: Runtime dispatch + // The RuntimeDispatch class is responsible for performing a runtime dispatch of the + // accessor. This is required when the access either depends on whether compressed oops + // is being used, or it depends on which GC implementation was chosen (e.g. requires GC + // barriers). The way it works is that a function pointer initially pointing to an + // accessor resolution function gets called for each access. Upon first invocation, + // it resolves which accessor to be used in future invocations and patches the + // function pointer to this new accessor. + + template + struct RuntimeDispatch: AllStatic {}; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _store_func; + + static void store_init(void* addr, T value); + + static inline void store(void* addr, T value) { + _store_func(addr, value); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _store_at_func; + + static void store_at_init(oop base, ptrdiff_t offset, T value); + + static inline void store_at(oop base, ptrdiff_t offset, T value) { + _store_at_func(base, offset, value); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _load_func; + + static T load_init(void* addr); + + static inline T load(void* addr) { + return _load_func(addr); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _load_at_func; + + static T load_at_init(oop base, ptrdiff_t offset); + + static inline T load_at(oop base, ptrdiff_t offset) { + return _load_at_func(base, offset); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _atomic_cmpxchg_func; + + static T atomic_cmpxchg_init(T new_value, void* addr, T compare_value); + + static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) { + return _atomic_cmpxchg_func(new_value, addr, compare_value); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _atomic_cmpxchg_at_func; + + static T atomic_cmpxchg_at_init(T new_value, oop base, ptrdiff_t offset, T compare_value); + + static inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { + return _atomic_cmpxchg_at_func(new_value, base, offset, compare_value); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _atomic_xchg_func; + + static T atomic_xchg_init(T new_value, void* addr); + + static inline T atomic_xchg(T new_value, void* addr) { + return _atomic_xchg_func(new_value, addr); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _atomic_xchg_at_func; + + static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset); + + static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { + return _atomic_xchg_at_func(new_value, base, offset); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _arraycopy_func; + + static bool arraycopy_init(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length); + + static inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) { + return _arraycopy_func(src_obj, dst_obj, src, dst, length); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _clone_func; + + static void clone_init(oop src, oop dst, size_t size); + + static inline void clone(oop src, oop dst, size_t size) { + _clone_func(src, dst, size); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _resolve_func; + + static oop resolve_init(oop obj); + + static inline oop resolve(oop obj) { + return _resolve_func(obj); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _equals_func; + + static bool equals_init(oop o1, oop o2); + + static inline bool equals(oop o1, oop o2) { + return _equals_func(o1, o2); + } + }; + + // Initialize the function pointers to point to the resolving function. + template + typename AccessFunction::type + RuntimeDispatch::_store_func = &store_init; + + template + typename AccessFunction::type + RuntimeDispatch::_store_at_func = &store_at_init; + + template + typename AccessFunction::type + RuntimeDispatch::_load_func = &load_init; + + template + typename AccessFunction::type + RuntimeDispatch::_load_at_func = &load_at_init; + + template + typename AccessFunction::type + RuntimeDispatch::_atomic_cmpxchg_func = &atomic_cmpxchg_init; + + template + typename AccessFunction::type + RuntimeDispatch::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init; + + template + typename AccessFunction::type + RuntimeDispatch::_atomic_xchg_func = &atomic_xchg_init; + + template + typename AccessFunction::type + RuntimeDispatch::_atomic_xchg_at_func = &atomic_xchg_at_init; + + template + typename AccessFunction::type + RuntimeDispatch::_arraycopy_func = &arraycopy_init; + + template + typename AccessFunction::type + RuntimeDispatch::_clone_func = &clone_init; + + template + typename AccessFunction::type + RuntimeDispatch::_resolve_func = &resolve_init; + + template + typename AccessFunction::type + RuntimeDispatch::_equals_func = &equals_init; + + // Step 3: Pre-runtime dispatching. + // The PreRuntimeDispatch class is responsible for filtering the barrier strength + // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime + // dispatch point. Otherwise it goes through a runtime check if hardwiring was + // not possible. + struct PreRuntimeDispatch: AllStatic { + template + struct CanHardwireRaw: public IntegralConstant< + bool, + !HasDecorator::value || // primitive access + !HasDecorator::value || // don't care about compressed oops (oop* address) + HasDecorator::value> // we can infer we use compressed oops (narrowOop* address) + {}; + + static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP; + + template + static bool is_hardwired_primitive() { + return !HasDecorator::value && + !HasDecorator::value; + } + + template + inline static typename EnableIf< + HasDecorator::value && CanHardwireRaw::value>::type + store(void* addr, T value) { + typedef RawAccessBarrier Raw; + if (HasDecorator::value) { + Raw::oop_store(addr, value); + } else { + Raw::store(addr, value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && !CanHardwireRaw::value>::type + store(void* addr, T value) { + if (UseCompressedOops) { + const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; + PreRuntimeDispatch::store(addr, value); + } else { + const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; + PreRuntimeDispatch::store(addr, value); + } + } + + template + inline static typename EnableIf< + !HasDecorator::value>::type + store(void* addr, T value) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + PreRuntimeDispatch::store(addr, value); + } else { + RuntimeDispatch::store(addr, value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value>::type + store_at(oop base, ptrdiff_t offset, T value) { + store(field_addr(base, offset), value); + } + + template + inline static typename EnableIf< + !HasDecorator::value>::type + store_at(oop base, ptrdiff_t offset, T value) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + PreRuntimeDispatch::store_at(base, offset, value); + } else { + RuntimeDispatch::store_at(base, offset, value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && CanHardwireRaw::value, T>::type + load(void* addr) { + typedef RawAccessBarrier Raw; + if (HasDecorator::value) { + return Raw::template oop_load(addr); + } else { + return Raw::template load(addr); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && !CanHardwireRaw::value, T>::type + load(void* addr) { + if (UseCompressedOops) { + const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; + return PreRuntimeDispatch::load(addr); + } else { + const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; + return PreRuntimeDispatch::load(addr); + } + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + load(void* addr) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::load(addr); + } else { + return RuntimeDispatch::load(addr); + } + } + + template + inline static typename EnableIf< + HasDecorator::value, T>::type + load_at(oop base, ptrdiff_t offset) { + return load(field_addr(base, offset)); + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + load_at(oop base, ptrdiff_t offset) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::load_at(base, offset); + } else { + return RuntimeDispatch::load_at(base, offset); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && CanHardwireRaw::value, T>::type + atomic_cmpxchg(T new_value, void* addr, T compare_value) { + typedef RawAccessBarrier Raw; + if (HasDecorator::value) { + return Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); + } else { + return Raw::atomic_cmpxchg(new_value, addr, compare_value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && !CanHardwireRaw::value, T>::type + atomic_cmpxchg(T new_value, void* addr, T compare_value) { + if (UseCompressedOops) { + const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } else { + const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + atomic_cmpxchg(T new_value, void* addr, T compare_value) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } else { + return RuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value, T>::type + atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { + return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::atomic_cmpxchg_at(new_value, base, offset, compare_value); + } else { + return RuntimeDispatch::atomic_cmpxchg_at(new_value, base, offset, compare_value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && CanHardwireRaw::value, T>::type + atomic_xchg(T new_value, void* addr) { + typedef RawAccessBarrier Raw; + if (HasDecorator::value) { + return Raw::oop_atomic_xchg(new_value, addr); + } else { + return Raw::atomic_xchg(new_value, addr); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && !CanHardwireRaw::value, T>::type + atomic_xchg(T new_value, void* addr) { + if (UseCompressedOops) { + const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } else { + const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + atomic_xchg(T new_value, void* addr) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } else { + return RuntimeDispatch::atomic_xchg(new_value, addr); + } + } + + template + inline static typename EnableIf< + HasDecorator::value, T>::type + atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { + return atomic_xchg(new_value, field_addr(base, offset)); + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::atomic_xchg(new_value, base, offset); + } else { + return RuntimeDispatch::atomic_xchg_at(new_value, base, offset); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && CanHardwireRaw::value, bool>::type + arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + typedef RawAccessBarrier Raw; + if (HasDecorator::value) { + return Raw::oop_arraycopy(src_obj, dst_obj, src, dst, length); + } else { + return Raw::arraycopy(src_obj, dst_obj, src, dst, length); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && !CanHardwireRaw::value, bool>::type + arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + if (UseCompressedOops) { + const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } else { + const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } + } + + template + inline static typename EnableIf< + !HasDecorator::value, bool>::type + arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } else { + return RuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } + } + + template + inline static typename EnableIf< + HasDecorator::value>::type + clone(oop src, oop dst, size_t size) { + typedef RawAccessBarrier Raw; + Raw::clone(src, dst, size); + } + + template + inline static typename EnableIf< + !HasDecorator::value>::type + clone(oop src, oop dst, size_t size) { + RuntimeDispatch::clone(src, dst, size); + } + + template + inline static typename EnableIf< + HasDecorator::value, oop>::type + resolve(oop obj) { + typedef RawAccessBarrier Raw; + return Raw::resolve(obj); + } + + template + inline static typename EnableIf< + !HasDecorator::value, oop>::type + resolve(oop obj) { + return RuntimeDispatch::resolve(obj); + } + + template + inline static typename EnableIf< + HasDecorator::value, bool>::type + equals(oop o1, oop o2) { + typedef RawAccessBarrier Raw; + return Raw::equals(o1, o2); + } + + template + inline static typename EnableIf< + !HasDecorator::value, bool>::type + equals(oop o1, oop o2) { + return RuntimeDispatch::equals(o1, o2); + } + }; + + // This class adds implied decorators that follow according to decorator rules. + // For example adding default reference strength and default memory ordering + // semantics. + template + struct DecoratorFixup: AllStatic { + // If no reference strength has been picked, then strong will be picked + static const DecoratorSet ref_strength_default = input_decorators | + (((ON_DECORATOR_MASK & input_decorators) == 0 && (INTERNAL_VALUE_IS_OOP & input_decorators) != 0) ? + ON_STRONG_OOP_REF : INTERNAL_EMPTY); + // If no memory ordering has been picked, unordered will be picked + static const DecoratorSet memory_ordering_default = ref_strength_default | + ((MO_DECORATOR_MASK & ref_strength_default) == 0 ? MO_UNORDERED : INTERNAL_EMPTY); + // If no barrier strength has been picked, normal will be used + static const DecoratorSet barrier_strength_default = memory_ordering_default | + ((AS_DECORATOR_MASK & memory_ordering_default) == 0 ? AS_NORMAL : INTERNAL_EMPTY); + // Heap array accesses imply it is a heap access + static const DecoratorSet heap_array_is_in_heap = barrier_strength_default | + ((IN_HEAP_ARRAY & barrier_strength_default) != 0 ? IN_HEAP : INTERNAL_EMPTY); + static const DecoratorSet conc_root_is_root = heap_array_is_in_heap | + ((IN_CONCURRENT_ROOT & heap_array_is_in_heap) != 0 ? IN_ROOT : INTERNAL_EMPTY); + static const DecoratorSet archive_root_is_root = conc_root_is_root | + ((IN_ARCHIVE_ROOT & conc_root_is_root) != 0 ? IN_ROOT : INTERNAL_EMPTY); + static const DecoratorSet value = archive_root_is_root | BT_BUILDTIME_DECORATORS; + }; + + // Step 2: Reduce types. + // Enforce that for non-oop types, T and P have to be strictly the same. + // P is the type of the address and T is the type of the values. + // As for oop types, it is allow to send T in {narrowOop, oop} and + // P in {narrowOop, oop, HeapWord*}. The following rules apply according to + // the subsequent table. (columns are P, rows are T) + // | | HeapWord | oop | narrowOop | + // | oop | rt-comp | hw-none | hw-comp | + // | narrowOop | x | x | hw-none | + // + // x means not allowed + // rt-comp means it must be checked at runtime whether the oop is compressed. + // hw-none means it is statically known the oop will not be compressed. + // hw-comp means it is statically known the oop will be compressed. + + template + inline void store_reduce_types(T* addr, T value) { + PreRuntimeDispatch::store(addr, value); + } + + template + inline void store_reduce_types(narrowOop* addr, oop value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + PreRuntimeDispatch::store(addr, value); + } + + template + inline void store_reduce_types(narrowOop* addr, narrowOop value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + PreRuntimeDispatch::store(addr, value); + } + + template + inline void store_reduce_types(HeapWord* addr, oop value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; + PreRuntimeDispatch::store(addr, value); + } + + template + inline T atomic_cmpxchg_reduce_types(T new_value, T* addr, T compare_value) { + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + + template + inline oop atomic_cmpxchg_reduce_types(oop new_value, narrowOop* addr, oop compare_value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + + template + inline narrowOop atomic_cmpxchg_reduce_types(narrowOop new_value, narrowOop* addr, narrowOop compare_value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + + template + inline oop atomic_cmpxchg_reduce_types(oop new_value, + HeapWord* addr, + oop compare_value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + + template + inline T atomic_xchg_reduce_types(T new_value, T* addr) { + const DecoratorSet expanded_decorators = decorators; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } + + template + inline oop atomic_xchg_reduce_types(oop new_value, narrowOop* addr) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } + + template + inline narrowOop atomic_xchg_reduce_types(narrowOop new_value, narrowOop* addr) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } + + template + inline oop atomic_xchg_reduce_types(oop new_value, HeapWord* addr) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } + + template + inline T load_reduce_types(T* addr) { + return PreRuntimeDispatch::load(addr); + } + + template + inline typename OopOrNarrowOop::type load_reduce_types(narrowOop* addr) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::load::type>(addr); + } + + template + inline oop load_reduce_types(HeapWord* addr) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; + return PreRuntimeDispatch::load(addr); + } + + template + inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } + + template + inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, HeapWord* src, HeapWord* dst, size_t length) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } + + template + inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, narrowOop* src, narrowOop* dst, size_t length) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } + + // Step 1: Set default decorators. This step remembers if a type was volatile + // and then sets the MO_VOLATILE decorator by default. Otherwise, a default + // memory ordering is set for the access, and the implied decorator rules + // are applied to select sensible defaults for decorators that have not been + // explicitly set. For example, default object referent strength is set to strong. + // This step also decays the types passed in (e.g. getting rid of CV qualifiers + // and references from the types). This step also perform some type verification + // that the passed in types make sense. + + template + static void verify_types(){ + // If this fails to compile, then you have sent in something that is + // not recognized as a valid primitive type to a primitive Access function. + STATIC_ASSERT((HasDecorator::value || // oops have already been validated + (IsPointer::value || IsIntegral::value) || + IsFloatingPoint::value)); // not allowed primitive type + } + + template + inline void store(P* addr, T value) { + verify_types(); + typedef typename Decay

::type DecayedP; + typedef typename Decay::type DecayedT; + DecayedT decayed_value = value; + // If a volatile address is passed in but no memory ordering decorator, + // set the memory ordering to MO_VOLATILE by default. + const DecoratorSet expanded_decorators = DecoratorFixup< + (IsVolatile

::value && !HasDecorator::value) ? + (MO_VOLATILE | decorators) : decorators>::value; + store_reduce_types(const_cast(addr), decayed_value); + } + + template + inline void store_at(oop base, ptrdiff_t offset, T value) { + verify_types(); + typedef typename Decay::type DecayedT; + DecayedT decayed_value = value; + const DecoratorSet expanded_decorators = DecoratorFixup::value ? + INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; + PreRuntimeDispatch::store_at(base, offset, decayed_value); + } + + template + inline T load(P* addr) { + verify_types(); + typedef typename Decay

::type DecayedP; + typedef typename Conditional::value, + typename OopOrNarrowOop::type, + typename Decay::type>::type DecayedT; + // If a volatile address is passed in but no memory ordering decorator, + // set the memory ordering to MO_VOLATILE by default. + const DecoratorSet expanded_decorators = DecoratorFixup< + (IsVolatile

::value && !HasDecorator::value) ? + (MO_VOLATILE | decorators) : decorators>::value; + return load_reduce_types(const_cast(addr)); + } + + template + inline T load_at(oop base, ptrdiff_t offset) { + verify_types(); + typedef typename Conditional::value, + typename OopOrNarrowOop::type, + typename Decay::type>::type DecayedT; + // Expand the decorators (figure out sensible defaults) + // Potentially remember if we need compressed oop awareness + const DecoratorSet expanded_decorators = DecoratorFixup::value ? + INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; + return PreRuntimeDispatch::load_at(base, offset); + } + + template + inline T atomic_cmpxchg(T new_value, P* addr, T compare_value) { + verify_types(); + typedef typename Decay

::type DecayedP; + typedef typename Decay::type DecayedT; + DecayedT new_decayed_value = new_value; + DecayedT compare_decayed_value = compare_value; + const DecoratorSet expanded_decorators = DecoratorFixup< + (!HasDecorator::value) ? + (MO_SEQ_CST | decorators) : decorators>::value; + return atomic_cmpxchg_reduce_types(new_decayed_value, + const_cast(addr), + compare_decayed_value); + } + + template + inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { + verify_types(); + typedef typename Decay::type DecayedT; + DecayedT new_decayed_value = new_value; + DecayedT compare_decayed_value = compare_value; + // Determine default memory ordering + const DecoratorSet expanded_decorators = DecoratorFixup< + (!HasDecorator::value) ? + (MO_SEQ_CST | decorators) : decorators>::value; + // Potentially remember that we need compressed oop awareness + const DecoratorSet final_decorators = expanded_decorators | + (HasDecorator::value ? + INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY); + return PreRuntimeDispatch::atomic_cmpxchg_at(new_decayed_value, base, + offset, compare_decayed_value); + } + + template + inline T atomic_xchg(T new_value, P* addr) { + verify_types(); + typedef typename Decay

::type DecayedP; + typedef typename Decay::type DecayedT; + DecayedT new_decayed_value = new_value; + // atomic_xchg is only available in SEQ_CST flavour. + const DecoratorSet expanded_decorators = DecoratorFixup::value; + return atomic_xchg_reduce_types(new_decayed_value, + const_cast(addr)); + } + + template + inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { + verify_types(); + typedef typename Decay::type DecayedT; + DecayedT new_decayed_value = new_value; + // atomic_xchg is only available in SEQ_CST flavour. + const DecoratorSet expanded_decorators = DecoratorFixup::value ? + INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; + return PreRuntimeDispatch::atomic_xchg_at(new_decayed_value, base, offset); + } + + template + inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + STATIC_ASSERT((HasDecorator::value || + (IsSame::value || IsIntegral::value) || + IsFloatingPoint::value)); // arraycopy allows type erased void elements + typedef typename Decay::type DecayedT; + const DecoratorSet expanded_decorators = DecoratorFixup::value; + return arraycopy_reduce_types(src_obj, dst_obj, + const_cast(src), + const_cast(dst), + length); + } + + template + inline void clone(oop src, oop dst, size_t size) { + const DecoratorSet expanded_decorators = DecoratorFixup::value; + PreRuntimeDispatch::clone(src, dst, size); + } + + template + inline oop resolve(oop obj) { + const DecoratorSet expanded_decorators = DecoratorFixup::value; + return PreRuntimeDispatch::resolve(obj); + } + + template + inline bool equals(oop o1, oop o2) { + const DecoratorSet expanded_decorators = DecoratorFixup::value; + return PreRuntimeDispatch::equals(o1, o2); + } +} + +#endif // SHARE_OOPS_ACCESSBACKEND_HPP