diff --git a/src/hotspot/share/oops/access.cpp b/src/hotspot/share/oops/access.cpp new file mode 100644 index 0000000..5234f44 --- /dev/null +++ b/src/hotspot/share/oops/access.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "oops/access.inline.hpp" +#include "oops/accessDecorators.hpp" + +// This macro allows instantiating selected accesses to be usable from the +// access.hpp file, to break dependencies to the access.inline.hpp file. +#define INSTANTIATE_HPP_ACCESS(decorators, T, barrier_type) \ + template struct RuntimeDispatch::value, T, barrier_type> + +namespace AccessInternal { + INSTANTIATE_HPP_ACCESS(INTERNAL_EMPTY, oop, BARRIER_EQUALS); +} diff --git a/src/hotspot/share/oops/access.hpp b/src/hotspot/share/oops/access.hpp index 5ccfcbd..7c9c38a 100644 --- a/src/hotspot/share/oops/access.hpp +++ b/src/hotspot/share/oops/access.hpp @@ -22,16 +22,17 @@ * */ -#ifndef SHARE_VM_RUNTIME_ACCESS_HPP -#define SHARE_VM_RUNTIME_ACCESS_HPP +#ifndef SHARE_OOPS_ACCESS_HPP +#define SHARE_OOPS_ACCESS_HPP #include "memory/allocation.hpp" -#include "metaprogramming/decay.hpp" -#include "metaprogramming/integralConstant.hpp" +#include "oops/accessBackend.hpp" +#include "oops/accessDecorators.hpp" #include "oops/oopsHierarchy.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" + // = GENERAL = // Access is an API for performing accesses with declarative semantics. Each access can have a number of "decorators". // A decorator is an attribute or property that affects the way a memory access is performed in some way. @@ -39,11 +40,12 @@ // e.g. strength of references, strength of GC barriers, or whether compression should be applied or not. // Some decorators are set at buildtime, such as whether primitives require GC barriers or not, others // at callsites such as whether an access is in the heap or not, and others are resolved at runtime -// such as GC-specific barriers and encoding/decoding compressed oops. +// such as GC-specific barriers and encoding/decoding compressed oops. For more information about what +// decorators are available, cf. oops/accessDecorators.hpp. // By pipelining handling of these decorators, the design of the Access API allows separation of concern // over the different orthogonal concerns of decorators, while providing a powerful way of // expressing these orthogonal semantic properties in a unified way. - +// // == OPERATIONS == // * load: Load a value from an address. // * load_at: Load a value from an internal pointer relative to a base object. @@ -57,220 +59,40 @@ // * clone: Clone the contents of an object to a newly allocated object. // * resolve: Resolve a stable to-space invariant oop that is guaranteed not to relocate its payload until a subsequent thread transition. // * equals: Object equality, e.g. when different copies of the same objects are in use (from-space vs. to-space) -typedef uint64_t DecoratorSet; - -// == Internal Decorators - do not use == -// * INTERNAL_EMPTY: This is the name for the empty decorator set (in absence of other decorators). -// * INTERNAL_CONVERT_COMPRESSED_OOPS: This is an oop access that will require converting an oop -// to a narrowOop or vice versa, if UseCompressedOops is known to be set. -// * INTERNAL_VALUE_IS_OOP: Remember that the involved access is on oop rather than primitive. -const DecoratorSet INTERNAL_EMPTY = UCONST64(0); -const DecoratorSet INTERNAL_CONVERT_COMPRESSED_OOP = UCONST64(1) << 1; -const DecoratorSet INTERNAL_VALUE_IS_OOP = UCONST64(1) << 2; - -// == Internal build-time Decorators == -// * INTERNAL_BT_BARRIER_ON_PRIMITIVES: This is set in the barrierSetConfig.hpp file. -// * INTERNAL_BT_TO_SPACE_INVARIANT: This is set in the barrierSetConfig.hpp file iff -// no GC is bundled in the build that is to-space invariant. -const DecoratorSet INTERNAL_BT_BARRIER_ON_PRIMITIVES = UCONST64(1) << 3; -const DecoratorSet INTERNAL_BT_TO_SPACE_INVARIANT = UCONST64(1) << 4; - -// == Internal run-time Decorators == -// * INTERNAL_RT_USE_COMPRESSED_OOPS: This decorator will be set in runtime resolved -// access backends iff UseCompressedOops is true. -const DecoratorSet INTERNAL_RT_USE_COMPRESSED_OOPS = UCONST64(1) << 5; - -const DecoratorSet INTERNAL_DECORATOR_MASK = INTERNAL_CONVERT_COMPRESSED_OOP | INTERNAL_VALUE_IS_OOP | - INTERNAL_BT_BARRIER_ON_PRIMITIVES | INTERNAL_RT_USE_COMPRESSED_OOPS; - -// == Memory Ordering Decorators == -// The memory ordering decorators can be described in the following way: -// === Decorator Rules === -// The different types of memory ordering guarantees have a strict order of strength. -// Explicitly specifying the stronger ordering implies that the guarantees of the weaker -// property holds too. The names come from the C++11 atomic operations, and typically -// have a JMM equivalent property. -// The equivalence may be viewed like this: -// MO_UNORDERED is equivalent to JMM plain. -// MO_VOLATILE has no equivalence in JMM, because it's a C++ thing. -// MO_RELAXED is equivalent to JMM opaque. -// MO_ACQUIRE is equivalent to JMM acquire. -// MO_RELEASE is equivalent to JMM release. -// MO_SEQ_CST is equivalent to JMM volatile. // -// === Stores === -// * MO_UNORDERED (Default): No guarantees. -// - The compiler and hardware are free to reorder aggressively. And they will. -// * MO_VOLATILE: Volatile stores (in the C++ sense). -// - The stores are not reordered by the compiler (but possibly the HW) w.r.t. other -// volatile accesses in program order (but possibly non-volatile accesses). -// * MO_RELAXED: Relaxed atomic stores. -// - The stores are atomic. -// - Guarantees from volatile stores hold. -// * MO_RELEASE: Releasing stores. -// - The releasing store will make its preceding memory accesses observable to memory accesses -// subsequent to an acquiring load observing this releasing store. -// - Guarantees from relaxed stores hold. -// * MO_SEQ_CST: Sequentially consistent stores. -// - The stores are observed in the same order by MO_SEQ_CST loads on other processors -// - Preceding loads and stores in program order are not reordered with subsequent loads and stores in program order. -// - Guarantees from releasing stores hold. -// === Loads === -// * MO_UNORDERED (Default): No guarantees -// - The compiler and hardware are free to reorder aggressively. And they will. -// * MO_VOLATILE: Volatile loads (in the C++ sense). -// - The loads are not reordered by the compiler (but possibly the HW) w.r.t. other -// volatile accesses in program order (but possibly non-volatile accesses). -// * MO_RELAXED: Relaxed atomic loads. -// - The stores are atomic. -// - Guarantees from volatile loads hold. -// * MO_ACQUIRE: Acquiring loads. -// - An acquiring load will make subsequent memory accesses observe the memory accesses -// preceding the releasing store that the acquiring load observed. -// - Guarantees from relaxed loads hold. -// * MO_SEQ_CST: Sequentially consistent loads. -// - These loads observe MO_SEQ_CST stores in the same order on other processors -// - Preceding loads and stores in program order are not reordered with subsequent loads and stores in program order. -// - Guarantees from acquiring loads hold. -// === Atomic Cmpxchg === -// * MO_RELAXED: Atomic but relaxed cmpxchg. -// - Guarantees from MO_RELAXED loads and MO_RELAXED stores hold unconditionally. -// * MO_SEQ_CST: Sequentially consistent cmpxchg. -// - Guarantees from MO_SEQ_CST loads and MO_SEQ_CST stores hold unconditionally. -// === Atomic Xchg === -// * MO_RELAXED: Atomic but relaxed atomic xchg. -// - Guarantees from MO_RELAXED loads and MO_RELAXED stores hold. -// * MO_SEQ_CST: Sequentially consistent xchg. -// - Guarantees from MO_SEQ_CST loads and MO_SEQ_CST stores hold. -const DecoratorSet MO_UNORDERED = UCONST64(1) << 6; -const DecoratorSet MO_VOLATILE = UCONST64(1) << 7; -const DecoratorSet MO_RELAXED = UCONST64(1) << 8; -const DecoratorSet MO_ACQUIRE = UCONST64(1) << 9; -const DecoratorSet MO_RELEASE = UCONST64(1) << 10; -const DecoratorSet MO_SEQ_CST = UCONST64(1) << 11; -const DecoratorSet MO_DECORATOR_MASK = MO_UNORDERED | MO_VOLATILE | MO_RELAXED | - MO_ACQUIRE | MO_RELEASE | MO_SEQ_CST; - -// === Barrier Strength Decorators === -// * AS_RAW: The access will translate into a raw memory access, hence ignoring all semantic concerns -// except memory ordering and compressed oops. This will bypass runtime function pointer dispatching -// in the pipeline and hardwire to raw accesses without going trough the GC access barriers. -// - Accesses on oop* translate to raw memory accesses without runtime checks -// - Accesses on narrowOop* translate to encoded/decoded memory accesses without runtime checks -// - Accesses on HeapWord* translate to a runtime check choosing one of the above -// - Accesses on other types translate to raw memory accesses without runtime checks -// * AS_DEST_NOT_INITIALIZED: This property can be important to e.g. SATB barriers by -// marking that the previous value is uninitialized nonsense rather than a real value. -// * AS_NO_KEEPALIVE: The barrier is used only on oop references and will not keep any involved objects -// alive, regardless of the type of reference being accessed. It will however perform the memory access -// in a consistent way w.r.t. e.g. concurrent compaction, so that the right field is being accessed, -// or maintain, e.g. intergenerational or interregional pointers if applicable. This should be used with -// extreme caution in isolated scopes. -// * AS_NORMAL: The accesses will be resolved to an accessor on the BarrierSet class, giving the -// responsibility of performing the access and what barriers to be performed to the GC. This is the default. -// Note that primitive accesses will only be resolved on the barrier set if the appropriate build-time -// decorator for enabling primitive barriers is enabled for the build. -const DecoratorSet AS_RAW = UCONST64(1) << 12; -const DecoratorSet AS_DEST_NOT_INITIALIZED = UCONST64(1) << 13; -const DecoratorSet AS_NO_KEEPALIVE = UCONST64(1) << 14; -const DecoratorSet AS_NORMAL = UCONST64(1) << 15; -const DecoratorSet AS_DECORATOR_MASK = AS_RAW | AS_DEST_NOT_INITIALIZED | - AS_NO_KEEPALIVE | AS_NORMAL; - -// === Reference Strength Decorators === -// These decorators only apply to accesses on oop-like types (oop/narrowOop). -// * ON_STRONG_OOP_REF: Memory access is performed on a strongly reachable reference. -// * ON_WEAK_OOP_REF: The memory access is performed on a weakly reachable reference. -// * ON_PHANTOM_OOP_REF: The memory access is performed on a phantomly reachable reference. -// This is the same ring of strength as jweak and weak oops in the VM. -// * ON_UNKNOWN_OOP_REF: The memory access is performed on a reference of unknown strength. -// This could for example come from the unsafe API. -// * Default (no explicit reference strength specified): ON_STRONG_OOP_REF -const DecoratorSet ON_STRONG_OOP_REF = UCONST64(1) << 16; -const DecoratorSet ON_WEAK_OOP_REF = UCONST64(1) << 17; -const DecoratorSet ON_PHANTOM_OOP_REF = UCONST64(1) << 18; -const DecoratorSet ON_UNKNOWN_OOP_REF = UCONST64(1) << 19; -const DecoratorSet ON_DECORATOR_MASK = ON_STRONG_OOP_REF | ON_WEAK_OOP_REF | - ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF; - -// === Access Location === -// Accesses can take place in, e.g. the heap, old or young generation and different native roots. -// The location is important to the GC as it may imply different actions. The following decorators are used: -// * IN_HEAP: The access is performed in the heap. Many barriers such as card marking will -// be omitted if this decorator is not set. -// * IN_HEAP_ARRAY: The access is performed on a heap allocated array. This is sometimes a special case -// for some GCs, and implies that it is an IN_HEAP. -// * IN_ROOT: The access is performed in an off-heap data structure pointing into the Java heap. -// * IN_CONCURRENT_ROOT: The access is performed in an off-heap data structure pointing into the Java heap, -// but is notably not scanned during safepoints. This is sometimes a special case for some GCs and -// implies that it is also an IN_ROOT. -const DecoratorSet IN_HEAP = UCONST64(1) << 20; -const DecoratorSet IN_HEAP_ARRAY = UCONST64(1) << 21; -const DecoratorSet IN_ROOT = UCONST64(1) << 22; -const DecoratorSet IN_CONCURRENT_ROOT = UCONST64(1) << 23; -const DecoratorSet IN_ARCHIVE_ROOT = UCONST64(1) << 24; -const DecoratorSet IN_DECORATOR_MASK = IN_HEAP | IN_HEAP_ARRAY | - IN_ROOT | IN_CONCURRENT_ROOT | - IN_ARCHIVE_ROOT; - -// == Value Decorators == -// * OOP_NOT_NULL: This property can make certain barriers faster such as compressing oops. -const DecoratorSet OOP_NOT_NULL = UCONST64(1) << 25; -const DecoratorSet OOP_DECORATOR_MASK = OOP_NOT_NULL; - -// == Arraycopy Decorators == -// * ARRAYCOPY_CHECKCAST: This property means that the class of the objects in source -// are not guaranteed to be subclasses of the class of the destination array. This requires -// a check-cast barrier during the copying operation. If this is not set, it is assumed -// that the array is covariant: (the source array type is-a destination array type) -// * ARRAYCOPY_DISJOINT: This property means that it is known that the two array ranges -// are disjoint. -// * ARRAYCOPY_ARRAYOF: The copy is in the arrayof form. -// * ARRAYCOPY_ATOMIC: The accesses have to be atomic over the size of its elements. -// * ARRAYCOPY_ALIGNED: The accesses have to be aligned on a HeapWord. -const DecoratorSet ARRAYCOPY_CHECKCAST = UCONST64(1) << 26; -const DecoratorSet ARRAYCOPY_DISJOINT = UCONST64(1) << 27; -const DecoratorSet ARRAYCOPY_ARRAYOF = UCONST64(1) << 28; -const DecoratorSet ARRAYCOPY_ATOMIC = UCONST64(1) << 29; -const DecoratorSet ARRAYCOPY_ALIGNED = UCONST64(1) << 30; -const DecoratorSet ARRAYCOPY_DECORATOR_MASK = ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT | - ARRAYCOPY_DISJOINT | ARRAYCOPY_ARRAYOF | - ARRAYCOPY_ATOMIC | ARRAYCOPY_ALIGNED; - -// The HasDecorator trait can help at compile-time determining whether a decorator set -// has an intersection with a certain other decorator set -template -struct HasDecorator: public IntegralConstant {}; +// == IMPLEMENTATION == +// Each access goes through the following steps in a template pipeline. +// There are essentially 5 steps for each access: +// * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers +// and sets default decorators to sensible values. +// * Step 2: Reduce types. This step makes sure there is only a single T type and not +// multiple types. The P type of the address and T type of the value must +// match. +// * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be +// avoided, and in that case avoids it (calling raw accesses or +// primitive accesses in a build that does not require primitive GC barriers) +// * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding +// BarrierSet::AccessBarrier accessor that attaches GC-required barriers +// to the access. +// * Step 5.a: Barrier resolution. This step is invoked the first time a runtime-dispatch +// happens for an access. The appropriate BarrierSet::AccessBarrier accessor +// is resolved, then the function pointer is updated to that accessor for +// future invocations. +// * Step 5.b: Post-runtime dispatch. This step now casts previously unknown types such +// as the address type of an oop on the heap (is it oop* or narrowOop*) to +// the appropriate type. It also splits sufficiently orthogonal accesses into +// different functions, such as whether the access involves oops or primitives +// and whether the access is performed on the heap or outside. Then the +// appropriate BarrierSet::AccessBarrier is called to perform the access. +// +// The implementation of step 1-4 resides in in accessBackend.hpp, to allow selected +// accesses to be accessible from only access.hpp, as opposed to access.inline.hpp. +// Steps 5.a and 5.b require knowledge about the GC backends, and therefore needs to +// include the various GC backend .inline.hpp headers. Their implementation resides in +// access.inline.hpp. The accesses that are allowed through the access.hpp file +// must be instantiated in access.cpp using the INSTANTIATE_HPP_ACCESS macro. namespace AccessInternal { - template - struct OopOrNarrowOopInternal: AllStatic { - typedef oop type; - }; - - template <> - struct OopOrNarrowOopInternal: AllStatic { - typedef narrowOop type; - }; - - // This metafunction returns a canonicalized oop/narrowOop type for a passed - // in oop-like types passed in from oop_* overloads where the user has sworn - // that the passed in values should be oop-like (e.g. oop, oopDesc*, arrayOop, - // narrowOoop, instanceOopDesc*, and random other things). - // In the oop_* overloads, it must hold that if the passed in type T is not - // narrowOop, then it by contract has to be one of many oop-like types implicitly - // convertible to oop, and hence returns oop as the canonical oop type. - // If it turns out it was not, then the implicit conversion to oop will fail - // to compile, as desired. - template - struct OopOrNarrowOop: AllStatic { - typedef typename OopOrNarrowOopInternal::type>::type type; - }; - - inline void* field_addr(oop base, ptrdiff_t byte_offset) { - return reinterpret_cast(reinterpret_cast((void*)base) + byte_offset); - } - template void store_at(oop base, ptrdiff_t offset, T value); @@ -579,4 +401,41 @@ class HeapAccess: public Access {}; template class RootAccess: public Access {}; -#endif // SHARE_VM_RUNTIME_ACCESS_HPP +template +template +void Access::verify_decorators() { + STATIC_ASSERT((~expected_decorators & decorators) == 0); // unexpected decorator used + const DecoratorSet barrier_strength_decorators = decorators & AS_DECORATOR_MASK; + STATIC_ASSERT(barrier_strength_decorators == 0 || ( // make sure barrier strength decorators are disjoint if set + (barrier_strength_decorators ^ AS_NO_KEEPALIVE) == 0 || + (barrier_strength_decorators ^ AS_DEST_NOT_INITIALIZED) == 0 || + (barrier_strength_decorators ^ AS_RAW) == 0 || + (barrier_strength_decorators ^ AS_NORMAL) == 0 + )); + const DecoratorSet ref_strength_decorators = decorators & ON_DECORATOR_MASK; + STATIC_ASSERT(ref_strength_decorators == 0 || ( // make sure ref strength decorators are disjoint if set + (ref_strength_decorators ^ ON_STRONG_OOP_REF) == 0 || + (ref_strength_decorators ^ ON_WEAK_OOP_REF) == 0 || + (ref_strength_decorators ^ ON_PHANTOM_OOP_REF) == 0 || + (ref_strength_decorators ^ ON_UNKNOWN_OOP_REF) == 0 + )); + const DecoratorSet memory_ordering_decorators = decorators & MO_DECORATOR_MASK; + STATIC_ASSERT(memory_ordering_decorators == 0 || ( // make sure memory ordering decorators are disjoint if set + (memory_ordering_decorators ^ MO_UNORDERED) == 0 || + (memory_ordering_decorators ^ MO_VOLATILE) == 0 || + (memory_ordering_decorators ^ MO_RELAXED) == 0 || + (memory_ordering_decorators ^ MO_ACQUIRE) == 0 || + (memory_ordering_decorators ^ MO_RELEASE) == 0 || + (memory_ordering_decorators ^ MO_SEQ_CST) == 0 + )); + const DecoratorSet location_decorators = decorators & IN_DECORATOR_MASK; + STATIC_ASSERT(location_decorators == 0 || ( // make sure location decorators are disjoint if set + (location_decorators ^ IN_ROOT) == 0 || + (location_decorators ^ IN_HEAP) == 0 || + (location_decorators ^ (IN_HEAP | IN_HEAP_ARRAY)) == 0 || + (location_decorators ^ (IN_ROOT | IN_CONCURRENT_ROOT)) == 0 || + (location_decorators ^ (IN_ROOT | IN_ARCHIVE_ROOT)) == 0 + )); +} + +#endif // SHARE_OOPS_ACCESS_HPP diff --git a/src/hotspot/share/oops/access.inline.hpp b/src/hotspot/share/oops/access.inline.hpp index c5b76d5..c02a0c7 100644 --- a/src/hotspot/share/oops/access.inline.hpp +++ b/src/hotspot/share/oops/access.inline.hpp @@ -22,43 +22,28 @@ * */ -#ifndef SHARE_VM_RUNTIME_ACCESS_INLINE_HPP -#define SHARE_VM_RUNTIME_ACCESS_INLINE_HPP +#ifndef SHARE_OOPS_ACCESS_INLINE_HPP +#define SHARE_OOPS_ACCESS_INLINE_HPP #include "gc/shared/barrierSetConfig.inline.hpp" -#include "metaprogramming/conditional.hpp" -#include "metaprogramming/isFloatingPoint.hpp" -#include "metaprogramming/isIntegral.hpp" -#include "metaprogramming/isPointer.hpp" -#include "metaprogramming/isVolatile.hpp" #include "oops/access.hpp" #include "oops/accessBackend.inline.hpp" -#include "runtime/atomic.hpp" -#include "runtime/orderAccess.inline.hpp" -// This file outlines the template pipeline of accesses going through the Access -// API. There are essentially 5 steps for each access. -// * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers -// and sets default decorators to sensible values. -// * Step 2: Reduce types. This step makes sure there is only a single T type and not -// multiple types. The P type of the address and T type of the value must -// match. -// * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be -// avoided, and in that case avoids it (calling raw accesses or -// primitive accesses in a build that does not require primitive GC barriers) -// * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding -// BarrierSet::AccessBarrier accessor that attaches GC-required barriers -// to the access. -// * Step 5: Post-runtime dispatch. This step now casts previously unknown types such -// as the address type of an oop on the heap (is it oop* or narrowOop*) to -// the appropriate type. It also splits sufficiently orthogonal accesses into -// different functions, such as whether the access involves oops or primitives -// and whether the access is performed on the heap or outside. Then the -// appropriate BarrierSet::AccessBarrier is called to perform the access. +// This file outlines the last 2 steps of the template pipeline of accesses going through +// the Access API. +// * Step 5.a: Barrier resolution. This step is invoked the first time a runtime-dispatch +// happens for an access. The appropriate BarrierSet::AccessBarrier accessor +// is resolved, then the function pointer is updated to that accessor for +// future invocations. +// * Step 5.b: Post-runtime dispatch. This step now casts previously unknown types such +// as the address type of an oop on the heap (is it oop* or narrowOop*) to +// the appropriate type. It also splits sufficiently orthogonal accesses into +// different functions, such as whether the access involves oops or primitives +// and whether the access is performed on the heap or outside. Then the +// appropriate BarrierSet::AccessBarrier is called to perform the access. namespace AccessInternal { - - // Step 5: Post-runtime dispatch. + // Step 5.b: Post-runtime dispatch. // This class is the last step before calling the BarrierSet::AccessBarrier. // Here we make sure to figure out types that were not known prior to the // runtime dispatch, such as whether an oop on the heap is oop or narrowOop. @@ -286,7 +271,7 @@ namespace AccessInternal { } }; - // Step 4: Runtime dispatch + // Step 5.a: Barrier resolution // The RuntimeDispatch class is responsible for performing a runtime dispatch of the // accessor. This is required when the access either depends on whether compressed oops // is being used, or it depends on which GC implementation was chosen (e.g. requires GC @@ -295,929 +280,89 @@ namespace AccessInternal { // it resolves which accessor to be used in future invocations and patches the // function pointer to this new accessor. - template - struct RuntimeDispatch: AllStatic {}; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _store_func; - - static void store_init(void* addr, T value) { - func_t function = BarrierResolver::resolve_barrier(); - _store_func = function; - function(addr, value); - } - - static inline void store(void* addr, T value) { - _store_func(addr, value); - } - }; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _store_at_func; - - static void store_at_init(oop base, ptrdiff_t offset, T value) { - func_t function = BarrierResolver::resolve_barrier(); - _store_at_func = function; - function(base, offset, value); - } - - static inline void store_at(oop base, ptrdiff_t offset, T value) { - _store_at_func(base, offset, value); - } - }; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _load_func; - - static T load_init(void* addr) { - func_t function = BarrierResolver::resolve_barrier(); - _load_func = function; - return function(addr); - } - - static inline T load(void* addr) { - return _load_func(addr); - } - }; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _load_at_func; - - static T load_at_init(oop base, ptrdiff_t offset) { - func_t function = BarrierResolver::resolve_barrier(); - _load_at_func = function; - return function(base, offset); - } - - static inline T load_at(oop base, ptrdiff_t offset) { - return _load_at_func(base, offset); - } - }; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _atomic_cmpxchg_func; - - static T atomic_cmpxchg_init(T new_value, void* addr, T compare_value) { - func_t function = BarrierResolver::resolve_barrier(); - _atomic_cmpxchg_func = function; - return function(new_value, addr, compare_value); - } - - static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) { - return _atomic_cmpxchg_func(new_value, addr, compare_value); - } - }; - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _atomic_cmpxchg_at_func; - - static T atomic_cmpxchg_at_init(T new_value, oop base, ptrdiff_t offset, T compare_value) { - func_t function = BarrierResolver::resolve_barrier(); - _atomic_cmpxchg_at_func = function; - return function(new_value, base, offset, compare_value); - } - - static inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { - return _atomic_cmpxchg_at_func(new_value, base, offset, compare_value); - } - }; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _atomic_xchg_func; - - static T atomic_xchg_init(T new_value, void* addr) { - func_t function = BarrierResolver::resolve_barrier(); - _atomic_xchg_func = function; - return function(new_value, addr); - } - - static inline T atomic_xchg(T new_value, void* addr) { - return _atomic_xchg_func(new_value, addr); - } - }; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _atomic_xchg_at_func; - - static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset) { - func_t function = BarrierResolver::resolve_barrier(); - _atomic_xchg_at_func = function; - return function(new_value, base, offset); - } - - static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { - return _atomic_xchg_at_func(new_value, base, offset); - } - }; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _arraycopy_func; - - static bool arraycopy_init(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) { - func_t function = BarrierResolver::resolve_barrier(); - _arraycopy_func = function; - return function(src_obj, dst_obj, src, dst, length); - } - - static inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) { - return _arraycopy_func(src_obj, dst_obj, src, dst, length); - } - }; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _clone_func; - - static void clone_init(oop src, oop dst, size_t size) { - func_t function = BarrierResolver::resolve_barrier(); - _clone_func = function; - function(src, dst, size); - } - - static inline void clone(oop src, oop dst, size_t size) { - _clone_func(src, dst, size); - } - }; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _resolve_func; - - static oop resolve_init(oop obj) { - func_t function = BarrierResolver::resolve_barrier(); - _resolve_func = function; - return function(obj); - } - - static inline oop resolve(oop obj) { - return _resolve_func(obj); - } - }; - - template - struct RuntimeDispatch: AllStatic { - typedef typename AccessFunction::type func_t; - static func_t _equals_func; - - static bool equals_init(oop o1, oop o2) { - func_t function = BarrierResolver::resolve_barrier(); - _equals_func = function; - return function(o1, o2); - } - - static inline bool equals(oop o1, oop o2) { - return _equals_func(o1, o2); - } - }; - - // Initialize the function pointers to point to the resolving function. - template - typename AccessFunction::type - RuntimeDispatch::_store_func = &store_init; - - template - typename AccessFunction::type - RuntimeDispatch::_store_at_func = &store_at_init; - - template - typename AccessFunction::type - RuntimeDispatch::_load_func = &load_init; - - template - typename AccessFunction::type - RuntimeDispatch::_load_at_func = &load_at_init; - - template - typename AccessFunction::type - RuntimeDispatch::_atomic_cmpxchg_func = &atomic_cmpxchg_init; - - template - typename AccessFunction::type - RuntimeDispatch::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init; - - template - typename AccessFunction::type - RuntimeDispatch::_atomic_xchg_func = &atomic_xchg_init; - - template - typename AccessFunction::type - RuntimeDispatch::_atomic_xchg_at_func = &atomic_xchg_at_init; - - template - typename AccessFunction::type - RuntimeDispatch::_arraycopy_func = &arraycopy_init; - - template - typename AccessFunction::type - RuntimeDispatch::_clone_func = &clone_init; - - template - typename AccessFunction::type - RuntimeDispatch::_resolve_func = &resolve_init; - - template - typename AccessFunction::type - RuntimeDispatch::_equals_func = &equals_init; - - // Step 3: Pre-runtime dispatching. - // The PreRuntimeDispatch class is responsible for filtering the barrier strength - // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime - // dispatch point. Otherwise it goes through a runtime check if hardwiring was - // not possible. - struct PreRuntimeDispatch: AllStatic { - template - struct CanHardwireRaw: public IntegralConstant< - bool, - !HasDecorator::value || // primitive access - !HasDecorator::value || // don't care about compressed oops (oop* address) - HasDecorator::value> // we can infer we use compressed oops (narrowOop* address) - {}; - - static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP; - - template - static bool is_hardwired_primitive() { - return !HasDecorator::value && - !HasDecorator::value; - } - - template - inline static typename EnableIf< - HasDecorator::value && CanHardwireRaw::value>::type - store(void* addr, T value) { - typedef RawAccessBarrier Raw; - if (HasDecorator::value) { - Raw::oop_store(addr, value); - } else { - Raw::store(addr, value); - } - } - - template - inline static typename EnableIf< - HasDecorator::value && !CanHardwireRaw::value>::type - store(void* addr, T value) { - if (UseCompressedOops) { - const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; - PreRuntimeDispatch::store(addr, value); - } else { - const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; - PreRuntimeDispatch::store(addr, value); - } - } - - template - inline static typename EnableIf< - !HasDecorator::value>::type - store(void* addr, T value) { - if (is_hardwired_primitive()) { - const DecoratorSet expanded_decorators = decorators | AS_RAW; - PreRuntimeDispatch::store(addr, value); - } else { - RuntimeDispatch::store(addr, value); - } - } - - template - inline static typename EnableIf< - HasDecorator::value>::type - store_at(oop base, ptrdiff_t offset, T value) { - store(field_addr(base, offset), value); - } - - template - inline static typename EnableIf< - !HasDecorator::value>::type - store_at(oop base, ptrdiff_t offset, T value) { - if (is_hardwired_primitive()) { - const DecoratorSet expanded_decorators = decorators | AS_RAW; - PreRuntimeDispatch::store_at(base, offset, value); - } else { - RuntimeDispatch::store_at(base, offset, value); - } - } - - template - inline static typename EnableIf< - HasDecorator::value && CanHardwireRaw::value, T>::type - load(void* addr) { - typedef RawAccessBarrier Raw; - if (HasDecorator::value) { - return Raw::template oop_load(addr); - } else { - return Raw::template load(addr); - } - } - - template - inline static typename EnableIf< - HasDecorator::value && !CanHardwireRaw::value, T>::type - load(void* addr) { - if (UseCompressedOops) { - const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; - return PreRuntimeDispatch::load(addr); - } else { - const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; - return PreRuntimeDispatch::load(addr); - } - } - - template - inline static typename EnableIf< - !HasDecorator::value, T>::type - load(void* addr) { - if (is_hardwired_primitive()) { - const DecoratorSet expanded_decorators = decorators | AS_RAW; - return PreRuntimeDispatch::load(addr); - } else { - return RuntimeDispatch::load(addr); - } - } - - template - inline static typename EnableIf< - HasDecorator::value, T>::type - load_at(oop base, ptrdiff_t offset) { - return load(field_addr(base, offset)); - } - - template - inline static typename EnableIf< - !HasDecorator::value, T>::type - load_at(oop base, ptrdiff_t offset) { - if (is_hardwired_primitive()) { - const DecoratorSet expanded_decorators = decorators | AS_RAW; - return PreRuntimeDispatch::load_at(base, offset); - } else { - return RuntimeDispatch::load_at(base, offset); - } - } - - template - inline static typename EnableIf< - HasDecorator::value && CanHardwireRaw::value, T>::type - atomic_cmpxchg(T new_value, void* addr, T compare_value) { - typedef RawAccessBarrier Raw; - if (HasDecorator::value) { - return Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); - } else { - return Raw::atomic_cmpxchg(new_value, addr, compare_value); - } - } - - template - inline static typename EnableIf< - HasDecorator::value && !CanHardwireRaw::value, T>::type - atomic_cmpxchg(T new_value, void* addr, T compare_value) { - if (UseCompressedOops) { - const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; - return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); - } else { - const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; - return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); - } - } - - template - inline static typename EnableIf< - !HasDecorator::value, T>::type - atomic_cmpxchg(T new_value, void* addr, T compare_value) { - if (is_hardwired_primitive()) { - const DecoratorSet expanded_decorators = decorators | AS_RAW; - return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); - } else { - return RuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); - } - } - - template - inline static typename EnableIf< - HasDecorator::value, T>::type - atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { - return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); - } - - template - inline static typename EnableIf< - !HasDecorator::value, T>::type - atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { - if (is_hardwired_primitive()) { - const DecoratorSet expanded_decorators = decorators | AS_RAW; - return PreRuntimeDispatch::atomic_cmpxchg_at(new_value, base, offset, compare_value); - } else { - return RuntimeDispatch::atomic_cmpxchg_at(new_value, base, offset, compare_value); - } - } - - template - inline static typename EnableIf< - HasDecorator::value && CanHardwireRaw::value, T>::type - atomic_xchg(T new_value, void* addr) { - typedef RawAccessBarrier Raw; - if (HasDecorator::value) { - return Raw::oop_atomic_xchg(new_value, addr); - } else { - return Raw::atomic_xchg(new_value, addr); - } - } - - template - inline static typename EnableIf< - HasDecorator::value && !CanHardwireRaw::value, T>::type - atomic_xchg(T new_value, void* addr) { - if (UseCompressedOops) { - const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; - return PreRuntimeDispatch::atomic_xchg(new_value, addr); - } else { - const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; - return PreRuntimeDispatch::atomic_xchg(new_value, addr); - } - } - - template - inline static typename EnableIf< - !HasDecorator::value, T>::type - atomic_xchg(T new_value, void* addr) { - if (is_hardwired_primitive()) { - const DecoratorSet expanded_decorators = decorators | AS_RAW; - return PreRuntimeDispatch::atomic_xchg(new_value, addr); - } else { - return RuntimeDispatch::atomic_xchg(new_value, addr); - } - } - - template - inline static typename EnableIf< - HasDecorator::value, T>::type - atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { - return atomic_xchg(new_value, field_addr(base, offset)); - } - - template - inline static typename EnableIf< - !HasDecorator::value, T>::type - atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { - if (is_hardwired_primitive()) { - const DecoratorSet expanded_decorators = decorators | AS_RAW; - return PreRuntimeDispatch::atomic_xchg(new_value, base, offset); - } else { - return RuntimeDispatch::atomic_xchg_at(new_value, base, offset); - } - } - - template - inline static typename EnableIf< - HasDecorator::value && CanHardwireRaw::value, bool>::type - arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { - typedef RawAccessBarrier Raw; - if (HasDecorator::value) { - return Raw::oop_arraycopy(src_obj, dst_obj, src, dst, length); - } else { - return Raw::arraycopy(src_obj, dst_obj, src, dst, length); - } - } - - template - inline static typename EnableIf< - HasDecorator::value && !CanHardwireRaw::value, bool>::type - arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { - if (UseCompressedOops) { - const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; - return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); - } else { - const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; - return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); - } - } - - template - inline static typename EnableIf< - !HasDecorator::value, bool>::type - arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { - if (is_hardwired_primitive()) { - const DecoratorSet expanded_decorators = decorators | AS_RAW; - return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); - } else { - return RuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); - } - } - - template - inline static typename EnableIf< - HasDecorator::value>::type - clone(oop src, oop dst, size_t size) { - typedef RawAccessBarrier Raw; - Raw::clone(src, dst, size); - } - - template - inline static typename EnableIf< - !HasDecorator::value>::type - clone(oop src, oop dst, size_t size) { - RuntimeDispatch::clone(src, dst, size); - } - - template - inline static typename EnableIf< - HasDecorator::value, oop>::type - resolve(oop obj) { - typedef RawAccessBarrier Raw; - return Raw::resolve(obj); - } - - template - inline static typename EnableIf< - !HasDecorator::value, oop>::type - resolve(oop obj) { - return RuntimeDispatch::resolve(obj); - } - - template - inline static typename EnableIf< - HasDecorator::value, bool>::type - equals(oop o1, oop o2) { - typedef RawAccessBarrier Raw; - return Raw::equals(o1, o2); - } - - template - inline static typename EnableIf< - !HasDecorator::value, bool>::type - equals(oop o1, oop o2) { - return RuntimeDispatch::equals(o1, o2); - } - }; - - // This class adds implied decorators that follow according to decorator rules. - // For example adding default reference strength and default memory ordering - // semantics. - template - struct DecoratorFixup: AllStatic { - // If no reference strength has been picked, then strong will be picked - static const DecoratorSet ref_strength_default = input_decorators | - (((ON_DECORATOR_MASK & input_decorators) == 0 && (INTERNAL_VALUE_IS_OOP & input_decorators) != 0) ? - ON_STRONG_OOP_REF : INTERNAL_EMPTY); - // If no memory ordering has been picked, unordered will be picked - static const DecoratorSet memory_ordering_default = ref_strength_default | - ((MO_DECORATOR_MASK & ref_strength_default) == 0 ? MO_UNORDERED : INTERNAL_EMPTY); - // If no barrier strength has been picked, normal will be used - static const DecoratorSet barrier_strength_default = memory_ordering_default | - ((AS_DECORATOR_MASK & memory_ordering_default) == 0 ? AS_NORMAL : INTERNAL_EMPTY); - // Heap array accesses imply it is a heap access - static const DecoratorSet heap_array_is_in_heap = barrier_strength_default | - ((IN_HEAP_ARRAY & barrier_strength_default) != 0 ? IN_HEAP : INTERNAL_EMPTY); - static const DecoratorSet conc_root_is_root = heap_array_is_in_heap | - ((IN_CONCURRENT_ROOT & heap_array_is_in_heap) != 0 ? IN_ROOT : INTERNAL_EMPTY); - static const DecoratorSet archive_root_is_root = conc_root_is_root | - ((IN_ARCHIVE_ROOT & conc_root_is_root) != 0 ? IN_ROOT : INTERNAL_EMPTY); - static const DecoratorSet value = archive_root_is_root | BT_BUILDTIME_DECORATORS; - }; - - // Step 2: Reduce types. - // Enforce that for non-oop types, T and P have to be strictly the same. - // P is the type of the address and T is the type of the values. - // As for oop types, it is allow to send T in {narrowOop, oop} and - // P in {narrowOop, oop, HeapWord*}. The following rules apply according to - // the subsequent table. (columns are P, rows are T) - // | | HeapWord | oop | narrowOop | - // | oop | rt-comp | hw-none | hw-comp | - // | narrowOop | x | x | hw-none | - // - // x means not allowed - // rt-comp means it must be checked at runtime whether the oop is compressed. - // hw-none means it is statically known the oop will not be compressed. - // hw-comp means it is statically known the oop will be compressed. - - template - inline void store_reduce_types(T* addr, T value) { - PreRuntimeDispatch::store(addr, value); - } - - template - inline void store_reduce_types(narrowOop* addr, oop value) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | - INTERNAL_RT_USE_COMPRESSED_OOPS; - PreRuntimeDispatch::store(addr, value); - } - - template - inline void store_reduce_types(narrowOop* addr, narrowOop value) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | - INTERNAL_RT_USE_COMPRESSED_OOPS; - PreRuntimeDispatch::store(addr, value); - } - - template - inline void store_reduce_types(HeapWord* addr, oop value) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; - PreRuntimeDispatch::store(addr, value); + void RuntimeDispatch::store_init(void* addr, T value) { + func_t function = BarrierResolver::resolve_barrier(); + _store_func = function; + function(addr, value); } template - inline T atomic_cmpxchg_reduce_types(T new_value, T* addr, T compare_value) { - return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); - } - - template - inline oop atomic_cmpxchg_reduce_types(oop new_value, narrowOop* addr, oop compare_value) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | - INTERNAL_RT_USE_COMPRESSED_OOPS; - return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); - } - - template - inline narrowOop atomic_cmpxchg_reduce_types(narrowOop new_value, narrowOop* addr, narrowOop compare_value) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | - INTERNAL_RT_USE_COMPRESSED_OOPS; - return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); - } - - template - inline oop atomic_cmpxchg_reduce_types(oop new_value, - HeapWord* addr, - oop compare_value) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; - return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); - } - - template - inline T atomic_xchg_reduce_types(T new_value, T* addr) { - const DecoratorSet expanded_decorators = decorators; - return PreRuntimeDispatch::atomic_xchg(new_value, addr); - } - - template - inline oop atomic_xchg_reduce_types(oop new_value, narrowOop* addr) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | - INTERNAL_RT_USE_COMPRESSED_OOPS; - return PreRuntimeDispatch::atomic_xchg(new_value, addr); - } - - template - inline narrowOop atomic_xchg_reduce_types(narrowOop new_value, narrowOop* addr) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | - INTERNAL_RT_USE_COMPRESSED_OOPS; - return PreRuntimeDispatch::atomic_xchg(new_value, addr); - } - - template - inline oop atomic_xchg_reduce_types(oop new_value, HeapWord* addr) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; - return PreRuntimeDispatch::atomic_xchg(new_value, addr); + void RuntimeDispatch::store_at_init(oop base, ptrdiff_t offset, T value) { + func_t function = BarrierResolver::resolve_barrier(); + _store_at_func = function; + function(base, offset, value); } template - inline T load_reduce_types(T* addr) { - return PreRuntimeDispatch::load(addr); + T RuntimeDispatch::load_init(void* addr) { + func_t function = BarrierResolver::resolve_barrier(); + _load_func = function; + return function(addr); } template - inline typename OopOrNarrowOop::type load_reduce_types(narrowOop* addr) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | - INTERNAL_RT_USE_COMPRESSED_OOPS; - return PreRuntimeDispatch::load::type>(addr); + T RuntimeDispatch::load_at_init(oop base, ptrdiff_t offset) { + func_t function = BarrierResolver::resolve_barrier(); + _load_at_func = function; + return function(base, offset); } template - inline oop load_reduce_types(HeapWord* addr) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; - return PreRuntimeDispatch::load(addr); + T RuntimeDispatch::atomic_cmpxchg_init(T new_value, void* addr, T compare_value) { + func_t function = BarrierResolver::resolve_barrier(); + _atomic_cmpxchg_func = function; + return function(new_value, addr, compare_value); } template - inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { - return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); - } - - template - inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, HeapWord* src, HeapWord* dst, size_t length) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; - return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); - } - - template - inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, narrowOop* src, narrowOop* dst, size_t length) { - const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | - INTERNAL_RT_USE_COMPRESSED_OOPS; - return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + T RuntimeDispatch::atomic_cmpxchg_at_init(T new_value, oop base, ptrdiff_t offset, T compare_value) { + func_t function = BarrierResolver::resolve_barrier(); + _atomic_cmpxchg_at_func = function; + return function(new_value, base, offset, compare_value); } - // Step 1: Set default decorators. This step remembers if a type was volatile - // and then sets the MO_VOLATILE decorator by default. Otherwise, a default - // memory ordering is set for the access, and the implied decorator rules - // are applied to select sensible defaults for decorators that have not been - // explicitly set. For example, default object referent strength is set to strong. - // This step also decays the types passed in (e.g. getting rid of CV qualifiers - // and references from the types). This step also perform some type verification - // that the passed in types make sense. - template - static void verify_types(){ - // If this fails to compile, then you have sent in something that is - // not recognized as a valid primitive type to a primitive Access function. - STATIC_ASSERT((HasDecorator::value || // oops have already been validated - (IsPointer::value || IsIntegral::value) || - IsFloatingPoint::value)); // not allowed primitive type - } - - template - inline void store(P* addr, T value) { - verify_types(); - typedef typename Decay

::type DecayedP; - typedef typename Decay::type DecayedT; - DecayedT decayed_value = value; - // If a volatile address is passed in but no memory ordering decorator, - // set the memory ordering to MO_VOLATILE by default. - const DecoratorSet expanded_decorators = DecoratorFixup< - (IsVolatile

::value && !HasDecorator::value) ? - (MO_VOLATILE | decorators) : decorators>::value; - store_reduce_types(const_cast(addr), decayed_value); + T RuntimeDispatch::atomic_xchg_init(T new_value, void* addr) { + func_t function = BarrierResolver::resolve_barrier(); + _atomic_xchg_func = function; + return function(new_value, addr); } template - inline void store_at(oop base, ptrdiff_t offset, T value) { - verify_types(); - typedef typename Decay::type DecayedT; - DecayedT decayed_value = value; - const DecoratorSet expanded_decorators = DecoratorFixup::value ? - INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; - PreRuntimeDispatch::store_at(base, offset, decayed_value); - } - - template - inline T load(P* addr) { - verify_types(); - typedef typename Decay

::type DecayedP; - typedef typename Conditional::value, - typename OopOrNarrowOop::type, - typename Decay::type>::type DecayedT; - // If a volatile address is passed in but no memory ordering decorator, - // set the memory ordering to MO_VOLATILE by default. - const DecoratorSet expanded_decorators = DecoratorFixup< - (IsVolatile

::value && !HasDecorator::value) ? - (MO_VOLATILE | decorators) : decorators>::value; - return load_reduce_types(const_cast(addr)); + T RuntimeDispatch::atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset) { + func_t function = BarrierResolver::resolve_barrier(); + _atomic_xchg_at_func = function; + return function(new_value, base, offset); } template - inline T load_at(oop base, ptrdiff_t offset) { - verify_types(); - typedef typename Conditional::value, - typename OopOrNarrowOop::type, - typename Decay::type>::type DecayedT; - // Expand the decorators (figure out sensible defaults) - // Potentially remember if we need compressed oop awareness - const DecoratorSet expanded_decorators = DecoratorFixup::value ? - INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; - return PreRuntimeDispatch::load_at(base, offset); - } - - template - inline T atomic_cmpxchg(T new_value, P* addr, T compare_value) { - verify_types(); - typedef typename Decay

::type DecayedP; - typedef typename Decay::type DecayedT; - DecayedT new_decayed_value = new_value; - DecayedT compare_decayed_value = compare_value; - const DecoratorSet expanded_decorators = DecoratorFixup< - (!HasDecorator::value) ? - (MO_SEQ_CST | decorators) : decorators>::value; - return atomic_cmpxchg_reduce_types(new_decayed_value, - const_cast(addr), - compare_decayed_value); + bool RuntimeDispatch::arraycopy_init(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) { + func_t function = BarrierResolver::resolve_barrier(); + _arraycopy_func = function; + return function(src_obj, dst_obj, src, dst, length); } template - inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { - verify_types(); - typedef typename Decay::type DecayedT; - DecayedT new_decayed_value = new_value; - DecayedT compare_decayed_value = compare_value; - // Determine default memory ordering - const DecoratorSet expanded_decorators = DecoratorFixup< - (!HasDecorator::value) ? - (MO_SEQ_CST | decorators) : decorators>::value; - // Potentially remember that we need compressed oop awareness - const DecoratorSet final_decorators = expanded_decorators | - (HasDecorator::value ? - INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY); - return PreRuntimeDispatch::atomic_cmpxchg_at(new_decayed_value, base, - offset, compare_decayed_value); - } - - template - inline T atomic_xchg(T new_value, P* addr) { - verify_types(); - typedef typename Decay

::type DecayedP; - typedef typename Decay::type DecayedT; - DecayedT new_decayed_value = new_value; - // atomic_xchg is only available in SEQ_CST flavour. - const DecoratorSet expanded_decorators = DecoratorFixup::value; - return atomic_xchg_reduce_types(new_decayed_value, - const_cast(addr)); + void RuntimeDispatch::clone_init(oop src, oop dst, size_t size) { + func_t function = BarrierResolver::resolve_barrier(); + _clone_func = function; + function(src, dst, size); } template - inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { - verify_types(); - typedef typename Decay::type DecayedT; - DecayedT new_decayed_value = new_value; - // atomic_xchg is only available in SEQ_CST flavour. - const DecoratorSet expanded_decorators = DecoratorFixup::value ? - INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; - return PreRuntimeDispatch::atomic_xchg_at(new_decayed_value, base, offset); + oop RuntimeDispatch::resolve_init(oop obj) { + func_t function = BarrierResolver::resolve_barrier(); + _resolve_func = function; + return function(obj); } template - inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { - STATIC_ASSERT((HasDecorator::value || - (IsSame::value || IsIntegral::value) || - IsFloatingPoint::value)); // arraycopy allows type erased void elements - typedef typename Decay::type DecayedT; - const DecoratorSet expanded_decorators = DecoratorFixup::value; - return arraycopy_reduce_types(src_obj, dst_obj, - const_cast(src), - const_cast(dst), - length); + bool RuntimeDispatch::equals_init(oop o1, oop o2) { + func_t function = BarrierResolver::resolve_barrier(); + _equals_func = function; + return function(o1, o2); } - - template - inline void clone(oop src, oop dst, size_t size) { - const DecoratorSet expanded_decorators = DecoratorFixup::value; - PreRuntimeDispatch::clone(src, dst, size); - } - - template - inline oop resolve(oop obj) { - const DecoratorSet expanded_decorators = DecoratorFixup::value; - return PreRuntimeDispatch::resolve(obj); - } - - template - inline bool equals(oop o1, oop o2) { - const DecoratorSet expanded_decorators = DecoratorFixup::value; - return PreRuntimeDispatch::equals(o1, o2); - } -} - -template -template -void Access::verify_decorators() { - STATIC_ASSERT((~expected_decorators & decorators) == 0); // unexpected decorator used - const DecoratorSet barrier_strength_decorators = decorators & AS_DECORATOR_MASK; - STATIC_ASSERT(barrier_strength_decorators == 0 || ( // make sure barrier strength decorators are disjoint if set - (barrier_strength_decorators ^ AS_NO_KEEPALIVE) == 0 || - (barrier_strength_decorators ^ AS_DEST_NOT_INITIALIZED) == 0 || - (barrier_strength_decorators ^ AS_RAW) == 0 || - (barrier_strength_decorators ^ AS_NORMAL) == 0 - )); - const DecoratorSet ref_strength_decorators = decorators & ON_DECORATOR_MASK; - STATIC_ASSERT(ref_strength_decorators == 0 || ( // make sure ref strength decorators are disjoint if set - (ref_strength_decorators ^ ON_STRONG_OOP_REF) == 0 || - (ref_strength_decorators ^ ON_WEAK_OOP_REF) == 0 || - (ref_strength_decorators ^ ON_PHANTOM_OOP_REF) == 0 || - (ref_strength_decorators ^ ON_UNKNOWN_OOP_REF) == 0 - )); - const DecoratorSet memory_ordering_decorators = decorators & MO_DECORATOR_MASK; - STATIC_ASSERT(memory_ordering_decorators == 0 || ( // make sure memory ordering decorators are disjoint if set - (memory_ordering_decorators ^ MO_UNORDERED) == 0 || - (memory_ordering_decorators ^ MO_VOLATILE) == 0 || - (memory_ordering_decorators ^ MO_RELAXED) == 0 || - (memory_ordering_decorators ^ MO_ACQUIRE) == 0 || - (memory_ordering_decorators ^ MO_RELEASE) == 0 || - (memory_ordering_decorators ^ MO_SEQ_CST) == 0 - )); - const DecoratorSet location_decorators = decorators & IN_DECORATOR_MASK; - STATIC_ASSERT(location_decorators == 0 || ( // make sure location decorators are disjoint if set - (location_decorators ^ IN_ROOT) == 0 || - (location_decorators ^ IN_HEAP) == 0 || - (location_decorators ^ (IN_HEAP | IN_HEAP_ARRAY)) == 0 || - (location_decorators ^ (IN_ROOT | IN_CONCURRENT_ROOT)) == 0 || - (location_decorators ^ (IN_ROOT | IN_ARCHIVE_ROOT)) == 0 - )); } -#endif // SHARE_VM_RUNTIME_ACCESS_INLINE_HPP +#endif // SHARE_OOPS_ACCESS_INLINE_HPP diff --git a/src/hotspot/share/oops/accessBackend.hpp b/src/hotspot/share/oops/accessBackend.hpp index a705392..abcf59a 100644 --- a/src/hotspot/share/oops/accessBackend.hpp +++ b/src/hotspot/share/oops/accessBackend.hpp @@ -22,16 +22,26 @@ * */ -#ifndef SHARE_VM_RUNTIME_ACCESSBACKEND_HPP -#define SHARE_VM_RUNTIME_ACCESSBACKEND_HPP +#ifndef SHARE_OOPS_ACCESSBACKEND_HPP +#define SHARE_OOPS_ACCESSBACKEND_HPP +#include "gc/shared/barrierSetConfig.hpp" +#include "memory/allocation.hpp" #include "metaprogramming/conditional.hpp" +#include "metaprogramming/decay.hpp" #include "metaprogramming/enableIf.hpp" #include "metaprogramming/integralConstant.hpp" +#include "metaprogramming/isFloatingPoint.hpp" +#include "metaprogramming/isIntegral.hpp" +#include "metaprogramming/isPointer.hpp" #include "metaprogramming/isSame.hpp" +#include "metaprogramming/isVolatile.hpp" +#include "oops/accessDecorators.hpp" +#include "oops/oopsHierarchy.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" + // This metafunction returns either oop or narrowOop depending on whether // an access needs to use compressed oops or not. template @@ -395,4 +405,894 @@ public: static bool equals(oop o1, oop o2) { return o1 == o2; } }; -#endif // SHARE_VM_RUNTIME_ACCESSBACKEND_HPP +// Below is the implementation of the first 4 steps of the template pipeline: +// * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers +// and sets default decorators to sensible values. +// * Step 2: Reduce types. This step makes sure there is only a single T type and not +// multiple types. The P type of the address and T type of the value must +// match. +// * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be +// avoided, and in that case avoids it (calling raw accesses or +// primitive accesses in a build that does not require primitive GC barriers) +// * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding +// BarrierSet::AccessBarrier accessor that attaches GC-required barriers +// to the access. + +namespace AccessInternal { + template + struct OopOrNarrowOopInternal: AllStatic { + typedef oop type; + }; + + template <> + struct OopOrNarrowOopInternal: AllStatic { + typedef narrowOop type; + }; + + // This metafunction returns a canonicalized oop/narrowOop type for a passed + // in oop-like types passed in from oop_* overloads where the user has sworn + // that the passed in values should be oop-like (e.g. oop, oopDesc*, arrayOop, + // narrowOoop, instanceOopDesc*, and random other things). + // In the oop_* overloads, it must hold that if the passed in type T is not + // narrowOop, then it by contract has to be one of many oop-like types implicitly + // convertible to oop, and hence returns oop as the canonical oop type. + // If it turns out it was not, then the implicit conversion to oop will fail + // to compile, as desired. + template + struct OopOrNarrowOop: AllStatic { + typedef typename OopOrNarrowOopInternal::type>::type type; + }; + + inline void* field_addr(oop base, ptrdiff_t byte_offset) { + return reinterpret_cast(reinterpret_cast((void*)base) + byte_offset); + } + // Step 4: Runtime dispatch + // The RuntimeDispatch class is responsible for performing a runtime dispatch of the + // accessor. This is required when the access either depends on whether compressed oops + // is being used, or it depends on which GC implementation was chosen (e.g. requires GC + // barriers). The way it works is that a function pointer initially pointing to an + // accessor resolution function gets called for each access. Upon first invocation, + // it resolves which accessor to be used in future invocations and patches the + // function pointer to this new accessor. + + template + struct RuntimeDispatch: AllStatic {}; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _store_func; + + static void store_init(void* addr, T value); + + static inline void store(void* addr, T value) { + _store_func(addr, value); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _store_at_func; + + static void store_at_init(oop base, ptrdiff_t offset, T value); + + static inline void store_at(oop base, ptrdiff_t offset, T value) { + _store_at_func(base, offset, value); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _load_func; + + static T load_init(void* addr); + + static inline T load(void* addr) { + return _load_func(addr); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _load_at_func; + + static T load_at_init(oop base, ptrdiff_t offset); + + static inline T load_at(oop base, ptrdiff_t offset) { + return _load_at_func(base, offset); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _atomic_cmpxchg_func; + + static T atomic_cmpxchg_init(T new_value, void* addr, T compare_value); + + static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) { + return _atomic_cmpxchg_func(new_value, addr, compare_value); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _atomic_cmpxchg_at_func; + + static T atomic_cmpxchg_at_init(T new_value, oop base, ptrdiff_t offset, T compare_value); + + static inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { + return _atomic_cmpxchg_at_func(new_value, base, offset, compare_value); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _atomic_xchg_func; + + static T atomic_xchg_init(T new_value, void* addr); + + static inline T atomic_xchg(T new_value, void* addr) { + return _atomic_xchg_func(new_value, addr); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _atomic_xchg_at_func; + + static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset); + + static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { + return _atomic_xchg_at_func(new_value, base, offset); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _arraycopy_func; + + static bool arraycopy_init(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length); + + static inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) { + return _arraycopy_func(src_obj, dst_obj, src, dst, length); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _clone_func; + + static void clone_init(oop src, oop dst, size_t size); + + static inline void clone(oop src, oop dst, size_t size) { + _clone_func(src, dst, size); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _resolve_func; + + static oop resolve_init(oop obj); + + static inline oop resolve(oop obj) { + return _resolve_func(obj); + } + }; + + template + struct RuntimeDispatch: AllStatic { + typedef typename AccessFunction::type func_t; + static func_t _equals_func; + + static bool equals_init(oop o1, oop o2); + + static inline bool equals(oop o1, oop o2) { + return _equals_func(o1, o2); + } + }; + + // Initialize the function pointers to point to the resolving function. + template + typename AccessFunction::type + RuntimeDispatch::_store_func = &store_init; + + template + typename AccessFunction::type + RuntimeDispatch::_store_at_func = &store_at_init; + + template + typename AccessFunction::type + RuntimeDispatch::_load_func = &load_init; + + template + typename AccessFunction::type + RuntimeDispatch::_load_at_func = &load_at_init; + + template + typename AccessFunction::type + RuntimeDispatch::_atomic_cmpxchg_func = &atomic_cmpxchg_init; + + template + typename AccessFunction::type + RuntimeDispatch::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init; + + template + typename AccessFunction::type + RuntimeDispatch::_atomic_xchg_func = &atomic_xchg_init; + + template + typename AccessFunction::type + RuntimeDispatch::_atomic_xchg_at_func = &atomic_xchg_at_init; + + template + typename AccessFunction::type + RuntimeDispatch::_arraycopy_func = &arraycopy_init; + + template + typename AccessFunction::type + RuntimeDispatch::_clone_func = &clone_init; + + template + typename AccessFunction::type + RuntimeDispatch::_resolve_func = &resolve_init; + + template + typename AccessFunction::type + RuntimeDispatch::_equals_func = &equals_init; + + // Step 3: Pre-runtime dispatching. + // The PreRuntimeDispatch class is responsible for filtering the barrier strength + // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime + // dispatch point. Otherwise it goes through a runtime check if hardwiring was + // not possible. + struct PreRuntimeDispatch: AllStatic { + template + struct CanHardwireRaw: public IntegralConstant< + bool, + !HasDecorator::value || // primitive access + !HasDecorator::value || // don't care about compressed oops (oop* address) + HasDecorator::value> // we can infer we use compressed oops (narrowOop* address) + {}; + + static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP; + + template + static bool is_hardwired_primitive() { + return !HasDecorator::value && + !HasDecorator::value; + } + + template + inline static typename EnableIf< + HasDecorator::value && CanHardwireRaw::value>::type + store(void* addr, T value) { + typedef RawAccessBarrier Raw; + if (HasDecorator::value) { + Raw::oop_store(addr, value); + } else { + Raw::store(addr, value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && !CanHardwireRaw::value>::type + store(void* addr, T value) { + if (UseCompressedOops) { + const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; + PreRuntimeDispatch::store(addr, value); + } else { + const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; + PreRuntimeDispatch::store(addr, value); + } + } + + template + inline static typename EnableIf< + !HasDecorator::value>::type + store(void* addr, T value) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + PreRuntimeDispatch::store(addr, value); + } else { + RuntimeDispatch::store(addr, value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value>::type + store_at(oop base, ptrdiff_t offset, T value) { + store(field_addr(base, offset), value); + } + + template + inline static typename EnableIf< + !HasDecorator::value>::type + store_at(oop base, ptrdiff_t offset, T value) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + PreRuntimeDispatch::store_at(base, offset, value); + } else { + RuntimeDispatch::store_at(base, offset, value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && CanHardwireRaw::value, T>::type + load(void* addr) { + typedef RawAccessBarrier Raw; + if (HasDecorator::value) { + return Raw::template oop_load(addr); + } else { + return Raw::template load(addr); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && !CanHardwireRaw::value, T>::type + load(void* addr) { + if (UseCompressedOops) { + const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; + return PreRuntimeDispatch::load(addr); + } else { + const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; + return PreRuntimeDispatch::load(addr); + } + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + load(void* addr) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::load(addr); + } else { + return RuntimeDispatch::load(addr); + } + } + + template + inline static typename EnableIf< + HasDecorator::value, T>::type + load_at(oop base, ptrdiff_t offset) { + return load(field_addr(base, offset)); + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + load_at(oop base, ptrdiff_t offset) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::load_at(base, offset); + } else { + return RuntimeDispatch::load_at(base, offset); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && CanHardwireRaw::value, T>::type + atomic_cmpxchg(T new_value, void* addr, T compare_value) { + typedef RawAccessBarrier Raw; + if (HasDecorator::value) { + return Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); + } else { + return Raw::atomic_cmpxchg(new_value, addr, compare_value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && !CanHardwireRaw::value, T>::type + atomic_cmpxchg(T new_value, void* addr, T compare_value) { + if (UseCompressedOops) { + const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } else { + const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + atomic_cmpxchg(T new_value, void* addr, T compare_value) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } else { + return RuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value, T>::type + atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { + return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::atomic_cmpxchg_at(new_value, base, offset, compare_value); + } else { + return RuntimeDispatch::atomic_cmpxchg_at(new_value, base, offset, compare_value); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && CanHardwireRaw::value, T>::type + atomic_xchg(T new_value, void* addr) { + typedef RawAccessBarrier Raw; + if (HasDecorator::value) { + return Raw::oop_atomic_xchg(new_value, addr); + } else { + return Raw::atomic_xchg(new_value, addr); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && !CanHardwireRaw::value, T>::type + atomic_xchg(T new_value, void* addr) { + if (UseCompressedOops) { + const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } else { + const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + atomic_xchg(T new_value, void* addr) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } else { + return RuntimeDispatch::atomic_xchg(new_value, addr); + } + } + + template + inline static typename EnableIf< + HasDecorator::value, T>::type + atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { + return atomic_xchg(new_value, field_addr(base, offset)); + } + + template + inline static typename EnableIf< + !HasDecorator::value, T>::type + atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::atomic_xchg(new_value, base, offset); + } else { + return RuntimeDispatch::atomic_xchg_at(new_value, base, offset); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && CanHardwireRaw::value, bool>::type + arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + typedef RawAccessBarrier Raw; + if (HasDecorator::value) { + return Raw::oop_arraycopy(src_obj, dst_obj, src, dst, length); + } else { + return Raw::arraycopy(src_obj, dst_obj, src, dst, length); + } + } + + template + inline static typename EnableIf< + HasDecorator::value && !CanHardwireRaw::value, bool>::type + arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + if (UseCompressedOops) { + const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } else { + const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } + } + + template + inline static typename EnableIf< + !HasDecorator::value, bool>::type + arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + if (is_hardwired_primitive()) { + const DecoratorSet expanded_decorators = decorators | AS_RAW; + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } else { + return RuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } + } + + template + inline static typename EnableIf< + HasDecorator::value>::type + clone(oop src, oop dst, size_t size) { + typedef RawAccessBarrier Raw; + Raw::clone(src, dst, size); + } + + template + inline static typename EnableIf< + !HasDecorator::value>::type + clone(oop src, oop dst, size_t size) { + RuntimeDispatch::clone(src, dst, size); + } + + template + inline static typename EnableIf< + HasDecorator::value, oop>::type + resolve(oop obj) { + typedef RawAccessBarrier Raw; + return Raw::resolve(obj); + } + + template + inline static typename EnableIf< + !HasDecorator::value, oop>::type + resolve(oop obj) { + return RuntimeDispatch::resolve(obj); + } + + template + inline static typename EnableIf< + HasDecorator::value, bool>::type + equals(oop o1, oop o2) { + typedef RawAccessBarrier Raw; + return Raw::equals(o1, o2); + } + + template + inline static typename EnableIf< + !HasDecorator::value, bool>::type + equals(oop o1, oop o2) { + return RuntimeDispatch::equals(o1, o2); + } + }; + + // This class adds implied decorators that follow according to decorator rules. + // For example adding default reference strength and default memory ordering + // semantics. + template + struct DecoratorFixup: AllStatic { + // If no reference strength has been picked, then strong will be picked + static const DecoratorSet ref_strength_default = input_decorators | + (((ON_DECORATOR_MASK & input_decorators) == 0 && (INTERNAL_VALUE_IS_OOP & input_decorators) != 0) ? + ON_STRONG_OOP_REF : INTERNAL_EMPTY); + // If no memory ordering has been picked, unordered will be picked + static const DecoratorSet memory_ordering_default = ref_strength_default | + ((MO_DECORATOR_MASK & ref_strength_default) == 0 ? MO_UNORDERED : INTERNAL_EMPTY); + // If no barrier strength has been picked, normal will be used + static const DecoratorSet barrier_strength_default = memory_ordering_default | + ((AS_DECORATOR_MASK & memory_ordering_default) == 0 ? AS_NORMAL : INTERNAL_EMPTY); + // Heap array accesses imply it is a heap access + static const DecoratorSet heap_array_is_in_heap = barrier_strength_default | + ((IN_HEAP_ARRAY & barrier_strength_default) != 0 ? IN_HEAP : INTERNAL_EMPTY); + static const DecoratorSet conc_root_is_root = heap_array_is_in_heap | + ((IN_CONCURRENT_ROOT & heap_array_is_in_heap) != 0 ? IN_ROOT : INTERNAL_EMPTY); + static const DecoratorSet archive_root_is_root = conc_root_is_root | + ((IN_ARCHIVE_ROOT & conc_root_is_root) != 0 ? IN_ROOT : INTERNAL_EMPTY); + static const DecoratorSet value = archive_root_is_root | BT_BUILDTIME_DECORATORS; + }; + + // Step 2: Reduce types. + // Enforce that for non-oop types, T and P have to be strictly the same. + // P is the type of the address and T is the type of the values. + // As for oop types, it is allow to send T in {narrowOop, oop} and + // P in {narrowOop, oop, HeapWord*}. The following rules apply according to + // the subsequent table. (columns are P, rows are T) + // | | HeapWord | oop | narrowOop | + // | oop | rt-comp | hw-none | hw-comp | + // | narrowOop | x | x | hw-none | + // + // x means not allowed + // rt-comp means it must be checked at runtime whether the oop is compressed. + // hw-none means it is statically known the oop will not be compressed. + // hw-comp means it is statically known the oop will be compressed. + + template + inline void store_reduce_types(T* addr, T value) { + PreRuntimeDispatch::store(addr, value); + } + + template + inline void store_reduce_types(narrowOop* addr, oop value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + PreRuntimeDispatch::store(addr, value); + } + + template + inline void store_reduce_types(narrowOop* addr, narrowOop value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + PreRuntimeDispatch::store(addr, value); + } + + template + inline void store_reduce_types(HeapWord* addr, oop value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; + PreRuntimeDispatch::store(addr, value); + } + + template + inline T atomic_cmpxchg_reduce_types(T new_value, T* addr, T compare_value) { + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + + template + inline oop atomic_cmpxchg_reduce_types(oop new_value, narrowOop* addr, oop compare_value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + + template + inline narrowOop atomic_cmpxchg_reduce_types(narrowOop new_value, narrowOop* addr, narrowOop compare_value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + + template + inline oop atomic_cmpxchg_reduce_types(oop new_value, + HeapWord* addr, + oop compare_value) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; + return PreRuntimeDispatch::atomic_cmpxchg(new_value, addr, compare_value); + } + + template + inline T atomic_xchg_reduce_types(T new_value, T* addr) { + const DecoratorSet expanded_decorators = decorators; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } + + template + inline oop atomic_xchg_reduce_types(oop new_value, narrowOop* addr) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } + + template + inline narrowOop atomic_xchg_reduce_types(narrowOop new_value, narrowOop* addr) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } + + template + inline oop atomic_xchg_reduce_types(oop new_value, HeapWord* addr) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; + return PreRuntimeDispatch::atomic_xchg(new_value, addr); + } + + template + inline T load_reduce_types(T* addr) { + return PreRuntimeDispatch::load(addr); + } + + template + inline typename OopOrNarrowOop::type load_reduce_types(narrowOop* addr) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::load::type>(addr); + } + + template + inline oop load_reduce_types(HeapWord* addr) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; + return PreRuntimeDispatch::load(addr); + } + + template + inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } + + template + inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, HeapWord* src, HeapWord* dst, size_t length) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } + + template + inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, narrowOop* src, narrowOop* dst, size_t length) { + const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | + INTERNAL_RT_USE_COMPRESSED_OOPS; + return PreRuntimeDispatch::arraycopy(src_obj, dst_obj, src, dst, length); + } + + // Step 1: Set default decorators. This step remembers if a type was volatile + // and then sets the MO_VOLATILE decorator by default. Otherwise, a default + // memory ordering is set for the access, and the implied decorator rules + // are applied to select sensible defaults for decorators that have not been + // explicitly set. For example, default object referent strength is set to strong. + // This step also decays the types passed in (e.g. getting rid of CV qualifiers + // and references from the types). This step also perform some type verification + // that the passed in types make sense. + + template + static void verify_types(){ + // If this fails to compile, then you have sent in something that is + // not recognized as a valid primitive type to a primitive Access function. + STATIC_ASSERT((HasDecorator::value || // oops have already been validated + (IsPointer::value || IsIntegral::value) || + IsFloatingPoint::value)); // not allowed primitive type + } + + template + inline void store(P* addr, T value) { + verify_types(); + typedef typename Decay

::type DecayedP; + typedef typename Decay::type DecayedT; + DecayedT decayed_value = value; + // If a volatile address is passed in but no memory ordering decorator, + // set the memory ordering to MO_VOLATILE by default. + const DecoratorSet expanded_decorators = DecoratorFixup< + (IsVolatile

::value && !HasDecorator::value) ? + (MO_VOLATILE | decorators) : decorators>::value; + store_reduce_types(const_cast(addr), decayed_value); + } + + template + inline void store_at(oop base, ptrdiff_t offset, T value) { + verify_types(); + typedef typename Decay::type DecayedT; + DecayedT decayed_value = value; + const DecoratorSet expanded_decorators = DecoratorFixup::value ? + INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; + PreRuntimeDispatch::store_at(base, offset, decayed_value); + } + + template + inline T load(P* addr) { + verify_types(); + typedef typename Decay

::type DecayedP; + typedef typename Conditional::value, + typename OopOrNarrowOop::type, + typename Decay::type>::type DecayedT; + // If a volatile address is passed in but no memory ordering decorator, + // set the memory ordering to MO_VOLATILE by default. + const DecoratorSet expanded_decorators = DecoratorFixup< + (IsVolatile

::value && !HasDecorator::value) ? + (MO_VOLATILE | decorators) : decorators>::value; + return load_reduce_types(const_cast(addr)); + } + + template + inline T load_at(oop base, ptrdiff_t offset) { + verify_types(); + typedef typename Conditional::value, + typename OopOrNarrowOop::type, + typename Decay::type>::type DecayedT; + // Expand the decorators (figure out sensible defaults) + // Potentially remember if we need compressed oop awareness + const DecoratorSet expanded_decorators = DecoratorFixup::value ? + INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; + return PreRuntimeDispatch::load_at(base, offset); + } + + template + inline T atomic_cmpxchg(T new_value, P* addr, T compare_value) { + verify_types(); + typedef typename Decay

::type DecayedP; + typedef typename Decay::type DecayedT; + DecayedT new_decayed_value = new_value; + DecayedT compare_decayed_value = compare_value; + const DecoratorSet expanded_decorators = DecoratorFixup< + (!HasDecorator::value) ? + (MO_SEQ_CST | decorators) : decorators>::value; + return atomic_cmpxchg_reduce_types(new_decayed_value, + const_cast(addr), + compare_decayed_value); + } + + template + inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { + verify_types(); + typedef typename Decay::type DecayedT; + DecayedT new_decayed_value = new_value; + DecayedT compare_decayed_value = compare_value; + // Determine default memory ordering + const DecoratorSet expanded_decorators = DecoratorFixup< + (!HasDecorator::value) ? + (MO_SEQ_CST | decorators) : decorators>::value; + // Potentially remember that we need compressed oop awareness + const DecoratorSet final_decorators = expanded_decorators | + (HasDecorator::value ? + INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY); + return PreRuntimeDispatch::atomic_cmpxchg_at(new_decayed_value, base, + offset, compare_decayed_value); + } + + template + inline T atomic_xchg(T new_value, P* addr) { + verify_types(); + typedef typename Decay

::type DecayedP; + typedef typename Decay::type DecayedT; + DecayedT new_decayed_value = new_value; + // atomic_xchg is only available in SEQ_CST flavour. + const DecoratorSet expanded_decorators = DecoratorFixup::value; + return atomic_xchg_reduce_types(new_decayed_value, + const_cast(addr)); + } + + template + inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { + verify_types(); + typedef typename Decay::type DecayedT; + DecayedT new_decayed_value = new_value; + // atomic_xchg is only available in SEQ_CST flavour. + const DecoratorSet expanded_decorators = DecoratorFixup::value ? + INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; + return PreRuntimeDispatch::atomic_xchg_at(new_decayed_value, base, offset); + } + + template + inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { + STATIC_ASSERT((HasDecorator::value || + (IsSame::value || IsIntegral::value) || + IsFloatingPoint::value)); // arraycopy allows type erased void elements + typedef typename Decay::type DecayedT; + const DecoratorSet expanded_decorators = DecoratorFixup::value; + return arraycopy_reduce_types(src_obj, dst_obj, + const_cast(src), + const_cast(dst), + length); + } + + template + inline void clone(oop src, oop dst, size_t size) { + const DecoratorSet expanded_decorators = DecoratorFixup::value; + PreRuntimeDispatch::clone(src, dst, size); + } + + template + inline oop resolve(oop obj) { + const DecoratorSet expanded_decorators = DecoratorFixup::value; + return PreRuntimeDispatch::resolve(obj); + } + + template + inline bool equals(oop o1, oop o2) { + const DecoratorSet expanded_decorators = DecoratorFixup::value; + return PreRuntimeDispatch::equals(o1, o2); + } +} + +#endif // SHARE_OOPS_ACCESSBACKEND_HPP diff --git a/src/hotspot/share/oops/accessDecorators.hpp b/src/hotspot/share/oops/accessDecorators.hpp new file mode 100644 index 0000000..8977969 --- /dev/null +++ b/src/hotspot/share/oops/accessDecorators.hpp @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_ACCESSDECORATORS_HPP +#define SHARE_OOPS_ACCESSDECORATORS_HPP + +// A decorator is an attribute or property that affects the way a memory access is performed in some way. +// There are different groups of decorators. Some have to do with memory ordering, others to do with, +// e.g. strength of references, strength of GC barriers, or whether compression should be applied or not. +// Some decorators are set at buildtime, such as whether primitives require GC barriers or not, others +// at callsites such as whether an access is in the heap or not, and others are resolved at runtime +// such as GC-specific barriers and encoding/decoding compressed oops. +typedef uint64_t DecoratorSet; + +// The HasDecorator trait can help at compile-time determining whether a decorator set +// has an intersection with a certain other decorator set +template +struct HasDecorator: public IntegralConstant {}; + +// == Internal Decorators - do not use == +// * INTERNAL_EMPTY: This is the name for the empty decorator set (in absence of other decorators). +// * INTERNAL_CONVERT_COMPRESSED_OOPS: This is an oop access that will require converting an oop +// to a narrowOop or vice versa, if UseCompressedOops is known to be set. +// * INTERNAL_VALUE_IS_OOP: Remember that the involved access is on oop rather than primitive. +const DecoratorSet INTERNAL_EMPTY = UCONST64(0); +const DecoratorSet INTERNAL_CONVERT_COMPRESSED_OOP = UCONST64(1) << 1; +const DecoratorSet INTERNAL_VALUE_IS_OOP = UCONST64(1) << 2; + +// == Internal build-time Decorators == +// * INTERNAL_BT_BARRIER_ON_PRIMITIVES: This is set in the barrierSetConfig.hpp file. +// * INTERNAL_BT_TO_SPACE_INVARIANT: This is set in the barrierSetConfig.hpp file iff +// no GC is bundled in the build that is to-space invariant. +const DecoratorSet INTERNAL_BT_BARRIER_ON_PRIMITIVES = UCONST64(1) << 3; +const DecoratorSet INTERNAL_BT_TO_SPACE_INVARIANT = UCONST64(1) << 4; + +// == Internal run-time Decorators == +// * INTERNAL_RT_USE_COMPRESSED_OOPS: This decorator will be set in runtime resolved +// access backends iff UseCompressedOops is true. +const DecoratorSet INTERNAL_RT_USE_COMPRESSED_OOPS = UCONST64(1) << 5; + +const DecoratorSet INTERNAL_DECORATOR_MASK = INTERNAL_CONVERT_COMPRESSED_OOP | INTERNAL_VALUE_IS_OOP | + INTERNAL_BT_BARRIER_ON_PRIMITIVES | INTERNAL_RT_USE_COMPRESSED_OOPS; + +// == Memory Ordering Decorators == +// The memory ordering decorators can be described in the following way: +// === Decorator Rules === +// The different types of memory ordering guarantees have a strict order of strength. +// Explicitly specifying the stronger ordering implies that the guarantees of the weaker +// property holds too. The names come from the C++11 atomic operations, and typically +// have a JMM equivalent property. +// The equivalence may be viewed like this: +// MO_UNORDERED is equivalent to JMM plain. +// MO_VOLATILE has no equivalence in JMM, because it's a C++ thing. +// MO_RELAXED is equivalent to JMM opaque. +// MO_ACQUIRE is equivalent to JMM acquire. +// MO_RELEASE is equivalent to JMM release. +// MO_SEQ_CST is equivalent to JMM volatile. +// +// === Stores === +// * MO_UNORDERED (Default): No guarantees. +// - The compiler and hardware are free to reorder aggressively. And they will. +// * MO_VOLATILE: Volatile stores (in the C++ sense). +// - The stores are not reordered by the compiler (but possibly the HW) w.r.t. other +// volatile accesses in program order (but possibly non-volatile accesses). +// * MO_RELAXED: Relaxed atomic stores. +// - The stores are atomic. +// - Guarantees from volatile stores hold. +// * MO_RELEASE: Releasing stores. +// - The releasing store will make its preceding memory accesses observable to memory accesses +// subsequent to an acquiring load observing this releasing store. +// - Guarantees from relaxed stores hold. +// * MO_SEQ_CST: Sequentially consistent stores. +// - The stores are observed in the same order by MO_SEQ_CST loads on other processors +// - Preceding loads and stores in program order are not reordered with subsequent loads and stores in program order. +// - Guarantees from releasing stores hold. +// === Loads === +// * MO_UNORDERED (Default): No guarantees +// - The compiler and hardware are free to reorder aggressively. And they will. +// * MO_VOLATILE: Volatile loads (in the C++ sense). +// - The loads are not reordered by the compiler (but possibly the HW) w.r.t. other +// volatile accesses in program order (but possibly non-volatile accesses). +// * MO_RELAXED: Relaxed atomic loads. +// - The loads are atomic. +// - Guarantees from volatile loads hold. +// * MO_ACQUIRE: Acquiring loads. +// - An acquiring load will make subsequent memory accesses observe the memory accesses +// preceding the releasing store that the acquiring load observed. +// - Guarantees from relaxed loads hold. +// * MO_SEQ_CST: Sequentially consistent loads. +// - These loads observe MO_SEQ_CST stores in the same order on other processors +// - Preceding loads and stores in program order are not reordered with subsequent loads and stores in program order. +// - Guarantees from acquiring loads hold. +// === Atomic Cmpxchg === +// * MO_RELAXED: Atomic but relaxed cmpxchg. +// - Guarantees from MO_RELAXED loads and MO_RELAXED stores hold unconditionally. +// * MO_SEQ_CST: Sequentially consistent cmpxchg. +// - Guarantees from MO_SEQ_CST loads and MO_SEQ_CST stores hold unconditionally. +// === Atomic Xchg === +// * MO_RELAXED: Atomic but relaxed atomic xchg. +// - Guarantees from MO_RELAXED loads and MO_RELAXED stores hold. +// * MO_SEQ_CST: Sequentially consistent xchg. +// - Guarantees from MO_SEQ_CST loads and MO_SEQ_CST stores hold. +const DecoratorSet MO_UNORDERED = UCONST64(1) << 6; +const DecoratorSet MO_VOLATILE = UCONST64(1) << 7; +const DecoratorSet MO_RELAXED = UCONST64(1) << 8; +const DecoratorSet MO_ACQUIRE = UCONST64(1) << 9; +const DecoratorSet MO_RELEASE = UCONST64(1) << 10; +const DecoratorSet MO_SEQ_CST = UCONST64(1) << 11; +const DecoratorSet MO_DECORATOR_MASK = MO_UNORDERED | MO_VOLATILE | MO_RELAXED | + MO_ACQUIRE | MO_RELEASE | MO_SEQ_CST; + +// === Barrier Strength Decorators === +// * AS_RAW: The access will translate into a raw memory access, hence ignoring all semantic concerns +// except memory ordering and compressed oops. This will bypass runtime function pointer dispatching +// in the pipeline and hardwire to raw accesses without going trough the GC access barriers. +// - Accesses on oop* translate to raw memory accesses without runtime checks +// - Accesses on narrowOop* translate to encoded/decoded memory accesses without runtime checks +// - Accesses on HeapWord* translate to a runtime check choosing one of the above +// - Accesses on other types translate to raw memory accesses without runtime checks +// * AS_DEST_NOT_INITIALIZED: This property can be important to e.g. SATB barriers by +// marking that the previous value is uninitialized nonsense rather than a real value. +// * AS_NO_KEEPALIVE: The barrier is used only on oop references and will not keep any involved objects +// alive, regardless of the type of reference being accessed. It will however perform the memory access +// in a consistent way w.r.t. e.g. concurrent compaction, so that the right field is being accessed, +// or maintain, e.g. intergenerational or interregional pointers if applicable. This should be used with +// extreme caution in isolated scopes. +// * AS_NORMAL: The accesses will be resolved to an accessor on the BarrierSet class, giving the +// responsibility of performing the access and what barriers to be performed to the GC. This is the default. +// Note that primitive accesses will only be resolved on the barrier set if the appropriate build-time +// decorator for enabling primitive barriers is enabled for the build. +const DecoratorSet AS_RAW = UCONST64(1) << 12; +const DecoratorSet AS_DEST_NOT_INITIALIZED = UCONST64(1) << 13; +const DecoratorSet AS_NO_KEEPALIVE = UCONST64(1) << 14; +const DecoratorSet AS_NORMAL = UCONST64(1) << 15; +const DecoratorSet AS_DECORATOR_MASK = AS_RAW | AS_DEST_NOT_INITIALIZED | + AS_NO_KEEPALIVE | AS_NORMAL; + +// === Reference Strength Decorators === +// These decorators only apply to accesses on oop-like types (oop/narrowOop). +// * ON_STRONG_OOP_REF: Memory access is performed on a strongly reachable reference. +// * ON_WEAK_OOP_REF: The memory access is performed on a weakly reachable reference. +// * ON_PHANTOM_OOP_REF: The memory access is performed on a phantomly reachable reference. +// This is the same ring of strength as jweak and weak oops in the VM. +// * ON_UNKNOWN_OOP_REF: The memory access is performed on a reference of unknown strength. +// This could for example come from the unsafe API. +// * Default (no explicit reference strength specified): ON_STRONG_OOP_REF +const DecoratorSet ON_STRONG_OOP_REF = UCONST64(1) << 16; +const DecoratorSet ON_WEAK_OOP_REF = UCONST64(1) << 17; +const DecoratorSet ON_PHANTOM_OOP_REF = UCONST64(1) << 18; +const DecoratorSet ON_UNKNOWN_OOP_REF = UCONST64(1) << 19; +const DecoratorSet ON_DECORATOR_MASK = ON_STRONG_OOP_REF | ON_WEAK_OOP_REF | + ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF; + +// === Access Location === +// Accesses can take place in, e.g. the heap, old or young generation and different native roots. +// The location is important to the GC as it may imply different actions. The following decorators are used: +// * IN_HEAP: The access is performed in the heap. Many barriers such as card marking will +// be omitted if this decorator is not set. +// * IN_HEAP_ARRAY: The access is performed on a heap allocated array. This is sometimes a special case +// for some GCs, and implies that it is an IN_HEAP. +// * IN_ROOT: The access is performed in an off-heap data structure pointing into the Java heap. +// * IN_CONCURRENT_ROOT: The access is performed in an off-heap data structure pointing into the Java heap, +// but is notably not scanned during safepoints. This is sometimes a special case for some GCs and +// implies that it is also an IN_ROOT. +const DecoratorSet IN_HEAP = UCONST64(1) << 20; +const DecoratorSet IN_HEAP_ARRAY = UCONST64(1) << 21; +const DecoratorSet IN_ROOT = UCONST64(1) << 22; +const DecoratorSet IN_CONCURRENT_ROOT = UCONST64(1) << 23; +const DecoratorSet IN_ARCHIVE_ROOT = UCONST64(1) << 24; +const DecoratorSet IN_DECORATOR_MASK = IN_HEAP | IN_HEAP_ARRAY | + IN_ROOT | IN_CONCURRENT_ROOT | + IN_ARCHIVE_ROOT; + +// == Value Decorators == +// * OOP_NOT_NULL: This property can make certain barriers faster such as compressing oops. +const DecoratorSet OOP_NOT_NULL = UCONST64(1) << 25; +const DecoratorSet OOP_DECORATOR_MASK = OOP_NOT_NULL; + +// == Arraycopy Decorators == +// * ARRAYCOPY_CHECKCAST: This property means that the class of the objects in source +// are not guaranteed to be subclasses of the class of the destination array. This requires +// a check-cast barrier during the copying operation. If this is not set, it is assumed +// that the array is covariant: (the source array type is-a destination array type) +// * ARRAYCOPY_DISJOINT: This property means that it is known that the two array ranges +// are disjoint. +// * ARRAYCOPY_ARRAYOF: The copy is in the arrayof form. +// * ARRAYCOPY_ATOMIC: The accesses have to be atomic over the size of its elements. +// * ARRAYCOPY_ALIGNED: The accesses have to be aligned on a HeapWord. +const DecoratorSet ARRAYCOPY_CHECKCAST = UCONST64(1) << 26; +const DecoratorSet ARRAYCOPY_DISJOINT = UCONST64(1) << 27; +const DecoratorSet ARRAYCOPY_ARRAYOF = UCONST64(1) << 28; +const DecoratorSet ARRAYCOPY_ATOMIC = UCONST64(1) << 29; +const DecoratorSet ARRAYCOPY_ALIGNED = UCONST64(1) << 30; +const DecoratorSet ARRAYCOPY_DECORATOR_MASK = ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT | + ARRAYCOPY_DISJOINT | ARRAYCOPY_ARRAYOF | + ARRAYCOPY_ATOMIC | ARRAYCOPY_ALIGNED; + +#endif // SHARE_OOPS_ACCESSDECORATORS_HPP diff --git a/src/hotspot/share/oops/oop.cpp b/src/hotspot/share/oops/oop.cpp index f7735a2..f4e6449 100644 --- a/src/hotspot/share/oops/oop.cpp +++ b/src/hotspot/share/oops/oop.cpp @@ -38,10 +38,6 @@ bool always_do_update_barrier = false; -bool oopDesc::equals(oop o1, oop o2) { - return Access<>::equals(o1, o2); -} - void oopDesc::print_on(outputStream* st) const { if (this == NULL) { st->print_cr("NULL"); diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 1df7763..467fc8d 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -142,7 +142,7 @@ class oopDesc { } } - static bool equals(oop o1, oop o2); + inline static bool equals(oop o1, oop o2) { return Access<>::equals(o1, o2); } // Access to fields in a instanceOop through these methods. template diff --git a/src/hotspot/share/runtime/handles.cpp b/src/hotspot/share/runtime/handles.cpp index ccc772c..4c902df 100644 --- a/src/hotspot/share/runtime/handles.cpp +++ b/src/hotspot/share/runtime/handles.cpp @@ -30,14 +30,6 @@ #include "runtime/handles.inline.hpp" #include "runtime/thread.inline.hpp" -bool Handle::operator == (oop o) const { - return oopDesc::equals(obj(), o); -} - -bool Handle::operator == (const Handle& h) const { - return oopDesc::equals(obj(), h.obj()); -} - #ifdef ASSERT oop* HandleArea::allocate_handle(oop obj) { assert(_handle_mark_nesting > 1, "memory leak: allocating handle outside HandleMark"); diff --git a/src/hotspot/share/runtime/handles.hpp b/src/hotspot/share/runtime/handles.hpp index 696d145..85285d5 100644 --- a/src/hotspot/share/runtime/handles.hpp +++ b/src/hotspot/share/runtime/handles.hpp @@ -78,8 +78,8 @@ class Handle { oop operator () () const { return obj(); } oop operator -> () const { return non_null_obj(); } - bool operator == (oop o) const; - bool operator == (const Handle& h) const; + bool operator == (oop o) const { return oopDesc::equals(obj(), o); } + bool operator == (const Handle& h) const { return oopDesc::equals(obj(), h.obj()); } // Null checks bool is_null() const { return _handle == NULL; } diff --git a/src/hotspot/share/services/memoryManager.hpp b/src/hotspot/share/services/memoryManager.hpp index 5bced05..46fe322 100644 --- a/src/hotspot/share/services/memoryManager.hpp +++ b/src/hotspot/share/services/memoryManager.hpp @@ -27,6 +27,7 @@ #include "gc/shared/gcCause.hpp" #include "memory/allocation.hpp" +#include "oops/oop.hpp" #include "oops/oopsHierarchy.hpp" #include "runtime/handles.hpp" #include "runtime/timer.hpp" diff --git a/src/hotspot/share/services/memoryPool.hpp b/src/hotspot/share/services/memoryPool.hpp index e2d2095..0d97fc3 100644 --- a/src/hotspot/share/services/memoryPool.hpp +++ b/src/hotspot/share/services/memoryPool.hpp @@ -26,6 +26,7 @@ #define SHARE_VM_SERVICES_MEMORYPOOL_HPP #include "memory/heap.hpp" +#include "oops/oop.hpp" #include "services/memoryUsage.hpp" #include "utilities/macros.hpp"