--- /dev/null	2017-03-07 11:44:12.271151064 +0100
+++ new/src/share/vm/runtime/access.inline.hpp	2017-04-25 16:46:38.431170893 +0200
@@ -0,0 +1,1224 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_RUNTIME_ACCESS_INLINE_HPP
+#define SHARE_VM_RUNTIME_ACCESS_INLINE_HPP
+
+#include "gc/shared/barrierSet.inline.hpp"
+#include "runtime/access.hpp"
+#include "runtime/accessBackend.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/orderAccess.inline.hpp"
+
+namespace AccessInternal {
+
+  template <DecoratorSet decorators, typename T, BarrierType type>
+  class AccessBarrierResolver: public AllStatic {
+  public:
+    static void* resolve_barrier();
+  };
+
+  template <DecoratorSet decorators>
+  class CloneAccessBarrierResolver: public AllStatic {
+  public:
+    static void* resolve_barrier();
+  };
+
+  template <DecoratorSet static_decorators, typename T, BarrierType type>
+  struct DispatcherFactory: public AllStatic {
+    static void* resolve_barrier_gc() {
+      return AccessBarrierResolver<static_decorators, T, type>::resolve_barrier();
+    }
+
+    static void* resolve_barrier_post_rt();
+
+    static void* resolve_barrier_rt() {
+      if (UseCompressedOops) {
+        return DispatcherFactory<static_decorators | RT_HAS_RUNTIME_DECORATOR | RT_USE_COMPRESSED_OOPS, T, type>::resolve_barrier_post_rt();
+      } else {
+        return DispatcherFactory<static_decorators | RT_HAS_RUNTIME_DECORATOR, T, type>::resolve_barrier_post_rt();
+      }
+    }
+
+    static void* resolve_barrier() {
+      return resolve_barrier_rt();
+    }
+  };
+
+  template <DecoratorSet static_decorators>
+  struct CloneDispatcherFactory: public AllStatic {
+    static void* resolve_barrier_gc() {
+      return CloneAccessBarrierResolver<static_decorators>::resolve_barrier();
+    }
+
+    static void* resolve_barrier_rt() {
+      if (UseCompressedOops) {
+        return CloneDispatcherFactory<static_decorators | RT_HAS_RUNTIME_DECORATOR | RT_USE_COMPRESSED_OOPS>::resolve_barrier_gc();
+      } else {
+        return CloneDispatcherFactory<static_decorators | RT_HAS_RUNTIME_DECORATOR>::resolve_barrier_gc();
+      }
+    }
+
+    static void* resolve_barrier() {
+      return resolve_barrier_rt();
+    }
+  };
+
+  template <DecoratorSet decorators>
+  struct CloneRuntimeDispatch {
+    typedef void (*clone_func_t)(oop src, oop dst, size_t size);
+
+    static void clone_init(oop src, oop dst, size_t size) {
+      _clone_func = (clone_func_t)CloneDispatcherFactory<decorators>::resolve_barrier();
+      _clone_func(src, dst, size);
+    }
+
+    static clone_func_t _clone_func;
+
+    static inline void clone(oop src, oop dst, size_t size) {
+      return _clone_func(src, dst, size);
+    }
+  };
+
+  template <DecoratorSet decorators, typename T>
+  struct RuntimeDispatch {
+    static void store_init(void* addr, T value) {
+      _store_func = (typename AccessFunctionTypes<decorators, T>::store_func_t)DispatcherFactory<decorators, T, BARRIER_STORE>::resolve_barrier();
+      _store_func(addr, value);
+    }
+
+    static void store_at_init(typename BaseType<decorators>::type base, ptrdiff_t offset, T value) {
+      _store_at_func = (typename AccessFunctionTypes<decorators, T>::store_at_func_t)DispatcherFactory<decorators, T, BARRIER_STORE_AT>::resolve_barrier();
+      _store_at_func(base, offset, value);
+    }
+
+    static T load_init(void* addr) {
+      _load_func = (typename AccessFunctionTypes<decorators, T>::load_func_t)DispatcherFactory<decorators, T, BARRIER_LOAD>::resolve_barrier();
+      return _load_func(addr);
+    }
+
+    static T load_at_init(typename BaseType<decorators>::type base, ptrdiff_t offset) {
+      _load_at_func = (typename AccessFunctionTypes<decorators, T>::load_at_func_t)DispatcherFactory<decorators, T, BARRIER_LOAD_AT>::resolve_barrier();
+      return _load_at_func(base, offset);
+    }
+
+    static T cas_init(T new_value, void* addr, T compare_value) {
+      _cas_func = (typename AccessFunctionTypes<decorators, T>::cas_func_t)DispatcherFactory<decorators, T, BARRIER_CAS>::resolve_barrier();
+      return _cas_func(new_value, addr, compare_value);
+    }
+
+    static T cas_at_init(T new_value, typename BaseType<decorators>::type base, ptrdiff_t offset, T compare_value) {
+      _cas_at_func = (typename AccessFunctionTypes<decorators, T>::cas_at_func_t)DispatcherFactory<decorators, T, BARRIER_CAS_AT>::resolve_barrier();
+      return _cas_at_func(new_value, base, offset, compare_value);
+    }
+
+    static T swap_init(T new_value, void* addr) {
+      _swap_func = (typename AccessFunctionTypes<decorators, T>::swap_func_t)DispatcherFactory<decorators, T, BARRIER_SWAP>::resolve_barrier();
+      return _swap_func(new_value, addr);
+    }
+
+    static T swap_at_init(T new_value, typename BaseType<decorators>::type base, ptrdiff_t offset) {
+      _swap_at_func = (typename AccessFunctionTypes<decorators, T>::swap_at_func_t)DispatcherFactory<decorators, T, BARRIER_SWAP_AT>::resolve_barrier();
+      return _swap_at_func(new_value, base, offset);
+    }
+
+    static bool copy_init(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
+      _copy_func = (typename AccessFunctionTypes<decorators, T>::copy_func_t)DispatcherFactory<decorators, T, BARRIER_COPY>::resolve_barrier();
+      return _copy_func(src_obj, dst_obj, src, dst, length);
+    }
+
+    static typename AccessFunctionTypes<decorators, T>::store_func_t    _store_func;
+    static typename AccessFunctionTypes<decorators, T>::store_at_func_t _store_at_func;
+    static typename AccessFunctionTypes<decorators, T>::load_func_t     _load_func;
+    static typename AccessFunctionTypes<decorators, T>::load_at_func_t  _load_at_func;
+    static typename AccessFunctionTypes<decorators, T>::cas_func_t      _cas_func;
+    static typename AccessFunctionTypes<decorators, T>::cas_at_func_t   _cas_at_func;
+    static typename AccessFunctionTypes<decorators, T>::swap_func_t     _swap_func;
+    static typename AccessFunctionTypes<decorators, T>::swap_at_func_t  _swap_at_func;
+    static typename AccessFunctionTypes<decorators, T>::copy_func_t     _copy_func;
+
+    static inline void store(void* addr, T value) {
+      _store_func(addr, value);
+    }
+
+    static inline void store_at(void* base, ptrdiff_t offset, T value) {
+      _store_at_func((typename BaseType<decorators>::type)base, offset, value);
+    }
+
+    static inline T load(void* addr) {
+      return _load_func(addr);
+    }
+
+    static inline T load_at(void* base, ptrdiff_t offset) {
+      return _load_at_func((typename BaseType<decorators>::type)base, offset);
+    }
+
+    static inline T cas(T new_value, void* addr, T compare_value) {
+      return _cas_func(new_value, addr, compare_value);
+    }
+
+    static inline T cas_at(T new_value, void* base, ptrdiff_t offset, T compare_value) {
+      return _cas_at_func(new_value, (typename BaseType<decorators>::type)base, offset, compare_value);
+    }
+
+    static inline T swap(T new_value, void* addr) {
+      return _swap_func(new_value, addr);
+    }
+
+    static inline T swap_at(T new_value, void* base, ptrdiff_t offset) {
+      return _swap_at_func(new_value, (typename BaseType<decorators>::type)base, offset);
+    }
+
+    static inline bool copy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
+      return _copy_func(src_obj, dst_obj, src, dst, length);
+    }
+  };
+
+  template <DecoratorSet decorators>
+  struct PreRuntimeDispatch {
+    typedef RawAccessBarrier<DecoratorIntersection<decorators, RAW_DECORATOR_MASK>::value> Raw;
+    typedef BasicAccessBarrier<DecoratorIntersection<decorators, BASIC_DECORATOR_MASK>::value> Basic;
+
+    enum {
+      CAN_HARDWIRE_BASIC = DecoratorTest<decorators>::HAS_USE_COMPRESSED_OOPS == DecoratorTest<decorators>::HAS_GC_CONVERT_COMPRESSED_OOP
+    };
+
+    template <DecoratorSet idecorators, typename T>
+    inline static typename EnableIf<DecoratorTest<idecorators>::HAS_ACCESS_RAW || DecoratorTest<idecorators>::HAS_ACCESS_BASIC, void>::type store(void* addr, T value) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
+        Raw::template store<T>(addr, value);
+      } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
+        if (!DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+          Basic::template store<T>(addr, value);
+        } else if (CAN_HARDWIRE_BASIC) {
+          Basic::template oop_store<T>(addr, value);
+        } else if (UseCompressedOops) {
+          const DecoratorSet expanded_decorators = decorators | RT_USE_COMPRESSED_OOPS;
+          PreRuntimeDispatch<expanded_decorators>::template store<expanded_decorators, T>(addr, value);
+        } else {
+          const DecoratorSet expanded_decorators = decorators & ~GC_CONVERT_COMPRESSED_OOP;
+          PreRuntimeDispatch<expanded_decorators>::template store<expanded_decorators, T>(addr, value);
+        }
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static typename EnableIf<!DecoratorTest<idecorators>::HAS_ACCESS_RAW && !DecoratorTest<idecorators>::HAS_ACCESS_BASIC, void>::type store(void* addr, T value) {
+      if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+        const DecoratorSet expanded_decorators = decorators | ACCESS_RAW;
+        PreRuntimeDispatch<expanded_decorators>::template store<expanded_decorators, T>(addr, value);
+      } else {
+        RuntimeDispatch<decorators, T>::store(addr, value);
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static typename EnableIf<DecoratorTest<idecorators>::HAS_ACCESS_RAW || DecoratorTest<idecorators>::HAS_ACCESS_BASIC, void>::type store_at(void* base, ptrdiff_t offset, T value) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
+        Raw::template store<T>(field_addr(base, offset), value);
+      } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
+        store<idecorators, T>(field_addr(base, offset), value);
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static typename EnableIf<!DecoratorTest<idecorators>::HAS_ACCESS_RAW && !DecoratorTest<idecorators>::HAS_ACCESS_BASIC, void>::type store_at(void* base, ptrdiff_t offset, T value) {
+      if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+        const DecoratorSet expanded_decorators = decorators | ACCESS_RAW;
+        PreRuntimeDispatch<expanded_decorators>::template store_at<expanded_decorators, T>(base, offset, value);
+      } else {
+        RuntimeDispatch<decorators, T>::store_at(base, offset, value);
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static typename EnableIf<DecoratorTest<idecorators>::HAS_ACCESS_RAW || DecoratorTest<idecorators>::HAS_ACCESS_BASIC, T>::type load(void* addr) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
+        return Raw::template load<T>(addr);
+      } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
+        if (!DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+          return Basic::template load<T>(addr);
+        } else if (CAN_HARDWIRE_BASIC) {
+          return Basic::template oop_load<T>(addr);
+        } else if (UseCompressedOops) {
+          const DecoratorSet expanded_decorators = decorators | RT_USE_COMPRESSED_OOPS;
+          return PreRuntimeDispatch<expanded_decorators>::template load<expanded_decorators, T>(addr);
+        } else {
+          const DecoratorSet expanded_decorators = decorators & ~GC_CONVERT_COMPRESSED_OOP;
+          return PreRuntimeDispatch<expanded_decorators>::template load<expanded_decorators, T>(addr);
+        }
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static typename EnableIf<!DecoratorTest<idecorators>::HAS_ACCESS_RAW && !DecoratorTest<idecorators>::HAS_ACCESS_BASIC, T>::type load(void* addr) {
+      if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+        const DecoratorSet expanded_decorators = decorators | ACCESS_RAW;
+        return PreRuntimeDispatch<expanded_decorators>::template load<expanded_decorators, T>(addr);
+      } else {
+        return RuntimeDispatch<decorators, T>::load(addr);
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static typename EnableIf<DecoratorTest<idecorators>::HAS_ACCESS_RAW || DecoratorTest<idecorators>::HAS_ACCESS_BASIC, T>::type load_at(void* base, ptrdiff_t offset) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
+        return Raw::template load<T>(field_addr(base, offset));
+      } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
+        return load<decorators, T>(field_addr(base, offset));
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static typename EnableIf<!DecoratorTest<idecorators>::HAS_ACCESS_RAW && !DecoratorTest<idecorators>::HAS_ACCESS_BASIC, T>::type load_at(void* base, ptrdiff_t offset) {
+      if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+        const DecoratorSet expanded_decorators = decorators | ACCESS_RAW;
+        return PreRuntimeDispatch<expanded_decorators>::template load_at<expanded_decorators, T>(base, offset);
+      } else {
+        return RuntimeDispatch<decorators, T>::load_at(base, offset);
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static T cas(T new_value, void* addr, T compare_value) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
+        return Raw::template cas<T>(new_value, addr, compare_value);
+      } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
+        if (!DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+          return Basic::template cas<T>(new_value, addr, compare_value);
+        } else if (CAN_HARDWIRE_BASIC) {
+          return Basic::template oop_cas<T>(new_value, addr, compare_value);
+        } else if (UseCompressedOops) {
+          const DecoratorSet expanded_decorators = decorators | RT_USE_COMPRESSED_OOPS;
+          return PreRuntimeDispatch<expanded_decorators>::template cas<expanded_decorators, T>(new_value, addr, compare_value);
+        } else {
+          const DecoratorSet expanded_decorators = decorators & ~GC_CONVERT_COMPRESSED_OOP;
+          return PreRuntimeDispatch<expanded_decorators>::template cas<expanded_decorators, T>(new_value, addr, compare_value);
+        }
+      } else if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+        return Raw::template cas<T>(new_value, addr, compare_value);
+      } else {
+        return RuntimeDispatch<decorators, T>::cas(new_value, addr, compare_value);
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static T cas_at(T new_value, void* base, ptrdiff_t offset, T compare_value) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
+        return Raw::template cas<T>(new_value, field_addr(base, offset), compare_value);
+      } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
+        return cas<idecorators, T>(new_value, field_addr(base, offset), compare_value);
+      } else if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+        return Raw::template cas<T>(new_value, field_addr(base, offset), compare_value);
+      } else {
+        return RuntimeDispatch<decorators, T>::cas_at(new_value, base, offset, compare_value);
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static T swap(T new_value, void* addr) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
+        return Raw::template swap<T>(new_value, addr);
+      } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
+        if (!DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+          return Basic::template swap<T>(new_value, addr);
+        } else if (CAN_HARDWIRE_BASIC) {
+          return Basic::template oop_swap<T>(new_value, addr);
+        } else if (UseCompressedOops) {
+          const DecoratorSet expanded_decorators = decorators | RT_USE_COMPRESSED_OOPS;
+          return PreRuntimeDispatch<expanded_decorators>::template swap<expanded_decorators, T>(new_value, addr);
+        } else {
+          const DecoratorSet expanded_decorators = decorators & ~GC_CONVERT_COMPRESSED_OOP;
+          return PreRuntimeDispatch<expanded_decorators>::template swap<expanded_decorators, T>(new_value, addr);
+        }
+      } else if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+        return Raw::template swap<T>(new_value, addr);
+      } else {
+        return RuntimeDispatch<decorators, T>::swap(new_value, addr);
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static T swap_at(T new_value, void* base, ptrdiff_t offset) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
+        return Raw::template swap<T>(new_value, field_addr(base, offset));
+      } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
+        return swap<idecorators, T>(new_value, field_addr(base, offset));
+      } else if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+        return Raw::template swap<T>(new_value, field_addr(base, offset));
+      } else {
+        return RuntimeDispatch<decorators, T>::swap_at(new_value, base, offset);
+      }
+    }
+
+    template <DecoratorSet idecorators, typename T>
+    inline static bool copy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
+        return Raw::template copy<T>(src, dst, length);
+      } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
+        if (!DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+          return Basic::template copy<T>(src_obj, dst_obj, src, dst, length);
+        } else {
+          return Basic::template oop_copy<T>(src_obj, dst_obj, src, dst, length);
+        }
+      } else if (!DecoratorTest<decorators>::HAS_BARRIER_ON_PRIMITIVES && !DecoratorTest<decorators>::HAS_VALUE_IS_OOP) {
+        return Raw::template copy<T>(src, dst, length);
+      } else {
+        return RuntimeDispatch<decorators, T>::copy(src_obj, dst_obj, src, dst, length);
+      }
+    }
+
+    template <DecoratorSet idecorators>
+    inline static void clone(oop src, oop dst, size_t size) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_RAW) {
+        return Raw::clone(src, dst, size);
+      } else if (DecoratorTest<decorators>::HAS_ACCESS_BASIC) {
+        return Basic::clone(src, dst, size);
+      } else {
+        return CloneRuntimeDispatch<decorators>::clone(src, dst, size);
+      }
+    }
+  };
+
+  template <DecoratorSet decorators>
+  struct BuildtimeDispatch {
+    template <typename T>
+    static void store(void* addr, T value);
+    template <typename T>
+    static void store_at(void* base, ptrdiff_t offset, T value);
+    template <typename T>
+    static T load(void* addr);
+    template <typename T>
+    static T load_at(void* base, ptrdiff_t offset);
+    template <typename T>
+    static T cas(T new_value, void* addr, T compare_value);
+    template <typename T>
+    static T cas_at(T new_value, void* base, ptrdiff_t offset, T compare_value);
+    template <typename T>
+    static T swap(T new_value, void* addr);
+    template <typename T>
+    static T swap_at(T new_value, void* base, ptrdiff_t offset);
+    template <typename T>
+    static bool copy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length);
+    static void clone(oop src, oop dst, size_t size);
+  };
+
+  /**
+   * This class adds implied properties that follow according to decorator rules
+   */
+  template <DecoratorSet input_decorators>
+  struct DecoratorFixup {
+    enum {
+      ref_strong_default = input_decorators | ((!DecoratorTest<input_decorators>::HAS_GC_ACCESS_ON_PHANTOM
+                                                && !DecoratorTest<input_decorators>::HAS_GC_ACCESS_ON_WEAK) ? GC_ACCESS_ON_STRONG : EMPTY_DECORATOR)
+    };
+    enum {
+      seqcst_release = ref_strong_default | (DecoratorTest<ref_strong_default>::HAS_MO_SEQ_CST ? MO_RELEASE : EMPTY_DECORATOR)
+    };
+    enum {
+      seqcst_acquire = seqcst_release | (DecoratorTest<seqcst_release>::HAS_MO_SEQ_CST ? MO_ACQUIRE : EMPTY_DECORATOR)
+    };
+    enum {
+      release_atomic = seqcst_acquire | (DecoratorTest<seqcst_acquire>::HAS_MO_RELEASE ? MO_ATOMIC : EMPTY_DECORATOR)
+    };
+    enum {
+      acquire_atomic = release_atomic | (DecoratorTest<release_atomic>::HAS_MO_ACQUIRE ? MO_ATOMIC : EMPTY_DECORATOR)
+    };
+    enum {
+      atomic_volatile = acquire_atomic | (DecoratorTest<acquire_atomic>::HAS_MO_ATOMIC ? MO_VOLATILE : EMPTY_DECORATOR)
+    };
+    enum {
+      novolatile_relaxed = atomic_volatile | (!DecoratorTest<atomic_volatile>::HAS_MO_VOLATILE ? MO_RELAXED : EMPTY_DECORATOR)
+    };
+    enum {
+      relaxed_root = novolatile_relaxed | ((DecoratorTest<novolatile_relaxed>::HAS_VALUE_IS_OOP &&
+                                            (DecoratorTest<novolatile_relaxed>::HAS_ACCESS_ON_NMETHOD ||
+                                             DecoratorTest<novolatile_relaxed>::HAS_ACCESS_ON_KLASS)) ?
+                                             ACCESS_ON_ROOT : EMPTY_DECORATOR)
+    };
+    enum {
+      value = relaxed_root
+    };
+  };
+
+  // Only let invocations where P and T are the same through
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<IsDerived<P, T>::value, void>::type
+  store_reduce_types(P* addr, T value) {
+    BuildtimeDispatch<decorators>::template store<T>((void*)addr, value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<!IsDerived<P, T>::value, void>::type
+  store_reduce_types(narrowOop* addr, oop value) {
+    const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP | RT_USE_COMPRESSED_OOPS;
+    BuildtimeDispatch<expanded_decorators>::template store<oop>((void*)addr, value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<!IsDerived<P, T>::value, void>::type
+  store_reduce_types(HeapWord* addr, oop value) {
+    const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP;
+    BuildtimeDispatch<expanded_decorators>::template store<oop>((void*)addr, value);
+  }
+
+  // If the decayed pointer and value types are the same, we decay their types and
+  // send them to the next stage in the pipeline
+  template <DecoratorSet decorators, typename P, typename T>
+  inline void store_decay_types(P* addr, T value) {
+    typedef typename Decay<P>::type DecayedP;
+    typedef typename Decay<T>::type DecayedT;
+    const DecoratorSet expanded_decorators = IsVolatile<P>::value ? DecoratorFixup<MO_VOLATILE | decorators>::value : decorators;
+    store_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedP*)addr, (DecayedT)value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  void store_at_reduce_types(P base, ptrdiff_t offset, T value);
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline void store_at_decay_types(P base, ptrdiff_t offset, T value) {
+    typedef typename Decay<P>::type DecayedP;
+    typedef typename Decay<T>::type DecayedT;
+    store_at_reduce_types<decorators, DecayedP, DecayedT>((DecayedP)base, offset, (DecayedT)value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  T cas_at_reduce_types(T new_value, P base, ptrdiff_t offset, T compare_value);
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T cas_at_decay_types(T new_value, P base, ptrdiff_t offset, T compare_value) {
+    typedef typename Decay<P>::type DecayedP;
+    typedef typename Decay<T>::type DecayedT;
+    const DecoratorSet memory_ordering = !DecoratorTest<decorators>::HAS_MO_SEQ_CST ? MO_ATOMIC : MO_SEQ_CST;
+    const DecoratorSet expanded_decorators = DecoratorFixup<decorators | memory_ordering>::value;
+    return cas_at_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedT)new_value, (DecayedP)base, offset, (DecayedT)compare_value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<IsDerived<P, T>::value, T>::type cas_reduce_types(T new_value, P* addr, T compare_value) {
+    return BuildtimeDispatch<decorators>::template cas<T>(new_value, (void*)addr, compare_value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<!IsDerived<P, T>::value, oop>::type cas_reduce_types(oop new_value, narrowOop* addr, oop compare_value) {
+    const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP | RT_USE_COMPRESSED_OOPS;
+    return BuildtimeDispatch<expanded_decorators>::template cas<oop>(new_value, (void*)addr, compare_value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<!IsDerived<P, T>::value, oop>::type cas_reduce_types(oop new_value, HeapWord* addr, oop compare_value) {
+    const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP;
+    return BuildtimeDispatch<expanded_decorators>::template cas<oop>(new_value, (void*)addr, compare_value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T cas_decay_types(T new_value, P* addr, T compare_value) {
+    typedef typename Decay<P>::type DecayedP;
+    typedef typename Decay<T>::type DecayedT;
+    const DecoratorSet memory_ordering = !DecoratorTest<decorators>::HAS_MO_SEQ_CST ? MO_ATOMIC : MO_SEQ_CST;
+    const DecoratorSet expanded_decorators = DecoratorFixup<decorators | memory_ordering>::value;
+    return (T)cas_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedT)new_value, (DecayedP*)addr, (DecayedT)compare_value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T swap_at_reduce_types(T new_value, P base, ptrdiff_t offset);
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T swap_at_decay_types(T new_value, P base, ptrdiff_t offset) {
+    typedef typename Decay<T>::type DecayedT;
+    typedef typename Decay<P>::type DecayedP;
+    const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
+    return swap_at_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedT)new_value, (DecayedP)base, offset);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<IsDerived<P, T>::value, T>::type swap_reduce_types(T new_value, P* addr) {
+    return BuildtimeDispatch<decorators>::template swap<T>(new_value, (void*)addr);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<!IsDerived<P, T>::value, oop>::type swap_reduce_types(oop new_value, narrowOop* addr) {
+    const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP | RT_USE_COMPRESSED_OOPS;
+    return BuildtimeDispatch<expanded_decorators>::template swap<oop>(new_value, (void*)addr);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<!IsDerived<P, T>::value, oop>::type
+  swap_reduce_types(oop new_value, HeapWord* addr) {
+    const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP;
+    return BuildtimeDispatch<expanded_decorators>::template swap<oop>(new_value, (void*)addr);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T swap_decay_types(T new_value, P* addr) {
+    typedef typename Decay<P>::type DecayedP;
+    typedef typename Decay<T>::type DecayedT;
+    const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
+    return (T)swap_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedT)new_value, (DecayedP*)addr);
+  }
+
+  template <DecoratorSet decorators, typename T>
+  inline typename EnableIf<!DecoratorTest<decorators>::HAS_VALUE_IS_OOP, bool>::type copy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
+    return BuildtimeDispatch<decorators | ACCESS_ON_HEAP>::template copy<T>(src_obj, dst_obj, src, dst, length);
+  }
+
+  template <DecoratorSet decorators, typename T>
+  inline typename EnableIf<DecoratorTest<decorators>::HAS_VALUE_IS_OOP, bool>::type copy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
+    return BuildtimeDispatch<decorators | ACCESS_ON_HEAP | GC_CONVERT_COMPRESSED_OOP>::template copy<T>(src_obj, dst_obj, src, dst, length);
+  }
+
+  template <DecoratorSet decorators, typename T>
+  inline bool copy_decay_types(arrayOop src_obj, arrayOop dst_obj, T *src, T *dst, size_t length) {
+    typedef typename Decay<T>::type DecayedT;
+    return copy_reduce_types<DecoratorFixup<decorators>::value, DecayedT>(src_obj, dst_obj, (DecayedT*)src, (DecayedT*)dst, length);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<IsDerived<P, T>::value, T>::type load_reduce_types(P* addr) {
+    return BuildtimeDispatch<decorators>::template load<T>((void*)addr);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<!IsDerived<P, T>::value, oop>::type load_reduce_types(narrowOop* addr) {
+    const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP | RT_USE_COMPRESSED_OOPS;
+    return BuildtimeDispatch<expanded_decorators>::template load<oop>((void*)addr);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline typename EnableIf<!IsDerived<P, T>::value, oop>::type load_reduce_types(HeapWord* addr) {
+    const DecoratorSet expanded_decorators = decorators | GC_CONVERT_COMPRESSED_OOP;
+    return BuildtimeDispatch<expanded_decorators>::template load<oop>((void*)addr);
+  }
+
+  // If the decayed pointer and value types are the same, we decay their types and
+  // send them to the next stage in the pipeline
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T load_decay_types(P* addr) {
+    typedef typename Decay<P>::type DecayedP;
+    typedef typename TypeIf<DecoratorTest<decorators>::HAS_VALUE_IS_OOP,
+                            typename OopOrNarrowOop<T>::type, typename Decay<T>::type>::type DecayedT;
+    const DecoratorSet expanded_decorators = IsVolatile<P>::value ? DecoratorFixup<MO_VOLATILE | decorators>::value : decorators;
+    return (T)load_reduce_types<expanded_decorators, DecayedP, DecayedT>((DecayedP*)addr);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  class DispatchLoadAtReduceTypes {
+  public:
+    static T load_at(P base, ptrdiff_t offset);
+  };
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T load_at_decay_types(P base, ptrdiff_t offset) {
+    typedef typename Decay<P>::type DecayedP;
+    typedef typename TypeIf<DecoratorTest<decorators>::HAS_VALUE_IS_OOP,
+                            typename OopOrNarrowOop<T>::type, typename Decay<T>::type>::type DecayedT;
+    return (T)DispatchLoadAtReduceTypes<decorators, DecayedP, DecayedT>::load_at((DecayedP)base, offset);
+  }
+
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline void store(P* addr, T value) {
+    store_decay_types<DecoratorFixup<decorators>::value, P, T>(addr, value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline void store_at(P base, ptrdiff_t offset, T value) {
+    store_at_decay_types<DecoratorFixup<decorators>::value, P, T>(base, offset, value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T load(P* addr) {
+    return load_decay_types<DecoratorFixup<decorators>::value, P, T>(addr);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T load_at(P base, ptrdiff_t offset) {
+    return load_at_decay_types<DecoratorFixup<decorators>::value, P, T>(base, offset);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T cas(T new_value, P* addr, T compare_value) {
+    return cas_decay_types<DecoratorFixup<decorators>::value, P, T>(new_value, addr, compare_value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T cas_at(T new_value, P base, ptrdiff_t offset, T compare_value) {
+    return cas_at_decay_types<DecoratorFixup<decorators>::value, P, T>(new_value, base, offset, compare_value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T swap(T new_value, P* addr) {
+    return swap_decay_types<DecoratorFixup<decorators>::value, P, T>(new_value, addr);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T swap_at(T new_value, P base, ptrdiff_t offset) {
+    return swap_at_decay_types<DecoratorFixup<decorators>::value, P, T>(new_value, base, offset);
+  }
+
+  template <DecoratorSet decorators, typename T>
+  inline bool copy(arrayOop src_obj, arrayOop dst_obj, T *src, T *dst, size_t length) {
+    return copy_decay_types<DecoratorFixup<decorators>::value, T>(src_obj, dst_obj, src, dst, length);
+  }
+
+  template <DecoratorSet decorators>
+  inline void clone(oop src, oop dst, size_t size) {
+    BuildtimeDispatch<DecoratorFixup<decorators>::value>::clone(src, dst, size);
+  }
+
+  template <DecoratorSet decorators, typename T>
+  typename AccessFunctionTypes<decorators, T>::store_func_t    RuntimeDispatch<decorators, T>::_store_func = &store_init;
+
+  template <DecoratorSet decorators, typename T>
+  typename AccessFunctionTypes<decorators, T>::store_at_func_t RuntimeDispatch<decorators, T>::_store_at_func = &store_at_init;
+
+  template <DecoratorSet decorators, typename T>
+  typename AccessFunctionTypes<decorators, T>::load_func_t     RuntimeDispatch<decorators, T>::_load_func = &load_init;
+
+  template <DecoratorSet decorators, typename T>
+  typename AccessFunctionTypes<decorators, T>::load_at_func_t  RuntimeDispatch<decorators, T>::_load_at_func = &load_at_init;
+
+  template <DecoratorSet decorators, typename T>
+  typename AccessFunctionTypes<decorators, T>::cas_func_t      RuntimeDispatch<decorators, T>::_cas_func = &cas_init;
+
+  template <DecoratorSet decorators, typename T>
+  typename AccessFunctionTypes<decorators, T>::cas_at_func_t   RuntimeDispatch<decorators, T>::_cas_at_func = &cas_at_init;
+
+  template <DecoratorSet decorators, typename T>
+  typename AccessFunctionTypes<decorators, T>::swap_func_t     RuntimeDispatch<decorators, T>::_swap_func = &swap_init;
+
+  template <DecoratorSet decorators, typename T>
+  typename AccessFunctionTypes<decorators, T>::swap_at_func_t  RuntimeDispatch<decorators, T>::_swap_at_func = &swap_at_init;
+
+  template <DecoratorSet decorators, typename T>
+  typename AccessFunctionTypes<decorators, T>::copy_func_t     RuntimeDispatch<decorators, T>::_copy_func = &copy_init;
+
+  template <DecoratorSet decorators>
+  typename CloneRuntimeDispatch<decorators>::clone_func_t      CloneRuntimeDispatch<decorators>::_clone_func = &clone_init;
+}
+
+template <DecoratorSet decorators>
+template <DecoratorSet internal_decorators, typename T>
+inline typename EnableIf<TestEncodable<internal_decorators, T>::value, T>::type
+BasicAccessBarrier<decorators>::decode_internal(typename EncodedType<internal_decorators, T>::type value) {
+  if (DecoratorTest<decorators>::HAS_VALUE_NOT_NULL) {
+    return oopDesc::decode_heap_oop_not_null(value);
+  } else {
+    return oopDesc::decode_heap_oop(value);
+  }
+}
+
+template <DecoratorSet decorators>
+template <DecoratorSet internal_decorators, typename T>
+inline typename EnableIf<TestEncodable<internal_decorators, T>::value,
+                         typename EncodedType<internal_decorators, T>::type>::type
+BasicAccessBarrier<decorators>::encode_internal(T value) {
+  if (DecoratorTest<decorators>::HAS_VALUE_NOT_NULL) {
+    return oopDesc::encode_heap_oop_not_null(value);
+  } else {
+    return oopDesc::encode_heap_oop(value);
+  }
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline void BasicAccessBarrier<decorators>::oop_store(void* addr, T value) {
+  typedef typename EncodedType<decorators, T>::type Encoded;
+  Encoded encoded = encode<T>(value);
+  Raw::template store<Encoded>((Encoded*)addr, encoded);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline void BasicAccessBarrier<decorators>::oop_store_at(void* base, ptrdiff_t offset, T value) {
+  oop_store(field_addr(base, offset), value);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline T BasicAccessBarrier<decorators>::oop_load(void* addr) {
+  typedef typename EncodedType<decorators, T>::type Encoded;
+  Encoded encoded = Raw::template load<Encoded>((Encoded*)addr);
+  return decode<T>(encoded);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline T BasicAccessBarrier<decorators>::oop_load_at(void* base, ptrdiff_t offset) {
+  return oop_load<T>(field_addr(base, offset));
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline T BasicAccessBarrier<decorators>::oop_cas(T new_value, void* addr, T compare_value) {
+  typedef typename EncodedType<decorators, T>::type Encoded;
+  Encoded encoded_new = encode<T>(new_value);
+  Encoded encoded_compare = encode<T>(compare_value);
+  Encoded encoded_result = Raw::template cas<Encoded>(encoded_new, (Encoded*)addr, encoded_compare);
+  return decode<T>(encoded_result);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline T BasicAccessBarrier<decorators>::oop_cas_at(T new_value, void* base, ptrdiff_t offset, T compare_value) {
+  return oop_cas<T>(new_value, field_addr(base, offset), compare_value);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline T BasicAccessBarrier<decorators>::oop_swap(T new_value, void* addr) {
+  typedef typename EncodedType<decorators, T>::type Encoded;
+  Encoded encoded_new = encode<T>(new_value);
+  Encoded encoded_result = Raw::template swap<Encoded>(encoded_new, (Encoded*)addr);
+  return decode<T>(encoded_result);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline T BasicAccessBarrier<decorators>::oop_swap_at(T new_value, void* base, ptrdiff_t offset) {
+  return oop_swap<T>(new_value, field_addr(base, offset));
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline bool BasicAccessBarrier<decorators>::oop_copy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) {
+  if (IsSame<T, HeapWord>::value) {
+    if (DecoratorTest<decorators>::NEEDS_OOP_COMPRESS) {
+      return Raw::template copy<narrowOop>((narrowOop*)src, (narrowOop*)dst, length);
+    } else {
+      return Raw::template copy<oop>((oop*)src, (oop*)dst, length);
+    }
+  } else {
+    return Raw::template copy<T>(src, dst, length);
+  }
+}
+
+template <DecoratorSet decorators>
+inline void BasicAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
+  Raw::clone(src, dst, size);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<IsPointerSize<T>::value, void>::type
+RawAccessBarrier<decorators>::atomic_store(void* addr, T value) {
+  Atomic::store_ptr((void*)value, (volatile void*)addr);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<!IsPointerSize<T>::value, void>::type
+RawAccessBarrier<decorators>::atomic_store(void* addr, T value) {
+  Atomic::store(IntegerType<T>::cast_to_signed(value), (volatile typename IntegerType<T>::signed_type*)addr);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<IsPointerSize<T>::value, void>::type
+RawAccessBarrier<decorators>::release_store(void* addr, T value) {
+  OrderAccess::release_store_ptr((volatile void*)addr, (void*)value);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<!IsPointerSize<T>::value, void>::type
+RawAccessBarrier<decorators>::release_store(void* addr, T value) {
+  OrderAccess::release_store((volatile typename IntegerType<T>::signed_type*)addr, IntegerType<T>::cast_to_signed(value));
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+typename EnableIf<IsPointerSize<T>::value, void>::type
+RawAccessBarrier<decorators>::release_store_fence(void* addr, T value) {
+  OrderAccess::release_store_ptr_fence((volatile void*)addr, (void*)value);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<!IsPointerSize<T>::value, void>::type
+RawAccessBarrier<decorators>::release_store_fence(void* addr, T value) {
+  OrderAccess::release_store_fence((volatile typename IntegerType<T>::signed_type*)addr, IntegerType<T>::cast_to_signed(value));
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::atomic_load(void* addr) {
+  return (T)Atomic::load_ptr((volatile void*)addr);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<!IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::atomic_load(void* addr) {
+  return (T)Atomic::load((volatile typename IntegerType<T>::signed_type*)addr);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::load_acquire(void* addr) {
+  return (T)OrderAccess::load_ptr_acquire((volatile void*)addr);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<!IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::load_acquire(void* addr) {
+  return (T)OrderAccess::load_acquire((volatile typename IntegerType<T>::signed_type*)addr);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::fence_load_acquire(void* addr) {
+  if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+    OrderAccess::fence();
+  }
+  return (T)OrderAccess::load_ptr_acquire((volatile void*)addr);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<!IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::fence_load_acquire(void* addr) {
+  if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+    OrderAccess::fence();
+  }
+  return (T)OrderAccess::load_acquire((volatile typename IntegerType<T>::signed_type*)addr);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::cas_relaxed(T new_value, void* addr, T compare_value) {
+  return (T)Atomic::cmpxchg_ptr((void*)new_value,
+                                (volatile void*)addr,
+                                (void*)compare_value,
+                                memory_order_relaxed);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<!IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::cas_relaxed(T new_value, void* addr, T compare_value) {
+  return (T)Atomic::cmpxchg((typename IntegerType<T>::signed_type)new_value,
+                            (volatile typename IntegerType<T>::signed_type*)addr,
+                            (typename IntegerType<T>::signed_type)compare_value,
+                            memory_order_relaxed);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::cas_seq_cst(T new_value, void* addr, T compare_value) {
+  return (T)Atomic::cmpxchg_ptr((void*)new_value,
+                                (volatile void*)addr,
+                                (void*)compare_value,
+                                memory_order_conservative);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<!IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::cas_seq_cst(T new_value, void* addr, T compare_value) {
+  return (T)Atomic::cmpxchg((typename IntegerType<T>::signed_type)new_value,
+                            (volatile typename IntegerType<T>::signed_type*)addr,
+                            (typename IntegerType<T>::signed_type)compare_value,
+                            memory_order_conservative);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::swap_seq_cst(T new_value, void* addr) {
+  return (T)Atomic::xchg_ptr((void*)new_value,
+                             (volatile void*)addr);
+}
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<!IsPointerSize<T>::value, T>::type
+RawAccessBarrier<decorators>::swap_seq_cst(T new_value, void* addr) {
+  return (T)Atomic::xchg((typename IntegerType<T>::signed_type)new_value,
+                         (volatile typename IntegerType<T>::signed_type*)addr);
+}
+
+template <DecoratorSet ds>
+template <typename T, DecoratorSet decorators>
+inline typename EnableIf<PossiblyLockedAccess<T, decorators>::value, T>::type
+RawAccessBarrier<ds>::load_maybe_locked(void* addr) {
+  STATIC_ASSERT(DecoratorTest<ds>::HAS_ACCESS_ON_ANONYMOUS); // why else?
+  if (AccessInternal::wide_atomic_needs_locking()) {
+    return load_not_locked<T>(addr);
+  } else {
+    return (T)AccessInternal::load_locked(addr);
+  }
+}
+
+template <DecoratorSet ds>
+template <typename T, DecoratorSet decorators>
+inline typename EnableIf<PossiblyLockedAccess<T, decorators>::value, void>::type
+RawAccessBarrier<ds>::store_maybe_locked(void* addr, T value) {
+  STATIC_ASSERT(DecoratorTest<ds>::HAS_ACCESS_ON_ANONYMOUS); // why else?
+  if (AccessInternal::wide_atomic_needs_locking()) {
+    store_not_locked<T>(addr, value);
+  } else {
+    AccessInternal::store_locked(addr, IntegerType<T>::cast_to_signed(value));
+  }
+}
+
+template <DecoratorSet ds>
+template <typename T, DecoratorSet decorators>
+inline typename EnableIf<PossiblyLockedAccess<T, decorators>::value, T>::type
+RawAccessBarrier<ds>::swap_maybe_locked(T new_value, void* addr) {
+  STATIC_ASSERT(DecoratorTest<ds>::HAS_ACCESS_ON_ANONYMOUS); // why else?
+  if (AccessInternal::wide_atomic_needs_locking()) {
+    return swap_not_locked<T>(new_value, addr);
+  } else {
+    return (T)AccessInternal::swap_locked(IntegerType<T>::cast_to_signed(new_value), addr);
+  }
+}
+
+template <DecoratorSet ds>
+template <typename T, DecoratorSet decorators>
+inline typename EnableIf<PossiblyLockedAccess<T, decorators>::value, T>::type
+RawAccessBarrier<ds>::cas_maybe_locked(T new_value, void* addr, T compare_value) {
+  STATIC_ASSERT(DecoratorTest<ds>::HAS_ACCESS_ON_ANONYMOUS); // why else?
+  if (AccessInternal::wide_atomic_needs_locking()) {
+    return cas_not_locked<T>(new_value, addr, compare_value);
+  } else {
+    return (T)AccessInternal::cas_locked(IntegerType<T>::cast_to_signed(new_value), addr,
+                                         IntegerType<T>::cast_to_signed(compare_value));
+  }
+}
+
+class RawAccessBarrierCopy: public AllStatic {
+public:
+  template <DecoratorSet decorators, typename T>
+  static inline typename EnableIf<DecoratorTest<decorators>::HAS_VALUE_IS_OOP &&
+                                  !IsSame<T, HeapWord>::value, void>::type
+  copy(T* src, T* dst, size_t length) {
+    if (DecoratorTest<decorators>::HAS_COPY_ARRAYOF) {
+      AccessInternal::copy_arrayof_conjoint_oops((void*)src, (void*)dst, length);
+    } else {
+      AccessInternal::copy_conjoint_oops((void*)src, (void*)dst, length);
+    }
+  }
+
+  template <DecoratorSet decorators, typename T>
+  static inline typename EnableIf<!DecoratorTest<decorators>::HAS_VALUE_IS_OOP ||
+                                  IsSame<T, HeapWord>::value, void>::type
+  copy(T* src, T* dst, size_t length) {
+    if (DecoratorTest<decorators>::HAS_DEST_CONJOINT) {
+      if (DecoratorTest<decorators>::HAS_ACCESS_ATOMIC) {
+        AccessInternal::copy_conjoint_memory_atomic((void*)src, (void*)dst, sizeof(T) * length);
+      } else {
+        AccessInternal::copy_conjoint_jbytes((void*)src, (void*)dst, sizeof(T) * length);
+      }
+    } else {
+      if (DecoratorTest<decorators>::HAS_ACCESS_ATOMIC) {
+        AccessInternal::copy_conjoint_memory_atomic((void*)src, (void*)dst, sizeof(T) * length);
+      } else if (sizeof(T) == HeapWordSize) {
+        AccessInternal::copy_disjoint_words((void*)src, (void*)dst, sizeof(T) * length);
+      } else {
+        AccessInternal::copy_conjoint_jbytes((void*)src, (void*)dst, sizeof(T) * length);
+      }
+    }
+  }
+};
+
+template <DecoratorSet decorators>
+template <typename T>
+inline typename EnableIf<!IsFloatLike<T>::value, bool>::type
+RawAccessBarrier<decorators>::copy(T* src, T* dst, size_t length) {
+  RawAccessBarrierCopy::copy<decorators, T>(src, dst, length);
+  return true;
+}
+
+template <DecoratorSet decorators>
+inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
+  assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
+  AccessInternal::copy_conjoint_jlongs_atomic((void*)src, (void*)dst,
+                                              (size_t)align_object_size(size) / HeapWordsPerLong);
+  // Clear the header
+  dst->init_mark();
+}
+
+namespace AccessInternal {
+  // The first time a barrier needs to be resolved at runtime, it needs
+  // to link it against the GC being used to patch a function pointer
+  // directing subsequent memory accesses to the resolved barrier
+  template <DecoratorSet decorators, typename T, BarrierType barrier_type>
+  void* AccessBarrierResolver<decorators, T, barrier_type>::resolve_barrier() {
+    return BarrierSet::barrier_set()->resolve_barrier<decorators, T, barrier_type>();
+  }
+
+  template <DecoratorSet decorators>
+  void* CloneAccessBarrierResolver<decorators>::resolve_barrier() {
+    return BarrierSet::barrier_set()->resolve_clone_barrier<decorators>();
+  }
+
+  template <DecoratorSet static_decorators, typename T, BarrierType type>
+  class PostRTDispatch: public AllStatic {
+    enum {
+      resolve_compressed_oops = type == BARRIER_COPY && DecoratorTest<static_decorators>::HAS_VALUE_IS_OOP &&
+                                IsSame<T, HeapWord>::value
+    };
+
+    template <bool resolve_compressed_oops>
+    static typename EnableIf<resolve_compressed_oops, void*>::type resolve_barrier_internal() {
+      if (DecoratorTest<static_decorators>::NEEDS_OOP_COMPRESS) {
+        return DispatcherFactory<static_decorators, narrowOop, type>::resolve_barrier_gc();
+      } else {
+        return DispatcherFactory<static_decorators, oop, type>::resolve_barrier_gc();
+      }
+    }
+
+    template <bool resolve_compressed_oops>
+    static typename EnableIf<!resolve_compressed_oops, void*>::type resolve_barrier_internal() {
+      return DispatcherFactory<static_decorators, T, type>::resolve_barrier_gc();
+    }
+  public:
+    static void* resolve_barrier() {
+      return resolve_barrier_internal<resolve_compressed_oops>();
+    }
+  };
+
+  template <DecoratorSet static_decorators, typename T, BarrierType type>
+  inline void* DispatcherFactory<static_decorators, T, type>::resolve_barrier_post_rt() {
+    return PostRTDispatch<static_decorators, T, type>::resolve_barrier();
+  }
+
+  template <DecoratorSet decorators>
+  template <typename T>
+  inline void BuildtimeDispatch<decorators>::store(void* addr, T value) {
+    const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
+    PreRuntimeDispatch<expanded_decorators>::template store<expanded_decorators, T>(addr, value);
+  }
+
+  template <DecoratorSet decorators>
+  template <typename T>
+  inline void BuildtimeDispatch<decorators>::store_at(void* base, ptrdiff_t offset, T value) {
+    const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
+    PreRuntimeDispatch<expanded_decorators>::template store_at<expanded_decorators, T>(base, offset, value);
+  }
+
+  template <DecoratorSet decorators>
+  template <typename T>
+  inline T BuildtimeDispatch<decorators>::load(void* addr) {
+    const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
+    return PreRuntimeDispatch<expanded_decorators>::template load<expanded_decorators, T>(addr);
+  }
+
+  template <DecoratorSet decorators>
+  template <typename T>
+  inline T BuildtimeDispatch<decorators>::load_at(void* base, ptrdiff_t offset) {
+    const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
+    return PreRuntimeDispatch<expanded_decorators>::template load_at<expanded_decorators, T>(base, offset);
+  }
+
+  template <DecoratorSet decorators>
+  template <typename T>
+  inline T BuildtimeDispatch<decorators>::cas(T new_value, void* addr, T compare_value) {
+    const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
+    return PreRuntimeDispatch<expanded_decorators>::template cas<expanded_decorators, T>(new_value, addr, compare_value);
+  }
+
+  template <DecoratorSet decorators>
+  template <typename T>
+  inline T BuildtimeDispatch<decorators>::cas_at(T new_value, void* base, ptrdiff_t offset, T compare_value) {
+    const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
+    return PreRuntimeDispatch<expanded_decorators>::template cas_at<expanded_decorators, T>(new_value, base, offset, compare_value);
+  }
+
+  template <DecoratorSet decorators>
+  template <typename T>
+  inline T BuildtimeDispatch<decorators>::swap(T new_value, void* addr) {
+    const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
+    return PreRuntimeDispatch<expanded_decorators>::template swap<expanded_decorators, T>(new_value, addr);
+  }
+
+  template <DecoratorSet decorators>
+  template <typename T>
+  inline T BuildtimeDispatch<decorators>::swap_at(T new_value, void* base, ptrdiff_t offset) {
+    const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
+    return PreRuntimeDispatch<expanded_decorators>::template swap_at<expanded_decorators, T>(new_value, base, offset);
+  }
+
+  template <DecoratorSet decorators>
+  template <typename T>
+  inline bool BuildtimeDispatch<decorators>::copy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) {
+    const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
+    return PreRuntimeDispatch<expanded_decorators>::template copy<expanded_decorators, T>(src_obj, dst_obj, src, dst, length);
+  }
+
+  template <DecoratorSet decorators>
+  inline void BuildtimeDispatch<decorators>::clone(oop src, oop dst, size_t size) {
+    const DecoratorSet expanded_decorators = decorators | BT_BUILDTIME_DECORATORS | BT_HAS_BUILDTIME_DECORATOR;
+    PreRuntimeDispatch<expanded_decorators>::template clone<expanded_decorators>(src, dst, size);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline void store_at_reduce_types(P base, ptrdiff_t offset, T value) {
+    const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
+    const DecoratorSet final_decorators = expanded_decorators |
+                                           ((DecoratorTest<expanded_decorators>::HAS_VALUE_IS_OOP &&
+                                             !DecoratorTest<expanded_decorators>::HAS_ACCESS_ON_ROOT) ?
+                                             GC_CONVERT_COMPRESSED_OOP : EMPTY_DECORATOR);
+    BuildtimeDispatch<final_decorators>::template store_at<T>((void*)base, offset, value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T DispatchLoadAtReduceTypes<decorators, P, T>::load_at(P base, ptrdiff_t offset) {
+    const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
+    const DecoratorSet final_decorators = expanded_decorators |
+                                           ((DecoratorTest<expanded_decorators>::HAS_VALUE_IS_OOP &&
+                                             !DecoratorTest<expanded_decorators>::HAS_ACCESS_ON_ROOT) ?
+                                             GC_CONVERT_COMPRESSED_OOP : EMPTY_DECORATOR);
+    return BuildtimeDispatch<final_decorators>::template load_at<T>((void*)base, offset);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T cas_at_reduce_types(T new_value, P base, ptrdiff_t offset, T compare_value) {
+    const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
+    const DecoratorSet final_decorators = expanded_decorators |
+                                           ((DecoratorTest<expanded_decorators>::HAS_VALUE_IS_OOP &&
+                                             !DecoratorTest<expanded_decorators>::HAS_ACCESS_ON_ROOT) ?
+                                             GC_CONVERT_COMPRESSED_OOP : EMPTY_DECORATOR);
+    return BuildtimeDispatch<final_decorators>::template cas_at<T>(new_value, (void*)base, offset, compare_value);
+  }
+
+  template <DecoratorSet decorators, typename P, typename T>
+  inline T swap_at_reduce_types(T new_value, P base, ptrdiff_t offset) {
+    const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
+    const DecoratorSet final_decorators = expanded_decorators |
+                                           ((DecoratorTest<expanded_decorators>::HAS_VALUE_IS_OOP &&
+                                             !DecoratorTest<expanded_decorators>::HAS_ACCESS_ON_ROOT) ?
+                                             GC_CONVERT_COMPRESSED_OOP : EMPTY_DECORATOR);
+    return BuildtimeDispatch<final_decorators>::template swap_at<T>(new_value, (void*)base, offset);
+  }
+}
+
+#endif // SHARE_VM_RUNTIME_ACCESS_INLINE_HPP