# HG changeset patch
# User rkennke
# Date 1559124081 -7200
#      Wed May 29 12:01:21 2019 +0200
# Node ID 854a291b42676f3ac01eeff439bbab7a30520774
# Parent  66fab7238237acd894f80f5f0d5a64fdb6b10b5b
[backport] 8224584: Shenandoah: Eliminate forwarding pointer word
Reviewed-by: shade, roland

diff -r 66fab7238237 src/cpu/aarch64/vm/macroAssembler_aarch64.cpp
--- a/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -3765,15 +3765,10 @@
 
   // verify_tlab();
 
-  int oop_extra_words = Universe::heap()->oop_extra_words();
-
   ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
   if (var_size_in_bytes == noreg) {
-    lea(end, Address(obj, con_size_in_bytes + oop_extra_words * HeapWordSize));
+    lea(end, Address(obj, con_size_in_bytes));
   } else {
-    if (oop_extra_words > 0) {
-      add(var_size_in_bytes, var_size_in_bytes, oop_extra_words * HeapWordSize);
-    }
     lea(end, Address(obj, var_size_in_bytes));
   }
   ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
@@ -3783,8 +3778,6 @@
   // update the tlab top pointer
   str(end, Address(rthread, JavaThread::tlab_top_offset()));
 
-  Universe::heap()->compile_prepare_oop(this, obj);
-
   // recover var_size_in_bytes if necessary
   if (var_size_in_bytes == end) {
     sub(var_size_in_bytes, var_size_in_bytes, obj);
diff -r 66fab7238237 src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp
--- a/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -40,17 +40,46 @@
 
 #define __ masm->
 
-void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) {
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
   assert(ShenandoahCASBarrier, "should be enabled");
   Label is_null;
   __ cbz(dst, is_null);
-  resolve_forward_pointer_not_null(masm, dst);
+  resolve_forward_pointer_not_null(masm, dst, tmp);
   __ bind(is_null);
 }
 
-void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
+// IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2, except those explicitely
+// passed in.
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
   assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled");
-  __ ldr(dst, Address(dst, ShenandoahForwarding::byte_offset()));
+  // The below loads the mark word, checks if the lowest two bits are
+  // set, and if so, clear the lowest two bits and copy the result
+  // to dst. Otherwise it leaves dst alone.
+  // Implementing this is surprisingly awkward. I do it here by:
+  // - Inverting the mark word
+  // - Test lowest two bits == 0
+  // - If so, set the lowest two bits
+  // - Invert the result back, and copy to dst
+
+  bool borrow_reg = (tmp == noreg);
+  if (borrow_reg) {
+    // No free registers available. Make one useful.
+    tmp = rscratch1;
+    __ push(RegSet::of(tmp), sp);
+  }
+
+  Label done;
+  __ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
+  __ eon(tmp, tmp, zr);
+  __ ands(zr, tmp, markOopDesc::lock_mask_in_place);
+  __ br(Assembler::NE, done);
+  __ orr(tmp, tmp, markOopDesc::marked_value);
+  __ eon(dst, tmp, zr);
+  __ bind(done);
+
+  if (borrow_reg) {
+    __ pop(RegSet::of(tmp), sp);
+  }
 }
 
 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) {
@@ -185,12 +214,3 @@
 #undef __
 
 #endif // COMPILER1
-
-#define __ masm->
-
-void ShenandoahHeap::compile_prepare_oop(MacroAssembler* masm, Register obj) {
-  __ add(obj, obj, ShenandoahForwarding::byte_size());
-  __ str(obj, Address(obj, -1 * HeapWordSize));
-}
-
-#undef __
diff -r 66fab7238237 src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp
--- a/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -36,8 +36,8 @@
 class ShenandoahBarrierSetAssembler : public CHeapObj<mtGC> {
 private:
 
-  void resolve_forward_pointer(MacroAssembler* masm, Register dst);
-  void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst);
+  void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
+  void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
 
   void load_reference_barrier_not_null(MacroAssembler* masm, Register dst);
 
diff -r 66fab7238237 src/cpu/x86/vm/macroAssembler_x86.cpp
--- a/src/cpu/x86/vm/macroAssembler_x86.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/cpu/x86/vm/macroAssembler_x86.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -4473,15 +4473,10 @@
 
   NOT_LP64(get_thread(thread));
 
-  uint oop_extra_words = Universe::heap()->oop_extra_words();
-
   movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
   if (var_size_in_bytes == noreg) {
-    lea(end, Address(obj, con_size_in_bytes + oop_extra_words * HeapWordSize));
+    lea(end, Address(obj, con_size_in_bytes));
   } else {
-    if (oop_extra_words > 0) {
-      addptr(var_size_in_bytes, oop_extra_words * HeapWordSize);
-    }
     lea(end, Address(obj, var_size_in_bytes, Address::times_1));
   }
   cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
@@ -4490,8 +4485,6 @@
   // update the tlab top pointer
   movptr(Address(thread, JavaThread::tlab_top_offset()), end);
 
-  Universe::heap()->compile_prepare_oop(this, obj);
-
   // recover var_size_in_bytes if necessary
   if (var_size_in_bytes == end) {
     subptr(var_size_in_bytes, obj);
diff -r 66fab7238237 src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp
--- a/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -40,18 +40,46 @@
 
 #define __ masm->
 
-void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) {
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
   assert(ShenandoahCASBarrier, "should be enabled");
   Label is_null;
   __ testptr(dst, dst);
   __ jcc(Assembler::zero, is_null);
-  resolve_forward_pointer_not_null(masm, dst);
+  resolve_forward_pointer_not_null(masm, dst, tmp);
   __ bind(is_null);
 }
 
-void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
+void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
   assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled");
-  __ movptr(dst, Address(dst, ShenandoahForwarding::byte_offset()));
+  // The below loads the mark word, checks if the lowest two bits are
+  // set, and if so, clear the lowest two bits and copy the result
+  // to dst. Otherwise it leaves dst alone.
+  // Implementing this is surprisingly awkward. I do it here by:
+  // - Inverting the mark word
+  // - Test lowest two bits == 0
+  // - If so, set the lowest two bits
+  // - Invert the result back, and copy to dst
+
+  bool borrow_reg = (tmp == noreg);
+  if (borrow_reg) {
+    // No free registers available. Make one useful.
+    tmp = rscratch1;
+    __ push(tmp);
+  }
+
+  Label done;
+  __ movptr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
+  __ notptr(tmp);
+  __ testb(tmp, markOopDesc::marked_value);
+  __ jccb(Assembler::notZero, done);
+  __ orptr(tmp, markOopDesc::marked_value);
+  __ notptr(tmp);
+  __ mov(dst, tmp);
+  __ bind(done);
+
+  if (borrow_reg) {
+    __ pop(tmp);
+  }
 }
 
 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) {
@@ -60,7 +88,7 @@
   Label done;
 
   Address gc_state(r15_thread, in_bytes(JavaThread::gc_state_offset()));
-  __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION);
+  __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
   __ jcc(Assembler::zero, done);
 
   {
@@ -254,16 +282,3 @@
 #undef __
 
 #endif // COMPILER1
-
-#define __ masm->
-
-void ShenandoahHeap::compile_prepare_oop(MacroAssembler* masm, Register obj) {
-#ifdef _LP64
-  __ incrementq(obj, ShenandoahForwarding::byte_size());
-#else
-  __ incrementl(obj, ShenandoahForwarding::byte_size());
-#endif
-  __ movptr(Address(obj, ShenandoahForwarding::byte_offset()), obj);
-}
-
-#undef __
diff -r 66fab7238237 src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp
--- a/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -36,8 +36,8 @@
 class ShenandoahBarrierSetAssembler : public CHeapObj<mtGC> {
 private:
 
-  void resolve_forward_pointer(MacroAssembler* masm, Register dst);
-  void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst);
+  void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
+  void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
 
   void load_reference_barrier_not_null(MacroAssembler* masm, Register dst);
 
diff -r 66fab7238237 src/cpu/x86/vm/templateTable_x86_64.cpp
--- a/src/cpu/x86/vm/templateTable_x86_64.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/cpu/x86/vm/templateTable_x86_64.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -3366,17 +3366,11 @@
     Universe::heap()->supports_inline_contig_alloc() && !CMSIncrementalMode;
 
   if (UseTLAB) {
-    uint oop_extra_words = Universe::heap()->oop_extra_words();
-    if (oop_extra_words > 0) {
-      __ addq(rdx, oop_extra_words * HeapWordSize);
-    }
-
     __ movptr(rax, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
     __ lea(rbx, Address(rax, rdx, Address::times_1));
     __ cmpptr(rbx, Address(r15_thread, in_bytes(JavaThread::tlab_end_offset())));
     __ jcc(Assembler::above, allow_shared_alloc ? allocate_shared : slow_case);
     __ movptr(Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())), rbx);
-    Universe::heap()->compile_prepare_oop(_masm, rax);
     if (ZeroTLAB) {
       // the fields have been already cleared
       __ jmp(initialize_header);
diff -r 66fab7238237 src/share/vm/asm/assembler.cpp
--- a/src/share/vm/asm/assembler.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/asm/assembler.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -26,7 +26,6 @@
 #include "asm/macroAssembler.hpp"
 #include "asm/macroAssembler.inline.hpp"
 #include "asm/codeBuffer.hpp"
-#include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/atomic.inline.hpp"
 #include "runtime/icache.hpp"
@@ -301,14 +300,6 @@
 bool MacroAssembler::needs_explicit_null_check(intptr_t offset) {
   // Exception handler checks the nmethod's implicit null checks table
   // only when this method returns false.
-#ifdef AARCH64
-  // AArch64 addresses passed from the signal handler may have
-  // their top 8 bits zeroed. That affects the case where
-  // Shenandoah tries to load a Brooks pointer via a null oop.
-  const uintptr_t address_bits = (uintptr_t)0xfffffffffffful;
-#else
-  const uintptr_t address_bits = ~(uintptr_t)0;
-#endif
 #ifdef _LP64
   if (UseCompressedOops && Universe::narrow_oop_base() != NULL) {
     assert (Universe::heap() != NULL, "java heap should be initialized");
@@ -316,21 +307,11 @@
     // the 'offset' is equal to [heap_base + offset] for
     // narrow oop implicit null checks.
     uintptr_t base = (uintptr_t)Universe::narrow_oop_base();
-    int adj = 0;
-    if (UseShenandoahGC) {
-      adj = ShenandoahForwarding::byte_offset();
-      assert(adj < 0, "no need for positive adjustments");
-    }
-    if ((uintptr_t)((offset - adj) & address_bits) >= base) {
+    if ((uintptr_t)offset >= base) {
       // Normalize offset for the next check.
       offset = (intptr_t)(pointer_delta((void*)offset, (void*)base, 1));
     }
   }
 #endif
-
-  if (UseShenandoahGC && ((offset & address_bits) == (ShenandoahForwarding::byte_offset() & address_bits))) {
-    return false;
-  }
-
   return offset < 0 || os::vm_page_size() <= offset;
 }
diff -r 66fab7238237 src/share/vm/ci/ciInstanceKlass.cpp
--- a/src/share/vm/ci/ciInstanceKlass.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/ci/ciInstanceKlass.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -28,7 +28,6 @@
 #include "ci/ciInstanceKlass.hpp"
 #include "ci/ciUtilities.hpp"
 #include "classfile/systemDictionary.hpp"
-#include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
 #include "memory/allocation.hpp"
 #include "memory/allocation.inline.hpp"
 #include "oops/oop.inline.hpp"
@@ -179,12 +178,12 @@
 //
 ciInstanceKlass* ciInstanceKlass::get_canonical_holder(int offset) {
   #ifdef ASSERT
-  if (!((offset >= 0 && offset < layout_helper()) || (UseShenandoahGC && offset == ShenandoahForwarding::byte_offset()))) {
+  if (!(offset >= 0 && offset < layout_helper())) {
     tty->print("*** get_canonical_holder(%d) on ", offset);
     this->print();
     tty->print_cr(" ***");
-    fatal("offset must be tame");
-  }
+  };
+  assert(offset >= 0 && offset < layout_helper(), "offset must be tame");
   #endif
 
   if (offset < instanceOopDesc::base_offset_in_bytes()) {
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/preservedMarks.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/share/vm/gc_implementation/shenandoah/preservedMarks.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc_implementation/shenandoah/preservedMarks.inline.hpp"
+#include "utilities/workgroup.hpp"
+#include "memory/allocation.inline.hpp"
+#include "memory/resourceArea.hpp"
+#include "oops/oop.inline.hpp"
+#include "utilities/macros.hpp"
+
+void PreservedMarks::restore() {
+  while (!_stack.is_empty()) {
+    const OopAndMarkOop elem = _stack.pop();
+    elem.set_mark();
+  }
+  assert_empty();
+}
+
+void PreservedMarks::adjust_during_full_gc() {
+  StackIterator<OopAndMarkOop, mtGC> iter(_stack);
+  while (!iter.is_empty()) {
+    OopAndMarkOop* elem = iter.next_addr();
+
+    oop obj = elem->get_oop();
+    if (obj->is_forwarded()) {
+      elem->set_oop(obj->forwardee());
+    }
+  }
+}
+
+void PreservedMarks::restore_and_increment(volatile size_t* const total_size_addr) {
+  const size_t stack_size = size();
+  restore();
+  // Only do the atomic add if the size is > 0.
+  if (stack_size > 0) {
+    Atomic::add(stack_size, (volatile jlong*)total_size_addr);
+  }
+}
+
+#ifndef PRODUCT
+void PreservedMarks::assert_empty() {
+  assert(_stack.is_empty(), err_msg("stack expected to be empty, size = " SIZE_FORMAT,
+                                    _stack.size()));
+  assert(_stack.cache_size() == 0,
+         err_msg("stack expected to have no cached segments, cache size = " SIZE_FORMAT,
+                 _stack.cache_size()));
+}
+#endif // ndef PRODUCT
+
+void RemoveForwardedPointerClosure::do_object(oop obj) {
+  if (obj->is_forwarded()) {
+    PreservedMarks::init_forwarded_mark(obj);
+  }
+}
+
+void PreservedMarksSet::init(uint num) {
+  assert(_stacks == NULL && _num == 0, "do not re-initialize");
+  assert(num > 0, "pre-condition");
+  if (_in_c_heap) {
+    _stacks = NEW_C_HEAP_ARRAY(Padded<PreservedMarks>, num, mtGC);
+  } else {
+    _stacks = NEW_RESOURCE_ARRAY(Padded<PreservedMarks>, num);
+  }
+  for (uint i = 0; i < num; i += 1) {
+    ::new (_stacks + i) PreservedMarks();
+  }
+  _num = num;
+
+  assert_empty();
+}
+
+class ParRestoreTask : public AbstractGangTask {
+private:
+  PreservedMarksSet* const _preserved_marks_set;
+  SequentialSubTasksDone _sub_tasks;
+  volatile size_t* const _total_size_addr;
+
+public:
+  virtual void work(uint worker_id) {
+    uint task_id = 0;
+    while (!_sub_tasks.is_task_claimed(/* reference */ task_id)) {
+      _preserved_marks_set->get(task_id)->restore_and_increment(_total_size_addr);
+    }
+    _sub_tasks.all_tasks_completed();
+  }
+
+  ParRestoreTask(uint worker_num,
+                 PreservedMarksSet* preserved_marks_set,
+                 volatile size_t* total_size_addr)
+      : AbstractGangTask("Parallel Preserved Mark Restoration"),
+        _preserved_marks_set(preserved_marks_set),
+        _total_size_addr(total_size_addr) {
+    _sub_tasks.set_n_threads(worker_num);
+    _sub_tasks.set_n_tasks(preserved_marks_set->num());
+  }
+};
+
+void PreservedMarksSet::reclaim() {
+  assert_empty();
+
+  for (uint i = 0; i < _num; i += 1) {
+    _stacks[i].~Padded<PreservedMarks>();
+  }
+
+  if (_in_c_heap) {
+    FREE_C_HEAP_ARRAY(Padded<PreservedMarks>, _stacks, mtGC);
+  } else {
+    // the array was resource-allocated, so nothing to do
+  }
+  _stacks = NULL;
+  _num = 0;
+}
+
+#ifndef PRODUCT
+void PreservedMarksSet::assert_empty() {
+  assert(_stacks != NULL && _num > 0, "should have been initialized");
+  for (uint i = 0; i < _num; i += 1) {
+    get(i)->assert_empty();
+  }
+}
+#endif // ndef PRODUCT
+
+void SharedRestorePreservedMarksTaskExecutor::restore(PreservedMarksSet* preserved_marks_set,
+                                                      volatile size_t* total_size_addr) {
+  if (_workers == NULL) {
+    for (uint i = 0; i < preserved_marks_set->num(); i += 1) {
+      *total_size_addr += preserved_marks_set->get(i)->size();
+      preserved_marks_set->get(i)->restore();
+    }
+  } else {
+    ParRestoreTask task(_workers->active_workers(), preserved_marks_set, total_size_addr);
+    _workers->run_task(&task);
+  }
+}
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/preservedMarks.hpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/share/vm/gc_implementation/shenandoah/preservedMarks.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHARED_PRESERVEDMARKS_HPP
+#define SHARE_VM_GC_SHARED_PRESERVEDMARKS_HPP
+
+#include "memory/allocation.hpp"
+#include "memory/padded.hpp"
+#include "oops/oop.hpp"
+#include "utilities/stack.hpp"
+
+class PreservedMarksSet;
+class WorkGang;
+
+class PreservedMarks {
+private:
+  class OopAndMarkOop {
+  private:
+    oop _o;
+    markOop _m;
+
+  public:
+    OopAndMarkOop(oop obj, markOop m) : _o(obj), _m(m) { }
+
+    oop get_oop() { return _o; }
+    inline void set_mark() const;
+    void set_oop(oop obj) { _o = obj; }
+  };
+  typedef Stack<OopAndMarkOop, mtGC> OopAndMarkOopStack;
+
+  OopAndMarkOopStack _stack;
+
+  inline bool should_preserve_mark(oop obj, markOop m) const;
+
+public:
+  size_t size() const { return _stack.size(); }
+  inline void push(oop obj, markOop m);
+  inline void push_if_necessary(oop obj, markOop m);
+  // Iterate over the stack, restore all preserved marks, and
+  // reclaim the memory taken up by the stack segments.
+  void restore();
+  // Iterate over the stack, adjust all preserved marks according
+  // to their forwarding location stored in the mark.
+  void adjust_during_full_gc();
+
+  void restore_and_increment(volatile size_t* const _total_size_addr);
+  inline static void init_forwarded_mark(oop obj);
+
+  // Assert the stack is empty and has no cached segments.
+  void assert_empty() PRODUCT_RETURN;
+
+  inline PreservedMarks();
+  ~PreservedMarks() { assert_empty(); }
+};
+
+class RemoveForwardedPointerClosure: public ObjectClosure {
+public:
+  virtual void do_object(oop obj);
+};
+
+class RestorePreservedMarksTaskExecutor {
+public:
+  void virtual restore(PreservedMarksSet* preserved_marks_set,
+                       volatile size_t* total_size_addr) = 0;
+};
+
+class SharedRestorePreservedMarksTaskExecutor : public RestorePreservedMarksTaskExecutor {
+private:
+    WorkGang* _workers;
+
+public:
+    SharedRestorePreservedMarksTaskExecutor(WorkGang* workers) : _workers(workers) { }
+
+    void restore(PreservedMarksSet* preserved_marks_set,
+                 volatile size_t* total_size_addr);
+
+};
+
+class PreservedMarksSet : public CHeapObj<mtGC> {
+private:
+  // true -> _stacks will be allocated in the C heap
+  // false -> _stacks will be allocated in the resource arena
+  const bool _in_c_heap;
+
+  // Number of stacks we have allocated (typically, one stack per GC worker).
+  // This should be >= 1 if the stacks have been initialized,
+  // or == 0 if they have not.
+  uint _num;
+
+  // Stack array (typically, one stack per GC worker) of length _num.
+  // This should be != NULL if the stacks have been initialized,
+  // or == NULL if they have not.
+  Padded<PreservedMarks>* _stacks;
+
+public:
+  uint num() const { return _num; }
+
+  // Return the i'th stack.
+  PreservedMarks* get(uint i = 0) const {
+    assert(_num > 0 && _stacks != NULL, "stacks should have been initialized");
+    assert(i < _num, "pre-condition");
+    return (_stacks + i);
+  }
+
+  // Allocate stack array.
+  void init(uint num);
+
+  // Iterate over all stacks, restore all preserved marks, and reclaim
+  // the memory taken up by the stack segments.
+  // Supported executors: SharedRestorePreservedMarksTaskExecutor (Serial, CMS, G1),
+  // PSRestorePreservedMarksTaskExecutor (PS).
+  inline void restore(RestorePreservedMarksTaskExecutor* executor);
+
+  // Reclaim stack array.
+  void reclaim();
+
+  // Assert all the stacks are empty and have no cached segments.
+  void assert_empty() PRODUCT_RETURN;
+
+  PreservedMarksSet(bool in_c_heap)
+      : _in_c_heap(in_c_heap), _num(0), _stacks(NULL) { }
+
+  ~PreservedMarksSet() {
+    assert(_stacks == NULL && _num == 0, "stacks should have been reclaimed");
+  }
+};
+
+#endif // SHARE_VM_GC_SHARED_PRESERVEDMARKS_HPP
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/preservedMarks.inline.hpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/share/vm/gc_implementation/shenandoah/preservedMarks.inline.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2016, 2018 Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHARED_PRESERVEDMARKS_INLINE_HPP
+#define SHARE_VM_GC_SHARED_PRESERVEDMARKS_INLINE_HPP
+
+#include "gc_implementation/shenandoah/preservedMarks.hpp"
+#include "gc_implementation/shenandoah/shenandoahLogging.hpp"
+#include "oops/oop.inline.hpp"
+#include "utilities/stack.inline.hpp"
+
+inline bool PreservedMarks::should_preserve_mark(oop obj, markOop m) const {
+  return m->must_be_preserved_for_promotion_failure(obj);
+}
+
+inline void PreservedMarks::push(oop obj, markOop m) {
+  assert(should_preserve_mark(obj, m), "pre-condition");
+  OopAndMarkOop elem(obj, m);
+  _stack.push(elem);
+}
+
+inline void PreservedMarks::push_if_necessary(oop obj, markOop m) {
+  if (should_preserve_mark(obj, m)) {
+    push(obj, m);
+  }
+}
+
+inline void PreservedMarks::init_forwarded_mark(oop obj) {
+  obj->init_mark();
+}
+
+inline void PreservedMarksSet::restore(RestorePreservedMarksTaskExecutor* executor) {
+  volatile size_t total_size = 0;
+
+#ifdef ASSERT
+  // This is to make sure the total_size we'll calculate below is correct.
+  size_t total_size_before = 0;
+  for (uint i = 0; i < _num; i += 1) {
+    total_size_before += get(i)->size();
+  }
+#endif // def ASSERT
+
+  executor->restore(this, &total_size);
+  assert_empty();
+
+  assert(total_size == total_size_before,
+         err_msg("total_size = " SIZE_FORMAT " before = " SIZE_FORMAT,
+                 total_size, total_size_before));
+
+  log_trace(gc)("Restored " SIZE_FORMAT " marks", total_size);
+}
+
+inline PreservedMarks::PreservedMarks()
+    : _stack(OopAndMarkOopStack::default_segment_size(),
+             // This stack should be used very infrequently so there's
+             // no point in caching stack segments (there will be a
+             // waste of space most of the time). So we set the max
+             // cache size to 0.
+             0 /* max_cache_size */) { }
+
+void PreservedMarks::OopAndMarkOop::set_mark() const {
+  _o->set_mark(_m);
+}
+
+#endif // SHARE_VM_GC_SHARED_PRESERVEDMARKS_INLINE_HPP
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahAsserts.cpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahAsserts.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahAsserts.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -58,12 +58,16 @@
   stringStream ss;
   r->print_on(&ss);
 
+  stringStream mw_ss;
+  obj->mark()->print_on(&mw_ss);
+
   ShenandoahMarkingContext* const ctx = heap->marking_context();
 
   msg.append("  " PTR_FORMAT " - klass " PTR_FORMAT " %s\n", p2i(obj), p2i(obj->klass()), obj->klass()->external_name());
   msg.append("    %3s allocated after mark start\n", ctx->allocated_after_mark_start((HeapWord *) obj)     ? "" : "not");
   msg.append("    %3s marked \n",                    ctx->is_marked(obj) ? "" : "not");
   msg.append("    %3s in collection set\n",          heap->in_collection_set(obj) ? "" : "not");
+  msg.append("  mark:%s\n", mw_ss.as_string());
   msg.append("  region: %s", ss.as_string());
 }
 
@@ -245,7 +249,7 @@
                   file, line);
   }
 
-  size_t alloc_size = obj->size() + ShenandoahForwarding::word_size();
+  size_t alloc_size = obj->size();
   if (alloc_size > ShenandoahHeapRegion::humongous_threshold_words()) {
     size_t idx = r->region_number();
     size_t num_regions = ShenandoahHeapRegion::required_regions(alloc_size * HeapWordSize);
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -26,7 +26,6 @@
 #include "gc_implementation/shenandoah/shenandoahAsserts.hpp"
 #include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
-#include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc_implementation/shenandoah/shenandoahHeuristics.hpp"
 #include "runtime/interfaceSupport.hpp"
@@ -295,7 +294,7 @@
       ShenandoahHeapRegion* r = _heap->heap_region_containing(obj);
       assert(r->is_cset(), "sanity");
 
-      HeapWord* cur = (HeapWord*)obj + obj->size() + ShenandoahForwarding::word_size();
+      HeapWord* cur = (HeapWord*)obj + obj->size();
 
       size_t count = 0;
       while ((cur < r->top()) && ctx->is_marked(oop(cur)) && (count++ < max)) {
@@ -303,7 +302,7 @@
         if (cur_oop == resolve_forwarded_not_null(cur_oop)) {
           _heap->evacuate_object(cur_oop, thread);
         }
-        cur = cur + cur_oop->size() + ShenandoahForwarding::word_size();
+        cur = cur + cur_oop->size();
       }
     }
 
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -25,7 +25,6 @@
 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP
 
 #include "gc_implementation/shenandoah/shenandoahAsserts.hpp"
-#include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
 #include "gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp"
 #include "gc_implementation/shenandoah/shenandoahConcurrentMark.hpp"
 #include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp"
@@ -71,7 +70,7 @@
 inline void ShenandoahConcurrentMark::count_liveness(jushort* live_data, oop obj) {
   size_t region_idx = _heap->heap_region_index_containing(obj);
   ShenandoahHeapRegion* region = _heap->get_region(region_idx);
-  size_t size = obj->size() + ShenandoahForwarding::word_size();
+  size_t size = obj->size();
 
   if (!region->is_humongous_start()) {
     assert(!region->is_humongous(), "Cannot have continuations here");
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahForwarding.hpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahForwarding.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahForwarding.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -28,68 +28,11 @@
 #include "utilities/globalDefinitions.hpp"
 
 class ShenandoahForwarding {
-  /*
-   * Notes:
-   *
-   *  a. It is important to have byte_offset and word_offset return constant
-   *     expressions, because that will allow to constant-fold forwarding ptr
-   *     accesses. This is not a problem in JIT compilers that would generate
-   *     the code once, but it is problematic in GC hotpath code.
-   *
-   *  b. With filler object mechanics, we may need to allocate more space for
-   *     the forwarding ptr to meet alignment requirements for objects. This
-   *     means *_offset and *_size calls are NOT interchangeable. The accesses
-   *     to forwarding ptrs should always be via *_offset. Storage size
-   *     calculations should always be via *_size.
-   */
-
 public:
-  /* Offset from the object start, in HeapWords. */
-  static inline int word_offset() {
-    return -1; // exactly one HeapWord
-  }
-
-  /* Offset from the object start, in bytes. */
-  static inline int byte_offset() {
-    return -HeapWordSize; // exactly one HeapWord
-  }
-
-  /* Allocated size, in HeapWords. */
-  static inline uint word_size() {
-    return (uint) MinObjAlignment;
-  }
-
-  /* Allocated size, in bytes */
-  static inline uint byte_size() {
-    return (uint) MinObjAlignmentInBytes;
-  }
-
-  /* Assert basic stuff once at startup. */
-  static void initial_checks() {
-    guarantee (MinObjAlignment > 0, "sanity, word_size is correct");
-    guarantee (MinObjAlignmentInBytes > 0, "sanity, byte_size is correct");
-  }
-
-  /* Initializes forwarding pointer (to self).
-   */
-  static inline void initialize(oop obj);
-
   /* Gets forwardee from the given object.
    */
   static inline oop get_forwardee(oop obj);
 
-  /* Tries to atomically update forwardee in $holder object to $update.
-   * Assumes $holder points at itself.
-   * Asserts $holder is in from-space.
-   * Asserts $update is in to-space.
-   */
-  static inline oop try_update_forwardee(oop obj, oop update);
-
-  /* Sets raw value for forwardee slot.
-   * THIS IS DANGEROUS: USERS HAVE TO INITIALIZE/SET FORWARDEE BACK AFTER THEY ARE DONE.
-   */
-  static inline void set_forwardee_raw(oop obj, HeapWord* update);
-
   /* Returns the raw value from forwardee slot.
    */
   static inline HeapWord* get_forwardee_raw(oop obj);
@@ -99,8 +42,21 @@
    */
   static inline HeapWord* get_forwardee_raw_unchecked(oop obj);
 
-private:
-  static inline HeapWord** forward_ptr_addr(oop obj);
+  /**
+   * Returns true if the object is forwarded, false otherwise.
+   */
+  static inline bool is_forwarded(oop obj);
+
+  /* Tries to atomically update forwardee in $holder object to $update.
+   * Assumes $holder points at itself.
+   * Asserts $holder is in from-space.
+   * Asserts $update is in to-space.
+   *
+   * Returns the new object 'update' upon success, or
+   * the new forwardee that a competing thread installed.
+   */
+  static inline oop try_update_forwardee(oop obj, oop update);
+
 };
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_HPP
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahForwarding.inline.hpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahForwarding.inline.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahForwarding.inline.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -31,38 +31,42 @@
 #include "gc_implementation/shenandoah/shenandoahLogging.hpp"
 #include "runtime/atomic.hpp"
 
-inline HeapWord** ShenandoahForwarding::forward_ptr_addr(oop obj) {
-  return (HeapWord**)((HeapWord*) obj + word_offset());
-}
-
-inline void ShenandoahForwarding::initialize(oop obj) {
-  shenandoah_assert_in_heap(NULL, obj);
-  *forward_ptr_addr(obj) = (HeapWord*) obj;
-}
-
-inline void ShenandoahForwarding::set_forwardee_raw(oop obj, HeapWord* update) {
-  shenandoah_assert_in_heap(NULL, obj);
-  *forward_ptr_addr(obj) = update;
-}
-
 inline HeapWord* ShenandoahForwarding::get_forwardee_raw(oop obj) {
   shenandoah_assert_in_heap(NULL, obj);
-  return *forward_ptr_addr(obj);
+  return get_forwardee_raw_unchecked(obj);
 }
 
 inline HeapWord* ShenandoahForwarding::get_forwardee_raw_unchecked(oop obj) {
-  return *forward_ptr_addr(obj);
+  markOop mark = obj->mark();
+  if (mark->is_marked()) {
+    return (HeapWord*) mark->clear_lock_bits();
+  } else {
+    return (HeapWord*) obj;
+  }
 }
 
 inline oop ShenandoahForwarding::get_forwardee(oop obj) {
   shenandoah_assert_correct(NULL, obj);
-  return oop(*forward_ptr_addr(obj));
+  return oop(get_forwardee_raw_unchecked(obj));
+}
+
+inline bool ShenandoahForwarding::is_forwarded(oop obj) {
+  return obj->mark()->is_marked();
 }
 
 inline oop ShenandoahForwarding::try_update_forwardee(oop obj, oop update) {
-  oop result = (oop) Atomic::cmpxchg_ptr(update, forward_ptr_addr(obj), obj);
-  shenandoah_assert_correct_except(NULL, obj, result != obj);
-  return result;
+  markOop old_mark = obj->mark();
+  if (old_mark->is_marked()) {
+    return (oop) old_mark->clear_lock_bits();
+  }
+
+  markOop new_mark = markOopDesc::encode_pointer_as_mark(update);
+  markOop prev_mark = obj->cas_set_mark(new_mark, old_mark);
+  if (prev_mark == old_mark) {
+    return update;
+  } else {
+    return (oop) prev_mark->clear_lock_bits();
+  }
 }
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -30,7 +30,6 @@
 
 #include "gc_implementation/shenandoah/shenandoahAllocTracker.hpp"
 #include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
-#include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
 #include "gc_implementation/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc_implementation/shenandoah/shenandoahCollectionSet.hpp"
 #include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
@@ -127,8 +126,6 @@
 jint ShenandoahHeap::initialize() {
   CollectedHeap::pre_initialize();
 
-  ShenandoahForwarding::initial_checks();
-
   initialize_heuristics();
 
   //
@@ -829,12 +826,9 @@
 
 HeapWord*  ShenandoahHeap::mem_allocate(size_t size,
                                         bool*  gc_overhead_limit_was_exceeded) {
-  ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size + ShenandoahForwarding::word_size());
-  HeapWord* filler = allocate_memory(req);
-  HeapWord* result = filler + ShenandoahForwarding::word_size();
-  if (filler != NULL) {
-    ShenandoahForwarding::initialize(oop(result));
-
+  ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
+  HeapWord* result = allocate_memory(req);
+  if (result != NULL) {
     assert(! in_collection_set(result), "never allocate in targetted region");
     return result;
   } else {
@@ -852,7 +846,7 @@
 
   void do_object(oop p) {
     shenandoah_assert_marked(NULL, p);
-    if (p == ShenandoahBarrierSet::resolve_forwarded_not_null(p)) {
+    if (!p->is_forwarded()) {
       _heap->evacuate_object(p, _thread);
     }
   }
@@ -929,8 +923,8 @@
 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
 
-  oop humongous_obj = oop(start->bottom() + ShenandoahForwarding::word_size());
-  size_t size = humongous_obj->size() + ShenandoahForwarding::word_size();
+  oop humongous_obj = oop(start->bottom());
+  size_t size = humongous_obj->size();
   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
   size_t index = start->region_number() + required_regions - 1;
 
@@ -1752,17 +1746,6 @@
   set_gc_state_mask(EVACUATION, in_progress);
 }
 
-HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
-  // Initialize forwarding pointer for the next object
-  HeapWord* result = obj + ShenandoahForwarding::word_size();
-  ShenandoahForwarding::initialize(oop(result));
-  return result;
-}
-
-uint ShenandoahHeap::oop_extra_words() {
-  return ShenandoahForwarding::word_size();
-}
-
 void ShenandoahHeap::ref_processing_init() {
   MemRegion mr = reserved_region();
 
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -519,15 +519,9 @@
   HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size);
 
 public:
-#ifndef CC_INTERP
-  void compile_prepare_oop(MacroAssembler* masm, Register obj);
-#endif
-
   HeapWord* allocate_memory(ShenandoahAllocRequest& request);
   HeapWord* mem_allocate(size_t size, bool* what);
 
-  uint oop_extra_words();
-
   void notify_mutator_alloc_words(size_t words, bool waste);
 
   // Shenandoah supports TLAB allocation
@@ -539,8 +533,6 @@
   size_t max_tlab_size() const;
   size_t tlab_used(Thread* ignored) const;
 
-  HeapWord* tlab_post_allocation_setup(HeapWord* obj);
-
   void resize_tlabs();
   void resize_all_tlabs();
 
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahHeap.inline.hpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.inline.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeap.inline.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -181,51 +181,46 @@
 
   assert(thread->is_evac_allowed(), "must be enclosed in in oom-evac scope");
 
-  size_t size_no_fwdptr = (size_t) p->size();
-  size_t size_with_fwdptr = size_no_fwdptr + ShenandoahForwarding::word_size();
+  size_t size = p->size();
 
   assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
 
   bool alloc_from_gclab = true;
-  HeapWord* filler = NULL;
+  HeapWord* copy = NULL;
 
 #ifdef ASSERT
   if (ShenandoahOOMDuringEvacALot &&
       (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
-        filler = NULL;
+        copy = NULL;
   } else {
 #endif
     if (UseTLAB) {
-      filler = allocate_from_gclab(thread, size_with_fwdptr);
+      copy = allocate_from_gclab(thread, size);
     }
-    if (filler == NULL) {
-      ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size_with_fwdptr);
-      filler = allocate_memory(req);
+    if (copy == NULL) {
+      ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size);
+      copy = allocate_memory(req);
       alloc_from_gclab = false;
     }
 #ifdef ASSERT
   }
 #endif
 
-  if (filler == NULL) {
-    control_thread()->handle_alloc_failure_evac(size_with_fwdptr);
+  if (copy == NULL) {
+    control_thread()->handle_alloc_failure_evac(size);
 
     _oom_evac_handler.handle_out_of_memory_during_evacuation();
 
     return ShenandoahBarrierSet::resolve_forwarded(p);
   }
 
-  // Copy the object and initialize its forwarding ptr:
-  HeapWord* copy = filler + ShenandoahForwarding::word_size();
-  oop copy_val = oop(copy);
-
-  Copy::aligned_disjoint_words((HeapWord*) p, copy, size_no_fwdptr);
-  ShenandoahForwarding::initialize(oop(copy));
+  // Copy the object:
+  Copy::aligned_disjoint_words((HeapWord*) p, copy, size);
 
   // Try to install the new forwarding pointer.
+  oop copy_val = oop(copy);
   oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
-
-  if (result == p) {
+  if (result == copy_val) {
     // Successfully evacuated. Our copy is now the public one!
     shenandoah_assert_correct(NULL, copy_val);
     return copy_val;
@@ -242,11 +237,11 @@
     // have to explicitly overwrite the copy with the filler object. With that overwrite,
     // we have to keep the fwdptr initialized and pointing to our (stale) copy.
     if (alloc_from_gclab) {
-      thread->gclab().rollback(size_with_fwdptr);
+      thread->gclab().rollback(size);
     } else {
-      fill_with_object(copy, size_no_fwdptr);
+      fill_with_object(copy, size);
+      shenandoah_assert_correct(NULL, copy_val);
     }
-    shenandoah_assert_correct(NULL, copy_val);
     shenandoah_assert_correct(NULL, result);
     return result;
   }
@@ -308,7 +303,7 @@
 
 template<class T>
 inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) {
-  assert(ShenandoahForwarding::word_offset() < 0, "skip_delta calculation below assumes the forwarding ptr is before obj");
+  assert(! region->is_humongous_continuation(), "no humongous continuation regions here");
 
   ShenandoahMarkingContext* const ctx = complete_marking_context();
   assert(ctx->is_complete(), "sanity");
@@ -316,10 +311,9 @@
   MarkBitMap* mark_bit_map = ctx->mark_bit_map();
   HeapWord* tams = ctx->top_at_mark_start(region);
 
-  size_t skip_bitmap_delta = ShenandoahForwarding::word_size() + 1;
-  size_t skip_objsize_delta = ShenandoahForwarding::word_size() /* + actual obj.size() below */;
-  HeapWord* start = region->bottom() + ShenandoahForwarding::word_size();
-  HeapWord* end = MIN2(tams + ShenandoahForwarding::word_size(), region->end());
+  size_t skip_bitmap_delta = 1;
+  HeapWord* start = region->bottom();
+  HeapWord* end = MIN2(tams, region->end());
 
   // Step 1. Scan below the TAMS based on bitmap data.
   HeapWord* limit_bitmap = MIN2(limit, tams);
@@ -349,7 +343,7 @@
     do {
       avail = 0;
       for (int c = 0; (c < dist) && (cb < limit_bitmap); c++) {
-        Prefetch::read(cb, ShenandoahForwarding::byte_offset());
+        Prefetch::read(cb, oopDesc::mark_offset_in_bytes());
         slots[avail++] = cb;
         cb += skip_bitmap_delta;
         if (cb < limit_bitmap) {
@@ -386,9 +380,9 @@
   // Step 2. Accurate size-based traversal, happens past the TAMS.
   // This restarts the scan at TAMS, which makes sure we traverse all objects,
   // regardless of what happened at Step 1.
-  HeapWord* cs = tams + ShenandoahForwarding::word_size();
+  HeapWord* cs = tams;
   while (cs < limit) {
-    assert (cs > tams,  err_msg("only objects past TAMS here: "   PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams)));
+    assert (cs >= tams, err_msg("only objects past TAMS here: "   PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams)));
     assert (cs < limit, err_msg("only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit)));
     oop obj = oop(cs);
     int size = obj->size();
@@ -396,7 +390,7 @@
     assert(obj->is_oop(), "sanity");
     assert(_marking_context->is_marked(obj), "object expected to be marked");
     cl->do_object(obj);
-    cs += size + skip_objsize_delta;
+    cs += size;
   }
 }
 
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegion.cpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegion.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeapRegion.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -24,7 +24,6 @@
 #include "precompiled.hpp"
 
 #include "memory/allocation.hpp"
-#include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc_implementation/shenandoah/shenandoahHeapRegion.hpp"
 #include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp"
@@ -457,11 +456,11 @@
   if (p >= top()) {
     return top();
   } else {
-    HeapWord* last = bottom() + ShenandoahForwarding::word_size();
+    HeapWord* last = bottom();
     HeapWord* cur = last;
     while (cur <= p) {
       last = cur;
-      cur += oop(cur)->size() + ShenandoahForwarding::word_size();
+      cur += oop(cur)->size();
     }
     shenandoah_assert_correct(NULL, oop(last));
     return last;
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahHeuristics.cpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahHeuristics.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahHeuristics.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -24,7 +24,6 @@
 #include "precompiled.hpp"
 
 #include "gc_interface/gcCause.hpp"
-#include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
 #include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc_implementation/shenandoah/shenandoahHeapRegion.hpp"
@@ -155,7 +154,7 @@
       // Reclaim humongous regions here, and count them as the immediate garbage
 #ifdef ASSERT
       bool reg_live = region->has_live();
-      bool bm_live = ctx->is_marked(oop(region->bottom() + ShenandoahForwarding::word_size()));
+      bool bm_live = ctx->is_marked(oop(region->bottom()));
       assert(reg_live == bm_live,
              err_msg("Humongous liveness and marks should agree. Region live: %s; Bitmap live: %s; Region Live Words: " SIZE_FORMAT,
                      BOOL_TO_STR(reg_live), BOOL_TO_STR(bm_live), region->get_live_data_words()));
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.cpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -26,6 +26,7 @@
 #include "code/codeCache.hpp"
 #include "gc_implementation/shenandoah/shenandoahGCTraceTime.hpp"
 #include "gc_implementation/shared/gcTimer.hpp"
+#include "gc_implementation/shenandoah/preservedMarks.inline.hpp"
 #include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
 #include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
 #include "gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp"
@@ -46,11 +47,16 @@
 #include "gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp"
 #include "memory/metaspace.hpp"
 #include "oops/oop.inline.hpp"
+#include "runtime/biasedLocking.hpp"
 #include "runtime/thread.hpp"
 #include "utilities/copy.hpp"
 #include "utilities/growableArray.hpp"
 #include "utilities/workgroup.hpp"
 
+ShenandoahMarkCompact::ShenandoahMarkCompact() :
+  _gc_timer(NULL),
+  _preserved_marks(new PreservedMarksSet(true)) {}
+
 void ShenandoahMarkCompact::initialize(GCTimer* gc_timer) {
   _gc_timer = gc_timer;
 }
@@ -115,6 +121,10 @@
 
     // e. Set back forwarded objects bit back, in case some steps above dropped it.
     heap->set_has_forwarded_objects(has_forwarded_objects);
+
+    // The rest of prologue:
+    BiasedLocking::preserve_marks();
+    _preserved_marks->init(heap->workers()->active_workers());
   }
 
   heap->make_parsable(true);
@@ -155,6 +165,16 @@
     phase4_compact_objects(worker_slices);
   }
 
+  {
+    // Epilogue
+    SharedRestorePreservedMarksTaskExecutor exec(heap->workers());
+    _preserved_marks->restore(&exec);
+    BiasedLocking::restore_marks();
+    _preserved_marks->reclaim();
+
+    JvmtiExport::gc_epilogue();
+  }
+
   // Resize metaspace
   MetaspaceGC::compute_new_size();
 
@@ -231,6 +251,7 @@
 
 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
 private:
+  PreservedMarks*          const _preserved_marks;
   ShenandoahHeap*          const _heap;
   GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
   int _empty_regions_pos;
@@ -239,7 +260,10 @@
   HeapWord* _compact_point;
 
 public:
-  ShenandoahPrepareForCompactionObjectClosure(GrowableArray<ShenandoahHeapRegion*>& empty_regions, ShenandoahHeapRegion* to_region) :
+  ShenandoahPrepareForCompactionObjectClosure(PreservedMarks* preserved_marks,
+                                              GrowableArray<ShenandoahHeapRegion*>& empty_regions,
+                                              ShenandoahHeapRegion* to_region) :
+    _preserved_marks(preserved_marks),
     _heap(ShenandoahHeap::heap()),
     _empty_regions(empty_regions),
     _empty_regions_pos(0),
@@ -269,7 +293,7 @@
     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
     assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
 
-    size_t obj_size = p->size() + ShenandoahForwarding::word_size();
+    size_t obj_size = p->size();
     if (_compact_point + obj_size > _to_region->end()) {
       finish_region();
 
@@ -292,13 +316,15 @@
     // Object fits into current region, record new location:
     assert(_compact_point + obj_size <= _to_region->end(), "must fit");
     shenandoah_assert_not_forwarded(NULL, p);
-    ShenandoahForwarding::set_forwardee_raw(p, _compact_point + ShenandoahForwarding::word_size());
+    _preserved_marks->push_if_necessary(p, p->mark());
+    p->forward_to(oop(_compact_point));
     _compact_point += obj_size;
   }
 };
 
 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
 private:
+  PreservedMarksSet*        const _preserved_marks;
   ShenandoahHeap*           const _heap;
   ShenandoahHeapRegionSet** const _worker_slices;
   ShenandoahRegionIterator        _heap_regions;
@@ -321,9 +347,10 @@
   }
 
 public:
-  ShenandoahPrepareForCompactionTask(ShenandoahHeapRegionSet** worker_slices) :
-          AbstractGangTask("Shenandoah Prepare For Compaction Task"),
-          _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
+  ShenandoahPrepareForCompactionTask(PreservedMarksSet* preserved_marks, ShenandoahHeapRegionSet** worker_slices) :
+    AbstractGangTask("Shenandoah Prepare For Compaction Task"),
+    _preserved_marks(preserved_marks),
+    _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
   }
 
   void work(uint worker_id) {
@@ -339,7 +366,7 @@
     // Remember empty regions and reuse them as needed.
     ResourceMark rm;
     GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
-    ShenandoahPrepareForCompactionObjectClosure cl(empty_regions, from_region);
+    ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
     while (from_region != NULL) {
       cl.set_from_region(from_region);
       if (from_region->has_live()) {
@@ -379,8 +406,8 @@
   size_t to_begin = heap->num_regions();
   size_t to_end = heap->num_regions();
 
-  for (size_t c = heap->num_regions() - 1; c > 0; c--) {
-    ShenandoahHeapRegion *r = heap->get_region(c);
+  for (size_t c = heap->num_regions(); c > 0; c--) {
+    ShenandoahHeapRegion *r = heap->get_region(c - 1);
     if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
       // To-region candidate: record this, and continue scan
       to_begin = r->region_number();
@@ -389,15 +416,16 @@
 
     if (r->is_humongous_start() && r->is_move_allowed()) {
       // From-region candidate: movable humongous region
-      oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
-      size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
+      oop old_obj = oop(r->bottom());
+      size_t words_size = old_obj->size();
       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 
       size_t start = to_end - num_regions;
 
       if (start >= to_begin && start != r->region_number()) {
         // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
-        ShenandoahForwarding::set_forwardee_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahForwarding::word_size());
+        _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
+        old_obj->forward_to(oop(heap->get_region(start)->bottom()));
         to_end = start;
         continue;
       }
@@ -445,7 +473,7 @@
 
   void heap_region_do(ShenandoahHeapRegion* r) {
     if (r->is_humongous_start()) {
-      oop humongous_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
+      oop humongous_obj = oop(r->bottom());
       if (!_ctx->is_marked(humongous_obj)) {
         assert(!r->has_live(),
                err_msg("Region " SIZE_FORMAT " is not marked, should not have live", r->region_number()));
@@ -485,7 +513,7 @@
   // Compute the new addresses for regular objects
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
-    ShenandoahPrepareForCompactionTask prepare_task(worker_slices);
+    ShenandoahPrepareForCompactionTask prepare_task(_preserved_marks, worker_slices);
     heap->workers()->run_task(&prepare_task);
   }
 
@@ -507,9 +535,11 @@
     if (! oopDesc::is_null(o)) {
       oop obj = oopDesc::decode_heap_oop_not_null(o);
       assert(_ctx->is_marked(obj), "must be marked");
-      oop forw = oop(ShenandoahForwarding::get_forwardee_raw(obj));
-      oopDesc::encode_store_heap_oop(p, forw);
-    }
+      if (obj->is_forwarded()) {
+        oop forw = obj->forwardee();
+        oopDesc::encode_store_heap_oop(p, forw);
+      }
+   }
   }
 
 public:
@@ -562,11 +592,12 @@
 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
 private:
   ShenandoahRootProcessor* _rp;
-
+  PreservedMarksSet* _preserved_marks;
 public:
-  ShenandoahAdjustRootPointersTask(ShenandoahRootProcessor* rp) :
+  ShenandoahAdjustRootPointersTask(ShenandoahRootProcessor* rp, PreservedMarksSet* preserved_marks) :
     AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
-    _rp(rp) {}
+    _rp(rp),
+    _preserved_marks(preserved_marks) {}
 
   void work(uint worker_id) {
     ShenandoahAdjustPointersClosure cl;
@@ -577,6 +608,7 @@
     _rp->process_all_roots(&cl, &cl,
                            &adjust_cld_closure,
                            &adjust_code_closure, NULL, worker_id);
+    _preserved_marks->get(worker_id)->adjust_during_full_gc();
   }
 };
 
@@ -590,7 +622,7 @@
   {
     COMPILER2_PRESENT(DerivedPointerTable::clear());
     ShenandoahRootProcessor rp(heap, nworkers, ShenandoahPhaseTimings::full_gc_roots);
-    ShenandoahAdjustRootPointersTask task(&rp);
+    ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks);
     workers->run_task(&task);
     COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
   }
@@ -609,13 +641,13 @@
   void do_object(oop p) {
     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
     size_t size = (size_t)p->size();
-    HeapWord* compact_to = ShenandoahForwarding::get_forwardee_raw(p);
-    HeapWord* compact_from = (HeapWord*) p;
-    if (compact_from != compact_to) {
+    if (p->is_forwarded()) {
+      HeapWord* compact_from = (HeapWord*) p;
+      HeapWord* compact_to = (HeapWord*) p->forwardee();
       Copy::aligned_conjoint_words(compact_from, compact_to, size);
+      oop new_obj = oop(compact_to);
+      new_obj->init_mark();
     }
-    oop new_obj = oop(compact_to);
-    ShenandoahForwarding::initialize(new_obj);
   }
 };
 
@@ -706,31 +738,30 @@
 
   ShenandoahHeap* heap = ShenandoahHeap::heap();
 
-  for (size_t c = heap->num_regions() - 1; c > 0; c--) {
-    ShenandoahHeapRegion* r = heap->get_region(c);
+  for (size_t c = heap->num_regions(); c > 0; c--) {
+    ShenandoahHeapRegion* r = heap->get_region(c - 1);
     if (r->is_humongous_start()) {
-      oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
-      size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
+      oop old_obj = oop(r->bottom());
+      if (!old_obj->is_forwarded()) {
+        // No need to move the object, it stays at the same slot
+        continue;
+      }
+      size_t words_size = old_obj->size();
       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 
       size_t old_start = r->region_number();
       size_t old_end   = old_start + num_regions - 1;
-      size_t new_start = heap->heap_region_index_containing(ShenandoahForwarding::get_forwardee_raw(old_obj));
+      size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
       size_t new_end   = new_start + num_regions - 1;
-
-      if (old_start == new_start) {
-        // No need to move the object, it stays at the same slot
-        continue;
-      }
-
+      assert(old_start != new_start, "must be real move");
       assert (r->is_move_allowed(), "should be movable");
 
       Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
                                    heap->get_region(new_start)->bottom(),
                                    ShenandoahHeapRegion::region_size_words()*num_regions);
 
-      oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahForwarding::word_size());
-      ShenandoahForwarding::initialize(new_obj);
+      oop new_obj = oop(heap->get_region(new_start)->bottom());
+      new_obj->init_mark();
 
       {
         for (size_t c = old_start; c <= old_end; c++) {
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.hpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -49,12 +49,19 @@
  * where it does sliding compaction, without interfering with other threads.
  */
 
+class PreservedMarksSet;
+
 class ShenandoahMarkCompact : public CHeapObj<mtGC> {
+  friend class ShenandoahPrepareForCompactionObjectClosure;
 private:
   GCTimer* _gc_timer;
 
+  PreservedMarksSet* _preserved_marks;
+
 public:
+  ShenandoahMarkCompact();
   void initialize(GCTimer* gc_timer);
+
   void do_it(GCCause::Cause gc_cause);
 
 private:
@@ -65,7 +72,6 @@
 
   void calculate_target_humongous_objects();
   void compact_humongous_objects();
-
 };
 
 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKCOMPACT_HPP
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahSupport.cpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahSupport.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahSupport.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -36,6 +36,12 @@
 #include "opto/runtime.hpp"
 #include "opto/subnode.hpp"
 
+#ifdef _LP64
+#define LoadXNode LoadLNode
+#else
+#define LoadXNode LoadINode
+#endif
+
 bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
   if (C->shenandoah_barriers_count() > 0) {
     C->clear_major_progress();
@@ -1382,7 +1388,7 @@
     assert(val->bottom_type()->make_oopptr(), "need oop");
     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
 
-    enum { _heap_stable = 1, _not_cset, _not_equal, _evac_path, _null_path, PATH_LIMIT };
+    enum { _heap_stable = 1, _not_cset, _fwded, _evac_path, _null_path, PATH_LIMIT };
     Node* region = new (phase->C) RegionNode(PATH_LIMIT);
     Node* val_phi = new (phase->C) PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
@@ -1432,36 +1438,48 @@
       IfNode* iff = unc_ctrl->in(0)->as_If();
       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
     }
-    Node* addr = new (phase->C) AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahForwarding::byte_offset()));
+    Node* addr = new (phase->C) AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(oopDesc::mark_offset_in_bytes()));
     phase->register_new_node(addr, ctrl);
-    assert(val->bottom_type()->isa_oopptr(), "what else?");
-    const TypePtr* obj_type =  val->bottom_type()->is_oopptr();
-    const TypePtr* adr_type = TypeRawPtr::BOTTOM;
-    Node* fwd = new (phase->C) LoadPNode(ctrl, raw_mem, addr, adr_type, obj_type, MemNode::unordered);
-    phase->register_new_node(fwd, ctrl);
+    assert(new_val->bottom_type()->isa_oopptr(), "what else?");
+    Node* markword = new (phase->C) LoadXNode(ctrl, raw_mem, addr, TypeRawPtr::BOTTOM, TypeX_X, MemNode::unordered);
+    phase->register_new_node(markword, ctrl);
+
+    // Test if object is forwarded. This is the case if lowest two bits are set.
+    Node* masked = new (phase->C) AndXNode(markword, phase->igvn().MakeConX(markOopDesc::lock_mask_in_place));
+    phase->register_new_node(masked, ctrl);
+    Node* cmp = new (phase->C) CmpXNode(masked, phase->igvn().MakeConX(markOopDesc::marked_value));
+    phase->register_new_node(cmp, ctrl);
 
     // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
-    Node* cmp = new (phase->C) CmpPNode(fwd, new_val);
-    phase->register_new_node(cmp, ctrl);
-    Node* bol = new (phase->C) BoolNode(cmp, BoolTest::eq);
+    Node* bol = new (phase->C) BoolNode(cmp, BoolTest::eq); // Equals 3 means it's forwarded
     phase->register_new_node(bol, ctrl);
 
-    IfNode* iff = new (phase->C) IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
-    if (reg2_ctrl == NULL) reg2_ctrl = iff;
+    IfNode* iff = new (phase->C) IfNode(ctrl, bol, PROB_LIKELY(0.999), COUNT_UNKNOWN);
     phase->register_control(iff, loop, ctrl);
-    Node* if_not_eq = new (phase->C) IfFalseNode(iff);
-    phase->register_control(if_not_eq, loop, iff);
-    Node* if_eq = new (phase->C) IfTrueNode(iff);
-    phase->register_control(if_eq, loop, iff);
+    Node* if_fwd = new (phase->C) IfTrueNode(iff);
+    phase->register_control(if_fwd, loop, iff);
+    Node* if_not_fwd = new (phase->C) IfFalseNode(iff);
+    phase->register_control(if_not_fwd, loop, iff);
+
+    // Decode forward pointer: since we already have the lowest bits, we can just subtract them
+    // from the mark word without the need for large immediate mask.
+    Node* masked2 = new (phase->C) SubXNode(markword, masked);
+    phase->register_new_node(masked2, if_fwd);
+    Node* fwdraw = new (phase->C) CastX2PNode(masked2);
+    fwdraw->init_req(0, if_fwd);
+    phase->register_new_node(fwdraw, if_fwd);
+    Node* fwd = new (phase->C) CheckCastPPNode(NULL, fwdraw, val->bottom_type());
+    phase->register_new_node(fwd, if_fwd);
 
     // Wire up not-equal-path in slots 3.
-    region->init_req(_not_equal, if_not_eq);
-    val_phi->init_req(_not_equal, fwd);
-    raw_mem_phi->init_req(_not_equal, raw_mem);
+    region->init_req(_fwded, if_fwd);
+    val_phi->init_req(_fwded, fwd);
+    raw_mem_phi->init_req(_fwded, raw_mem);
 
     // Call lrb-stub and wire up that path in slots 4
     Node* result_mem = NULL;
-    ctrl = if_eq;
+    ctrl = if_not_fwd;
+    fwd = new_val;
     call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase);
     region->init_req(_evac_path, ctrl);
     val_phi->init_req(_evac_path, fwd);
diff -r 66fab7238237 src/share/vm/gc_implementation/shenandoah/shenandoahVerifier.cpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahVerifier.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahVerifier.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -136,7 +136,7 @@
           // skip
           break;
         case ShenandoahVerifier::_verify_liveness_complete:
-          Atomic::add(obj->size() + ShenandoahForwarding::word_size(), &_ld[obj_reg->region_number()]);
+          Atomic::add((uint) obj->size(), &_ld[obj_reg->region_number()]);
           // fallthrough for fast failure for un-live regions:
         case ShenandoahVerifier::_verify_liveness_conservative:
           verify(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
@@ -504,7 +504,7 @@
 
   virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
     jlong processed = 0;
-    HeapWord* obj = r->bottom() + ShenandoahForwarding::word_size();
+    HeapWord* obj = r->bottom();
     if (_heap->complete_marking_context()->is_marked((oop)obj)) {
       verify_and_follow(obj, stack, cl, &processed);
     }
@@ -518,12 +518,12 @@
 
     // Bitmaps, before TAMS
     if (tams > r->bottom()) {
-      HeapWord* start = r->bottom() + ShenandoahForwarding::word_size();
+      HeapWord* start = r->bottom();
       HeapWord* addr = mark_bit_map->getNextMarkedWordAddress(start, tams);
 
       while (addr < tams) {
         verify_and_follow(addr, stack, cl, &processed);
-        addr += ShenandoahForwarding::word_size();
+        addr += 1;
         if (addr < tams) {
           addr = mark_bit_map->getNextMarkedWordAddress(addr, tams);
         }
@@ -533,11 +533,11 @@
     // Size-based, after TAMS
     {
       HeapWord* limit = r->top();
-      HeapWord* addr = tams + ShenandoahForwarding::word_size();
+      HeapWord* addr = tams;
 
       while (addr < limit) {
         verify_and_follow(addr, stack, cl, &processed);
-        addr += oop(addr)->size() + ShenandoahForwarding::word_size();
+        addr += oop(addr)->size();
       }
     }
 
diff -r 66fab7238237 src/share/vm/gc_interface/collectedHeap.cpp
--- a/src/share/vm/gc_interface/collectedHeap.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_interface/collectedHeap.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -302,7 +302,7 @@
 #endif // ASSERT
   }
   thread->tlab().fill(obj, obj + size, new_tlab_size);
-  return Universe::heap()->tlab_post_allocation_setup(obj);
+  return obj;
 }
 
 void CollectedHeap::flush_deferred_store_barrier(JavaThread* thread) {
@@ -609,15 +609,6 @@
 }
 #endif
 
-HeapWord* CollectedHeap::tlab_post_allocation_setup(HeapWord* obj) {
-  return obj;
-}
-
-uint CollectedHeap::oop_extra_words() {
-  // Default implementation doesn't need extra space for oops.
-  return 0;
-}
-
 void CollectedHeap::shutdown() {
   // Default implementation does nothing.
 }
@@ -626,12 +617,6 @@
   // Default implementation does nothing.
 }
 
-#ifndef CC_INTERP
-void CollectedHeap::compile_prepare_oop(MacroAssembler* masm, Register obj) {
-  // Default implementation does nothing.
-}
-#endif
-
 bool CollectedHeap::supports_object_pinning() const {
   return false;
 }
diff -r 66fab7238237 src/share/vm/gc_interface/collectedHeap.hpp
--- a/src/share/vm/gc_interface/collectedHeap.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_interface/collectedHeap.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -199,8 +199,6 @@
 
   virtual CollectedHeap::Name kind() const { return CollectedHeap::Abstract; }
 
-  virtual HeapWord* tlab_post_allocation_setup(HeapWord* obj);
-
   /**
    * Returns JNI error code JNI_ENOMEM if memory could not be allocated,
    * and JNI_OK on success.
@@ -327,12 +325,6 @@
   inline static void post_allocation_install_obj_klass(KlassHandle klass,
                                                        oop obj);
 
-  virtual uint oop_extra_words();
-
-#ifndef CC_INTERP
-  virtual void compile_prepare_oop(MacroAssembler* masm, Register obj);
-#endif
-
   // Raw memory allocation facilities
   // The obj and array allocate methods are covers for these methods.
   // mem_allocate() should never be
diff -r 66fab7238237 src/share/vm/gc_interface/collectedHeap.inline.hpp
--- a/src/share/vm/gc_interface/collectedHeap.inline.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/gc_interface/collectedHeap.inline.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -180,10 +180,8 @@
 HeapWord* CollectedHeap::allocate_from_tlab(KlassHandle klass, Thread* thread, size_t size) {
   assert(UseTLAB, "should use UseTLAB");
 
-  size += Universe::heap()->oop_extra_words();
   HeapWord* obj = thread->tlab().allocate(size);
   if (obj != NULL) {
-    obj = Universe::heap()->tlab_post_allocation_setup(obj);
     return obj;
   }
   // Otherwise...
diff -r 66fab7238237 src/share/vm/memory/threadLocalAllocBuffer.cpp
--- a/src/share/vm/memory/threadLocalAllocBuffer.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/memory/threadLocalAllocBuffer.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -124,8 +124,7 @@
       }
     }
 
-    HeapWord* obj = Universe::heap()->tlab_post_allocation_setup(top());
-    CollectedHeap::fill_with_object(obj, hard_end(), retire);
+    CollectedHeap::fill_with_object(top(), hard_end(), retire);
 
     if (retire || ZeroTLAB) {  // "Reset" the TLAB
       set_start(NULL);
@@ -310,11 +309,6 @@
   return thread;
 }
 
-size_t ThreadLocalAllocBuffer::end_reserve() {
-  int reserve_size = typeArrayOopDesc::header_size(T_INT) + Universe::heap()->oop_extra_words();
-  return MAX2(reserve_size, VM_Version::reserve_for_allocation_prefetch());
-}
-
 void ThreadLocalAllocBuffer::rollback(size_t size) {
   HeapWord* old_top = top();
   if (old_top != NULL) { // Pathological case: we accept that we can't rollback.
diff -r 66fab7238237 src/share/vm/memory/threadLocalAllocBuffer.hpp
--- a/src/share/vm/memory/threadLocalAllocBuffer.hpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/memory/threadLocalAllocBuffer.hpp	Mon Oct 28 15:51:53 2019 +0100
@@ -123,6 +123,12 @@
   // Allocate size HeapWords. The memory is NOT initialized to zero.
   inline HeapWord* allocate(size_t size);
 
+  // Reserve space at the end of TLAB
+  static size_t end_reserve() {
+    int reserve_size = typeArrayOopDesc::header_size(T_INT);
+    return MAX2(reserve_size, VM_Version::reserve_for_allocation_prefetch());
+  }
+
   // Resize based on amount of allocation, etc.
   void resize();
 
@@ -132,7 +138,6 @@
   // Rolls back a single allocation of the given size.
   void rollback(size_t size);
 
-  static size_t end_reserve();
   static size_t alignment_reserve()              { return align_object_size(end_reserve()); }
   static size_t alignment_reserve_in_bytes()     { return alignment_reserve() * HeapWordSize; }
 
diff -r 66fab7238237 src/share/vm/opto/macro.cpp
--- a/src/share/vm/opto/macro.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/opto/macro.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -1305,14 +1305,6 @@
 
     transform_later(old_eden_top);
     // Add to heap top to get a new heap top
-
-    Node* init_size_in_bytes = size_in_bytes;
-    if (UseShenandoahGC) {
-      // Allocate several words more for the Shenandoah forwarding pointer.
-      size_in_bytes = new (C) AddXNode(size_in_bytes, _igvn.MakeConX(ShenandoahForwarding::byte_size()));
-      transform_later(size_in_bytes);
-    }
-
     Node *new_eden_top = new (C) AddPNode(top(), old_eden_top, size_in_bytes);
     transform_later(new_eden_top);
     // Check for needing a GC; compare against heap end
@@ -1403,16 +1395,10 @@
                                    0, new_alloc_bytes, T_LONG);
     }
 
-    if (UseShenandoahGC) {
-      // Bump up object for Shenandoah forwarding pointer.
-      fast_oop = new (C) AddPNode(top(), fast_oop, _igvn.MakeConX(ShenandoahForwarding::byte_size()));
-      transform_later(fast_oop);
-    }
-
     InitializeNode* init = alloc->initialization();
     fast_oop_rawmem = initialize_object(alloc,
                                         fast_oop_ctrl, fast_oop_rawmem, fast_oop,
-                                        klass_node, length, init_size_in_bytes);
+                                        klass_node, length, size_in_bytes);
 
     // If initialization is performed by an array copy, any required
     // MemBarStoreStore was already added. If the object does not
@@ -1692,11 +1678,6 @@
       header_size = Klass::layout_helper_header_size(k->layout_helper());
   }
 
-  if (UseShenandoahGC) {
-    // Initialize Shenandoah forwarding pointer to point to the object itself.
-    rawmem = make_store(control, rawmem, object, ShenandoahForwarding::byte_offset(), object, T_OBJECT);
-  }
-
   // Clear the object body, if necessary.
   if (init == NULL) {
     // The init has somehow disappeared; be cautious and clear everything.
diff -r 66fab7238237 src/share/vm/opto/memnode.cpp
--- a/src/share/vm/opto/memnode.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/opto/memnode.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -1525,7 +1525,7 @@
     // as to alignment, which will therefore produce the smallest
     // possible base offset.
     const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
-    const bool off_beyond_header = (off != ShenandoahForwarding::byte_offset() || !UseShenandoahGC) && ((uint)off >= (uint)min_base_off);
+    const bool off_beyond_header = ((uint)off >= (uint)min_base_off);
 
     // Try to constant-fold a stable array element.
     if (FoldStableValues && ary->is_stable()) {
diff -r 66fab7238237 src/share/vm/opto/type.cpp
--- a/src/share/vm/opto/type.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/opto/type.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -2502,8 +2502,6 @@
   if (_offset != 0) {
     if (_offset == oopDesc::klass_offset_in_bytes()) {
       _is_ptr_to_narrowklass = UseCompressedClassPointers;
-    } else if (UseShenandoahGC && _offset == ShenandoahForwarding::byte_offset()) {
-      // Shenandoah doesn't support compressed forwarding pointers
     } else if (klass() == NULL) {
       // Array with unknown body type
       assert(this->isa_aryptr(), "only arrays without klass");
@@ -2529,8 +2527,7 @@
           assert(this->isa_instptr(), "must be an instance ptr.");
           _is_ptr_to_narrowoop = false;
         } else if (klass() == ciEnv::current()->Class_klass() &&
-                   _offset >= InstanceMirrorKlass::offset_of_static_fields() &&
-                   !UseShenandoahGC) {
+                   _offset >= InstanceMirrorKlass::offset_of_static_fields()) {
           // Static fields
           assert(o != NULL, "must be constant");
           ciInstanceKlass* k = o->as_instance()->java_lang_Class_klass()->as_instance_klass();
diff -r 66fab7238237 src/share/vm/prims/jvmtiEnv.cpp
--- a/src/share/vm/prims/jvmtiEnv.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/prims/jvmtiEnv.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -308,9 +308,9 @@
       !java_lang_Class::is_primitive(mirror)) {
     Klass* k = java_lang_Class::as_Klass(mirror);
     assert(k != NULL, "class for non-primitive mirror must exist");
-    *size_ptr = (jlong)(k->size() + Universe::heap()->oop_extra_words()) * wordSize;
+    *size_ptr = (jlong)k->size() * wordSize;
   } else {
-    *size_ptr = (jlong)(mirror->size() + Universe::heap()->oop_extra_words()) * wordSize;
+    *size_ptr = (jlong)mirror->size() * wordSize;
     }
   return JVMTI_ERROR_NONE;
 } /* end GetObjectSize */
diff -r 66fab7238237 src/share/vm/prims/jvmtiExport.cpp
--- a/src/share/vm/prims/jvmtiExport.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/prims/jvmtiExport.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -704,7 +704,7 @@
  public:
    JvmtiVMObjectAllocEventMark(JavaThread *thread, oop obj) : JvmtiClassEventMark(thread, oop_to_klass(obj)) {
      _jobj = (jobject)to_jobject(obj);
-     _size = (obj->size() + Universe::heap()->oop_extra_words()) * wordSize;
+     _size = obj->size() * wordSize;
    };
    jobject jni_jobject() { return _jobj; }
    jlong size() { return _size; }
diff -r 66fab7238237 src/share/vm/prims/whitebox.cpp
--- a/src/share/vm/prims/whitebox.cpp	Wed May 08 20:45:30 2019 +0200
+++ b/src/share/vm/prims/whitebox.cpp	Mon Oct 28 15:51:53 2019 +0100
@@ -305,7 +305,7 @@
 
 WB_ENTRY(jlong, WB_GetObjectSize(JNIEnv* env, jobject o, jobject obj))
   oop p = JNIHandles::resolve(obj);
-  return (p->size() + Universe::heap()->oop_extra_words()) * HeapWordSize;
+  return p->size() * HeapWordSize;
 WB_END
 
 #if INCLUDE_ALL_GCS