# HG changeset patch
# Parent  b25269d2e08d0e5f6886f36afc1e85664376237a

diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/cpu/x86/assembler_x86.cpp
--- a/src/hotspot/cpu/x86/assembler_x86.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -9231,4 +9231,13 @@
   emit_operand(dst, src);
 }
 
+void Assembler::btr(Register dst, int imm8) {
+  InstructionMark im(this);
+  int encode = prefixq_and_encode(dst->encoding());
+  emit_int8(0x0f);
+  emit_int8(0xba);
+  emit_int8(0xf0 | encode);
+  emit_int8(imm8);
+}
+
 #endif // !LP64
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/cpu/x86/assembler_x86.hpp
--- a/src/hotspot/cpu/x86/assembler_x86.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -2192,6 +2192,8 @@
   void blendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len);
   void vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len);
 
+  void btr(Register dst, int imm8);
+
  protected:
   // Next instructions require address alignment 16 bytes SSE mode.
   // They should be called only from corresponding MacroAssembler instructions.
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -1703,7 +1703,7 @@
     // get object class
     // not a safepoint as obj null check happens earlier
 #ifdef _LP64
-    if (UseCompressedClassPointers) {
+    if (UseCompressedClassPointers || UseShenandoahGC) {
       __ load_klass(Rtmp1, obj);
       __ cmpptr(k_RInfo, Rtmp1);
     } else {
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -319,10 +319,15 @@
 
 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
   assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
-  __ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
+  Label done;
+  __ testb(Address(dst, oopDesc::klass_offset_in_bytes()), 1);
+  __ jccb(Assembler::zero, done);
+  __ movptr(dst, Address(dst, oopDesc::klass_offset_in_bytes()));
+  __ shrptr(dst, 1);
+  __ shlptr(dst, 1);
+  __ bind(done);
 }
 
-
 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
   if (ShenandoahWriteBarrier) {
     write_barrier_impl(masm, dst);
@@ -516,55 +521,6 @@
   }
 }
 
-void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
-                                                  Register thread, Register obj,
-                                                  Register var_size_in_bytes,
-                                                  int con_size_in_bytes,
-                                                  Register t1, Register t2,
-                                                  Label& slow_case) {
-  assert_different_registers(obj, t1, t2);
-  assert_different_registers(obj, var_size_in_bytes, t1);
-  Register end = t2;
-  if (!thread->is_valid()) {
-#ifdef _LP64
-    thread = r15_thread;
-#else
-    assert(t1->is_valid(), "need temp reg");
-    thread = t1;
-    __ get_thread(thread);
-#endif
-  }
-
-  __ verify_tlab();
-
-  __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
-  if (var_size_in_bytes == noreg) {
-    __ lea(end, Address(obj, con_size_in_bytes + ShenandoahBrooksPointer::byte_size()));
-  } else {
-    __ addptr(var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
-    __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
-  }
-  __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
-  __ jcc(Assembler::above, slow_case);
-
-  // update the tlab top pointer
-  __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
-
-  // Initialize brooks pointer
-#ifdef _LP64
-  __ incrementq(obj, ShenandoahBrooksPointer::byte_size());
-#else
-  __ incrementl(obj, ShenandoahBrooksPointer::byte_size());
-#endif
-  __ movptr(Address(obj, ShenandoahBrooksPointer::byte_offset()), obj);
-
-  // recover var_size_in_bytes if necessary
-  if (var_size_in_bytes == end) {
-    __ subptr(var_size_in_bytes, obj);
-  }
-  __ verify_tlab();
-}
-
 void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
   bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
   bool is_write = (decorators & ACCESS_WRITE) != 0;
@@ -859,6 +815,29 @@
 
 }
 
+void ShenandoahBarrierSetAssembler::gen_read_barrier_stub(LIR_Assembler* ce, ShenandoahReadBarrierStub* stub) {
+  __ bind(*stub->entry());
+
+  Label done;
+  Register obj = stub->obj()->as_register();
+  Register res = stub->result()->as_register();
+
+  if (res != obj) {
+    __ mov(res, obj);
+  }
+
+  // Check for null.
+  if (stub->needs_null_check()) {
+    __ testptr(res, res);
+    __ jcc(Assembler::zero, done);
+  }
+
+  read_barrier_not_null_impl(ce->masm(), res);
+
+  __ bind(done);
+  __ jmp(*stub->continuation());
+}
+
 void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
   __ bind(*stub->entry());
 
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -29,6 +29,7 @@
 #ifdef COMPILER1
 class LIR_Assembler;
 class ShenandoahPreBarrierStub;
+class ShenandoahReadBarrierStub;
 class ShenandoahWriteBarrierStub;
 class StubAssembler;
 class StubCodeGenerator;
@@ -55,11 +56,12 @@
                                     bool tosca_live,
                                     bool expand_call);
 
-  void read_barrier(MacroAssembler* masm, Register dst);
-  void read_barrier_impl(MacroAssembler* masm, Register dst);
-
-  void read_barrier_not_null(MacroAssembler* masm, Register dst);
-  void read_barrier_not_null_impl(MacroAssembler* masm, Register dst);
+public:
+  static void read_barrier(MacroAssembler* masm, Register dst);
+  static void read_barrier_not_null(MacroAssembler* masm, Register dst);
+private:
+  static void read_barrier_impl(MacroAssembler* masm, Register dst);
+  static void read_barrier_not_null_impl(MacroAssembler* masm, Register dst);
 
   void write_barrier(MacroAssembler* masm, Register dst);
   void write_barrier_impl(MacroAssembler* masm, Register dst);
@@ -78,6 +80,7 @@
 #ifdef COMPILER1
   void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
   void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub);
+  void gen_read_barrier_stub(LIR_Assembler* ce, ShenandoahReadBarrierStub* stub);
   void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
 #endif
 
@@ -103,13 +106,6 @@
   virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2);
   virtual void obj_equals(MacroAssembler* masm, Register src1, Address src2);
 
-  virtual void tlab_allocate(MacroAssembler* masm,
-                             Register thread, Register obj,
-                             Register var_size_in_bytes,
-                             int con_size_in_bytes,
-                             Register t1, Register t2,
-                             Label& slow_case);
-
   virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj);
 
   virtual void barrier_stubs_init();
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoah_x86_64.ad	Tue Dec 25 00:11:08 2018 +0100
@@ -25,45 +25,48 @@
 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
 %}
 
-instruct shenandoahRB(rRegP dst, rRegP src, rFlagsReg cr) %{
+instruct shenandoahRB(rRegP dst, rRegP src, rRegP tmp, rFlagsReg cr) %{
   match(Set dst (ShenandoahReadBarrier src));
-  effect(DEF dst, USE src);
-  ins_cost(125); // XXX
+  effect(TEMP_DEF dst, USE src, TEMP tmp, KILL cr);
+  ins_cost(300); // XXX
   format %{ "shenandoah_rb $dst, $src" %}
   ins_encode %{
     Register d = $dst$$Register;
     Register s = $src$$Register;
-    __ movptr(d, Address(s, ShenandoahBrooksPointer::byte_offset()));
+    Register t = $tmp$$Register;
+    guarantee(s != t, "src and tmp must be different");
+    guarantee(d != t, "dst and tmp must be different");
+    __ movptr(t, Address(s, oopDesc::klass_offset_in_bytes()));
+    if (s != d) {
+      __ movptr(d, s);
+    }
+    __ testptr(t, 1);
+    __ btr(t, 0);
+    __ cmovptr(Assembler::carrySet, d, t);
   %}
-  ins_pipe(ialu_reg_mem);
+  ins_pipe(pipe_slow);
 %}
 
-instruct shenandoahRBNarrow(rRegP dst, rRegN src) %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_shift() == 0));
-  match(Set dst (ShenandoahReadBarrier (DecodeN src)));
-  effect(DEF dst, USE src);
-  ins_cost(125); // XXX
-  format %{ "shenandoah_rb $dst, $src" %}
+instruct shenandoahLoadKlass(rRegP dst, memory mem)
+%{
+  predicate(UseShenandoahGC);
+  match(Set dst (LoadKlass mem));
+
+  ins_cost(300); // XXX
+  format %{ "movq    $dst, $mem\t# class" %}
   ins_encode %{
     Register d = $dst$$Register;
-    Register s = $src$$Register;
-    __ movptr(d, Address(r12, s, Address::times_1, ShenandoahBrooksPointer::byte_offset()));
+    Address s = $mem$$Address;
+    Label done;
+    __ movptr(d, s);
+    __ testptr(d, 1);
+    __ jccb(Assembler::zero, done);
+    __ shrptr(d, 1);
+    __ shlptr(d, 1);
+    __ movptr(d, Address(d, oopDesc::klass_offset_in_bytes()));
+    __ bind(done);
   %}
-  ins_pipe(ialu_reg_mem);
-%}
-
-instruct shenandoahRBNarrowShift(rRegP dst, rRegN src) %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
-  match(Set dst (ShenandoahReadBarrier (DecodeN src)));
-  effect(DEF dst, USE src);
-  ins_cost(125); // XXX
-  format %{ "shenandoah_rb $dst, $src" %}
-  ins_encode %{
-    Register d = $dst$$Register;
-    Register s = $src$$Register;
-    __ movptr(d, Address(r12, s, Address::times_8, ShenandoahBrooksPointer::byte_offset()));
-  %}
-  ins_pipe(ialu_reg_mem);
+  ins_pipe(pipe_slow); // XXX
 %}
 
 instruct compareAndSwapP_shenandoah(rRegI res,
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/cpu/x86/interp_masm_x86.cpp
--- a/src/hotspot/cpu/x86/interp_masm_x86.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -55,7 +55,7 @@
   testptr(obj, obj);
   jccb(Assembler::notZero, update);
   orptr(mdo_addr, TypeEntries::null_seen);
-  jmpb(next);
+  jmp(next);
 
   bind(update);
   load_klass(obj, obj);
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/cpu/x86/macroAssembler_x86.cpp
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -5037,7 +5037,20 @@
     decode_klass_not_null(dst);
   } else
 #endif
-    movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+  {
+    if (UseShenandoahGC) {
+      Label done;
+      movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+      testptr(dst, 1);
+      jccb(Assembler::zero, done);
+      shrptr(dst, 1);
+      shlptr(dst, 1);
+      movptr(dst, Address(dst, oopDesc::klass_offset_in_bytes()));
+      bind(done);
+    } else {
+      movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
+    }
+  }
 }
 
 void MacroAssembler::load_prototype_header(Register dst, Register src) {
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/cpu/x86/x86_64.ad
--- a/src/hotspot/cpu/x86/x86_64.ad	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/cpu/x86/x86_64.ad	Tue Dec 25 00:11:08 2018 +0100
@@ -5310,6 +5310,7 @@
 // Load Klass Pointer
 instruct loadKlass(rRegP dst, memory mem)
 %{
+  predicate(!UseShenandoahGC);
   match(Set dst (LoadKlass mem));
 
   ins_cost(125); // XXX
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -51,6 +51,11 @@
   bs->gen_write_barrier_stub(ce, this);
 }
 
+void ShenandoahReadBarrierStub::emit_code(LIR_Assembler* ce) {
+  ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
+  bs->gen_read_barrier_stub(ce, this);
+}
+
 void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) {
   // First we test whether marking is in progress.
   BasicType flag_type;
@@ -115,17 +120,16 @@
 
 LIR_Opr ShenandoahBarrierSetC1::read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
   assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "Should be enabled");
-  LabelObj* done = new LabelObj();
   LIR_Opr result = gen->new_register(T_OBJECT);
+
+  obj = ensure_in_register(gen, obj);
+
   __ move(obj, result);
-  if (need_null_check) {
-    __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL));
-    __ branch(lir_cond_equal, T_LONG, done->label());
-  }
-  LIR_Address* brooks_ptr_address = gen->generate_address(result, ShenandoahBrooksPointer::byte_offset(), T_ADDRESS);
-  __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none);
 
-  __ branch_destination(done->label());
+  CodeStub* slow = new ShenandoahReadBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check);
+  __ jump(slow);
+  __ branch_destination(slow->continuation());
+
   return result;
 }
 
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp
--- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -117,6 +117,38 @@
 #endif // PRODUCT
 };
 
+class ShenandoahReadBarrierStub: public CodeStub {
+  friend class ShenandoahBarrierSetC1;
+private:
+  LIR_Opr _obj;
+  LIR_Opr _result;
+  CodeEmitInfo* _info;
+  bool _needs_null_check;
+
+public:
+  ShenandoahReadBarrierStub(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, bool needs_null_check) :
+          _obj(obj), _result(result), _info(info), _needs_null_check(needs_null_check)
+  {
+    assert(_obj->is_register(), "should be register");
+    assert(_result->is_register(), "should be register");
+  }
+
+  LIR_Opr obj() const { return _obj; }
+  LIR_Opr result() const { return _result; }
+  CodeEmitInfo* info() const { return _info; }
+  bool needs_null_check() const { return _needs_null_check; }
+
+  virtual void emit_code(LIR_Assembler* e);
+  virtual void visit(LIR_OpVisitState* visitor) {
+    visitor->do_slow_case();
+    visitor->do_input(_obj);
+    visitor->do_temp(_result);
+  }
+#ifndef PRODUCT
+  virtual void print_name(outputStream* out) const { out->print("ShenandoahWritePreBarrierStub"); }
+#endif // PRODUCT
+};
+
 class LIR_OpShenandoahCompareAndSwap : public LIR_Op {
  friend class LIR_OpVisitState;
 
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -870,30 +870,6 @@
   }
 }
 
-Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
-                                           Node*& i_o, Node*& needgc_ctrl,
-                                           Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
-                                           intx prefetch_lines) const {
-  PhaseIterGVN& igvn = macro->igvn();
-
-  // Allocate several words more for the Shenandoah brooks pointer.
-  size_in_bytes = new AddXNode(size_in_bytes, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));
-  macro->transform_later(size_in_bytes);
-
-  Node* fast_oop = BarrierSetC2::obj_allocate(macro, ctrl, mem, toobig_false, size_in_bytes,
-                                              i_o, needgc_ctrl, fast_oop_ctrl, fast_oop_rawmem,
-                                              prefetch_lines);
-
-  // Bump up object for Shenandoah brooks pointer.
-  fast_oop = new AddPNode(macro->top(), fast_oop, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));
-  macro->transform_later(fast_oop);
-
-  // Initialize Shenandoah brooks pointer to point to the object itself.
-  fast_oop_rawmem = macro->make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, ShenandoahBrooksPointer::byte_offset(), fast_oop, T_OBJECT);
-
-  return fast_oop;
-}
-
 // Support for GC barriers emitted during parsing
 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
   if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
@@ -1291,7 +1267,7 @@
 
 bool ShenandoahBarrierSetC2::flatten_gc_alias_type(const TypePtr*& adr_type) const {
   int offset = adr_type->offset();
-  if (offset == ShenandoahBrooksPointer::byte_offset()) {
+  if (offset == -8) {
     if (adr_type->isa_aryptr()) {
       adr_type = TypeAryPtr::make(adr_type->ptr(), adr_type->isa_aryptr()->ary(), adr_type->isa_aryptr()->klass(), false, offset);
     } else if (adr_type->isa_instptr()) {
@@ -1349,7 +1325,7 @@
 
 #ifdef ASSERT
 bool ShenandoahBarrierSetC2::verify_gc_alias_type(const TypePtr* adr_type, int offset) const {
-  if (offset == ShenandoahBrooksPointer::byte_offset() &&
+  if (offset == -8 &&
       (adr_type->base() == Type::AryPtr || adr_type->base() == Type::OopPtr)) {
     return true;
   } else {
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -108,11 +108,6 @@
 
   virtual Node* resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const;
 
-  virtual Node* obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,
-                             Node*& i_o, Node*& needgc_ctrl,
-                             Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
-                             intx prefetch_lines) const;
-
   // These are general helper methods used by C2
   virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const;
   virtual void clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const;
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -97,7 +97,7 @@
     return false;
   }
 
-  if (n->is_CallJava() || n->Opcode() == Op_CallLeafNoFP) {
+  if (n->is_CallJava() || n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {
     return true;
   }
 
@@ -105,6 +105,7 @@
   if (type == Type::TOP) {
     return false;
   }
+  assert(type->isa_ptr(), "must be");
   if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) {
     return false;
   }
@@ -2920,12 +2921,8 @@
       IfNode* iff = unc_ctrl->in(0)->as_If();
       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
     }
-    Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset()));
-    phase->register_new_node(addr, ctrl);
-    assert(val->bottom_type()->isa_oopptr(), "what else?");
-    const TypePtr* obj_type =  val->bottom_type()->is_oopptr();
-    const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
-    Node* fwd = new LoadPNode(ctrl, wb_mem, addr, adr_type, obj_type, MemNode::unordered);
+
+    Node* fwd = new ShenandoahReadBarrierNode(ctrl, wb_mem, new_val);
     phase->register_new_node(fwd, ctrl);
 
     // Only branch to WB stub if object is not forwarded; otherwise reply with fwd ptr
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -72,7 +72,7 @@
   static Node* skip_through_barrier(Node* n);
 
   static const TypeOopPtr* brooks_pointer_type(const Type* t) {
-    return t->is_oopptr()->cast_to_nonconst()->add_offset(ShenandoahBrooksPointer::byte_offset())->is_oopptr();
+    return t->is_oopptr()->cast_to_nonconst()->add_offset(-8)->is_oopptr();
   }
 
   virtual const TypePtr* adr_type() const {
@@ -81,7 +81,7 @@
     }
     //const TypePtr* adr_type = in(MemNode::Address)->bottom_type()->is_ptr();
     const TypePtr* adr_type = brooks_pointer_type(bottom_type());
-    assert(adr_type->offset() == ShenandoahBrooksPointer::byte_offset(), "sane offset");
+    assert(adr_type->offset() == -8, "sane offset");
     assert(Compile::current()->alias_type(adr_type)->is_rewritable(), "brooks ptr must be rewritable");
     return adr_type;
   }
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -192,6 +192,9 @@
     FLAG_SET_DEFAULT(TLABAllocationWeight, 90);
   }
 
+  // UseCompressedClassPointers not supported
+  FLAG_SET_DEFAULT(UseCompressedClassPointers, false);
+
   // Shenandoah needs more C2 nodes to compile some methods with lots of barriers.
   // NodeLimitFudgeFactor needs to stay the same relative to MaxNodeLimit.
 #ifdef COMPILER2
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -251,7 +251,7 @@
                   file, line);
   }
 
-  size_t alloc_size = obj->size() + ShenandoahBrooksPointer::word_size();
+  size_t alloc_size = obj->size();
   if (alloc_size > ShenandoahHeapRegion::humongous_threshold_words()) {
     size_t idx = r->region_number();
     size_t num_regions = ShenandoahHeapRegion::required_regions(alloc_size * HeapWordSize);
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -261,7 +261,7 @@
       ShenandoahHeapRegion* r = _heap->heap_region_containing(obj);
       assert(r->is_cset(), "sanity");
 
-      HeapWord* cur = (HeapWord*)obj + obj->size() + ShenandoahBrooksPointer::word_size();
+      HeapWord* cur = (HeapWord*)obj + obj->size();
 
       size_t count = 0;
       while ((cur < r->top()) && ctx->is_marked(oop(cur)) && (count++ < max)) {
@@ -269,7 +269,7 @@
         if (oopDesc::equals_raw(cur_oop, resolve_forwarded_not_null(cur_oop))) {
           _heap->evacuate_object(cur_oop, thread);
         }
-        cur = cur + cur_oop->size() + ShenandoahBrooksPointer::word_size();
+        cur = cur + cur_oop->size();
       }
     }
 
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -36,7 +36,12 @@
 }
 
 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {
-  return ShenandoahBrooksPointer::forwardee(p);
+  intptr_t klassptr = reinterpret_cast<intptr_t>(p->klass_raw());
+  if ((klassptr & 1) == 0) {
+    return p;
+  } else {
+    return oop(reinterpret_cast<HeapWord*>(klassptr & ~1));
+  }
 }
 
 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahBrooksPointer.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBrooksPointer.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBrooksPointer.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -44,52 +44,16 @@
    */
 
 public:
-  /* Offset from the object start, in HeapWords. */
-  static inline int word_offset() {
-    return -1; // exactly one HeapWord
-  }
-
-  /* Offset from the object start, in bytes. */
-  static inline int byte_offset() {
-    return -HeapWordSize; // exactly one HeapWord
-  }
-
-  /* Allocated size, in HeapWords. */
-  static inline uint word_size() {
-    return (uint) MinObjAlignment;
-  }
-
-  /* Allocated size, in bytes */
-  static inline uint byte_size() {
-    return (uint) MinObjAlignmentInBytes;
-  }
-
   /* Assert basic stuff once at startup. */
   static void initial_checks() {
     guarantee (MinObjAlignment > 0, "sanity, word_size is correct");
     guarantee (MinObjAlignmentInBytes > 0, "sanity, byte_size is correct");
   }
 
-  /* Initializes Brooks pointer (to self).
-   */
-  static inline void initialize(oop obj);
-
   /* Gets forwardee from the given object.
    */
   static inline oop forwardee(oop obj);
 
-  /* Tries to atomically update forwardee in $holder object to $update.
-   * Assumes $holder points at itself.
-   * Asserts $holder is in from-space.
-   * Asserts $update is in to-space.
-   */
-  static inline oop try_update_forwardee(oop obj, oop update);
-
-  /* Sets raw value for forwardee slot.
-   * THIS IS DANGEROUS: USERS HAVE TO INITIALIZE/SET FORWARDEE BACK AFTER THEY ARE DONE.
-   */
-  static inline void set_raw(oop obj, HeapWord* update);
-
   /* Returns the raw value from forwardee slot.
    */
   static inline HeapWord* get_raw(oop obj);
@@ -98,9 +62,6 @@
    * Used for quick verification.
    */
   static inline HeapWord* get_raw_unchecked(oop obj);
-
-private:
-  static inline HeapWord** brooks_ptr_addr(oop obj);
 };
 
 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHBROOKSPOINTER_HPP
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahBrooksPointer.inline.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahBrooksPointer.inline.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBrooksPointer.inline.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -28,38 +28,23 @@
 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"
 #include "runtime/atomic.hpp"
 
-inline HeapWord** ShenandoahBrooksPointer::brooks_ptr_addr(oop obj) {
-  return (HeapWord**)((HeapWord*) obj + word_offset());
-}
-
-inline void ShenandoahBrooksPointer::initialize(oop obj) {
-  shenandoah_assert_in_heap(NULL, obj);
-  *brooks_ptr_addr(obj) = (HeapWord*) obj;
-}
-
-inline void ShenandoahBrooksPointer::set_raw(oop obj, HeapWord* update) {
-  shenandoah_assert_in_heap(NULL, obj);
-  *brooks_ptr_addr(obj) = update;
-}
-
 inline HeapWord* ShenandoahBrooksPointer::get_raw(oop obj) {
   shenandoah_assert_in_heap(NULL, obj);
-  return *brooks_ptr_addr(obj);
+  return get_raw_unchecked(obj);
 }
 
 inline HeapWord* ShenandoahBrooksPointer::get_raw_unchecked(oop obj) {
-  return *brooks_ptr_addr(obj);
+  intptr_t klassptr = reinterpret_cast<intptr_t>(obj->klass_raw());
+  if ((klassptr & 1) == 0) {
+    return (HeapWord*) obj;
+  } else {
+    return reinterpret_cast<HeapWord*>(klassptr & ~1);
+  }
 }
 
 inline oop ShenandoahBrooksPointer::forwardee(oop obj) {
   shenandoah_assert_correct(NULL, obj);
-  return oop(*brooks_ptr_addr(obj));
-}
-
-inline oop ShenandoahBrooksPointer::try_update_forwardee(oop obj, oop update) {
-  oop result = (oop) Atomic::cmpxchg(update, (oop*)brooks_ptr_addr(obj), obj);
-  shenandoah_assert_correct_except(NULL, obj, !oopDesc::equals_raw(result, obj));
-  return result;
+  return oop(get_raw_unchecked(obj));
 }
 
 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHBROOKSPOINTER_INLINE_HPP
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -69,7 +69,7 @@
 inline void ShenandoahConcurrentMark::count_liveness(jushort* live_data, oop obj) {
   size_t region_idx = _heap->heap_region_index_containing(obj);
   ShenandoahHeapRegion* region = _heap->get_region(region_idx);
-  size_t size = obj->size() + ShenandoahBrooksPointer::word_size();
+  size_t size = obj->size();
 
   if (!region->is_humongous_start()) {
     assert(!region->is_humongous(), "Cannot have continuations here");
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -769,49 +769,6 @@
   return _free_set->allocate(req, in_new_region);
 }
 
-class ShenandoahMemAllocator : public MemAllocator {
-private:
-  MemAllocator& _initializer;
-public:
-  ShenandoahMemAllocator(MemAllocator& initializer, Klass* klass, size_t word_size, Thread* thread) :
-  MemAllocator(klass, word_size + ShenandoahBrooksPointer::word_size(), thread),
-    _initializer(initializer) {}
-
-protected:
-  virtual HeapWord* mem_allocate(Allocation& allocation) const {
-    HeapWord* result = MemAllocator::mem_allocate(allocation);
-    // Initialize brooks-pointer
-    if (result != NULL) {
-      result += ShenandoahBrooksPointer::word_size();
-      ShenandoahBrooksPointer::initialize(oop(result));
-      assert(! ShenandoahHeap::heap()->in_collection_set(result), "never allocate in targetted region");
-    }
-    return result;
-  }
-
-  virtual oop initialize(HeapWord* mem) const {
-     return _initializer.initialize(mem);
-  }
-};
-
-oop ShenandoahHeap::obj_allocate(Klass* klass, int size, TRAPS) {
-  ObjAllocator initializer(klass, size, THREAD);
-  ShenandoahMemAllocator allocator(initializer, klass, size, THREAD);
-  return allocator.allocate();
-}
-
-oop ShenandoahHeap::array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS) {
-  ObjArrayAllocator initializer(klass, size, length, do_zero, THREAD);
-  ShenandoahMemAllocator allocator(initializer, klass, size, THREAD);
-  return allocator.allocate();
-}
-
-oop ShenandoahHeap::class_allocate(Klass* klass, int size, TRAPS) {
-  ClassAllocator initializer(klass, size, THREAD);
-  ShenandoahMemAllocator allocator(initializer, klass, size, THREAD);
-  return allocator.allocate();
-}
-
 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
                                         bool*  gc_overhead_limit_was_exceeded) {
   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
@@ -854,15 +811,6 @@
   return NULL;
 }
 
-void ShenandoahHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap) {
-  HeapWord* obj = tlab_post_allocation_setup(start);
-  CollectedHeap::fill_with_object(obj, end);
-}
-
-size_t ShenandoahHeap::min_dummy_object_size() const {
-  return CollectedHeap::min_dummy_object_size() + ShenandoahBrooksPointer::word_size();
-}
-
 class ShenandoahEvacuateUpdateRootsClosure: public BasicOopIterateClosure {
 private:
   ShenandoahHeap* _heap;
@@ -990,8 +938,8 @@
 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
 
-  oop humongous_obj = oop(start->bottom() + ShenandoahBrooksPointer::word_size());
-  size_t size = humongous_obj->size() + ShenandoahBrooksPointer::word_size();
+  oop humongous_obj = oop(start->bottom());
+  size_t size = humongous_obj->size();
   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
   size_t index = start->region_number() + required_regions - 1;
 
@@ -1809,13 +1757,6 @@
   set_gc_state_mask(EVACUATION, in_progress);
 }
 
-HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
-  // Initialize Brooks pointer for the next object
-  HeapWord* result = obj + ShenandoahBrooksPointer::word_size();
-  ShenandoahBrooksPointer::initialize(oop(result));
-  return result;
-}
-
 ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
   _mark_context(ShenandoahHeap::heap()->marking_context()) {
 }
@@ -2782,14 +2723,6 @@
   }
 }
 
-size_t ShenandoahHeap::obj_size(oop obj) const {
-  return CollectedHeap::obj_size(obj) + ShenandoahBrooksPointer::word_size();
-}
-
-ptrdiff_t ShenandoahHeap::cell_header_size() const {
-  return ShenandoahBrooksPointer::byte_size();
-}
-
 BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
   return ShenandoahHeap::heap()->has_forwarded_objects() ? reinterpret_cast<BoolObjectClosure*>(&_fwd_alive_cl)
                                                          : reinterpret_cast<BoolObjectClosure*>(&_alive_cl);
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -553,9 +553,6 @@
 
   bool is_in(const void* p) const;
 
-  size_t obj_size(oop obj) const;
-  virtual ptrdiff_t cell_header_size() const;
-
   // All objects can potentially move
   bool is_scavengable(oop obj) { return true; };
 
@@ -611,10 +608,6 @@
                                                size_t size,
                                                Metaspace::MetadataType mdtype);
 
-  oop obj_allocate(Klass* klass, int size, TRAPS);
-  oop array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS);
-  oop class_allocate(Klass* klass, int size, TRAPS);
-
   void notify_mutator_alloc_words(size_t words, bool waste);
 
   // Shenandoah supports TLAB allocation
@@ -626,10 +619,6 @@
   size_t max_tlab_size() const;
   size_t tlab_used(Thread* ignored) const;
 
-  HeapWord* tlab_post_allocation_setup(HeapWord* obj);
-  void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap);
-  size_t min_dummy_object_size() const;
-
   void resize_tlabs();
 
   void ensure_parsability(bool retire_tlabs);
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -264,8 +264,13 @@
 
   assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
 
-  size_t size_no_fwdptr = (size_t) p->size();
-  size_t size_with_fwdptr = size_no_fwdptr + ShenandoahBrooksPointer::word_size();
+  Klass* klass = p->klass_raw();
+  intptr_t klassptr = reinterpret_cast<intptr_t>(klass);
+  if ((klassptr & 1) == 1) {
+    return oop(reinterpret_cast<HeapWord*>(klassptr & ~1));
+  }
+
+  size_t size = (size_t) p->size_given_klass(klass);
 
   assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
 
@@ -279,10 +284,10 @@
   } else {
 #endif
     if (UseTLAB) {
-      filler = allocate_from_gclab(thread, size_with_fwdptr);
+      filler = allocate_from_gclab(thread, size);
     }
     if (filler == NULL) {
-      ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size_with_fwdptr);
+      ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size);
       filler = allocate_memory(req);
       alloc_from_gclab = false;
     }
@@ -291,7 +296,7 @@
 #endif
 
   if (filler == NULL) {
-    control_thread()->handle_alloc_failure_evac(size_with_fwdptr);
+    control_thread()->handle_alloc_failure_evac(size);
 
     _oom_evac_handler.handle_out_of_memory_during_evacuation();
 
@@ -299,16 +304,18 @@
   }
 
   // Copy the object and initialize its forwarding ptr:
-  HeapWord* copy = filler + ShenandoahBrooksPointer::word_size();
+  HeapWord* copy = filler;
   oop copy_val = oop(copy);
 
-  Copy::aligned_disjoint_words((HeapWord*) p, copy, size_no_fwdptr);
-  ShenandoahBrooksPointer::initialize(oop(copy));
+  Copy::aligned_disjoint_words((HeapWord*) p, copy, size);
+  // ShenandoahBrooksPointer::initialize(oop(copy));
 
   // Try to install the new forwarding pointer.
-  oop result = ShenandoahBrooksPointer::try_update_forwardee(p, copy_val);
+  intptr_t fwdptr = reinterpret_cast<intptr_t>(copy) | 1;
+  intptr_t* addr = reinterpret_cast<intptr_t*>(p->klass_addr());
+  intptr_t result = Atomic::cmpxchg(fwdptr, addr, klassptr);
 
-  if (oopDesc::equals_raw(result, p)) {
+  if (result == klassptr) {
     // Successfully evacuated. Our copy is now the public one!
     shenandoah_assert_correct(NULL, copy_val);
     return copy_val;
@@ -325,13 +332,13 @@
     // have to explicitly overwrite the copy with the filler object. With that overwrite,
     // we have to keep the fwdptr initialized and pointing to our (stale) copy.
     if (alloc_from_gclab) {
-      ShenandoahThreadLocalData::gclab(thread)->undo_allocation(filler, size_with_fwdptr);
+      ShenandoahThreadLocalData::gclab(thread)->undo_allocation(filler, size);
     } else {
-      fill_with_object(copy, size_no_fwdptr);
+      fill_with_object(copy, size);
     }
-    shenandoah_assert_correct(NULL, copy_val);
-    shenandoah_assert_correct(NULL, result);
-    return result;
+    // shenandoah_assert_correct(NULL, copy_val);
+    shenandoah_assert_correct(NULL, oop(reinterpret_cast<HeapWord*>(result & ~1)));
+    return oop(reinterpret_cast<HeapWord*>(result & ~1));
   }
 }
 
@@ -395,7 +402,6 @@
 
 template<class T>
 inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) {
-  assert(ShenandoahBrooksPointer::word_offset() < 0, "skip_delta calculation below assumes the forwarding ptr is before obj");
   assert(! region->is_humongous_continuation(), "no humongous continuation regions here");
 
   ShenandoahMarkingContext* const ctx = complete_marking_context();
@@ -404,10 +410,9 @@
   MarkBitMap* mark_bit_map = ctx->mark_bit_map();
   HeapWord* tams = ctx->top_at_mark_start(region);
 
-  size_t skip_bitmap_delta = ShenandoahBrooksPointer::word_size() + 1;
-  size_t skip_objsize_delta = ShenandoahBrooksPointer::word_size() /* + actual obj.size() below */;
-  HeapWord* start = region->bottom() + ShenandoahBrooksPointer::word_size();
-  HeapWord* end = MIN2(tams + ShenandoahBrooksPointer::word_size(), region->end());
+  size_t skip_bitmap_delta = 1;
+  HeapWord* start = region->bottom();
+  HeapWord* end = MIN2(tams, region->end());
 
   // Step 1. Scan below the TAMS based on bitmap data.
   HeapWord* limit_bitmap = MIN2(limit, tams);
@@ -437,7 +442,7 @@
     do {
       avail = 0;
       for (int c = 0; (c < dist) && (cb < limit_bitmap); c++) {
-        Prefetch::read(cb, ShenandoahBrooksPointer::byte_offset());
+        Prefetch::read(cb, oopDesc::klass_offset_in_bytes());
         slots[avail++] = cb;
         cb += skip_bitmap_delta;
         if (cb < limit_bitmap) {
@@ -472,16 +477,16 @@
   // Step 2. Accurate size-based traversal, happens past the TAMS.
   // This restarts the scan at TAMS, which makes sure we traverse all objects,
   // regardless of what happened at Step 1.
-  HeapWord* cs = tams + ShenandoahBrooksPointer::word_size();
+  HeapWord* cs = tams;
   while (cs < limit) {
-    assert (cs > tams,  "only objects past TAMS here: "   PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams));
+    assert (cs >= tams,  "only objects past TAMS here: "   PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams));
     assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit));
     oop obj = oop(cs);
     assert(oopDesc::is_oop(obj), "sanity");
     assert(ctx->is_marked(obj), "object expected to be marked");
     int size = obj->size();
     cl->do_object(obj);
-    cs += size + skip_objsize_delta;
+    cs += size;
   }
 }
 
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -454,12 +454,12 @@
 
 void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
   assert(! is_humongous(), "no humongous region here");
-  HeapWord* obj_addr = bottom() + ShenandoahBrooksPointer::word_size();
+  HeapWord* obj_addr = bottom();
   HeapWord* t = top();
   // Could call objects iterate, but this is easier.
   while (obj_addr < t) {
     oop obj = oop(obj_addr);
-    obj_addr += obj->oop_iterate_size(blk) + ShenandoahBrooksPointer::word_size();
+    obj_addr += obj->oop_iterate_size(blk);
   }
 }
 
@@ -468,7 +468,7 @@
   // Find head.
   ShenandoahHeapRegion* r = humongous_start_region();
   assert(r->is_humongous_start(), "need humongous head here");
-  oop obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
+  oop obj = oop(r->bottom());
   obj->oop_iterate(blk, MemRegion(bottom(), top()));
 }
 
@@ -507,11 +507,11 @@
   if (p >= top()) {
     return top();
   } else {
-    HeapWord* last = bottom() + ShenandoahBrooksPointer::word_size();
+    HeapWord* last = bottom();
     HeapWord* cur = last;
     while (cur <= p) {
       last = cur;
-      cur += oop(cur)->size() + ShenandoahBrooksPointer::word_size();
+      cur += oop(cur)->size();
     }
     shenandoah_assert_correct(NULL, oop(last));
     return last;
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -164,7 +164,7 @@
       // Reclaim humongous regions here, and count them as the immediate garbage
 #ifdef ASSERT
       bool reg_live = region->has_live();
-      bool bm_live = ctx->is_marked(oop(region->bottom() + ShenandoahBrooksPointer::word_size()));
+      bool bm_live = ctx->is_marked(oop(region->bottom()));
       assert(reg_live == bm_live,
              "Humongous liveness and marks should agree. Region live: %s; Bitmap live: %s; Region Live Words: " SIZE_FORMAT,
              BOOL_TO_STR(reg_live), BOOL_TO_STR(bm_live), region->get_live_data_words());
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -233,6 +233,7 @@
 
 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
 private:
+  ShenandoahMarkCompact*   const _mark_compact;
   ShenandoahHeap*          const _heap;
   GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
   int _empty_regions_pos;
@@ -241,7 +242,8 @@
   HeapWord* _compact_point;
 
 public:
-  ShenandoahPrepareForCompactionObjectClosure(GrowableArray<ShenandoahHeapRegion*>& empty_regions, ShenandoahHeapRegion* to_region) :
+  ShenandoahPrepareForCompactionObjectClosure(ShenandoahMarkCompact* mc, GrowableArray<ShenandoahHeapRegion*>& empty_regions, ShenandoahHeapRegion* to_region) :
+    _mark_compact(mc),
     _heap(ShenandoahHeap::heap()),
     _empty_regions(empty_regions),
     _empty_regions_pos(0),
@@ -271,7 +273,7 @@
     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
     assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
 
-    size_t obj_size = p->size() + ShenandoahBrooksPointer::word_size();
+    size_t obj_size = p->size();
     if (_compact_point + obj_size > _to_region->end()) {
       finish_region();
 
@@ -294,13 +296,15 @@
     // Object fits into current region, record new location:
     assert(_compact_point + obj_size <= _to_region->end(), "must fit");
     shenandoah_assert_not_forwarded(NULL, p);
-    ShenandoahBrooksPointer::set_raw(p, _compact_point + ShenandoahBrooksPointer::word_size());
+    _mark_compact->preserve_mark(p);
+    p->forward_to(oop(_compact_point));
     _compact_point += obj_size;
   }
 };
 
 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
 private:
+  ShenandoahMarkCompact*    const _mark_compact;
   ShenandoahHeap*           const _heap;
   ShenandoahHeapRegionSet** const _worker_slices;
   ShenandoahRegionIterator        _heap_regions;
@@ -323,8 +327,9 @@
   }
 
 public:
-  ShenandoahPrepareForCompactionTask(ShenandoahHeapRegionSet** worker_slices) :
+  ShenandoahPrepareForCompactionTask(ShenandoahMarkCompact* mc, ShenandoahHeapRegionSet** worker_slices) :
     AbstractGangTask("Shenandoah Prepare For Compaction Task"),
+    _mark_compact(mc),
     _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
   }
 
@@ -340,7 +345,7 @@
     // Remember empty regions and reuse them as needed.
     ResourceMark rm;
     GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
-    ShenandoahPrepareForCompactionObjectClosure cl(empty_regions, from_region);
+    ShenandoahPrepareForCompactionObjectClosure cl(_mark_compact, empty_regions, from_region);
     while (from_region != NULL) {
       cl.set_from_region(from_region);
       if (from_region->has_live()) {
@@ -390,15 +395,16 @@
 
     if (r->is_humongous_start() && r->is_move_allowed()) {
       // From-region candidate: movable humongous region
-      oop old_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
-      size_t words_size = old_obj->size() + ShenandoahBrooksPointer::word_size();
+      oop old_obj = oop(r->bottom());
+      size_t words_size = old_obj->size();
       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 
       size_t start = to_end - num_regions;
 
       if (start >= to_begin && start != r->region_number()) {
         // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
-        ShenandoahBrooksPointer::set_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahBrooksPointer::word_size());
+        preserve_mark(old_obj);
+        old_obj->forward_to(oop(heap->get_region(start)->bottom()));
         to_end = start;
         continue;
       }
@@ -446,7 +452,7 @@
 
   void heap_region_do(ShenandoahHeapRegion* r) {
     if (r->is_humongous_start()) {
-      oop humongous_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
+      oop humongous_obj = oop(r->bottom());
       if (!_ctx->is_marked(humongous_obj)) {
         assert(!r->has_live(),
                "Region " SIZE_FORMAT " is not marked, should not have live", r->region_number());
@@ -487,7 +493,7 @@
   // Compute the new addresses for regular objects
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
-    ShenandoahPrepareForCompactionTask prepare_task(worker_slices);
+    ShenandoahPrepareForCompactionTask prepare_task(this, worker_slices);
     heap->workers()->run_task(&prepare_task);
   }
 
@@ -509,7 +515,7 @@
     if (!CompressedOops::is_null(o)) {
       oop obj = CompressedOops::decode_not_null(o);
       assert(_ctx->is_marked(obj), "must be marked");
-      oop forw = oop(ShenandoahBrooksPointer::get_raw(obj));
+      oop forw = obj->forwardee();
       RawAccess<IS_NOT_NULL>::oop_store(p, forw);
     }
   }
@@ -534,7 +540,6 @@
   }
   void do_object(oop p) {
     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
-    HeapWord* forw = ShenandoahBrooksPointer::get_raw(p);
     p->oop_iterate(&_cl);
   }
 };
@@ -605,6 +610,8 @@
 
   ShenandoahAdjustPointersTask adjust_pointers_task;
   workers->run_task(&adjust_pointers_task);
+
+  adjust_marks();
 }
 
 class ShenandoahCompactObjectsClosure : public ObjectClosure {
@@ -619,13 +626,13 @@
   void do_object(oop p) {
     assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
     size_t size = (size_t)p->size();
-    HeapWord* compact_to = ShenandoahBrooksPointer::get_raw(p);
+    HeapWord* compact_to = (HeapWord*) oop(p)->forwardee();
     HeapWord* compact_from = (HeapWord*) p;
     if (compact_from != compact_to) {
       Copy::aligned_conjoint_words(compact_from, compact_to, size);
     }
     oop new_obj = oop(compact_to);
-    ShenandoahBrooksPointer::initialize(new_obj);
+    new_obj->init_mark();
   }
 };
 
@@ -719,13 +726,13 @@
   for (size_t c = heap->num_regions() - 1; c > 0; c--) {
     ShenandoahHeapRegion* r = heap->get_region(c);
     if (r->is_humongous_start()) {
-      oop old_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
-      size_t words_size = old_obj->size() + ShenandoahBrooksPointer::word_size();
+      oop old_obj = oop(r->bottom());
+      size_t words_size = old_obj->size();
       size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 
       size_t old_start = r->region_number();
       size_t old_end   = old_start + num_regions - 1;
-      size_t new_start = heap->heap_region_index_containing(ShenandoahBrooksPointer::get_raw(old_obj));
+      size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
       size_t new_end   = new_start + num_regions - 1;
 
       if (old_start == new_start) {
@@ -739,8 +746,8 @@
                                    heap->get_region(new_start)->bottom(),
                                    ShenandoahHeapRegion::region_size_words()*num_regions);
 
-      oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahBrooksPointer::word_size());
-      ShenandoahBrooksPointer::initialize(new_obj);
+      oop new_obj = oop(heap->get_region(new_start)->bottom());
+      new_obj->init_mark();
 
       {
         for (size_t c = old_start; c <= old_end; c++) {
@@ -814,6 +821,8 @@
     heap->workers()->run_task(&compact_task);
   }
 
+  restore_marks();
+
   // Compact humongous objects after regular object moves
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
@@ -842,3 +851,32 @@
 
   heap->clear_cancelled_gc();
 }
+
+void ShenandoahMarkCompact::preserve_mark(oop obj) {
+  markOop mark = obj->mark_raw();
+  if (mark->must_be_preserved(obj)) {
+    _preserved_mark_stack.push(mark);
+    _preserved_oop_stack.push(obj);
+  }
+}
+
+void ShenandoahMarkCompact::restore_marks() {
+  while (!_preserved_oop_stack.is_empty()) {
+    oop obj       = _preserved_oop_stack.pop();
+    markOop mark  = _preserved_mark_stack.pop();
+    obj->set_mark_raw(mark);
+  }
+}
+
+void ShenandoahMarkCompact::adjust_marks() {
+  assert( _preserved_oop_stack.size() == _preserved_mark_stack.size(),
+          "inconsistent preserved oop stacks");
+
+  StackIterator<oop, mtGC> iter(_preserved_oop_stack);
+  while (!iter.is_empty()) {
+    oop* p = iter.next_addr();
+    oop obj = *p;
+    oop fwd = obj->forwardee();
+    *p = fwd;
+  }
+}
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.hpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -51,21 +51,34 @@
 
 class ShenandoahMarkCompact : public CHeapObj<mtGC> {
 private:
-  GCTimer* _gc_timer;
+  GCTimer *_gc_timer;
+
+  Stack <markOop, mtGC> _preserved_mark_stack;
+  Stack <oop, mtGC> _preserved_oop_stack;
 
 public:
-  void initialize(GCTimer* gc_timer);
+  ShenandoahMarkCompact() : _gc_timer(NULL), _preserved_mark_stack(), _preserved_oop_stack() {}
+  void initialize(GCTimer *gc_timer);
+
   void do_it(GCCause::Cause gc_cause);
 
 private:
   void phase1_mark_heap();
-  void phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices);
+
+  void phase2_calculate_target_addresses(ShenandoahHeapRegionSet **worker_slices);
+
   void phase3_update_references();
-  void phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices);
+
+  void phase4_compact_objects(ShenandoahHeapRegionSet **worker_slices);
 
   void calculate_target_humongous_objects();
+
   void compact_humongous_objects();
 
+public:
+  void preserve_mark(oop obj);
+  void restore_marks();
+  void adjust_marks();
 };
 
 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKCOMPACT_HPP
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -639,7 +639,7 @@
         bool candidate = traversal_regions->is_in(r) && !r->has_live() && not_allocated;
         if (r->is_humongous_start() && candidate) {
           // Trash humongous.
-          HeapWord* humongous_obj = r->bottom() + ShenandoahBrooksPointer::word_size();
+          HeapWord* humongous_obj = r->bottom();
           assert(!ctx->is_marked(oop(humongous_obj)), "must not be marked");
           r->make_trash_immediate();
           while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) {
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp	Tue Dec 25 00:11:08 2018 +0100
@@ -138,7 +138,7 @@
           // skip
           break;
         case ShenandoahVerifier::_verify_liveness_complete:
-          Atomic::add(obj->size() + ShenandoahBrooksPointer::word_size(), &_ld[obj_reg->region_number()]);
+          Atomic::add((uint) obj->size(), &_ld[obj_reg->region_number()]);
           // fallthrough for fast failure for un-live regions:
         case ShenandoahVerifier::_verify_liveness_conservative:
           check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
@@ -529,7 +529,7 @@
 
   virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
     size_t processed = 0;
-    HeapWord* obj = r->bottom() + ShenandoahBrooksPointer::word_size();
+    HeapWord* obj = r->bottom();
     if (_heap->complete_marking_context()->is_marked((oop)obj)) {
       verify_and_follow(obj, stack, cl, &processed);
     }
@@ -543,12 +543,11 @@
 
     // Bitmaps, before TAMS
     if (tams > r->bottom()) {
-      HeapWord* start = r->bottom() + ShenandoahBrooksPointer::word_size();
+      HeapWord* start = r->bottom();
       HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
 
       while (addr < tams) {
         verify_and_follow(addr, stack, cl, &processed);
-        addr += ShenandoahBrooksPointer::word_size();
         if (addr < tams) {
           addr = mark_bit_map->get_next_marked_addr(addr, tams);
         }
@@ -558,11 +557,11 @@
     // Size-based, after TAMS
     {
       HeapWord* limit = r->top();
-      HeapWord* addr = tams + ShenandoahBrooksPointer::word_size();
+      HeapWord* addr = tams;
 
       while (addr < limit) {
         verify_and_follow(addr, stack, cl, &processed);
-        addr += oop(addr)->size() + ShenandoahBrooksPointer::word_size();
+        addr += oop(addr)->size();
       }
     }
 
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/oops/oop.hpp
--- a/src/hotspot/share/oops/oop.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/oops/oop.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -80,6 +80,7 @@
   inline void init_mark_raw();
 
   inline Klass* klass() const;
+  inline Klass* klass_raw() const;
   inline Klass* klass_or_null() const volatile;
   inline Klass* klass_or_null_acquire() const volatile;
   static inline Klass** klass_addr(HeapWord* mem);
@@ -125,6 +126,7 @@
 
  protected:
   inline oop        as_oop() const { return const_cast<oopDesc*>(this); }
+  inline oop        as_oop() const volatile { return const_cast<oopDesc*>(this); }
 
  public:
   // field addresses in oop
diff -r b25269d2e08d -r 49aa454b8e7c src/hotspot/share/oops/oop.inline.hpp
--- a/src/hotspot/share/oops/oop.inline.hpp	Thu Dec 20 22:55:52 2018 +0100
+++ b/src/hotspot/share/oops/oop.inline.hpp	Tue Dec 25 00:11:08 2018 +0100
@@ -90,6 +90,25 @@
   if (UseCompressedClassPointers) {
     return Klass::decode_klass_not_null(_metadata._compressed_klass);
   } else {
+    if (UseShenandoahGC) {
+      Klass* kl = _metadata._klass;
+      intptr_t klptr = reinterpret_cast<intptr_t>(kl);
+      if ((klptr & 1) == 0) {
+        return kl;
+      } else {
+        oop fwd = oop(reinterpret_cast<HeapWord*>(klptr & ~1));
+        return fwd->_metadata._klass;
+      }
+    } else {
+      return _metadata._klass;
+    }
+  }
+}
+
+Klass* oopDesc::klass_raw() const {
+  if (UseCompressedClassPointers) {
+    return Klass::decode_klass_not_null(_metadata._compressed_klass);
+  } else {
     return _metadata._klass;
   }
 }
@@ -98,7 +117,18 @@
   if (UseCompressedClassPointers) {
     return Klass::decode_klass(_metadata._compressed_klass);
   } else {
-    return _metadata._klass;
+    if (UseShenandoahGC) {
+      Klass* kl = _metadata._klass;
+      intptr_t klptr = reinterpret_cast<intptr_t>(kl);
+      if ((klptr & 1) == 0) {
+        return kl;
+      } else {
+        oop fwd = oop(reinterpret_cast<HeapWord*>(klptr & ~1));
+        return fwd->_metadata._klass;
+      }
+    } else {
+      return _metadata._klass;
+    }
   }
 }