# HG changeset patch
# User rkennke
# Date 1568832980 -7200
#      Wed Sep 18 20:56:20 2019 +0200
# Node ID b16b63a399f38d657f5f9a545270e74e3f3d3fa3
# Parent a70feeebb9b0dee40f653caa08ff2c904b06470c
[backport] 8231087: Shenandoah: Self-fixing load reference barriers for C1/C2
Reviewed-by: shade

diff -r a70feeebb9b0 src/cpu/aarch64/vm/c1_LIRGenerator_aarch64.cpp
--- a/src/cpu/aarch64/vm/c1_LIRGenerator_aarch64.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/cpu/aarch64/vm/c1_LIRGenerator_aarch64.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -1444,7 +1444,7 @@
     __ xchg(LIR_OprFact::address(addr), data, dst, tmp);
 #if INCLUDE_ALL_GCS
     if (UseShenandoahGC && is_obj) {
-      LIR_Opr tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, dst);
+      LIR_Opr tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, dst, LIR_OprFact::addressConst(0));
       __ move(tmp, dst);
     }
 #endif
diff -r a70feeebb9b0 src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp
--- a/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -1307,7 +1307,12 @@
         
         __ push_call_clobbered_registers();
         f.load_argument(0, r0);
-        __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
+        f.load_argument(1, r1);
+        if (UseCompressedOops) {
+          __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow));
+        } else {
+          __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup));
+        }
         __ blr(lr);
         __ mov(rscratch1, r0);
         __ pop_call_clobbered_registers();
diff -r a70feeebb9b0 src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp
--- a/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -245,6 +245,7 @@
 
   Register obj = stub->obj()->as_register();
   Register res = stub->result()->as_register();
+  Register addr = stub->addr()->as_register_lo();
   Register tmp1 = stub->tmp1()->as_register();
   Register tmp2 = stub->tmp2()->as_register();
 
@@ -277,6 +278,7 @@
 
   __ bind(slow_path);
   ce->store_parameter(res, 0);
+  ce->store_parameter(addr, 1);
   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_lrb_slow_id)));
 
   __ b(*stub->continuation());
diff -r a70feeebb9b0 src/cpu/x86/vm/c1_LIRGenerator_x86.cpp
--- a/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -1503,7 +1503,7 @@
 
 #if INCLUDE_ALL_GCS
     if (UseShenandoahGC && is_obj) {
-      LIR_Opr tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, dst);
+      LIR_Opr tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, dst, LIR_OprFact::addressConst(0));
       __ move(tmp, dst);
     }
 #endif
diff -r a70feeebb9b0 src/cpu/x86/vm/c1_Runtime1_x86.cpp
--- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -1785,8 +1785,19 @@
         // arg0 : object to be resolved
         
         save_live_registers(sasm, 1);
-        f.load_argument(0, LP64_ONLY(c_rarg0) NOT_LP64(rax));
-        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), LP64_ONLY(c_rarg0) NOT_LP64(rax));
+#ifdef _LP64
+        f.load_argument(0, c_rarg0);
+        f.load_argument(1, c_rarg1);
+        if (UseCompressedOops) {
+          __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow), c_rarg0, c_rarg1);
+        } else {
+          __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup), c_rarg0, c_rarg1);
+        }
+#else
+        f.load_argument(0, rax);
+        f.load_argument(1, rbx);
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup), rax, rbx);
+#endif
         restore_live_registers_except_rax(sasm, true);
 
       }
diff -r a70feeebb9b0 src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp
--- a/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -412,8 +412,10 @@
   Label done;
   Register obj = stub->obj()->as_register();
   Register res = stub->result()->as_register();
+  Register addr = stub->addr()->as_register();
   Register tmp1 = stub->tmp1()->as_register();
   Register tmp2 = stub->tmp2()->as_register();
+  assert_different_registers(obj, res, addr, tmp1, tmp2);
 
   Label slow_path;
 
@@ -442,31 +444,10 @@
 #endif
   __ jcc(Assembler::zero, *stub->continuation());
 
-  // Test if object is resolved.
-  __ movptr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
-  // Test if both lowest bits are set. We trick it by negating the bits
-  // then test for both bits clear.
-  __ notptr(tmp1);
-#ifdef _LP64
-  __ testb(tmp1, markOopDesc::marked_value);
-#else
-  // On x86_32, C1 register allocator can give us the register without 8-bit support.
-  // Do the full-register access and test to avoid compilation failures.
-  __ testptr(tmp1, markOopDesc::marked_value);
-#endif
-
-  __ jccb(Assembler::notZero, slow_path);
-  // Clear both lower bits. It's still inverted, so set them, and then invert back.
-  __ orptr(tmp1, markOopDesc::marked_value);
-  __ notptr(tmp1);
-  // At this point, tmp1 contains the decoded forwarding pointer.
-  __ mov(res, tmp1);
-
-  __ jmp(*stub->continuation());
-
   __ bind(slow_path);
   ce->store_parameter(res, 0);
- __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_lrb_slow_id)));
+  ce->store_parameter(addr, 1);
+  __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_lrb_slow_id)));
 
   __ jmp(*stub->continuation());
 }
diff -r a70feeebb9b0 src/share/vm/c1/c1_LIRGenerator.cpp
--- a/src/share/vm/c1/c1_LIRGenerator.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/c1/c1_LIRGenerator.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -1234,8 +1234,9 @@
 #if INCLUDE_ALL_GCS
   if (UseShenandoahGC) {
     LIR_Opr tmp = new_register(T_OBJECT);
-    __ load(referent_field_adr, tmp, info);
-    tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, tmp);
+    LIR_Opr addr = ShenandoahBarrierSet::barrier_set()->bsc1()->resolve_address(this, referent_field_adr, T_OBJECT, NULL);
+    __ load(addr->as_address_ptr(), tmp, info);
+    tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, tmp, addr);
     __ move(tmp, result);
   } else
 #endif
@@ -1831,16 +1832,16 @@
 #if INCLUDE_ALL_GCS
   if (UseShenandoahGC && (field_type == T_OBJECT || field_type == T_ARRAY)) {
     LIR_Opr tmp = new_register(T_OBJECT);
-    if (is_volatile && !needs_patching) {
-      volatile_field_load(address, tmp, info);
+    LIR_Opr addr = ShenandoahBarrierSet::barrier_set()->bsc1()->resolve_address(this, address, field_type, needs_patching ? info : NULL);
+    if (is_volatile) {
+      volatile_field_load(addr->as_address_ptr(), tmp, info);
     } else {
-      LIR_PatchCode patch_code = needs_patching ? lir_patch_normal : lir_patch_none;
-      __ load(address, tmp, info, patch_code);
+      __ load(addr->as_address_ptr(), tmp, info);
     }
     if (is_volatile && os::is_MP()) {
       __ membar_acquire();
     }
-    tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, tmp);
+    tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, tmp, addr);
     __ move(tmp, reg);
   } else
 #endif
@@ -1976,8 +1977,9 @@
 #if INCLUDE_ALL_GCS
   if (UseShenandoahGC && (x->elt_type() == T_OBJECT || x->elt_type() == T_ARRAY)) {
     LIR_Opr tmp = new_register(T_OBJECT);
-    __ move(array_addr, tmp, null_check_info);
-    tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, tmp);
+    LIR_Opr addr = ShenandoahBarrierSet::barrier_set()->bsc1()->resolve_address(this, array_addr, x->elt_type(), NULL);
+    __ move(addr->as_address_ptr(), tmp, null_check_info);
+    tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, tmp, addr);
     __ move(tmp, result);
   } else
 #endif
@@ -2275,7 +2277,7 @@
   if (UseShenandoahGC && (type == T_OBJECT || type == T_ARRAY)) {
     LIR_Opr tmp = new_register(T_OBJECT);
     get_Object_unsafe(tmp, src.result(), off.result(), type, x->is_volatile());
-    tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, tmp);
+    tmp = ShenandoahBarrierSet::barrier_set()->bsc1()->load_reference_barrier(this, tmp, LIR_OprFact::addressConst(0));
     __ move(tmp, value);
   } else
 #endif
diff -r a70feeebb9b0 src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.cpp
--- a/src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -52,18 +52,20 @@
   return ShenandoahBarrierSet::barrier_set()->bsc1();
 }
 
-LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj) {
+LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr) {
   if (ShenandoahLoadRefBarrier) {
-    return load_reference_barrier_impl(gen, obj);
+    return load_reference_barrier_impl(gen, obj, addr);
   } else {
     return obj;
   }
 }
 
-LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj) {
+LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr) {
   assert(ShenandoahLoadRefBarrier, "Should be enabled");
   obj = ensure_in_register(gen, obj);
   assert(obj->is_register(), "must be a register at this point");
+  addr = ensure_in_register(gen, addr);
+  assert(addr->is_register(), "must be a register at this point");
   LIR_Opr result = gen->result_register_for(obj->value_type());
   __ move(obj, result);
   LIR_Opr tmp1 = gen->new_register(T_OBJECT);
@@ -91,7 +93,7 @@
   }
   __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
 
-  CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, result, tmp1, tmp2);
+  CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2);
   __ branch(lir_cond_notEqual, T_INT, slow);
   __ branch_destination(slow->continuation());
 
@@ -100,10 +102,18 @@
 
 LIR_Opr ShenandoahBarrierSetC1::ensure_in_register(LIRGenerator* gen, LIR_Opr obj) {
   if (!obj->is_register()) {
-    LIR_Opr obj_reg = gen->new_register(T_OBJECT);
+    LIR_Opr obj_reg;
     if (obj->is_constant()) {
+      obj_reg = gen->new_register(T_OBJECT);
       __ move(obj, obj_reg);
     } else {
+#ifdef AARCH64
+      // AArch64 expects double-size register.
+      obj_reg = gen->new_pointer_register();
+#else
+      // x86 expects single-size register.
+      obj_reg = gen->new_register(T_OBJECT);
+#endif
       __ leal(obj, obj_reg);
     }
     obj = obj_reg;
@@ -118,3 +128,15 @@
   }
   return obj;
 }
+
+LIR_Opr ShenandoahBarrierSetC1::resolve_address(LIRGenerator* gen, LIR_Address* addr, BasicType type, CodeEmitInfo* patch_emit_info) {
+  LIR_Opr addr_opr = LIR_OprFact::address(addr);
+
+  LIR_Opr resolved_addr = gen->new_pointer_register();
+  if (patch_emit_info != NULL) {
+    __ leal(addr_opr, resolved_addr, lir_patch_normal, new CodeEmitInfo(patch_emit_info));
+  } else {
+    __ leal(addr_opr, resolved_addr);
+  }
+  return LIR_OprFact::address(new LIR_Address(resolved_addr, type));
+}
diff -r a70feeebb9b0 src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.hpp
--- a/src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.hpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.hpp	Wed Nov 25 13:06:29 2020 -0500
@@ -28,26 +28,30 @@
 #include "memory/allocation.hpp"
 
 class LIRGenerator;
+class LIRItem;
 
 class ShenandoahLoadReferenceBarrierStub: public CodeStub {
   friend class ShenandoahBarrierSetC1;
 private:
   LIR_Opr _obj;
+  LIR_Opr _addr;
   LIR_Opr _result;
   LIR_Opr _tmp1;
   LIR_Opr _tmp2;
 
 public:
-  ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2) :
-    _obj(obj), _result(result), _tmp1(tmp1), _tmp2(tmp2)
+  ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr addr, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2) :
+    _obj(obj), _addr(addr), _result(result), _tmp1(tmp1), _tmp2(tmp2)
   {
     assert(_obj->is_register(), "should be register");
+    assert(_addr->is_register(), "should be register");
     assert(_result->is_register(), "should be register");
     assert(_tmp1->is_register(), "should be register");
     assert(_tmp2->is_register(), "should be register");
   }
 
   LIR_Opr obj() const { return _obj; }
+  LIR_Opr addr() const { return _addr; }
   LIR_Opr result() const { return _result; }
   LIR_Opr tmp1() const { return _tmp1; }
   LIR_Opr tmp2() const { return _tmp2; }
@@ -56,6 +60,9 @@
   virtual void visit(LIR_OpVisitState* visitor) {
     visitor->do_slow_case();
     visitor->do_input(_obj);
+    visitor->do_temp(_obj);
+    visitor->do_input(_addr);
+    visitor->do_temp(_addr);
     visitor->do_temp(_result);
     visitor->do_temp(_tmp1);
     visitor->do_temp(_tmp2);
@@ -71,12 +78,14 @@
 public:
   static ShenandoahBarrierSetC1* bsc1();
 
-  LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj);
+  LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr);
   LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool patch);
+
+  LIR_Opr resolve_address(LIRGenerator* gen, LIR_Address* addr, BasicType type, CodeEmitInfo* patch_emit_info);
+
 private:
-  LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj);
+  LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr);
   LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj);
-
 };
 
 #endif // SHARE_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP
diff -r a70feeebb9b0 src/share/vm/gc_implementation/shenandoah/c2/shenandoahBarrierSetC2.cpp
--- a/src/share/vm/gc_implementation/shenandoah/c2/shenandoahBarrierSetC2.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/gc_implementation/shenandoah/c2/shenandoahBarrierSetC2.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -47,9 +47,11 @@
 }
 
 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type() {
-  const Type **fields = TypeTuple::fields(1);
+  const Type **fields = TypeTuple::fields(2);
   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
-  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
+  fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM;   // original load address
+
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
 
   // create result type (range)
   fields = TypeTuple::fields(1);
diff -r a70feeebb9b0 src/share/vm/gc_implementation/shenandoah/c2/shenandoahSupport.cpp
--- a/src/share/vm/gc_implementation/shenandoah/c2/shenandoahSupport.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/gc_implementation/shenandoah/c2/shenandoahSupport.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -1,3 +1,4 @@
+
 /*
  * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved.
  *
@@ -998,7 +999,7 @@
   phase->register_new_node(cset_bool,      old_ctrl);
 }
 
-void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) {
+void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) {
   IdealLoopTree*loop = phase->get_loop(ctrl);
   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr();
 
@@ -1009,8 +1010,12 @@
   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
   phase->register_new_node(mm, ctrl);
 
+  address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
+          CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow) :
+          CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup);
+
   Node* call = new (phase->C) CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(),
-                                           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier),
+                                           target,
                                            "shenandoah_load_reference_barrier", TypeRawPtr::BOTTOM);
   call->init_req(TypeFunc::Control, ctrl);
   call->init_req(TypeFunc::I_O, phase->C->top());
@@ -1018,6 +1023,7 @@
   call->init_req(TypeFunc::FramePtr, phase->C->top());
   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
   call->init_req(TypeFunc::Parms, val);
+  call->init_req(TypeFunc::Parms+1, load_addr);
   phase->register_control(call, loop, ctrl);
   ctrl = new (phase->C) ProjNode(call, TypeFunc::Control);
   phase->register_control(ctrl, loop, call);
@@ -1381,7 +1387,7 @@
     assert(val->bottom_type()->make_oopptr(), "need oop");
     assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
 
-    enum { _heap_stable = 1, _not_cset, _fwded, _evac_path, _null_path, PATH_LIMIT };
+    enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT };
     Node* region = new (phase->C) RegionNode(PATH_LIMIT);
     Node* val_phi = new (phase->C) PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
@@ -1431,49 +1437,44 @@
       IfNode* iff = unc_ctrl->in(0)->as_If();
       phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
     }
-    Node* addr = new (phase->C) AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(oopDesc::mark_offset_in_bytes()));
-    phase->register_new_node(addr, ctrl);
-    assert(new_val->bottom_type()->isa_oopptr(), "what else?");
-    Node* markword = new (phase->C) LoadXNode(ctrl, raw_mem, addr, TypeRawPtr::BOTTOM, TypeX_X, MemNode::unordered);
-    phase->register_new_node(markword, ctrl);
-
-    // Test if object is forwarded. This is the case if lowest two bits are set.
-    Node* masked = new (phase->C) AndXNode(markword, phase->igvn().MakeConX(markOopDesc::lock_mask_in_place));
-    phase->register_new_node(masked, ctrl);
-    Node* cmp = new (phase->C) CmpXNode(masked, phase->igvn().MakeConX(markOopDesc::marked_value));
-    phase->register_new_node(cmp, ctrl);
-
-    // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
-    Node* bol = new (phase->C) BoolNode(cmp, BoolTest::eq); // Equals 3 means it's forwarded
-    phase->register_new_node(bol, ctrl);
-
-    IfNode* iff = new (phase->C) IfNode(ctrl, bol, PROB_LIKELY(0.999), COUNT_UNKNOWN);
-    phase->register_control(iff, loop, ctrl);
-    Node* if_fwd = new (phase->C) IfTrueNode(iff);
-    phase->register_control(if_fwd, loop, iff);
-    Node* if_not_fwd = new (phase->C) IfFalseNode(iff);
-    phase->register_control(if_not_fwd, loop, iff);
-
-    // Decode forward pointer: since we already have the lowest bits, we can just subtract them
-    // from the mark word without the need for large immediate mask.
-    Node* masked2 = new (phase->C) SubXNode(markword, masked);
-    phase->register_new_node(masked2, if_fwd);
-    Node* fwdraw = new (phase->C) CastX2PNode(masked2);
-    fwdraw->init_req(0, if_fwd);
-    phase->register_new_node(fwdraw, if_fwd);
-    Node* fwd = new (phase->C) CheckCastPPNode(NULL, fwdraw, val->bottom_type());
-    phase->register_new_node(fwd, if_fwd);
-
-    // Wire up not-equal-path in slots 3.
-    region->init_req(_fwded, if_fwd);
-    val_phi->init_req(_fwded, fwd);
-    raw_mem_phi->init_req(_fwded, raw_mem);
 
     // Call lrb-stub and wire up that path in slots 4
     Node* result_mem = NULL;
-    ctrl = if_not_fwd;
-    fwd = new_val;
-    call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase);
+
+    Node* fwd = new_val;
+    Node* addr;
+    if (ShenandoahSelfFixing) {
+      VectorSet visited(Thread::current()->resource_area());
+      addr = get_load_addr(phase, visited, lrb);
+    } else {
+      addr = phase->igvn().zerocon(T_OBJECT);
+    }
+    if (addr->Opcode() == Op_AddP) {
+      Node* orig_base = addr->in(AddPNode::Base);
+      Node* base = new (phase->C) CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type());
+      phase->register_new_node(base, ctrl);
+      if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) {
+        // Field access
+        addr = addr->clone();
+        addr->set_req(AddPNode::Base, base);
+        addr->set_req(AddPNode::Address, base);
+        phase->register_new_node(addr, ctrl);
+      } else {
+        Node* addr2 = addr->in(AddPNode::Address);
+        if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) &&
+              addr2->in(AddPNode::Base) == orig_base) {
+          addr2 = addr2->clone();
+          addr2->set_req(AddPNode::Base, base);
+          addr2->set_req(AddPNode::Address, base);
+          phase->register_new_node(addr2, ctrl);
+          addr = addr->clone();
+          addr->set_req(AddPNode::Base, base);
+          addr->set_req(AddPNode::Address, addr2);
+          phase->register_new_node(addr, ctrl);
+        }
+      }
+    }
+    call_lrb_stub(ctrl, fwd, addr, result_mem, raw_mem, false, phase);
     region->init_req(_evac_path, ctrl);
     val_phi->init_req(_evac_path, fwd);
     raw_mem_phi->init_req(_evac_path, result_mem);
@@ -1526,6 +1527,68 @@
 
 }
 
+Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
+  if (visited.test_set(in->_idx)) {
+    return NULL;
+  }
+  switch (in->Opcode()) {
+    case Op_Proj:
+      return get_load_addr(phase, visited, in->in(0));
+    case Op_CastPP:
+    case Op_CheckCastPP:
+    case Op_DecodeN:
+    case Op_EncodeP:
+      return get_load_addr(phase, visited, in->in(1));
+    case Op_LoadN:
+    case Op_LoadP:
+      return in->in(MemNode::Address);
+    case Op_GetAndSetN:
+    case Op_GetAndSetP:
+      // Those instructions would just have stored a different
+      // value into the field. No use to attempt to fix it at this point.
+      return phase->igvn().zerocon(T_OBJECT);
+    case Op_CMoveP:
+    case Op_CMoveN: {
+      Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue));
+      Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse));
+      // Handle unambiguous cases: single address reported on both branches.
+      if (t != NULL && f == NULL) return t;
+      if (t == NULL && f != NULL) return f;
+      if (t != NULL && t == f)    return t;
+      // Ambiguity.
+      return phase->igvn().zerocon(T_OBJECT);
+    }
+    case Op_Phi: {
+      Node* addr = NULL;
+      for (uint i = 1; i < in->req(); i++) {
+        Node* addr1 = get_load_addr(phase, visited, in->in(i));
+        if (addr == NULL) {
+          addr = addr1;
+        }
+        if (addr != addr1) {
+          return phase->igvn().zerocon(T_OBJECT);
+        }
+      }
+      return addr;
+    }
+    case Op_ShenandoahLoadReferenceBarrier:
+      return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
+    case Op_CallDynamicJava:
+    case Op_CallLeaf:
+    case Op_CallStaticJava:
+    case Op_ConN:
+    case Op_ConP:
+    case Op_Parm:
+      return phase->igvn().zerocon(T_OBJECT);
+    default:
+#ifdef ASSERT
+      fatal(err_msg("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]));
+#endif
+      return phase->igvn().zerocon(T_OBJECT);
+  }
+
+}
+
 void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
   IdealLoopTree *loop = phase->get_loop(iff);
   Node* loop_head = loop->_head;
diff -r a70feeebb9b0 src/share/vm/gc_implementation/shenandoah/c2/shenandoahSupport.hpp
--- a/src/share/vm/gc_implementation/shenandoah/c2/shenandoahSupport.hpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/gc_implementation/shenandoah/c2/shenandoahSupport.hpp	Wed Nov 25 13:06:29 2020 -0500
@@ -63,7 +63,7 @@
   static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase);
   static void test_gc_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
                             PhaseIdealLoop* phase, int flags);
-  static void call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase);
+  static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase);
   static Node* clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase);
   static void fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses,
                              PhaseIdealLoop* phase);
@@ -74,6 +74,7 @@
   static void fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase);
   static IfNode* find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase);
 
+  static Node* get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* lrb);
 public:
   static bool is_dominator(Node* d_c, Node* n_c, Node* d, Node* n, PhaseIdealLoop* phase);
   static bool is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase);
diff -r a70feeebb9b0 src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.hpp	Wed Nov 25 13:06:29 2020 -0500
@@ -110,7 +110,11 @@
 
   oop load_reference_barrier(oop obj);
   oop load_reference_barrier_not_null(oop obj);
-  inline oop load_reference_barrier_mutator(oop obj);
+  inline oop load_reference_barrier_mutator(oop obj, oop* load_addr);
+  inline oop load_reference_barrier_mutator(oop obj, narrowOop* load_addr);
+
+  template <class T>
+  inline oop load_reference_barrier_mutator_work(oop obj, T* load_addr);
 
   oop oop_atomic_cmpxchg_in_heap(oop new_value, volatile HeapWord* dest, oop compare_value);
 
diff -r a70feeebb9b0 src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp	Wed Nov 25 13:06:29 2020 -0500
@@ -48,9 +48,18 @@
   return ShenandoahForwarding::get_forwardee_mutator(p);
 }
 
-inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj) {
+inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, oop* load_addr) {
+  return load_reference_barrier_mutator_work(obj, load_addr);
+}
+
+inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, narrowOop* load_addr) {
+  return load_reference_barrier_mutator_work(obj, load_addr);
+}
+
+template <class T>
+oop ShenandoahBarrierSet::load_reference_barrier_mutator_work(oop obj, T* load_addr) {
   assert(ShenandoahLoadRefBarrier, "should be enabled");
-  shenandoah_assert_in_cset(NULL, obj);
+  shenandoah_assert_in_cset(load_addr, obj);
 
   oop fwd = resolve_forwarded_not_null_mutator(obj);
   if (obj == fwd) {
@@ -60,6 +69,11 @@
     fwd = _heap->evacuate_object(obj, Thread::current());
   }
 
+  if (load_addr != NULL && fwd != obj) {
+    // Since we are here and we know the load address, update the reference.
+    ShenandoahHeap::cas_oop(fwd, load_addr, obj);
+  }
+
   return fwd;
 }
 
diff -r a70feeebb9b0 src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.cpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.cpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.cpp	Wed Nov 25 13:06:29 2020 -0500
@@ -48,8 +48,15 @@
 JRT_END
 
 JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier(oopDesc* src))
-  oop result = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(oop(src));
-  return (oopDesc*) result;
+  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, (oop*)NULL);
+JRT_END
+
+JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_fixup(oopDesc* src, oop* load_addr))
+  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
+JRT_END
+
+JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_fixup_narrow(oopDesc* src, narrowOop* load_addr))
+  return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
 JRT_END
 
 IRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_interpreter(oopDesc* src))
diff -r a70feeebb9b0 src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.hpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.hpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoahRuntime.hpp	Wed Nov 25 13:06:29 2020 -0500
@@ -39,6 +39,8 @@
   static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread);
 
   static oopDesc* load_reference_barrier(oopDesc* src);
+  static oopDesc* load_reference_barrier_fixup(oopDesc* src, oop* load_addr);
+  static oopDesc* load_reference_barrier_fixup_narrow(oopDesc* src, narrowOop* load_addr);
   static oopDesc* load_reference_barrier_interpreter(oopDesc* src);
 
   static void shenandoah_clone_barrier(oopDesc* src);
diff -r a70feeebb9b0 src/share/vm/gc_implementation/shenandoah/shenandoah_globals.hpp
--- a/src/share/vm/gc_implementation/shenandoah/shenandoah_globals.hpp	Wed Nov 25 13:06:11 2020 -0500
+++ b/src/share/vm/gc_implementation/shenandoah/shenandoah_globals.hpp	Wed Nov 25 13:06:29 2020 -0500
@@ -345,6 +345,10 @@
   product(bool, ShenandoahLogWarning, false,                                \
           "Turns on logging in Shenandoah at warning level. ")              \
                                                                             \
+  diagnostic(bool, ShenandoahSelfFixing, true,                              \
+          "Fix references with load reference barrier. Disabling this "     \
+          "might degrade performance.")                                     \
+
 
 SHENANDOAH_FLAGS(DECLARE_DEVELOPER_FLAG, \
                  DECLARE_PD_DEVELOPER_FLAG,     \