# HG changeset patch
# Parent a8c1af3603ee1988f1bdc86183e32173ac7bdc35
[backport] 8228369: Shenandoah: Refactor LRB C1 stubs

diff -r a8c1af3603ee src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp
--- a/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp	Mon May 14 15:42:59 2018 +0200
+++ b/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp	Wed Nov 25 11:23:04 2020 -0500
@@ -45,6 +45,7 @@
 #if INCLUDE_ALL_GCS
 #include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
+#include "gc_implementation/shenandoah/shenandoahRuntime.hpp"
 #endif
 
 
@@ -1299,6 +1300,20 @@
 
       }
       break;
+    case shenandoah_lrb_slow_id:
+      {
+        StubFrame f(sasm, "shenandoah_load_reference_barrier", dont_gc_arguments);
+        // arg0 : object to be resolved
+        
+        __ push_call_clobbered_registers();
+        f.load_argument(0, r0);
+        __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
+        __ blr(lr);
+        __ mov(rscratch1, r0);
+        __ pop_call_clobbered_registers();
+        __ mov(r0, rscratch1);
+      }
+      break;
 #endif
 
     case predicate_failed_trap_id:
diff -r a8c1af3603ee src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp
--- a/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp	Mon May 14 15:42:59 2018 +0200
+++ b/src/cpu/aarch64/vm/shenandoahBarrierSetAssembler_aarch64.cpp	Wed Nov 25 11:23:04 2020 -0500
@@ -29,6 +29,7 @@
 #include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
 #include "gc_implementation/shenandoah/shenandoahHeap.hpp"
+#include "gc_implementation/shenandoah/shenandoahHeapRegion.hpp"
 #include "gc_implementation/shenandoah/shenandoahRuntime.hpp"
 #include "gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.hpp"
 #include "runtime/stubCodeGenerator.hpp"
@@ -240,23 +241,44 @@
 #define __ ce->masm()->
 
 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
+  __ bind(*stub->entry());
 
   Register obj = stub->obj()->as_register();
   Register res = stub->result()->as_register();
+  Register tmp1 = stub->tmp1()->as_register();
+  Register tmp2 = stub->tmp2()->as_register();
 
-  Label done;
-
-  __ bind(*stub->entry());
+  assert(res == r0, "result must arrive in r0");
 
   if (res != obj) {
     __ mov(res, obj);
   }
+
   // Check for null.
-  __ cbz(res, done);
+  __ cbz(res, *stub->continuation());
 
-  load_reference_barrier_not_null(ce->masm(), res);
+  // Check for object in cset.
+  __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
+  __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ ldrb(tmp2, Address(tmp2, tmp1));
+  __ cbz(tmp2, *stub->continuation());
 
-  __ bind(done);
+  // Check if object is already forwarded.
+  Label slow_path;
+  __ ldr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
+  __ eon(tmp1, tmp1, zr);
+  __ ands(zr, tmp1, markOopDesc::lock_mask_in_place);
+  __ br(Assembler::NE, slow_path);
+
+  // Decode forwarded object.
+  __ orr(tmp1, tmp1, markOopDesc::marked_value);
+  __ eon(res, tmp1, zr);
+  __ b(*stub->continuation());
+
+  __ bind(slow_path);
+  ce->store_parameter(res, 0);
+  __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_lrb_slow_id)));
+
   __ b(*stub->continuation());
 }
 
diff -r a8c1af3603ee src/cpu/x86/vm/c1_Runtime1_x86.cpp
--- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp	Mon May 14 15:42:59 2018 +0200
+++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp	Wed Nov 25 11:23:04 2020 -0500
@@ -40,6 +40,7 @@
 #include "vmreg_x86.inline.hpp"
 #if INCLUDE_ALL_GCS
 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
+#include "gc_implementation/shenandoah/shenandoahRuntime.hpp"
 #endif
 
 
@@ -1778,6 +1779,18 @@
 
       }
       break;
+    case shenandoah_lrb_slow_id:
+      {
+        StubFrame f(sasm, "shenandoah_load_reference_barrier", dont_gc_arguments);
+        // arg0 : object to be resolved
+        
+        save_live_registers(sasm, 1);
+        f.load_argument(0, c_rarg0);
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0);
+        restore_live_registers_except_rax(sasm, true);
+
+      }
+      break;
 #endif // INCLUDE_ALL_GCS
 
     case predicate_failed_trap_id:
diff -r a8c1af3603ee src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp
--- a/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp	Mon May 14 15:42:59 2018 +0200
+++ b/src/cpu/x86/vm/shenandoahBarrierSetAssembler_x86.cpp	Wed Nov 25 11:23:04 2020 -0500
@@ -412,6 +412,12 @@
   Label done;
   Register obj = stub->obj()->as_register();
   Register res = stub->result()->as_register();
+  Register tmp1 = stub->tmp1()->as_register();
+  Register tmp2 = stub->tmp2()->as_register();
+
+  Label slow_path;
+
+  assert(res == rax, "result must arrive in rax");
 
   if (res != obj) {
     __ mov(res, obj);
@@ -419,11 +425,35 @@
 
   // Check for null.
   __ testptr(res, res);
-  __ jcc(Assembler::zero, done);
+  __ jcc(Assembler::zero, *stub->continuation());
 
-  load_reference_barrier_not_null(ce->masm(), res);
+  // Check for object being in the collection set.
+  __ mov(tmp1, res);
+  __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+  __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
+  __ testbool(tmp2);
+  __ jcc(Assembler::zero, *stub->continuation());
 
-  __ bind(done);
+  // Test if object is resolved.
+  __ movptr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
+  // Test if both lowest bits are set. We trick it by negating the bits
+  // then test for both bits clear.
+  __ notptr(tmp1);
+  __ testb(tmp1, markOopDesc::marked_value);
+  __ jccb(Assembler::notZero, slow_path);
+  // Clear both lower bits. It's still inverted, so set them, and then invert back.
+  __ orptr(tmp1, markOopDesc::marked_value);
+  __ notptr(tmp1);
+  // At this point, tmp1 contains the decoded forwarding pointer.
+  __ mov(res, tmp1);
+
+  __ jmp(*stub->continuation());
+
+  __ bind(slow_path);
+  ce->store_parameter(res, 0);
+ __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_lrb_slow_id)));
+
   __ jmp(*stub->continuation());
 }
 
diff -r a8c1af3603ee src/share/vm/c1/c1_Runtime1.cpp
--- a/src/share/vm/c1/c1_Runtime1.cpp	Mon May 14 15:42:59 2018 +0200
+++ b/src/share/vm/c1/c1_Runtime1.cpp	Wed Nov 25 11:23:04 2020 -0500
@@ -202,6 +202,7 @@
     case dtrace_object_alloc_id:
     case g1_pre_barrier_slow_id:
     case g1_post_barrier_slow_id:
+    case shenandoah_lrb_slow_id:
     case slow_subtype_check_id:
     case fpu2long_stub_id:
     case unwind_exception_id:
diff -r a8c1af3603ee src/share/vm/c1/c1_Runtime1.hpp
--- a/src/share/vm/c1/c1_Runtime1.hpp	Mon May 14 15:42:59 2018 +0200
+++ b/src/share/vm/c1/c1_Runtime1.hpp	Wed Nov 25 11:23:04 2020 -0500
@@ -70,6 +70,7 @@
   stub(load_appendix_patching)       \
   stub(g1_pre_barrier_slow)          \
   stub(g1_post_barrier_slow)         \
+  stub(shenandoah_lrb_slow)          \
   stub(fpu2long_stub)                \
   stub(counter_overflow)             \
   stub(predicate_failed_trap)        \
diff -r a8c1af3603ee src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.cpp
--- a/src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.cpp	Mon May 14 15:42:59 2018 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.cpp	Wed Nov 25 11:23:04 2020 -0500
@@ -64,8 +64,10 @@
   assert(ShenandoahLoadRefBarrier, "Should be enabled");
   obj = ensure_in_register(gen, obj);
   assert(obj->is_register(), "must be a register at this point");
-  LIR_Opr result = gen->new_register(T_OBJECT);
+  LIR_Opr result = gen->result_register_for(obj->value_type());
   __ move(obj, result);
+  LIR_Opr tmp1 = gen->new_register(T_OBJECT);
+  LIR_Opr tmp2 = gen->new_register(T_OBJECT);
 
   LIR_Opr thrd = gen->getThreadPointer();
   LIR_Address* active_flag_addr =
@@ -89,7 +91,7 @@
   }
   __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
 
-  CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, result);
+  CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, result, tmp1, tmp2);
   __ branch(lir_cond_notEqual, T_INT, slow);
   __ branch_destination(slow->continuation());
 
diff -r a8c1af3603ee src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.hpp
--- a/src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.hpp	Mon May 14 15:42:59 2018 +0200
+++ b/src/share/vm/gc_implementation/shenandoah/c1/shenandoahBarrierSetC1.hpp	Wed Nov 25 11:23:04 2020 -0500
@@ -34,23 +34,31 @@
 private:
   LIR_Opr _obj;
   LIR_Opr _result;
+  LIR_Opr _tmp1;
+  LIR_Opr _tmp2;
 
 public:
-  ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr result) :
-    _obj(obj), _result(result)
+  ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2) :
+    _obj(obj), _result(result), _tmp1(tmp1), _tmp2(tmp2)
   {
     assert(_obj->is_register(), "should be register");
     assert(_result->is_register(), "should be register");
+    assert(_tmp1->is_register(), "should be register");
+    assert(_tmp2->is_register(), "should be register");
   }
 
   LIR_Opr obj() const { return _obj; }
   LIR_Opr result() const { return _result; }
+  LIR_Opr tmp1() const { return _tmp1; }
+  LIR_Opr tmp2() const { return _tmp2; }
 
   virtual void emit_code(LIR_Assembler* e);
   virtual void visit(LIR_OpVisitState* visitor) {
     visitor->do_slow_case();
     visitor->do_input(_obj);
     visitor->do_temp(_result);
+    visitor->do_temp(_tmp1);
+    visitor->do_temp(_tmp2);
   }
 #ifndef PRODUCT
   virtual void print_name(outputStream* out) const { out->print("ShenandoahLoadReferenceBarrierStub"); }