# HG changeset patch
# Parent 74e8612336a7df83f2f282995b6c5ab409e10570

diff -r 74e8612336a7 src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp	Fri Aug 09 06:07:40 2019 -0400
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp	Fri Aug 09 08:29:55 2019 -0400
@@ -472,27 +472,6 @@
   }
 }
 
-// Generate cset check. If obj is not in cset, branch to done label, otherwise fall through
-// obj: Register holding the oop, preserved
-// tmp1, tmp2: temp registers, trashed
-void ShenandoahBarrierSetAssembler::gen_cset_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& done) {
-  __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
-  __ lsr(tmp1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
-  __ ldrb(tmp2, Address(tmp2, tmp1));
-  __ tbz(tmp2, 0, done);
-}
-
-// Generate check if object is resolved. Branch to not_resolved label, if not. Otherwise return resolved
-// object in obj register.
-// obj: object, resolved object on normal return
-// tmp1, tmp2: temp registers, trashed
-void ShenandoahBarrierSetAssembler::gen_resolved_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& resolved) {
-  __ mov(tmp2, obj);
-  resolve_forward_pointer_not_null(masm, obj, tmp1);
-  __ cmp(tmp2, obj);
-  __ br(Assembler::EQ, resolved);
-}
-
 #undef __
 
 #ifdef COMPILER1
@@ -530,8 +509,6 @@
   Register tmp1 = stub->tmp1()->as_register();
   Register tmp2 = stub->tmp2()->as_register();
 
-  Label slow_path;
-
   assert(res == r0, "result must arrive in r0");
 
   if (res != obj) {
@@ -541,8 +518,22 @@
   // Check for null.
   __ cbz(res, *stub->continuation());
 
-  gen_cset_check(ce->masm(), res, tmp1, tmp2, *stub->continuation());
-  gen_resolved_check(ce->masm(), res, tmp1, tmp2, slow_path);
+  // Check for object in cset.
+  __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
+  __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ ldrb(tmp2, Address(tmp2, tmp1));
+  __ cbz(tmp2, *stub->continuation());
+
+  // Check if object is already forwarded.
+  Label slow_path;
+  __ ldr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
+  __ eon(tmp1, tmp1, zr);
+  __ ands(zr, tmp1, markOopDesc::lock_mask_in_place);
+  __ br(Assembler::NE, slow_path);
+
+  // Decode forwarded object.
+  __ orr(tmp1, tmp1, markOopDesc::marked_value);
+  __ eon(res, tmp1, zr);
   __ b(*stub->continuation());
 
   __ bind(slow_path);
@@ -643,12 +634,26 @@
   StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
   address start = __ pc();
 
-  Label done, call_rt;
-  gen_cset_check(cgen->assembler(), r0, rscratch1, rscratch2, done);
-  gen_resolved_check(cgen->assembler(), r0, rscratch1, rscratch2, call_rt);
+  Label work, done;
+  __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
+  __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ ldrb(rscratch2, Address(rscratch2, rscratch1));
+  __ tbnz(rscratch2, 0, work);
+  __ ret(lr);
+  __ bind(work);
+
+  Label slow_path;
+  __ ldr(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
+  __ eon(rscratch1, rscratch1, zr);
+  __ ands(zr, rscratch1, markOopDesc::lock_mask_in_place);
+  __ br(Assembler::NE, slow_path);
+
+  // Decode forwarded object.
+  __ orr(rscratch1, rscratch1, markOopDesc::marked_value);
+  __ eon(r0, rscratch1, zr);
   __ ret(lr);
 
-  __ bind(call_rt);
+  __ bind(slow_path);
   __ enter(); // required for proper stackwalking of RuntimeStub frame
 
   __ push_call_clobbered_registers();
@@ -660,7 +665,6 @@
   __ mov(r0, rscratch1);
 
   __ leave(); // required for proper stackwalking of RuntimeStub frame
-
   __ bind(done);
   __ ret(lr);
 
diff -r 74e8612336a7 src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
--- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp	Fri Aug 09 06:07:40 2019 -0400
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp	Fri Aug 09 08:29:55 2019 -0400
@@ -62,17 +62,6 @@
 
   address generate_shenandoah_lrb(StubCodeGenerator* cgen);
 
-  // Generate cset check. If obj is not in cset, branch to done label, otherwise fall through
-  // obj: Register holding the oop, preserved
-  // tmp1, tmp2: temp registers, trashed
-  void gen_cset_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& done);
-
-  // Generate check if object is resolved. Branch to not_resolved label, if not. Otherwise return resolved
-  // object in obj register.
-  // obj: object, resolved object on normal return
-  // tmp1, tmp2: temp registers, trashed
-  void gen_resolved_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& resolved);
-
 public:
   static address shenandoah_lrb();
 
diff -r 74e8612336a7 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Fri Aug 09 06:07:40 2019 -0400
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Fri Aug 09 08:29:55 2019 -0400
@@ -736,44 +736,6 @@
   }
 }
 
-// Generate cset check. If obj is not in cset, branch to done label, otherwise fall through
-// obj: Register holding the oop, preserved
-// tmp1, tmp2: temp registers, trashed
-void ShenandoahBarrierSetAssembler::gen_cset_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& done) {
-  // Check for object being in the collection set.
-  // TODO: Can we use only 1 register here?
-  // The source object arrives here in rax.
-  // live: rax
-  // live: tmp1
-  __ mov(tmp1, obj);
-  __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
-  // live: tmp2
-  __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
-  __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
-  // unlive: tmp1
-  __ testbool(tmp2);
-  // unlive: tmp2
-  __ jcc(Assembler::zero, done);
-}
-
-// Generate check if object is resolved. Branch to resolved label, if not. Otherwise return resolved
-// object in obj register.
-// obj: object, resolved object on normal return
-// tmp: temp register, trashed
-void ShenandoahBarrierSetAssembler::gen_resolved_check(MacroAssembler* masm, Register obj, Register tmp, Label& not_resolved) {
-  __ movptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
-  // Test if both lowest bits are set. We trick it by negating the bits
-  // then test for both bits clear.
-  __ notptr(tmp);
-  __ testb(tmp, markOopDesc::marked_value);
-  __ jccb(Assembler::notZero, not_resolved);
-  // Clear both lower bits. It's still inverted, so set them, and then invert back.
-  __ orptr(tmp, markOopDesc::marked_value);
-  __ notptr(tmp);
-  // At this point, tmp2 contains the decoded forwarding pointer.
-  __ mov(obj, tmp);
-}
-
 #undef __
 
 #ifdef COMPILER1
@@ -825,8 +787,26 @@
   __ testptr(res, res);
   __ jcc(Assembler::zero, *stub->continuation());
 
-  gen_cset_check(ce->masm(), res, tmp1, tmp2, *stub->continuation());
-  gen_resolved_check(ce->masm(), rax, tmp1, slow_path);
+  // Check for object being in the collection set.
+  __ mov(tmp1, res);
+  __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+  __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
+  __ testbool(tmp2);
+  __ jcc(Assembler::zero, *stub->continuation());
+
+  // Test if object is resolved.
+  __ movptr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
+  // Test if both lowest bits are set. We trick it by negating the bits
+  // then test for both bits clear.
+  __ notptr(tmp1);
+  __ testb(tmp1, markOopDesc::marked_value);
+  __ jccb(Assembler::notZero, slow_path);
+  // Clear both lower bits. It's still inverted, so set them, and then invert back.
+  __ orptr(tmp1, markOopDesc::marked_value);
+  __ notptr(tmp1);
+  // At this point, tmp1 contains the decoded forwarding pointer.
+  __ mov(res, tmp1);
 
   __ jmp(*stub->continuation());
 
@@ -937,11 +917,30 @@
   __ push(tmp1);
   __ push(tmp2);
 
-  gen_cset_check(cgen->assembler(), rax, tmp1, tmp2, done);
+  // Check for object being in the collection set.
+  __ mov(tmp1, rax);
+  __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+  __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
+  __ testbool(tmp2);
+  __ jccb(Assembler::notZero, resolve_oop);
+  __ pop(tmp2);
+  __ pop(tmp1);
+  __ ret(0);
 
+  // Test if object is already resolved.
   __ bind(resolve_oop);
-
-  gen_resolved_check(cgen->assembler(), rax, tmp2, slow_path);
+  __ movptr(tmp2, Address(rax, oopDesc::mark_offset_in_bytes()));
+  // Test if both lowest bits are set. We trick it by negating the bits
+  // then test for both bits clear.
+  __ notptr(tmp2);
+  __ testb(tmp2, markOopDesc::marked_value);
+  __ jccb(Assembler::notZero, slow_path);
+  // Clear both lower bits. It's still inverted, so set them, and then invert back.
+  __ orptr(tmp2, markOopDesc::marked_value);
+  __ notptr(tmp2);
+  // At this point, tmp2 contains the decoded forwarding pointer.
+  __ mov(rax, tmp2);
 
   __ bind(done);
   __ pop(tmp2);
diff -r 74e8612336a7 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Fri Aug 09 06:07:40 2019 -0400
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Fri Aug 09 08:29:55 2019 -0400
@@ -64,17 +64,6 @@
 
   address generate_shenandoah_lrb(StubCodeGenerator* cgen);
 
-  // Generate cset check. If obj is not in cset, branch to done label, otherwise fall through
-  // obj: Register holding the oop, preserved
-  // tmp1, tmp2: temp registers, trashed
-  void gen_cset_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& done);
-
-  // Generate check if object is resolved. Branch to resolved label, if not. Otherwise return resolved
-  // object in obj register.
-  // obj: object, resolved object on normal return
-  // tmp: temp register, trashed
-  void gen_resolved_check(MacroAssembler* masm, Register obj, Register tmp, Label& not_resolved);
-
 public:
   static address shenandoah_lrb();