# HG changeset patch
# User rkennke
# Date 1569875951 -7200
#      Mon Sep 30 22:39:11 2019 +0200
# Node ID 946cdbf9cb3c24d837ade302ca8c8b4948b4b4e5
# Parent  cece7402158018330c3418e76a45d86c1c4b8b43
[mq]: simpler-cas-obj.patch

diff -r cece74021580 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Mon Sep 30 17:54:11 2019 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Tue Oct 01 14:16:53 2019 +0200
@@ -247,54 +247,6 @@
   __ bind(done);
 }
 
-void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
-  assert(ShenandoahCASBarrier, "should be enabled");
-  Label is_null;
-  __ testptr(dst, dst);
-  __ jcc(Assembler::zero, is_null);
-  resolve_forward_pointer_not_null(masm, dst, tmp);
-  __ bind(is_null);
-}
-
-void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
-  assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled");
-  // The below loads the mark word, checks if the lowest two bits are
-  // set, and if so, clear the lowest two bits and copy the result
-  // to dst. Otherwise it leaves dst alone.
-  // Implementing this is surprisingly awkward. I do it here by:
-  // - Inverting the mark word
-  // - Test lowest two bits == 0
-  // - If so, set the lowest two bits
-  // - Invert the result back, and copy to dst
-
-  bool borrow_reg = (tmp == noreg);
-  if (borrow_reg) {
-    // No free registers available. Make one useful.
-    tmp = LP64_ONLY(rscratch1) NOT_LP64(rdx);
-    if (tmp == dst) {
-      tmp = LP64_ONLY(rscratch2) NOT_LP64(rcx);
-    }
-    __ push(tmp);
-  }
-
-  assert_different_registers(dst, tmp);
-
-  Label done;
-  __ movptr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
-  __ notptr(tmp);
-  __ testb(tmp, markWord::marked_value);
-  __ jccb(Assembler::notZero, done);
-  __ orptr(tmp, markWord::marked_value);
-  __ notptr(tmp);
-  __ mov(dst, tmp);
-  __ bind(done);
-
-  if (borrow_reg) {
-    __ pop(tmp);
-  }
-}
-
-
 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) {
   assert(ShenandoahLoadRefBarrier, "Should be enabled");
 
@@ -573,7 +525,7 @@
   assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
   assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
 
-  Label retry, done;
+  Label done, done_failure, exit;
 
   // Remember oldval for retry logic below
 #ifdef _LP64
@@ -598,41 +550,70 @@
   }
   __ jcc(Assembler::equal, done, true);
 
+  // Avoid mid-path when witness is NULL
+  __ testptr(oldval, oldval);
+  __ jcc(Assembler::zero, done_failure, true);
+
+  // Avoid mid-path when heap is stable.
+  const Register thread = NOT_LP64(tmp2) LP64_ONLY(r15_thread);
+  NOT_LP64(__ get_thread(thread);)
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
+  __ jcc(Assembler::zero, done_failure, true);
+
   // Step 2. CAS had failed. This may be a false negative.
   //
   // The trouble comes when we compare the to-space pointer with the from-space
-  // pointer to the same object. To resolve this, it will suffice to resolve both
-  // oldval and the value from memory -- this will give both to-space pointers.
+  // pointer to the same object. To resolve this, it will suffice to resolve
+  // the value from memory -- this will give both to-space pointers.
   // If they mismatch, then it was a legitimate failure.
   //
 #ifdef _LP64
   if (UseCompressedOops) {
-    __ decode_heap_oop(tmp1);
-  }
-#endif
-  resolve_forward_pointer(masm, tmp1);
-
-#ifdef _LP64
-  if (UseCompressedOops) {
     __ movl(tmp2, oldval);
     __ decode_heap_oop(tmp2);
+    __ decode_heap_oop(tmp1);
   } else
 #endif
   {
     __ movptr(tmp2, oldval);
   }
-  resolve_forward_pointer(masm, tmp2);
+
+  // Decode offending in-memory value.
+  // Test if-forwarded
+  __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markWord::marked_value);
+  __ jcc(Assembler::noParity, done_failure, true);// When odd number of bits, then not forwarded
+  __ jcc(Assembler::zero, done_failure, true);    // When it's 00, then also not forwarded
 
+  // Load and mask forwarding pointer
+  __ movptr(tmp2, Address(tmp2, oopDesc::mark_offset_in_bytes()));
+  __ shrptr(tmp2, 2);
+  __ shlptr(tmp2, 2);
+
+  // Now we have the forwarded offender in tmp2.
+  // Compare and if they don't match, we have legitimate failure
   __ cmpptr(tmp1, tmp2);
-  __ jcc(Assembler::notEqual, done, true);
+  __ jcc(Assembler::notEqual, done_failure, true);
 
-  // Step 3. Try to CAS again with resolved to-space pointers.
-  //
-  // Corner case: it may happen that somebody stored the from-space pointer
-  // to memory while we were preparing for retry. Therefore, we can fail again
-  // on retry, and so need to do this in loop, always resolving the failure
-  // witness.
-  __ bind(retry);
+  // Step 3. Fix in-memory value by CASing the forwarded copy
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ encode_heap_oop(tmp2);
+  }
+#endif
+
+  if (os::is_MP()) __ lock();
+#ifdef _LP64
+  if (UseCompressedOops) {
+    __ cmpxchgl(tmp2, addr);
+  } else
+#endif
+  {
+    __ cmpxchgptr(tmp2, addr);
+  }
+
+  // Step 4. Safe final CAS
+  __ movptr(oldval, tmp2);
   if (os::is_MP()) __ lock();
 #ifdef _LP64
   if (UseCompressedOops) {
@@ -642,25 +623,19 @@
   {
     __ cmpxchgptr(newval, addr);
   }
-  __ jcc(Assembler::equal, done, true);
+
+  __ jmp(done, true);
 
-#ifdef _LP64
-  if (UseCompressedOops) {
-    __ movl(tmp2, oldval);
-    __ decode_heap_oop(tmp2);
-  } else
-#endif
-  {
-    __ movptr(tmp2, oldval);
+  __ bind(done_failure);
+  if (!exchange) {
+    assert(res != NULL, "need result register");
+    __ xorptr(res, res);
+    __ jmp(exit, true);
   }
-  resolve_forward_pointer(masm, tmp2);
-
-  __ cmpptr(tmp1, tmp2);
-  __ jcc(Assembler::equal, retry, true);
 
   // Step 4. If we need a boolean result out of CAS, check the flag again,
-  // and promote the result. Note that we handle the flag from both the CAS
-  // itself and from the retry loop.
+  // and promote the result. Note that we handle the flag from both the
+  // 1st and 2nd CAS.
   __ bind(done);
   if (!exchange) {
     assert(res != NULL, "need result register");
@@ -678,6 +653,7 @@
     __ bind(res_non_zero);
 #endif
   }
+  __ bind(exit);
 }
 
 #undef __
diff -r cece74021580 src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
--- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Mon Sep 30 17:54:11 2019 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Tue Oct 01 14:16:53 2019 +0200
@@ -55,9 +55,6 @@
                                     bool tosca_live,
                                     bool expand_call);
 
-  void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
-  void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
-
   void load_reference_barrier_not_null(MacroAssembler* masm, Register dst);
 
   void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp);