# HG changeset patch
# User rkennke
# Date 1583425199 -3600
#      Thu Mar 05 17:19:59 2020 +0100
# Node ID 4574d265157c7b517fa678a0a9ef2e11de1f0f7f
# Parent  e3f1f696063f3c4bb12f29aef02058720be8f047
8240315: Shenandoah: Rename ShLBN::get_barrier_strength()
* * *
[mq]: betterweaks.patch
* * *
[mq]: betterweaks2.patch

diff -r e3f1f696063f src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp	Wed Mar 04 19:23:13 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp	Fri Mar 06 07:40:42 2020 +0100
@@ -563,9 +563,7 @@
 
     if (on_weak_ref) {
       // Use the pre-barrier to record the value in the referent field
-      satb_write_barrier_pre(kit, false /* do_load */,
-                             NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
-                             load /* pre_val */, T_OBJECT);
+      load = shenandoah_enqueue_barrier(kit, load);
       // Add memory barrier to prevent commoning reads from this field
       // across safepoint since GC can change its value.
       kit->insert_mem_bar(Op_MemBarCPUOrder);
diff -r e3f1f696063f src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp	Wed Mar 04 19:23:13 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp	Fri Mar 06 07:40:42 2020 +0100
@@ -863,7 +863,7 @@
 }
 
 void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
-                                                  PhaseIdealLoop* phase) {
+                                                  PhaseIdealLoop* phase, int flags) {
   IdealLoopTree* loop = phase->get_loop(ctrl);
   Node* thread = new ThreadLocalNode();
   phase->register_new_node(thread, ctrl);
@@ -877,7 +877,7 @@
 
   Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
   phase->register_new_node(gc_state, ctrl);
-  Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED));
+  Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(flags));
   phase->register_new_node(heap_stable_and, ctrl);
   Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
   phase->register_new_node(heap_stable_cmp, ctrl);
@@ -891,7 +891,7 @@
   ctrl = new IfTrueNode(heap_stable_iff);
   phase->register_control(ctrl, loop, heap_stable_iff);
 
-  assert(is_heap_stable_test(heap_stable_iff), "Should match the shape");
+  assert(is_heap_state_test(heap_stable_iff, flags), "Should match the shape");
 }
 
 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
@@ -1144,7 +1144,7 @@
   Node_List clones;
   for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
-    if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
+    if (!lrb->is_lrb_useful()) {
       continue;
     }
 
@@ -1355,7 +1355,7 @@
 
   for (int i = 0; i < state->load_reference_barriers_count(); i++) {
     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
-    if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
+    if (!lrb->is_lrb_useful()) {
       continue;
     }
     Node* ctrl = phase->get_ctrl(lrb);
@@ -1374,7 +1374,7 @@
   Unique_Node_List uses_to_ignore;
   for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
     ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
-    if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
+    if (!lrb->is_lrb_useful()) {
       phase->igvn().replace_node(lrb, lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
       continue;
     }
@@ -1417,7 +1417,7 @@
     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
 
     // Stable path.
-    test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
+    test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::HAS_FORWARDED);
     IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
 
     // Heap stable case
@@ -1558,6 +1558,11 @@
       continue;
     }
 
+    if (!((ShenandoahEnqueueBarrierNode*)barrier)->is_useful()) {
+      phase->igvn().replace_node(barrier, pre_val);
+      continue;
+    }
+
     Node* ctrl = phase->get_ctrl(barrier);
 
     if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
@@ -1588,7 +1593,7 @@
     Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
 
     // Stable path.
-    test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
+    test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::TRAVERSAL | ShenandoahHeap::MARKING);
     region->init_req(_heap_stable, heap_stable_ctrl);
     phi->init_req(_heap_stable, raw_mem);
 
@@ -2111,95 +2116,163 @@
   return t->is_oopptr();
 }
 
-int ShenandoahEnqueueBarrierNode::needed(Node* n) {
-  if (n == NULL ||
-      n->is_Allocate() ||
-      n->Opcode() == Op_ShenandoahEnqueueBarrier ||
-      n->bottom_type() == TypePtr::NULL_PTR ||
-      (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL)) {
-    return NotNeeded;
-  }
-  if (n->is_Phi() ||
-      n->is_CMove()) {
-    return MaybeNeeded;
-  }
-  return Needed;
+bool ShenandoahEnqueueBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) {
+  Unique_Node_List visited;
+  return needs_barrier_impl(phase, n, visited);
 }
 
-Node* ShenandoahEnqueueBarrierNode::next(Node* n) {
-  for (;;) {
-    if (n == NULL) {
-      return n;
-    } else if (n->bottom_type() == TypePtr::NULL_PTR) {
-      return n;
-    } else if (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL) {
-      return n;
-    } else if (n->is_ConstraintCast() ||
-               n->Opcode() == Op_DecodeN ||
-               n->Opcode() == Op_EncodeP) {
-      n = n->in(1);
-    } else if (n->is_Proj()) {
-      n = n->in(0);
-    } else {
-      return n;
+bool ShenandoahEnqueueBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) {
+  if (n == NULL) return false;
+  if (visited.member(n)) {
+    return false; // Been there.
+  }
+  visited.push(n);
+
+  if (n->is_Allocate()) {
+    // tty->print_cr("optimize barrier on alloc");
+    return false;
+  }
+  if (n->is_Call()) {
+    // tty->print_cr("optimize barrier on call");
+    return false;
+  }
+
+  const Type* type = phase->type(n);
+  if (type == Type::TOP) {
+    return false;
+  }
+  if (type->make_ptr()->higher_equal(TypePtr::NULL_PTR)) {
+    // tty->print_cr("optimize barrier on null");
+    return false;
+  }
+  if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) {
+    // tty->print_cr("optimize barrier on constant");
+    return false; // Needed, apparently
+  }
+
+  switch (n->Opcode()) {
+    case Op_ShenandoahEnqueueBarrier:
+      return false;
+    case Op_Parm:
+      return false;
+    case Op_LoadN:
+    case Op_LoadP:
+    case Op_GetAndSetP:
+    case Op_GetAndSetN:
+    case Op_CompareAndExchangeP:
+    case Op_CompareAndExchangeN:
+      return true;
+    case Op_Proj:
+      return needs_barrier_impl(phase, n->in(0), visited);
+    case Op_DecodeN:
+    case Op_EncodeP:
+    case Op_CheckCastPP:
+    case Op_CastPP:
+      return needs_barrier_impl(phase, n->in(1), visited);
+    case Op_ShenandoahLoadReferenceBarrier:
+      return needs_barrier_impl(phase, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), visited);
+    case Op_CMoveN:
+    case Op_CMoveP:
+      return needs_barrier_impl(phase, n->in(2), visited) ||
+             needs_barrier_impl(phase, n->in(3), visited);
+    case Op_Phi: {
+      for (uint i = 1; i < n->req(); i++) {
+        if (needs_barrier_impl(phase, n->in(i), visited)) return true;
+      }
+      return false;
+    }
+    default:
+      break;
+  }
+#ifdef ASSERT
+  tty->print("need enqueue barrier on?: ");
+  tty->print_cr("ins:");
+  n->dump(2);
+  tty->print_cr("outs:");
+  n->dump(-2);
+  ShouldNotReachHere();
+#endif
+  return true;
+}
+
+bool ShenandoahEnqueueBarrierNode::is_useful() {
+  Unique_Node_List visited;
+  Node_Stack stack(0);
+  stack.push(this, 0);
+
+  bool useful = false;
+  while (!useful && stack.size() > 0) {
+    Node* n = stack.node();
+    if (visited.member(n)) {
+      stack.pop();
+      continue;
+    }
+    visited.push(n);
+    bool visit_users = false;
+    switch (n->Opcode()) {
+      case Op_CallDynamicJava:
+      case Op_CallStaticJava:
+      case Op_CompareAndExchangeN:
+      case Op_CompareAndExchangeP:
+      case Op_CompareAndSwapN:
+      case Op_CompareAndSwapP:
+      case Op_GetAndSetN:
+      case Op_GetAndSetP:
+      case Op_Return:
+      case Op_StoreN:
+      case Op_StoreP:
+        useful = true;
+        break;
+      case Op_AddP:
+      case Op_Allocate:
+      case Op_AllocateArray:
+      case Op_ArrayCopy:
+      case Op_CmpP:
+      case Op_LoadL:
+      case Op_SafePoint:
+        // useful = false;
+        break;
+      case Op_CastPP:
+      case Op_CheckCastPP:
+      case Op_CMoveN:
+      case Op_CMoveP:
+      case Op_EncodeP:
+      case Op_Phi:
+      case Op_ShenandoahEnqueueBarrier:
+        visit_users = true;
+        break;
+      default: {
+#ifdef ASSERT
+        //fatal("Unknown node in get_barrier_strength: %s", NodeClassNames[n->Opcode()]);
+        useful = true;
+#else
+        // Default to strong: better to have excess barriers, rather than miss some.
+        useful = true;
+#endif
+      }
+    }
+
+    stack.pop();
+    if (visit_users) {
+      for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+        Node* user = n->fast_out(i);
+        if (user != NULL) {
+          stack.push(user, 0);
+        }
+      }
     }
   }
-  ShouldNotReachHere();
-  return NULL;
+  return useful;
 }
 
 Node* ShenandoahEnqueueBarrierNode::Identity(PhaseGVN* phase) {
   PhaseIterGVN* igvn = phase->is_IterGVN();
-
-  Node* n = next(in(1));
-
-  int cont = needed(n);
-
-  if (cont == NotNeeded) {
-    return in(1);
-  } else if (cont == MaybeNeeded) {
-    if (igvn == NULL) {
-      phase->record_for_igvn(this);
-      return this;
-    } else {
-      ResourceMark rm;
-      Unique_Node_List wq;
-      uint wq_i = 0;
-
-      for (;;) {
-        if (n->is_Phi()) {
-          for (uint i = 1; i < n->req(); i++) {
-            Node* m = n->in(i);
-            if (m != NULL) {
-              wq.push(m);
-            }
-          }
-        } else {
-          assert(n->is_CMove(), "nothing else here");
-          Node* m = n->in(CMoveNode::IfFalse);
-          wq.push(m);
-          m = n->in(CMoveNode::IfTrue);
-          wq.push(m);
-        }
-        Node* orig_n = NULL;
-        do {
-          if (wq_i >= wq.size()) {
-            return in(1);
-          }
-          n = wq.at(wq_i);
-          wq_i++;
-          orig_n = n;
-          n = next(n);
-          cont = needed(n);
-          if (cont == Needed) {
-            return this;
-          }
-        } while (cont != MaybeNeeded || (orig_n != n && wq.member(n)));
-      }
-    }
+  Node* value = in(1);
+  if (!needs_barrier(phase, value)) {
+    return value;
+  } else {
+    return this;
   }
-
-  return this;
 }
 
 #ifdef ASSERT
@@ -3150,7 +3223,7 @@
   return true;
 }
 
-ShenandoahLoadReferenceBarrierNode::Strength ShenandoahLoadReferenceBarrierNode::get_barrier_strength() {
+bool ShenandoahLoadReferenceBarrierNode::is_lrb_useful() {
   Unique_Node_List visited;
   Node_Stack stack(0);
   stack.push(this, 0);
@@ -3158,8 +3231,8 @@
   // Look for strongest strength: go over nodes looking for STRONG ones.
   // Stop once we encountered STRONG. Otherwise, walk until we ran out of nodes,
   // and then the overall strength is NONE.
-  Strength strength = NONE;
-  while (strength != STRONG && stack.size() > 0) {
+  bool useful = false;
+  while (!useful && stack.size() > 0) {
     Node* n = stack.node();
     if (visited.member(n)) {
       stack.pop();
@@ -3206,7 +3279,6 @@
       case Op_GetAndAddI:
       case Op_GetAndAddB:
       case Op_GetAndAddS:
-      case Op_ShenandoahEnqueueBarrier:
       case Op_FastLock:
       case Op_FastUnlock:
       case Op_Rethrow:
@@ -3235,14 +3307,14 @@
       case Op_StrIndexOfChar:
       case Op_HasNegatives:
         // Known to require barriers
-        strength = STRONG;
+        useful = true;
         break;
       case Op_CmpP: {
         if (n->in(1)->bottom_type()->higher_equal(TypePtr::NULL_PTR) ||
             n->in(2)->bottom_type()->higher_equal(TypePtr::NULL_PTR)) {
           // One of the sides is known null, no need for barrier.
         } else {
-          strength = STRONG;
+          useful = true;
         }
         break;
       }
@@ -3268,7 +3340,7 @@
           // Loading the constant does not require barriers: it should be handled
           // as part of GC roots already.
         } else {
-          strength = STRONG;
+          useful = true;
         }
         break;
       }
@@ -3284,15 +3356,16 @@
       case Op_CMoveP:
       case Op_Phi:
       case Op_ShenandoahLoadReferenceBarrier:
+      case Op_ShenandoahEnqueueBarrier:
         // Whether or not these need the barriers depends on their users
         visit_users = true;
         break;
       default: {
 #ifdef ASSERT
-        fatal("Unknown node in get_barrier_strength: %s", NodeClassNames[n->Opcode()]);
+        fatal("Unknown node in is_lrb_useful: %s", NodeClassNames[n->Opcode()]);
 #else
         // Default to strong: better to have excess barriers, rather than miss some.
-        strength = STRONG;
+        useful = true;
 #endif
       }
     }
@@ -3307,7 +3380,7 @@
       }
     }
   }
-  return strength;
+  return useful;
 }
 
 CallStaticJavaNode* ShenandoahLoadReferenceBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) {
diff -r e3f1f696063f src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp	Wed Mar 04 19:23:13 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp	Fri Mar 06 07:40:42 2020 +0100
@@ -58,7 +58,7 @@
   static void follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase);
   static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase);
   static void test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
-                               PhaseIdealLoop* phase);
+                               PhaseIdealLoop* phase, int flags);
   static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase);
   static Node* clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase);
   static void fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses,
@@ -96,12 +96,13 @@
   Node* Identity(PhaseGVN* phase);
 
   int Opcode() const;
+  bool is_useful();
 
 private:
   enum { Needed, NotNeeded, MaybeNeeded };
 
-  static int needed(Node* n);
-  static Node* next(Node* n);
+  bool needs_barrier(PhaseGVN* phase, Node* n);
+  bool needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited);
 };
 
 class MemoryGraphFixer : public ResourceObj {
@@ -230,10 +231,6 @@
     ValueIn
   };
 
-  enum Strength {
-    NONE, STRONG
-  };
-
   ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val);
 
   virtual int Opcode() const;
@@ -251,7 +248,7 @@
     return sizeof(*this);
   }
 
-  Strength get_barrier_strength();
+  bool is_lrb_useful();
   CallStaticJavaNode* pin_and_expand_null_check(PhaseIterGVN& igvn);
 
 private:
diff -r e3f1f696063f src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp	Wed Mar 04 19:23:13 2020 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp	Fri Mar 06 07:40:42 2020 +0100
@@ -177,21 +177,28 @@
 private:
   ShenandoahConcMarkSATBBufferClosure* _satb_cl;
   int _thread_parity;
+  OopClosure* const _cl;
+  CodeBlobClosure* const _code_cl;
 
 public:
-  ShenandoahSATBThreadsClosure(ShenandoahConcMarkSATBBufferClosure* satb_cl) :
+  ShenandoahSATBThreadsClosure(ShenandoahConcMarkSATBBufferClosure* satb_cl, OopClosure* cl, CodeBlobClosure* code_cl) :
     _satb_cl(satb_cl),
-    _thread_parity(Threads::thread_claim_parity()) {}
+    _thread_parity(Threads::thread_claim_parity()),
+    _cl(cl), _code_cl(code_cl) {}
 
   void do_thread(Thread* thread) {
     if (thread->is_Java_thread()) {
       if (thread->claim_oops_do(true, _thread_parity)) {
         JavaThread* jt = (JavaThread*)thread;
         ShenandoahThreadLocalData::satb_mark_queue(jt).apply_closure_and_empty(_satb_cl);
+        ResourceMark rm;
+        thread->oops_do(_cl, _code_cl);
       }
     } else if (thread->is_VM_thread()) {
       if (thread->claim_oops_do(true, _thread_parity)) {
         ShenandoahBarrierSet::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl);
+        ResourceMark rm;
+        thread->oops_do(_cl, _code_cl);
       }
     }
   }
@@ -219,12 +226,22 @@
     // full-gc.
     {
       ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
-      ShenandoahConcMarkSATBBufferClosure cl(q);
+      ShenandoahConcMarkSATBBufferClosure buf_cl(q);
       ShenandoahSATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
-      while (satb_mq_set.apply_closure_to_completed_buffer(&cl));
-      ShenandoahSATBThreadsClosure tc(&cl);
-      Threads::threads_do(&tc);
-    }
+      while (satb_mq_set.apply_closure_to_completed_buffer(&buf_cl));
+      ReferenceProcessor* rp = heap->ref_processor();
+      if (heap->has_forwarded_objects()) {
+        ShenandoahMarkResolveRefsClosure cl(q, rp);
+        CodeBlobToOopClosure code_cl(&cl, false);
+        ShenandoahSATBThreadsClosure tc(&buf_cl, &cl, &code_cl);
+        Threads::threads_do(&tc);
+      } else {
+        ShenandoahMarkRefsClosure cl(q, rp);
+        CodeBlobToOopClosure code_cl(&cl, false);
+        ShenandoahSATBThreadsClosure tc(&buf_cl, &cl, &code_cl);
+        Threads::threads_do(&tc);
+      }
+   }
 
     ReferenceProcessor* rp;
     if (heap->process_references()) {