# HG changeset patch
# Parent 333e3e31243d5004b26513e371edf757f6fb4526

diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
@@ -2597,16 +2597,15 @@
 
 
 void ShenandoahWriteBarrierNode::test_evacuation_in_progress(Node* ctrl, Node* val, Node*& raw_mem,
-                                                             Node*& evac_in_progress, Node*& evac_not_in_progress,
-                                                             Node*& heap_stable, Node*& null_val,
+                                                             Node*& heap_unstable_ctrl,
+                                                             Node*& heap_stable_ctrl, Node*& null_ctrl,
                                                              PhaseIdealLoop* phase) {
   IdealLoopTree *loop = phase->get_loop(ctrl);
-  Node* heap_not_stable = NULL;
   Node* unused_gc_state = NULL;
 
-  test_heap_stable(ctrl, raw_mem, unused_gc_state, heap_stable, heap_not_stable, phase);
-
-  ctrl = heap_not_stable;
+  test_heap_stable(ctrl, raw_mem, unused_gc_state, heap_stable_ctrl, heap_unstable_ctrl, phase);
+
+  ctrl = heap_unstable_ctrl;
 
   const Type* val_t = phase->igvn().type(val);
 
@@ -2621,39 +2620,10 @@
     phase->register_control(not_null, loop, null_iff);
     Node* null = new IfFalseNode(null_iff);
     phase->register_control(null, loop, null_iff);
-    null_val = null;
-    ctrl = not_null;
+    null_ctrl = null;
+    heap_unstable_ctrl = not_null;
   }
 
-  Node* thread = new ThreadLocalNode();
-  phase->register_new_node(thread, ctrl);
-  Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
-  phase->set_ctrl(offset, phase->C->root());
-  Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
-  phase->register_new_node(gc_state_addr, ctrl);
-  uint gc_state_idx = Compile::AliasIdxRaw;
-  const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
-  debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
-
-  Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
-  phase->register_new_node(gc_state, ctrl);
-
-  Node* evacuation_in_progress = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL));
-  phase->register_new_node(evacuation_in_progress, ctrl);
-  Node* evacuation_in_progress_cmp = new CmpINode(evacuation_in_progress, phase->igvn().zerocon(T_INT));
-  phase->register_new_node(evacuation_in_progress_cmp, ctrl);
-  Node* evacuation_in_progress_test = new BoolNode(evacuation_in_progress_cmp, BoolTest::ne);
-  phase->register_new_node(evacuation_in_progress_test, ctrl);
-  IfNode* evacuation_iff = new IfNode(ctrl, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
-  phase->register_control(evacuation_iff, loop, ctrl);
-
-  assert(is_evacuation_in_progress_test(evacuation_iff), "Should match the shape");
-  assert(is_gc_state_load(gc_state), "Should match the shape");
-
-  evac_not_in_progress = new IfFalseNode(evacuation_iff);
-  phase->register_control(evac_not_in_progress, loop, evacuation_iff);
-  evac_in_progress = new IfTrueNode(evacuation_iff);
-  phase->register_control(evac_in_progress, loop, evacuation_iff);
 }
 
 Node* ShenandoahWriteBarrierNode::clone_null_check(Node*& c, Node* val, Node* unc_ctrl,
@@ -2803,10 +2773,10 @@
     Node* in_cset_fast_test_success = new IfFalseNode(in_cset_fast_test_iff);
     phase->register_control(in_cset_fast_test_success, loop, in_cset_fast_test_iff);
 
-    region->init_req(3, in_cset_fast_test_success);
-    val_phi->init_req(3, rbtrue);
-    mem_phi->init_req(3, wb_mem);
-    raw_mem_phi->init_req(3, raw_mem);
+    region->init_req(2, in_cset_fast_test_success);
+    val_phi->init_req(2, rbtrue);
+    mem_phi->init_req(2, wb_mem);
+    raw_mem_phi->init_req(2, raw_mem);
 
     Node* in_cset_fast_test_failure = new IfTrueNode(in_cset_fast_test_iff);
     phase->register_control(in_cset_fast_test_failure, loop, in_cset_fast_test_iff);
@@ -2815,24 +2785,12 @@
   }
 }
 
-void ShenandoahWriteBarrierNode::evacuation_in_progress(Node* c, Node* val, Node* evacuation_iff, Node* unc, Node* unc_ctrl,
-                                                        Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi,
-                                                        Node* raw_mem_phi, Node* unc_region, int alias, Unique_Node_List& uses,
+void ShenandoahWriteBarrierNode::evacuation_in_progress(Node* c, Node* val,
+                                                        Node* raw_mem, Node* wb_mem,
+                                                        Node* region, Node* val_phi, Node* mem_phi,
+                                                        Node* raw_mem_phi, int alias,
                                                         PhaseIdealLoop* phase) {
-  evacuation_in_progress_null_check(c, val, evacuation_iff, unc, unc_ctrl, unc_region, uses, phase);
-
-  IdealLoopTree *loop = phase->get_loop(c);
-
-  Node* rbtrue;
-  if (ShenandoahWriteBarrierRB) {
-    rbtrue = new ShenandoahReadBarrierNode(c, wb_mem, val);
-    phase->register_new_node(rbtrue, c);
-  } else {
-    rbtrue = val;
-  }
-
-  Node* in_cset_fast_test_failure = NULL;
-  in_cset_fast_test(c, rbtrue, raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, phase);
+  IdealLoopTree*loop = phase->get_loop(c);
 
   // The slow path stub consumes and produces raw memory in addition
   // to the existing memory edges
@@ -2849,7 +2807,7 @@
   call->init_req(TypeFunc::Memory, mm);
   call->init_req(TypeFunc::FramePtr, phase->C->top());
   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
-  call->init_req(TypeFunc::Parms, rbtrue);
+  call->init_req(TypeFunc::Parms, val);
   phase->register_control(call, loop, c);
   Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
   phase->register_control(ctrl_proj, loop, call);
@@ -2859,10 +2817,10 @@
   phase->register_new_node(res_proj, call);
   Node* res = new CheckCastPPNode(ctrl_proj, res_proj, phase->igvn().type(val)->is_oopptr()->cast_to_nonconst());
   phase->register_new_node(res, ctrl_proj);
-  region->init_req(2, ctrl_proj);
-  val_phi->init_req(2, res);
-  mem_phi->init_req(2, mem_proj);
-  raw_mem_phi->init_req(2, mem_proj);
+  region->init_req(4, ctrl_proj);
+  val_phi->init_req(4, res);
+  mem_phi->init_req(4, mem_proj);
+  raw_mem_phi->init_req(4, mem_proj);
 }
 
 void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
@@ -3000,32 +2958,26 @@
     Node* raw_mem = fixer.find_mem(ctrl, barrier);
     Node* init_raw_mem = raw_mem;
     Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
-    Node* evac_in_progress = NULL;
-    Node* evac_not_in_progress = NULL;
-    Node* heap_stable = NULL;
-    Node* null_val = NULL;
+    Node* heap_unstable_ctrl = NULL;
+    Node* heap_stable_ctrl = NULL;
+    Node* null_ctrl = NULL;
     uint last = phase->C->unique();
 
-    Node* stable_test_region = new RegionNode(3);
-    Node* stable_test_phi = PhiNode::make(stable_test_region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
-    Node* unstable_region = new RegionNode(5);
-    Node* unstable_phi = PhiNode::make(unstable_region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
-
-    test_evacuation_in_progress(ctrl, pre_val, raw_mem, evac_in_progress, evac_not_in_progress, heap_stable, null_val, phase);
-
-    stable_test_region->init_req(1, heap_stable);
-    stable_test_region->init_req(2, unstable_region);
-    stable_test_phi->init_req(1, raw_mem);
-    stable_test_phi->init_req(2, unstable_phi);
-
-    if (null_val != NULL) {
-      unstable_region->init_req(1, null_val);
-      unstable_phi->init_req(1, raw_mem);
+    Node* region = new RegionNode(5);
+    Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+    test_evacuation_in_progress(ctrl, pre_val, raw_mem, heap_unstable_ctrl, heap_stable_ctrl, null_ctrl, phase);
+
+    // Stable path.
+    region->init_req(1, heap_stable_ctrl);
+    phi->init_req(1, raw_mem);
+    // Null path
+    if (null_ctrl != NULL) {
+      region->init_req(2, null_ctrl);
+      phi->init_req(2, raw_mem);
     }
-    unstable_region->init_req(2, evac_not_in_progress);
-    unstable_phi->init_req(2, raw_mem);
-
-    ctrl = evac_in_progress;
+
+    ctrl = heap_unstable_ctrl;
 
     const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
     const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
@@ -3067,8 +3019,9 @@
     Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered);
     phase->register_new_node(index_update, ctrl);
 
-    unstable_region->init_req(3, ctrl);
-    unstable_phi->init_req(3, index_update);
+    // Fast-path case
+    region->init_req(3, ctrl);
+    phi->init_req(3, index_update);
 
     ctrl = full;
 
@@ -3093,23 +3046,22 @@
     Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
     phase->register_new_node(mem_proj, call);
 
-    unstable_region->init_req(4, ctrl_proj);
-    unstable_phi->init_req(4, mem_proj);
-
-    phase->register_control(unstable_region, loop, null_val != NULL ? null_val->in(0) : evac_in_progress->in(0));
-    phase->register_new_node(unstable_phi, unstable_region);
-    phase->register_control(stable_test_region, loop, heap_stable->in(0));
-    phase->register_new_node(stable_test_phi, stable_test_region);
-
-    fix_ctrl(barrier, stable_test_region, fixer, uses, uses_to_ignore, last, phase);
+    // Slow-path case
+    region->init_req(4, ctrl_proj);
+    phi->init_req(4, mem_proj);
+
+    phase->register_control(region, loop, heap_stable_ctrl->in(0));
+    phase->register_new_node(phi, region);
+
+    fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
     for(uint next = 0; next < uses.size(); next++ ) {
       Node *n = uses.at(next);
       assert(phase->get_ctrl(n) == init_ctrl, "bad control");
       assert(n != init_raw_mem, "should leave input raw mem above the barrier");
-      phase->set_ctrl(n, stable_test_region);
+      phase->set_ctrl(n, region);
       follow_barrier_uses(n, init_ctrl, uses, phase);
     }
-    fixer.fix_mem(init_ctrl, stable_test_region, init_raw_mem, raw_mem_for_ctrl, stable_test_phi, uses);
+    fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
 
     phase->igvn().replace_node(barrier, pre_val);
   }
@@ -3120,6 +3072,7 @@
 
     uint last = phase->C->unique();
     Node* ctrl = phase->get_ctrl(wb);
+    Node* orig_ctrl = ctrl;
 
     Node* raw_mem = fixer.find_mem(ctrl, wb);
     Node* init_raw_mem = raw_mem;
@@ -3149,55 +3102,76 @@
       uncasted_val = val->in(1);
     }
 
-    Node* evac_in_progress = NULL;
-    Node* evac_not_in_progress = NULL;
     Node* heap_stable_ctrl = NULL;
+    Node* heap_unstable_ctrl = NULL;
     Node* null_ctrl = NULL;
-    test_evacuation_in_progress(ctrl, val, raw_mem, evac_in_progress, evac_not_in_progress, heap_stable_ctrl, null_ctrl, phase);
-    IfNode* evacuation_iff = evac_in_progress->in(0)->as_If();
+    test_evacuation_in_progress(ctrl, val, raw_mem, heap_unstable_ctrl, heap_stable_ctrl, null_ctrl, phase);
     IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
 
-    Node* evacuation_region = new RegionNode(5);
-    Node* evacuation_val_phi = new PhiNode(evacuation_region, uncasted_val->bottom_type()->is_oopptr()->cast_to_nonconst());
-    Node* evacuation_mem_phi = PhiNode::make(evacuation_region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type());
-    Node* evacuation_raw_mem_phi = PhiNode::make(evacuation_region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
-    Node* region = new RegionNode(3);
+    Node* orig_val = uncasted_val;
+    Node* region = new RegionNode(6);
     Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()->cast_to_nonconst());
     Node* mem_phi = PhiNode::make(region, wb_mem, Type::MEMORY, phase->C->alias_type(wb->adr_type())->adr_type());
     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
 
+    // Heap stable case
+    region->init_req(1, heap_stable_ctrl);
+    val_phi->init_req(1, orig_val);
+    mem_phi->init_req(1, wb_mem);
+    raw_mem_phi->init_req(1, raw_mem);
+
+    // Null case
     if (null_ctrl != NULL) {
-      evacuation_region->init_req(4, null_ctrl);
-      evacuation_val_phi->init_req(4, phase->igvn().zerocon(T_OBJECT));
-      evacuation_mem_phi->init_req(4, wb_mem);
-      evacuation_raw_mem_phi->init_req(4, raw_mem);
+      region->init_req(5, null_ctrl);
+      // TODO: This probably loads a new NULL into register, instead of
+      // reusing the (just null-tested) original value.
+      val_phi->init_req(5, phase->igvn().zerocon(T_OBJECT));
+      mem_phi->init_req(5, wb_mem);
+      raw_mem_phi->init_req(5, raw_mem);
     } else {
-      evacuation_region->del_req(4);
-      evacuation_val_phi->del_req(4);
-      evacuation_mem_phi->del_req(4);
-      evacuation_raw_mem_phi->del_req(4);
+      region->del_req(5);
+      val_phi->del_req(5);
+      mem_phi->del_req(5);
+      raw_mem_phi->del_req(5);
     }
 
-    Node* unc_region = NULL;
-    evacuation_not_in_progress(evac_not_in_progress, val, unc_ctrl, raw_mem, wb_mem,
-                               evacuation_region, evacuation_val_phi, evacuation_mem_phi, evacuation_raw_mem_phi, unc_region,
-                               phase);
-
-    heap_stable(heap_stable_ctrl, val, unc_ctrl, init_raw_mem, init_wb_mem, region, val_phi, mem_phi, raw_mem_phi,
-                unc_region, phase);
-
-    evacuation_in_progress(evac_in_progress, val, evacuation_iff, unc, unc_ctrl,
-                           raw_mem, wb_mem, evacuation_region, evacuation_val_phi, evacuation_mem_phi, evacuation_raw_mem_phi,
-                           unc_region, alias, uses,
-                           phase);
-    region->init_req(2, evacuation_region);
-    val_phi->init_req(2, evacuation_val_phi);
-    mem_phi->init_req(2, evacuation_mem_phi);
-    raw_mem_phi->init_req(2, evacuation_raw_mem_phi);
-    phase->register_control(evacuation_region, loop, evacuation_iff);
-    phase->register_new_node(evacuation_val_phi, evacuation_region);
-    phase->register_new_node(evacuation_mem_phi, evacuation_region);
-    phase->register_new_node(evacuation_raw_mem_phi, evacuation_region);
+    // Test for in-cset.
+    // Wires !in_cset(obj) to slot 2 of region and phis
+    ctrl = heap_unstable_ctrl;
+    in_cset_fast_test(ctrl, orig_val, raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, phase);
+
+    // Resolve object when orig-value is in cset.
+    Node* rb;
+    if (ShenandoahWriteBarrierRB) {
+      rb = new ShenandoahReadBarrierNode(ctrl, wb_mem, orig_val);
+      phase->register_new_node(rb, ctrl);
+    } else {
+      rb = val;
+    }
+
+    // Only branch to WB stub if rb == orig-value.
+    Node* cmp = new CmpPNode(rb, orig_val);
+    phase->register_new_node(cmp, ctrl);
+    Node* bol = new BoolNode(cmp, BoolTest::eq);
+    phase->register_new_node(bol, ctrl);
+
+    IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
+    phase->register_control(iff, loop, ctrl);
+    Node* if_not_eq = new IfFalseNode(iff);
+    phase->register_control(if_not_eq, loop, iff);
+    Node* if_eq = new IfTrueNode(iff);
+    phase->register_control(if_eq, loop, iff);
+
+    // Wire up not-equal-path in slots 3.
+    region->init_req(3, if_not_eq);
+    val_phi->init_req(3, rb);
+    mem_phi->init_req(3, wb_mem);
+    raw_mem_phi->init_req(3, raw_mem);
+
+    // Call wb-stub and wire up that path in slots 4
+    evacuation_in_progress(if_eq, rb, raw_mem, wb_mem,
+                           region, val_phi, mem_phi, raw_mem_phi,
+                           alias, phase);
 
     phase->register_control(region, loop, heap_stable_iff);
 
@@ -3208,6 +3182,8 @@
 
     fix_ctrl(wb, region, fixer, uses, uses_to_ignore, last, phase);
 
+    ctrl = orig_ctrl;
+
     phase->igvn().replace_input_of(wbproj, 0, phase->C->top());
     phase->lazy_replace(wbproj, mem_phi);
 
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
@@ -199,15 +199,15 @@
   static void test_heap_stable(Node* ctrl, Node* raw_mem, Node*& gc_state, Node*& heap_stable,
                                Node*& heap_not_stable, PhaseIdealLoop* phase);
   static void test_evacuation_in_progress(Node* ctrl, Node* val, Node*& raw_mem,
-                                          Node*& evac_in_progress, Node*& evac_not_in_progress,
-                                          Node*& heap_stable, Node*& null_val,
+                                          Node*& heap_unstable_ctrl,
+                                          Node*& heap_stable_ctrl, Node*& null_ctrl,
                                           PhaseIdealLoop* phase);
   static void evacuation_not_in_progress(Node* c, Node* v, Node* unc_ctrl, Node* raw_mem, Node* wb_mem, Node* region,
                                          Node* val_phi, Node* mem_phi, Node* raw_mem_phi, Node*& unc_region,
                                          PhaseIdealLoop* phase);
-  static void evacuation_in_progress(Node* c, Node* val, Node* evacuation_iff, Node* unc, Node* unc_ctrl,
+  static void evacuation_in_progress(Node* c, Node* val,
                                      Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi,
-                                     Node* raw_mem_phi, Node* unc_region, int alias, Unique_Node_List& uses,
+                                     Node* raw_mem_phi, int alias,
                                      PhaseIdealLoop* phase);
   static void heap_stable(Node* c, Node* val, Node* unc_ctrl, Node* raw_mem, Node* wb_mem, Node* region,
                           Node* val_phi, Node* mem_phi, Node* raw_mem_phi, Node* unc_region, PhaseIdealLoop* phase);