--- old/src/hotspot/share/opto/loopopts.cpp 2018-11-26 21:30:19.599962610 +0100 +++ new/src/hotspot/share/opto/loopopts.cpp 2018-11-26 21:30:19.438964030 +0100 @@ -44,11 +44,14 @@ #if INCLUDE_ZGC #include "gc/z/c2/zBarrierSetC2.hpp" #endif +#if INCLUDE_SHENANDOAHGC +#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" +#endif //============================================================================= //------------------------------split_thru_phi--------------------------------- // Split Node 'n' through merge point if there is enough win. -Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) { +Node *PhaseIdealLoop::split_thru_phi(Node *n, Node *region, int policy, Node** out_mem_phi) { if (n->Opcode() == Op_ConvI2L && n->bottom_type() != TypeLong::LONG) { // ConvI2L may have type information on it which is unsafe to push up // so disable this for now @@ -78,6 +81,12 @@ } else { phi = PhiNode::make_blank(region, n); } + Node* mem_phi = NULL; +#if INCLUDE_SHENANDOAHGC + if (n->Opcode() == Op_ShenandoahWriteBarrier) { + mem_phi = new PhiNode(region, Type::MEMORY, C->alias_type(n->adr_type())->adr_type()); + } +#endif uint old_unique = C->unique(); for (uint i = 1; i < region->req(); i++) { Node *x; @@ -149,15 +158,38 @@ if (x != the_clone && the_clone != NULL) _igvn.remove_dead_node(the_clone); phi->set_req( i, x ); +#if INCLUDE_SHENANDOAHGC + if (mem_phi != NULL) { + if (x == the_clone) { + mem_phi->init_req(i, _igvn.transform(new ShenandoahWBMemProjNode(x))); + } else { + Node* mem = n->in(ShenandoahBarrierNode::Memory); + if (mem->is_Phi() && mem->in(0) == region) { + mem = mem->in(i); + } + mem_phi->init_req(i, mem); + } + } +#endif } // Too few wins? if (wins <= policy) { _igvn.remove_dead_node(phi); +#if INCLUDE_SHENANDOAHGC + if (mem_phi != NULL) { + _igvn.remove_dead_node(mem_phi); + } +#endif return NULL; } // Record Phi register_new_node( phi, region ); +#if INCLUDE_SHENANDOAHGC + if (mem_phi != NULL) { + register_new_node(mem_phi, region); + } +#endif for (uint i2 = 1; i2 < phi->req(); i2++) { Node *x = phi->in(i2); @@ -209,6 +241,25 @@ } } +#if INCLUDE_SHENANDOAHGC + if (mem_phi != NULL) { + for (uint i2 = 1; i2 < mem_phi->req(); i2++) { + Node *x = mem_phi->in(i2); + if (x->_idx >= old_unique) { + assert(x->Opcode() == Op_ShenandoahWBMemProj, "only for shenandoah memory"); + set_ctrl(x, get_ctrl(x->in(ShenandoahWBMemProjNode::WriteBarrier))); + IdealLoopTree* loop = get_loop(get_ctrl(x)); + if (loop->_child == NULL) { + loop->_body.push(x); + } + + } + } + } + if (mem_phi != NULL) { + *out_mem_phi = mem_phi; + } +#endif return phi; } @@ -918,6 +969,12 @@ return n; } +#if INCLUDE_SHENANDOAHGC + if (n->Opcode() == Op_ShenandoahReadBarrier) { + ((ShenandoahReadBarrierNode*)n)->try_move(n_ctrl, this); + } +#endif + // Attempt to remix address expressions for loop invariants Node *m = remix_address_expressions( n ); if( m ) return m; @@ -945,7 +1002,7 @@ // Check for having no control input; not pinned. Allow // dominating control. - if (n->in(0)) { + if (n->in(0) && !(n->Opcode() == Op_ShenandoahWriteBarrier && n->in(0) == n_blk)) { Node *dom = idom(n_blk); if (dom_lca(n->in(0), dom) != n->in(0)) { return n; @@ -970,17 +1027,27 @@ if( C->live_nodes() > 35000 ) return n; // Method too big // Split 'n' through the merge point if it is profitable - Node *phi = split_thru_phi( n, n_blk, policy ); + Node* mem_phi = NULL; + Node *phi = split_thru_phi(n, n_blk, policy, &mem_phi); if (!phi) return n; // Found a Phi to split thru! // Replace 'n' with the new phi - _igvn.replace_node( n, phi ); + if (n->Opcode() == Op_ShenandoahWriteBarrier) { + Node* proj = n->find_out_with(Op_ShenandoahWBMemProj); + _igvn.replace_node(proj, mem_phi); + } + _igvn.replace_node(n, phi); // Moved a load around the loop, 'en-registering' something. if (n_blk->is_Loop() && n->is_Load() && !phi->in(LoopNode::LoopBackControl)->is_Load()) C->set_major_progress(); + // Moved a barrier around the loop, 'en-registering' something. + if (n_blk->is_Loop() && n->is_ShenandoahBarrier() && + !phi->in(LoopNode::LoopBackControl)->is_ShenandoahBarrier()) + C->set_major_progress(); + return phi; } @@ -1019,6 +1086,11 @@ Node* m = n->fast_out(j); if (m->is_FastLock()) return false; +#if INCLUDE_SHENANDOAHGC + if (m->is_ShenandoahBarrier() && m->has_out_with(Op_FastLock)) { + return false; + } +#endif #ifdef _LP64 if (m->Opcode() == Op_ConvI2L) return false; @@ -1335,7 +1407,7 @@ // For inner loop uses get the preheader area. x_ctrl = place_near_use(x_ctrl); - if (n->is_Load()) { + if (n->is_Load() || n->Opcode() == Op_ShenandoahReadBarrier) { // For loads, add a control edge to a CFG node outside of the loop // to force them to not combine and return back inside the loop // during GVN optimization (4641526). @@ -1343,7 +1415,9 @@ // Because we are setting the actual control input, factor in // the result from get_late_ctrl() so we respect any // anti-dependences. (6233005). - x_ctrl = dom_lca(late_load_ctrl, x_ctrl); + if (n->is_Load()) { + x_ctrl = dom_lca(late_load_ctrl, x_ctrl); + } // Don't allow the control input to be a CFG splitting node. // Such nodes should only have ProjNodes as outs, e.g. IfNode @@ -1365,7 +1439,7 @@ // to fold a StoreP and an AddP together (as part of an // address expression) and the AddP and StoreP have // different controls. - if (!x->is_Load() && !x->is_DecodeNarrowPtr()) _igvn._worklist.yank(x); + if (!x->is_Load() && !x->is_DecodeNarrowPtr() && !x->is_ShenandoahBarrier() && !x->is_MergeMem()) _igvn._worklist.yank(x); } _igvn.remove_dead_node(n); } @@ -3132,7 +3206,7 @@ // if not pinned and not a load (which maybe anti-dependent on a store) // and not a CMove (Matcher expects only bool->cmove). - if ( n->in(0) == NULL && !n->is_Load() && !n->is_CMove() ) { + if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove() && n->Opcode() != Op_ShenandoahWBMemProj) { cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist ); sink_list.push(n); peel >>= n->_idx; // delete n from peel set.