< prev index next >
src/hotspot/share/opto/loopopts.cpp
Print this page
*** 42,56 ****
#include "opto/subnode.hpp"
#include "utilities/macros.hpp"
#if INCLUDE_ZGC
#include "gc/z/c2/zBarrierSetC2.hpp"
#endif
//=============================================================================
//------------------------------split_thru_phi---------------------------------
// Split Node 'n' through merge point if there is enough win.
! Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
if (n->Opcode() == Op_ConvI2L && n->bottom_type() != TypeLong::LONG) {
// ConvI2L may have type information on it which is unsafe to push up
// so disable this for now
return NULL;
}
--- 42,59 ----
#include "opto/subnode.hpp"
#include "utilities/macros.hpp"
#if INCLUDE_ZGC
#include "gc/z/c2/zBarrierSetC2.hpp"
#endif
+ #if INCLUDE_SHENANDOAHGC
+ #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
+ #endif
//=============================================================================
//------------------------------split_thru_phi---------------------------------
// Split Node 'n' through merge point if there is enough win.
! Node *PhaseIdealLoop::split_thru_phi(Node *n, Node *region, int policy, Node** out_mem_phi) {
if (n->Opcode() == Op_ConvI2L && n->bottom_type() != TypeLong::LONG) {
// ConvI2L may have type information on it which is unsafe to push up
// so disable this for now
return NULL;
}
*** 76,85 ****
--- 79,94 ----
int offset = t_oop->offset();
phi = new PhiNode(region, type, NULL, iid, index, offset);
} else {
phi = PhiNode::make_blank(region, n);
}
+ Node* mem_phi = NULL;
+ #if INCLUDE_SHENANDOAHGC
+ if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+ mem_phi = new PhiNode(region, Type::MEMORY, C->alias_type(n->adr_type())->adr_type());
+ }
+ #endif
uint old_unique = C->unique();
for (uint i = 1; i < region->req(); i++) {
Node *x;
Node* the_clone = NULL;
if (region->in(i) == C->top()) {
*** 147,165 ****
--- 156,197 ----
}
}
if (x != the_clone && the_clone != NULL)
_igvn.remove_dead_node(the_clone);
phi->set_req( i, x );
+ #if INCLUDE_SHENANDOAHGC
+ if (mem_phi != NULL) {
+ if (x == the_clone) {
+ mem_phi->init_req(i, _igvn.transform(new ShenandoahWBMemProjNode(x)));
+ } else {
+ Node* mem = n->in(ShenandoahBarrierNode::Memory);
+ if (mem->is_Phi() && mem->in(0) == region) {
+ mem = mem->in(i);
+ }
+ mem_phi->init_req(i, mem);
+ }
+ }
+ #endif
}
// Too few wins?
if (wins <= policy) {
_igvn.remove_dead_node(phi);
+ #if INCLUDE_SHENANDOAHGC
+ if (mem_phi != NULL) {
+ _igvn.remove_dead_node(mem_phi);
+ }
+ #endif
return NULL;
}
// Record Phi
register_new_node( phi, region );
+ #if INCLUDE_SHENANDOAHGC
+ if (mem_phi != NULL) {
+ register_new_node(mem_phi, region);
+ }
+ #endif
for (uint i2 = 1; i2 < phi->req(); i2++) {
Node *x = phi->in(i2);
// If we commoned up the cloned 'x' with another existing Node,
// the existing Node picks up a new use. We need to make the
*** 207,216 ****
--- 239,267 ----
if (!new_loop->_child)
new_loop->_body.push(x); // Collect body info
}
}
+ #if INCLUDE_SHENANDOAHGC
+ if (mem_phi != NULL) {
+ for (uint i2 = 1; i2 < mem_phi->req(); i2++) {
+ Node *x = mem_phi->in(i2);
+ if (x->_idx >= old_unique) {
+ assert(x->Opcode() == Op_ShenandoahWBMemProj, "only for shenandoah memory");
+ set_ctrl(x, get_ctrl(x->in(ShenandoahWBMemProjNode::WriteBarrier)));
+ IdealLoopTree* loop = get_loop(get_ctrl(x));
+ if (loop->_child == NULL) {
+ loop->_body.push(x);
+ }
+
+ }
+ }
+ }
+ if (mem_phi != NULL) {
+ *out_mem_phi = mem_phi;
+ }
+ #endif
return phi;
}
//------------------------------dominated_by------------------------------------
// Replace the dominated test with an obvious true or false. Place it on the
*** 916,925 ****
--- 967,982 ----
Node* res = try_move_store_before_loop(n, n_ctrl);
if (res != NULL) {
return n;
}
+ #if INCLUDE_SHENANDOAHGC
+ if (n->Opcode() == Op_ShenandoahReadBarrier) {
+ ((ShenandoahReadBarrierNode*)n)->try_move(n_ctrl, this);
+ }
+ #endif
+
// Attempt to remix address expressions for loop invariants
Node *m = remix_address_expressions( n );
if( m ) return m;
if (n->is_ConstraintCast()) {
*** 943,953 ****
// (messes up the canonical shape).
if( n_blk->is_CountedLoop() && n->Opcode() == Op_AddI ) return n;
// Check for having no control input; not pinned. Allow
// dominating control.
! if (n->in(0)) {
Node *dom = idom(n_blk);
if (dom_lca(n->in(0), dom) != n->in(0)) {
return n;
}
}
--- 1000,1010 ----
// (messes up the canonical shape).
if( n_blk->is_CountedLoop() && n->Opcode() == Op_AddI ) return n;
// Check for having no control input; not pinned. Allow
// dominating control.
! if (n->in(0) && !(n->Opcode() == Op_ShenandoahWriteBarrier && n->in(0) == n_blk)) {
Node *dom = idom(n_blk);
if (dom_lca(n->in(0), dom) != n->in(0)) {
return n;
}
}
*** 968,988 ****
// Use same limit as split_if_with_blocks_post
if( C->live_nodes() > 35000 ) return n; // Method too big
// Split 'n' through the merge point if it is profitable
! Node *phi = split_thru_phi( n, n_blk, policy );
if (!phi) return n;
// Found a Phi to split thru!
// Replace 'n' with the new phi
! _igvn.replace_node( n, phi );
// Moved a load around the loop, 'en-registering' something.
if (n_blk->is_Loop() && n->is_Load() &&
!phi->in(LoopNode::LoopBackControl)->is_Load())
C->set_major_progress();
return phi;
}
static bool merge_point_too_heavy(Compile* C, Node* region) {
// Bail out if the region and its phis have too many users.
--- 1025,1055 ----
// Use same limit as split_if_with_blocks_post
if( C->live_nodes() > 35000 ) return n; // Method too big
// Split 'n' through the merge point if it is profitable
! Node* mem_phi = NULL;
! Node *phi = split_thru_phi(n, n_blk, policy, &mem_phi);
if (!phi) return n;
// Found a Phi to split thru!
// Replace 'n' with the new phi
! if (n->Opcode() == Op_ShenandoahWriteBarrier) {
! Node* proj = n->find_out_with(Op_ShenandoahWBMemProj);
! _igvn.replace_node(proj, mem_phi);
! }
! _igvn.replace_node(n, phi);
// Moved a load around the loop, 'en-registering' something.
if (n_blk->is_Loop() && n->is_Load() &&
!phi->in(LoopNode::LoopBackControl)->is_Load())
C->set_major_progress();
+ // Moved a barrier around the loop, 'en-registering' something.
+ if (n_blk->is_Loop() && n->is_ShenandoahBarrier() &&
+ !phi->in(LoopNode::LoopBackControl)->is_ShenandoahBarrier())
+ C->set_major_progress();
+
return phi;
}
static bool merge_point_too_heavy(Compile* C, Node* region) {
// Bail out if the region and its phis have too many users.
*** 1017,1026 ****
--- 1084,1098 ----
if (n->is_Phi()) {
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* m = n->fast_out(j);
if (m->is_FastLock())
return false;
+ #if INCLUDE_SHENANDOAHGC
+ if (m->is_ShenandoahBarrier() && m->has_out_with(Op_FastLock)) {
+ return false;
+ }
+ #endif
#ifdef _LP64
if (m->Opcode() == Op_ConvI2L)
return false;
if (m->is_CastII() && m->isa_CastII()->has_range_check()) {
return false;
*** 1333,1351 ****
// Find control for 'x' next to use but not inside inner loops.
// For inner loop uses get the preheader area.
x_ctrl = place_near_use(x_ctrl);
! if (n->is_Load()) {
// For loads, add a control edge to a CFG node outside of the loop
// to force them to not combine and return back inside the loop
// during GVN optimization (4641526).
//
// Because we are setting the actual control input, factor in
// the result from get_late_ctrl() so we respect any
// anti-dependences. (6233005).
! x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
// Don't allow the control input to be a CFG splitting node.
// Such nodes should only have ProjNodes as outs, e.g. IfNode
// should only have IfTrueNode and IfFalseNode (4985384).
x_ctrl = find_non_split_ctrl(x_ctrl);
--- 1405,1425 ----
// Find control for 'x' next to use but not inside inner loops.
// For inner loop uses get the preheader area.
x_ctrl = place_near_use(x_ctrl);
! if (n->is_Load() || n->Opcode() == Op_ShenandoahReadBarrier) {
// For loads, add a control edge to a CFG node outside of the loop
// to force them to not combine and return back inside the loop
// during GVN optimization (4641526).
//
// Because we are setting the actual control input, factor in
// the result from get_late_ctrl() so we respect any
// anti-dependences. (6233005).
! if (n->is_Load()) {
! x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
! }
// Don't allow the control input to be a CFG splitting node.
// Such nodes should only have ProjNodes as outs, e.g. IfNode
// should only have IfTrueNode and IfFalseNode (4985384).
x_ctrl = find_non_split_ctrl(x_ctrl);
*** 1363,1373 ****
// tried setting control edges on the x's to force them to
// not combine, but the matching gets worried when it tries
// to fold a StoreP and an AddP together (as part of an
// address expression) and the AddP and StoreP have
// different controls.
! if (!x->is_Load() && !x->is_DecodeNarrowPtr()) _igvn._worklist.yank(x);
}
_igvn.remove_dead_node(n);
}
}
}
--- 1437,1447 ----
// tried setting control edges on the x's to force them to
// not combine, but the matching gets worried when it tries
// to fold a StoreP and an AddP together (as part of an
// address expression) and the AddP and StoreP have
// different controls.
! if (!x->is_Load() && !x->is_DecodeNarrowPtr() && !x->is_ShenandoahBarrier() && !x->is_MergeMem()) _igvn._worklist.yank(x);
}
_igvn.remove_dead_node(n);
}
}
}
*** 3130,3140 ****
if ( !has_use_internal_to_set(n, peel, loop) ) {
// if not pinned and not a load (which maybe anti-dependent on a store)
// and not a CMove (Matcher expects only bool->cmove).
! if ( n->in(0) == NULL && !n->is_Load() && !n->is_CMove() ) {
cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist );
sink_list.push(n);
peel >>= n->_idx; // delete n from peel set.
not_peel <<= n->_idx; // add n to not_peel set.
peel_list.remove(i);
--- 3204,3214 ----
if ( !has_use_internal_to_set(n, peel, loop) ) {
// if not pinned and not a load (which maybe anti-dependent on a store)
// and not a CMove (Matcher expects only bool->cmove).
! if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove() && n->Opcode() != Op_ShenandoahWBMemProj) {
cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist );
sink_list.push(n);
peel >>= n->_idx; // delete n from peel set.
not_peel <<= n->_idx; // add n to not_peel set.
peel_list.remove(i);
< prev index next >