< prev index next >
src/share/vm/opto/loopopts.cpp
Print this page
*** 31,40 ****
--- 31,41 ----
#include "opto/matcher.hpp"
#include "opto/mulnode.hpp"
#include "opto/movenode.hpp"
#include "opto/opaquenode.hpp"
#include "opto/rootnode.hpp"
+ #include "opto/shenandoahSupport.hpp"
#include "opto/subnode.hpp"
//=============================================================================
//------------------------------split_thru_phi---------------------------------
// Split Node 'n' through merge point if there is enough win.
*** 65,74 ****
--- 66,78 ----
Node *x;
Node* the_clone = NULL;
if (region->in(i) == C->top()) {
x = C->top(); // Dead path? Use a dead data op
} else {
+ #ifdef ASSERT
+ if (n->is_ShenandoahBarrier()) { n->as_ShenandoahBarrier()->check_invariants(); }
+ #endif
x = n->clone(); // Else clone up the data op
the_clone = x; // Remember for possible deletion.
// Alter data node to use pre-phi inputs
if (n->in(0) == region)
x->set_req( 0, region->in(i) );
*** 109,118 ****
--- 113,123 ----
_igvn.set_type(x, t);
// If x is a TypeNode, capture any more-precise type permanently into Node
// otherwise it will be not updated during igvn->transform since
// igvn->type(x) is set to x->Value() already.
x->raise_bottom_type(t);
+ if (x->Opcode() != Op_ShenandoahWriteBarrier) {
Node *y = x->Identity(&_igvn);
if (y != x) {
wins++;
x = y;
} else {
*** 125,134 ****
--- 130,143 ----
// We do not need register_new_node_with_optimizer
// because set_type has already been called.
_igvn._worklist.push(x);
}
}
+ } else {
+ // wins++;
+ _igvn._worklist.push(x);
+ }
}
if (x != the_clone && the_clone != NULL)
_igvn.remove_dead_node(the_clone);
phi->set_req( i, x );
}
*** 192,201 ****
--- 201,242 ----
}
return phi;
}
+ void PhaseIdealLoop::split_mem_thru_phi(Node* n, Node* r, Node* phi) {
+ if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+ #ifdef ASSERT
+ n->as_ShenandoahBarrier()->check_invariants();
+ #endif
+ if (n->has_out_with(Op_ShenandoahWBMemProj)) {
+ Node* old_mem_phi = n->in(ShenandoahBarrierNode::Memory);
+ assert(r->is_Region(), "need region to control phi");
+ assert(phi->is_Phi(), "expect phi");
+ Node* memphi = PhiNode::make(r, old_mem_phi, Type::MEMORY, C->alias_type(n->adr_type())->adr_type());
+ for (uint i = 1; i < r->req(); i++) {
+ Node* wb = phi->in(i);
+ if (wb->Opcode() == Op_ShenandoahWriteBarrier) {
+ assert(! wb->has_out_with(Op_ShenandoahWBMemProj), "new clone does not have mem proj");
+ Node* new_proj = new ShenandoahWBMemProjNode(wb);
+ register_new_node(new_proj, r->in(i));
+ memphi->set_req(i, new_proj);
+ } else {
+ if (old_mem_phi->is_Phi() && old_mem_phi->in(0) == r) {
+ memphi->set_req(i, old_mem_phi->in(i));
+ }
+ }
+ }
+ register_new_node(memphi, r);
+ Node* old_mem_out = n->find_out_with(Op_ShenandoahWBMemProj);
+ assert(old_mem_out != NULL, "expect memory projection");
+ _igvn.replace_node(old_mem_out, memphi);
+ }
+ assert(! n->has_out_with(Op_ShenandoahWBMemProj), "no more memory outs");
+ }
+ }
+
//------------------------------dominated_by------------------------------------
// Replace the dominated test with an obvious true or false. Place it on the
// IGVN worklist for later cleanup. Move control-dependent data Nodes on the
// live path up to the dominating control.
void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff, bool flip, bool exclude_loop_predicate ) {
*** 932,947 ****
--- 973,994 ----
Node *phi = split_thru_phi( n, n_blk, policy );
if (!phi) return n;
// Found a Phi to split thru!
// Replace 'n' with the new phi
+ split_mem_thru_phi(n, n_blk, phi);
_igvn.replace_node( n, phi );
// Moved a load around the loop, 'en-registering' something.
if (n_blk->is_Loop() && n->is_Load() &&
!phi->in(LoopNode::LoopBackControl)->is_Load())
C->set_major_progress();
+ // Moved a barrier around the loop, 'en-registering' something.
+ if (n_blk->is_Loop() && n->is_ShenandoahBarrier() &&
+ !phi->in(LoopNode::LoopBackControl)->is_ShenandoahBarrier())
+ C->set_major_progress();
+
return phi;
}
static bool merge_point_too_heavy(Compile* C, Node* region) {
// Bail out if the region and its phis have too many users.
*** 1080,1092 ****
--- 1127,1141 ----
Node *phi = split_thru_phi( n, n_ctrl, policy );
if( !phi ) return;
// Found a Phi to split thru!
// Replace 'n' with the new phi
+ assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet");
_igvn.replace_node( n, phi );
// Now split the bool up thru the phi
+ assert(bol->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet");
Node *bolphi = split_thru_phi( bol, n_ctrl, -1 );
guarantee(bolphi != NULL, "null boolean phi node");
_igvn.replace_node( bol, bolphi );
assert( iff->in(1) == bolphi, "" );
*** 1094,1103 ****
--- 1143,1153 ----
if( bolphi->Value(&_igvn)->singleton() )
return;
// Conditional-move? Must split up now
if( !iff->is_If() ) {
+ assert(iff->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet");
Node *cmovphi = split_thru_phi( iff, n_ctrl, -1 );
_igvn.replace_node( iff, cmovphi );
return;
}
*** 1227,1237 ****
// tried setting control edges on the x's to force them to
// not combine, but the matching gets worried when it tries
// to fold a StoreP and an AddP together (as part of an
// address expression) and the AddP and StoreP have
// different controls.
! if (!x->is_Load() && !x->is_DecodeNarrowPtr()) _igvn._worklist.yank(x);
}
_igvn.remove_dead_node(n);
}
}
}
--- 1277,1287 ----
// tried setting control edges on the x's to force them to
// not combine, but the matching gets worried when it tries
// to fold a StoreP and an AddP together (as part of an
// address expression) and the AddP and StoreP have
// different controls.
! if (!x->is_Load() && !x->is_DecodeNarrowPtr() && !x->is_ShenandoahBarrier()) _igvn._worklist.yank(x);
}
_igvn.remove_dead_node(n);
}
}
}
*** 1500,1509 ****
--- 1550,1569 ----
// Step 1: Clone the loop body. Make the old->new mapping.
uint i;
for( i = 0; i < loop->_body.size(); i++ ) {
Node *old = loop->_body.at(i);
+ #ifdef ASSERT
+ if (old->Opcode() == Op_ShenandoahWriteBarrier) {
+ Node* memproj = old->find_out_with(Op_ShenandoahWBMemProj);
+ assert(memproj == NULL || (loop->is_member(get_loop(has_ctrl(memproj) ? get_ctrl(memproj) : memproj))), "WB's mem-proj must be in loop too");
+ }
+ if (old->Opcode() == Op_ShenandoahWBMemProj) {
+ Node* wb = old->in(0);
+ assert(loop->is_member(get_loop(has_ctrl(wb) ? get_ctrl(wb) : wb)), "WB must be in loop too");
+ }
+ #endif
Node *nnn = old->clone();
old_new.map( old->_idx, nnn );
if (C->do_vector_loop()) {
cm.verify_insert_and_clone(old, nnn, cm.clone_idx());
}
*** 1690,1713 ****
// merge point. NOW we do need a Phi here. Each loop exit value
// is now merged with the peeled body exit; each exit gets its own
// private Phi and those Phis need to be merged here.
Node *phi;
if( prev->is_Region() ) {
! if( idx == 0 ) { // Updating control edge?
phi = prev; // Just use existing control
} else { // Else need a new Phi
phi = PhiNode::make( prev, old );
// Now recursively fix up the new uses of old!
! for( uint i = 1; i < prev->req(); i++ ) {
worklist.push(phi); // Onto worklist once for each 'old' input
}
}
} else {
// Get new RegionNode merging old and new loop exits
prev = old_new[prev->_idx];
assert( prev, "just made this in step 7" );
! if( idx == 0 ) { // Updating control edge?
phi = prev; // Just use existing control
} else { // Else need a new Phi
// Make a new Phi merging data values properly
phi = PhiNode::make( prev, old );
phi->set_req( 1, nnn );
--- 1750,1774 ----
// merge point. NOW we do need a Phi here. Each loop exit value
// is now merged with the peeled body exit; each exit gets its own
// private Phi and those Phis need to be merged here.
Node *phi;
if( prev->is_Region() ) {
! if( idx == 0 && use->Opcode() != Op_ShenandoahWBMemProj) { // Updating control edge?
phi = prev; // Just use existing control
} else { // Else need a new Phi
phi = PhiNode::make( prev, old );
// Now recursively fix up the new uses of old!
! uint first = use->Opcode() != Op_ShenandoahWBMemProj ? 1 : 0;
! for( uint i = first; i < prev->req(); i++ ) {
worklist.push(phi); // Onto worklist once for each 'old' input
}
}
} else {
// Get new RegionNode merging old and new loop exits
prev = old_new[prev->_idx];
assert( prev, "just made this in step 7" );
! if( idx == 0 && use->Opcode() != Op_ShenandoahWBMemProj) { // Updating control edge?
phi = prev; // Just use existing control
} else { // Else need a new Phi
// Make a new Phi merging data values properly
phi = PhiNode::make( prev, old );
phi->set_req( 1, nnn );
*** 2186,2195 ****
--- 2247,2257 ----
//------------------------------ clone_for_use_outside_loop -------------------------------------
// clone "n" for uses that are outside of loop
int PhaseIdealLoop::clone_for_use_outside_loop( IdealLoopTree *loop, Node* n, Node_List& worklist ) {
int cloned = 0;
assert(worklist.size() == 0, "should be empty");
+ assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers yet");
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* use = n->fast_out(j);
if( !loop->is_member(get_loop(has_ctrl(use) ? get_ctrl(use) : use)) ) {
worklist.push(use);
}
*** 2202,2211 ****
--- 2264,2274 ----
if (use->in(j) == n) break;
}
assert(j < use->req(), "must be there");
// clone "n" and insert it between the inputs of "n" and the use outside the loop
+ assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers");
Node* n_clone = n->clone();
_igvn.replace_input_of(use, j, n_clone);
cloned++;
Node* use_c;
if (!use->is_Phi()) {
*** 2248,2257 ****
--- 2311,2321 ----
worklist.push(use);
}
}
if (worklist.size() > 0) {
// clone "n" and insert it between inputs of "n" and the use
+ assert(n->Opcode() != Op_ShenandoahWriteBarrier, "not with write barriers");
Node* n_clone = n->clone();
loop->_body.push(n_clone);
_igvn.register_new_node_with_optimizer(n_clone);
set_ctrl(n_clone, get_ctrl(n));
sink_list.push(n_clone);
< prev index next >