src/share/vm/opto/loopopts.cpp
Index
Unified diffs
Context diffs
Sdiffs
Wdiffs
Patch
New
Old
Previous File
Next File
*** old/src/share/vm/opto/loopopts.cpp Fri Oct 21 15:43:58 2011
--- new/src/share/vm/opto/loopopts.cpp Fri Oct 21 15:43:58 2011
*** 26,35 ****
--- 26,36 ----
#include "memory/allocation.inline.hpp"
#include "opto/addnode.hpp"
#include "opto/connode.hpp"
#include "opto/divnode.hpp"
#include "opto/loopnode.hpp"
+ #include "opto/matcher.hpp"
#include "opto/mulnode.hpp"
#include "opto/rootnode.hpp"
#include "opto/subnode.hpp"
//=============================================================================
*** 470,519 ****
--- 471,524 ----
// of the CFG diamond is now speculatively executed. This code has to be
// "cheap enough". We are pretty much limited to CFG diamonds that merge
// 1 or 2 items with a total of 1 or 2 ops executed speculatively.
Node *PhaseIdealLoop::conditional_move( Node *region ) {
! assert( region->is_Region(), "sanity check" );
! if( region->req() != 3 ) return NULL;
! if (region->req() != 3) return NULL;
// Check for CFG diamond
Node *lp = region->in(1);
Node *rp = region->in(2);
! if( !lp || !rp ) return NULL;
! if (!lp || !rp) return NULL;
Node *lp_c = lp->in(0);
! if( lp_c == NULL || lp_c != rp->in(0) || !lp_c->is_If() ) return NULL;
! if (lp_c == NULL || lp_c != rp->in(0) || !lp_c->is_If()) return NULL;
IfNode *iff = lp_c->as_If();
// Check for highly predictable branch. No point in CMOV'ing if
// we are going to predict accurately all the time.
// %%% This hides patterns produced by utility methods like Math.min.
if( iff->_prob < PROB_UNLIKELY_MAG(3) ||
iff->_prob > PROB_LIKELY_MAG(3) )
return NULL;
// Check for ops pinned in an arm of the diamond.
// Can't remove the control flow in this case
! if( lp->outcnt() > 1 ) return NULL;
! if( rp->outcnt() > 1 ) return NULL;
! if (lp->outcnt() > 1) return NULL;
! if (rp->outcnt() > 1) return NULL;
+ IdealLoopTree* r_loop = get_loop(region);
+ assert(r_loop == get_loop(iff), "sanity");
+ // Always convert to CMOVE if all results are used only outside this loop.
+ bool used_inside_loop = (r_loop == _ltree_root);
+
// Check profitability
int cost = 0;
int phis = 0;
for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
Node *out = region->fast_out(i);
! if( !out->is_Phi() ) continue; // Ignore other control edges, etc
! if (!out->is_Phi()) continue; // Ignore other control edges, etc
phis++;
PhiNode* phi = out->as_Phi();
switch (phi->type()->basic_type()) {
case T_LONG:
cost++; // Probably encodes as 2 CMOV's
case T_INT: // These all CMOV fine
+ BasicType bt = phi->type()->basic_type();
+ switch (bt) {
case T_FLOAT:
! case T_DOUBLE: {
case T_ADDRESS: // (RawPtr)
+ cost += Matcher::float_cmove_cost(); // Could be very expensive
+ break;
+ }
+ case T_LONG: {
+ cost += Matcher::long_cmove_cost(); // May encodes as 2 CMOV's
+ }
+ case T_INT: // These all CMOV fine
+ case T_ADDRESS: { // (RawPtr)
cost++;
break;
+ }
case T_NARROWOOP: // Fall through
case T_OBJECT: { // Base oops are OK, but not derived oops
const TypeOopPtr *tp = phi->type()->make_ptr()->isa_oopptr();
// Derived pointers are Bad (tm): what's the Base (for GC purposes) of a
// CMOVE'd derived pointer? It's a CMOVE'd derived base. Thus
*** 522,573 ****
--- 527,603 ----
// and good. But if the base is dead, we'll not make a CMOVE. Later
// the allocator will have to produce a base by creating a CMOVE of the
// relevant bases. This puts the allocator in the business of
// manufacturing expensive instructions, generally a bad plan.
// Just Say No to Conditionally-Moved Derived Pointers.
! if( tp && tp->offset() != 0 )
! if (tp && tp->offset() != 0)
return NULL;
cost++;
break;
}
default:
return NULL; // In particular, can't do memory or I/O
}
// Add in cost any speculative ops
! for( uint j = 1; j < region->req(); j++ ) {
! for (uint j = 1; j < region->req(); j++) {
Node *proj = region->in(j);
Node *inp = phi->in(j);
if (get_ctrl(inp) == proj) { // Found local op
cost++;
// Check for a chain of dependent ops; these will all become
// speculative in a CMOV.
! for( uint k = 1; k < inp->req(); k++ )
! for (uint k = 1; k < inp->req(); k++)
if (get_ctrl(inp->in(k)) == proj)
! return NULL; // Too much speculative goo
! cost += ConditionalMoveLimit; // Too much speculative goo
}
}
// See if the Phi is used by a Cmp or Narrow oop Decode/Encode.
// This will likely Split-If, a higher-payoff operation.
for (DUIterator_Fast kmax, k = phi->fast_outs(kmax); k < kmax; k++) {
Node* use = phi->fast_out(k);
! if( use->is_Cmp() || use->is_DecodeN() || use->is_EncodeP() )
! return NULL;
! if (use->is_Cmp() || use->is_DecodeN() || use->is_EncodeP())
! cost += ConditionalMoveLimit;
+ // Is there a use inside the loop?
+ // Note: check only basic types since CMoveP is pinned.
+ if (!used_inside_loop && is_java_primitive(bt)) {
+ IdealLoopTree* u_loop = get_loop(has_ctrl(use) ? get_ctrl(use) : use);
+ if (r_loop == u_loop || r_loop->is_member(u_loop)) {
+ used_inside_loop = true;
}
}
if( cost >= ConditionalMoveLimit ) return NULL; // Too much goo
+ }
+ }
Node* bol = iff->in(1);
! assert( bol->Opcode() == Op_Bool, "" );
int cmp_op = bol->in(1)->Opcode();
// It is expensive to generate flags from a float compare.
// Avoid duplicated float compare.
! if( phis > 1 && (cmp_op == Op_CmpF || cmp_op == Op_CmpD)) return NULL;
! if (phis > 1 && (cmp_op == Op_CmpF || cmp_op == Op_CmpD)) return NULL;
+ float infrequent_prob = PROB_UNLIKELY_MAG(3);
+ // Ignore cost and blocks frequency if CMOVE can be moved outside the loop.
+ if (used_inside_loop) {
+ if (cost >= ConditionalMoveLimit) return NULL; // Too much goo
+
+ // BlockLayoutByFrequency optimization moves infrequent branch
+ // from hot path. No point in CMOV'ing in such case (110 is used
+ // instead of 100 to take into account not exactness of float value).
+ if (BlockLayoutByFrequency) {
+ infrequent_prob = MAX2(infrequent_prob, (float)BlockLayoutMinDiamondPercentage/110.0f);
+ }
+ }
+ // Check for highly predictable branch. No point in CMOV'ing if
+ // we are going to predict accurately all the time.
+ if (iff->_prob < infrequent_prob ||
+ iff->_prob > (1.0f - infrequent_prob))
+ return NULL;
+
// --------------
// Now replace all Phis with CMOV's
Node *cmov_ctrl = iff->in(0);
uint flip = (lp->Opcode() == Op_IfTrue);
! while( 1 ) {
! while (1) {
PhiNode* phi = NULL;
for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
Node *out = region->fast_out(i);
if (out->is_Phi()) {
phi = out->as_Phi();
*** 574,592 ****
--- 604,622 ----
break;
}
}
if (phi == NULL) break;
#ifndef PRODUCT
! if( PrintOpto && VerifyLoopOptimizations ) tty->print_cr("CMOV");
! if (PrintOpto && VerifyLoopOptimizations) tty->print_cr("CMOV");
#endif
// Move speculative ops
! for( uint j = 1; j < region->req(); j++ ) {
! for (uint j = 1; j < region->req(); j++) {
Node *proj = region->in(j);
Node *inp = phi->in(j);
if (get_ctrl(inp) == proj) { // Found local op
#ifndef PRODUCT
! if( PrintOpto && VerifyLoopOptimizations ) {
! if (PrintOpto && VerifyLoopOptimizations) {
tty->print(" speculate: ");
inp->dump();
}
#endif
set_ctrl(inp, cmov_ctrl);
*** 594,604 ****
--- 624,641 ----
}
Node *cmov = CMoveNode::make( C, cmov_ctrl, iff->in(1), phi->in(1+flip), phi->in(2-flip), _igvn.type(phi) );
register_new_node( cmov, cmov_ctrl );
_igvn.replace_node( phi, cmov );
#ifndef PRODUCT
! if( VerifyLoopOptimizations ) verify();
! if (TraceLoopOpts) {
+ tty->print("CMOV ");
+ r_loop->dump_head();
+ if (Verbose)
+ bol->in(1)->dump(1);
+ cmov->dump(1);
+ }
+ if (VerifyLoopOptimizations) verify();
#endif
}
// The useless CFG diamond will fold up later; see the optimization in
// RegionNode::Ideal.
*** 674,691 ****
--- 711,728 ----
// Use same limit as split_if_with_blocks_post
if( C->unique() > 35000 ) return n; // Method too big
// Split 'n' through the merge point if it is profitable
Node *phi = split_thru_phi( n, n_blk, policy );
! if( !phi ) return n;
! if (!phi) return n;
// Found a Phi to split thru!
// Replace 'n' with the new phi
_igvn.replace_node( n, phi );
// Moved a load around the loop, 'en-registering' something.
! if( n_blk->Opcode() == Op_Loop && n->is_Load() &&
- !phi->in(LoopNode::LoopBackControl)->is_Load() )
! if (n_blk->is_Loop() && n->is_Load() &&
! !phi->in(LoopNode::LoopBackControl)->is_Load())
C->set_major_progress();
return phi;
}
src/share/vm/opto/loopopts.cpp
Index
Unified diffs
Context diffs
Sdiffs
Wdiffs
Patch
New
Old
Previous File
Next File