src/share/vm/opto/loopopts.cpp
Index Unified diffs Context diffs Sdiffs Wdiffs Patch New Old Previous File Next File
*** old/src/share/vm/opto/loopopts.cpp	Fri Oct 21 15:43:58 2011
--- new/src/share/vm/opto/loopopts.cpp	Fri Oct 21 15:43:58 2011

*** 26,35 **** --- 26,36 ---- #include "memory/allocation.inline.hpp" #include "opto/addnode.hpp" #include "opto/connode.hpp" #include "opto/divnode.hpp" #include "opto/loopnode.hpp" + #include "opto/matcher.hpp" #include "opto/mulnode.hpp" #include "opto/rootnode.hpp" #include "opto/subnode.hpp" //=============================================================================
*** 470,519 **** --- 471,524 ---- // of the CFG diamond is now speculatively executed. This code has to be // "cheap enough". We are pretty much limited to CFG diamonds that merge // 1 or 2 items with a total of 1 or 2 ops executed speculatively. Node *PhaseIdealLoop::conditional_move( Node *region ) { ! assert( region->is_Region(), "sanity check" ); ! if( region->req() != 3 ) return NULL; ! if (region->req() != 3) return NULL; // Check for CFG diamond Node *lp = region->in(1); Node *rp = region->in(2); ! if( !lp || !rp ) return NULL; ! if (!lp || !rp) return NULL; Node *lp_c = lp->in(0); ! if( lp_c == NULL || lp_c != rp->in(0) || !lp_c->is_If() ) return NULL; ! if (lp_c == NULL || lp_c != rp->in(0) || !lp_c->is_If()) return NULL; IfNode *iff = lp_c->as_If(); // Check for highly predictable branch. No point in CMOV'ing if // we are going to predict accurately all the time. // %%% This hides patterns produced by utility methods like Math.min. if( iff->_prob < PROB_UNLIKELY_MAG(3) || iff->_prob > PROB_LIKELY_MAG(3) ) return NULL; // Check for ops pinned in an arm of the diamond. // Can't remove the control flow in this case ! if( lp->outcnt() > 1 ) return NULL; ! if( rp->outcnt() > 1 ) return NULL; ! if (lp->outcnt() > 1) return NULL; ! if (rp->outcnt() > 1) return NULL; + IdealLoopTree* r_loop = get_loop(region); + assert(r_loop == get_loop(iff), "sanity"); + // Always convert to CMOVE if all results are used only outside this loop. + bool used_inside_loop = (r_loop == _ltree_root); + // Check profitability int cost = 0; int phis = 0; for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { Node *out = region->fast_out(i); ! if( !out->is_Phi() ) continue; // Ignore other control edges, etc ! if (!out->is_Phi()) continue; // Ignore other control edges, etc phis++; PhiNode* phi = out->as_Phi(); switch (phi->type()->basic_type()) { case T_LONG: cost++; // Probably encodes as 2 CMOV's case T_INT: // These all CMOV fine + BasicType bt = phi->type()->basic_type(); + switch (bt) { case T_FLOAT: ! case T_DOUBLE: { case T_ADDRESS: // (RawPtr) + cost += Matcher::float_cmove_cost(); // Could be very expensive + break; + } + case T_LONG: { + cost += Matcher::long_cmove_cost(); // May encodes as 2 CMOV's + } + case T_INT: // These all CMOV fine + case T_ADDRESS: { // (RawPtr) cost++; break; + } case T_NARROWOOP: // Fall through case T_OBJECT: { // Base oops are OK, but not derived oops const TypeOopPtr *tp = phi->type()->make_ptr()->isa_oopptr(); // Derived pointers are Bad (tm): what's the Base (for GC purposes) of a // CMOVE'd derived pointer? It's a CMOVE'd derived base. Thus
*** 522,573 **** --- 527,603 ---- // and good. But if the base is dead, we'll not make a CMOVE. Later // the allocator will have to produce a base by creating a CMOVE of the // relevant bases. This puts the allocator in the business of // manufacturing expensive instructions, generally a bad plan. // Just Say No to Conditionally-Moved Derived Pointers. ! if( tp && tp->offset() != 0 ) ! if (tp && tp->offset() != 0) return NULL; cost++; break; } default: return NULL; // In particular, can't do memory or I/O } // Add in cost any speculative ops ! for( uint j = 1; j < region->req(); j++ ) { ! for (uint j = 1; j < region->req(); j++) { Node *proj = region->in(j); Node *inp = phi->in(j); if (get_ctrl(inp) == proj) { // Found local op cost++; // Check for a chain of dependent ops; these will all become // speculative in a CMOV. ! for( uint k = 1; k < inp->req(); k++ ) ! for (uint k = 1; k < inp->req(); k++) if (get_ctrl(inp->in(k)) == proj) ! return NULL; // Too much speculative goo ! cost += ConditionalMoveLimit; // Too much speculative goo } } // See if the Phi is used by a Cmp or Narrow oop Decode/Encode. // This will likely Split-If, a higher-payoff operation. for (DUIterator_Fast kmax, k = phi->fast_outs(kmax); k < kmax; k++) { Node* use = phi->fast_out(k); ! if( use->is_Cmp() || use->is_DecodeN() || use->is_EncodeP() ) ! return NULL; ! if (use->is_Cmp() || use->is_DecodeN() || use->is_EncodeP()) ! cost += ConditionalMoveLimit; + // Is there a use inside the loop? + // Note: check only basic types since CMoveP is pinned. + if (!used_inside_loop && is_java_primitive(bt)) { + IdealLoopTree* u_loop = get_loop(has_ctrl(use) ? get_ctrl(use) : use); + if (r_loop == u_loop || r_loop->is_member(u_loop)) { + used_inside_loop = true; } } if( cost >= ConditionalMoveLimit ) return NULL; // Too much goo + } + } Node* bol = iff->in(1); ! assert( bol->Opcode() == Op_Bool, "" ); int cmp_op = bol->in(1)->Opcode(); // It is expensive to generate flags from a float compare. // Avoid duplicated float compare. ! if( phis > 1 && (cmp_op == Op_CmpF || cmp_op == Op_CmpD)) return NULL; ! if (phis > 1 && (cmp_op == Op_CmpF || cmp_op == Op_CmpD)) return NULL; + float infrequent_prob = PROB_UNLIKELY_MAG(3); + // Ignore cost and blocks frequency if CMOVE can be moved outside the loop. + if (used_inside_loop) { + if (cost >= ConditionalMoveLimit) return NULL; // Too much goo + + // BlockLayoutByFrequency optimization moves infrequent branch + // from hot path. No point in CMOV'ing in such case (110 is used + // instead of 100 to take into account not exactness of float value). + if (BlockLayoutByFrequency) { + infrequent_prob = MAX2(infrequent_prob, (float)BlockLayoutMinDiamondPercentage/110.0f); + } + } + // Check for highly predictable branch. No point in CMOV'ing if + // we are going to predict accurately all the time. + if (iff->_prob < infrequent_prob || + iff->_prob > (1.0f - infrequent_prob)) + return NULL; + // -------------- // Now replace all Phis with CMOV's Node *cmov_ctrl = iff->in(0); uint flip = (lp->Opcode() == Op_IfTrue); ! while( 1 ) { ! while (1) { PhiNode* phi = NULL; for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { Node *out = region->fast_out(i); if (out->is_Phi()) { phi = out->as_Phi();
*** 574,592 **** --- 604,622 ---- break; } } if (phi == NULL) break; #ifndef PRODUCT ! if( PrintOpto && VerifyLoopOptimizations ) tty->print_cr("CMOV"); ! if (PrintOpto && VerifyLoopOptimizations) tty->print_cr("CMOV"); #endif // Move speculative ops ! for( uint j = 1; j < region->req(); j++ ) { ! for (uint j = 1; j < region->req(); j++) { Node *proj = region->in(j); Node *inp = phi->in(j); if (get_ctrl(inp) == proj) { // Found local op #ifndef PRODUCT ! if( PrintOpto && VerifyLoopOptimizations ) { ! if (PrintOpto && VerifyLoopOptimizations) { tty->print(" speculate: "); inp->dump(); } #endif set_ctrl(inp, cmov_ctrl);
*** 594,604 **** --- 624,641 ---- } Node *cmov = CMoveNode::make( C, cmov_ctrl, iff->in(1), phi->in(1+flip), phi->in(2-flip), _igvn.type(phi) ); register_new_node( cmov, cmov_ctrl ); _igvn.replace_node( phi, cmov ); #ifndef PRODUCT ! if( VerifyLoopOptimizations ) verify(); ! if (TraceLoopOpts) { + tty->print("CMOV "); + r_loop->dump_head(); + if (Verbose) + bol->in(1)->dump(1); + cmov->dump(1); + } + if (VerifyLoopOptimizations) verify(); #endif } // The useless CFG diamond will fold up later; see the optimization in // RegionNode::Ideal.
*** 674,691 **** --- 711,728 ---- // Use same limit as split_if_with_blocks_post if( C->unique() > 35000 ) return n; // Method too big // Split 'n' through the merge point if it is profitable Node *phi = split_thru_phi( n, n_blk, policy ); ! if( !phi ) return n; ! if (!phi) return n; // Found a Phi to split thru! // Replace 'n' with the new phi _igvn.replace_node( n, phi ); // Moved a load around the loop, 'en-registering' something. ! if( n_blk->Opcode() == Op_Loop && n->is_Load() && - !phi->in(LoopNode::LoopBackControl)->is_Load() ) ! if (n_blk->is_Loop() && n->is_Load() && ! !phi->in(LoopNode::LoopBackControl)->is_Load()) C->set_major_progress(); return phi; }

src/share/vm/opto/loopopts.cpp
Index Unified diffs Context diffs Sdiffs Wdiffs Patch New Old Previous File Next File