--- old/src/share/vm/opto/loopnode.cpp Wed Aug 12 15:10:32 2009 +++ new/src/share/vm/opto/loopnode.cpp Wed Aug 12 15:10:31 2009 @@ -1420,13 +1420,12 @@ } //============================================================================= -//------------------------------PhaseIdealLoop--------------------------------- +//----------------------------build_and_optimize------------------------------- // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups. -PhaseIdealLoop::PhaseIdealLoop( PhaseIterGVN &igvn, const PhaseIdealLoop *verify_me, bool do_split_ifs ) - : PhaseTransform(Ideal_Loop), - _igvn(igvn), - _dom_lca_tags(C->comp_arena()) { +void PhaseIdealLoop::build_and_optimize(bool do_split_ifs) { + int old_progress = C->major_progress(); + // Reset major-progress flag for the driver's heuristics C->clear_major_progress(); @@ -1465,18 +1464,20 @@ } // No loops after all - if( !_ltree_root->_child ) C->set_has_loops(false); + if( !_ltree_root->_child && !_verify_only ) C->set_has_loops(false); // There should always be an outer loop containing the Root and Return nodes. // If not, we have a degenerate empty program. Bail out in this case. if (!has_node(C->root())) { - C->clear_major_progress(); - C->record_method_not_compilable("empty program detected during loop optimization"); + if (!_verify_only) { + C->clear_major_progress(); + C->record_method_not_compilable("empty program detected during loop optimization"); + } return; } // Nothing to do, so get out - if( !C->has_loops() && !do_split_ifs && !verify_me) { + if( !C->has_loops() && !do_split_ifs && !_verify_me && !_verify_only ) { _igvn.optimize(); // Cleanup NeverBranches return; } @@ -1486,7 +1487,7 @@ // Split shared headers and insert loop landing pads. // Do not bother doing this on the Root loop of course. - if( !verify_me && _ltree_root->_child ) { + if( !_verify_me && !_verify_only && _ltree_root->_child ) { if( _ltree_root->_child->beautify_loops( this ) ) { // Re-build loop tree! _ltree_root->_child = NULL; @@ -1515,24 +1516,26 @@ Dominators(); - // As a side effect, Dominators removed any unreachable CFG paths - // into RegionNodes. It doesn't do this test against Root, so - // we do it here. - for( uint i = 1; i < C->root()->req(); i++ ) { - if( !_nodes[C->root()->in(i)->_idx] ) { // Dead path into Root? - _igvn.hash_delete(C->root()); - C->root()->del_req(i); - _igvn._worklist.push(C->root()); - i--; // Rerun same iteration on compressed edges + if (!_verify_only) { + // As a side effect, Dominators removed any unreachable CFG paths + // into RegionNodes. It doesn't do this test against Root, so + // we do it here. + for( uint i = 1; i < C->root()->req(); i++ ) { + if( !_nodes[C->root()->in(i)->_idx] ) { // Dead path into Root? + _igvn.hash_delete(C->root()); + C->root()->del_req(i); + _igvn._worklist.push(C->root()); + i--; // Rerun same iteration on compressed edges + } } + + // Given dominators, try to find inner loops with calls that must + // always be executed (call dominates loop tail). These loops do + // not need a separate safepoint. + Node_List cisstack(a); + _ltree_root->check_safepts(visited, cisstack); } - // Given dominators, try to find inner loops with calls that must - // always be executed (call dominates loop tail). These loops do - // not need a separate safepoint. - Node_List cisstack(a); - _ltree_root->check_safepts(visited, cisstack); - // Walk the DATA nodes and place into loops. Find earliest control // node. For CFG nodes, the _nodes array starts out and remains // holding the associated IdealLoopTree pointer. For DATA nodes, the @@ -1548,11 +1551,11 @@ // it will be processed among C->top() inputs worklist.push( C->top() ); visited.set( C->top()->_idx ); // Set C->top() as visited now - build_loop_early( visited, worklist, nstack, verify_me ); + build_loop_early( visited, worklist, nstack ); // Given early legal placement, try finding counted loops. This placement // is good enough to discover most loop invariants. - if( !verify_me ) + if( !_verify_me && !_verify_only ) _ltree_root->counted_loop( this ); // Find latest loop placement. Find ideal loop placement. @@ -1562,16 +1565,25 @@ worklist.push( C->root() ); NOT_PRODUCT( C->verify_graph_edges(); ) worklist.push( C->top() ); - build_loop_late( visited, worklist, nstack, verify_me ); + build_loop_late( visited, worklist, nstack ); + if (_verify_only) { + // restore major progress flag + for (int i = 0; i < old_progress; i++) + C->set_major_progress(); + assert(C->unique() == unique, "verification mode made Nodes? ? ?"); + assert(_igvn._worklist.size() == 0, "shouldn't push anything"); + return; + } + // clear out the dead code while(_deadlist.size()) { - igvn.remove_globally_dead_node(_deadlist.pop()); + _igvn.remove_globally_dead_node(_deadlist.pop()); } #ifndef PRODUCT C->verify_graph_edges(); - if( verify_me ) { // Nested verify pass? + if( _verify_me ) { // Nested verify pass? // Check to see if the verify mode is broken assert(C->unique() == unique, "non-optimize mode made Nodes? ? ?"); return; @@ -1678,7 +1690,7 @@ void PhaseIdealLoop::verify() const { int old_progress = C->major_progress(); ResourceMark rm; - PhaseIdealLoop loop_verify( _igvn, this, false ); + PhaseIdealLoop loop_verify( _igvn, this ); VectorSet visited(Thread::current()->resource_area()); fail = 0; @@ -2138,39 +2150,41 @@ // optimizing an infinite loop? l = _ltree_root; // Oops, found infinite loop - // Insert the NeverBranch between 'm' and it's control user. - NeverBranchNode *iff = new (C, 1) NeverBranchNode( m ); - _igvn.register_new_node_with_optimizer(iff); - set_loop(iff, l); - Node *if_t = new (C, 1) CProjNode( iff, 0 ); - _igvn.register_new_node_with_optimizer(if_t); - set_loop(if_t, l); + if (!_verify_only) { + // Insert the NeverBranch between 'm' and it's control user. + NeverBranchNode *iff = new (C, 1) NeverBranchNode( m ); + _igvn.register_new_node_with_optimizer(iff); + set_loop(iff, l); + Node *if_t = new (C, 1) CProjNode( iff, 0 ); + _igvn.register_new_node_with_optimizer(if_t); + set_loop(if_t, l); - Node* cfg = NULL; // Find the One True Control User of m - for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) { - Node* x = m->fast_out(j); - if (x->is_CFG() && x != m && x != iff) - { cfg = x; break; } - } - assert(cfg != NULL, "must find the control user of m"); - uint k = 0; // Probably cfg->in(0) - while( cfg->in(k) != m ) k++; // But check incase cfg is a Region - cfg->set_req( k, if_t ); // Now point to NeverBranch + Node* cfg = NULL; // Find the One True Control User of m + for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) { + Node* x = m->fast_out(j); + if (x->is_CFG() && x != m && x != iff) + { cfg = x; break; } + } + assert(cfg != NULL, "must find the control user of m"); + uint k = 0; // Probably cfg->in(0) + while( cfg->in(k) != m ) k++; // But check incase cfg is a Region + cfg->set_req( k, if_t ); // Now point to NeverBranch - // Now create the never-taken loop exit - Node *if_f = new (C, 1) CProjNode( iff, 1 ); - _igvn.register_new_node_with_optimizer(if_f); - set_loop(if_f, l); - // Find frame ptr for Halt. Relies on the optimizer - // V-N'ing. Easier and quicker than searching through - // the program structure. - Node *frame = new (C, 1) ParmNode( C->start(), TypeFunc::FramePtr ); - _igvn.register_new_node_with_optimizer(frame); - // Halt & Catch Fire - Node *halt = new (C, TypeFunc::Parms) HaltNode( if_f, frame ); - _igvn.register_new_node_with_optimizer(halt); - set_loop(halt, l); - C->root()->add_req(halt); + // Now create the never-taken loop exit + Node *if_f = new (C, 1) CProjNode( iff, 1 ); + _igvn.register_new_node_with_optimizer(if_f); + set_loop(if_f, l); + // Find frame ptr for Halt. Relies on the optimizer + // V-N'ing. Easier and quicker than searching through + // the program structure. + Node *frame = new (C, 1) ParmNode( C->start(), TypeFunc::FramePtr ); + _igvn.register_new_node_with_optimizer(frame); + // Halt & Catch Fire + Node *halt = new (C, TypeFunc::Parms) HaltNode( if_f, frame ); + _igvn.register_new_node_with_optimizer(halt); + set_loop(halt, l); + C->root()->add_req(halt); + } set_loop(C->root(), _ltree_root); } } @@ -2177,15 +2191,17 @@ // Weeny check for irreducible. This child was already visited (this // IS the post-work phase). Is this child's loop header post-visited // as well? If so, then I found another entry into the loop. - while( is_postvisited(l->_head) ) { - // found irreducible - l->_irreducible = 1; // = true - l = l->_parent; - _has_irreducible_loops = true; - // Check for bad CFG here to prevent crash, and bailout of compile - if (l == NULL) { - C->record_method_not_compilable("unhandled CFG detected during loop optimization"); - return pre_order; + if (!_verify_only) { + while( is_postvisited(l->_head) ) { + // found irreducible + l->_irreducible = 1; // = true + l = l->_parent; + _has_irreducible_loops = true; + // Check for bad CFG here to prevent crash, and bailout of compile + if (l == NULL) { + C->record_method_not_compilable("unhandled CFG detected during loop optimization"); + return pre_order; + } } } @@ -2253,7 +2269,7 @@ // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. // First pass computes the earliest controlling node possible. This is the // controlling input with the deepest dominating depth. -void PhaseIdealLoop::build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack, const PhaseIdealLoop *verify_me ) { +void PhaseIdealLoop::build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) { while (worklist.size() != 0) { // Use local variables nstack_top_n & nstack_top_i to cache values // on nstack's top. @@ -2285,7 +2301,7 @@ // (the old code here would yank a 2nd safepoint after seeing a // first one, even though the 1st did not dominate in the loop body // and thus could be avoided indefinitely) - if( !verify_me && ilt->_has_sfpt && n->Opcode() == Op_SafePoint && + if( !_verify_only && !_verify_me && ilt->_has_sfpt && n->Opcode() == Op_SafePoint && is_deleteable_safept(n)) { Node *in = n->in(TypeFunc::Control); lazy_replace(n,in); // Pull safepoint now @@ -2408,12 +2424,31 @@ return LCA; } -//------------------------------get_late_ctrl---------------------------------- -// Compute latest legal control. -Node *PhaseIdealLoop::get_late_ctrl( Node *n, Node *early ) { - assert(early != NULL, "early control should not be NULL"); +bool PhaseIdealLoop::verify_dominance(Node* n, Node* use, Node* LCA, Node* early) { + bool had_error = false; +#ifdef ASSERT + if (early != C->root()) { + // Make sure that there's a dominance path from use to LCA + Node* d = use; + while (d != LCA) { + d = idom(d); + if (d == C->root()) { + tty->print_cr("*** Use %d isn't dominated by def %s", use->_idx, n->_idx); + n->dump(); + use->dump(); + had_error = true; + break; + } + } + } +#endif + return had_error; +} + +Node* PhaseIdealLoop::compute_lca_of_uses(Node* n, Node* early, bool verify) { // Compute LCA over list of uses + bool had_error = false; Node *LCA = NULL; for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && LCA != early; i++) { Node* c = n->fast_out(i); @@ -2423,7 +2458,9 @@ for( uint j=1; jreq(); j++ ) {// For all inputs if( c->in(j) == n ) { // Found matching input? Node *use = c->in(0)->in(j); + if (_verify_only && use->is_top()) continue; LCA = dom_lca_for_get_late_ctrl( LCA, use, n ); + if (verify) had_error = verify_dominance(n, use, LCA, early) || had_error; } } } else { @@ -2430,9 +2467,26 @@ // For CFG data-users, use is in the block just prior Node *use = has_ctrl(c) ? get_ctrl(c) : c->in(0); LCA = dom_lca_for_get_late_ctrl( LCA, use, n ); + if (verify) had_error = verify_dominance(n, use, LCA, early) || had_error; } } + assert(!had_error, "bad dominance"); + return LCA; +} +//------------------------------get_late_ctrl---------------------------------- +// Compute latest legal control. +Node *PhaseIdealLoop::get_late_ctrl( Node *n, Node *early ) { + assert(early != NULL, "early control should not be NULL"); + + Node* LCA = compute_lca_of_uses(n, early); +#ifdef ASSERT + if (LCA == C->root() && LCA != early) { + // def doesn't dominate uses so print some useful debugging output + compute_lca_of_uses(n, early, true); + } +#endif + // if this is a load, check for anti-dependent stores // We use a conservative algorithm to identify potential interfering // instructions and for rescheduling the load. The users of the memory @@ -2576,7 +2630,7 @@ //------------------------------build_loop_late-------------------------------- // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. // Second pass finds latest legal placement, and ideal loop placement. -void PhaseIdealLoop::build_loop_late( VectorSet &visited, Node_List &worklist, Node_Stack &nstack, const PhaseIdealLoop *verify_me ) { +void PhaseIdealLoop::build_loop_late( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) { while (worklist.size() != 0) { Node *n = worklist.pop(); // Only visit once @@ -2612,7 +2666,7 @@ } } else { // All of n's children have been processed, complete post-processing. - build_loop_late_post(n, verify_me); + build_loop_late_post(n); if (nstack.is_empty()) { // Finished all nodes on stack. // Process next node on the worklist. @@ -2631,9 +2685,9 @@ //------------------------------build_loop_late_post--------------------------- // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. // Second pass finds latest legal placement, and ideal loop placement. -void PhaseIdealLoop::build_loop_late_post( Node *n, const PhaseIdealLoop *verify_me ) { +void PhaseIdealLoop::build_loop_late_post( Node *n ) { - if (n->req() == 2 && n->Opcode() == Op_ConvI2L && !C->major_progress()) { + if (n->req() == 2 && n->Opcode() == Op_ConvI2L && !C->major_progress() && !_verify_only) { _igvn._worklist.push(n); // Maybe we'll normalize it, if no more loops. } @@ -2714,6 +2768,7 @@ if( get_loop(legal)->_nest < get_loop(least)->_nest ) least = legal; } + assert(early == legal || legal != C->root(), "bad dominance of inputs"); // Try not to place code on a loop entry projection // which can inhibit range check elimination. @@ -2731,8 +2786,8 @@ #ifdef ASSERT // If verifying, verify that 'verify_me' has a legal location // and choose it as our location. - if( verify_me ) { - Node *v_ctrl = verify_me->get_ctrl_no_update(n); + if( _verify_me ) { + Node *v_ctrl = _verify_me->get_ctrl_no_update(n); Node *legal = LCA; while( early != legal ) { // While not at earliest legal if( legal == v_ctrl ) break; // Check for prior good location