--- old/src/share/vm/opto/ifnode.cpp 2015-11-09 11:56:26.024095254 +0100 +++ new/src/share/vm/opto/ifnode.cpp 2015-11-09 11:56:25.569690215 +0100 @@ -306,12 +306,16 @@ Node *b_c = phase->transform(new BoolNode(cmp_c,b->_test._test)); Node *b_x = phase->transform(new BoolNode(cmp_x,b->_test._test)); // Make the IfNode - IfNode *iff_c = new IfNode(region_c,b_c,iff->_prob,iff->_fcnt); + IfNode* iff_c = iff->clone()->as_If(); + iff_c->set_req(0, region_c); + iff_c->set_req(1, b_c); igvn->set_type_bottom(iff_c); igvn->_worklist.push(iff_c); hook->init_req(2, iff_c); - IfNode *iff_x = new IfNode(region_x,b_x,iff->_prob, iff->_fcnt); + IfNode* iff_x = iff->clone()->as_If(); + iff_x->set_req(0, region_x); + iff_x->set_req(1, b_x); igvn->set_type_bottom(iff_x); igvn->_worklist.push(iff_x); hook->init_req(3, iff_x); @@ -496,7 +500,7 @@ // Return 0 if not a range check. Return 1 if a range check and set index and // offset. Return 2 if we had to negate the test. Index is NULL if the check // is versus a constant. -int IfNode::is_range_check(Node* &range, Node* &index, jint &offset) { +int RangeCheckNode::is_range_check(Node* &range, Node* &index, jint &offset) { int flip_test = 0; Node* l = NULL; Node* r = NULL; @@ -724,7 +728,7 @@ return ctrl != NULL && ctrl->is_Proj() && ctrl->in(0) != NULL && - ctrl->in(0)->is_If() && + ctrl->in(0)->Opcode() == Op_If && ctrl->in(0)->outcnt() == 2 && ctrl->in(0)->as_If()->cmpi_folds(igvn) && // Must compare same value @@ -972,8 +976,8 @@ if (failtype->_lo > failtype->_hi) { // previous if determines the result of this if so // replace Bool with constant - igvn->hash_delete(this); - set_req(1, igvn->intcon(success->_con)); + igvn->_worklist.push(in(1)); + igvn->replace_input_of(this, 1, igvn->intcon(success->_con)); return true; } } @@ -992,7 +996,8 @@ Node* newbool = igvn->transform(new BoolNode(newcmp, cond)); igvn->replace_input_of(dom_iff, 1, igvn->intcon(proj->_con)); - set_req(1, newbool); + igvn->_worklist.push(in(1)); + igvn->replace_input_of(this, 1, newbool); return true; } @@ -1002,7 +1007,10 @@ // Merge the branches that trap for this If and the dominating If into // a single region that branches to the uncommon trap for the // dominating If -void IfNode::merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn) { +Node* IfNode::merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn) { + Node* res = this; + assert(success->in(0) == this, "bad projection"); + ProjNode* otherproj = proj->other_if_proj(); CallStaticJavaNode* unc = success->is_uncommon_trap_proj(Deoptimization::Reason_none); @@ -1038,6 +1046,8 @@ trap_request = Deoptimization::make_trap_request(Deoptimization::Reason_range_check, action); improve_address_types(l, r, fail, igvn); + + res = igvn->transform(new RangeCheckNode(in(0), in(1), _prob, _fcnt)); } else if (unc != dom_unc) { // If we trap we won't know what CmpI would have caused the trap // so use a special trap reason to mark this pair of CmpI nodes as @@ -1047,6 +1057,7 @@ trap_request = Deoptimization::make_trap_request(Deoptimization::Reason_unstable_fused_if, action); } igvn->replace_input_of(dom_unc, TypeFunc::Parms, igvn->intcon(trap_request)); + return res; } // If we are turning 2 CmpI nodes into a CmpU that follows the pattern @@ -1240,8 +1251,7 @@ if (has_only_uncommon_traps(dom_cmp, success, fail, igvn) && // Next call modifies graph so must be last fold_compares_helper(dom_cmp, success, fail, igvn)) { - merge_uncommon_traps(dom_cmp, success, fail, igvn); - return this; + return merge_uncommon_traps(dom_cmp, success, fail, igvn); } return NULL; } else if (ctrl->in(0) != NULL && @@ -1260,8 +1270,7 @@ // Next call modifies graph so must be last fold_compares_helper(dom_cmp, success, fail, igvn)) { reroute_side_effect_free_unc(other_cmp, dom_cmp, igvn); - merge_uncommon_traps(dom_cmp, success, fail, igvn); - return this; + return merge_uncommon_traps(dom_cmp, success, fail, igvn); } } } @@ -1342,14 +1351,10 @@ jint off; }; -//------------------------------Ideal------------------------------------------ -// Return a node which is more "ideal" than the current node. Strip out -// control copies -Node *IfNode::Ideal(PhaseGVN *phase, bool can_reshape) { +Node* IfNode::Ideal_common(PhaseGVN *phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) return this; // No Def-Use info? if (!can_reshape) return NULL; - PhaseIterGVN *igvn = phase->is_IterGVN(); // Don't bother trying to transform a dead if if (in(0)->is_top()) return NULL; @@ -1365,212 +1370,69 @@ if (idt_if != NULL) return idt_if; // Try to split the IF + PhaseIterGVN *igvn = phase->is_IterGVN(); Node *s = split_if(this, igvn); if (s != NULL) return s; + return NodeSentinel; +} + +//------------------------------Ideal------------------------------------------ +// Return a node which is more "ideal" than the current node. Strip out +// control copies +Node* IfNode::Ideal(PhaseGVN *phase, bool can_reshape) { + Node* res = Ideal_common(phase, can_reshape); + if (res != NodeSentinel) { + return res; + } + // Check for people making a useless boolean: things like // if( (x < y ? true : false) ) { ... } // Replace with if( x < y ) { ... } Node *bol2 = remove_useless_bool(this, phase); if( bol2 ) return bol2; - // Setup to scan up the CFG looking for a dominating test - Node *dom = in(0); - Node *prev_dom = this; + if (in(0) == NULL) return NULL; // Dead loop? - // Check for range-check vs other kinds of tests - Node *index1, *range1; - jint offset1; - int flip1 = is_range_check(range1, index1, offset1); - if( flip1 ) { - // Try to remove extra range checks. All 'up_one_dom' gives up at merges - // so all checks we inspect post-dominate the top-most check we find. - // If we are going to fail the current check and we reach the top check - // then we are guaranteed to fail, so just start interpreting there. - // We 'expand' the top 3 range checks to include all post-dominating - // checks. - - // The top 3 range checks seen - const int NRC =3; - RangeCheck prev_checks[NRC]; - int nb_checks = 0; - - // Low and high offsets seen so far - jint off_lo = offset1; - jint off_hi = offset1; - - bool found_immediate_dominator = false; - - // Scan for the top checks and collect range of offsets - for (int dist = 0; dist < 999; dist++) { // Range-Check scan limit - if (dom->Opcode() == Op_If && // Not same opcode? - prev_dom->in(0) == dom) { // One path of test does dominate? - if (dom == this) return NULL; // dead loop - // See if this is a range check - Node *index2, *range2; - jint offset2; - int flip2 = dom->as_If()->is_range_check(range2, index2, offset2); - // See if this is a _matching_ range check, checking against - // the same array bounds. - if (flip2 == flip1 && range2 == range1 && index2 == index1 && - dom->outcnt() == 2) { - if (nb_checks == 0 && dom->in(1) == in(1)) { - // Found an immediately dominating test at the same offset. - // This kind of back-to-back test can be eliminated locally, - // and there is no need to search further for dominating tests. - assert(offset2 == offset1, "Same test but different offsets"); - found_immediate_dominator = true; - break; - } - // Gather expanded bounds - off_lo = MIN2(off_lo,offset2); - off_hi = MAX2(off_hi,offset2); - // Record top NRC range checks - prev_checks[nb_checks%NRC].ctl = prev_dom; - prev_checks[nb_checks%NRC].off = offset2; - nb_checks++; - } - } - prev_dom = dom; - dom = up_one_dom(dom); - if (!dom) break; - } - - if (!found_immediate_dominator) { - // Attempt to widen the dominating range check to cover some later - // ones. Since range checks "fail" by uncommon-trapping to the - // interpreter, widening a check can make us speculatively enter - // the interpreter. If we see range-check deopt's, do not widen! - if (!phase->C->allow_range_check_smearing()) return NULL; - - // Didn't find prior covering check, so cannot remove anything. - if (nb_checks == 0) { - return NULL; - } - // Constant indices only need to check the upper bound. - // Non-constant indices must check both low and high. - int chk0 = (nb_checks - 1) % NRC; - if (index1) { - if (nb_checks == 1) { - return NULL; - } else { - // If the top range check's constant is the min or max of - // all constants we widen the next one to cover the whole - // range of constants. - RangeCheck rc0 = prev_checks[chk0]; - int chk1 = (nb_checks - 2) % NRC; - RangeCheck rc1 = prev_checks[chk1]; - if (rc0.off == off_lo) { - adjust_check(rc1.ctl, range1, index1, flip1, off_hi, igvn); - prev_dom = rc1.ctl; - } else if (rc0.off == off_hi) { - adjust_check(rc1.ctl, range1, index1, flip1, off_lo, igvn); - prev_dom = rc1.ctl; - } else { - // If the top test's constant is not the min or max of all - // constants, we need 3 range checks. We must leave the - // top test unchanged because widening it would allow the - // accesses it protects to successfully read/write out of - // bounds. - if (nb_checks == 2) { - return NULL; - } - int chk2 = (nb_checks - 3) % NRC; - RangeCheck rc2 = prev_checks[chk2]; - // The top range check a+i covers interval: -a <= i < length-a - // The second range check b+i covers interval: -b <= i < length-b - if (rc1.off <= rc0.off) { - // if b <= a, we change the second range check to: - // -min_of_all_constants <= i < length-min_of_all_constants - // Together top and second range checks now cover: - // -min_of_all_constants <= i < length-a - // which is more restrictive than -b <= i < length-b: - // -b <= -min_of_all_constants <= i < length-a <= length-b - // The third check is then changed to: - // -max_of_all_constants <= i < length-max_of_all_constants - // so 2nd and 3rd checks restrict allowed values of i to: - // -min_of_all_constants <= i < length-max_of_all_constants - adjust_check(rc1.ctl, range1, index1, flip1, off_lo, igvn); - adjust_check(rc2.ctl, range1, index1, flip1, off_hi, igvn); - } else { - // if b > a, we change the second range check to: - // -max_of_all_constants <= i < length-max_of_all_constants - // Together top and second range checks now cover: - // -a <= i < length-max_of_all_constants - // which is more restrictive than -b <= i < length-b: - // -b < -a <= i < length-max_of_all_constants <= length-b - // The third check is then changed to: - // -max_of_all_constants <= i < length-max_of_all_constants - // so 2nd and 3rd checks restrict allowed values of i to: - // -min_of_all_constants <= i < length-max_of_all_constants - adjust_check(rc1.ctl, range1, index1, flip1, off_hi, igvn); - adjust_check(rc2.ctl, range1, index1, flip1, off_lo, igvn); - } - prev_dom = rc2.ctl; - } - } - } else { - RangeCheck rc0 = prev_checks[chk0]; - // 'Widen' the offset of the 1st and only covering check - adjust_check(rc0.ctl, range1, index1, flip1, off_hi, igvn); - // Test is now covered by prior checks, dominate it out - prev_dom = rc0.ctl; - } - } - - } else { // Scan for an equivalent test + PhaseIterGVN *igvn = phase->is_IterGVN(); + Node* result = fold_compares(igvn); + if (result != NULL) { + return result; + } - Node *cmp; - int dist = 0; // Cutoff limit for search - int op = Opcode(); - if( op == Op_If && - (cmp=in(1)->in(1))->Opcode() == Op_CmpP ) { - if( cmp->in(2) != NULL && // make sure cmp is not already dead - cmp->in(2)->bottom_type() == TypePtr::NULL_PTR ) { - dist = 64; // Limit for null-pointer scans - } else { - dist = 4; // Do not bother for random pointer tests - } + // Scan for an equivalent test + Node *cmp; + int dist = 0; // Cutoff limit for search + int op = Opcode(); + if( op == Op_If && + (cmp=in(1)->in(1))->Opcode() == Op_CmpP ) { + if( cmp->in(2) != NULL && // make sure cmp is not already dead + cmp->in(2)->bottom_type() == TypePtr::NULL_PTR ) { + dist = 64; // Limit for null-pointer scans } else { - dist = 4; // Limit for random junky scans - } - - // Normal equivalent-test check. - if( !dom ) return NULL; // Dead loop? - - Node* result = fold_compares(igvn); - if (result != NULL) { - return result; - } - - // Search up the dominator tree for an If with an identical test - while( dom->Opcode() != op || // Not same opcode? - dom->in(1) != in(1) || // Not same input 1? - (req() == 3 && dom->in(2) != in(2)) || // Not same input 2? - prev_dom->in(0) != dom ) { // One path of test does not dominate? - if( dist < 0 ) return NULL; - - dist--; - prev_dom = dom; - dom = up_one_dom( dom ); - if( !dom ) return NULL; + dist = 4; // Do not bother for random pointer tests } + } else { + dist = 4; // Limit for random junky scans + } - // Check that we did not follow a loop back to ourselves - if( this == dom ) - return NULL; + Node* prev_dom = search_identical(dist); - if( dist > 2 ) // Add to count of NULL checks elided - explicit_null_checks_elided++; + if (prev_dom == NULL) { + return NULL; + } - } // End of Else scan for an equivalent test + // Replace dominated IfNode + return dominated_by(prev_dom, igvn); +} - // Hit! Remove this IF +//------------------------------dominated_by----------------------------------- +Node* IfNode::dominated_by(Node* prev_dom, PhaseIterGVN *igvn) { #ifndef PRODUCT - if( TraceIterativeGVN ) { + if (TraceIterativeGVN) { tty->print(" Removing IfNode: "); this->dump(); } - if( VerifyOpto && !phase->allow_progress() ) { + if (VerifyOpto && !igvn->allow_progress()) { // Found an equivalent dominating test, // we can not guarantee reaching a fix-point for these during iterativeGVN // since intervening nodes may not change. @@ -1578,16 +1440,6 @@ } #endif - // Replace dominated IfNode - dominated_by( prev_dom, igvn ); - - // Must return either the original node (now dead) or a new node - // (Do not return a top here, since that would break the uniqueness of top.) - return new ConINode(TypeInt::ZERO); -} - -//------------------------------dominated_by----------------------------------- -void IfNode::dominated_by( Node *prev_dom, PhaseIterGVN *igvn ) { igvn->hash_delete(this); // Remove self to prevent spurious V-N Node *idom = in(0); // Need opcode to decide which way 'this' test goes @@ -1635,6 +1487,38 @@ // Kill the IfNode igvn->remove_dead_node(this); + + // Must return either the original node (now dead) or a new node + // (Do not return a top here, since that would break the uniqueness of top.) + return new ConINode(TypeInt::ZERO); +} + +Node* IfNode::search_identical(int dist) { + // Setup to scan up the CFG looking for a dominating test + Node* dom = in(0); + Node* prev_dom = this; + int op = Opcode(); + // Search up the dominator tree for an If with an identical test + while( dom->Opcode() != op || // Not same opcode? + dom->in(1) != in(1) || // Not same input 1? + (req() == 3 && dom->in(2) != in(2)) || // Not same input 2? + prev_dom->in(0) != dom ) { // One path of test does not dominate? + if( dist < 0 ) return NULL; + + dist--; + prev_dom = dom; + dom = up_one_dom( dom ); + if( !dom ) return NULL; + } + + // Check that we did not follow a loop back to ourselves + if( this == dom ) + return NULL; + + if( dist > 2 ) // Add to count of NULL checks elided + explicit_null_checks_elided++; + + return prev_dom; } //------------------------------Identity--------------------------------------- @@ -1726,7 +1610,9 @@ assert( igvn, "Test is not canonical in parser?" ); // The IF node never really changes, but it needs to be cloned - iff = new IfNode( iff->in(0), b, 1.0-iff->_prob, iff->_fcnt); + iff = iff->clone()->as_If(); + iff->set_req(1, b); + iff->_prob = 1.0-iff->_prob; Node *prior = igvn->hash_find_insert(iff); if( prior ) { @@ -1751,3 +1637,167 @@ // Progress return iff; } + +Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) { + Node* res = Ideal_common(phase, can_reshape); + if (res != NodeSentinel) { + return res; + } + + PhaseIterGVN *igvn = phase->is_IterGVN(); + // Setup to scan up the CFG looking for a dominating test + Node* prev_dom = this; + + // Check for range-check vs other kinds of tests + Node* index1; + Node* range1; + jint offset1; + int flip1 = is_range_check(range1, index1, offset1); + if (flip1) { + Node* dom = in(0); + // Try to remove extra range checks. All 'up_one_dom' gives up at merges + // so all checks we inspect post-dominate the top-most check we find. + // If we are going to fail the current check and we reach the top check + // then we are guaranteed to fail, so just start interpreting there. + // We 'expand' the top 3 range checks to include all post-dominating + // checks. + + // The top 3 range checks seen + const int NRC =3; + RangeCheck prev_checks[NRC]; + int nb_checks = 0; + + // Low and high offsets seen so far + jint off_lo = offset1; + jint off_hi = offset1; + + bool found_immediate_dominator = false; + + // Scan for the top checks and collect range of offsets + for (int dist = 0; dist < 999; dist++) { // Range-Check scan limit + if (dom->Opcode() == Op_RangeCheck && // Not same opcode? + prev_dom->in(0) == dom) { // One path of test does dominate? + if (dom == this) return NULL; // dead loop + // See if this is a range check + Node* index2; + Node* range2; + jint offset2; + int flip2 = dom->as_RangeCheck()->is_range_check(range2, index2, offset2); + // See if this is a _matching_ range check, checking against + // the same array bounds. + if (flip2 == flip1 && range2 == range1 && index2 == index1 && + dom->outcnt() == 2) { + if (nb_checks == 0 && dom->in(1) == in(1)) { + // Found an immediately dominating test at the same offset. + // This kind of back-to-back test can be eliminated locally, + // and there is no need to search further for dominating tests. + assert(offset2 == offset1, "Same test but different offsets"); + found_immediate_dominator = true; + break; + } + // Gather expanded bounds + off_lo = MIN2(off_lo,offset2); + off_hi = MAX2(off_hi,offset2); + // Record top NRC range checks + prev_checks[nb_checks%NRC].ctl = prev_dom; + prev_checks[nb_checks%NRC].off = offset2; + nb_checks++; + } + } + prev_dom = dom; + dom = up_one_dom(dom); + if (!dom) break; + } + + if (!found_immediate_dominator) { + // Attempt to widen the dominating range check to cover some later + // ones. Since range checks "fail" by uncommon-trapping to the + // interpreter, widening a check can make us speculatively enter + // the interpreter. If we see range-check deopt's, do not widen! + if (!phase->C->allow_range_check_smearing()) return NULL; + + // Didn't find prior covering check, so cannot remove anything. + if (nb_checks == 0) { + return NULL; + } + // Constant indices only need to check the upper bound. + // Non-constant indices must check both low and high. + int chk0 = (nb_checks - 1) % NRC; + if (index1) { + if (nb_checks == 1) { + return NULL; + } else { + // If the top range check's constant is the min or max of + // all constants we widen the next one to cover the whole + // range of constants. + RangeCheck rc0 = prev_checks[chk0]; + int chk1 = (nb_checks - 2) % NRC; + RangeCheck rc1 = prev_checks[chk1]; + if (rc0.off == off_lo) { + adjust_check(rc1.ctl, range1, index1, flip1, off_hi, igvn); + prev_dom = rc1.ctl; + } else if (rc0.off == off_hi) { + adjust_check(rc1.ctl, range1, index1, flip1, off_lo, igvn); + prev_dom = rc1.ctl; + } else { + // If the top test's constant is not the min or max of all + // constants, we need 3 range checks. We must leave the + // top test unchanged because widening it would allow the + // accesses it protects to successfully read/write out of + // bounds. + if (nb_checks == 2) { + return NULL; + } + int chk2 = (nb_checks - 3) % NRC; + RangeCheck rc2 = prev_checks[chk2]; + // The top range check a+i covers interval: -a <= i < length-a + // The second range check b+i covers interval: -b <= i < length-b + if (rc1.off <= rc0.off) { + // if b <= a, we change the second range check to: + // -min_of_all_constants <= i < length-min_of_all_constants + // Together top and second range checks now cover: + // -min_of_all_constants <= i < length-a + // which is more restrictive than -b <= i < length-b: + // -b <= -min_of_all_constants <= i < length-a <= length-b + // The third check is then changed to: + // -max_of_all_constants <= i < length-max_of_all_constants + // so 2nd and 3rd checks restrict allowed values of i to: + // -min_of_all_constants <= i < length-max_of_all_constants + adjust_check(rc1.ctl, range1, index1, flip1, off_lo, igvn); + adjust_check(rc2.ctl, range1, index1, flip1, off_hi, igvn); + } else { + // if b > a, we change the second range check to: + // -max_of_all_constants <= i < length-max_of_all_constants + // Together top and second range checks now cover: + // -a <= i < length-max_of_all_constants + // which is more restrictive than -b <= i < length-b: + // -b < -a <= i < length-max_of_all_constants <= length-b + // The third check is then changed to: + // -max_of_all_constants <= i < length-max_of_all_constants + // so 2nd and 3rd checks restrict allowed values of i to: + // -min_of_all_constants <= i < length-max_of_all_constants + adjust_check(rc1.ctl, range1, index1, flip1, off_hi, igvn); + adjust_check(rc2.ctl, range1, index1, flip1, off_lo, igvn); + } + prev_dom = rc2.ctl; + } + } + } else { + RangeCheck rc0 = prev_checks[chk0]; + // 'Widen' the offset of the 1st and only covering check + adjust_check(rc0.ctl, range1, index1, flip1, off_hi, igvn); + // Test is now covered by prior checks, dominate it out + prev_dom = rc0.ctl; + } + } + } else { + prev_dom = search_identical(4); + + if (prev_dom == NULL) { + return NULL; + } + } + + // Replace dominated IfNode + return dominated_by(prev_dom, igvn); +}