--- old/src/share/vm/opto/loopUnswitch.cpp 2015-11-09 11:56:25.188294087 +0100 +++ new/src/share/vm/opto/loopUnswitch.cpp 2015-11-09 11:56:24.699617106 +0100 @@ -132,7 +132,7 @@ head->as_CountedLoop()->set_normal_loop(); } - ProjNode* proj_true = create_slow_version_of_loop(loop, old_new); + ProjNode* proj_true = create_slow_version_of_loop(loop, old_new, unswitch_iff->Opcode()); #ifdef ASSERT Node* uniqc = proj_true->unique_ctrl_out(); @@ -222,7 +222,8 @@ // and inserting an if to select fast-slow versions. // Return control projection of the entry to the fast version. ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop, - Node_List &old_new) { + Node_List &old_new, + int opcode) { LoopNode* head = loop->_head->as_Loop(); bool counted_loop = head->is_CountedLoop(); Node* entry = head->in(LoopNode::EntryControl); @@ -235,7 +236,8 @@ register_node(opq, outer_loop, entry, dom_depth(entry)); Node *bol = new Conv2BNode(opq); register_node(bol, outer_loop, entry, dom_depth(entry)); - IfNode* iff = new IfNode(entry, bol, PROB_MAX, COUNT_UNKNOWN); + IfNode* iff = (opcode == Op_RangeCheck) ? new RangeCheckNode(entry, bol, PROB_MAX, COUNT_UNKNOWN) : + new IfNode(entry, bol, PROB_MAX, COUNT_UNKNOWN); register_node(iff, outer_loop, entry, dom_depth(entry)); ProjNode* iffast = new IfTrueNode(iff); register_node(iffast, outer_loop, iff, dom_depth(iff)); --- old/src/share/vm/opto/multnode.cpp 2015-11-09 11:56:25.240997628 +0100 +++ new/src/share/vm/opto/multnode.cpp 2015-11-09 11:56:24.734756349 +0100 @@ -44,14 +44,14 @@ //------------------------------proj_out--------------------------------------- // Get a named projection ProjNode* MultiNode::proj_out(uint which_proj) const { - assert(Opcode() != Op_If || which_proj == (uint)true || which_proj == (uint)false, "must be 1 or 0"); - assert(Opcode() != Op_If || outcnt() == 2, "bad if #1"); + assert((Opcode() != Op_If && Opcode() != Op_RangeCheck) || which_proj == (uint)true || which_proj == (uint)false, "must be 1 or 0"); + assert((Opcode() != Op_If && Opcode() != Op_RangeCheck) || outcnt() == 2, "bad if #1"); for( DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++ ) { Node *p = fast_out(i); if (p->is_Proj()) { ProjNode *proj = p->as_Proj(); if (proj->_con == which_proj) { - assert(Opcode() != Op_If || proj->Opcode() == (which_proj?Op_IfTrue:Op_IfFalse), "bad if #2"); + assert((Opcode() != Op_If && Opcode() != Op_RangeCheck) || proj->Opcode() == (which_proj ? Op_IfTrue : Op_IfFalse), "bad if #2"); return proj; } } else { --- old/src/share/vm/opto/parse2.cpp 2015-11-09 11:56:25.391454189 +0100 +++ new/src/share/vm/opto/parse2.cpp 2015-11-09 11:56:24.783420623 +0100 @@ -136,8 +136,16 @@ BoolTest::mask btest = BoolTest::lt; tst = _gvn.transform( new BoolNode(chk, btest) ); } + RangeCheckNode* rc = new RangeCheckNode(control(), tst, PROB_MAX, COUNT_UNKNOWN); + _gvn.set_type(rc, rc->Value(&_gvn)); + if (!tst->is_Con()) { + record_for_igvn(rc); + } + set_control(_gvn.transform(new IfTrueNode(rc))); // Branch to failure if out of bounds - { BuildCutout unless(this, tst, PROB_MAX); + { + PreserveJVMState pjvms(this); + set_control(_gvn.transform(new IfFalseNode(rc))); if (C->allow_range_check_smearing()) { // Do not use builtin_throw, since range checks are sometimes // made more stringent by an optimistic transformation. --- old/src/share/vm/opto/loopTransform.cpp 2015-11-09 11:56:25.597041674 +0100 +++ new/src/share/vm/opto/loopTransform.cpp 2015-11-09 11:56:25.017951429 +0100 @@ -290,7 +290,7 @@ if (ctrl->is_top()) return false; // Found dead test on live IF? No peeling! // Standard IF only has one input value to check for loop invariance - assert( test->Opcode() == Op_If || test->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added"); + assert(test->Opcode() == Op_If || test->Opcode() == Op_CountedLoopEnd || test->Opcode() == Op_RangeCheck, "Check this code when new subtype is added"); // Condition is not a member of this loop? if( !is_member(phase->get_loop(ctrl)) && is_loop_exit(test) ) @@ -856,7 +856,8 @@ // loop-invariant. for (uint i = 0; i < _body.size(); i++) { Node *iff = _body[i]; - if (iff->Opcode() == Op_If) { // Test? + if (iff->Opcode() == Op_If || + iff->Opcode() == Op_RangeCheck) { // Test? // Comparing trip+off vs limit Node *bol = iff->in(1); @@ -2035,8 +2036,8 @@ // loop-invariant. for( uint i = 0; i < loop->_body.size(); i++ ) { Node *iff = loop->_body[i]; - if( iff->Opcode() == Op_If ) { // Test? - + if (iff->Opcode() == Op_If || + iff->Opcode() == Op_RangeCheck) { // Test? // Test is an IfNode, has 2 projections. If BOTH are in the loop // we need loop unswitching instead of iteration splitting. Node *exit = loop->is_loop_exit(iff); --- old/src/share/vm/opto/node.hpp 2015-11-09 11:56:25.622552395 +0100 +++ new/src/share/vm/opto/node.hpp 2015-11-09 11:56:24.969787765 +0100 @@ -125,6 +125,7 @@ class PhiNode; class Pipeline; class ProjNode; +class RangeCheckNode; class RegMask; class RegionNode; class RootNode; @@ -584,6 +585,7 @@ DEFINE_CLASS_ID(Jump, PCTable, 1) DEFINE_CLASS_ID(If, MultiBranch, 1) DEFINE_CLASS_ID(CountedLoopEnd, If, 0) + DEFINE_CLASS_ID(RangeCheck, If, 1) DEFINE_CLASS_ID(NeverBranch, MultiBranch, 2) DEFINE_CLASS_ID(Start, Multi, 2) DEFINE_CLASS_ID(MemBar, Multi, 3) @@ -758,6 +760,7 @@ DEFINE_CLASS_QUERY(FastLock) DEFINE_CLASS_QUERY(FastUnlock) DEFINE_CLASS_QUERY(If) + DEFINE_CLASS_QUERY(RangeCheck) DEFINE_CLASS_QUERY(IfFalse) DEFINE_CLASS_QUERY(IfTrue) DEFINE_CLASS_QUERY(Initialize) --- old/src/share/vm/opto/classes.hpp 2015-11-09 11:56:25.725296082 +0100 +++ new/src/share/vm/opto/classes.hpp 2015-11-09 11:56:25.158534682 +0100 @@ -138,6 +138,7 @@ macro(Halt) macro(HasNegatives) macro(If) +macro(RangeCheck) macro(IfFalse) macro(IfTrue) macro(Initialize) --- old/src/share/vm/opto/loopopts.cpp 2015-11-09 11:56:25.811686091 +0100 +++ new/src/share/vm/opto/loopopts.cpp 2015-11-09 11:56:25.196631411 +0100 @@ -206,7 +206,7 @@ // prevdom is the dominating projection of the dominating test. assert( iff->is_If(), "" ); - assert( iff->Opcode() == Op_If || iff->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added"); + assert(iff->Opcode() == Op_If || iff->Opcode() == Op_CountedLoopEnd || iff->Opcode() == Op_RangeCheck, "Check this code when new subtype is added"); int pop = prevdom->Opcode(); assert( pop == Op_IfFalse || pop == Op_IfTrue, "" ); if (flip) { @@ -1123,7 +1123,8 @@ int n_op = n->Opcode(); // Check for an IF being dominated by another IF same test - if (n_op == Op_If) { + if (n_op == Op_If || + n_op == Op_RangeCheck) { Node *bol = n->in(1); uint max = bol->outcnt(); // Check for same test used more than once? @@ -1944,8 +1945,11 @@ BoolNode* bol = new BoolNode(cmp, relop); register_node(bol, loop, proj2, ddepth); - - IfNode* new_if = new IfNode(proj2, bol, iff->_prob, iff->_fcnt); + + int opcode = iff->Opcode(); + assert(opcode == Op_If || opcode == Op_RangeCheck, "unexpected opcode"); + IfNode* new_if = (opcode == Op_If) ? new IfNode(proj2, bol, iff->_prob, iff->_fcnt): + new RangeCheckNode(proj2, bol, iff->_prob, iff->_fcnt); register_node(new_if, loop, proj2, ddepth); proj->set_req(0, new_if); // reattach --- old/src/share/vm/opto/cfgnode.hpp 2015-11-09 11:56:25.770550452 +0100 +++ new/src/share/vm/opto/cfgnode.hpp 2015-11-09 11:56:25.329109838 +0100 @@ -270,7 +270,6 @@ virtual uint size_of() const { return sizeof(*this); } private: - ProjNode* range_check_trap_proj(int& flip, Node*& l, Node*& r); ProjNode* range_check_trap_proj() { int flip_test = 0; Node* l = NULL; @@ -283,7 +282,7 @@ bool is_ctrl_folds(Node* ctrl, PhaseIterGVN* igvn); bool has_shared_region(ProjNode* proj, ProjNode*& success, ProjNode*& fail); bool has_only_uncommon_traps(ProjNode* proj, ProjNode*& success, ProjNode*& fail, PhaseIterGVN* igvn); - static void merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn); + Node* merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn); static void improve_address_types(Node* l, Node* r, ProjNode* fail, PhaseIterGVN* igvn); bool is_cmp_with_loadrange(ProjNode* proj); bool is_null_check(ProjNode* proj, PhaseIterGVN* igvn); @@ -292,6 +291,12 @@ ProjNode* uncommon_trap_proj(CallStaticJavaNode*& call) const; bool fold_compares_helper(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn); +protected: + ProjNode* range_check_trap_proj(int& flip, Node*& l, Node*& r); + Node* Ideal_common(PhaseGVN *phase, bool can_reshape); + Node* dominated_by(Node* prev_dom, PhaseIterGVN* igvn); + Node* search_identical(int dist); + public: // Degrees of branch prediction probability by order of magnitude: @@ -375,8 +380,6 @@ virtual const Type *Value( PhaseTransform *phase ) const; virtual int required_outcnt() const { return 2; } virtual const RegMask &out_RegMask() const; - void dominated_by(Node* prev_dom, PhaseIterGVN* igvn); - int is_range_check(Node* &range, Node* &index, jint &offset); Node* fold_compares(PhaseIterGVN* phase); static Node* up_one_dom(Node* curr, bool linear_only = false); @@ -391,6 +394,20 @@ #endif }; +class RangeCheckNode : public IfNode { +private: + int is_range_check(Node* &range, Node* &index, jint &offset); + +public: + RangeCheckNode(Node* control, Node *b, float p, float fcnt) + : IfNode(control, b, p, fcnt) { + init_class_id(Class_RangeCheck); + } + + virtual int Opcode() const; + virtual Node* Ideal(PhaseGVN *phase, bool can_reshape); +}; + class IfProjNode : public CProjNode { public: IfProjNode(IfNode *ifnode, uint idx) : CProjNode(ifnode,idx) {} --- old/src/share/vm/opto/compile.cpp 2015-11-09 11:56:25.841012642 +0100 +++ new/src/share/vm/opto/compile.cpp 2015-11-09 11:56:25.104962581 +0100 @@ -3181,6 +3181,13 @@ n->set_req(MemBarNode::Precedent, top()); } break; + case Op_RangeCheck: { + RangeCheckNode* rc = n->as_RangeCheck(); + Node* iff = new IfNode(rc->in(0), rc->in(1), rc->_prob, rc->_fcnt); + n->subsume_by(iff, this); + frc._tests.push(iff); + break; + } default: assert( !n->is_Call(), "" ); assert( !n->is_Mem(), "" ); @@ -3189,8 +3196,9 @@ } // Collect CFG split points - if (n->is_MultiBranch()) + if (n->is_MultiBranch() && !n->is_RangeCheck()) { frc._tests.push(n); + } } //------------------------------final_graph_reshaping_walk--------------------- --- old/src/share/vm/opto/loopnode.hpp 2015-11-09 11:56:25.968969837 +0100 +++ new/src/share/vm/opto/loopnode.hpp 2015-11-09 11:56:25.355331093 +0100 @@ -916,7 +916,8 @@ // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry, - Deoptimization::DeoptReason reason); + Deoptimization::DeoptReason reason, + int opcode); void register_control(Node* n, IdealLoopTree *loop, Node* pred); // Clone loop predicates to cloned loops (peeled, unswitched) @@ -966,7 +967,8 @@ // Create a slow version of the loop by cloning the loop // and inserting an if to select fast-slow versions. ProjNode* create_slow_version_of_loop(IdealLoopTree *loop, - Node_List &old_new); + Node_List &old_new, + int opcode); // Clone a loop and return the clone head (clone_loop_head). // Added nodes include int(1), int(0) - disconnected, If, IfTrue, IfFalse, --- old/src/share/vm/opto/loopPredicate.cpp 2015-11-09 11:56:25.980916724 +0100 +++ new/src/share/vm/opto/loopPredicate.cpp 2015-11-09 11:56:25.396056169 +0100 @@ -91,7 +91,8 @@ // The true projecttion (if_cont) of the new_iff is returned. // This code is also used to clone predicates to cloned loops. ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry, - Deoptimization::DeoptReason reason) { + Deoptimization::DeoptReason reason, + int opcode) { assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); IfNode* iff = cont_proj->in(0)->as_If(); @@ -133,8 +134,13 @@ } // Create new_iff IdealLoopTree* lp = get_loop(entry); - IfNode *new_iff = iff->clone()->as_If(); - new_iff->set_req(0, entry); + IfNode* new_iff = NULL; + if (opcode == Op_If) { + new_iff = new IfNode(entry, iff->in(1), iff->_prob, iff->_fcnt); + } else { + assert(opcode == Op_RangeCheck, "no other if variant here"); + new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt); + } register_control(new_iff, lp, entry); Node *if_cont = new IfTrueNode(new_iff); Node *if_uct = new IfFalseNode(new_iff); @@ -183,7 +189,8 @@ //------------------------------create_new_if_for_predicate------------------------ // Create a new if below new_entry for the predicate to be cloned (IGVN optimization) ProjNode* PhaseIterGVN::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry, - Deoptimization::DeoptReason reason) { + Deoptimization::DeoptReason reason, + int opcode) { assert(new_entry != 0, "only used for clone predicate"); assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); IfNode* iff = cont_proj->in(0)->as_If(); @@ -208,8 +215,13 @@ } // Create new_iff in new location. - IfNode *new_iff = iff->clone()->as_If(); - new_iff->set_req(0, new_entry); + IfNode* new_iff = NULL; + if (opcode == Op_If) { + new_iff = new IfNode(new_entry, iff->in(1), iff->_prob, iff->_fcnt); + } else { + assert(opcode == Op_RangeCheck, "no other if variant here"); + new_iff = new RangeCheckNode(new_entry, iff->in(1), iff->_prob, iff->_fcnt); + } register_new_node_with_optimizer(new_iff); Node *if_cont = new IfTrueNode(new_iff); @@ -249,9 +261,9 @@ PhaseIterGVN* igvn) { ProjNode* new_predicate_proj; if (loop_phase != NULL) { - new_predicate_proj = loop_phase->create_new_if_for_predicate(predicate_proj, new_entry, reason); + new_predicate_proj = loop_phase->create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If); } else { - new_predicate_proj = igvn->create_new_if_for_predicate(predicate_proj, new_entry, reason); + new_predicate_proj = igvn->create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If); } IfNode* iff = new_predicate_proj->in(0)->as_If(); Node* ctrl = iff->in(0); @@ -714,7 +726,8 @@ while (current_proj != head) { if (loop == get_loop(current_proj) && // still in the loop ? current_proj->is_Proj() && // is a projection ? - current_proj->in(0)->Opcode() == Op_If) { // is a if projection ? + (current_proj->in(0)->Opcode() == Op_If || + current_proj->in(0)->Opcode() == Op_RangeCheck)) { // is a if projection ? if_proj_list.push(current_proj); } current_proj = idom(current_proj); @@ -753,7 +766,8 @@ if (invar.is_invariant(bol)) { // Invariant test new_predicate_proj = create_new_if_for_predicate(predicate_proj, NULL, - Deoptimization::Reason_predicate); + Deoptimization::Reason_predicate, + iff->Opcode()); Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0); BoolNode* new_predicate_bol = invar.clone(bol, ctrl)->as_Bool(); @@ -797,8 +811,8 @@ // lower_bound test will dominate the upper bound test and all // cloned or created nodes will use the lower bound test as // their declared control. - ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate); - ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate); + ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate, iff->Opcode()); + ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate, iff->Opcode()); assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate"); Node *ctrl = lower_bound_proj->in(0)->as_If()->in(0); --- old/src/share/vm/opto/ifnode.cpp 2015-11-09 11:56:26.024095254 +0100 +++ new/src/share/vm/opto/ifnode.cpp 2015-11-09 11:56:25.569690215 +0100 @@ -306,12 +306,16 @@ Node *b_c = phase->transform(new BoolNode(cmp_c,b->_test._test)); Node *b_x = phase->transform(new BoolNode(cmp_x,b->_test._test)); // Make the IfNode - IfNode *iff_c = new IfNode(region_c,b_c,iff->_prob,iff->_fcnt); + IfNode* iff_c = iff->clone()->as_If(); + iff_c->set_req(0, region_c); + iff_c->set_req(1, b_c); igvn->set_type_bottom(iff_c); igvn->_worklist.push(iff_c); hook->init_req(2, iff_c); - IfNode *iff_x = new IfNode(region_x,b_x,iff->_prob, iff->_fcnt); + IfNode* iff_x = iff->clone()->as_If(); + iff_x->set_req(0, region_x); + iff_x->set_req(1, b_x); igvn->set_type_bottom(iff_x); igvn->_worklist.push(iff_x); hook->init_req(3, iff_x); @@ -496,7 +500,7 @@ // Return 0 if not a range check. Return 1 if a range check and set index and // offset. Return 2 if we had to negate the test. Index is NULL if the check // is versus a constant. -int IfNode::is_range_check(Node* &range, Node* &index, jint &offset) { +int RangeCheckNode::is_range_check(Node* &range, Node* &index, jint &offset) { int flip_test = 0; Node* l = NULL; Node* r = NULL; @@ -724,7 +728,7 @@ return ctrl != NULL && ctrl->is_Proj() && ctrl->in(0) != NULL && - ctrl->in(0)->is_If() && + ctrl->in(0)->Opcode() == Op_If && ctrl->in(0)->outcnt() == 2 && ctrl->in(0)->as_If()->cmpi_folds(igvn) && // Must compare same value @@ -972,8 +976,8 @@ if (failtype->_lo > failtype->_hi) { // previous if determines the result of this if so // replace Bool with constant - igvn->hash_delete(this); - set_req(1, igvn->intcon(success->_con)); + igvn->_worklist.push(in(1)); + igvn->replace_input_of(this, 1, igvn->intcon(success->_con)); return true; } } @@ -992,7 +996,8 @@ Node* newbool = igvn->transform(new BoolNode(newcmp, cond)); igvn->replace_input_of(dom_iff, 1, igvn->intcon(proj->_con)); - set_req(1, newbool); + igvn->_worklist.push(in(1)); + igvn->replace_input_of(this, 1, newbool); return true; } @@ -1002,7 +1007,10 @@ // Merge the branches that trap for this If and the dominating If into // a single region that branches to the uncommon trap for the // dominating If -void IfNode::merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn) { +Node* IfNode::merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn) { + Node* res = this; + assert(success->in(0) == this, "bad projection"); + ProjNode* otherproj = proj->other_if_proj(); CallStaticJavaNode* unc = success->is_uncommon_trap_proj(Deoptimization::Reason_none); @@ -1038,6 +1046,8 @@ trap_request = Deoptimization::make_trap_request(Deoptimization::Reason_range_check, action); improve_address_types(l, r, fail, igvn); + + res = igvn->transform(new RangeCheckNode(in(0), in(1), _prob, _fcnt)); } else if (unc != dom_unc) { // If we trap we won't know what CmpI would have caused the trap // so use a special trap reason to mark this pair of CmpI nodes as @@ -1047,6 +1057,7 @@ trap_request = Deoptimization::make_trap_request(Deoptimization::Reason_unstable_fused_if, action); } igvn->replace_input_of(dom_unc, TypeFunc::Parms, igvn->intcon(trap_request)); + return res; } // If we are turning 2 CmpI nodes into a CmpU that follows the pattern @@ -1240,8 +1251,7 @@ if (has_only_uncommon_traps(dom_cmp, success, fail, igvn) && // Next call modifies graph so must be last fold_compares_helper(dom_cmp, success, fail, igvn)) { - merge_uncommon_traps(dom_cmp, success, fail, igvn); - return this; + return merge_uncommon_traps(dom_cmp, success, fail, igvn); } return NULL; } else if (ctrl->in(0) != NULL && @@ -1260,8 +1270,7 @@ // Next call modifies graph so must be last fold_compares_helper(dom_cmp, success, fail, igvn)) { reroute_side_effect_free_unc(other_cmp, dom_cmp, igvn); - merge_uncommon_traps(dom_cmp, success, fail, igvn); - return this; + return merge_uncommon_traps(dom_cmp, success, fail, igvn); } } } @@ -1342,14 +1351,10 @@ jint off; }; -//------------------------------Ideal------------------------------------------ -// Return a node which is more "ideal" than the current node. Strip out -// control copies -Node *IfNode::Ideal(PhaseGVN *phase, bool can_reshape) { +Node* IfNode::Ideal_common(PhaseGVN *phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) return this; // No Def-Use info? if (!can_reshape) return NULL; - PhaseIterGVN *igvn = phase->is_IterGVN(); // Don't bother trying to transform a dead if if (in(0)->is_top()) return NULL; @@ -1365,212 +1370,69 @@ if (idt_if != NULL) return idt_if; // Try to split the IF + PhaseIterGVN *igvn = phase->is_IterGVN(); Node *s = split_if(this, igvn); if (s != NULL) return s; + return NodeSentinel; +} + +//------------------------------Ideal------------------------------------------ +// Return a node which is more "ideal" than the current node. Strip out +// control copies +Node* IfNode::Ideal(PhaseGVN *phase, bool can_reshape) { + Node* res = Ideal_common(phase, can_reshape); + if (res != NodeSentinel) { + return res; + } + // Check for people making a useless boolean: things like // if( (x < y ? true : false) ) { ... } // Replace with if( x < y ) { ... } Node *bol2 = remove_useless_bool(this, phase); if( bol2 ) return bol2; - // Setup to scan up the CFG looking for a dominating test - Node *dom = in(0); - Node *prev_dom = this; + if (in(0) == NULL) return NULL; // Dead loop? - // Check for range-check vs other kinds of tests - Node *index1, *range1; - jint offset1; - int flip1 = is_range_check(range1, index1, offset1); - if( flip1 ) { - // Try to remove extra range checks. All 'up_one_dom' gives up at merges - // so all checks we inspect post-dominate the top-most check we find. - // If we are going to fail the current check and we reach the top check - // then we are guaranteed to fail, so just start interpreting there. - // We 'expand' the top 3 range checks to include all post-dominating - // checks. - - // The top 3 range checks seen - const int NRC =3; - RangeCheck prev_checks[NRC]; - int nb_checks = 0; - - // Low and high offsets seen so far - jint off_lo = offset1; - jint off_hi = offset1; - - bool found_immediate_dominator = false; - - // Scan for the top checks and collect range of offsets - for (int dist = 0; dist < 999; dist++) { // Range-Check scan limit - if (dom->Opcode() == Op_If && // Not same opcode? - prev_dom->in(0) == dom) { // One path of test does dominate? - if (dom == this) return NULL; // dead loop - // See if this is a range check - Node *index2, *range2; - jint offset2; - int flip2 = dom->as_If()->is_range_check(range2, index2, offset2); - // See if this is a _matching_ range check, checking against - // the same array bounds. - if (flip2 == flip1 && range2 == range1 && index2 == index1 && - dom->outcnt() == 2) { - if (nb_checks == 0 && dom->in(1) == in(1)) { - // Found an immediately dominating test at the same offset. - // This kind of back-to-back test can be eliminated locally, - // and there is no need to search further for dominating tests. - assert(offset2 == offset1, "Same test but different offsets"); - found_immediate_dominator = true; - break; - } - // Gather expanded bounds - off_lo = MIN2(off_lo,offset2); - off_hi = MAX2(off_hi,offset2); - // Record top NRC range checks - prev_checks[nb_checks%NRC].ctl = prev_dom; - prev_checks[nb_checks%NRC].off = offset2; - nb_checks++; - } - } - prev_dom = dom; - dom = up_one_dom(dom); - if (!dom) break; - } - - if (!found_immediate_dominator) { - // Attempt to widen the dominating range check to cover some later - // ones. Since range checks "fail" by uncommon-trapping to the - // interpreter, widening a check can make us speculatively enter - // the interpreter. If we see range-check deopt's, do not widen! - if (!phase->C->allow_range_check_smearing()) return NULL; - - // Didn't find prior covering check, so cannot remove anything. - if (nb_checks == 0) { - return NULL; - } - // Constant indices only need to check the upper bound. - // Non-constant indices must check both low and high. - int chk0 = (nb_checks - 1) % NRC; - if (index1) { - if (nb_checks == 1) { - return NULL; - } else { - // If the top range check's constant is the min or max of - // all constants we widen the next one to cover the whole - // range of constants. - RangeCheck rc0 = prev_checks[chk0]; - int chk1 = (nb_checks - 2) % NRC; - RangeCheck rc1 = prev_checks[chk1]; - if (rc0.off == off_lo) { - adjust_check(rc1.ctl, range1, index1, flip1, off_hi, igvn); - prev_dom = rc1.ctl; - } else if (rc0.off == off_hi) { - adjust_check(rc1.ctl, range1, index1, flip1, off_lo, igvn); - prev_dom = rc1.ctl; - } else { - // If the top test's constant is not the min or max of all - // constants, we need 3 range checks. We must leave the - // top test unchanged because widening it would allow the - // accesses it protects to successfully read/write out of - // bounds. - if (nb_checks == 2) { - return NULL; - } - int chk2 = (nb_checks - 3) % NRC; - RangeCheck rc2 = prev_checks[chk2]; - // The top range check a+i covers interval: -a <= i < length-a - // The second range check b+i covers interval: -b <= i < length-b - if (rc1.off <= rc0.off) { - // if b <= a, we change the second range check to: - // -min_of_all_constants <= i < length-min_of_all_constants - // Together top and second range checks now cover: - // -min_of_all_constants <= i < length-a - // which is more restrictive than -b <= i < length-b: - // -b <= -min_of_all_constants <= i < length-a <= length-b - // The third check is then changed to: - // -max_of_all_constants <= i < length-max_of_all_constants - // so 2nd and 3rd checks restrict allowed values of i to: - // -min_of_all_constants <= i < length-max_of_all_constants - adjust_check(rc1.ctl, range1, index1, flip1, off_lo, igvn); - adjust_check(rc2.ctl, range1, index1, flip1, off_hi, igvn); - } else { - // if b > a, we change the second range check to: - // -max_of_all_constants <= i < length-max_of_all_constants - // Together top and second range checks now cover: - // -a <= i < length-max_of_all_constants - // which is more restrictive than -b <= i < length-b: - // -b < -a <= i < length-max_of_all_constants <= length-b - // The third check is then changed to: - // -max_of_all_constants <= i < length-max_of_all_constants - // so 2nd and 3rd checks restrict allowed values of i to: - // -min_of_all_constants <= i < length-max_of_all_constants - adjust_check(rc1.ctl, range1, index1, flip1, off_hi, igvn); - adjust_check(rc2.ctl, range1, index1, flip1, off_lo, igvn); - } - prev_dom = rc2.ctl; - } - } - } else { - RangeCheck rc0 = prev_checks[chk0]; - // 'Widen' the offset of the 1st and only covering check - adjust_check(rc0.ctl, range1, index1, flip1, off_hi, igvn); - // Test is now covered by prior checks, dominate it out - prev_dom = rc0.ctl; - } - } - - } else { // Scan for an equivalent test + PhaseIterGVN *igvn = phase->is_IterGVN(); + Node* result = fold_compares(igvn); + if (result != NULL) { + return result; + } - Node *cmp; - int dist = 0; // Cutoff limit for search - int op = Opcode(); - if( op == Op_If && - (cmp=in(1)->in(1))->Opcode() == Op_CmpP ) { - if( cmp->in(2) != NULL && // make sure cmp is not already dead - cmp->in(2)->bottom_type() == TypePtr::NULL_PTR ) { - dist = 64; // Limit for null-pointer scans - } else { - dist = 4; // Do not bother for random pointer tests - } + // Scan for an equivalent test + Node *cmp; + int dist = 0; // Cutoff limit for search + int op = Opcode(); + if( op == Op_If && + (cmp=in(1)->in(1))->Opcode() == Op_CmpP ) { + if( cmp->in(2) != NULL && // make sure cmp is not already dead + cmp->in(2)->bottom_type() == TypePtr::NULL_PTR ) { + dist = 64; // Limit for null-pointer scans } else { - dist = 4; // Limit for random junky scans - } - - // Normal equivalent-test check. - if( !dom ) return NULL; // Dead loop? - - Node* result = fold_compares(igvn); - if (result != NULL) { - return result; - } - - // Search up the dominator tree for an If with an identical test - while( dom->Opcode() != op || // Not same opcode? - dom->in(1) != in(1) || // Not same input 1? - (req() == 3 && dom->in(2) != in(2)) || // Not same input 2? - prev_dom->in(0) != dom ) { // One path of test does not dominate? - if( dist < 0 ) return NULL; - - dist--; - prev_dom = dom; - dom = up_one_dom( dom ); - if( !dom ) return NULL; + dist = 4; // Do not bother for random pointer tests } + } else { + dist = 4; // Limit for random junky scans + } - // Check that we did not follow a loop back to ourselves - if( this == dom ) - return NULL; + Node* prev_dom = search_identical(dist); - if( dist > 2 ) // Add to count of NULL checks elided - explicit_null_checks_elided++; + if (prev_dom == NULL) { + return NULL; + } - } // End of Else scan for an equivalent test + // Replace dominated IfNode + return dominated_by(prev_dom, igvn); +} - // Hit! Remove this IF +//------------------------------dominated_by----------------------------------- +Node* IfNode::dominated_by(Node* prev_dom, PhaseIterGVN *igvn) { #ifndef PRODUCT - if( TraceIterativeGVN ) { + if (TraceIterativeGVN) { tty->print(" Removing IfNode: "); this->dump(); } - if( VerifyOpto && !phase->allow_progress() ) { + if (VerifyOpto && !igvn->allow_progress()) { // Found an equivalent dominating test, // we can not guarantee reaching a fix-point for these during iterativeGVN // since intervening nodes may not change. @@ -1578,16 +1440,6 @@ } #endif - // Replace dominated IfNode - dominated_by( prev_dom, igvn ); - - // Must return either the original node (now dead) or a new node - // (Do not return a top here, since that would break the uniqueness of top.) - return new ConINode(TypeInt::ZERO); -} - -//------------------------------dominated_by----------------------------------- -void IfNode::dominated_by( Node *prev_dom, PhaseIterGVN *igvn ) { igvn->hash_delete(this); // Remove self to prevent spurious V-N Node *idom = in(0); // Need opcode to decide which way 'this' test goes @@ -1635,6 +1487,38 @@ // Kill the IfNode igvn->remove_dead_node(this); + + // Must return either the original node (now dead) or a new node + // (Do not return a top here, since that would break the uniqueness of top.) + return new ConINode(TypeInt::ZERO); +} + +Node* IfNode::search_identical(int dist) { + // Setup to scan up the CFG looking for a dominating test + Node* dom = in(0); + Node* prev_dom = this; + int op = Opcode(); + // Search up the dominator tree for an If with an identical test + while( dom->Opcode() != op || // Not same opcode? + dom->in(1) != in(1) || // Not same input 1? + (req() == 3 && dom->in(2) != in(2)) || // Not same input 2? + prev_dom->in(0) != dom ) { // One path of test does not dominate? + if( dist < 0 ) return NULL; + + dist--; + prev_dom = dom; + dom = up_one_dom( dom ); + if( !dom ) return NULL; + } + + // Check that we did not follow a loop back to ourselves + if( this == dom ) + return NULL; + + if( dist > 2 ) // Add to count of NULL checks elided + explicit_null_checks_elided++; + + return prev_dom; } //------------------------------Identity--------------------------------------- @@ -1726,7 +1610,9 @@ assert( igvn, "Test is not canonical in parser?" ); // The IF node never really changes, but it needs to be cloned - iff = new IfNode( iff->in(0), b, 1.0-iff->_prob, iff->_fcnt); + iff = iff->clone()->as_If(); + iff->set_req(1, b); + iff->_prob = 1.0-iff->_prob; Node *prior = igvn->hash_find_insert(iff); if( prior ) { @@ -1751,3 +1637,167 @@ // Progress return iff; } + +Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) { + Node* res = Ideal_common(phase, can_reshape); + if (res != NodeSentinel) { + return res; + } + + PhaseIterGVN *igvn = phase->is_IterGVN(); + // Setup to scan up the CFG looking for a dominating test + Node* prev_dom = this; + + // Check for range-check vs other kinds of tests + Node* index1; + Node* range1; + jint offset1; + int flip1 = is_range_check(range1, index1, offset1); + if (flip1) { + Node* dom = in(0); + // Try to remove extra range checks. All 'up_one_dom' gives up at merges + // so all checks we inspect post-dominate the top-most check we find. + // If we are going to fail the current check and we reach the top check + // then we are guaranteed to fail, so just start interpreting there. + // We 'expand' the top 3 range checks to include all post-dominating + // checks. + + // The top 3 range checks seen + const int NRC =3; + RangeCheck prev_checks[NRC]; + int nb_checks = 0; + + // Low and high offsets seen so far + jint off_lo = offset1; + jint off_hi = offset1; + + bool found_immediate_dominator = false; + + // Scan for the top checks and collect range of offsets + for (int dist = 0; dist < 999; dist++) { // Range-Check scan limit + if (dom->Opcode() == Op_RangeCheck && // Not same opcode? + prev_dom->in(0) == dom) { // One path of test does dominate? + if (dom == this) return NULL; // dead loop + // See if this is a range check + Node* index2; + Node* range2; + jint offset2; + int flip2 = dom->as_RangeCheck()->is_range_check(range2, index2, offset2); + // See if this is a _matching_ range check, checking against + // the same array bounds. + if (flip2 == flip1 && range2 == range1 && index2 == index1 && + dom->outcnt() == 2) { + if (nb_checks == 0 && dom->in(1) == in(1)) { + // Found an immediately dominating test at the same offset. + // This kind of back-to-back test can be eliminated locally, + // and there is no need to search further for dominating tests. + assert(offset2 == offset1, "Same test but different offsets"); + found_immediate_dominator = true; + break; + } + // Gather expanded bounds + off_lo = MIN2(off_lo,offset2); + off_hi = MAX2(off_hi,offset2); + // Record top NRC range checks + prev_checks[nb_checks%NRC].ctl = prev_dom; + prev_checks[nb_checks%NRC].off = offset2; + nb_checks++; + } + } + prev_dom = dom; + dom = up_one_dom(dom); + if (!dom) break; + } + + if (!found_immediate_dominator) { + // Attempt to widen the dominating range check to cover some later + // ones. Since range checks "fail" by uncommon-trapping to the + // interpreter, widening a check can make us speculatively enter + // the interpreter. If we see range-check deopt's, do not widen! + if (!phase->C->allow_range_check_smearing()) return NULL; + + // Didn't find prior covering check, so cannot remove anything. + if (nb_checks == 0) { + return NULL; + } + // Constant indices only need to check the upper bound. + // Non-constant indices must check both low and high. + int chk0 = (nb_checks - 1) % NRC; + if (index1) { + if (nb_checks == 1) { + return NULL; + } else { + // If the top range check's constant is the min or max of + // all constants we widen the next one to cover the whole + // range of constants. + RangeCheck rc0 = prev_checks[chk0]; + int chk1 = (nb_checks - 2) % NRC; + RangeCheck rc1 = prev_checks[chk1]; + if (rc0.off == off_lo) { + adjust_check(rc1.ctl, range1, index1, flip1, off_hi, igvn); + prev_dom = rc1.ctl; + } else if (rc0.off == off_hi) { + adjust_check(rc1.ctl, range1, index1, flip1, off_lo, igvn); + prev_dom = rc1.ctl; + } else { + // If the top test's constant is not the min or max of all + // constants, we need 3 range checks. We must leave the + // top test unchanged because widening it would allow the + // accesses it protects to successfully read/write out of + // bounds. + if (nb_checks == 2) { + return NULL; + } + int chk2 = (nb_checks - 3) % NRC; + RangeCheck rc2 = prev_checks[chk2]; + // The top range check a+i covers interval: -a <= i < length-a + // The second range check b+i covers interval: -b <= i < length-b + if (rc1.off <= rc0.off) { + // if b <= a, we change the second range check to: + // -min_of_all_constants <= i < length-min_of_all_constants + // Together top and second range checks now cover: + // -min_of_all_constants <= i < length-a + // which is more restrictive than -b <= i < length-b: + // -b <= -min_of_all_constants <= i < length-a <= length-b + // The third check is then changed to: + // -max_of_all_constants <= i < length-max_of_all_constants + // so 2nd and 3rd checks restrict allowed values of i to: + // -min_of_all_constants <= i < length-max_of_all_constants + adjust_check(rc1.ctl, range1, index1, flip1, off_lo, igvn); + adjust_check(rc2.ctl, range1, index1, flip1, off_hi, igvn); + } else { + // if b > a, we change the second range check to: + // -max_of_all_constants <= i < length-max_of_all_constants + // Together top and second range checks now cover: + // -a <= i < length-max_of_all_constants + // which is more restrictive than -b <= i < length-b: + // -b < -a <= i < length-max_of_all_constants <= length-b + // The third check is then changed to: + // -max_of_all_constants <= i < length-max_of_all_constants + // so 2nd and 3rd checks restrict allowed values of i to: + // -min_of_all_constants <= i < length-max_of_all_constants + adjust_check(rc1.ctl, range1, index1, flip1, off_hi, igvn); + adjust_check(rc2.ctl, range1, index1, flip1, off_lo, igvn); + } + prev_dom = rc2.ctl; + } + } + } else { + RangeCheck rc0 = prev_checks[chk0]; + // 'Widen' the offset of the 1st and only covering check + adjust_check(rc0.ctl, range1, index1, flip1, off_hi, igvn); + // Test is now covered by prior checks, dominate it out + prev_dom = rc0.ctl; + } + } + } else { + prev_dom = search_identical(4); + + if (prev_dom == NULL) { + return NULL; + } + } + + // Replace dominated IfNode + return dominated_by(prev_dom, igvn); +} --- old/src/share/vm/opto/phaseX.hpp 2015-11-09 11:56:28.618743168 +0100 +++ new/src/share/vm/opto/phaseX.hpp 2015-11-09 11:56:28.345263674 +0100 @@ -521,7 +521,8 @@ Node* clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check); // Create a new if below new_entry for the predicate to be cloned ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry, - Deoptimization::DeoptReason reason); + Deoptimization::DeoptReason reason, + int opcode); void remove_speculative_types(); void check_no_speculative_types() {