< prev index next >
src/hotspot/share/opto/loopTransform.cpp
Print this page
@@ -133,15 +133,49 @@
}
//------------------------------compute_profile_trip_cnt----------------------------
// Compute loop trip count from profile data as
// (backedge_count + loop_exit_count) / loop_exit_count
-void IdealLoopTree::compute_profile_trip_cnt( PhaseIdealLoop *phase ) {
- if (!_head->is_CountedLoop()) {
+
+float IdealLoopTree::compute_profile_trip_cnt_helper(Node* n) {
+ if (n->is_If()) {
+ IfNode *iff = n->as_If();
+ if (iff->_fcnt != COUNT_UNKNOWN && iff->_prob != PROB_UNKNOWN) {
+ Node *exit = is_loop_exit(iff);
+ if (exit) {
+ float exit_prob = iff->_prob;
+ if (exit->Opcode() == Op_IfFalse) exit_prob = 1.0 - exit_prob;
+ if (exit_prob > PROB_MIN) {
+ float exit_cnt = iff->_fcnt * exit_prob;
+ return exit_cnt;
+ }
+ }
+ }
+ }
+ if (n->is_Jump()) {
+ JumpNode *jmp = n->as_Jump();
+ if (jmp->_fcnt != COUNT_UNKNOWN) {
+ float* probs = jmp->_probs;
+ float exit_prob = 0;
+ PhaseIdealLoop *phase = _phase;
+ for (DUIterator_Fast imax, i = jmp->fast_outs(imax); i < imax; i++) {
+ JumpProjNode* u = jmp->fast_out(i)->as_JumpProj();
+ if (!is_member(_phase->get_loop(u))) {
+ exit_prob += probs[u->_con];
+ }
+ }
+ return exit_prob * jmp->_fcnt;
+ }
+ }
+ return 0;
+}
+
+void IdealLoopTree::compute_profile_trip_cnt(PhaseIdealLoop *phase) {
+ if (!_head->is_Loop()) {
return;
}
- CountedLoopNode* head = _head->as_CountedLoop();
+ LoopNode* head = _head->as_Loop();
if (head->profile_trip_cnt() != COUNT_UNKNOWN) {
return; // Already computed
}
float trip_cnt = (float)max_jint; // default is big
@@ -149,46 +183,57 @@
while (back != head) {
if ((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
back->in(0) &&
back->in(0)->is_If() &&
back->in(0)->as_If()->_fcnt != COUNT_UNKNOWN &&
- back->in(0)->as_If()->_prob != PROB_UNKNOWN) {
+ back->in(0)->as_If()->_prob != PROB_UNKNOWN &&
+ (back->Opcode() == Op_IfTrue ? 1-back->in(0)->as_If()->_prob : back->in(0)->as_If()->_prob) > PROB_MIN) {
break;
}
back = phase->idom(back);
}
if (back != head) {
assert((back->Opcode() == Op_IfTrue || back->Opcode() == Op_IfFalse) &&
back->in(0), "if-projection exists");
IfNode* back_if = back->in(0)->as_If();
- float loop_back_cnt = back_if->_fcnt * back_if->_prob;
+ float loop_back_cnt = back_if->_fcnt * (back->Opcode() == Op_IfTrue ? back_if->_prob : (1 - back_if->_prob));
// Now compute a loop exit count
float loop_exit_cnt = 0.0f;
+ if (_child == NULL) {
for( uint i = 0; i < _body.size(); i++ ) {
Node *n = _body[i];
- if( n->is_If() ) {
- IfNode *iff = n->as_If();
- if( iff->_fcnt != COUNT_UNKNOWN && iff->_prob != PROB_UNKNOWN ) {
- Node *exit = is_loop_exit(iff);
- if( exit ) {
- float exit_prob = iff->_prob;
- if (exit->Opcode() == Op_IfFalse) exit_prob = 1.0 - exit_prob;
- if (exit_prob > PROB_MIN) {
- float exit_cnt = iff->_fcnt * exit_prob;
- loop_exit_cnt += exit_cnt;
+ loop_exit_cnt += compute_profile_trip_cnt_helper(n);
+ }
+ } else {
+ ResourceMark rm;
+ Unique_Node_List wq;
+ wq.push(back);
+ for (uint i = 0; i < wq.size(); i++) {
+ Node *n = wq.at(i);
+ assert(n->is_CFG(), "only control nodes");
+ if (n != head) {
+ if (n->is_Region()) {
+ for (uint j = 1; j < n->req(); j++) {
+ wq.push(n->in(j));
}
+ } else {
+ loop_exit_cnt += compute_profile_trip_cnt_helper(n);
+ wq.push(n->in(0));
}
}
}
+
}
if (loop_exit_cnt > 0.0f) {
trip_cnt = (loop_back_cnt + loop_exit_cnt) / loop_exit_cnt;
} else {
// No exit count so use
trip_cnt = loop_back_cnt;
}
+ } else {
+ head->mark_profile_trip_failed();
}
#ifndef PRODUCT
if (TraceProfileTripCount) {
tty->print_cr("compute_profile_trip_cnt lp: %d cnt: %f\n", head->_idx, trip_cnt);
}
@@ -1012,34 +1057,26 @@
// loop is never executed). When that happens, range check
// CastII/ConvI2L nodes cause some data paths to die. For consistency,
// the control paths must die too but the range checks were removed by
// predication. The range checks that we add here guarantee that they
// do.
-void PhaseIdealLoop::duplicate_predicates(CountedLoopNode* pre_head, Node* min_taken, Node* castii,
- IdealLoopTree* outer_loop, LoopNode* outer_main_head,
- uint dd_main_head) {
- if (UseLoopPredicate) {
- Node* entry = pre_head->in(LoopNode::EntryControl);
- Node* predicate = NULL;
- predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
+void PhaseIdealLoop::duplicate_predicates_helper(Node* predicate, Node* castii, IdealLoopTree* outer_loop,
+ LoopNode* outer_main_head, uint dd_main_head) {
if (predicate != NULL) {
- entry = entry->in(0)->in(0);
- }
- predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
- if (predicate != NULL) {
- IfNode* iff = entry->in(0)->as_If();
- ProjNode* uncommon_proj = iff->proj_out(1 - entry->as_Proj()->_con);
+ IfNode* iff = predicate->in(0)->as_If();
+ ProjNode* uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
assert(iff->in(1)->in(1)->Opcode() == Op_Opaque1, "unexpected predicate shape");
- entry = entry->in(0)->in(0);
- Node* prev_proj = min_taken;
- while (entry != NULL && entry->is_Proj() && entry->in(0)->is_If()) {
- uncommon_proj = entry->in(0)->as_If()->proj_out(1 - entry->as_Proj()->_con);
+ predicate = predicate->in(0)->in(0);
+ Node* current_proj = outer_main_head->in(LoopNode::EntryControl);
+ Node* prev_proj = current_proj;
+ while (predicate != NULL && predicate->is_Proj() && predicate->in(0)->is_If()) {
+ uncommon_proj = predicate->in(0)->as_If()->proj_out(1 - predicate->as_Proj()->_con);
if (uncommon_proj->unique_ctrl_out() != rgn)
break;
- iff = entry->in(0)->as_If();
+ iff = predicate->in(0)->as_If();
if (iff->in(1)->Opcode() == Op_Opaque4) {
Node_Stack to_clone(2);
to_clone.push(iff->in(1), 1);
uint current = C->unique();
Node* result = NULL;
@@ -1072,11 +1109,11 @@
if (op == Op_Opaque1) {
if (n->_idx < current) {
n = n->clone();
}
n->set_req(i, castii);
- register_new_node(n, min_taken);
+ register_new_node(n, current_proj);
to_clone.set_node(n);
}
for (;;) {
Node* cur = to_clone.node();
uint j = to_clone.index();
@@ -1092,21 +1129,21 @@
Node* next = to_clone.node();
j = to_clone.index();
if (cur->_idx >= current) {
if (next->_idx < current) {
next = next->clone();
- register_new_node(next, min_taken);
+ register_new_node(next, current_proj);
to_clone.set_node(next);
}
assert(next->in(j) != cur, "input should have been cloned");
next->set_req(j, cur);
}
}
} while (result == NULL);
assert(result->_idx >= current, "new node expected");
- Node* proj = entry->clone();
+ Node* proj = predicate->clone();
Node* other_proj = uncommon_proj->clone();
Node* new_iff = iff->clone();
new_iff->set_req(1, result);
proj->set_req(0, new_iff);
other_proj->set_req(0, new_iff);
@@ -1123,18 +1160,41 @@
register_control(other_proj, _ltree_root, new_iff);
register_control(halt, _ltree_root, other_proj);
prev_proj = proj;
}
- entry = entry->in(0)->in(0);
+ predicate = predicate->in(0)->in(0);
}
+ if (prev_proj != current_proj) {
_igvn.replace_input_of(outer_main_head, LoopNode::EntryControl, prev_proj);
set_idom(outer_main_head, prev_proj, dd_main_head);
}
}
}
+void PhaseIdealLoop::duplicate_predicates(CountedLoopNode* pre_head, Node* castii, IdealLoopTree* outer_loop,
+ LoopNode* outer_main_head, uint dd_main_head) {
+ if (UseLoopPredicate) {
+ Node* entry = pre_head->in(LoopNode::EntryControl);
+ Node* predicate = NULL;
+ predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
+ if (predicate != NULL) {
+ entry = entry->in(0)->in(0);
+ }
+ Node* profile_predicate = NULL;
+ if (UseProfiledLoopPredicate) {
+ profile_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
+ if (profile_predicate != NULL) {
+ entry = skip_loop_predicates(entry);
+ }
+ }
+ predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
+ duplicate_predicates_helper(predicate, castii, outer_loop, outer_main_head, dd_main_head);
+ duplicate_predicates_helper(profile_predicate, castii, outer_loop, outer_main_head, dd_main_head);
+ }
+}
+
//------------------------------insert_pre_post_loops--------------------------
// Insert pre and post loops. If peel_only is set, the pre-loop can not have
// more iterations added. It acts as a 'peel' only, no lower-bound RCE, no
// alignment. Useful to unroll loops that do no array accesses.
void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ) {
@@ -1274,11 +1334,11 @@
// dependencies.
// CastII for the main loop:
Node* castii = cast_incr_before_loop( pre_incr, min_taken, main_head );
assert(castii != NULL, "no castII inserted");
- duplicate_predicates(pre_head, min_taken, castii, outer_loop, outer_main_head, dd_main_head);
+ duplicate_predicates(pre_head, castii, outer_loop, outer_main_head, dd_main_head);
// Step B4: Shorten the pre-loop to run only 1 iteration (for now).
// RCE and alignment may change this later.
Node *cmp_end = pre_end->cmp_node();
assert( cmp_end->in(2) == limit, "" );
@@ -2811,11 +2871,11 @@
needs_guard = (init_t->_lo <= limit_t->_hi);
}
}
if (needs_guard) {
// Check for an obvious zero trip guard.
- Node* inctrl = PhaseIdealLoop::skip_loop_predicates(cl->skip_predicates());
+ Node* inctrl = PhaseIdealLoop::skip_all_loop_predicates(cl->skip_predicates());
if (inctrl->Opcode() == Op_IfTrue || inctrl->Opcode() == Op_IfFalse) {
bool maybe_swapped = (inctrl->Opcode() == Op_IfFalse);
// The test should look like just the backedge of a CountedLoop
Node* iff = inctrl->in(0);
if (iff->is_If()) {
< prev index next >