1394
1395 // See if a shared loop-varying computation has no loop-varying uses.
1396 // Happens if something is only used for JVM state in uncommon trap exits,
1397 // like various versions of induction variable+offset. Clone the
1398 // computation per usage to allow it to sink out of the loop.
1399 if (has_ctrl(n) && !n->in(0)) {// n not dead and has no control edge (can float about)
1400 Node *n_ctrl = get_ctrl(n);
1401 IdealLoopTree *n_loop = get_loop(n_ctrl);
1402 if( n_loop != _ltree_root ) {
1403 DUIterator_Fast imax, i = n->fast_outs(imax);
1404 for (; i < imax; i++) {
1405 Node* u = n->fast_out(i);
1406 if( !has_ctrl(u) ) break; // Found control user
1407 IdealLoopTree *u_loop = get_loop(get_ctrl(u));
1408 if( u_loop == n_loop ) break; // Found loop-varying use
1409 if( n_loop->is_member( u_loop ) ) break; // Found use in inner loop
1410 if( u->Opcode() == Op_Opaque1 ) break; // Found loop limit, bugfix for 4677003
1411 }
1412 bool did_break = (i < imax); // Did we break out of the previous loop?
1413 if (!did_break && n->outcnt() > 1) { // All uses in outer loops!
1414 Node *late_load_ctrl = NULL;
1415 if (n->is_Load()) {
1416 // If n is a load, get and save the result from get_late_ctrl(),
1417 // to be later used in calculating the control for n's clones.
1418 clear_dom_lca_tags();
1419 late_load_ctrl = get_late_ctrl(n, n_ctrl);
1420 }
1421 // If n is a load, and the late control is the same as the current
1422 // control, then the cloning of n is a pointless exercise, because
1423 // GVN will ensure that we end up where we started.
1424 if (!n->is_Load() || late_load_ctrl != n_ctrl) {
1425 for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin; ) {
1426 Node *u = n->last_out(j); // Clone private computation per use
1427 _igvn.rehash_node_delayed(u);
1428 Node *x = n->clone(); // Clone computation
1429 Node *x_ctrl = NULL;
1430 if( u->is_Phi() ) {
1431 // Replace all uses of normal nodes. Replace Phi uses
1432 // individually, so the separate Nodes can sink down
1433 // different paths.
1434 uint k = 1;
1435 while( u->in(k) != n ) k++;
1436 u->set_req( k, x );
1437 // x goes next to Phi input path
1438 x_ctrl = u->in(0)->in(k);
1439 --j;
1440 } else { // Normal use
1441 // Replace all uses
1442 for( uint k = 0; k < u->req(); k++ ) {
1443 if( u->in(k) == n ) {
1444 u->set_req( k, x );
1445 --j;
1446 }
1447 }
1448 x_ctrl = get_ctrl(u);
1449 }
1450
1451 // Find control for 'x' next to use but not inside inner loops.
1452 // For inner loop uses get the preheader area.
1453 x_ctrl = place_near_use(x_ctrl);
1454
1455 if (n->is_Load()) {
1456 // For loads, add a control edge to a CFG node outside of the loop
1457 // to force them to not combine and return back inside the loop
1458 // during GVN optimization (4641526).
1459 //
1460 // Because we are setting the actual control input, factor in
1461 // the result from get_late_ctrl() so we respect any
1462 // anti-dependences. (6233005).
1463 x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
1464
1465 // Don't allow the control input to be a CFG splitting node.
1466 // Such nodes should only have ProjNodes as outs, e.g. IfNode
1467 // should only have IfTrueNode and IfFalseNode (4985384).
1468 x_ctrl = find_non_split_ctrl(x_ctrl);
1469
1470 IdealLoopTree* x_loop = get_loop(x_ctrl);
1471 Node* x_head = x_loop->_head;
1472 if (x_head->is_Loop() && (x_head->is_OuterStripMinedLoop() || x_head->as_Loop()->is_strip_mined()) && is_dominator(n_ctrl, x_head)) {
1473 // Anti dependence analysis is sometimes too
1474 // conservative: a store in the outer strip mined loop
1475 // can prevent a load from floating out of the outer
1476 // strip mined loop but the load may not be referenced
1477 // from the safepoint: loop strip mining verification
1478 // code reports a problem in that case. Make sure the
1479 // load is not moved in the outer strip mined loop in
1480 // that case.
1481 x_ctrl = x_head->as_Loop()->skip_strip_mined()->in(LoopNode::EntryControl);
1482 }
1483 assert(dom_depth(n_ctrl) <= dom_depth(x_ctrl), "n is later than its clone");
|
1394
1395 // See if a shared loop-varying computation has no loop-varying uses.
1396 // Happens if something is only used for JVM state in uncommon trap exits,
1397 // like various versions of induction variable+offset. Clone the
1398 // computation per usage to allow it to sink out of the loop.
1399 if (has_ctrl(n) && !n->in(0)) {// n not dead and has no control edge (can float about)
1400 Node *n_ctrl = get_ctrl(n);
1401 IdealLoopTree *n_loop = get_loop(n_ctrl);
1402 if( n_loop != _ltree_root ) {
1403 DUIterator_Fast imax, i = n->fast_outs(imax);
1404 for (; i < imax; i++) {
1405 Node* u = n->fast_out(i);
1406 if( !has_ctrl(u) ) break; // Found control user
1407 IdealLoopTree *u_loop = get_loop(get_ctrl(u));
1408 if( u_loop == n_loop ) break; // Found loop-varying use
1409 if( n_loop->is_member( u_loop ) ) break; // Found use in inner loop
1410 if( u->Opcode() == Op_Opaque1 ) break; // Found loop limit, bugfix for 4677003
1411 }
1412 bool did_break = (i < imax); // Did we break out of the previous loop?
1413 if (!did_break && n->outcnt() > 1) { // All uses in outer loops!
1414 Node* late_load_ctrl = NULL;
1415 Node* outer_strip_mined_loop_exit = NULL;
1416 if (n->is_Load()) {
1417 // If n is a load, get and save the result from get_late_ctrl(),
1418 // to be later used in calculating the control for n's clones.
1419 clear_dom_lca_tags();
1420 late_load_ctrl = get_late_ctrl(n, n_ctrl);
1421 if (n_loop->head()->is_Loop() && n_loop->head()->as_Loop()->is_strip_mined() && !n_loop->head()->is_OuterStripMinedLoop()
1422 && n_loop->tail()->in(0) == late_load_ctrl->in(0)) {
1423 // late_load_ctrl is a loop exit of an inner loop of an outer strip mined loop.
1424 // Use the outer strip mined loop exit instead later if it dominates x_ctrl.
1425 outer_strip_mined_loop_exit = n_loop->_parent->head()->as_OuterStripMinedLoop()->outer_loop_exit();
1426 }
1427 }
1428
1429 // If n is a load, and the late control is the same as the current
1430 // control, then the cloning of n is a pointless exercise, because
1431 // GVN will ensure that we end up where we started.
1432 if (!n->is_Load() || late_load_ctrl != n_ctrl) {
1433 for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin; ) {
1434 Node *u = n->last_out(j); // Clone private computation per use
1435 _igvn.rehash_node_delayed(u);
1436 Node *x = n->clone(); // Clone computation
1437 Node *x_ctrl = NULL;
1438 if( u->is_Phi() ) {
1439 // Replace all uses of normal nodes. Replace Phi uses
1440 // individually, so the separate Nodes can sink down
1441 // different paths.
1442 uint k = 1;
1443 while( u->in(k) != n ) k++;
1444 u->set_req( k, x );
1445 // x goes next to Phi input path
1446 x_ctrl = u->in(0)->in(k);
1447 --j;
1448 } else { // Normal use
1449 // Replace all uses
1450 for( uint k = 0; k < u->req(); k++ ) {
1451 if( u->in(k) == n ) {
1452 u->set_req( k, x );
1453 --j;
1454 }
1455 }
1456 x_ctrl = get_ctrl(u);
1457 }
1458
1459 // Find control for 'x' next to use but not inside inner loops.
1460 // For inner loop uses get the preheader area.
1461 x_ctrl = place_near_use(x_ctrl);
1462
1463 if (n->is_Load()) {
1464 // For loads, add a control edge to a CFG node outside of the loop
1465 // to force them to not combine and return back inside the loop
1466 // during GVN optimization (4641526).
1467 //
1468 // Because we are setting the actual control input, factor in the result from get_late_ctrl() so we respect
1469 // any anti-dependences (6233005). If late_load_ctrl is an inner loop exit inside an outer strip mined loop
1470 // whose exit dominates the latest possible placement for x, then use this outer strip mined loop exit
1471 // instead of the inner loop exit (= late_load_ctrl) to move the load completely out of the loop.
1472 // This still respects any anti-dependencies in get_late_ctrl().
1473 if (outer_strip_mined_loop_exit != NULL && is_dominator(outer_strip_mined_loop_exit, get_late_ctrl(x, late_load_ctrl))) {
1474 assert(dom_lca(outer_strip_mined_loop_exit, x_ctrl) == outer_strip_mined_loop_exit, "must be the same result");
1475 x_ctrl = outer_strip_mined_loop_exit;
1476 } else {
1477 x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
1478 }
1479
1480 // Don't allow the control input to be a CFG splitting node.
1481 // Such nodes should only have ProjNodes as outs, e.g. IfNode
1482 // should only have IfTrueNode and IfFalseNode (4985384).
1483 x_ctrl = find_non_split_ctrl(x_ctrl);
1484
1485 IdealLoopTree* x_loop = get_loop(x_ctrl);
1486 Node* x_head = x_loop->_head;
1487 if (x_head->is_Loop() && (x_head->is_OuterStripMinedLoop() || x_head->as_Loop()->is_strip_mined()) && is_dominator(n_ctrl, x_head)) {
1488 // Anti dependence analysis is sometimes too
1489 // conservative: a store in the outer strip mined loop
1490 // can prevent a load from floating out of the outer
1491 // strip mined loop but the load may not be referenced
1492 // from the safepoint: loop strip mining verification
1493 // code reports a problem in that case. Make sure the
1494 // load is not moved in the outer strip mined loop in
1495 // that case.
1496 x_ctrl = x_head->as_Loop()->skip_strip_mined()->in(LoopNode::EntryControl);
1497 }
1498 assert(dom_depth(n_ctrl) <= dom_depth(x_ctrl), "n is later than its clone");
|