1507 #ifndef PRODUCT
1508 if (TraceLoopOpts) {
1509 tty->print("MaxUnroll %d ", cl->trip_count());
1510 loop->dump_head();
1511 }
1512 #endif
1513
1514 // If loop is tripping an odd number of times, peel odd iteration
1515 if ((cl->trip_count() & 1) == 1) {
1516 do_peeling(loop, old_new);
1517 }
1518
1519 // Now its tripping an even number of times remaining. Double loop body.
1520 // Do not adjust pre-guards; they are not needed and do not exist.
1521 if (cl->trip_count() > 0) {
1522 assert((cl->trip_count() & 1) == 0, "missed peeling");
1523 do_unroll(loop, old_new, false);
1524 }
1525 }
1526
1527 //------------------------------dominates_backedge---------------------------------
1528 // Returns true if ctrl is executed on every complete iteration
1529 bool IdealLoopTree::dominates_backedge(Node* ctrl) {
1530 assert(ctrl->is_CFG(), "must be control");
1531 Node* backedge = _head->as_Loop()->in(LoopNode::LoopBackControl);
1532 return _phase->dom_lca_internal(ctrl, backedge) == ctrl;
1533 }
1534
1535 //------------------------------adjust_limit-----------------------------------
1536 // Helper function for add_constraint().
1537 Node* PhaseIdealLoop::adjust_limit(int stride_con, Node * scale, Node *offset, Node *rc_limit, Node *loop_limit, Node *pre_ctrl) {
1538 // Compute "I :: (limit-offset)/scale"
1539 Node *con = new SubINode(rc_limit, offset);
1540 register_new_node(con, pre_ctrl);
1541 Node *X = new DivINode(0, con, scale);
1542 register_new_node(X, pre_ctrl);
1543
1544 // Adjust loop limit
1545 loop_limit = (stride_con > 0)
1546 ? (Node*)(new MinINode(loop_limit, X))
2344 bool may_rce_align = !policy_peel_only(phase) || should_rce || should_align;
2345
2346 // If we have any of these conditions (RCE, alignment, unrolling) met, then
2347 // we switch to the pre-/main-/post-loop model. This model also covers
2348 // peeling.
2349 if (should_rce || should_align || should_unroll) {
2350 if (cl->is_normal_loop()) // Convert to 'pre/main/post' loops
2351 phase->insert_pre_post_loops(this,old_new, !may_rce_align);
2352
2353 // Adjust the pre- and main-loop limits to let the pre and post loops run
2354 // with full checks, but the main-loop with no checks. Remove said
2355 // checks from the main body.
2356 if (should_rce)
2357 phase->do_range_check(this,old_new);
2358
2359 // Double loop body for unrolling. Adjust the minimum-trip test (will do
2360 // twice as many iterations as before) and the main body limit (only do
2361 // an even number of trips). If we are peeling, we might enable some RCE
2362 // and we'd rather unroll the post-RCE'd loop SO... do not unroll if
2363 // peeling.
2364 if (should_unroll && !should_peel)
2365 phase->do_unroll(this,old_new, true);
2366
2367 // Adjust the pre-loop limits to align the main body
2368 // iterations.
2369 if (should_align)
2370 Unimplemented();
2371
2372 } else { // Else we have an unchanged counted loop
2373 if (should_peel) // Might want to peel but do nothing else
2374 phase->do_peeling(this,old_new);
2375 }
2376 return true;
2377 }
2378
2379
2380 //=============================================================================
2381 //------------------------------iteration_split--------------------------------
2382 bool IdealLoopTree::iteration_split( PhaseIdealLoop *phase, Node_List &old_new ) {
2383 // Recursively iteration split nested loops
2384 if (_child && !_child->iteration_split(phase, old_new))
2385 return false;
|
1507 #ifndef PRODUCT
1508 if (TraceLoopOpts) {
1509 tty->print("MaxUnroll %d ", cl->trip_count());
1510 loop->dump_head();
1511 }
1512 #endif
1513
1514 // If loop is tripping an odd number of times, peel odd iteration
1515 if ((cl->trip_count() & 1) == 1) {
1516 do_peeling(loop, old_new);
1517 }
1518
1519 // Now its tripping an even number of times remaining. Double loop body.
1520 // Do not adjust pre-guards; they are not needed and do not exist.
1521 if (cl->trip_count() > 0) {
1522 assert((cl->trip_count() & 1) == 0, "missed peeling");
1523 do_unroll(loop, old_new, false);
1524 }
1525 }
1526
1527 void PhaseIdealLoop::mark_reductions(IdealLoopTree *loop) {
1528 if (SuperWordReductions == false) return;
1529
1530 CountedLoopNode* loop_head = loop->_head->as_CountedLoop();
1531 if (loop_head->unrolled_count() > 1) {
1532 return;
1533 }
1534
1535 Node* trip_phi = loop_head->phi();
1536 for (DUIterator_Fast imax, i = loop_head->fast_outs(imax); i < imax; i++) {
1537 Node* phi = loop_head->fast_out(i);
1538 if (phi->is_Phi() && phi->outcnt() > 0 && phi != trip_phi) {
1539 // For definitions which are loop inclusive and not tripcounts.
1540 Node* def_node = phi->in(LoopNode::LoopBackControl);
1541
1542 if (def_node != NULL) {
1543 Node* n_ctrl = get_ctrl(def_node);
1544 if (n_ctrl != NULL && loop->is_member(get_loop(n_ctrl))) {
1545 // Now test it to see if it fits the standard pattern for a reduction operator.
1546 int opc = def_node->Opcode();
1547 if (opc != ReductionNode::opcode(opc, def_node->bottom_type()->basic_type())) {
1548 if (!def_node->is_reduction()) { // Not marked yet
1549 // To be a reduction, the arithmetic node must have the phi as input and provide a def to it
1550 for (unsigned j = 1; j < def_node->req(); j++) {
1551 Node* in = def_node->in(j);
1552 if (in == phi) {
1553 def_node->add_flag(Node::Flag_is_reduction);
1554 break;
1555 }
1556 }
1557 }
1558 }
1559 }
1560 }
1561 }
1562 }
1563 }
1564
1565 //------------------------------dominates_backedge---------------------------------
1566 // Returns true if ctrl is executed on every complete iteration
1567 bool IdealLoopTree::dominates_backedge(Node* ctrl) {
1568 assert(ctrl->is_CFG(), "must be control");
1569 Node* backedge = _head->as_Loop()->in(LoopNode::LoopBackControl);
1570 return _phase->dom_lca_internal(ctrl, backedge) == ctrl;
1571 }
1572
1573 //------------------------------adjust_limit-----------------------------------
1574 // Helper function for add_constraint().
1575 Node* PhaseIdealLoop::adjust_limit(int stride_con, Node * scale, Node *offset, Node *rc_limit, Node *loop_limit, Node *pre_ctrl) {
1576 // Compute "I :: (limit-offset)/scale"
1577 Node *con = new SubINode(rc_limit, offset);
1578 register_new_node(con, pre_ctrl);
1579 Node *X = new DivINode(0, con, scale);
1580 register_new_node(X, pre_ctrl);
1581
1582 // Adjust loop limit
1583 loop_limit = (stride_con > 0)
1584 ? (Node*)(new MinINode(loop_limit, X))
2382 bool may_rce_align = !policy_peel_only(phase) || should_rce || should_align;
2383
2384 // If we have any of these conditions (RCE, alignment, unrolling) met, then
2385 // we switch to the pre-/main-/post-loop model. This model also covers
2386 // peeling.
2387 if (should_rce || should_align || should_unroll) {
2388 if (cl->is_normal_loop()) // Convert to 'pre/main/post' loops
2389 phase->insert_pre_post_loops(this,old_new, !may_rce_align);
2390
2391 // Adjust the pre- and main-loop limits to let the pre and post loops run
2392 // with full checks, but the main-loop with no checks. Remove said
2393 // checks from the main body.
2394 if (should_rce)
2395 phase->do_range_check(this,old_new);
2396
2397 // Double loop body for unrolling. Adjust the minimum-trip test (will do
2398 // twice as many iterations as before) and the main body limit (only do
2399 // an even number of trips). If we are peeling, we might enable some RCE
2400 // and we'd rather unroll the post-RCE'd loop SO... do not unroll if
2401 // peeling.
2402 if (should_unroll && !should_peel) {
2403 phase->mark_reductions(this);
2404 phase->do_unroll(this, old_new, true);
2405 }
2406
2407 // Adjust the pre-loop limits to align the main body
2408 // iterations.
2409 if (should_align)
2410 Unimplemented();
2411
2412 } else { // Else we have an unchanged counted loop
2413 if (should_peel) // Might want to peel but do nothing else
2414 phase->do_peeling(this,old_new);
2415 }
2416 return true;
2417 }
2418
2419
2420 //=============================================================================
2421 //------------------------------iteration_split--------------------------------
2422 bool IdealLoopTree::iteration_split( PhaseIdealLoop *phase, Node_List &old_new ) {
2423 // Recursively iteration split nested loops
2424 if (_child && !_child->iteration_split(phase, old_new))
2425 return false;
|