1507 #ifndef PRODUCT 1508 if (TraceLoopOpts) { 1509 tty->print("MaxUnroll %d ", cl->trip_count()); 1510 loop->dump_head(); 1511 } 1512 #endif 1513 1514 // If loop is tripping an odd number of times, peel odd iteration 1515 if ((cl->trip_count() & 1) == 1) { 1516 do_peeling(loop, old_new); 1517 } 1518 1519 // Now its tripping an even number of times remaining. Double loop body. 1520 // Do not adjust pre-guards; they are not needed and do not exist. 1521 if (cl->trip_count() > 0) { 1522 assert((cl->trip_count() & 1) == 0, "missed peeling"); 1523 do_unroll(loop, old_new, false); 1524 } 1525 } 1526 1527 //------------------------------dominates_backedge--------------------------------- 1528 // Returns true if ctrl is executed on every complete iteration 1529 bool IdealLoopTree::dominates_backedge(Node* ctrl) { 1530 assert(ctrl->is_CFG(), "must be control"); 1531 Node* backedge = _head->as_Loop()->in(LoopNode::LoopBackControl); 1532 return _phase->dom_lca_internal(ctrl, backedge) == ctrl; 1533 } 1534 1535 //------------------------------adjust_limit----------------------------------- 1536 // Helper function for add_constraint(). 1537 Node* PhaseIdealLoop::adjust_limit(int stride_con, Node * scale, Node *offset, Node *rc_limit, Node *loop_limit, Node *pre_ctrl) { 1538 // Compute "I :: (limit-offset)/scale" 1539 Node *con = new SubINode(rc_limit, offset); 1540 register_new_node(con, pre_ctrl); 1541 Node *X = new DivINode(0, con, scale); 1542 register_new_node(X, pre_ctrl); 1543 1544 // Adjust loop limit 1545 loop_limit = (stride_con > 0) 1546 ? (Node*)(new MinINode(loop_limit, X)) 2344 bool may_rce_align = !policy_peel_only(phase) || should_rce || should_align; 2345 2346 // If we have any of these conditions (RCE, alignment, unrolling) met, then 2347 // we switch to the pre-/main-/post-loop model. This model also covers 2348 // peeling. 2349 if (should_rce || should_align || should_unroll) { 2350 if (cl->is_normal_loop()) // Convert to 'pre/main/post' loops 2351 phase->insert_pre_post_loops(this,old_new, !may_rce_align); 2352 2353 // Adjust the pre- and main-loop limits to let the pre and post loops run 2354 // with full checks, but the main-loop with no checks. Remove said 2355 // checks from the main body. 2356 if (should_rce) 2357 phase->do_range_check(this,old_new); 2358 2359 // Double loop body for unrolling. Adjust the minimum-trip test (will do 2360 // twice as many iterations as before) and the main body limit (only do 2361 // an even number of trips). If we are peeling, we might enable some RCE 2362 // and we'd rather unroll the post-RCE'd loop SO... do not unroll if 2363 // peeling. 2364 if (should_unroll && !should_peel) 2365 phase->do_unroll(this,old_new, true); 2366 2367 // Adjust the pre-loop limits to align the main body 2368 // iterations. 2369 if (should_align) 2370 Unimplemented(); 2371 2372 } else { // Else we have an unchanged counted loop 2373 if (should_peel) // Might want to peel but do nothing else 2374 phase->do_peeling(this,old_new); 2375 } 2376 return true; 2377 } 2378 2379 2380 //============================================================================= 2381 //------------------------------iteration_split-------------------------------- 2382 bool IdealLoopTree::iteration_split( PhaseIdealLoop *phase, Node_List &old_new ) { 2383 // Recursively iteration split nested loops 2384 if (_child && !_child->iteration_split(phase, old_new)) 2385 return false; | 1507 #ifndef PRODUCT 1508 if (TraceLoopOpts) { 1509 tty->print("MaxUnroll %d ", cl->trip_count()); 1510 loop->dump_head(); 1511 } 1512 #endif 1513 1514 // If loop is tripping an odd number of times, peel odd iteration 1515 if ((cl->trip_count() & 1) == 1) { 1516 do_peeling(loop, old_new); 1517 } 1518 1519 // Now its tripping an even number of times remaining. Double loop body. 1520 // Do not adjust pre-guards; they are not needed and do not exist. 1521 if (cl->trip_count() > 0) { 1522 assert((cl->trip_count() & 1) == 0, "missed peeling"); 1523 do_unroll(loop, old_new, false); 1524 } 1525 } 1526 1527 void PhaseIdealLoop::mark_reductions(IdealLoopTree *loop) { 1528 if (SuperWordReductions == false) return; 1529 1530 CountedLoopNode* loop_head = loop->_head->as_CountedLoop(); 1531 if (loop_head->unrolled_count() > 1) { 1532 return; 1533 } 1534 1535 Node* trip_phi = loop_head->phi(); 1536 for (DUIterator_Fast imax, i = loop_head->fast_outs(imax); i < imax; i++) { 1537 Node* phi = loop_head->fast_out(i); 1538 if (phi->is_Phi() && phi->outcnt() > 0 && phi != trip_phi) { 1539 // For definitions which are loop inclusive and not tripcounts. 1540 Node* def_node = phi->in(LoopNode::LoopBackControl); 1541 1542 if (def_node != NULL) { 1543 Node* n_ctrl = get_ctrl(def_node); 1544 if (n_ctrl != NULL && loop->is_member(get_loop(n_ctrl))) { 1545 // Now test it to see if it fits the standard pattern for a reduction operator. 1546 int opc = def_node->Opcode(); 1547 if (opc != ReductionNode::opcode(opc, def_node->bottom_type()->basic_type())) { 1548 if (!def_node->is_reduction()) { // Not marked yet 1549 // To be a reduction, the arithmetic node must have the phi as input and provide a def to it 1550 for (unsigned j = 1; j < def_node->req(); j++) { 1551 Node* in = def_node->in(j); 1552 if (in == phi) { 1553 def_node->add_flag(Node::Flag_is_reduction); 1554 break; 1555 } 1556 } 1557 } 1558 } 1559 } 1560 } 1561 } 1562 } 1563 } 1564 1565 //------------------------------dominates_backedge--------------------------------- 1566 // Returns true if ctrl is executed on every complete iteration 1567 bool IdealLoopTree::dominates_backedge(Node* ctrl) { 1568 assert(ctrl->is_CFG(), "must be control"); 1569 Node* backedge = _head->as_Loop()->in(LoopNode::LoopBackControl); 1570 return _phase->dom_lca_internal(ctrl, backedge) == ctrl; 1571 } 1572 1573 //------------------------------adjust_limit----------------------------------- 1574 // Helper function for add_constraint(). 1575 Node* PhaseIdealLoop::adjust_limit(int stride_con, Node * scale, Node *offset, Node *rc_limit, Node *loop_limit, Node *pre_ctrl) { 1576 // Compute "I :: (limit-offset)/scale" 1577 Node *con = new SubINode(rc_limit, offset); 1578 register_new_node(con, pre_ctrl); 1579 Node *X = new DivINode(0, con, scale); 1580 register_new_node(X, pre_ctrl); 1581 1582 // Adjust loop limit 1583 loop_limit = (stride_con > 0) 1584 ? (Node*)(new MinINode(loop_limit, X)) 2382 bool may_rce_align = !policy_peel_only(phase) || should_rce || should_align; 2383 2384 // If we have any of these conditions (RCE, alignment, unrolling) met, then 2385 // we switch to the pre-/main-/post-loop model. This model also covers 2386 // peeling. 2387 if (should_rce || should_align || should_unroll) { 2388 if (cl->is_normal_loop()) // Convert to 'pre/main/post' loops 2389 phase->insert_pre_post_loops(this,old_new, !may_rce_align); 2390 2391 // Adjust the pre- and main-loop limits to let the pre and post loops run 2392 // with full checks, but the main-loop with no checks. Remove said 2393 // checks from the main body. 2394 if (should_rce) 2395 phase->do_range_check(this,old_new); 2396 2397 // Double loop body for unrolling. Adjust the minimum-trip test (will do 2398 // twice as many iterations as before) and the main body limit (only do 2399 // an even number of trips). If we are peeling, we might enable some RCE 2400 // and we'd rather unroll the post-RCE'd loop SO... do not unroll if 2401 // peeling. 2402 if (should_unroll && !should_peel) { 2403 phase->mark_reductions(this); 2404 phase->do_unroll(this, old_new, true); 2405 } 2406 2407 // Adjust the pre-loop limits to align the main body 2408 // iterations. 2409 if (should_align) 2410 Unimplemented(); 2411 2412 } else { // Else we have an unchanged counted loop 2413 if (should_peel) // Might want to peel but do nothing else 2414 phase->do_peeling(this,old_new); 2415 } 2416 return true; 2417 } 2418 2419 2420 //============================================================================= 2421 //------------------------------iteration_split-------------------------------- 2422 bool IdealLoopTree::iteration_split( PhaseIdealLoop *phase, Node_List &old_new ) { 2423 // Recursively iteration split nested loops 2424 if (_child && !_child->iteration_split(phase, old_new)) 2425 return false; |