2421 if (!iteration_split_impl(phase, old_new)) {
2422 return false;
2423 }
2424 } else if (policy_unswitching(phase)) {
2425 phase->do_unswitching(this, old_new);
2426 }
2427 }
2428
2429 // Minor offset re-organization to remove loop-fallout uses of
2430 // trip counter when there was no major reshaping.
2431 phase->reorg_offsets(this);
2432
2433 if (_next && !_next->iteration_split(phase, old_new))
2434 return false;
2435 return true;
2436 }
2437
2438
2439 //=============================================================================
2440 // Process all the loops in the loop tree and replace any fill
2441 // patterns with an intrisc version.
2442 bool PhaseIdealLoop::do_intrinsify_fill() {
2443 bool changed = false;
2444 for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
2445 IdealLoopTree* lpt = iter.current();
2446 changed |= intrinsify_fill(lpt);
2447 }
2448 return changed;
2449 }
2450
2451
2452 // Examine an inner loop looking for a a single store of an invariant
2453 // value in a unit stride loop,
2454 bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value,
2455 Node*& shift, Node*& con) {
2456 const char* msg = NULL;
2457 Node* msg_node = NULL;
2458
2459 store_value = NULL;
2460 con = NULL;
2461 shift = NULL;
2519 // Make sure there is an appropriate fill routine
2520 BasicType t = store->as_Mem()->memory_type();
2521 const char* fill_name;
2522 if (msg == NULL &&
2523 StubRoutines::select_fill_function(t, false, fill_name) == NULL) {
2524 msg = "unsupported store";
2525 msg_node = store;
2526 }
2527
2528 if (msg != NULL) {
2529 #ifndef PRODUCT
2530 if (TraceOptimizeFill) {
2531 tty->print_cr("not fill intrinsic candidate: %s", msg);
2532 if (msg_node != NULL) msg_node->dump();
2533 }
2534 #endif
2535 return false;
2536 }
2537
2538 // Make sure the address expression can be handled. It should be
2539 // head->phi * elsize + con. head->phi might have a ConvI2L.
2540 Node* elements[4];
2541 Node* conv = NULL;
2542 bool found_index = false;
2543 int count = store->in(MemNode::Address)->as_AddP()->unpack_offsets(elements, ARRAY_SIZE(elements));
2544 for (int e = 0; e < count; e++) {
2545 Node* n = elements[e];
2546 if (n->is_Con() && con == NULL) {
2547 con = n;
2548 } else if (n->Opcode() == Op_LShiftX && shift == NULL) {
2549 Node* value = n->in(1);
2550 #ifdef _LP64
2551 if (value->Opcode() == Op_ConvI2L) {
2552 conv = value;
2553 value = value->in(1);
2554 }
2555 #endif
2556 if (value != head->phi()) {
2557 msg = "unhandled shift in address";
2558 } else {
2559 if (type2aelembytes(store->as_Mem()->memory_type(), true) != (1 << n->in(2)->get_int())) {
2560 msg = "scale doesn't match";
2561 } else {
2562 found_index = true;
2563 shift = n;
2564 }
2565 }
2566 } else if (n->Opcode() == Op_ConvI2L && conv == NULL) {
2567 if (n->in(1) == head->phi()) {
2568 found_index = true;
2569 conv = n;
2570 } else {
2571 msg = "unhandled input to ConvI2L";
2572 }
2573 } else if (n == head->phi()) {
2574 // no shift, check below for allowed cases
2575 found_index = true;
2576 } else {
2577 msg = "unhandled node in address";
2578 msg_node = n;
2579 }
2580 }
2581
2582 if (count == -1) {
2583 msg = "malformed address expression";
2584 msg_node = store;
2585 }
2586
2587 if (!found_index) {
2588 msg = "missing use of index";
2589 }
2608 VectorSet ok(Thread::current()->resource_area());
2609
2610 // store related values are ok
2611 ok.set(store->_idx);
2612 ok.set(store->in(MemNode::Memory)->_idx);
2613
2614 CountedLoopEndNode* loop_exit = head->loopexit();
2615 guarantee(loop_exit != NULL, "no loop exit node");
2616
2617 // Loop structure is ok
2618 ok.set(head->_idx);
2619 ok.set(loop_exit->_idx);
2620 ok.set(head->phi()->_idx);
2621 ok.set(head->incr()->_idx);
2622 ok.set(loop_exit->cmp_node()->_idx);
2623 ok.set(loop_exit->in(1)->_idx);
2624
2625 // Address elements are ok
2626 if (con) ok.set(con->_idx);
2627 if (shift) ok.set(shift->_idx);
2628 if (conv) ok.set(conv->_idx);
2629
2630 for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
2631 Node* n = lpt->_body.at(i);
2632 if (n->outcnt() == 0) continue; // Ignore dead
2633 if (ok.test(n->_idx)) continue;
2634 // Backedge projection is ok
2635 if (n->is_IfTrue() && n->in(0) == loop_exit) continue;
2636 if (!n->is_AddP()) {
2637 msg = "unhandled node";
2638 msg_node = n;
2639 break;
2640 }
2641 }
2642
2643 // Make sure no unexpected values are used outside the loop
2644 for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
2645 Node* n = lpt->_body.at(i);
2646 // These values can be replaced with other nodes if they are used
2647 // outside the loop.
|
2421 if (!iteration_split_impl(phase, old_new)) {
2422 return false;
2423 }
2424 } else if (policy_unswitching(phase)) {
2425 phase->do_unswitching(this, old_new);
2426 }
2427 }
2428
2429 // Minor offset re-organization to remove loop-fallout uses of
2430 // trip counter when there was no major reshaping.
2431 phase->reorg_offsets(this);
2432
2433 if (_next && !_next->iteration_split(phase, old_new))
2434 return false;
2435 return true;
2436 }
2437
2438
2439 //=============================================================================
2440 // Process all the loops in the loop tree and replace any fill
2441 // patterns with an intrinsic version.
2442 bool PhaseIdealLoop::do_intrinsify_fill() {
2443 bool changed = false;
2444 for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
2445 IdealLoopTree* lpt = iter.current();
2446 changed |= intrinsify_fill(lpt);
2447 }
2448 return changed;
2449 }
2450
2451
2452 // Examine an inner loop looking for a a single store of an invariant
2453 // value in a unit stride loop,
2454 bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value,
2455 Node*& shift, Node*& con) {
2456 const char* msg = NULL;
2457 Node* msg_node = NULL;
2458
2459 store_value = NULL;
2460 con = NULL;
2461 shift = NULL;
2519 // Make sure there is an appropriate fill routine
2520 BasicType t = store->as_Mem()->memory_type();
2521 const char* fill_name;
2522 if (msg == NULL &&
2523 StubRoutines::select_fill_function(t, false, fill_name) == NULL) {
2524 msg = "unsupported store";
2525 msg_node = store;
2526 }
2527
2528 if (msg != NULL) {
2529 #ifndef PRODUCT
2530 if (TraceOptimizeFill) {
2531 tty->print_cr("not fill intrinsic candidate: %s", msg);
2532 if (msg_node != NULL) msg_node->dump();
2533 }
2534 #endif
2535 return false;
2536 }
2537
2538 // Make sure the address expression can be handled. It should be
2539 // head->phi * elsize + con. head->phi might have a ConvI2L(CastII()).
2540 Node* elements[4];
2541 Node* cast = NULL;
2542 Node* conv = NULL;
2543 bool found_index = false;
2544 int count = store->in(MemNode::Address)->as_AddP()->unpack_offsets(elements, ARRAY_SIZE(elements));
2545 for (int e = 0; e < count; e++) {
2546 Node* n = elements[e];
2547 if (n->is_Con() && con == NULL) {
2548 con = n;
2549 } else if (n->Opcode() == Op_LShiftX && shift == NULL) {
2550 Node* value = n->in(1);
2551 #ifdef _LP64
2552 if (value->Opcode() == Op_ConvI2L) {
2553 conv = value;
2554 value = value->in(1);
2555 }
2556 if (value->Opcode() == Op_CastII &&
2557 value->as_CastII()->has_range_check()) {
2558 // Skip range check dependent CastII nodes
2559 cast = value;
2560 value = value->in(1);
2561 }
2562 #endif
2563 if (value != head->phi()) {
2564 msg = "unhandled shift in address";
2565 } else {
2566 if (type2aelembytes(store->as_Mem()->memory_type(), true) != (1 << n->in(2)->get_int())) {
2567 msg = "scale doesn't match";
2568 } else {
2569 found_index = true;
2570 shift = n;
2571 }
2572 }
2573 } else if (n->Opcode() == Op_ConvI2L && conv == NULL) {
2574 conv = n;
2575 n = n->in(1);
2576 if (n->Opcode() == Op_CastII &&
2577 n->as_CastII()->has_range_check()) {
2578 // Skip range check dependent CastII nodes
2579 cast = n;
2580 n = n->in(1);
2581 }
2582 if (n == head->phi()) {
2583 found_index = true;
2584 } else {
2585 msg = "unhandled input to ConvI2L";
2586 }
2587 } else if (n == head->phi()) {
2588 // no shift, check below for allowed cases
2589 found_index = true;
2590 } else {
2591 msg = "unhandled node in address";
2592 msg_node = n;
2593 }
2594 }
2595
2596 if (count == -1) {
2597 msg = "malformed address expression";
2598 msg_node = store;
2599 }
2600
2601 if (!found_index) {
2602 msg = "missing use of index";
2603 }
2622 VectorSet ok(Thread::current()->resource_area());
2623
2624 // store related values are ok
2625 ok.set(store->_idx);
2626 ok.set(store->in(MemNode::Memory)->_idx);
2627
2628 CountedLoopEndNode* loop_exit = head->loopexit();
2629 guarantee(loop_exit != NULL, "no loop exit node");
2630
2631 // Loop structure is ok
2632 ok.set(head->_idx);
2633 ok.set(loop_exit->_idx);
2634 ok.set(head->phi()->_idx);
2635 ok.set(head->incr()->_idx);
2636 ok.set(loop_exit->cmp_node()->_idx);
2637 ok.set(loop_exit->in(1)->_idx);
2638
2639 // Address elements are ok
2640 if (con) ok.set(con->_idx);
2641 if (shift) ok.set(shift->_idx);
2642 if (cast) ok.set(cast->_idx);
2643 if (conv) ok.set(conv->_idx);
2644
2645 for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
2646 Node* n = lpt->_body.at(i);
2647 if (n->outcnt() == 0) continue; // Ignore dead
2648 if (ok.test(n->_idx)) continue;
2649 // Backedge projection is ok
2650 if (n->is_IfTrue() && n->in(0) == loop_exit) continue;
2651 if (!n->is_AddP()) {
2652 msg = "unhandled node";
2653 msg_node = n;
2654 break;
2655 }
2656 }
2657
2658 // Make sure no unexpected values are used outside the loop
2659 for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
2660 Node* n = lpt->_body.at(i);
2661 // These values can be replaced with other nodes if they are used
2662 // outside the loop.
|