2643 if (!iteration_split_impl(phase, old_new)) {
2644 return false;
2645 }
2646 } else if (policy_unswitching(phase)) {
2647 phase->do_unswitching(this, old_new);
2648 }
2649 }
2650
2651 // Minor offset re-organization to remove loop-fallout uses of
2652 // trip counter when there was no major reshaping.
2653 phase->reorg_offsets(this);
2654
2655 if (_next && !_next->iteration_split(phase, old_new))
2656 return false;
2657 return true;
2658 }
2659
2660
2661 //=============================================================================
2662 // Process all the loops in the loop tree and replace any fill
2663 // patterns with an intrisc version.
2664 bool PhaseIdealLoop::do_intrinsify_fill() {
2665 bool changed = false;
2666 for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
2667 IdealLoopTree* lpt = iter.current();
2668 changed |= intrinsify_fill(lpt);
2669 }
2670 return changed;
2671 }
2672
2673
2674 // Examine an inner loop looking for a a single store of an invariant
2675 // value in a unit stride loop,
2676 bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value,
2677 Node*& shift, Node*& con) {
2678 const char* msg = NULL;
2679 Node* msg_node = NULL;
2680
2681 store_value = NULL;
2682 con = NULL;
2683 shift = NULL;
2741 // Make sure there is an appropriate fill routine
2742 BasicType t = store->as_Mem()->memory_type();
2743 const char* fill_name;
2744 if (msg == NULL &&
2745 StubRoutines::select_fill_function(t, false, fill_name) == NULL) {
2746 msg = "unsupported store";
2747 msg_node = store;
2748 }
2749
2750 if (msg != NULL) {
2751 #ifndef PRODUCT
2752 if (TraceOptimizeFill) {
2753 tty->print_cr("not fill intrinsic candidate: %s", msg);
2754 if (msg_node != NULL) msg_node->dump();
2755 }
2756 #endif
2757 return false;
2758 }
2759
2760 // Make sure the address expression can be handled. It should be
2761 // head->phi * elsize + con. head->phi might have a ConvI2L.
2762 Node* elements[4];
2763 Node* conv = NULL;
2764 bool found_index = false;
2765 int count = store->in(MemNode::Address)->as_AddP()->unpack_offsets(elements, ARRAY_SIZE(elements));
2766 for (int e = 0; e < count; e++) {
2767 Node* n = elements[e];
2768 if (n->is_Con() && con == NULL) {
2769 con = n;
2770 } else if (n->Opcode() == Op_LShiftX && shift == NULL) {
2771 Node* value = n->in(1);
2772 #ifdef _LP64
2773 if (value->Opcode() == Op_ConvI2L) {
2774 conv = value;
2775 value = value->in(1);
2776 }
2777 #endif
2778 if (value != head->phi()) {
2779 msg = "unhandled shift in address";
2780 } else {
2781 if (type2aelembytes(store->as_Mem()->memory_type(), true) != (1 << n->in(2)->get_int())) {
2782 msg = "scale doesn't match";
2783 } else {
2784 found_index = true;
2785 shift = n;
2786 }
2787 }
2788 } else if (n->Opcode() == Op_ConvI2L && conv == NULL) {
2789 if (n->in(1) == head->phi()) {
2790 found_index = true;
2791 conv = n;
2792 } else {
2793 msg = "unhandled input to ConvI2L";
2794 }
2795 } else if (n == head->phi()) {
2796 // no shift, check below for allowed cases
2797 found_index = true;
2798 } else {
2799 msg = "unhandled node in address";
2800 msg_node = n;
2801 }
2802 }
2803
2804 if (count == -1) {
2805 msg = "malformed address expression";
2806 msg_node = store;
2807 }
2808
2809 if (!found_index) {
2810 msg = "missing use of index";
2811 }
2830 VectorSet ok(Thread::current()->resource_area());
2831
2832 // store related values are ok
2833 ok.set(store->_idx);
2834 ok.set(store->in(MemNode::Memory)->_idx);
2835
2836 CountedLoopEndNode* loop_exit = head->loopexit();
2837 guarantee(loop_exit != NULL, "no loop exit node");
2838
2839 // Loop structure is ok
2840 ok.set(head->_idx);
2841 ok.set(loop_exit->_idx);
2842 ok.set(head->phi()->_idx);
2843 ok.set(head->incr()->_idx);
2844 ok.set(loop_exit->cmp_node()->_idx);
2845 ok.set(loop_exit->in(1)->_idx);
2846
2847 // Address elements are ok
2848 if (con) ok.set(con->_idx);
2849 if (shift) ok.set(shift->_idx);
2850 if (conv) ok.set(conv->_idx);
2851
2852 for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
2853 Node* n = lpt->_body.at(i);
2854 if (n->outcnt() == 0) continue; // Ignore dead
2855 if (ok.test(n->_idx)) continue;
2856 // Backedge projection is ok
2857 if (n->is_IfTrue() && n->in(0) == loop_exit) continue;
2858 if (!n->is_AddP()) {
2859 msg = "unhandled node";
2860 msg_node = n;
2861 break;
2862 }
2863 }
2864
2865 // Make sure no unexpected values are used outside the loop
2866 for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
2867 Node* n = lpt->_body.at(i);
2868 // These values can be replaced with other nodes if they are used
2869 // outside the loop.
|
2643 if (!iteration_split_impl(phase, old_new)) {
2644 return false;
2645 }
2646 } else if (policy_unswitching(phase)) {
2647 phase->do_unswitching(this, old_new);
2648 }
2649 }
2650
2651 // Minor offset re-organization to remove loop-fallout uses of
2652 // trip counter when there was no major reshaping.
2653 phase->reorg_offsets(this);
2654
2655 if (_next && !_next->iteration_split(phase, old_new))
2656 return false;
2657 return true;
2658 }
2659
2660
2661 //=============================================================================
2662 // Process all the loops in the loop tree and replace any fill
2663 // patterns with an intrinsic version.
2664 bool PhaseIdealLoop::do_intrinsify_fill() {
2665 bool changed = false;
2666 for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
2667 IdealLoopTree* lpt = iter.current();
2668 changed |= intrinsify_fill(lpt);
2669 }
2670 return changed;
2671 }
2672
2673
2674 // Examine an inner loop looking for a a single store of an invariant
2675 // value in a unit stride loop,
2676 bool PhaseIdealLoop::match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value,
2677 Node*& shift, Node*& con) {
2678 const char* msg = NULL;
2679 Node* msg_node = NULL;
2680
2681 store_value = NULL;
2682 con = NULL;
2683 shift = NULL;
2741 // Make sure there is an appropriate fill routine
2742 BasicType t = store->as_Mem()->memory_type();
2743 const char* fill_name;
2744 if (msg == NULL &&
2745 StubRoutines::select_fill_function(t, false, fill_name) == NULL) {
2746 msg = "unsupported store";
2747 msg_node = store;
2748 }
2749
2750 if (msg != NULL) {
2751 #ifndef PRODUCT
2752 if (TraceOptimizeFill) {
2753 tty->print_cr("not fill intrinsic candidate: %s", msg);
2754 if (msg_node != NULL) msg_node->dump();
2755 }
2756 #endif
2757 return false;
2758 }
2759
2760 // Make sure the address expression can be handled. It should be
2761 // head->phi * elsize + con. head->phi might have a ConvI2L(CastII()).
2762 Node* elements[4];
2763 Node* cast = NULL;
2764 Node* conv = NULL;
2765 bool found_index = false;
2766 int count = store->in(MemNode::Address)->as_AddP()->unpack_offsets(elements, ARRAY_SIZE(elements));
2767 for (int e = 0; e < count; e++) {
2768 Node* n = elements[e];
2769 if (n->is_Con() && con == NULL) {
2770 con = n;
2771 } else if (n->Opcode() == Op_LShiftX && shift == NULL) {
2772 Node* value = n->in(1);
2773 #ifdef _LP64
2774 if (value->Opcode() == Op_ConvI2L) {
2775 conv = value;
2776 value = value->in(1);
2777 }
2778 if (value->Opcode() == Op_CastII &&
2779 value->as_CastII()->has_range_check()) {
2780 // Skip range check dependent CastII nodes
2781 cast = value;
2782 value = value->in(1);
2783 }
2784 #endif
2785 if (value != head->phi()) {
2786 msg = "unhandled shift in address";
2787 } else {
2788 if (type2aelembytes(store->as_Mem()->memory_type(), true) != (1 << n->in(2)->get_int())) {
2789 msg = "scale doesn't match";
2790 } else {
2791 found_index = true;
2792 shift = n;
2793 }
2794 }
2795 } else if (n->Opcode() == Op_ConvI2L && conv == NULL) {
2796 conv = n;
2797 n = n->in(1);
2798 if (n->Opcode() == Op_CastII &&
2799 n->as_CastII()->has_range_check()) {
2800 // Skip range check dependent CastII nodes
2801 cast = n;
2802 n = n->in(1);
2803 }
2804 if (n == head->phi()) {
2805 found_index = true;
2806 } else {
2807 msg = "unhandled input to ConvI2L";
2808 }
2809 } else if (n == head->phi()) {
2810 // no shift, check below for allowed cases
2811 found_index = true;
2812 } else {
2813 msg = "unhandled node in address";
2814 msg_node = n;
2815 }
2816 }
2817
2818 if (count == -1) {
2819 msg = "malformed address expression";
2820 msg_node = store;
2821 }
2822
2823 if (!found_index) {
2824 msg = "missing use of index";
2825 }
2844 VectorSet ok(Thread::current()->resource_area());
2845
2846 // store related values are ok
2847 ok.set(store->_idx);
2848 ok.set(store->in(MemNode::Memory)->_idx);
2849
2850 CountedLoopEndNode* loop_exit = head->loopexit();
2851 guarantee(loop_exit != NULL, "no loop exit node");
2852
2853 // Loop structure is ok
2854 ok.set(head->_idx);
2855 ok.set(loop_exit->_idx);
2856 ok.set(head->phi()->_idx);
2857 ok.set(head->incr()->_idx);
2858 ok.set(loop_exit->cmp_node()->_idx);
2859 ok.set(loop_exit->in(1)->_idx);
2860
2861 // Address elements are ok
2862 if (con) ok.set(con->_idx);
2863 if (shift) ok.set(shift->_idx);
2864 if (cast) ok.set(cast->_idx);
2865 if (conv) ok.set(conv->_idx);
2866
2867 for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
2868 Node* n = lpt->_body.at(i);
2869 if (n->outcnt() == 0) continue; // Ignore dead
2870 if (ok.test(n->_idx)) continue;
2871 // Backedge projection is ok
2872 if (n->is_IfTrue() && n->in(0) == loop_exit) continue;
2873 if (!n->is_AddP()) {
2874 msg = "unhandled node";
2875 msg_node = n;
2876 break;
2877 }
2878 }
2879
2880 // Make sure no unexpected values are used outside the loop
2881 for (uint i = 0; msg == NULL && i < lpt->_body.size(); i++) {
2882 Node* n = lpt->_body.at(i);
2883 // These values can be replaced with other nodes if they are used
2884 // outside the loop.
|