961 assert(inner != NULL && outer != NULL, "missing loop in strip mined nest");
962 Node* outer_tail = outer->in(LoopNode::LoopBackControl);
963 Node* outer_le = outer_tail->in(0);
964 assert(outer_le->Opcode() == Op_OuterStripMinedLoopEnd, "tail of outer loop should be an If");
965 Node* sfpt = outer_le->in(0);
966 assert(sfpt->Opcode() == Op_SafePoint, "where's the safepoint?");
967 Node* inner_out = sfpt->in(0);
968 if (inner_out->outcnt() != 1) {
969 ResourceMark rm;
970 Unique_Node_List wq;
971
972 for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) {
973 Node* u = inner_out->fast_out(i);
974 if (u == sfpt) {
975 continue;
976 }
977 wq.clear();
978 wq.push(u);
979 bool found_sfpt = false;
980 for (uint next = 0; next < wq.size() && !found_sfpt; next++) {
981 Node *n = wq.at(next);
982 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && !found_sfpt; i++) {
983 Node* u = n->fast_out(i);
984 if (u == sfpt) {
985 found_sfpt = true;
986 }
987 if (!u->is_CFG()) {
988 wq.push(u);
989 }
990 }
991 }
992 assert(found_sfpt, "no node in loop that's not input to safepoint");
993 }
994 }
995 CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd();
996 assert(cle == inner->loopexit_or_null(), "mismatch");
997 bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0;
998 if (has_skeleton) {
999 assert(expect_skeleton == 1 || expect_skeleton == -1, "unexpected skeleton node");
1000 assert(outer->outcnt() == 2, "only phis");
1001 } else {
1002 assert(expect_skeleton == 0 || expect_skeleton == -1, "no skeleton node?");
1003 uint phis = 0;
1004 for (DUIterator_Fast imax, i = inner->fast_outs(imax); i < imax; i++) {
1005 Node* u = inner->fast_out(i);
1006 if (u->is_Phi()) {
1007 phis++;
1008 }
1009 }
1010 for (DUIterator_Fast imax, i = outer->fast_outs(imax); i < imax; i++) {
1011 Node* u = outer->fast_out(i);
1012 assert(u == outer || u == inner || u->is_Phi(), "nothing between inner and outer loop");
1013 }
1014 uint stores = 0;
1015 for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) {
1016 Node* u = inner_out->fast_out(i);
1017 if (u->is_Store()) {
1018 stores++;
1019 }
1020 }
1021 assert(outer->outcnt() >= phis + 2 && outer->outcnt() <= phis + 2 + stores + 1, "only phis");
1022 }
1023 assert(sfpt->outcnt() == 1, "no data node");
1024 assert(outer_tail->outcnt() == 1 || !has_skeleton, "no data node");
1025 }
1026 #endif
1027 }
1028
1029 //=============================================================================
1030 //------------------------------Ideal------------------------------------------
1031 // Return a node which is more "ideal" than the current node.
1032 // Attempt to convert into a counted-loop.
1033 Node *CountedLoopNode::Ideal(PhaseGVN *phase, bool can_reshape) {
1034 return RegionNode::Ideal(phase, can_reshape);
1035 }
1036
2693 if (n2->in(0) != c2) {
2694 _igvn.hash_delete(n2);
2695 n2->set_req(0, c2);
2696 _igvn.hash_insert(n2);
2697 _igvn._worklist.push(n2);
2698 progress = true;
2699 }
2700 }
2701 }
2702 }
2703
2704 return progress;
2705 }
2706
2707
2708 //=============================================================================
2709 //----------------------------build_and_optimize-------------------------------
2710 // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to
2711 // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups.
2712 void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) {
2713 bool do_split_ifs = (mode == LoopOptsDefault || mode == LoopOptsLastRound);
2714 bool skip_loop_opts = (mode == LoopOptsNone);
2715
2716 int old_progress = C->major_progress();
2717 uint orig_worklist_size = _igvn._worklist.size();
2718
2719 // Reset major-progress flag for the driver's heuristics
2720 C->clear_major_progress();
2721
2722 #ifndef PRODUCT
2723 // Capture for later assert
2724 uint unique = C->unique();
2725 _loop_invokes++;
2726 _loop_work += unique;
2727 #endif
2728
2729 // True if the method has at least 1 irreducible loop
2730 _has_irreducible_loops = false;
2731
2732 _created_loop_node = false;
2733
2734 Arena *a = Thread::current()->resource_area();
2853 worklist.push( C->top() );
2854 visited.set( C->top()->_idx ); // Set C->top() as visited now
2855 build_loop_early( visited, worklist, nstack );
2856
2857 // Given early legal placement, try finding counted loops. This placement
2858 // is good enough to discover most loop invariants.
2859 if (!_verify_me && !_verify_only && !strip_mined_loops_expanded) {
2860 _ltree_root->counted_loop( this );
2861 }
2862
2863 // Find latest loop placement. Find ideal loop placement.
2864 visited.Clear();
2865 init_dom_lca_tags();
2866 // Need C->root() on worklist when processing outs
2867 worklist.push( C->root() );
2868 NOT_PRODUCT( C->verify_graph_edges(); )
2869 worklist.push( C->top() );
2870 build_loop_late( visited, worklist, nstack );
2871
2872 if (_verify_only) {
2873 // restore major progress flag
2874 for (int i = 0; i < old_progress; i++)
2875 C->set_major_progress();
2876 assert(C->unique() == unique, "verification mode made Nodes? ? ?");
2877 assert(_igvn._worklist.size() == orig_worklist_size, "shouldn't push anything");
2878 return;
2879 }
2880
2881 // clear out the dead code after build_loop_late
2882 while (_deadlist.size()) {
2883 _igvn.remove_globally_dead_node(_deadlist.pop());
2884 }
2885
2886 if (stop_early) {
2887 assert(do_expensive_nodes, "why are we here?");
2888 if (process_expensive_nodes()) {
2889 // If we made some progress when processing expensive nodes then
2890 // the IGVN may modify the graph in a way that will allow us to
2891 // make some more progress: we need to try processing expensive
2892 // nodes again.
2893 C->set_major_progress();
2894 }
2895 _igvn.optimize();
2897 }
2898
2899 // Some parser-inserted loop predicates could never be used by loop
2900 // predication or they were moved away from loop during some optimizations.
2901 // For example, peeling. Eliminate them before next loop optimizations.
2902 eliminate_useless_predicates();
2903
2904 #ifndef PRODUCT
2905 C->verify_graph_edges();
2906 if (_verify_me) { // Nested verify pass?
2907 // Check to see if the verify mode is broken
2908 assert(C->unique() == unique, "non-optimize mode made Nodes? ? ?");
2909 return;
2910 }
2911 if(VerifyLoopOptimizations) verify();
2912 if(TraceLoopOpts && C->has_loops()) {
2913 _ltree_root->dump();
2914 }
2915 #endif
2916
2917 if (skip_loop_opts) {
2918 // restore major progress flag
2919 for (int i = 0; i < old_progress; i++) {
2920 C->set_major_progress();
2921 }
2922
2923 // Cleanup any modified bits
2924 _igvn.optimize();
2925
2926 if (C->log() != NULL) {
2927 log_loop_tree(_ltree_root, _ltree_root, C->log());
2928 }
2929 return;
2930 }
2931
2932 if (bs->optimize_loops(this, mode, visited, nstack, worklist)) {
2933 _igvn.optimize();
2934 if (C->log() != NULL) {
2935 log_loop_tree(_ltree_root, _ltree_root, C->log());
2936 }
2937 return;
2938 }
2939
2940 if (ReassociateInvariants) {
2941 AutoNodeBudget node_budget(this, AutoNodeBudget::NO_BUDGET_CHECK);
2948 // check for vectorized loops, any reassociation of invariants was already done
2949 if (is_counted && lpt->_head->as_CountedLoop()->is_unroll_only()) continue;
2950
2951 lpt->reassociate_invariants(this);
2952
2953 // Because RCE opportunities can be masked by split_thru_phi,
2954 // look for RCE candidates and inhibit split_thru_phi
2955 // on just their loop-phi's for this pass of loop opts
2956 if (SplitIfBlocks && do_split_ifs) {
2957 if (lpt->policy_range_check(this)) {
2958 lpt->_rce_candidate = 1; // = true
2959 }
2960 }
2961 }
2962 }
2963
2964 // Check for aggressive application of split-if and other transforms
2965 // that require basic-block info (like cloning through Phi's)
2966 if( SplitIfBlocks && do_split_ifs ) {
2967 visited.Clear();
2968 split_if_with_blocks( visited, nstack, mode == LoopOptsLastRound );
2969 NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); );
2970 if (mode == LoopOptsLastRound) {
2971 C->set_major_progress();
2972 }
2973 }
2974
2975 if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) {
2976 C->set_major_progress();
2977 }
2978
2979 // Perform loop predication before iteration splitting
2980 if (C->has_loops() && !C->major_progress() && (C->predicate_count() > 0)) {
2981 _ltree_root->_child->loop_predication(this);
2982 }
2983
2984 if (OptimizeFill && UseLoopPredicate && C->has_loops() && !C->major_progress()) {
2985 if (do_intrinsify_fill()) {
2986 C->set_major_progress();
2987 }
2988 }
2989
2990 // Perform iteration-splitting on inner loops. Split iterations to avoid
2991 // range checks or one-shot null checks.
2992
3087 void PhaseIdealLoop::print_statistics() {
3088 tty->print_cr("PhaseIdealLoop=%d, sum _unique=%d", _loop_invokes, _loop_work);
3089 }
3090
3091 //------------------------------verify-----------------------------------------
3092 // Build a verify-only PhaseIdealLoop, and see that it agrees with me.
3093 static int fail; // debug only, so its multi-thread dont care
3094 void PhaseIdealLoop::verify() const {
3095 int old_progress = C->major_progress();
3096 ResourceMark rm;
3097 PhaseIdealLoop loop_verify( _igvn, this );
3098 VectorSet visited(Thread::current()->resource_area());
3099
3100 fail = 0;
3101 verify_compare( C->root(), &loop_verify, visited );
3102 assert( fail == 0, "verify loops failed" );
3103 // Verify loop structure is the same
3104 _ltree_root->verify_tree(loop_verify._ltree_root, NULL);
3105 // Reset major-progress. It was cleared by creating a verify version of
3106 // PhaseIdealLoop.
3107 for( int i=0; i<old_progress; i++ )
3108 C->set_major_progress();
3109 }
3110
3111 //------------------------------verify_compare---------------------------------
3112 // Make sure me and the given PhaseIdealLoop agree on key data structures
3113 void PhaseIdealLoop::verify_compare( Node *n, const PhaseIdealLoop *loop_verify, VectorSet &visited ) const {
3114 if( !n ) return;
3115 if( visited.test_set( n->_idx ) ) return;
3116 if( !_nodes[n->_idx] ) { // Unreachable
3117 assert( !loop_verify->_nodes[n->_idx], "both should be unreachable" );
3118 return;
3119 }
3120
3121 uint i;
3122 for( i = 0; i < n->req(); i++ )
3123 verify_compare( n->in(i), loop_verify, visited );
3124
3125 // Check the '_nodes' block/loop structure
3126 i = n->_idx;
3127 if( has_ctrl(n) ) { // We have control; verify has loop or ctrl
3128 if( _nodes[i] != loop_verify->_nodes[i] &&
4218 // state), Mods/Loads can float around. So free them up.
4219 switch( n->Opcode() ) {
4220 case Op_DivI:
4221 case Op_DivF:
4222 case Op_DivD:
4223 case Op_ModI:
4224 case Op_ModF:
4225 case Op_ModD:
4226 case Op_LoadB: // Same with Loads; they can sink
4227 case Op_LoadUB: // during loop optimizations.
4228 case Op_LoadUS:
4229 case Op_LoadD:
4230 case Op_LoadF:
4231 case Op_LoadI:
4232 case Op_LoadKlass:
4233 case Op_LoadNKlass:
4234 case Op_LoadL:
4235 case Op_LoadS:
4236 case Op_LoadP:
4237 case Op_LoadBarrierSlowReg:
4238 case Op_LoadBarrierWeakSlowReg:
4239 case Op_LoadN:
4240 case Op_LoadRange:
4241 case Op_LoadD_unaligned:
4242 case Op_LoadL_unaligned:
4243 case Op_StrComp: // Does a bunch of load-like effects
4244 case Op_StrEquals:
4245 case Op_StrIndexOf:
4246 case Op_StrIndexOfChar:
4247 case Op_AryEq:
4248 case Op_HasNegatives:
4249 pinned = false;
4250 }
4251 if( pinned ) {
4252 IdealLoopTree *chosen_loop = get_loop(n->is_CFG() ? n : get_ctrl(n));
4253 if( !chosen_loop->_child ) // Inner loop?
4254 chosen_loop->_body.push(n); // Collect inner loops
4255 return;
4256 }
4257 } else { // No slot zero
4258 if( n->is_CFG() ) { // CFG with no slot 0 is dead
|
961 assert(inner != NULL && outer != NULL, "missing loop in strip mined nest");
962 Node* outer_tail = outer->in(LoopNode::LoopBackControl);
963 Node* outer_le = outer_tail->in(0);
964 assert(outer_le->Opcode() == Op_OuterStripMinedLoopEnd, "tail of outer loop should be an If");
965 Node* sfpt = outer_le->in(0);
966 assert(sfpt->Opcode() == Op_SafePoint, "where's the safepoint?");
967 Node* inner_out = sfpt->in(0);
968 if (inner_out->outcnt() != 1) {
969 ResourceMark rm;
970 Unique_Node_List wq;
971
972 for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) {
973 Node* u = inner_out->fast_out(i);
974 if (u == sfpt) {
975 continue;
976 }
977 wq.clear();
978 wq.push(u);
979 bool found_sfpt = false;
980 for (uint next = 0; next < wq.size() && !found_sfpt; next++) {
981 Node* n = wq.at(next);
982 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && !found_sfpt; i++) {
983 Node* u = n->fast_out(i);
984 if (u == sfpt) {
985 found_sfpt = true;
986 }
987 if (!u->is_CFG()) {
988 wq.push(u);
989 }
990 }
991 }
992 assert(found_sfpt, "no node in loop that's not input to safepoint");
993 }
994 }
995
996 if (UseZGC && !inner_out->in(0)->is_CountedLoopEnd()) {
997 // In some very special cases there can be a load that has no other uses than the
998 // counted loop safepoint. Then its loadbarrier will be placed between the inner
999 // loop exit and the safepoint. This is very rare
1000
1001 Node* ifnode = inner_out->in(1)->in(0);
1002 // Region->IfTrue->If == Region->Iffalse->If
1003 if (ifnode == inner_out->in(2)->in(0)) {
1004 inner_out = ifnode->in(0);
1005 }
1006 }
1007
1008 CountedLoopEndNode *cle = inner_out->in(0)->as_CountedLoopEnd();
1009 assert(cle == inner->loopexit_or_null(), "mismatch");
1010 bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0;
1011 if (has_skeleton) {
1012 assert(expect_skeleton == 1 || expect_skeleton == -1, "unexpected skeleton node");
1013 assert(outer->outcnt() == 2, "only phis");
1014 } else {
1015 assert(expect_skeleton == 0 || expect_skeleton == -1, "no skeleton node?");
1016 uint phis = 0;
1017 for (DUIterator_Fast imax, i = inner->fast_outs(imax); i < imax; i++) {
1018 Node *u = inner->fast_out(i);
1019 if (u->is_Phi()) {
1020 phis++;
1021 }
1022 }
1023 for (DUIterator_Fast imax, i = outer->fast_outs(imax); i < imax; i++) {
1024 Node *u = outer->fast_out(i);
1025 assert(u == outer || u == inner || u->is_Phi(), "nothing between inner and outer loop");
1026 }
1027 uint stores = 0;
1028 for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) {
1029 Node *u = inner_out->fast_out(i);
1030 if (u->is_Store()) {
1031 stores++;
1032 }
1033 }
1034 assert(outer->outcnt() >= phis + 2 && outer->outcnt() <= phis + 2 + stores + 1, "only phis");
1035 }
1036 assert(sfpt->outcnt() == 1, "no data node");
1037 assert(outer_tail->outcnt() == 1 || !has_skeleton, "no data node");
1038 }
1039 #endif
1040 }
1041
1042 //=============================================================================
1043 //------------------------------Ideal------------------------------------------
1044 // Return a node which is more "ideal" than the current node.
1045 // Attempt to convert into a counted-loop.
1046 Node *CountedLoopNode::Ideal(PhaseGVN *phase, bool can_reshape) {
1047 return RegionNode::Ideal(phase, can_reshape);
1048 }
1049
2706 if (n2->in(0) != c2) {
2707 _igvn.hash_delete(n2);
2708 n2->set_req(0, c2);
2709 _igvn.hash_insert(n2);
2710 _igvn._worklist.push(n2);
2711 progress = true;
2712 }
2713 }
2714 }
2715 }
2716
2717 return progress;
2718 }
2719
2720
2721 //=============================================================================
2722 //----------------------------build_and_optimize-------------------------------
2723 // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to
2724 // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups.
2725 void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) {
2726 bool do_split_ifs = (mode == LoopOptsDefault);
2727 bool skip_loop_opts = (mode == LoopOptsNone || mode == LoopOptsBarrierInsertion);
2728
2729 int old_progress = C->major_progress();
2730 uint orig_worklist_size = _igvn._worklist.size();
2731
2732 // Reset major-progress flag for the driver's heuristics
2733 C->clear_major_progress();
2734
2735 #ifndef PRODUCT
2736 // Capture for later assert
2737 uint unique = C->unique();
2738 _loop_invokes++;
2739 _loop_work += unique;
2740 #endif
2741
2742 // True if the method has at least 1 irreducible loop
2743 _has_irreducible_loops = false;
2744
2745 _created_loop_node = false;
2746
2747 Arena *a = Thread::current()->resource_area();
2866 worklist.push( C->top() );
2867 visited.set( C->top()->_idx ); // Set C->top() as visited now
2868 build_loop_early( visited, worklist, nstack );
2869
2870 // Given early legal placement, try finding counted loops. This placement
2871 // is good enough to discover most loop invariants.
2872 if (!_verify_me && !_verify_only && !strip_mined_loops_expanded) {
2873 _ltree_root->counted_loop( this );
2874 }
2875
2876 // Find latest loop placement. Find ideal loop placement.
2877 visited.Clear();
2878 init_dom_lca_tags();
2879 // Need C->root() on worklist when processing outs
2880 worklist.push( C->root() );
2881 NOT_PRODUCT( C->verify_graph_edges(); )
2882 worklist.push( C->top() );
2883 build_loop_late( visited, worklist, nstack );
2884
2885 if (_verify_only) {
2886 C->restore_major_progress(old_progress);
2887 assert(C->unique() == unique, "verification mode made Nodes? ? ?");
2888 assert(_igvn._worklist.size() == orig_worklist_size, "shouldn't push anything");
2889 return;
2890 }
2891
2892 // clear out the dead code after build_loop_late
2893 while (_deadlist.size()) {
2894 _igvn.remove_globally_dead_node(_deadlist.pop());
2895 }
2896
2897 if (stop_early) {
2898 assert(do_expensive_nodes, "why are we here?");
2899 if (process_expensive_nodes()) {
2900 // If we made some progress when processing expensive nodes then
2901 // the IGVN may modify the graph in a way that will allow us to
2902 // make some more progress: we need to try processing expensive
2903 // nodes again.
2904 C->set_major_progress();
2905 }
2906 _igvn.optimize();
2908 }
2909
2910 // Some parser-inserted loop predicates could never be used by loop
2911 // predication or they were moved away from loop during some optimizations.
2912 // For example, peeling. Eliminate them before next loop optimizations.
2913 eliminate_useless_predicates();
2914
2915 #ifndef PRODUCT
2916 C->verify_graph_edges();
2917 if (_verify_me) { // Nested verify pass?
2918 // Check to see if the verify mode is broken
2919 assert(C->unique() == unique, "non-optimize mode made Nodes? ? ?");
2920 return;
2921 }
2922 if(VerifyLoopOptimizations) verify();
2923 if(TraceLoopOpts && C->has_loops()) {
2924 _ltree_root->dump();
2925 }
2926 #endif
2927
2928 if (mode == LoopOptsBarrierInsertion) {
2929 // Expand late nodes here
2930 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
2931 bs->barrier_insertion(this);
2932 }
2933
2934 if (skip_loop_opts) {
2935 // restore major progress flag
2936 C->restore_major_progress(old_progress);
2937
2938 // Cleanup any modified bits
2939 _igvn.optimize();
2940
2941 if (C->log() != NULL) {
2942 log_loop_tree(_ltree_root, _ltree_root, C->log());
2943 }
2944 return;
2945 }
2946
2947 if (bs->optimize_loops(this, mode, visited, nstack, worklist)) {
2948 _igvn.optimize();
2949 if (C->log() != NULL) {
2950 log_loop_tree(_ltree_root, _ltree_root, C->log());
2951 }
2952 return;
2953 }
2954
2955 if (ReassociateInvariants) {
2956 AutoNodeBudget node_budget(this, AutoNodeBudget::NO_BUDGET_CHECK);
2963 // check for vectorized loops, any reassociation of invariants was already done
2964 if (is_counted && lpt->_head->as_CountedLoop()->is_unroll_only()) continue;
2965
2966 lpt->reassociate_invariants(this);
2967
2968 // Because RCE opportunities can be masked by split_thru_phi,
2969 // look for RCE candidates and inhibit split_thru_phi
2970 // on just their loop-phi's for this pass of loop opts
2971 if (SplitIfBlocks && do_split_ifs) {
2972 if (lpt->policy_range_check(this)) {
2973 lpt->_rce_candidate = 1; // = true
2974 }
2975 }
2976 }
2977 }
2978
2979 // Check for aggressive application of split-if and other transforms
2980 // that require basic-block info (like cloning through Phi's)
2981 if( SplitIfBlocks && do_split_ifs ) {
2982 visited.Clear();
2983 split_if_with_blocks( visited, nstack);
2984 NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); );
2985 }
2986
2987 if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) {
2988 C->set_major_progress();
2989 }
2990
2991 // Perform loop predication before iteration splitting
2992 if (C->has_loops() && !C->major_progress() && (C->predicate_count() > 0)) {
2993 _ltree_root->_child->loop_predication(this);
2994 }
2995
2996 if (OptimizeFill && UseLoopPredicate && C->has_loops() && !C->major_progress()) {
2997 if (do_intrinsify_fill()) {
2998 C->set_major_progress();
2999 }
3000 }
3001
3002 // Perform iteration-splitting on inner loops. Split iterations to avoid
3003 // range checks or one-shot null checks.
3004
3099 void PhaseIdealLoop::print_statistics() {
3100 tty->print_cr("PhaseIdealLoop=%d, sum _unique=%d", _loop_invokes, _loop_work);
3101 }
3102
3103 //------------------------------verify-----------------------------------------
3104 // Build a verify-only PhaseIdealLoop, and see that it agrees with me.
3105 static int fail; // debug only, so its multi-thread dont care
3106 void PhaseIdealLoop::verify() const {
3107 int old_progress = C->major_progress();
3108 ResourceMark rm;
3109 PhaseIdealLoop loop_verify( _igvn, this );
3110 VectorSet visited(Thread::current()->resource_area());
3111
3112 fail = 0;
3113 verify_compare( C->root(), &loop_verify, visited );
3114 assert( fail == 0, "verify loops failed" );
3115 // Verify loop structure is the same
3116 _ltree_root->verify_tree(loop_verify._ltree_root, NULL);
3117 // Reset major-progress. It was cleared by creating a verify version of
3118 // PhaseIdealLoop.
3119 C->restore_major_progress(old_progress);
3120 }
3121
3122 //------------------------------verify_compare---------------------------------
3123 // Make sure me and the given PhaseIdealLoop agree on key data structures
3124 void PhaseIdealLoop::verify_compare( Node *n, const PhaseIdealLoop *loop_verify, VectorSet &visited ) const {
3125 if( !n ) return;
3126 if( visited.test_set( n->_idx ) ) return;
3127 if( !_nodes[n->_idx] ) { // Unreachable
3128 assert( !loop_verify->_nodes[n->_idx], "both should be unreachable" );
3129 return;
3130 }
3131
3132 uint i;
3133 for( i = 0; i < n->req(); i++ )
3134 verify_compare( n->in(i), loop_verify, visited );
3135
3136 // Check the '_nodes' block/loop structure
3137 i = n->_idx;
3138 if( has_ctrl(n) ) { // We have control; verify has loop or ctrl
3139 if( _nodes[i] != loop_verify->_nodes[i] &&
4229 // state), Mods/Loads can float around. So free them up.
4230 switch( n->Opcode() ) {
4231 case Op_DivI:
4232 case Op_DivF:
4233 case Op_DivD:
4234 case Op_ModI:
4235 case Op_ModF:
4236 case Op_ModD:
4237 case Op_LoadB: // Same with Loads; they can sink
4238 case Op_LoadUB: // during loop optimizations.
4239 case Op_LoadUS:
4240 case Op_LoadD:
4241 case Op_LoadF:
4242 case Op_LoadI:
4243 case Op_LoadKlass:
4244 case Op_LoadNKlass:
4245 case Op_LoadL:
4246 case Op_LoadS:
4247 case Op_LoadP:
4248 case Op_LoadBarrierSlowReg:
4249 case Op_LoadN:
4250 case Op_LoadRange:
4251 case Op_LoadD_unaligned:
4252 case Op_LoadL_unaligned:
4253 case Op_StrComp: // Does a bunch of load-like effects
4254 case Op_StrEquals:
4255 case Op_StrIndexOf:
4256 case Op_StrIndexOfChar:
4257 case Op_AryEq:
4258 case Op_HasNegatives:
4259 pinned = false;
4260 }
4261 if( pinned ) {
4262 IdealLoopTree *chosen_loop = get_loop(n->is_CFG() ? n : get_ctrl(n));
4263 if( !chosen_loop->_child ) // Inner loop?
4264 chosen_loop->_body.push(n); // Collect inner loops
4265 return;
4266 }
4267 } else { // No slot zero
4268 if( n->is_CFG() ) { // CFG with no slot 0 is dead
|