1007 }
1008
1009 static bool merge_point_safe(Node* region) {
1010 // 4799512: Stop split_if_with_blocks from splitting a block with a ConvI2LNode
1011 // having a PhiNode input. This sidesteps the dangerous case where the split
1012 // ConvI2LNode may become TOP if the input Value() does not
1013 // overlap the ConvI2L range, leaving a node which may not dominate its
1014 // uses.
1015 // A better fix for this problem can be found in the BugTraq entry, but
1016 // expediency for Mantis demands this hack.
1017 // 6855164: If the merge point has a FastLockNode with a PhiNode input, we stop
1018 // split_if_with_blocks from splitting a block because we could not move around
1019 // the FastLockNode.
1020 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1021 Node* n = region->fast_out(i);
1022 if (n->is_Phi()) {
1023 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
1024 Node* m = n->fast_out(j);
1025 if (m->is_FastLock())
1026 return false;
1027 #ifdef _LP64
1028 if (m->Opcode() == Op_ConvI2L)
1029 return false;
1030 if (m->is_CastII() && m->isa_CastII()->has_range_check()) {
1031 return false;
1032 }
1033 #endif
1034 }
1035 }
1036 }
1037 return true;
1038 }
1039
1040
1041 //------------------------------place_near_use---------------------------------
1042 // Place some computation next to use but not inside inner loops.
1043 // For inner loop uses move it to the preheader area.
1044 Node *PhaseIdealLoop::place_near_use(Node *useblock) const {
1045 IdealLoopTree *u_loop = get_loop( useblock );
1046 if (u_loop->_irreducible) {
1293 Node* u = n->fast_out(i);
1294 if( !has_ctrl(u) ) break; // Found control user
1295 IdealLoopTree *u_loop = get_loop(get_ctrl(u));
1296 if( u_loop == n_loop ) break; // Found loop-varying use
1297 if( n_loop->is_member( u_loop ) ) break; // Found use in inner loop
1298 if( u->Opcode() == Op_Opaque1 ) break; // Found loop limit, bugfix for 4677003
1299 }
1300 bool did_break = (i < imax); // Did we break out of the previous loop?
1301 if (!did_break && n->outcnt() > 1) { // All uses in outer loops!
1302 Node *late_load_ctrl = NULL;
1303 if (n->is_Load()) {
1304 // If n is a load, get and save the result from get_late_ctrl(),
1305 // to be later used in calculating the control for n's clones.
1306 clear_dom_lca_tags();
1307 late_load_ctrl = get_late_ctrl(n, n_ctrl);
1308 }
1309 // If n is a load, and the late control is the same as the current
1310 // control, then the cloning of n is a pointless exercise, because
1311 // GVN will ensure that we end up where we started.
1312 if (!n->is_Load() || late_load_ctrl != n_ctrl) {
1313 for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin; ) {
1314 Node *u = n->last_out(j); // Clone private computation per use
1315 _igvn.rehash_node_delayed(u);
1316 Node *x = n->clone(); // Clone computation
1317 Node *x_ctrl = NULL;
1318 if( u->is_Phi() ) {
1319 // Replace all uses of normal nodes. Replace Phi uses
1320 // individually, so the separate Nodes can sink down
1321 // different paths.
1322 uint k = 1;
1323 while( u->in(k) != n ) k++;
1324 u->set_req( k, x );
1325 // x goes next to Phi input path
1326 x_ctrl = u->in(0)->in(k);
1327 --j;
1328 } else { // Normal use
1329 // Replace all uses
1330 for( uint k = 0; k < u->req(); k++ ) {
1331 if( u->in(k) == n ) {
1332 u->set_req( k, x );
1333 --j;
1334 }
1335 }
1336 x_ctrl = get_ctrl(u);
1337 }
1338
1339 // Find control for 'x' next to use but not inside inner loops.
1340 // For inner loop uses get the preheader area.
1341 x_ctrl = place_near_use(x_ctrl);
1342
1343 if (n->is_Load()) {
1344 // For loads, add a control edge to a CFG node outside of the loop
1345 // to force them to not combine and return back inside the loop
1346 // during GVN optimization (4641526).
1347 //
1348 // Because we are setting the actual control input, factor in
1349 // the result from get_late_ctrl() so we respect any
1350 // anti-dependences. (6233005).
1351 x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
1352
1353 // Don't allow the control input to be a CFG splitting node.
1354 // Such nodes should only have ProjNodes as outs, e.g. IfNode
1355 // should only have IfTrueNode and IfFalseNode (4985384).
1356 x_ctrl = find_non_split_ctrl(x_ctrl);
1357 assert(dom_depth(n_ctrl) <= dom_depth(x_ctrl), "n is later than its clone");
1358
1359 x->set_req(0, x_ctrl);
1360 }
1361 register_new_node(x, x_ctrl);
1362
3120 // Evacuate nodes in peel region into the not_peeled region if possible
3121 uint new_phi_cnt = 0;
3122 uint cloned_for_outside_use = 0;
3123 for (i = 0; i < peel_list.size();) {
3124 Node* n = peel_list.at(i);
3125 #if !defined(PRODUCT)
3126 if (TracePartialPeeling) n->dump();
3127 #endif
3128 bool incr = true;
3129 if ( !n->is_CFG() ) {
3130
3131 if ( has_use_in_set(n, not_peel) ) {
3132
3133 // If not used internal to the peeled region,
3134 // move "n" from peeled to not_peeled region.
3135
3136 if ( !has_use_internal_to_set(n, peel, loop) ) {
3137
3138 // if not pinned and not a load (which maybe anti-dependent on a store)
3139 // and not a CMove (Matcher expects only bool->cmove).
3140 if ( n->in(0) == NULL && !n->is_Load() && !n->is_CMove() ) {
3141 cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist );
3142 sink_list.push(n);
3143 peel >>= n->_idx; // delete n from peel set.
3144 not_peel <<= n->_idx; // add n to not_peel set.
3145 peel_list.remove(i);
3146 incr = false;
3147 #if !defined(PRODUCT)
3148 if (TracePartialPeeling) {
3149 tty->print_cr("sink to not_peeled region: %d newbb: %d",
3150 n->_idx, get_ctrl(n)->_idx);
3151 }
3152 #endif
3153 }
3154 } else {
3155 // Otherwise check for special def-use cases that span
3156 // the peel/not_peel boundary such as bool->if
3157 clone_for_special_use_inside_loop( loop, n, not_peel, sink_list, worklist );
3158 new_phi_cnt++;
3159 }
3160 }
|
1007 }
1008
1009 static bool merge_point_safe(Node* region) {
1010 // 4799512: Stop split_if_with_blocks from splitting a block with a ConvI2LNode
1011 // having a PhiNode input. This sidesteps the dangerous case where the split
1012 // ConvI2LNode may become TOP if the input Value() does not
1013 // overlap the ConvI2L range, leaving a node which may not dominate its
1014 // uses.
1015 // A better fix for this problem can be found in the BugTraq entry, but
1016 // expediency for Mantis demands this hack.
1017 // 6855164: If the merge point has a FastLockNode with a PhiNode input, we stop
1018 // split_if_with_blocks from splitting a block because we could not move around
1019 // the FastLockNode.
1020 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1021 Node* n = region->fast_out(i);
1022 if (n->is_Phi()) {
1023 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
1024 Node* m = n->fast_out(j);
1025 if (m->is_FastLock())
1026 return false;
1027 #if INCLUDE_SHENANDOAHGC
1028 if (m->is_ShenandoahBarrier() && m->has_out_with(Op_FastLock)) {
1029 return false;
1030 }
1031 #endif
1032 #ifdef _LP64
1033 if (m->Opcode() == Op_ConvI2L)
1034 return false;
1035 if (m->is_CastII() && m->isa_CastII()->has_range_check()) {
1036 return false;
1037 }
1038 #endif
1039 }
1040 }
1041 }
1042 return true;
1043 }
1044
1045
1046 //------------------------------place_near_use---------------------------------
1047 // Place some computation next to use but not inside inner loops.
1048 // For inner loop uses move it to the preheader area.
1049 Node *PhaseIdealLoop::place_near_use(Node *useblock) const {
1050 IdealLoopTree *u_loop = get_loop( useblock );
1051 if (u_loop->_irreducible) {
1298 Node* u = n->fast_out(i);
1299 if( !has_ctrl(u) ) break; // Found control user
1300 IdealLoopTree *u_loop = get_loop(get_ctrl(u));
1301 if( u_loop == n_loop ) break; // Found loop-varying use
1302 if( n_loop->is_member( u_loop ) ) break; // Found use in inner loop
1303 if( u->Opcode() == Op_Opaque1 ) break; // Found loop limit, bugfix for 4677003
1304 }
1305 bool did_break = (i < imax); // Did we break out of the previous loop?
1306 if (!did_break && n->outcnt() > 1) { // All uses in outer loops!
1307 Node *late_load_ctrl = NULL;
1308 if (n->is_Load()) {
1309 // If n is a load, get and save the result from get_late_ctrl(),
1310 // to be later used in calculating the control for n's clones.
1311 clear_dom_lca_tags();
1312 late_load_ctrl = get_late_ctrl(n, n_ctrl);
1313 }
1314 // If n is a load, and the late control is the same as the current
1315 // control, then the cloning of n is a pointless exercise, because
1316 // GVN will ensure that we end up where we started.
1317 if (!n->is_Load() || late_load_ctrl != n_ctrl) {
1318 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1319 for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin; ) {
1320 Node *u = n->last_out(j); // Clone private computation per use
1321 _igvn.rehash_node_delayed(u);
1322 Node *x = n->clone(); // Clone computation
1323 Node *x_ctrl = NULL;
1324 if( u->is_Phi() ) {
1325 // Replace all uses of normal nodes. Replace Phi uses
1326 // individually, so the separate Nodes can sink down
1327 // different paths.
1328 uint k = 1;
1329 while( u->in(k) != n ) k++;
1330 u->set_req( k, x );
1331 // x goes next to Phi input path
1332 x_ctrl = u->in(0)->in(k);
1333 --j;
1334 } else { // Normal use
1335 // Replace all uses
1336 for( uint k = 0; k < u->req(); k++ ) {
1337 if( u->in(k) == n ) {
1338 u->set_req( k, x );
1339 --j;
1340 }
1341 }
1342 x_ctrl = get_ctrl(u);
1343 }
1344
1345 // Find control for 'x' next to use but not inside inner loops.
1346 // For inner loop uses get the preheader area.
1347 x_ctrl = place_near_use(x_ctrl);
1348
1349 if (bs->sink_node(this, n, x, x_ctrl, n_ctrl)) {
1350 continue;
1351 }
1352
1353 if (n->is_Load()) {
1354 // For loads, add a control edge to a CFG node outside of the loop
1355 // to force them to not combine and return back inside the loop
1356 // during GVN optimization (4641526).
1357 //
1358 // Because we are setting the actual control input, factor in
1359 // the result from get_late_ctrl() so we respect any
1360 // anti-dependences. (6233005).
1361 x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
1362
1363 // Don't allow the control input to be a CFG splitting node.
1364 // Such nodes should only have ProjNodes as outs, e.g. IfNode
1365 // should only have IfTrueNode and IfFalseNode (4985384).
1366 x_ctrl = find_non_split_ctrl(x_ctrl);
1367 assert(dom_depth(n_ctrl) <= dom_depth(x_ctrl), "n is later than its clone");
1368
1369 x->set_req(0, x_ctrl);
1370 }
1371 register_new_node(x, x_ctrl);
1372
3130 // Evacuate nodes in peel region into the not_peeled region if possible
3131 uint new_phi_cnt = 0;
3132 uint cloned_for_outside_use = 0;
3133 for (i = 0; i < peel_list.size();) {
3134 Node* n = peel_list.at(i);
3135 #if !defined(PRODUCT)
3136 if (TracePartialPeeling) n->dump();
3137 #endif
3138 bool incr = true;
3139 if ( !n->is_CFG() ) {
3140
3141 if ( has_use_in_set(n, not_peel) ) {
3142
3143 // If not used internal to the peeled region,
3144 // move "n" from peeled to not_peeled region.
3145
3146 if ( !has_use_internal_to_set(n, peel, loop) ) {
3147
3148 // if not pinned and not a load (which maybe anti-dependent on a store)
3149 // and not a CMove (Matcher expects only bool->cmove).
3150 if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove() && n->Opcode() != Op_ShenandoahWBMemProj) {
3151 cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist );
3152 sink_list.push(n);
3153 peel >>= n->_idx; // delete n from peel set.
3154 not_peel <<= n->_idx; // add n to not_peel set.
3155 peel_list.remove(i);
3156 incr = false;
3157 #if !defined(PRODUCT)
3158 if (TracePartialPeeling) {
3159 tty->print_cr("sink to not_peeled region: %d newbb: %d",
3160 n->_idx, get_ctrl(n)->_idx);
3161 }
3162 #endif
3163 }
3164 } else {
3165 // Otherwise check for special def-use cases that span
3166 // the peel/not_peel boundary such as bool->if
3167 clone_for_special_use_inside_loop( loop, n, not_peel, sink_list, worklist );
3168 new_phi_cnt++;
3169 }
3170 }
|