1212
1213 if (adjust_min_trip) {
1214 // Step 2: Adjust the trip limit if it is called for.
1215 // The adjustment amount is -stride. Need to make sure if the
1216 // adjustment underflows or overflows, then the main loop is skipped.
1217 Node* cmp = loop_end->cmp_node();
1218 assert(cmp->in(2) == limit, "sanity");
1219 assert(opaq != NULL && opaq->in(1) == limit, "sanity");
1220
1221 // Verify that policy_unroll result is still valid.
1222 const TypeInt* limit_type = _igvn.type(limit)->is_int();
1223 assert(stride_con > 0 && ((limit_type->_hi - stride_con) < limit_type->_hi) ||
1224 stride_con < 0 && ((limit_type->_lo - stride_con) > limit_type->_lo), "sanity");
1225
1226 if (limit->is_Con()) {
1227 // The check in policy_unroll and the assert above guarantee
1228 // no underflow if limit is constant.
1229 new_limit = _igvn.intcon(limit->get_int() - stride_con);
1230 set_ctrl(new_limit, C->root());
1231 } else {
1232 if (stride_con > 0 && ((limit_type->_lo - stride_con) < limit_type->_lo) ||
1233 stride_con < 0 && ((limit_type->_hi - stride_con) > limit_type->_hi)) {
1234 // No underflow.
1235 new_limit = new (C, 3) SubINode(limit, stride);
1236 } else {
1237 // (limit - stride) may underflow.
1238 // Clamp the adjustment value with MININT or MAXINT:
1239 //
1240 // new_limit = limit-stride
1241 // if (stride > 0)
1242 // new_limit = (limit < new_limit) ? MININT : new_limit;
1243 // else
1244 // new_limit = (limit > new_limit) ? MAXINT : new_limit;
1245 //
1246 BoolTest::mask bt = loop_end->test_trip();
1247 assert(bt == BoolTest::lt || bt == BoolTest::gt, "canonical test is expected");
1248 Node* adj_max = _igvn.intcon((stride_con > 0) ? min_jint : max_jint);
1249 set_ctrl(adj_max, C->root());
1250 Node* old_limit = NULL;
1251 Node* adj_limit = NULL;
1261 // use original limit as old limit.
1262 old_limit = bol->in(1)->in(1);
1263 // Adjust previous adjusted limit.
1264 adj_limit = limit->in(CMoveNode::IfFalse);
1265 adj_limit = new (C, 3) SubINode(adj_limit, stride);
1266 } else {
1267 old_limit = limit;
1268 adj_limit = new (C, 3) SubINode(limit, stride);
1269 }
1270 assert(old_limit != NULL && adj_limit != NULL, "");
1271 register_new_node( adj_limit, ctrl ); // adjust amount
1272 Node* adj_cmp = new (C, 3) CmpINode(old_limit, adj_limit);
1273 register_new_node( adj_cmp, ctrl );
1274 Node* adj_bool = new (C, 2) BoolNode(adj_cmp, bt);
1275 register_new_node( adj_bool, ctrl );
1276 new_limit = new (C, 4) CMoveINode(adj_bool, adj_limit, adj_max, TypeInt::INT);
1277 }
1278 register_new_node(new_limit, ctrl);
1279 }
1280 assert(new_limit != NULL, "");
1281 if (limit->outcnt() == 2) {
1282 // Replace old limit if it is used only in loop tests.
1283 _igvn.replace_node(limit, new_limit);
1284 } else {
1285 // Replace in loop test.
1286 _igvn.hash_delete(cmp);
1287 cmp->set_req(2, new_limit);
1288
1289 // Step 3: Find the min-trip test guaranteed before a 'main' loop.
1290 // Make it a 1-trip test (means at least 2 trips).
1291
1292 // Guard test uses an 'opaque' node which is not shared. Hence I
1293 // can edit it's inputs directly. Hammer in the new limit for the
1294 // minimum-trip guard.
1295 assert(opaq->outcnt() == 1, "");
1296 _igvn.hash_delete(opaq);
1297 opaq->set_req(1, new_limit);
1298 }
1299 }
1300
1301 // Adjust max trip count. The trip count is intentionally rounded
1302 // down here (e.g. 15-> 7-> 3-> 1) because if we unwittingly over-unroll,
1303 // the main, unrolled, part of the loop will never execute as it is protected
1304 // by the min-trip test. See bug 4834191 for a case where we over-unrolled
1305 // and later determined that part of the unrolled loop was dead.
1306 loop_head->set_trip_count(old_trip_count / 2);
1307
1308 // Double the count of original iterations in the unrolled loop body.
1309 loop_head->double_unrolled_count();
1310
1311 } else { // LoopLimitCheck
1312
1313 // Adjust max trip count. The trip count is intentionally rounded
1314 // down here (e.g. 15-> 7-> 3-> 1) because if we unwittingly over-unroll,
1315 // the main, unrolled, part of the loop will never execute as it is protected
1316 // by the min-trip test. See bug 4834191 for a case where we over-unrolled
1317 // and later determined that part of the unrolled loop was dead.
1318 loop_head->set_trip_count(loop_head->trip_count() / 2);
1319
|
1212
1213 if (adjust_min_trip) {
1214 // Step 2: Adjust the trip limit if it is called for.
1215 // The adjustment amount is -stride. Need to make sure if the
1216 // adjustment underflows or overflows, then the main loop is skipped.
1217 Node* cmp = loop_end->cmp_node();
1218 assert(cmp->in(2) == limit, "sanity");
1219 assert(opaq != NULL && opaq->in(1) == limit, "sanity");
1220
1221 // Verify that policy_unroll result is still valid.
1222 const TypeInt* limit_type = _igvn.type(limit)->is_int();
1223 assert(stride_con > 0 && ((limit_type->_hi - stride_con) < limit_type->_hi) ||
1224 stride_con < 0 && ((limit_type->_lo - stride_con) > limit_type->_lo), "sanity");
1225
1226 if (limit->is_Con()) {
1227 // The check in policy_unroll and the assert above guarantee
1228 // no underflow if limit is constant.
1229 new_limit = _igvn.intcon(limit->get_int() - stride_con);
1230 set_ctrl(new_limit, C->root());
1231 } else {
1232 // Limit is not constant.
1233 {
1234 // Separate limit by Opaque node in case it is an incremented
1235 // variable from previous loop to avoid using pre-incremented
1236 // value which could increase register pressure.
1237 // Otherwise reorg_offsets() optimization will create a separate
1238 // Opaque node for each use of trip-counter and as result
1239 // zero trip guard limit will be different from loop limit.
1240 assert(has_ctrl(opaq), "should have it");
1241 Node* opaq_ctrl = get_ctrl(opaq);
1242 limit = new (C, 2) Opaque2Node( C, limit );
1243 register_new_node( limit, opaq_ctrl );
1244 }
1245 if (stride_con > 0 && ((limit_type->_lo - stride_con) < limit_type->_lo) ||
1246 stride_con < 0 && ((limit_type->_hi - stride_con) > limit_type->_hi)) {
1247 // No underflow.
1248 new_limit = new (C, 3) SubINode(limit, stride);
1249 } else {
1250 // (limit - stride) may underflow.
1251 // Clamp the adjustment value with MININT or MAXINT:
1252 //
1253 // new_limit = limit-stride
1254 // if (stride > 0)
1255 // new_limit = (limit < new_limit) ? MININT : new_limit;
1256 // else
1257 // new_limit = (limit > new_limit) ? MAXINT : new_limit;
1258 //
1259 BoolTest::mask bt = loop_end->test_trip();
1260 assert(bt == BoolTest::lt || bt == BoolTest::gt, "canonical test is expected");
1261 Node* adj_max = _igvn.intcon((stride_con > 0) ? min_jint : max_jint);
1262 set_ctrl(adj_max, C->root());
1263 Node* old_limit = NULL;
1264 Node* adj_limit = NULL;
1274 // use original limit as old limit.
1275 old_limit = bol->in(1)->in(1);
1276 // Adjust previous adjusted limit.
1277 adj_limit = limit->in(CMoveNode::IfFalse);
1278 adj_limit = new (C, 3) SubINode(adj_limit, stride);
1279 } else {
1280 old_limit = limit;
1281 adj_limit = new (C, 3) SubINode(limit, stride);
1282 }
1283 assert(old_limit != NULL && adj_limit != NULL, "");
1284 register_new_node( adj_limit, ctrl ); // adjust amount
1285 Node* adj_cmp = new (C, 3) CmpINode(old_limit, adj_limit);
1286 register_new_node( adj_cmp, ctrl );
1287 Node* adj_bool = new (C, 2) BoolNode(adj_cmp, bt);
1288 register_new_node( adj_bool, ctrl );
1289 new_limit = new (C, 4) CMoveINode(adj_bool, adj_limit, adj_max, TypeInt::INT);
1290 }
1291 register_new_node(new_limit, ctrl);
1292 }
1293 assert(new_limit != NULL, "");
1294 // Replace in loop test.
1295 _igvn.hash_delete(cmp);
1296 cmp->set_req(2, new_limit);
1297
1298 // Step 3: Find the min-trip test guaranteed before a 'main' loop.
1299 // Make it a 1-trip test (means at least 2 trips).
1300
1301 // Guard test uses an 'opaque' node which is not shared. Hence I
1302 // can edit it's inputs directly. Hammer in the new limit for the
1303 // minimum-trip guard.
1304 assert(opaq->outcnt() == 1, "");
1305 _igvn.hash_delete(opaq);
1306 opaq->set_req(1, new_limit);
1307 }
1308
1309 // Adjust max trip count. The trip count is intentionally rounded
1310 // down here (e.g. 15-> 7-> 3-> 1) because if we unwittingly over-unroll,
1311 // the main, unrolled, part of the loop will never execute as it is protected
1312 // by the min-trip test. See bug 4834191 for a case where we over-unrolled
1313 // and later determined that part of the unrolled loop was dead.
1314 loop_head->set_trip_count(old_trip_count / 2);
1315
1316 // Double the count of original iterations in the unrolled loop body.
1317 loop_head->double_unrolled_count();
1318
1319 } else { // LoopLimitCheck
1320
1321 // Adjust max trip count. The trip count is intentionally rounded
1322 // down here (e.g. 15-> 7-> 3-> 1) because if we unwittingly over-unroll,
1323 // the main, unrolled, part of the loop will never execute as it is protected
1324 // by the min-trip test. See bug 4834191 for a case where we over-unrolled
1325 // and later determined that part of the unrolled loop was dead.
1326 loop_head->set_trip_count(loop_head->trip_count() / 2);
1327
|