113 unswitch_iff = iff;
114 }
115 }
116 }
117 }
118 }
119 n = n_dom;
120 }
121
122 Node* array;
123 if (unswitch_iff == NULL || unswitch_iff->is_flattened_array_check(&_igvn, array)) {
124 // collect all flattened array checks
125 for (uint i = 0; i < loop->_body.size(); i++) {
126 Node* n = loop->_body.at(i);
127 if (n->is_If() && n->as_If()->is_flattened_array_check(&_igvn, array) &&
128 loop->is_invariant(n->in(1)) &&
129 !loop->is_loop_exit(n)) {
130 flattened_checks.push(n);
131 }
132 }
133 unswitch_iff = NULL;
134 }
135
136 return unswitch_iff;
137 }
138
139 //------------------------------do_unswitching-----------------------------
140 // Clone loop with an invariant test (that does not exit) and
141 // insert a clone of the test that selects which version to
142 // execute.
143 void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) {
144
145 // Find first invariant test that doesn't exit the loop
146 LoopNode *head = loop->_head->as_Loop();
147
148 Node_List flattened_checks;
149 IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop, flattened_checks);
150 assert(unswitch_iff != NULL || flattened_checks.size() > 0, "should be at least one");
151 if (unswitch_iff == NULL) {
152 unswitch_iff = flattened_checks.at(0)->as_If();
153 }
179 if (n != NULL) {
180 predicate = n;
181 entry = skip_loop_predicates(entry);
182 }
183 }
184 if (predicate != NULL && UseProfiledLoopPredicate) {
185 entry = find_predicate(entry);
186 if (entry != NULL) predicate = entry;
187 }
188 if (predicate != NULL) predicate = predicate->in(0);
189 assert(proj_true->is_IfTrue() &&
190 (predicate == NULL && uniqc == head && !head->is_strip_mined() ||
191 predicate == NULL && uniqc == head->in(LoopNode::EntryControl) && head->is_strip_mined() ||
192 predicate != NULL && uniqc == predicate), "by construction");
193 #endif
194 // Increment unswitch count
195 LoopNode* head_clone = old_new[head->_idx]->as_Loop();
196 int nct = head->unswitch_count() + 1;
197 head->set_unswitch_count(nct);
198 head_clone->set_unswitch_count(nct);
199 head_clone->mark_flattened_arrays();
200
201 // Add test to new "if" outside of loop
202 IfNode* invar_iff = proj_true->in(0)->as_If();
203 Node* invar_iff_c = invar_iff->in(0);
204 invar_iff->_prob = unswitch_iff->_prob;
205 if (flattened_checks.size() > 0) {
206 // Flattened array checks are used in
207 // Parse::array_store()/Parse::array_load() to switch between a
208 // legacy object array access and a flattened value array
209 // access. We want the performance impact on legacy accesses to be
210 // as small as possible so we make 2 copies of the loops: a fast
211 // one where all accesses are known to be legacy, a slow one where
212 // some accesses are to flattened arrays. Flattened array checks
213 // can be removed from the first one but not from the second one
214 // as it can have a mix of flattened/legacy accesses.
215 BoolNode* bol = unswitch_iff->in(1)->clone()->as_Bool();
216 register_new_node(bol, invar_iff->in(0));
217 Node* cmp = bol->in(1)->clone();
218 register_new_node(cmp, invar_iff->in(0));
219 bol->set_req(1, cmp);
220 Node* in1 = NULL;
221 for (uint i = 0; i < flattened_checks.size(); i++) {
222 Node* v = flattened_checks.at(i)->in(1)->in(1)->in(1);
223 v = new AndINode(v, _igvn.intcon(Klass::_lh_array_tag_vt_value));
224 register_new_node(v, invar_iff->in(0));
225 if (in1 == NULL) {
226 in1 = v;
227 } else {
228 in1 = new OrINode(in1, v);
229 register_new_node(in1, invar_iff->in(0));
230 }
231 }
232 cmp->set_req(1, in1);
233 invar_iff->set_req(1, bol);
234 } else {
235 BoolNode* bol = unswitch_iff->in(1)->as_Bool();
236 invar_iff->set_req(1, bol);
237 }
238
239 ProjNode* proj_false = invar_iff->proj_out(0)->as_Proj();
240
241 // Hoist invariant casts out of each loop to the appropriate
242 // control projection.
243
244 Node_List worklist;
245
246 for (DUIterator_Fast imax, i = unswitch_iff->fast_outs(imax); i < imax; i++) {
247 ProjNode* proj= unswitch_iff->fast_out(i)->as_Proj();
248 // Copy to a worklist for easier manipulation
249 for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) {
250 Node* use = proj->fast_out(j);
251 if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) {
252 worklist.push(use);
253 }
254 }
255 ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj();
256 while (worklist.size() > 0) {
257 Node* use = worklist.pop();
258 Node* nuse = use->clone();
259 nuse->set_req(0, invar_proj);
260 _igvn.replace_input_of(use, 1, nuse);
261 register_new_node(nuse, invar_proj);
262 // Same for the clone
263 Node* use_clone = old_new[use->_idx];
264 _igvn.replace_input_of(use_clone, 1, nuse);
265 }
266 }
267
268 IfNode* unswitch_iff_clone = old_new[unswitch_iff->_idx]->as_If();
269 if (flattened_checks.size() > 0) {
270 for (uint i = 0; i < flattened_checks.size(); i++) {
271 IfNode* iff = flattened_checks.at(i)->as_If();
272 _igvn.rehash_node_delayed(iff);
273 short_circuit_if(iff, proj_true);
274 }
275 } else {
276 // Hardwire the control paths in the loops into if(true) and if(false)
277 _igvn.rehash_node_delayed(unswitch_iff);
278 short_circuit_if(unswitch_iff, proj_true);
279
280 _igvn.rehash_node_delayed(unswitch_iff_clone);
281 short_circuit_if(unswitch_iff_clone, proj_false);
282 }
283
284 // Reoptimize loops
285 loop->record_for_igvn();
286 for(int i = loop->_body.size() - 1; i >= 0 ; i--) {
287 Node *n = loop->_body[i];
288 Node *n_clone = old_new[n->_idx];
289 _igvn._worklist.push(n_clone);
290 }
291
292 #ifndef PRODUCT
293 if (TraceLoopUnswitching) {
|
113 unswitch_iff = iff;
114 }
115 }
116 }
117 }
118 }
119 n = n_dom;
120 }
121
122 Node* array;
123 if (unswitch_iff == NULL || unswitch_iff->is_flattened_array_check(&_igvn, array)) {
124 // collect all flattened array checks
125 for (uint i = 0; i < loop->_body.size(); i++) {
126 Node* n = loop->_body.at(i);
127 if (n->is_If() && n->as_If()->is_flattened_array_check(&_igvn, array) &&
128 loop->is_invariant(n->in(1)) &&
129 !loop->is_loop_exit(n)) {
130 flattened_checks.push(n);
131 }
132 }
133 if (flattened_checks.size() > 1) {
134 unswitch_iff = NULL;
135 } else {
136 flattened_checks.clear();
137 }
138 }
139
140 return unswitch_iff;
141 }
142
143 //------------------------------do_unswitching-----------------------------
144 // Clone loop with an invariant test (that does not exit) and
145 // insert a clone of the test that selects which version to
146 // execute.
147 void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) {
148
149 // Find first invariant test that doesn't exit the loop
150 LoopNode *head = loop->_head->as_Loop();
151
152 Node_List flattened_checks;
153 IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop, flattened_checks);
154 assert(unswitch_iff != NULL || flattened_checks.size() > 0, "should be at least one");
155 if (unswitch_iff == NULL) {
156 unswitch_iff = flattened_checks.at(0)->as_If();
157 }
183 if (n != NULL) {
184 predicate = n;
185 entry = skip_loop_predicates(entry);
186 }
187 }
188 if (predicate != NULL && UseProfiledLoopPredicate) {
189 entry = find_predicate(entry);
190 if (entry != NULL) predicate = entry;
191 }
192 if (predicate != NULL) predicate = predicate->in(0);
193 assert(proj_true->is_IfTrue() &&
194 (predicate == NULL && uniqc == head && !head->is_strip_mined() ||
195 predicate == NULL && uniqc == head->in(LoopNode::EntryControl) && head->is_strip_mined() ||
196 predicate != NULL && uniqc == predicate), "by construction");
197 #endif
198 // Increment unswitch count
199 LoopNode* head_clone = old_new[head->_idx]->as_Loop();
200 int nct = head->unswitch_count() + 1;
201 head->set_unswitch_count(nct);
202 head_clone->set_unswitch_count(nct);
203 if (flattened_checks.size() > 0) {
204 head->mark_flattened_arrays();
205 }
206
207 // Add test to new "if" outside of loop
208 IfNode* invar_iff = proj_true->in(0)->as_If();
209 Node* invar_iff_c = invar_iff->in(0);
210 invar_iff->_prob = unswitch_iff->_prob;
211 if (flattened_checks.size() > 0) {
212 // Flattened array checks are used in
213 // Parse::array_store()/Parse::array_load() to switch between a
214 // legacy object array access and a flattened value array
215 // access. We want the performance impact on legacy accesses to be
216 // as small as possible so we make 2 copies of the loops: a fast
217 // one where all accesses are known to be legacy, a slow one where
218 // some accesses are to flattened arrays. Flattened array checks
219 // can be removed from the first one but not from the second one
220 // as it can have a mix of flattened/legacy accesses.
221 BoolNode* bol = unswitch_iff->in(1)->clone()->as_Bool();
222 register_new_node(bol, invar_iff->in(0));
223 Node* cmp = bol->in(1)->clone();
224 register_new_node(cmp, invar_iff->in(0));
225 bol->set_req(1, cmp);
226 Node* in1 = NULL;
227 for (uint i = 0; i < flattened_checks.size(); i++) {
228 Node* v = flattened_checks.at(i)->in(1)->in(1)->in(1);
229 if (in1 == NULL) {
230 in1 = v;
231 } else {
232 if (cmp->Opcode() == Op_CmpL) {
233 in1 = new OrLNode(in1, v);
234 } else {
235 in1 = new OrINode(in1, v);
236 }
237 register_new_node(in1, invar_iff->in(0));
238 }
239 }
240 cmp->set_req(1, in1);
241 invar_iff->set_req(1, bol);
242 } else {
243 BoolNode* bol = unswitch_iff->in(1)->as_Bool();
244 invar_iff->set_req(1, bol);
245 }
246
247 ProjNode* proj_false = invar_iff->proj_out(0)->as_Proj();
248
249 // Hoist invariant casts out of each loop to the appropriate
250 // control projection.
251
252 Node_List worklist;
253
254 if (flattened_checks.size() > 0) {
255 for (uint i = 0; i < flattened_checks.size(); i++) {
256 IfNode* iff = flattened_checks.at(i)->as_If();
257 ProjNode* proj= iff->proj_out(0)->as_Proj();
258 // Copy to a worklist for easier manipulation
259 for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) {
260 Node* use = proj->fast_out(j);
261 if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) {
262 worklist.push(use);
263 }
264 }
265 ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj();
266 while (worklist.size() > 0) {
267 Node* use = worklist.pop();
268 Node* nuse = use->clone();
269 nuse->set_req(0, invar_proj);
270 _igvn.replace_input_of(use, 1, nuse);
271 register_new_node(nuse, invar_proj);
272 // Same for the clone
273 Node* use_clone = old_new[use->_idx];
274 _igvn.replace_input_of(use_clone, 1, nuse);
275 }
276 }
277 } else {
278 for (DUIterator_Fast imax, i = unswitch_iff->fast_outs(imax); i < imax; i++) {
279 ProjNode* proj= unswitch_iff->fast_out(i)->as_Proj();
280 // Copy to a worklist for easier manipulation
281 for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) {
282 Node* use = proj->fast_out(j);
283 if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) {
284 worklist.push(use);
285 }
286 }
287 ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj();
288 while (worklist.size() > 0) {
289 Node* use = worklist.pop();
290 Node* nuse = use->clone();
291 nuse->set_req(0, invar_proj);
292 _igvn.replace_input_of(use, 1, nuse);
293 register_new_node(nuse, invar_proj);
294 // Same for the clone
295 Node* use_clone = old_new[use->_idx];
296 _igvn.replace_input_of(use_clone, 1, nuse);
297 }
298 }
299 }
300
301 IfNode* unswitch_iff_clone = old_new[unswitch_iff->_idx]->as_If();
302 if (flattened_checks.size() > 0) {
303 for (uint i = 0; i < flattened_checks.size(); i++) {
304 IfNode* iff = flattened_checks.at(i)->as_If();
305 _igvn.rehash_node_delayed(iff);
306 short_circuit_if(old_new[iff->_idx]->as_If(), proj_false);
307 }
308 } else {
309 // Hardwire the control paths in the loops into if(true) and if(false)
310 _igvn.rehash_node_delayed(unswitch_iff);
311 short_circuit_if(unswitch_iff, proj_true);
312
313 _igvn.rehash_node_delayed(unswitch_iff_clone);
314 short_circuit_if(unswitch_iff_clone, proj_false);
315 }
316
317 // Reoptimize loops
318 loop->record_for_igvn();
319 for(int i = loop->_body.size() - 1; i >= 0 ; i--) {
320 Node *n = loop->_body[i];
321 Node *n_clone = old_new[n->_idx];
322 _igvn._worklist.push(n_clone);
323 }
324
325 #ifndef PRODUCT
326 if (TraceLoopUnswitching) {
|