6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "opto/opaquenode.hpp"
27 #include "opto/phaseX.hpp"
28
29 //=============================================================================
30 // Do not allow value-numbering
31 uint Opaque1Node::hash() const { return NO_HASH; }
32 uint Opaque1Node::cmp( const Node &n ) const {
33 return (&n == this); // Always fail except on self
34 }
35
36 //------------------------------Identity---------------------------------------
37 // If _major_progress, then more loop optimizations follow. Do NOT remove
38 // the opaque Node until no more loop ops can happen. Note the timing of
39 // _major_progress; it's set in the major loop optimizations THEN comes the
40 // call to IterGVN and any chance of hitting this code. Hence there's no
41 // phase-ordering problem with stripping Opaque1 in IGVN followed by some
42 // more loop optimizations that require it.
43 Node* Opaque1Node::Identity(PhaseGVN* phase) {
44 return phase->C->major_progress() ? this : in(1);
45 }
49 // value-numbering, most Ideal calls or Identity functions. This Node is
50 // specifically designed to prevent the pre-increment value of a loop trip
51 // counter from being live out of the bottom of the loop (hence causing the
52 // pre- and post-increment values both being live and thus requiring an extra
53 // temp register and an extra move). If we "accidentally" optimize through
54 // this kind of a Node, we'll get slightly pessimal, but correct, code. Thus
55 // it's OK to be slightly sloppy on optimizations here.
56
57 // Do not allow value-numbering
58 uint Opaque2Node::hash() const { return NO_HASH; }
59 uint Opaque2Node::cmp( const Node &n ) const {
60 return (&n == this); // Always fail except on self
61 }
62
63 Node* Opaque4Node::Identity(PhaseGVN* phase) {
64 return phase->C->major_progress() ? this : in(2);
65 }
66
67 const Type* Opaque4Node::Value(PhaseGVN* phase) const {
68 return phase->type(in(1));
69 }
70
71 //=============================================================================
72
73 uint ProfileBooleanNode::hash() const { return NO_HASH; }
74 uint ProfileBooleanNode::cmp( const Node &n ) const {
75 return (&n == this);
76 }
77
78 Node *ProfileBooleanNode::Ideal(PhaseGVN *phase, bool can_reshape) {
79 if (can_reshape && _delay_removal) {
80 _delay_removal = false;
81 return this;
82 } else {
83 return NULL;
84 }
85 }
86
87 Node* ProfileBooleanNode::Identity(PhaseGVN* phase) {
88 if (_delay_removal) {
|
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "opto/addnode.hpp"
27 #include "opto/cfgnode.hpp"
28 #include "opto/connode.hpp"
29 #include "opto/divnode.hpp"
30 #include "opto/loopnode.hpp"
31 #include "opto/opaquenode.hpp"
32 #include "opto/phaseX.hpp"
33
34 //=============================================================================
35 // Do not allow value-numbering
36 uint Opaque1Node::hash() const { return NO_HASH; }
37 uint Opaque1Node::cmp( const Node &n ) const {
38 return (&n == this); // Always fail except on self
39 }
40
41 //------------------------------Identity---------------------------------------
42 // If _major_progress, then more loop optimizations follow. Do NOT remove
43 // the opaque Node until no more loop ops can happen. Note the timing of
44 // _major_progress; it's set in the major loop optimizations THEN comes the
45 // call to IterGVN and any chance of hitting this code. Hence there's no
46 // phase-ordering problem with stripping Opaque1 in IGVN followed by some
47 // more loop optimizations that require it.
48 Node* Opaque1Node::Identity(PhaseGVN* phase) {
49 return phase->C->major_progress() ? this : in(1);
50 }
54 // value-numbering, most Ideal calls or Identity functions. This Node is
55 // specifically designed to prevent the pre-increment value of a loop trip
56 // counter from being live out of the bottom of the loop (hence causing the
57 // pre- and post-increment values both being live and thus requiring an extra
58 // temp register and an extra move). If we "accidentally" optimize through
59 // this kind of a Node, we'll get slightly pessimal, but correct, code. Thus
60 // it's OK to be slightly sloppy on optimizations here.
61
62 // Do not allow value-numbering
63 uint Opaque2Node::hash() const { return NO_HASH; }
64 uint Opaque2Node::cmp( const Node &n ) const {
65 return (&n == this); // Always fail except on self
66 }
67
68 Node* Opaque4Node::Identity(PhaseGVN* phase) {
69 return phase->C->major_progress() ? this : in(2);
70 }
71
72 const Type* Opaque4Node::Value(PhaseGVN* phase) const {
73 return phase->type(in(1));
74 }
75
76 CountedLoopNode* Opaque5Node::inner_loop() const {
77 if (outcnt() != 1) {
78 return NULL;
79 }
80 Node* cmp = unique_out();
81 if (cmp == NULL || cmp->outcnt() != 1 || cmp->Opcode() != Op_CmpI) {
82 return NULL;
83 }
84 Node* test = cmp->unique_out();
85 if (test == NULL || test->outcnt() != 1 || test->Opcode() != Op_Bool) {
86 return NULL;
87 }
88 Node* lex = test->unique_out();
89 if (lex == NULL || lex->Opcode() != Op_If) {
90 return NULL;
91 }
92 IfNode* le = lex->as_If();
93 Node* le_tail = le->proj_out(true);
94 if (le_tail == NULL) {
95 return NULL;
96 }
97 Node* lx = le_tail->unique_ctrl_out();
98 if (lx == NULL || !lx->is_Loop()) {
99 return NULL;
100 }
101 LoopNode* l = lx->as_Loop();
102 if (!lx->as_Loop()->is_strip_mined() ||
103 le->in(0) == NULL ||
104 le->in(0)->in(0) == NULL) {
105 return NULL;
106 }
107 Node* inner_clex = le->in(0)->in(0)->in(0);
108 if (inner_clex == NULL || !inner_clex->is_CountedLoopEnd()) {
109 return NULL;
110 }
111 CountedLoopEndNode* inner_cle = inner_clex->as_CountedLoopEnd();
112 Node* inner_clx = l->unique_ctrl_out();
113 if (inner_clx == NULL || !inner_clx->is_CountedLoop()) {
114 return NULL;
115 }
116 CountedLoopNode* inner_cl = inner_clx->as_CountedLoop();
117 assert(inner_cl->is_strip_mined(), "inner loop should be strip mined");
118 return inner_cl;
119 }
120
121
122 Node* Opaque5Node::adjust_strip_mined_loop(PhaseGVN* phase) {
123 // Look for the outer & inner strip mined loop, reduce number of
124 // iterations of the inner loop, set exit condition of outer loop,
125 // construct required phi nodes for outer loop.
126 CountedLoopNode* inner_cl = inner_loop();
127 PhaseIterGVN *igvn = phase->is_IterGVN();
128 Node* inner_iv_phi = inner_cl->phi();
129 if (inner_iv_phi == NULL) {
130 return NULL;
131 }
132 CountedLoopEndNode* inner_cle = inner_cl->loopexit();
133 Node* cmp = inner_cl->outer_cmp();
134 BoolNode* bol = inner_cl->outer_bol();
135 assert(cmp->in(1) == inner_cle->cmp_node()->in(1), "broken comparison");
136 assert(bol->_test._test == inner_cle->test_trip(), "broken comparison");
137
138 int stride = inner_cl->stride_con();
139 jlong scaled_iters_long = ((jlong)LoopStripMiningIter) * ABS(stride);
140 int scaled_iters = (int)scaled_iters_long;
141 int short_scaled_iters = LoopStripMiningIterShortLoop* ABS(stride);
142 const TypeInt* inner_iv_t = phase->type(inner_iv_phi)->is_int();
143 jlong iter_estimate = (jlong)inner_iv_t->_hi - (jlong)inner_iv_t->_lo;
144 assert(iter_estimate > 0, "broken");
145 if ((jlong)scaled_iters != scaled_iters_long || iter_estimate <= short_scaled_iters) {
146 // Remove outer loop and safepoint (too few iterations)
147 Node* outer_sfpt = inner_cl->outer_safepoint();
148 Node* outer_out = inner_cl->outer_loop_exit();
149 igvn->replace_node(outer_out, outer_sfpt->in(0));
150 igvn->replace_input_of(outer_sfpt, 0, igvn->C->top());
151 inner_cl->clear_strip_mined();
152 return igvn->C->top();
153 }
154
155 Node* cle_tail = inner_cle->proj_out(true);
156 ResourceMark rm;
157 Node_List old_new;
158 if (cle_tail->outcnt() > 1) {
159 // Look for nodes on backedge of inner loop and clone them
160 Unique_Node_List backedge_nodes;
161 for (DUIterator_Fast imax, i = cle_tail->fast_outs(imax); i < imax; i++) {
162 Node* u = cle_tail->fast_out(i);
163 if (u != inner_cl) {
164 assert(!u->is_CFG(), "control flow on the backedge?");
165 backedge_nodes.push(u);
166 }
167 }
168 uint last = igvn->C->unique();
169 for (uint next = 0; next < backedge_nodes.size(); next++) {
170 Node* n = backedge_nodes.at(next);
171 old_new.map(n->_idx, n->clone());
172 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
173 Node* u = n->fast_out(i);
174 assert(!u->is_CFG(), "broken");
175 if (u->_idx >= last) {
176 continue;
177 }
178 if (!u->is_Phi()) {
179 backedge_nodes.push(u);
180 } else {
181 assert(u->in(0) == inner_cl, "strange phi on the backedge");
182 }
183 }
184 }
185 // Put the clones on the outer loop backedge
186 Node* le_tail = inner_cl->outer_loop_tail();
187 for (uint next = 0; next < backedge_nodes.size(); next++) {
188 Node *n = old_new[backedge_nodes.at(next)->_idx];
189 for (uint i = 1; i < n->req(); i++) {
190 if (n->in(i) != NULL && old_new[n->in(i)->_idx] != NULL) {
191 n->set_req(i, old_new[n->in(i)->_idx]);
192 }
193 }
194 if (n->in(0) != NULL) {
195 assert(n->in(0) == cle_tail, "node not on backedge?");
196 n->set_req(0, le_tail);
197 }
198 igvn->register_new_node_with_optimizer(n);
199 }
200 }
201
202 Node* iv_phi = NULL;
203 // Make a clone of each phi in the inner loop
204 // for the outer loop
205 Node* l = inner_cl->outer_loop();
206 for (uint i = 0; i < inner_cl->outcnt(); i++) {
207 Node* u = inner_cl->raw_out(i);
208 if (u->is_Phi()) {
209 assert(u->in(0) == inner_cl, "inconsistent");
210 Node* phi = u->clone();
211 phi->set_req(0, l);
212 Node* be = old_new[phi->in(LoopNode::LoopBackControl)->_idx];
213 if (be != NULL) {
214 phi->set_req(LoopNode::LoopBackControl, be);
215 }
216 phi = igvn->transform(phi);
217 igvn->replace_input_of(u, LoopNode::EntryControl, phi);
218 if (u == inner_iv_phi) {
219 iv_phi = phi;
220 }
221 }
222 }
223 Node* cle_out = inner_cle->proj_out(false);
224 if (cle_out->outcnt() > 1) {
225 // Look for chains of stores that were sunk
226 // out of the inner loop and are in the outer loop
227 for (DUIterator_Fast imax, i = cle_out->fast_outs(imax); i < imax; i++) {
228 Node* u = cle_out->fast_out(i);
229 if (u->is_Store()) {
230 Node* first = u;
231 for(;;) {
232 Node* next = first->in(MemNode::Memory);
233 if (!next->is_Store() || next->in(0) != cle_out) {
234 break;
235 }
236 first = next;
237 }
238 Node* last = u;
239 for(;;) {
240 Node* next = NULL;
241 for (DUIterator_Fast jmax, j = last->fast_outs(jmax); j < jmax; j++) {
242 Node* uu = last->fast_out(j);
243 if (uu->is_Store() && uu->in(0) == cle_out) {
244 assert(next == NULL, "only one in the outer loop");
245 next = uu;
246 }
247 }
248 if (next == NULL) {
249 break;
250 }
251 last = next;
252 }
253 Node* phi = NULL;
254 for (DUIterator_Fast jmax, j = l->fast_outs(jmax); j < jmax; j++) {
255 Node* uu = l->fast_out(j);
256 if (uu->is_Phi()) {
257 Node* be = uu->in(LoopNode::LoopBackControl);
258 while (be->is_Store() && old_new[be->_idx] != NULL) {
259 ShouldNotReachHere();
260 be = be->in(MemNode::Memory);
261 }
262 if (be == last || be == first->in(MemNode::Memory)) {
263 assert(phi == NULL, "only one phi");
264 phi = uu;
265 }
266 }
267 }
268 #ifdef ASSERT
269 for (DUIterator_Fast jmax, j = l->fast_outs(jmax); j < jmax; j++) {
270 Node* uu = l->fast_out(j);
271 if (uu->is_Phi() && uu->bottom_type() == Type::MEMORY) {
272 if (uu->adr_type() == igvn->C->get_adr_type(igvn->C->get_alias_index(u->adr_type()))) {
273 assert(phi == uu, "what's that phi?");
274 } else if (uu->adr_type() == TypePtr::BOTTOM) {
275 Node* n = uu->in(LoopNode::LoopBackControl);
276 uint limit = igvn->C->live_nodes();
277 uint i = 0;
278 while (n != uu) {
279 i++;
280 assert(i < limit, "infinite loop");
281 if (n->is_Proj()) {
282 n = n->in(0);
283 } else if (n->is_SafePoint() || n->is_MemBar()) {
284 n = n->in(TypeFunc::Memory);
285 } else if (n->is_Phi()) {
286 n = n->in(1);
287 } else if (n->is_MergeMem()) {
288 n = n->as_MergeMem()->memory_at(igvn->C->get_alias_index(u->adr_type()));
289 } else if (n->is_Store() || n->is_LoadStore() || n->is_ClearArray()) {
290 n = n->in(MemNode::Memory);
291 } else {
292 n->dump();
293 ShouldNotReachHere();
294 }
295 }
296 }
297 }
298 }
299 #endif
300 if (phi == NULL) {
301 // If the an entire chains was sunk, the
302 // inner loop has no phi for that memory
303 // slice, create one for the outer loop
304 phi = PhiNode::make(l, first->in(MemNode::Memory), Type::MEMORY,
305 igvn->C->get_adr_type(igvn->C->get_alias_index(u->adr_type())));
306 phi->set_req(LoopNode::LoopBackControl, last);
307 phi = igvn->transform(phi);
308 igvn->replace_input_of(first, MemNode::Memory, phi);
309 } else {
310 // Or fix the outer loop fix to include
311 // that chain of stores.
312 Node* be = phi->in(LoopNode::LoopBackControl);
313 while (be->is_Store() && old_new[be->_idx] != NULL) {
314 ShouldNotReachHere();
315 be = be->in(MemNode::Memory);
316 }
317 if (be == first->in(MemNode::Memory)) {
318 if (be == phi->in(LoopNode::LoopBackControl)) {
319 igvn->replace_input_of(phi, LoopNode::LoopBackControl, last);
320 } else {
321 igvn->replace_input_of(be, MemNode::Memory, last);
322 }
323 } else {
324 #ifdef ASSERT
325 if (be == phi->in(LoopNode::LoopBackControl)) {
326 assert(phi->in(LoopNode::LoopBackControl) == last, "");
327 } else {
328 assert(be->in(MemNode::Memory) == last, "");
329 }
330 #endif
331 }
332 }
333 }
334 }
335 }
336
337 if (iv_phi != NULL) {
338 // Now adjust the inner loop's exit condition
339 Node* limit = inner_cl->limit();
340 Node* sub = NULL;
341 if (stride > 0) {
342 sub = igvn->transform(new SubINode(limit, iv_phi));
343 } else {
344 sub = igvn->transform(new SubINode(iv_phi, limit));
345 }
346 Node* min = igvn->transform(new MinINode(sub, igvn->intcon(scaled_iters)));
347 Node* new_limit = NULL;
348 if (stride > 0) {
349 new_limit = igvn->transform(new AddINode(min, iv_phi));
350 } else {
351 new_limit = igvn->transform(new SubINode(iv_phi, min));
352 }
353 igvn->replace_input_of(inner_cle->cmp_node(), 2, new_limit);
354 if (iter_estimate <= scaled_iters_long) {
355 // We would only go through one iteration of
356 // the outer loop: drop the outer loop but
357 // keep the safepoint so we don't run for
358 // too long without a safepoint
359 igvn->replace_input_of(inner_cl->outer_loop_end(), 1, igvn->intcon(0));
360 inner_cl->clear_strip_mined();
361 }
362 return limit;
363 }
364 return NULL;
365 }
366
367 //=============================================================================
368
369 uint ProfileBooleanNode::hash() const { return NO_HASH; }
370 uint ProfileBooleanNode::cmp( const Node &n ) const {
371 return (&n == this);
372 }
373
374 Node *ProfileBooleanNode::Ideal(PhaseGVN *phase, bool can_reshape) {
375 if (can_reshape && _delay_removal) {
376 _delay_removal = false;
377 return this;
378 } else {
379 return NULL;
380 }
381 }
382
383 Node* ProfileBooleanNode::Identity(PhaseGVN* phase) {
384 if (_delay_removal) {
|