104 int unswitch_max() { return _unswitch_max; }
105 int unswitch_count() { return _unswitch_count; }
106
107 int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; }
108 void set_has_been_range_checked() { _postloop_flags |= LoopRCEChecked; }
109 int is_rce_post_loop() const { return _postloop_flags & RCEPostLoop; }
110 void set_is_rce_post_loop() { _postloop_flags |= RCEPostLoop; }
111
112 void set_unswitch_count(int val) {
113 assert (val <= unswitch_max(), "too many unswitches");
114 _unswitch_count = val;
115 }
116
117 LoopNode(Node *entry, Node *backedge) : RegionNode(3), _loop_flags(0), _unswitch_count(0), _postloop_flags(0) {
118 init_class_id(Class_Loop);
119 init_req(EntryControl, entry);
120 init_req(LoopBackControl, backedge);
121 }
122
123 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
124 virtual int Opcode() const;
125 bool can_be_counted_loop(PhaseTransform* phase) const {
126 return req() == 3 && in(0) != NULL &&
127 in(1) != NULL && phase->type(in(1)) != Type::TOP &&
128 in(2) != NULL && phase->type(in(2)) != Type::TOP;
129 }
130 bool is_valid_counted_loop() const;
131 #ifndef PRODUCT
132 virtual void dump_spec(outputStream *st) const;
133 #endif
134 };
135
136 //------------------------------Counted Loops----------------------------------
137 // Counted loops are all trip-counted loops, with exactly 1 trip-counter exit
138 // path (and maybe some other exit paths). The trip-counter exit is always
139 // last in the loop. The trip-counter have to stride by a constant;
140 // the exit value is also loop invariant.
141
142 // CountedLoopNodes and CountedLoopEndNodes come in matched pairs. The
143 // CountedLoopNode has the incoming loop control and the loop-back-control
144 // which is always the IfTrue before the matching CountedLoopEndNode. The
181 int _unrolled_count_log2;
182
183 // Node count prior to last unrolling - used to decide if
184 // unroll,optimize,unroll,optimize,... is making progress
185 int _node_count_before_unroll;
186
187 // If slp analysis is performed we record the maximum
188 // vector mapped unroll factor here
189 int _slp_maximum_unroll_factor;
190
191 public:
192 CountedLoopNode( Node *entry, Node *backedge )
193 : LoopNode(entry, backedge), _main_idx(0), _trip_count(max_juint),
194 _profile_trip_cnt(COUNT_UNKNOWN), _unrolled_count_log2(0),
195 _node_count_before_unroll(0), _slp_maximum_unroll_factor(0) {
196 init_class_id(Class_CountedLoop);
197 // Initialize _trip_count to the largest possible value.
198 // Will be reset (lower) if the loop's trip count is known.
199 }
200
201 virtual int Opcode() const;
202 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
203
204 Node *init_control() const { return in(EntryControl); }
205 Node *back_control() const { return in(LoopBackControl); }
206 CountedLoopEndNode *loopexit() const;
207 Node *init_trip() const;
208 Node *stride() const;
209 int stride_con() const;
210 bool stride_is_con() const;
211 Node *limit() const;
212 Node *incr() const;
213 Node *phi() const;
214
215 // Match increment with optional truncation
216 static Node* match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInt** trunc_type);
217
218 // A 'main' loop has a pre-loop and a post-loop. The 'main' loop
219 // can run short a few iterations and may start a few iterations in.
220 // It will be RCE'd and unrolled and aligned.
221
277 int node_count_before_unroll() { return _node_count_before_unroll; }
278 void set_slp_max_unroll(int unroll_factor) { _slp_maximum_unroll_factor = unroll_factor; }
279 int slp_max_unroll() const { return _slp_maximum_unroll_factor; }
280
281 #ifndef PRODUCT
282 virtual void dump_spec(outputStream *st) const;
283 #endif
284 };
285
286 //------------------------------CountedLoopEndNode-----------------------------
287 // CountedLoopEndNodes end simple trip counted loops. They act much like
288 // IfNodes.
289 class CountedLoopEndNode : public IfNode {
290 public:
291 enum { TestControl, TestValue };
292
293 CountedLoopEndNode( Node *control, Node *test, float prob, float cnt )
294 : IfNode( control, test, prob, cnt) {
295 init_class_id(Class_CountedLoopEnd);
296 }
297 virtual int Opcode() const;
298
299 Node *cmp_node() const { return (in(TestValue)->req() >=2) ? in(TestValue)->in(1) : NULL; }
300 Node *incr() const { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }
301 Node *limit() const { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }
302 Node *stride() const { Node *tmp = incr (); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }
303 Node *init_trip() const { Node *tmp = phi (); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }
304 int stride_con() const;
305 bool stride_is_con() const { Node *tmp = stride (); return (tmp != NULL && tmp->is_Con()); }
306 BoolTest::mask test_trip() const { return in(TestValue)->as_Bool()->_test._test; }
307 PhiNode *phi() const {
308 Node *tmp = incr();
309 if (tmp && tmp->req() == 3) {
310 Node* phi = tmp->in(1);
311 if (phi->is_Phi()) {
312 return phi->as_Phi();
313 }
314 }
315 return NULL;
316 }
317 CountedLoopNode *loopnode() const {
349 inline bool CountedLoopNode::stride_is_con() const { return loopexit() && loopexit()->stride_is_con(); }
350 inline Node *CountedLoopNode::limit() const { return loopexit() ? loopexit()->limit() : NULL; }
351 inline Node *CountedLoopNode::incr() const { return loopexit() ? loopexit()->incr() : NULL; }
352 inline Node *CountedLoopNode::phi() const { return loopexit() ? loopexit()->phi() : NULL; }
353
354 //------------------------------LoopLimitNode-----------------------------
355 // Counted Loop limit node which represents exact final iterator value:
356 // trip_count = (limit - init_trip + stride - 1)/stride
357 // final_value= trip_count * stride + init_trip.
358 // Use HW instructions to calculate it when it can overflow in integer.
359 // Note, final_value should fit into integer since counted loop has
360 // limit check: limit <= max_int-stride.
361 class LoopLimitNode : public Node {
362 enum { Init=1, Limit=2, Stride=3 };
363 public:
364 LoopLimitNode( Compile* C, Node *init, Node *limit, Node *stride ) : Node(0,init,limit,stride) {
365 // Put it on the Macro nodes list to optimize during macro nodes expansion.
366 init_flags(Flag_is_macro);
367 C->add_macro_node(this);
368 }
369 virtual int Opcode() const;
370 virtual const Type *bottom_type() const { return TypeInt::INT; }
371 virtual uint ideal_reg() const { return Op_RegI; }
372 virtual const Type* Value(PhaseGVN* phase) const;
373 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
374 virtual Node* Identity(PhaseGVN* phase);
375 };
376
377 // -----------------------------IdealLoopTree----------------------------------
378 class IdealLoopTree : public ResourceObj {
379 public:
380 IdealLoopTree *_parent; // Parent in loop tree
381 IdealLoopTree *_next; // Next sibling in loop tree
382 IdealLoopTree *_child; // First child in loop tree
383
384 // The head-tail backedge defines the loop.
385 // If tail is NULL then this loop has multiple backedges as part of the
386 // same loop. During cleanup I'll peel off the multiple backedges; merge
387 // them at the loop bottom and flow 1 real backedge into the loop.
388 Node *_head; // Head of loop
389 Node *_tail; // Tail of loop
|
104 int unswitch_max() { return _unswitch_max; }
105 int unswitch_count() { return _unswitch_count; }
106
107 int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; }
108 void set_has_been_range_checked() { _postloop_flags |= LoopRCEChecked; }
109 int is_rce_post_loop() const { return _postloop_flags & RCEPostLoop; }
110 void set_is_rce_post_loop() { _postloop_flags |= RCEPostLoop; }
111
112 void set_unswitch_count(int val) {
113 assert (val <= unswitch_max(), "too many unswitches");
114 _unswitch_count = val;
115 }
116
117 LoopNode(Node *entry, Node *backedge) : RegionNode(3), _loop_flags(0), _unswitch_count(0), _postloop_flags(0) {
118 init_class_id(Class_Loop);
119 init_req(EntryControl, entry);
120 init_req(LoopBackControl, backedge);
121 }
122
123 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
124 virtual uint Opcode() const;
125 bool can_be_counted_loop(PhaseTransform* phase) const {
126 return req() == 3 && in(0) != NULL &&
127 in(1) != NULL && phase->type(in(1)) != Type::TOP &&
128 in(2) != NULL && phase->type(in(2)) != Type::TOP;
129 }
130 bool is_valid_counted_loop() const;
131 #ifndef PRODUCT
132 virtual void dump_spec(outputStream *st) const;
133 #endif
134 };
135
136 //------------------------------Counted Loops----------------------------------
137 // Counted loops are all trip-counted loops, with exactly 1 trip-counter exit
138 // path (and maybe some other exit paths). The trip-counter exit is always
139 // last in the loop. The trip-counter have to stride by a constant;
140 // the exit value is also loop invariant.
141
142 // CountedLoopNodes and CountedLoopEndNodes come in matched pairs. The
143 // CountedLoopNode has the incoming loop control and the loop-back-control
144 // which is always the IfTrue before the matching CountedLoopEndNode. The
181 int _unrolled_count_log2;
182
183 // Node count prior to last unrolling - used to decide if
184 // unroll,optimize,unroll,optimize,... is making progress
185 int _node_count_before_unroll;
186
187 // If slp analysis is performed we record the maximum
188 // vector mapped unroll factor here
189 int _slp_maximum_unroll_factor;
190
191 public:
192 CountedLoopNode( Node *entry, Node *backedge )
193 : LoopNode(entry, backedge), _main_idx(0), _trip_count(max_juint),
194 _profile_trip_cnt(COUNT_UNKNOWN), _unrolled_count_log2(0),
195 _node_count_before_unroll(0), _slp_maximum_unroll_factor(0) {
196 init_class_id(Class_CountedLoop);
197 // Initialize _trip_count to the largest possible value.
198 // Will be reset (lower) if the loop's trip count is known.
199 }
200
201 virtual uint Opcode() const;
202 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
203
204 Node *init_control() const { return in(EntryControl); }
205 Node *back_control() const { return in(LoopBackControl); }
206 CountedLoopEndNode *loopexit() const;
207 Node *init_trip() const;
208 Node *stride() const;
209 int stride_con() const;
210 bool stride_is_con() const;
211 Node *limit() const;
212 Node *incr() const;
213 Node *phi() const;
214
215 // Match increment with optional truncation
216 static Node* match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInt** trunc_type);
217
218 // A 'main' loop has a pre-loop and a post-loop. The 'main' loop
219 // can run short a few iterations and may start a few iterations in.
220 // It will be RCE'd and unrolled and aligned.
221
277 int node_count_before_unroll() { return _node_count_before_unroll; }
278 void set_slp_max_unroll(int unroll_factor) { _slp_maximum_unroll_factor = unroll_factor; }
279 int slp_max_unroll() const { return _slp_maximum_unroll_factor; }
280
281 #ifndef PRODUCT
282 virtual void dump_spec(outputStream *st) const;
283 #endif
284 };
285
286 //------------------------------CountedLoopEndNode-----------------------------
287 // CountedLoopEndNodes end simple trip counted loops. They act much like
288 // IfNodes.
289 class CountedLoopEndNode : public IfNode {
290 public:
291 enum { TestControl, TestValue };
292
293 CountedLoopEndNode( Node *control, Node *test, float prob, float cnt )
294 : IfNode( control, test, prob, cnt) {
295 init_class_id(Class_CountedLoopEnd);
296 }
297 virtual uint Opcode() const;
298
299 Node *cmp_node() const { return (in(TestValue)->req() >=2) ? in(TestValue)->in(1) : NULL; }
300 Node *incr() const { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }
301 Node *limit() const { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }
302 Node *stride() const { Node *tmp = incr (); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }
303 Node *init_trip() const { Node *tmp = phi (); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }
304 int stride_con() const;
305 bool stride_is_con() const { Node *tmp = stride (); return (tmp != NULL && tmp->is_Con()); }
306 BoolTest::mask test_trip() const { return in(TestValue)->as_Bool()->_test._test; }
307 PhiNode *phi() const {
308 Node *tmp = incr();
309 if (tmp && tmp->req() == 3) {
310 Node* phi = tmp->in(1);
311 if (phi->is_Phi()) {
312 return phi->as_Phi();
313 }
314 }
315 return NULL;
316 }
317 CountedLoopNode *loopnode() const {
349 inline bool CountedLoopNode::stride_is_con() const { return loopexit() && loopexit()->stride_is_con(); }
350 inline Node *CountedLoopNode::limit() const { return loopexit() ? loopexit()->limit() : NULL; }
351 inline Node *CountedLoopNode::incr() const { return loopexit() ? loopexit()->incr() : NULL; }
352 inline Node *CountedLoopNode::phi() const { return loopexit() ? loopexit()->phi() : NULL; }
353
354 //------------------------------LoopLimitNode-----------------------------
355 // Counted Loop limit node which represents exact final iterator value:
356 // trip_count = (limit - init_trip + stride - 1)/stride
357 // final_value= trip_count * stride + init_trip.
358 // Use HW instructions to calculate it when it can overflow in integer.
359 // Note, final_value should fit into integer since counted loop has
360 // limit check: limit <= max_int-stride.
361 class LoopLimitNode : public Node {
362 enum { Init=1, Limit=2, Stride=3 };
363 public:
364 LoopLimitNode( Compile* C, Node *init, Node *limit, Node *stride ) : Node(0,init,limit,stride) {
365 // Put it on the Macro nodes list to optimize during macro nodes expansion.
366 init_flags(Flag_is_macro);
367 C->add_macro_node(this);
368 }
369 virtual uint Opcode() const;
370 virtual const Type *bottom_type() const { return TypeInt::INT; }
371 virtual uint ideal_reg() const { return Op_RegI; }
372 virtual const Type* Value(PhaseGVN* phase) const;
373 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
374 virtual Node* Identity(PhaseGVN* phase);
375 };
376
377 // -----------------------------IdealLoopTree----------------------------------
378 class IdealLoopTree : public ResourceObj {
379 public:
380 IdealLoopTree *_parent; // Parent in loop tree
381 IdealLoopTree *_next; // Next sibling in loop tree
382 IdealLoopTree *_child; // First child in loop tree
383
384 // The head-tail backedge defines the loop.
385 // If tail is NULL then this loop has multiple backedges as part of the
386 // same loop. During cleanup I'll peel off the multiple backedges; merge
387 // them at the loop bottom and flow 1 real backedge into the loop.
388 Node *_head; // Head of loop
389 Node *_tail; // Tail of loop
|