59 virtual uint size_of() const { return sizeof(*this); }
60 protected:
61 uint _loop_flags;
62 // Names for flag bitfields
63 enum { Normal=0, Pre=1, Main=2, Post=3, PreMainPostFlagsMask=3,
64 MainHasNoPreLoop=4,
65 HasExactTripCount=8,
66 InnerLoop=16,
67 PartialPeelLoop=32,
68 PartialPeelFailed=64,
69 HasReductions=128,
70 WasSlpAnalyzed=256,
71 PassedSlpAnalysis=512,
72 DoUnrollOnly=1024,
73 VectorizedLoop=2048,
74 HasAtomicPostLoop=4096,
75 HasRangeChecks=8192,
76 IsMultiversioned=16384,
77 StripMined=32768,
78 SubwordLoop=65536,
79 ProfileTripFailed=131072};
80 char _unswitch_count;
81 enum { _unswitch_max=3 };
82 char _postloop_flags;
83 enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 };
84
85 // Expected trip count from profile data
86 float _profile_trip_cnt;
87
88 public:
89 // Names for edge indices
90 enum { Self=0, EntryControl, LoopBackControl };
91
92 bool is_inner_loop() const { return _loop_flags & InnerLoop; }
93 void set_inner_loop() { _loop_flags |= InnerLoop; }
94
95 bool range_checks_present() const { return _loop_flags & HasRangeChecks; }
96 bool is_multiversioned() const { return _loop_flags & IsMultiversioned; }
97 bool is_vectorized_loop() const { return _loop_flags & VectorizedLoop; }
98 bool is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }
99 void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }
100 bool partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
101 bool is_strip_mined() const { return _loop_flags & StripMined; }
102 bool is_profile_trip_failed() const { return _loop_flags & ProfileTripFailed; }
103 bool is_subword_loop() const { return _loop_flags & SubwordLoop; }
104
105 void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }
106 void mark_has_reductions() { _loop_flags |= HasReductions; }
107 void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; }
108 void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; }
109 void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; }
110 void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; }
111 void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; }
112 void mark_has_range_checks() { _loop_flags |= HasRangeChecks; }
113 void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; }
114 void mark_strip_mined() { _loop_flags |= StripMined; }
115 void clear_strip_mined() { _loop_flags &= ~StripMined; }
116 void mark_profile_trip_failed() { _loop_flags |= ProfileTripFailed; }
117 void mark_subword_loop() { _loop_flags |= SubwordLoop; }
118
119 int unswitch_max() { return _unswitch_max; }
120 int unswitch_count() { return _unswitch_count; }
121
122 int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; }
123 void set_has_been_range_checked() { _postloop_flags |= LoopRCEChecked; }
124 int is_rce_post_loop() const { return _postloop_flags & RCEPostLoop; }
125 void set_is_rce_post_loop() { _postloop_flags |= RCEPostLoop; }
126
127 void set_unswitch_count(int val) {
128 assert (val <= unswitch_max(), "too many unswitches");
129 _unswitch_count = val;
130 }
131
132 void set_profile_trip_cnt(float ptc) { _profile_trip_cnt = ptc; }
133 float profile_trip_cnt() { return _profile_trip_cnt; }
134
135 LoopNode(Node *entry, Node *backedge)
136 : RegionNode(3), _loop_flags(0), _unswitch_count(0),
137 _postloop_flags(0), _profile_trip_cnt(COUNT_UNKNOWN) {
1202 Node_List &old_new,
1203 int opcode,
1204 CloneLoopMode mode);
1205
1206 // Clone a loop and return the clone head (clone_loop_head).
1207 // Added nodes include int(1), int(0) - disconnected, If, IfTrue, IfFalse,
1208 // This routine was created for usage in CountedLoopReserveKit.
1209 //
1210 // int(1) -> If -> IfTrue -> original_loop_head
1211 // |
1212 // V
1213 // IfFalse -> clone_loop_head (returned by function pointer)
1214 //
1215 LoopNode* create_reserve_version_of_loop(IdealLoopTree *loop, CountedLoopReserveKit* lk);
1216 // Clone loop with an invariant test (that does not exit) and
1217 // insert a clone of the test that selects which version to
1218 // execute.
1219 void do_unswitching (IdealLoopTree *loop, Node_List &old_new);
1220
1221 // Find candidate "if" for unswitching
1222 IfNode* find_unswitching_candidate(const IdealLoopTree *loop) const;
1223
1224 // Range Check Elimination uses this function!
1225 // Constrain the main loop iterations so the affine function:
1226 // low_limit <= scale_con * I + offset < upper_limit
1227 // always holds true. That is, either increase the number of iterations in
1228 // the pre-loop or the post-loop until the condition holds true in the main
1229 // loop. Scale_con, offset and limit are all loop invariant.
1230 void add_constraint( int stride_con, int scale_con, Node *offset, Node *low_limit, Node *upper_limit, Node *pre_ctrl, Node **pre_limit, Node **main_limit );
1231 // Helper function for add_constraint().
1232 Node* adjust_limit(int stride_con, Node * scale, Node *offset, Node *rc_limit, Node *loop_limit, Node *pre_ctrl, bool round_up);
1233
1234 // Partially peel loop up through last_peel node.
1235 bool partial_peel( IdealLoopTree *loop, Node_List &old_new );
1236
1237 // Create a scheduled list of nodes control dependent on ctrl set.
1238 void scheduled_nodelist( IdealLoopTree *loop, VectorSet& ctrl, Node_List &sched );
1239 // Has a use in the vector set
1240 bool has_use_in_set( Node* n, VectorSet& vset );
1241 // Has use internal to the vector set (ie. not in a phi at the loop head)
1242 bool has_use_internal_to_set( Node* n, VectorSet& vset, IdealLoopTree *loop );
|
59 virtual uint size_of() const { return sizeof(*this); }
60 protected:
61 uint _loop_flags;
62 // Names for flag bitfields
63 enum { Normal=0, Pre=1, Main=2, Post=3, PreMainPostFlagsMask=3,
64 MainHasNoPreLoop=4,
65 HasExactTripCount=8,
66 InnerLoop=16,
67 PartialPeelLoop=32,
68 PartialPeelFailed=64,
69 HasReductions=128,
70 WasSlpAnalyzed=256,
71 PassedSlpAnalysis=512,
72 DoUnrollOnly=1024,
73 VectorizedLoop=2048,
74 HasAtomicPostLoop=4096,
75 HasRangeChecks=8192,
76 IsMultiversioned=16384,
77 StripMined=32768,
78 SubwordLoop=65536,
79 ProfileTripFailed=131072,
80 FlattenedArrays=262144};
81 char _unswitch_count;
82 enum { _unswitch_max=3 };
83 char _postloop_flags;
84 enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 };
85
86 // Expected trip count from profile data
87 float _profile_trip_cnt;
88
89 public:
90 // Names for edge indices
91 enum { Self=0, EntryControl, LoopBackControl };
92
93 bool is_inner_loop() const { return _loop_flags & InnerLoop; }
94 void set_inner_loop() { _loop_flags |= InnerLoop; }
95
96 bool range_checks_present() const { return _loop_flags & HasRangeChecks; }
97 bool is_multiversioned() const { return _loop_flags & IsMultiversioned; }
98 bool is_vectorized_loop() const { return _loop_flags & VectorizedLoop; }
99 bool is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }
100 void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }
101 bool partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
102 bool is_strip_mined() const { return _loop_flags & StripMined; }
103 bool is_profile_trip_failed() const { return _loop_flags & ProfileTripFailed; }
104 bool is_subword_loop() const { return _loop_flags & SubwordLoop; }
105 bool is_flattened_arrays() const { return _loop_flags & FlattenedArrays; }
106
107 void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }
108 void mark_has_reductions() { _loop_flags |= HasReductions; }
109 void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; }
110 void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; }
111 void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; }
112 void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; }
113 void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; }
114 void mark_has_range_checks() { _loop_flags |= HasRangeChecks; }
115 void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; }
116 void mark_strip_mined() { _loop_flags |= StripMined; }
117 void clear_strip_mined() { _loop_flags &= ~StripMined; }
118 void mark_profile_trip_failed() { _loop_flags |= ProfileTripFailed; }
119 void mark_subword_loop() { _loop_flags |= SubwordLoop; }
120 void mark_flattened_arrays() { _loop_flags |= FlattenedArrays; }
121
122 int unswitch_max() { return _unswitch_max; }
123 int unswitch_count() { return _unswitch_count; }
124
125 int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; }
126 void set_has_been_range_checked() { _postloop_flags |= LoopRCEChecked; }
127 int is_rce_post_loop() const { return _postloop_flags & RCEPostLoop; }
128 void set_is_rce_post_loop() { _postloop_flags |= RCEPostLoop; }
129
130 void set_unswitch_count(int val) {
131 assert (val <= unswitch_max(), "too many unswitches");
132 _unswitch_count = val;
133 }
134
135 void set_profile_trip_cnt(float ptc) { _profile_trip_cnt = ptc; }
136 float profile_trip_cnt() { return _profile_trip_cnt; }
137
138 LoopNode(Node *entry, Node *backedge)
139 : RegionNode(3), _loop_flags(0), _unswitch_count(0),
140 _postloop_flags(0), _profile_trip_cnt(COUNT_UNKNOWN) {
1205 Node_List &old_new,
1206 int opcode,
1207 CloneLoopMode mode);
1208
1209 // Clone a loop and return the clone head (clone_loop_head).
1210 // Added nodes include int(1), int(0) - disconnected, If, IfTrue, IfFalse,
1211 // This routine was created for usage in CountedLoopReserveKit.
1212 //
1213 // int(1) -> If -> IfTrue -> original_loop_head
1214 // |
1215 // V
1216 // IfFalse -> clone_loop_head (returned by function pointer)
1217 //
1218 LoopNode* create_reserve_version_of_loop(IdealLoopTree *loop, CountedLoopReserveKit* lk);
1219 // Clone loop with an invariant test (that does not exit) and
1220 // insert a clone of the test that selects which version to
1221 // execute.
1222 void do_unswitching (IdealLoopTree *loop, Node_List &old_new);
1223
1224 // Find candidate "if" for unswitching
1225 IfNode* find_unswitching_candidate(const IdealLoopTree *loop, Node_List& flattened_checks) const;
1226
1227 // Range Check Elimination uses this function!
1228 // Constrain the main loop iterations so the affine function:
1229 // low_limit <= scale_con * I + offset < upper_limit
1230 // always holds true. That is, either increase the number of iterations in
1231 // the pre-loop or the post-loop until the condition holds true in the main
1232 // loop. Scale_con, offset and limit are all loop invariant.
1233 void add_constraint( int stride_con, int scale_con, Node *offset, Node *low_limit, Node *upper_limit, Node *pre_ctrl, Node **pre_limit, Node **main_limit );
1234 // Helper function for add_constraint().
1235 Node* adjust_limit(int stride_con, Node * scale, Node *offset, Node *rc_limit, Node *loop_limit, Node *pre_ctrl, bool round_up);
1236
1237 // Partially peel loop up through last_peel node.
1238 bool partial_peel( IdealLoopTree *loop, Node_List &old_new );
1239
1240 // Create a scheduled list of nodes control dependent on ctrl set.
1241 void scheduled_nodelist( IdealLoopTree *loop, VectorSet& ctrl, Node_List &sched );
1242 // Has a use in the vector set
1243 bool has_use_in_set( Node* n, VectorSet& vset );
1244 // Has use internal to the vector set (ie. not in a phi at the loop head)
1245 bool has_use_internal_to_set( Node* n, VectorSet& vset, IdealLoopTree *loop );
|