< prev index next >
src/hotspot/share/opto/loopnode.hpp
Print this page
*** 36,45 ****
--- 36,46 ----
class CountedLoopNode;
class IdealLoopTree;
class LoopNode;
class Node;
class OuterStripMinedLoopEndNode;
+ class PathFrequency;
class PhaseIdealLoop;
class CountedLoopReserveKit;
class VectorSet;
class Invariance;
struct small_cache;
*** 55,65 ****
class LoopNode : public RegionNode {
// Size is bigger to hold the flags. However, the flags do not change
// the semantics so it does not appear in the hash & cmp functions.
virtual uint size_of() const { return sizeof(*this); }
protected:
! short _loop_flags;
// Names for flag bitfields
enum { Normal=0, Pre=1, Main=2, Post=3, PreMainPostFlagsMask=3,
MainHasNoPreLoop=4,
HasExactTripCount=8,
InnerLoop=16,
--- 56,66 ----
class LoopNode : public RegionNode {
// Size is bigger to hold the flags. However, the flags do not change
// the semantics so it does not appear in the hash & cmp functions.
virtual uint size_of() const { return sizeof(*this); }
protected:
! uint _loop_flags;
// Names for flag bitfields
enum { Normal=0, Pre=1, Main=2, Post=3, PreMainPostFlagsMask=3,
MainHasNoPreLoop=4,
HasExactTripCount=8,
InnerLoop=16,
*** 71,100 ****
DoUnrollOnly=1024,
VectorizedLoop=2048,
HasAtomicPostLoop=4096,
HasRangeChecks=8192,
IsMultiversioned=16384,
! StripMined=32768};
char _unswitch_count;
enum { _unswitch_max=3 };
char _postloop_flags;
enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 };
public:
// Names for edge indices
enum { Self=0, EntryControl, LoopBackControl };
! int is_inner_loop() const { return _loop_flags & InnerLoop; }
void set_inner_loop() { _loop_flags |= InnerLoop; }
! int range_checks_present() const { return _loop_flags & HasRangeChecks; }
! int is_multiversioned() const { return _loop_flags & IsMultiversioned; }
! int is_vectorized_loop() const { return _loop_flags & VectorizedLoop; }
! int is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }
void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }
! int partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
! int is_strip_mined() const { return _loop_flags & StripMined; }
void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }
void mark_has_reductions() { _loop_flags |= HasReductions; }
void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; }
void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; }
--- 72,106 ----
DoUnrollOnly=1024,
VectorizedLoop=2048,
HasAtomicPostLoop=4096,
HasRangeChecks=8192,
IsMultiversioned=16384,
! StripMined=32768,
! ProfileTripFailed=65536};
char _unswitch_count;
enum { _unswitch_max=3 };
char _postloop_flags;
enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 };
+ // Expected trip count from profile data
+ float _profile_trip_cnt;
+
public:
// Names for edge indices
enum { Self=0, EntryControl, LoopBackControl };
! uint is_inner_loop() const { return _loop_flags & InnerLoop; }
void set_inner_loop() { _loop_flags |= InnerLoop; }
! uint range_checks_present() const { return _loop_flags & HasRangeChecks; }
! uint is_multiversioned() const { return _loop_flags & IsMultiversioned; }
! uint is_vectorized_loop() const { return _loop_flags & VectorizedLoop; }
! uint is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }
void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }
! uint partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
! uint is_strip_mined() const { return _loop_flags & StripMined; }
! uint is_profile_trip_failed() const { return _loop_flags & ProfileTripFailed; }
void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }
void mark_has_reductions() { _loop_flags |= HasReductions; }
void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; }
void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; }
*** 103,112 ****
--- 109,119 ----
void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; }
void mark_has_range_checks() { _loop_flags |= HasRangeChecks; }
void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; }
void mark_strip_mined() { _loop_flags |= StripMined; }
void clear_strip_mined() { _loop_flags &= ~StripMined; }
+ void mark_profile_trip_failed() { _loop_flags |= ProfileTripFailed; }
int unswitch_max() { return _unswitch_max; }
int unswitch_count() { return _unswitch_count; }
int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; }
*** 117,127 ****
void set_unswitch_count(int val) {
assert (val <= unswitch_max(), "too many unswitches");
_unswitch_count = val;
}
! LoopNode(Node *entry, Node *backedge) : RegionNode(3), _loop_flags(0), _unswitch_count(0), _postloop_flags(0) {
init_class_id(Class_Loop);
init_req(EntryControl, entry);
init_req(LoopBackControl, backedge);
}
--- 124,139 ----
void set_unswitch_count(int val) {
assert (val <= unswitch_max(), "too many unswitches");
_unswitch_count = val;
}
! void set_profile_trip_cnt(float ptc) { _profile_trip_cnt = ptc; }
! float profile_trip_cnt() { return _profile_trip_cnt; }
!
! LoopNode(Node *entry, Node *backedge)
! : RegionNode(3), _loop_flags(0), _unswitch_count(0),
! _postloop_flags(0), _profile_trip_cnt(COUNT_UNKNOWN) {
init_class_id(Class_Loop);
init_req(EntryControl, entry);
init_req(LoopBackControl, backedge);
}
*** 184,196 ****
node_idx_t _main_idx;
// Known trip count calculated by compute_exact_trip_count()
uint _trip_count;
- // Expected trip count from profile data
- float _profile_trip_cnt;
-
// Log2 of original loop bodies in unrolled loop
int _unrolled_count_log2;
// Node count prior to last unrolling - used to decide if
// unroll,optimize,unroll,optimize,... is making progress
--- 196,205 ----
*** 201,212 ****
int _slp_maximum_unroll_factor;
public:
CountedLoopNode( Node *entry, Node *backedge )
: LoopNode(entry, backedge), _main_idx(0), _trip_count(max_juint),
! _profile_trip_cnt(COUNT_UNKNOWN), _unrolled_count_log2(0),
! _node_count_before_unroll(0), _slp_maximum_unroll_factor(0) {
init_class_id(Class_CountedLoop);
// Initialize _trip_count to the largest possible value.
// Will be reset (lower) if the loop's trip count is known.
}
--- 210,221 ----
int _slp_maximum_unroll_factor;
public:
CountedLoopNode( Node *entry, Node *backedge )
: LoopNode(entry, backedge), _main_idx(0), _trip_count(max_juint),
! _unrolled_count_log2(0), _node_count_before_unroll(0),
! _slp_maximum_unroll_factor(0) {
init_class_id(Class_CountedLoop);
// Initialize _trip_count to the largest possible value.
// Will be reset (lower) if the loop's trip count is known.
}
*** 243,262 ****
// so the following main loop 'knows' that it is striding down cache
// lines.
// A 'main' loop that is ONLY unrolled or peeled, never RCE'd or
// Aligned, may be missing it's pre-loop.
! int is_normal_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }
! int is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; }
! int is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; }
! int is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; }
! int is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; }
! int was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; }
! int has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; }
! int do_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; }
! int is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }
! int has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; }
void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; }
int main_idx() const { return _main_idx; }
--- 252,271 ----
// so the following main loop 'knows' that it is striding down cache
// lines.
// A 'main' loop that is ONLY unrolled or peeled, never RCE'd or
// Aligned, may be missing it's pre-loop.
! uint is_normal_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }
! uint is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; }
! uint is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; }
! uint is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; }
! uint is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; }
! uint was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; }
! uint has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; }
! uint do_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; }
! uint is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }
! uint has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; }
void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; }
int main_idx() const { return _main_idx; }
*** 278,290 ****
}
void set_notpassed_slp() {
_loop_flags &= ~PassedSlpAnalysis;
}
- void set_profile_trip_cnt(float ptc) { _profile_trip_cnt = ptc; }
- float profile_trip_cnt() { return _profile_trip_cnt; }
-
void double_unrolled_count() { _unrolled_count_log2++; }
int unrolled_count() { return 1 << MIN2(_unrolled_count_log2, BitsPerInt-3); }
void set_node_count_before_unroll(int ct) { _node_count_before_unroll = ct; }
int node_count_before_unroll() { return _node_count_before_unroll; }
--- 287,296 ----
*** 299,308 ****
--- 305,315 ----
virtual SafePointNode* outer_safepoint() const;
// If this is a main loop in a pre/main/post loop nest, walk over
// the predicates that were inserted by
// duplicate_predicates()/add_range_check_predicate()
+ static Node* skip_predicates_from_entry(Node* ctrl);
Node* skip_predicates();
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
*** 586,595 ****
--- 593,603 ----
// Compute loop trip count if possible
void compute_trip_count(PhaseIdealLoop* phase);
// Compute loop trip count from profile data
+ float compute_profile_trip_cnt_helper(Node* n);
void compute_profile_trip_cnt( PhaseIdealLoop *phase );
// Reassociate invariant expressions.
void reassociate_invariants(PhaseIdealLoop *phase);
// Reassociate invariant add and subtract expressions.
*** 728,740 ****
}
return ctrl;
}
Node* cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop);
! void duplicate_predicates(CountedLoopNode* pre_head, Node *min_taken, Node* castii,
! IdealLoopTree* outer_loop, LoopNode* outer_main_head,
! uint dd_main_head);
public:
static bool is_canonical_loop_entry(CountedLoopNode* cl);
--- 736,749 ----
}
return ctrl;
}
Node* cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop);
! void duplicate_predicates_helper(Node* predicate, Node* castii, IdealLoopTree* outer_loop,
! LoopNode* outer_main_head, uint dd_main_head);
! void duplicate_predicates(CountedLoopNode* pre_head, Node* castii, IdealLoopTree* outer_loop,
! LoopNode* outer_main_head, uint dd_main_head);
public:
static bool is_canonical_loop_entry(CountedLoopNode* cl);
*** 1059,1068 ****
--- 1068,1078 ----
bool clone_limit_check,
PhaseIdealLoop* loop_phase,
PhaseIterGVN* igvn);
Node* clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check);
+ static Node* skip_all_loop_predicates(Node* entry);
static Node* skip_loop_predicates(Node* entry);
// Find a good location to insert a predicate
static ProjNode* find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason);
// Find a predicate
*** 1073,1088 ****
Node* init, Node* limit, jint stride,
Node* range, bool upper, bool &overflow);
// Implementation of the loop predication to promote checks outside the loop
bool loop_predication_impl(IdealLoopTree *loop);
ProjNode* insert_skeleton_predicate(IfNode* iff, IdealLoopTree *loop,
ProjNode* proj, ProjNode *predicate_proj,
ProjNode* upper_bound_proj,
int scale, Node* offset,
Node* init, Node* limit, jint stride,
! Node* rng, bool& overflow);
Node* add_range_check_predicate(IdealLoopTree* loop, CountedLoopNode* cl,
Node* predicate_proj, int scale_con, Node* offset,
Node* limit, jint stride_con);
// Helper function to collect predicate for eliminating the useless ones
--- 1083,1106 ----
Node* init, Node* limit, jint stride,
Node* range, bool upper, bool &overflow);
// Implementation of the loop predication to promote checks outside the loop
bool loop_predication_impl(IdealLoopTree *loop);
+ bool loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* proj, ProjNode *predicate_proj,
+ CountedLoopNode *cl, ConNode* zero, Invariance& invar,
+ Deoptimization::DeoptReason reason);
+ bool loop_predication_should_follow_branches(IdealLoopTree *loop, ProjNode *predicate_proj, float& loop_trip_cnt);
+ void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt,
+ PathFrequency& pf, Node_Stack& stack, VectorSet& seen,
+ Node_List& if_proj_list);
ProjNode* insert_skeleton_predicate(IfNode* iff, IdealLoopTree *loop,
ProjNode* proj, ProjNode *predicate_proj,
ProjNode* upper_bound_proj,
int scale, Node* offset,
Node* init, Node* limit, jint stride,
! Node* rng, bool& overflow,
! Deoptimization::DeoptReason reason);
Node* add_range_check_predicate(IdealLoopTree* loop, CountedLoopNode* cl,
Node* predicate_proj, int scale_con, Node* offset,
Node* limit, jint stride_con);
// Helper function to collect predicate for eliminating the useless ones
< prev index next >