< prev index next >
src/hotspot/share/opto/loopnode.hpp
Print this page
*** 35,44 ****
--- 35,45 ----
class CountedLoopEndNode;
class CountedLoopNode;
class IdealLoopTree;
class LoopNode;
class Node;
+ class Opaque5Node;
class PhaseIdealLoop;
class CountedLoopReserveKit;
class VectorSet;
class Invariance;
struct small_cache;
*** 69,107 ****
PassedSlpAnalysis=512,
DoUnrollOnly=1024,
VectorizedLoop=2048,
HasAtomicPostLoop=4096,
HasRangeChecks=8192,
! IsMultiversioned=16384};
char _unswitch_count;
enum { _unswitch_max=3 };
char _postloop_flags;
enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 };
public:
// Names for edge indices
enum { Self=0, EntryControl, LoopBackControl };
! int is_inner_loop() const { return _loop_flags & InnerLoop; }
void set_inner_loop() { _loop_flags |= InnerLoop; }
! int range_checks_present() const { return _loop_flags & HasRangeChecks; }
! int is_multiversioned() const { return _loop_flags & IsMultiversioned; }
! int is_vectorized_loop() const { return _loop_flags & VectorizedLoop; }
! int is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }
void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }
! int partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }
void mark_has_reductions() { _loop_flags |= HasReductions; }
void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; }
void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; }
void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; }
void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; }
void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; }
void mark_has_range_checks() { _loop_flags |= HasRangeChecks; }
void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; }
int unswitch_max() { return _unswitch_max; }
int unswitch_count() { return _unswitch_count; }
int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; }
--- 70,112 ----
PassedSlpAnalysis=512,
DoUnrollOnly=1024,
VectorizedLoop=2048,
HasAtomicPostLoop=4096,
HasRangeChecks=8192,
! IsMultiversioned=16384,
! StripMined=32768};
char _unswitch_count;
enum { _unswitch_max=3 };
char _postloop_flags;
enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 };
public:
// Names for edge indices
enum { Self=0, EntryControl, LoopBackControl };
! uint is_inner_loop() const { return _loop_flags & InnerLoop; }
void set_inner_loop() { _loop_flags |= InnerLoop; }
! uint range_checks_present() const { return _loop_flags & HasRangeChecks; }
! uint is_multiversioned() const { return _loop_flags & IsMultiversioned; }
! uint is_vectorized_loop() const { return _loop_flags & VectorizedLoop; }
! uint is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }
void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }
! uint partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
! uint is_strip_mined() const { return _loop_flags & StripMined; }
void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }
void mark_has_reductions() { _loop_flags |= HasReductions; }
void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; }
void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; }
void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; }
void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; }
void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; }
void mark_has_range_checks() { _loop_flags |= HasRangeChecks; }
void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; }
+ void mark_strip_mined() { _loop_flags |= StripMined; }
+ void clear_strip_mined() { _loop_flags &= ~StripMined; }
int unswitch_max() { return _unswitch_max; }
int unswitch_count() { return _unswitch_count; }
int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; }
*** 129,138 ****
--- 134,153 ----
}
bool is_valid_counted_loop() const;
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
+
+ void verify_strip_mined(int expect_opaq) const;
+ virtual LoopNode* skip_strip_mined(int expect_opaq = 1) { return this; }
+ virtual IfTrueNode* outer_loop_tail() const;
+ virtual IfNode* outer_loop_end() const;
+ virtual IfFalseNode* outer_loop_exit() const;
+ virtual SafePointNode* outer_safepoint() const;
+ virtual BoolNode* outer_bol() const;
+ virtual CmpINode* outer_cmp() const;
+ virtual Opaque5Node* outer_opaq() const;
};
//------------------------------Counted Loops----------------------------------
// Counted loops are all trip-counted loops, with exactly 1 trip-counter exit
// path (and maybe some other exit paths). The trip-counter exit is always
*** 230,249 ****
// so the following main loop 'knows' that it is striding down cache
// lines.
// A 'main' loop that is ONLY unrolled or peeled, never RCE'd or
// Aligned, may be missing it's pre-loop.
! int is_normal_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }
! int is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; }
! int is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; }
! int is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; }
! int is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; }
! int was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; }
! int has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; }
! int do_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; }
! int is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }
! int has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; }
void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; }
int main_idx() const { return _main_idx; }
--- 245,264 ----
// so the following main loop 'knows' that it is striding down cache
// lines.
// A 'main' loop that is ONLY unrolled or peeled, never RCE'd or
// Aligned, may be missing it's pre-loop.
! uint is_normal_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }
! uint is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; }
! uint is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; }
! uint is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; }
! uint is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; }
! uint was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; }
! uint has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; }
! uint do_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; }
! uint is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }
! uint has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; }
void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; }
int main_idx() const { return _main_idx; }
*** 276,285 ****
--- 291,310 ----
void set_node_count_before_unroll(int ct) { _node_count_before_unroll = ct; }
int node_count_before_unroll() { return _node_count_before_unroll; }
void set_slp_max_unroll(int unroll_factor) { _slp_maximum_unroll_factor = unroll_factor; }
int slp_max_unroll() const { return _slp_maximum_unroll_factor; }
+ virtual LoopNode* skip_strip_mined(int expect_opaq = 1);
+ LoopNode* outer_loop() const;
+ virtual IfTrueNode* outer_loop_tail() const;
+ virtual IfNode* outer_loop_end() const;
+ virtual IfFalseNode* outer_loop_exit() const;
+ virtual SafePointNode* outer_safepoint() const;
+ virtual BoolNode* outer_bol() const;
+ virtual CmpINode* outer_cmp() const;
+ virtual Opaque5Node* outer_opaq() const;
+
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
*** 778,787 ****
--- 803,813 ----
// Place Data nodes in some loop nest
void build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack );
void build_loop_late ( VectorSet &visited, Node_List &worklist, Node_Stack &nstack );
void build_loop_late_post ( Node* n );
+ void verify_strip_mined_scheduling(Node *n, Node* least);
// Array of immediate dominance info for each CFG node indexed by node idx
private:
uint _idom_size;
Node **_idom; // Array of immediate dominators
*** 875,885 ****
bool _has_irreducible_loops;
// Per-Node transform
virtual Node *transform( Node *a_node ) { return 0; }
! bool is_counted_loop( Node *x, IdealLoopTree *loop );
Node* exact_limit( IdealLoopTree *loop );
// Return a post-walked LoopNode
IdealLoopTree *get_loop( Node *n ) const {
--- 901,914 ----
bool _has_irreducible_loops;
// Per-Node transform
virtual Node *transform( Node *a_node ) { return 0; }
! bool is_counted_loop(Node* x, IdealLoopTree*& loop);
! IdealLoopTree* create_outer_strip_mined_loop(BoolNode *test, Node *cmp, Node *init_control,
! IdealLoopTree* loop, float cl_prob, float le_fcnt,
! Node*& entry_control, Node*& iffalse);
Node* exact_limit( IdealLoopTree *loop );
// Return a post-walked LoopNode
IdealLoopTree *get_loop( Node *n ) const {
*** 906,917 ****
// the clone loop to dominate the original. Used in construction of
// pre-main-post loop sequence.
// When nonnull, the clone and original are side-by-side, both are
// dominated by the passed in side_by_side_idom node. Used in
// construction of unswitched loops.
void clone_loop( IdealLoopTree *loop, Node_List &old_new, int dom_depth,
! Node* side_by_side_idom = NULL);
// If we got the effect of peeling, either by actually peeling or by
// making a pre-loop which must execute at least once, we can remove
// all loop-invariant dominated tests in the main body.
void peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new );
--- 935,962 ----
// the clone loop to dominate the original. Used in construction of
// pre-main-post loop sequence.
// When nonnull, the clone and original are side-by-side, both are
// dominated by the passed in side_by_side_idom node. Used in
// construction of unswitched loops.
+ enum CloneLoopMode {
+ IgnoreStripMined = 0, // Only clone inner strip mined loop
+ CloneIncludesStripMined = 1, // clone both inner and outer strip mined loops
+ ControlAroundStripMined = 2 // Only clone inner strip mined loop,
+ // result control flow branches
+ // either to inner clone or outer
+ // strip mined loop.
+ };
void clone_loop( IdealLoopTree *loop, Node_List &old_new, int dom_depth,
! CloneLoopMode mode, Node* side_by_side_idom = NULL);
! void clone_loop_handle_data_uses(Node* old, Node_List &old_new,
! IdealLoopTree* loop, IdealLoopTree* companion_loop,
! Node_List*& split_if_set, Node_List*& split_bool_set,
! Node_List*& split_cex_set, Node_List& worklist,
! uint new_counter, CloneLoopMode mode);
! void clone_outer_loop(LoopNode* head, CloneLoopMode mode, IdealLoopTree *loop,
! IdealLoopTree* outer_loop, int dd, Node_List &old_new,
! Node_List& extra_data_nodes);
// If we got the effect of peeling, either by actually peeling or by
// making a pre-loop which must execute at least once, we can remove
// all loop-invariant dominated tests in the main body.
void peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new );
*** 1018,1028 ****
// Create a slow version of the loop by cloning the loop
// and inserting an if to select fast-slow versions.
ProjNode* create_slow_version_of_loop(IdealLoopTree *loop,
Node_List &old_new,
! int opcode);
// Clone a loop and return the clone head (clone_loop_head).
// Added nodes include int(1), int(0) - disconnected, If, IfTrue, IfFalse,
// This routine was created for usage in CountedLoopReserveKit.
//
--- 1063,1074 ----
// Create a slow version of the loop by cloning the loop
// and inserting an if to select fast-slow versions.
ProjNode* create_slow_version_of_loop(IdealLoopTree *loop,
Node_List &old_new,
! int opcode,
! CloneLoopMode mode);
// Clone a loop and return the clone head (clone_loop_head).
// Added nodes include int(1), int(0) - disconnected, If, IfTrue, IfFalse,
// This routine was created for usage in CountedLoopReserveKit.
//
< prev index next >