< prev index next >
src/hotspot/share/opto/loopnode.hpp
Print this page
@@ -35,10 +35,11 @@
class CountedLoopEndNode;
class CountedLoopNode;
class IdealLoopTree;
class LoopNode;
class Node;
+class Opaque5Node;
class PhaseIdealLoop;
class CountedLoopReserveKit;
class VectorSet;
class Invariance;
struct small_cache;
@@ -69,39 +70,43 @@
PassedSlpAnalysis=512,
DoUnrollOnly=1024,
VectorizedLoop=2048,
HasAtomicPostLoop=4096,
HasRangeChecks=8192,
- IsMultiversioned=16384};
+ IsMultiversioned=16384,
+ StripMined=32768};
char _unswitch_count;
enum { _unswitch_max=3 };
char _postloop_flags;
enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 };
public:
// Names for edge indices
enum { Self=0, EntryControl, LoopBackControl };
- int is_inner_loop() const { return _loop_flags & InnerLoop; }
+ uint is_inner_loop() const { return _loop_flags & InnerLoop; }
void set_inner_loop() { _loop_flags |= InnerLoop; }
- int range_checks_present() const { return _loop_flags & HasRangeChecks; }
- int is_multiversioned() const { return _loop_flags & IsMultiversioned; }
- int is_vectorized_loop() const { return _loop_flags & VectorizedLoop; }
- int is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }
+ uint range_checks_present() const { return _loop_flags & HasRangeChecks; }
+ uint is_multiversioned() const { return _loop_flags & IsMultiversioned; }
+ uint is_vectorized_loop() const { return _loop_flags & VectorizedLoop; }
+ uint is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }
void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }
- int partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
+ uint partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
+ uint is_strip_mined() const { return _loop_flags & StripMined; }
void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }
void mark_has_reductions() { _loop_flags |= HasReductions; }
void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; }
void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; }
void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; }
void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; }
void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; }
void mark_has_range_checks() { _loop_flags |= HasRangeChecks; }
void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; }
+ void mark_strip_mined() { _loop_flags |= StripMined; }
+ void clear_strip_mined() { _loop_flags &= ~StripMined; }
int unswitch_max() { return _unswitch_max; }
int unswitch_count() { return _unswitch_count; }
int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; }
@@ -129,10 +134,20 @@
}
bool is_valid_counted_loop() const;
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
+
+ void verify_strip_mined(int expect_opaq) const;
+ virtual LoopNode* skip_strip_mined(int expect_opaq = 1) { return this; }
+ virtual IfTrueNode* outer_loop_tail() const;
+ virtual IfNode* outer_loop_end() const;
+ virtual IfFalseNode* outer_loop_exit() const;
+ virtual SafePointNode* outer_safepoint() const;
+ virtual BoolNode* outer_bol() const;
+ virtual CmpINode* outer_cmp() const;
+ virtual Opaque5Node* outer_opaq() const;
};
//------------------------------Counted Loops----------------------------------
// Counted loops are all trip-counted loops, with exactly 1 trip-counter exit
// path (and maybe some other exit paths). The trip-counter exit is always
@@ -230,20 +245,20 @@
// so the following main loop 'knows' that it is striding down cache
// lines.
// A 'main' loop that is ONLY unrolled or peeled, never RCE'd or
// Aligned, may be missing it's pre-loop.
- int is_normal_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }
- int is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; }
- int is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; }
- int is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; }
- int is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; }
- int was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; }
- int has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; }
- int do_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; }
- int is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }
- int has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; }
+ uint is_normal_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }
+ uint is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; }
+ uint is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; }
+ uint is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; }
+ uint is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; }
+ uint was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; }
+ uint has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; }
+ uint do_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; }
+ uint is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }
+ uint has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; }
void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; }
int main_idx() const { return _main_idx; }
@@ -276,10 +291,20 @@
void set_node_count_before_unroll(int ct) { _node_count_before_unroll = ct; }
int node_count_before_unroll() { return _node_count_before_unroll; }
void set_slp_max_unroll(int unroll_factor) { _slp_maximum_unroll_factor = unroll_factor; }
int slp_max_unroll() const { return _slp_maximum_unroll_factor; }
+ virtual LoopNode* skip_strip_mined(int expect_opaq = 1);
+ LoopNode* outer_loop() const;
+ virtual IfTrueNode* outer_loop_tail() const;
+ virtual IfNode* outer_loop_end() const;
+ virtual IfFalseNode* outer_loop_exit() const;
+ virtual SafePointNode* outer_safepoint() const;
+ virtual BoolNode* outer_bol() const;
+ virtual CmpINode* outer_cmp() const;
+ virtual Opaque5Node* outer_opaq() const;
+
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
};
@@ -778,10 +803,11 @@
// Place Data nodes in some loop nest
void build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack );
void build_loop_late ( VectorSet &visited, Node_List &worklist, Node_Stack &nstack );
void build_loop_late_post ( Node* n );
+ void verify_strip_mined_scheduling(Node *n, Node* least);
// Array of immediate dominance info for each CFG node indexed by node idx
private:
uint _idom_size;
Node **_idom; // Array of immediate dominators
@@ -875,11 +901,14 @@
bool _has_irreducible_loops;
// Per-Node transform
virtual Node *transform( Node *a_node ) { return 0; }
- bool is_counted_loop( Node *x, IdealLoopTree *loop );
+ bool is_counted_loop(Node* x, IdealLoopTree*& loop);
+ IdealLoopTree* create_outer_strip_mined_loop(BoolNode *test, Node *cmp, Node *init_control,
+ IdealLoopTree* loop, float cl_prob, float le_fcnt,
+ Node*& entry_control, Node*& iffalse);
Node* exact_limit( IdealLoopTree *loop );
// Return a post-walked LoopNode
IdealLoopTree *get_loop( Node *n ) const {
@@ -906,12 +935,28 @@
// the clone loop to dominate the original. Used in construction of
// pre-main-post loop sequence.
// When nonnull, the clone and original are side-by-side, both are
// dominated by the passed in side_by_side_idom node. Used in
// construction of unswitched loops.
+ enum CloneLoopMode {
+ IgnoreStripMined = 0, // Only clone inner strip mined loop
+ CloneIncludesStripMined = 1, // clone both inner and outer strip mined loops
+ ControlAroundStripMined = 2 // Only clone inner strip mined loop,
+ // result control flow branches
+ // either to inner clone or outer
+ // strip mined loop.
+ };
void clone_loop( IdealLoopTree *loop, Node_List &old_new, int dom_depth,
- Node* side_by_side_idom = NULL);
+ CloneLoopMode mode, Node* side_by_side_idom = NULL);
+ void clone_loop_handle_data_uses(Node* old, Node_List &old_new,
+ IdealLoopTree* loop, IdealLoopTree* companion_loop,
+ Node_List*& split_if_set, Node_List*& split_bool_set,
+ Node_List*& split_cex_set, Node_List& worklist,
+ uint new_counter, CloneLoopMode mode);
+ void clone_outer_loop(LoopNode* head, CloneLoopMode mode, IdealLoopTree *loop,
+ IdealLoopTree* outer_loop, int dd, Node_List &old_new,
+ Node_List& extra_data_nodes);
// If we got the effect of peeling, either by actually peeling or by
// making a pre-loop which must execute at least once, we can remove
// all loop-invariant dominated tests in the main body.
void peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new );
@@ -1018,11 +1063,12 @@
// Create a slow version of the loop by cloning the loop
// and inserting an if to select fast-slow versions.
ProjNode* create_slow_version_of_loop(IdealLoopTree *loop,
Node_List &old_new,
- int opcode);
+ int opcode,
+ CloneLoopMode mode);
// Clone a loop and return the clone head (clone_loop_head).
// Added nodes include int(1), int(0) - disconnected, If, IfTrue, IfFalse,
// This routine was created for usage in CountedLoopReserveKit.
//
< prev index next >