< prev index next >

src/hotspot/share/opto/loopnode.hpp

Print this page

        

*** 35,44 **** --- 35,45 ---- class CountedLoopEndNode; class CountedLoopNode; class IdealLoopTree; class LoopNode; class Node; + class Opaque5Node; class PhaseIdealLoop; class CountedLoopReserveKit; class VectorSet; class Invariance; struct small_cache;
*** 69,107 **** PassedSlpAnalysis=512, DoUnrollOnly=1024, VectorizedLoop=2048, HasAtomicPostLoop=4096, HasRangeChecks=8192, ! IsMultiversioned=16384}; char _unswitch_count; enum { _unswitch_max=3 }; char _postloop_flags; enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 }; public: // Names for edge indices enum { Self=0, EntryControl, LoopBackControl }; ! int is_inner_loop() const { return _loop_flags & InnerLoop; } void set_inner_loop() { _loop_flags |= InnerLoop; } ! int range_checks_present() const { return _loop_flags & HasRangeChecks; } ! int is_multiversioned() const { return _loop_flags & IsMultiversioned; } ! int is_vectorized_loop() const { return _loop_flags & VectorizedLoop; } ! int is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; } void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; } ! int partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; } void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; } void mark_has_reductions() { _loop_flags |= HasReductions; } void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; } void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; } void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; } void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; } void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; } void mark_has_range_checks() { _loop_flags |= HasRangeChecks; } void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; } int unswitch_max() { return _unswitch_max; } int unswitch_count() { return _unswitch_count; } int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; } --- 70,112 ---- PassedSlpAnalysis=512, DoUnrollOnly=1024, VectorizedLoop=2048, HasAtomicPostLoop=4096, HasRangeChecks=8192, ! IsMultiversioned=16384, ! StripMined=32768}; char _unswitch_count; enum { _unswitch_max=3 }; char _postloop_flags; enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 }; public: // Names for edge indices enum { Self=0, EntryControl, LoopBackControl }; ! uint is_inner_loop() const { return _loop_flags & InnerLoop; } void set_inner_loop() { _loop_flags |= InnerLoop; } ! uint range_checks_present() const { return _loop_flags & HasRangeChecks; } ! uint is_multiversioned() const { return _loop_flags & IsMultiversioned; } ! uint is_vectorized_loop() const { return _loop_flags & VectorizedLoop; } ! uint is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; } void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; } ! uint partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; } ! uint is_strip_mined() const { return _loop_flags & StripMined; } void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; } void mark_has_reductions() { _loop_flags |= HasReductions; } void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; } void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; } void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; } void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; } void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; } void mark_has_range_checks() { _loop_flags |= HasRangeChecks; } void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; } + void mark_strip_mined() { _loop_flags |= StripMined; } + void clear_strip_mined() { _loop_flags &= ~StripMined; } int unswitch_max() { return _unswitch_max; } int unswitch_count() { return _unswitch_count; } int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; }
*** 129,138 **** --- 134,153 ---- } bool is_valid_counted_loop() const; #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; #endif + + void verify_strip_mined(int expect_opaq) const; + virtual LoopNode* skip_strip_mined(int expect_opaq = 1) { return this; } + virtual IfTrueNode* outer_loop_tail() const; + virtual IfNode* outer_loop_end() const; + virtual IfFalseNode* outer_loop_exit() const; + virtual SafePointNode* outer_safepoint() const; + virtual BoolNode* outer_bol() const; + virtual CmpINode* outer_cmp() const; + virtual Opaque5Node* outer_opaq() const; }; //------------------------------Counted Loops---------------------------------- // Counted loops are all trip-counted loops, with exactly 1 trip-counter exit // path (and maybe some other exit paths). The trip-counter exit is always
*** 230,249 **** // so the following main loop 'knows' that it is striding down cache // lines. // A 'main' loop that is ONLY unrolled or peeled, never RCE'd or // Aligned, may be missing it's pre-loop. ! int is_normal_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; } ! int is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; } ! int is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; } ! int is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; } ! int is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; } ! int was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; } ! int has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; } ! int do_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; } ! int is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; } ! int has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; } void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; } int main_idx() const { return _main_idx; } --- 245,264 ---- // so the following main loop 'knows' that it is striding down cache // lines. // A 'main' loop that is ONLY unrolled or peeled, never RCE'd or // Aligned, may be missing it's pre-loop. ! uint is_normal_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; } ! uint is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; } ! uint is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; } ! uint is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; } ! uint is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; } ! uint was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; } ! uint has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; } ! uint do_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; } ! uint is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; } ! uint has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; } void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; } int main_idx() const { return _main_idx; }
*** 276,285 **** --- 291,310 ---- void set_node_count_before_unroll(int ct) { _node_count_before_unroll = ct; } int node_count_before_unroll() { return _node_count_before_unroll; } void set_slp_max_unroll(int unroll_factor) { _slp_maximum_unroll_factor = unroll_factor; } int slp_max_unroll() const { return _slp_maximum_unroll_factor; } + virtual LoopNode* skip_strip_mined(int expect_opaq = 1); + LoopNode* outer_loop() const; + virtual IfTrueNode* outer_loop_tail() const; + virtual IfNode* outer_loop_end() const; + virtual IfFalseNode* outer_loop_exit() const; + virtual SafePointNode* outer_safepoint() const; + virtual BoolNode* outer_bol() const; + virtual CmpINode* outer_cmp() const; + virtual Opaque5Node* outer_opaq() const; + #ifndef PRODUCT virtual void dump_spec(outputStream *st) const; #endif };
*** 778,787 **** --- 803,813 ---- // Place Data nodes in some loop nest void build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ); void build_loop_late ( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ); void build_loop_late_post ( Node* n ); + void verify_strip_mined_scheduling(Node *n, Node* least); // Array of immediate dominance info for each CFG node indexed by node idx private: uint _idom_size; Node **_idom; // Array of immediate dominators
*** 875,885 **** bool _has_irreducible_loops; // Per-Node transform virtual Node *transform( Node *a_node ) { return 0; } ! bool is_counted_loop( Node *x, IdealLoopTree *loop ); Node* exact_limit( IdealLoopTree *loop ); // Return a post-walked LoopNode IdealLoopTree *get_loop( Node *n ) const { --- 901,914 ---- bool _has_irreducible_loops; // Per-Node transform virtual Node *transform( Node *a_node ) { return 0; } ! bool is_counted_loop(Node* x, IdealLoopTree*& loop); ! IdealLoopTree* create_outer_strip_mined_loop(BoolNode *test, Node *cmp, Node *init_control, ! IdealLoopTree* loop, float cl_prob, float le_fcnt, ! Node*& entry_control, Node*& iffalse); Node* exact_limit( IdealLoopTree *loop ); // Return a post-walked LoopNode IdealLoopTree *get_loop( Node *n ) const {
*** 906,917 **** // the clone loop to dominate the original. Used in construction of // pre-main-post loop sequence. // When nonnull, the clone and original are side-by-side, both are // dominated by the passed in side_by_side_idom node. Used in // construction of unswitched loops. void clone_loop( IdealLoopTree *loop, Node_List &old_new, int dom_depth, ! Node* side_by_side_idom = NULL); // If we got the effect of peeling, either by actually peeling or by // making a pre-loop which must execute at least once, we can remove // all loop-invariant dominated tests in the main body. void peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new ); --- 935,962 ---- // the clone loop to dominate the original. Used in construction of // pre-main-post loop sequence. // When nonnull, the clone and original are side-by-side, both are // dominated by the passed in side_by_side_idom node. Used in // construction of unswitched loops. + enum CloneLoopMode { + IgnoreStripMined = 0, // Only clone inner strip mined loop + CloneIncludesStripMined = 1, // clone both inner and outer strip mined loops + ControlAroundStripMined = 2 // Only clone inner strip mined loop, + // result control flow branches + // either to inner clone or outer + // strip mined loop. + }; void clone_loop( IdealLoopTree *loop, Node_List &old_new, int dom_depth, ! CloneLoopMode mode, Node* side_by_side_idom = NULL); ! void clone_loop_handle_data_uses(Node* old, Node_List &old_new, ! IdealLoopTree* loop, IdealLoopTree* companion_loop, ! Node_List*& split_if_set, Node_List*& split_bool_set, ! Node_List*& split_cex_set, Node_List& worklist, ! uint new_counter, CloneLoopMode mode); ! void clone_outer_loop(LoopNode* head, CloneLoopMode mode, IdealLoopTree *loop, ! IdealLoopTree* outer_loop, int dd, Node_List &old_new, ! Node_List& extra_data_nodes); // If we got the effect of peeling, either by actually peeling or by // making a pre-loop which must execute at least once, we can remove // all loop-invariant dominated tests in the main body. void peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new );
*** 1018,1028 **** // Create a slow version of the loop by cloning the loop // and inserting an if to select fast-slow versions. ProjNode* create_slow_version_of_loop(IdealLoopTree *loop, Node_List &old_new, ! int opcode); // Clone a loop and return the clone head (clone_loop_head). // Added nodes include int(1), int(0) - disconnected, If, IfTrue, IfFalse, // This routine was created for usage in CountedLoopReserveKit. // --- 1063,1074 ---- // Create a slow version of the loop by cloning the loop // and inserting an if to select fast-slow versions. ProjNode* create_slow_version_of_loop(IdealLoopTree *loop, Node_List &old_new, ! int opcode, ! CloneLoopMode mode); // Clone a loop and return the clone head (clone_loop_head). // Added nodes include int(1), int(0) - disconnected, If, IfTrue, IfFalse, // This routine was created for usage in CountedLoopReserveKit. //
< prev index next >