< prev index next >
src/hotspot/share/opto/loopnode.cpp
Print this page
@@ -38,10 +38,14 @@
#include "opto/idealGraphPrinter.hpp"
#include "opto/loopnode.hpp"
#include "opto/mulnode.hpp"
#include "opto/rootnode.hpp"
#include "opto/superword.hpp"
+#include "utilities/macros.hpp"
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
+#endif
//=============================================================================
//------------------------------is_loop_iv-------------------------------------
// Determine if a node is Counted loop induction variable.
// The method is declared in node.hpp.
@@ -994,14 +998,14 @@
}
}
assert(found_sfpt, "no node in loop that's not input to safepoint");
}
}
- CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd();
- assert(cle == inner->loopexit_or_null(), "mismatch");
bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0;
if (has_skeleton) {
+ CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd();
+ assert(cle == inner->loopexit_or_null(), "mismatch");
assert(expect_skeleton == 1 || expect_skeleton == -1, "unexpected skeleton node");
assert(outer->outcnt() == 2, "only phis");
} else {
assert(expect_skeleton == 0 || expect_skeleton == -1, "no skeleton node?");
uint phis = 0;
@@ -1013,16 +1017,36 @@
}
for (DUIterator_Fast imax, i = outer->fast_outs(imax); i < imax; i++) {
Node* u = outer->fast_out(i);
assert(u == outer || u == inner || u->is_Phi(), "nothing between inner and outer loop");
}
+ Node* c = inner_out;
uint stores = 0;
- for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) {
- Node* u = inner_out->fast_out(i);
- if (u->is_Store()) {
- stores++;
+ for (;;) {
+ for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
+ Node* u = c->fast_out(i);
+ if (u->is_Store()) {
+ stores++;
+ }
}
+ if (c->in(0)->is_CountedLoopEnd()) {
+ break;
+ }
+#if INCLUDE_SHENANDOAHGC
+ assert(UseShenandoahGC, "only for shenandoah barriers");
+ assert(c->is_Region() && c->req() == 3, "region that ends barrier");
+ uint j = 1;
+ uint req = c->req();
+ for (; j < req; j++) {
+ Node* in = c->in(j);
+ if (in->is_IfProj() && ShenandoahWriteBarrierNode::is_heap_stable_test(in->in(0))) {
+ c = in->in(0)->in(0);
+ break;
+ }
+ }
+ assert(j < req, "should have found heap stable test");
+#endif
}
assert(outer->outcnt() >= phis + 2 && outer->outcnt() <= phis + 2 + stores + 1, "only phis");
}
assert(sfpt->outcnt() == 1, "no data node");
assert(outer_tail->outcnt() == 1 || !has_skeleton, "no data node");
@@ -2709,10 +2733,12 @@
// Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to
// its corresponding LoopNode. If 'optimize' is true, do some loop cleanups.
void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) {
bool do_split_ifs = (mode == LoopOptsDefault || mode == LoopOptsLastRound);
bool skip_loop_opts = (mode == LoopOptsNone);
+ bool shenandoah_opts = (mode == LoopOptsShenandoahExpand ||
+ mode == LoopOptsShenandoahPostExpand);
ResourceMark rm;
int old_progress = C->major_progress();
uint orig_worklist_size = _igvn._worklist.size();
@@ -2773,11 +2799,11 @@
}
return;
}
// Nothing to do, so get out
- bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only;
+ bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only && !shenandoah_opts;
bool do_expensive_nodes = C->should_optimize_expensive_nodes(_igvn);
if (stop_early && !do_expensive_nodes) {
_igvn.optimize(); // Cleanup NeverBranches
return;
}
@@ -2852,21 +2878,22 @@
visited.set( C->top()->_idx ); // Set C->top() as visited now
build_loop_early( visited, worklist, nstack );
// Given early legal placement, try finding counted loops. This placement
// is good enough to discover most loop invariants.
- if( !_verify_me && !_verify_only )
- _ltree_root->counted_loop( this );
+ if (!_verify_me && !_verify_only && !shenandoah_opts) {
+ _ltree_root->counted_loop(this);
+ }
// Find latest loop placement. Find ideal loop placement.
visited.Clear();
init_dom_lca_tags();
// Need C->root() on worklist when processing outs
worklist.push( C->root() );
NOT_PRODUCT( C->verify_graph_edges(); )
worklist.push( C->top() );
- build_loop_late( visited, worklist, nstack );
+ build_loop_late(visited, worklist, nstack, !shenandoah_opts);
if (_verify_only) {
// restore major progress flag
for (int i = 0; i < old_progress; i++)
C->set_major_progress();
@@ -2909,10 +2936,29 @@
if(TraceLoopOpts && C->has_loops()) {
_ltree_root->dump();
}
#endif
+#if INCLUDE_SHENANDOAHGC
+ if (mode == LoopOptsShenandoahExpand) {
+ assert(UseShenandoahGC, "only for shenandoah");
+ ShenandoahWriteBarrierNode::pin_and_expand(this);
+ } else if (mode == LoopOptsShenandoahPostExpand) {
+ assert(UseShenandoahGC, "only for shenandoah");
+ visited.Clear();
+ ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, this);
+ }
+
+ if (shenandoah_opts) {
+ _igvn.optimize();
+ if (C->log() != NULL) {
+ log_loop_tree(_ltree_root, _ltree_root, C->log());
+ }
+ return;
+ }
+#endif
+
if (skip_loop_opts) {
// restore major progress flag
for (int i = 0; i < old_progress; i++) {
C->set_major_progress();
}
@@ -2924,10 +2970,17 @@
log_loop_tree(_ltree_root, _ltree_root, C->log());
}
return;
}
+#if INCLUDE_SHENANDOAHGC
+ if (UseShenandoahGC) {
+ GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
+ ShenandoahWriteBarrierNode::optimize_before_expansion(this, memory_graph_fixers, false);
+ }
+#endif
+
if (ReassociateInvariants) {
// Reassociate invariants and prep for split_thru_phi
for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
IdealLoopTree* lpt = iter.current();
bool is_counted = lpt->is_counted();
@@ -3954,21 +4007,21 @@
Node* s = mem->fast_out(i);
worklist.push(s);
}
while(worklist.size() != 0 && LCA != early) {
Node* s = worklist.pop();
- if (s->is_Load() || s->Opcode() == Op_SafePoint ||
+ if (s->is_Load() || s->is_ShenandoahBarrier() || s->Opcode() == Op_SafePoint ||
(s->is_CallStaticJava() && s->as_CallStaticJava()->uncommon_trap_request() != 0)) {
continue;
} else if (s->is_MergeMem()) {
for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) {
Node* s1 = s->fast_out(i);
worklist.push(s1);
}
} else {
Node *sctrl = has_ctrl(s) ? get_ctrl(s) : s->in(0);
- assert(sctrl != NULL || s->outcnt() == 0, "must have control");
+ assert(sctrl != NULL || s->outcnt() == 0 || s->is_ShenandoahBarrier(), "must have control");
if (sctrl != NULL && !sctrl->is_top() && is_dominator(early, sctrl)) {
LCA = dom_lca_for_get_late_ctrl(LCA, sctrl, n);
}
}
}
@@ -4082,11 +4135,11 @@
}
//------------------------------build_loop_late--------------------------------
// Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping.
// Second pass finds latest legal placement, and ideal loop placement.
-void PhaseIdealLoop::build_loop_late( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) {
+void PhaseIdealLoop::build_loop_late(VectorSet &visited, Node_List &worklist, Node_Stack &nstack, bool verify_strip_mined) {
while (worklist.size() != 0) {
Node *n = worklist.pop();
// Only visit once
if (visited.test_set(n->_idx)) continue;
uint cnt = n->outcnt();
@@ -4118,11 +4171,11 @@
// push dead code onto a worklist
_deadlist.push(use);
}
} else {
// All of n's children have been processed, complete post-processing.
- build_loop_late_post(n);
+ build_loop_late_post(n, verify_strip_mined);
if (nstack.is_empty()) {
// Finished all nodes on stack.
// Process next node on the worklist.
break;
}
@@ -4134,11 +4187,11 @@
}
}
}
}
-// Verify that no data node is schedules in the outer loop of a strip
+// Verify that no data node is scheduled in the outer loop of a strip
// mined loop.
void PhaseIdealLoop::verify_strip_mined_scheduling(Node *n, Node* least) {
#ifdef ASSERT
if (get_loop(least)->_nest == 0) {
return;
@@ -4169,11 +4222,11 @@
//------------------------------build_loop_late_post---------------------------
// Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping.
// Second pass finds latest legal placement, and ideal loop placement.
-void PhaseIdealLoop::build_loop_late_post( Node *n ) {
+void PhaseIdealLoop::build_loop_late_post(Node *n, bool verify_strip_mined) {
if (n->req() == 2 && (n->Opcode() == Op_ConvI2L || n->Opcode() == Op_CastII) && !C->major_progress() && !_verify_only) {
_igvn._worklist.push(n); // Maybe we'll normalize it, if no more loops.
}
@@ -4220,13 +4273,19 @@
case Op_StrComp: // Does a bunch of load-like effects
case Op_StrEquals:
case Op_StrIndexOf:
case Op_StrIndexOfChar:
case Op_AryEq:
+#if INCLUDE_SHENANDOAHGC
+ case Op_ShenandoahReadBarrier:
+ case Op_ShenandoahWriteBarrier:
+ case Op_ShenandoahWBMemProj:
+#endif
case Op_HasNegatives:
pinned = false;
}
+
if( pinned ) {
IdealLoopTree *chosen_loop = get_loop(n->is_CFG() ? n : get_ctrl(n));
if( !chosen_loop->_child ) // Inner loop?
chosen_loop->_body.push(n); // Collect inner loops
return;
@@ -4325,17 +4384,29 @@
}
#endif
// Assign discovered "here or above" point
least = find_non_split_ctrl(least);
- verify_strip_mined_scheduling(n, least);
+ if (verify_strip_mined && !_verify_only) {
+ verify_strip_mined_scheduling(n, least);
+ }
set_ctrl(n, least);
// Collect inner loop bodies
IdealLoopTree *chosen_loop = get_loop(least);
if( !chosen_loop->_child ) // Inner loop?
chosen_loop->_body.push(n);// Collect inner loops
+
+ if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+ // The write barrier and its memory proj must have the same
+ // control otherwise some loop opts could put nodes (Phis) between
+ // them
+ Node* proj = n->find_out_with(Op_ShenandoahWBMemProj);
+ if (proj != NULL) {
+ set_ctrl_and_loop(proj, least);
+ }
+ }
}
#ifdef ASSERT
void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA) {
tty->print_cr("%s", msg);
@@ -4487,10 +4558,11 @@
}
}
}
}
}
+#endif
// Collect a R-P-O for the whole CFG.
// Result list is in post-order (scan backwards for RPO)
void PhaseIdealLoop::rpo( Node *start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list ) const {
stk.push(start, 0);
@@ -4509,11 +4581,10 @@
rpo_list.push(m);
stk.pop();
}
}
}
-#endif
//=============================================================================
//------------------------------LoopTreeIterator-----------------------------------
< prev index next >