< prev index next >
src/hotspot/share/opto/loopnode.cpp
Print this page
rev 54995 : 8224675: Late GC barrier insertion for ZGC
Reviewed-by:
*** 976,986 ****
}
wq.clear();
wq.push(u);
bool found_sfpt = false;
for (uint next = 0; next < wq.size() && !found_sfpt; next++) {
! Node *n = wq.at(next);
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && !found_sfpt; i++) {
Node* u = n->fast_out(i);
if (u == sfpt) {
found_sfpt = true;
}
--- 976,986 ----
}
wq.clear();
wq.push(u);
bool found_sfpt = false;
for (uint next = 0; next < wq.size() && !found_sfpt; next++) {
! Node* n = wq.at(next);
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && !found_sfpt; i++) {
Node* u = n->fast_out(i);
if (u == sfpt) {
found_sfpt = true;
}
*** 990,1021 ****
}
}
assert(found_sfpt, "no node in loop that's not input to safepoint");
}
}
! CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd();
assert(cle == inner->loopexit_or_null(), "mismatch");
bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0;
if (has_skeleton) {
assert(expect_skeleton == 1 || expect_skeleton == -1, "unexpected skeleton node");
assert(outer->outcnt() == 2, "only phis");
} else {
assert(expect_skeleton == 0 || expect_skeleton == -1, "no skeleton node?");
uint phis = 0;
for (DUIterator_Fast imax, i = inner->fast_outs(imax); i < imax; i++) {
! Node* u = inner->fast_out(i);
if (u->is_Phi()) {
phis++;
}
}
for (DUIterator_Fast imax, i = outer->fast_outs(imax); i < imax; i++) {
! Node* u = outer->fast_out(i);
assert(u == outer || u == inner || u->is_Phi(), "nothing between inner and outer loop");
}
uint stores = 0;
for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) {
! Node* u = inner_out->fast_out(i);
if (u->is_Store()) {
stores++;
}
}
assert(outer->outcnt() >= phis + 2 && outer->outcnt() <= phis + 2 + stores + 1, "only phis");
--- 990,1034 ----
}
}
assert(found_sfpt, "no node in loop that's not input to safepoint");
}
}
!
! if (UseZGC && !inner_out->in(0)->is_CountedLoopEnd()) {
! // In some very special cases there can be a load that has no other uses than the
! // counted loop safepoint. Then its loadbarrier will be placed between the inner
! // loop exit and the safepoint. This is very rare
!
! Node* ifnode = inner_out->in(1)->in(0);
! // Region->IfTrue->If == Region->Iffalse->If
! if (ifnode == inner_out->in(2)->in(0)) {
! inner_out = ifnode->in(0);
! }
! }
!
! CountedLoopEndNode *cle = inner_out->in(0)->as_CountedLoopEnd();
assert(cle == inner->loopexit_or_null(), "mismatch");
bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0;
if (has_skeleton) {
assert(expect_skeleton == 1 || expect_skeleton == -1, "unexpected skeleton node");
assert(outer->outcnt() == 2, "only phis");
} else {
assert(expect_skeleton == 0 || expect_skeleton == -1, "no skeleton node?");
uint phis = 0;
for (DUIterator_Fast imax, i = inner->fast_outs(imax); i < imax; i++) {
! Node *u = inner->fast_out(i);
if (u->is_Phi()) {
phis++;
}
}
for (DUIterator_Fast imax, i = outer->fast_outs(imax); i < imax; i++) {
! Node *u = outer->fast_out(i);
assert(u == outer || u == inner || u->is_Phi(), "nothing between inner and outer loop");
}
uint stores = 0;
for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) {
! Node *u = inner_out->fast_out(i);
if (u->is_Store()) {
stores++;
}
}
assert(outer->outcnt() >= phis + 2 && outer->outcnt() <= phis + 2 + stores + 1, "only phis");
*** 2708,2719 ****
//=============================================================================
//----------------------------build_and_optimize-------------------------------
// Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to
// its corresponding LoopNode. If 'optimize' is true, do some loop cleanups.
void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) {
! bool do_split_ifs = (mode == LoopOptsDefault || mode == LoopOptsLastRound);
! bool skip_loop_opts = (mode == LoopOptsNone);
int old_progress = C->major_progress();
uint orig_worklist_size = _igvn._worklist.size();
// Reset major-progress flag for the driver's heuristics
--- 2721,2732 ----
//=============================================================================
//----------------------------build_and_optimize-------------------------------
// Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to
// its corresponding LoopNode. If 'optimize' is true, do some loop cleanups.
void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) {
! bool do_split_ifs = (mode == LoopOptsDefault);
! bool skip_loop_opts = (mode == LoopOptsNone || mode == LoopOptsBarrierInsertion);
int old_progress = C->major_progress();
uint orig_worklist_size = _igvn._worklist.size();
// Reset major-progress flag for the driver's heuristics
*** 2868,2880 ****
NOT_PRODUCT( C->verify_graph_edges(); )
worklist.push( C->top() );
build_loop_late( visited, worklist, nstack );
if (_verify_only) {
! // restore major progress flag
! for (int i = 0; i < old_progress; i++)
! C->set_major_progress();
assert(C->unique() == unique, "verification mode made Nodes? ? ?");
assert(_igvn._worklist.size() == orig_worklist_size, "shouldn't push anything");
return;
}
--- 2881,2891 ----
NOT_PRODUCT( C->verify_graph_edges(); )
worklist.push( C->top() );
build_loop_late( visited, worklist, nstack );
if (_verify_only) {
! C->restore_major_progress(old_progress);
assert(C->unique() == unique, "verification mode made Nodes? ? ?");
assert(_igvn._worklist.size() == orig_worklist_size, "shouldn't push anything");
return;
}
*** 2912,2926 ****
if(TraceLoopOpts && C->has_loops()) {
_ltree_root->dump();
}
#endif
if (skip_loop_opts) {
// restore major progress flag
! for (int i = 0; i < old_progress; i++) {
! C->set_major_progress();
! }
// Cleanup any modified bits
_igvn.optimize();
if (C->log() != NULL) {
--- 2923,2941 ----
if(TraceLoopOpts && C->has_loops()) {
_ltree_root->dump();
}
#endif
+ if (mode == LoopOptsBarrierInsertion) {
+ // Expand late nodes here
+ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+ bs->barrier_insertion(this);
+ }
+
if (skip_loop_opts) {
// restore major progress flag
! C->restore_major_progress(old_progress);
// Cleanup any modified bits
_igvn.optimize();
if (C->log() != NULL) {
*** 2963,2977 ****
// Check for aggressive application of split-if and other transforms
// that require basic-block info (like cloning through Phi's)
if( SplitIfBlocks && do_split_ifs ) {
visited.Clear();
! split_if_with_blocks( visited, nstack, mode == LoopOptsLastRound );
NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); );
- if (mode == LoopOptsLastRound) {
- C->set_major_progress();
- }
}
if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) {
C->set_major_progress();
}
--- 2978,2989 ----
// Check for aggressive application of split-if and other transforms
// that require basic-block info (like cloning through Phi's)
if( SplitIfBlocks && do_split_ifs ) {
visited.Clear();
! split_if_with_blocks( visited, nstack);
NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); );
}
if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) {
C->set_major_progress();
}
*** 3102,3113 ****
assert( fail == 0, "verify loops failed" );
// Verify loop structure is the same
_ltree_root->verify_tree(loop_verify._ltree_root, NULL);
// Reset major-progress. It was cleared by creating a verify version of
// PhaseIdealLoop.
! for( int i=0; i<old_progress; i++ )
! C->set_major_progress();
}
//------------------------------verify_compare---------------------------------
// Make sure me and the given PhaseIdealLoop agree on key data structures
void PhaseIdealLoop::verify_compare( Node *n, const PhaseIdealLoop *loop_verify, VectorSet &visited ) const {
--- 3114,3124 ----
assert( fail == 0, "verify loops failed" );
// Verify loop structure is the same
_ltree_root->verify_tree(loop_verify._ltree_root, NULL);
// Reset major-progress. It was cleared by creating a verify version of
// PhaseIdealLoop.
! C->restore_major_progress(old_progress);
}
//------------------------------verify_compare---------------------------------
// Make sure me and the given PhaseIdealLoop agree on key data structures
void PhaseIdealLoop::verify_compare( Node *n, const PhaseIdealLoop *loop_verify, VectorSet &visited ) const {
*** 4233,4243 ****
case Op_LoadNKlass:
case Op_LoadL:
case Op_LoadS:
case Op_LoadP:
case Op_LoadBarrierSlowReg:
- case Op_LoadBarrierWeakSlowReg:
case Op_LoadN:
case Op_LoadRange:
case Op_LoadD_unaligned:
case Op_LoadL_unaligned:
case Op_StrComp: // Does a bunch of load-like effects
--- 4244,4253 ----
< prev index next >