1174 }
1175 assert(region != NULL, "");
1176 Node* phi = new PhiNode(region, n->bottom_type());
1177 for (uint j = 1; j < region->req(); j++) {
1178 Node* in = region->in(j);
1179 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1180 phi->init_req(j, n);
1181 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1182 phi->init_req(j, n_clone);
1183 } else {
1184 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1185 }
1186 }
1187 phase->register_new_node(phi, region);
1188 return phi;
1189 }
1190
1191 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1192 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1193
1194 // Collect raw memory state at CFG points in the entire graph and
1195 // record it in memory_nodes. Optimize the raw memory graph in the
1196 // process. Optimizing the memory graph also makes the memory graph
1197 // simpler.
1198 GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
1199
1200 Unique_Node_List uses;
1201 for (int i = 0; i < state->enqueue_barriers_count(); i++) {
1202 Node* barrier = state->enqueue_barrier(i);
1203 Node* ctrl = phase->get_ctrl(barrier);
1204 IdealLoopTree* loop = phase->get_loop(ctrl);
1205 if (loop->_head->is_OuterStripMinedLoop()) {
1206 // Expanding a barrier here will break loop strip mining
1207 // verification. Transform the loop so the loop nest doesn't
1208 // appear as strip mined.
1209 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
1210 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
1211 }
1212 }
1213
1214 Node_Stack stack(0);
1215 Node_List clones;
1216 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1217 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1218 if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
1219 continue;
1395 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1396 phase->igvn().rehash_node_delayed(u);
1397 int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase));
1398 assert(nb > 0, "should have replaced some uses");
1399 replaced = true;
1400 }
1401 }
1402 if (!replaced) {
1403 stack.set_index(idx+1);
1404 }
1405 }
1406 } else {
1407 stack.pop();
1408 clones.pop();
1409 }
1410 } while (stack.size() > 0);
1411 assert(stack.size() == 0 && clones.size() == 0, "");
1412 }
1413 }
1414
1415 // Expand load-reference-barriers
1416 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase);
1417 Unique_Node_List uses_to_ignore;
1418 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1419 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1420 if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
1421 phase->igvn().replace_node(lrb, lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1422 continue;
1423 }
1424 uint last = phase->C->unique();
1425 Node* ctrl = phase->get_ctrl(lrb);
1426 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1427
1428
1429 Node* orig_ctrl = ctrl;
1430
1431 Node* raw_mem = fixer.find_mem(ctrl, lrb);
1432 Node* init_raw_mem = raw_mem;
1433 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
1434 // int alias = phase->C->get_alias_index(lrb->adr_type());
1435
1436 IdealLoopTree *loop = phase->get_loop(ctrl);
1437 CallStaticJavaNode* unc = lrb->pin_and_expand_null_check(phase->igvn());
1438 Node* unc_ctrl = NULL;
1439 if (unc != NULL) {
1440 if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
1441 unc = NULL;
1442 } else {
1443 unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
1444 }
1445 }
1446
1447 Node* uncasted_val = val;
1448 if (unc != NULL) {
1449 uncasted_val = val->in(1);
1450 }
1451
1452 Node* heap_stable_ctrl = NULL;
1453 Node* null_ctrl = NULL;
1454
2578 Node* u = n->fast_out(i);
2579 if (!u->is_Root() && u->is_CFG() && u != n) {
2580 Node* m = _memory_nodes[u->_idx];
2581 if (u->is_Region() && (!u->is_OuterStripMinedLoop() || _include_lsm) &&
2582 !has_mem_phi(u) &&
2583 u->unique_ctrl_out()->Opcode() != Op_Halt) {
2584 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); });
2585 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); });
2586
2587 if (!mem_is_valid(m, u) || !m->is_Phi()) {
2588 bool push = true;
2589 bool create_phi = true;
2590 if (_phase->is_dominator(new_ctrl, u)) {
2591 create_phi = false;
2592 } else if (!_phase->C->has_irreducible_loop()) {
2593 IdealLoopTree* loop = _phase->get_loop(ctrl);
2594 bool do_check = true;
2595 IdealLoopTree* l = loop;
2596 create_phi = false;
2597 while (l != _phase->ltree_root()) {
2598 if (_phase->is_dominator(l->_head, u) && _phase->is_dominator(_phase->idom(u), l->_head)) {
2599 create_phi = true;
2600 do_check = false;
2601 break;
2602 }
2603 l = l->_parent;
2604 }
2605
2606 if (do_check) {
2607 assert(!create_phi, "");
2608 IdealLoopTree* u_loop = _phase->get_loop(u);
2609 if (u_loop != _phase->ltree_root() && u_loop->is_member(loop)) {
2610 Node* c = ctrl;
2611 while (!_phase->is_dominator(c, u_loop->tail())) {
2612 c = _phase->idom(c);
2613 }
2614 if (!_phase->is_dominator(c, u)) {
2615 do_check = false;
2616 }
2617 }
2618 }
|
1174 }
1175 assert(region != NULL, "");
1176 Node* phi = new PhiNode(region, n->bottom_type());
1177 for (uint j = 1; j < region->req(); j++) {
1178 Node* in = region->in(j);
1179 if (phase->is_dominator(projs.fallthrough_catchproj, in)) {
1180 phi->init_req(j, n);
1181 } else if (phase->is_dominator(projs.catchall_catchproj, in)) {
1182 phi->init_req(j, n_clone);
1183 } else {
1184 phi->init_req(j, create_phis_on_call_return(ctrl, in, n, n_clone, projs, phase));
1185 }
1186 }
1187 phase->register_new_node(phi, region);
1188 return phi;
1189 }
1190
1191 void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
1192 ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
1193
1194 Unique_Node_List uses;
1195 for (int i = 0; i < state->enqueue_barriers_count(); i++) {
1196 Node* barrier = state->enqueue_barrier(i);
1197 Node* ctrl = phase->get_ctrl(barrier);
1198 IdealLoopTree* loop = phase->get_loop(ctrl);
1199 if (loop->_head->is_OuterStripMinedLoop()) {
1200 // Expanding a barrier here will break loop strip mining
1201 // verification. Transform the loop so the loop nest doesn't
1202 // appear as strip mined.
1203 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
1204 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
1205 }
1206 }
1207
1208 Node_Stack stack(0);
1209 Node_List clones;
1210 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1211 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1212 if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
1213 continue;
1389 } else if (!phase->is_dominator(projs.fallthrough_catchproj, c)) {
1390 phase->igvn().rehash_node_delayed(u);
1391 int nb = u->replace_edge(n, create_phis_on_call_return(ctrl, c, n, n_clone, projs, phase));
1392 assert(nb > 0, "should have replaced some uses");
1393 replaced = true;
1394 }
1395 }
1396 if (!replaced) {
1397 stack.set_index(idx+1);
1398 }
1399 }
1400 } else {
1401 stack.pop();
1402 clones.pop();
1403 }
1404 } while (stack.size() > 0);
1405 assert(stack.size() == 0 && clones.size() == 0, "");
1406 }
1407 }
1408
1409 for (int i = 0; i < state->load_reference_barriers_count(); i++) {
1410 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1411 if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
1412 continue;
1413 }
1414 Node* ctrl = phase->get_ctrl(lrb);
1415 IdealLoopTree* loop = phase->get_loop(ctrl);
1416 if (loop->_head->is_OuterStripMinedLoop()) {
1417 // Expanding a barrier here will break loop strip mining
1418 // verification. Transform the loop so the loop nest doesn't
1419 // appear as strip mined.
1420 OuterStripMinedLoopNode* outer = loop->_head->as_OuterStripMinedLoop();
1421 hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
1422 }
1423 }
1424
1425 // Expand load-reference-barriers
1426 MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase);
1427 Unique_Node_List uses_to_ignore;
1428 for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
1429 ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
1430 if (lrb->get_barrier_strength() == ShenandoahLoadReferenceBarrierNode::NONE) {
1431 phase->igvn().replace_node(lrb, lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
1432 continue;
1433 }
1434 uint last = phase->C->unique();
1435 Node* ctrl = phase->get_ctrl(lrb);
1436 Node* val = lrb->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
1437
1438
1439 Node* orig_ctrl = ctrl;
1440
1441 Node* raw_mem = fixer.find_mem(ctrl, lrb);
1442 Node* init_raw_mem = raw_mem;
1443 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
1444
1445 IdealLoopTree *loop = phase->get_loop(ctrl);
1446 CallStaticJavaNode* unc = lrb->pin_and_expand_null_check(phase->igvn());
1447 Node* unc_ctrl = NULL;
1448 if (unc != NULL) {
1449 if (val->in(ShenandoahLoadReferenceBarrierNode::Control) != ctrl) {
1450 unc = NULL;
1451 } else {
1452 unc_ctrl = val->in(ShenandoahLoadReferenceBarrierNode::Control);
1453 }
1454 }
1455
1456 Node* uncasted_val = val;
1457 if (unc != NULL) {
1458 uncasted_val = val->in(1);
1459 }
1460
1461 Node* heap_stable_ctrl = NULL;
1462 Node* null_ctrl = NULL;
1463
2587 Node* u = n->fast_out(i);
2588 if (!u->is_Root() && u->is_CFG() && u != n) {
2589 Node* m = _memory_nodes[u->_idx];
2590 if (u->is_Region() && (!u->is_OuterStripMinedLoop() || _include_lsm) &&
2591 !has_mem_phi(u) &&
2592 u->unique_ctrl_out()->Opcode() != Op_Halt) {
2593 DEBUG_ONLY(if (trace) { tty->print("ZZZ region"); u->dump(); });
2594 DEBUG_ONLY(if (trace && m != NULL) { tty->print("ZZZ mem"); m->dump(); });
2595
2596 if (!mem_is_valid(m, u) || !m->is_Phi()) {
2597 bool push = true;
2598 bool create_phi = true;
2599 if (_phase->is_dominator(new_ctrl, u)) {
2600 create_phi = false;
2601 } else if (!_phase->C->has_irreducible_loop()) {
2602 IdealLoopTree* loop = _phase->get_loop(ctrl);
2603 bool do_check = true;
2604 IdealLoopTree* l = loop;
2605 create_phi = false;
2606 while (l != _phase->ltree_root()) {
2607 Node* head = l->_head;
2608 if (head->in(0) == NULL) {
2609 head = _phase->get_ctrl(head);
2610 }
2611 if (_phase->is_dominator(head, u) && _phase->is_dominator(_phase->idom(u), head)) {
2612 create_phi = true;
2613 do_check = false;
2614 break;
2615 }
2616 l = l->_parent;
2617 }
2618
2619 if (do_check) {
2620 assert(!create_phi, "");
2621 IdealLoopTree* u_loop = _phase->get_loop(u);
2622 if (u_loop != _phase->ltree_root() && u_loop->is_member(loop)) {
2623 Node* c = ctrl;
2624 while (!_phase->is_dominator(c, u_loop->tail())) {
2625 c = _phase->idom(c);
2626 }
2627 if (!_phase->is_dominator(c, u)) {
2628 do_check = false;
2629 }
2630 }
2631 }
|