1076 phase->register_control(in_cset_fast_test_iff, loop, ctrl);
1077
1078 not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
1079 phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
1080
1081 ctrl = new IfFalseNode(in_cset_fast_test_iff);
1082 phase->register_control(ctrl, loop, in_cset_fast_test_iff);
1083 }
1084
1085 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) {
1086 IdealLoopTree*loop = phase->get_loop(ctrl);
1087 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
1088
1089 // The slow path stub consumes and produces raw memory in addition
1090 // to the existing memory edges
1091 Node* base = find_bottom_mem(ctrl, phase);
1092 MergeMemNode* mm = MergeMemNode::make(base);
1093 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1094 phase->register_new_node(mm, ctrl);
1095
1096 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
1097 call->init_req(TypeFunc::Control, ctrl);
1098 call->init_req(TypeFunc::I_O, phase->C->top());
1099 call->init_req(TypeFunc::Memory, mm);
1100 call->init_req(TypeFunc::FramePtr, phase->C->top());
1101 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1102 call->init_req(TypeFunc::Parms, val);
1103 phase->register_control(call, loop, ctrl);
1104 ctrl = new ProjNode(call, TypeFunc::Control);
1105 phase->register_control(ctrl, loop, call);
1106 result_mem = new ProjNode(call, TypeFunc::Memory);
1107 phase->register_new_node(result_mem, call);
1108 val = new ProjNode(call, TypeFunc::Parms);
1109 phase->register_new_node(val, call);
1110 val = new CheckCastPPNode(ctrl, val, obj_type);
1111 phase->register_new_node(val, ctrl);
1112 }
1113
1114 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
1115 Node* ctrl = phase->get_ctrl(barrier);
1116 Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
1518
1519 // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
1520 Node* cmp = new CmpPNode(fwd, new_val);
1521 phase->register_new_node(cmp, ctrl);
1522 Node* bol = new BoolNode(cmp, BoolTest::eq);
1523 phase->register_new_node(bol, ctrl);
1524
1525 IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1526 if (reg2_ctrl == NULL) reg2_ctrl = iff;
1527 phase->register_control(iff, loop, ctrl);
1528 Node* if_not_eq = new IfFalseNode(iff);
1529 phase->register_control(if_not_eq, loop, iff);
1530 Node* if_eq = new IfTrueNode(iff);
1531 phase->register_control(if_eq, loop, iff);
1532
1533 // Wire up not-equal-path in slots 3.
1534 region->init_req(_not_equal, if_not_eq);
1535 val_phi->init_req(_not_equal, fwd);
1536 raw_mem_phi->init_req(_not_equal, raw_mem);
1537
1538 // Call wb-stub and wire up that path in slots 4
1539 Node* result_mem = NULL;
1540 ctrl = if_eq;
1541 call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase);
1542 region->init_req(_evac_path, ctrl);
1543 val_phi->init_req(_evac_path, fwd);
1544 raw_mem_phi->init_req(_evac_path, result_mem);
1545
1546 phase->register_control(region, loop, heap_stable_iff);
1547 Node* out_val = val_phi;
1548 phase->register_new_node(val_phi, region);
1549 phase->register_new_node(raw_mem_phi, region);
1550
1551 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1552
1553 ctrl = orig_ctrl;
1554
1555 if (unc != NULL) {
1556 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1557 Node* u = val->fast_out(i);
1558 Node* c = phase->ctrl_or_self(u);
1962 phase->igvn().replace_input_of(iff, 1, bol);
1963 }
1964 }
1965 }
1966 }
1967 }
1968 }
1969 }
1970
1971 #ifdef ASSERT
1972 void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) {
1973 const bool trace = false;
1974 ResourceMark rm;
1975 Unique_Node_List nodes;
1976 Unique_Node_List controls;
1977 Unique_Node_List memories;
1978
1979 nodes.push(root);
1980 for (uint next = 0; next < nodes.size(); next++) {
1981 Node *n = nodes.at(next);
1982 if (ShenandoahBarrierSetC2::is_shenandoah_wb_call(n)) {
1983 controls.push(n);
1984 if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
1985 for (uint next2 = 0; next2 < controls.size(); next2++) {
1986 Node *m = controls.at(next2);
1987 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
1988 Node* u = m->fast_out(i);
1989 if (u->is_CFG() && !u->is_Root() &&
1990 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
1991 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
1992 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
1993 controls.push(u);
1994 }
1995 }
1996 }
1997 memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
1998 for (uint next2 = 0; next2 < memories.size(); next2++) {
1999 Node *m = memories.at(next2);
2000 assert(m->bottom_type() == Type::MEMORY, "");
2001 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2002 Node* u = m->fast_out(i);
|
1076 phase->register_control(in_cset_fast_test_iff, loop, ctrl);
1077
1078 not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
1079 phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
1080
1081 ctrl = new IfFalseNode(in_cset_fast_test_iff);
1082 phase->register_control(ctrl, loop, in_cset_fast_test_iff);
1083 }
1084
1085 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, PhaseIdealLoop* phase) {
1086 IdealLoopTree*loop = phase->get_loop(ctrl);
1087 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
1088
1089 // The slow path stub consumes and produces raw memory in addition
1090 // to the existing memory edges
1091 Node* base = find_bottom_mem(ctrl, phase);
1092 MergeMemNode* mm = MergeMemNode::make(base);
1093 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1094 phase->register_new_node(mm, ctrl);
1095
1096 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT), "shenandoah_load_reference_barrier", TypeRawPtr::BOTTOM);
1097 call->init_req(TypeFunc::Control, ctrl);
1098 call->init_req(TypeFunc::I_O, phase->C->top());
1099 call->init_req(TypeFunc::Memory, mm);
1100 call->init_req(TypeFunc::FramePtr, phase->C->top());
1101 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
1102 call->init_req(TypeFunc::Parms, val);
1103 phase->register_control(call, loop, ctrl);
1104 ctrl = new ProjNode(call, TypeFunc::Control);
1105 phase->register_control(ctrl, loop, call);
1106 result_mem = new ProjNode(call, TypeFunc::Memory);
1107 phase->register_new_node(result_mem, call);
1108 val = new ProjNode(call, TypeFunc::Parms);
1109 phase->register_new_node(val, call);
1110 val = new CheckCastPPNode(ctrl, val, obj_type);
1111 phase->register_new_node(val, ctrl);
1112 }
1113
1114 void ShenandoahBarrierC2Support::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
1115 Node* ctrl = phase->get_ctrl(barrier);
1116 Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
1518
1519 // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr
1520 Node* cmp = new CmpPNode(fwd, new_val);
1521 phase->register_new_node(cmp, ctrl);
1522 Node* bol = new BoolNode(cmp, BoolTest::eq);
1523 phase->register_new_node(bol, ctrl);
1524
1525 IfNode* iff = new IfNode(ctrl, bol, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1526 if (reg2_ctrl == NULL) reg2_ctrl = iff;
1527 phase->register_control(iff, loop, ctrl);
1528 Node* if_not_eq = new IfFalseNode(iff);
1529 phase->register_control(if_not_eq, loop, iff);
1530 Node* if_eq = new IfTrueNode(iff);
1531 phase->register_control(if_eq, loop, iff);
1532
1533 // Wire up not-equal-path in slots 3.
1534 region->init_req(_not_equal, if_not_eq);
1535 val_phi->init_req(_not_equal, fwd);
1536 raw_mem_phi->init_req(_not_equal, raw_mem);
1537
1538 // Call lrb-stub and wire up that path in slots 4
1539 Node* result_mem = NULL;
1540 ctrl = if_eq;
1541 call_lrb_stub(ctrl, fwd, result_mem, raw_mem, phase);
1542 region->init_req(_evac_path, ctrl);
1543 val_phi->init_req(_evac_path, fwd);
1544 raw_mem_phi->init_req(_evac_path, result_mem);
1545
1546 phase->register_control(region, loop, heap_stable_iff);
1547 Node* out_val = val_phi;
1548 phase->register_new_node(val_phi, region);
1549 phase->register_new_node(raw_mem_phi, region);
1550
1551 fix_ctrl(lrb, region, fixer, uses, uses_to_ignore, last, phase);
1552
1553 ctrl = orig_ctrl;
1554
1555 if (unc != NULL) {
1556 for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
1557 Node* u = val->fast_out(i);
1558 Node* c = phase->ctrl_or_self(u);
1962 phase->igvn().replace_input_of(iff, 1, bol);
1963 }
1964 }
1965 }
1966 }
1967 }
1968 }
1969 }
1970
1971 #ifdef ASSERT
1972 void ShenandoahBarrierC2Support::verify_raw_mem(RootNode* root) {
1973 const bool trace = false;
1974 ResourceMark rm;
1975 Unique_Node_List nodes;
1976 Unique_Node_List controls;
1977 Unique_Node_List memories;
1978
1979 nodes.push(root);
1980 for (uint next = 0; next < nodes.size(); next++) {
1981 Node *n = nodes.at(next);
1982 if (ShenandoahBarrierSetC2::is_shenandoah_lrb_call(n)) {
1983 controls.push(n);
1984 if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
1985 for (uint next2 = 0; next2 < controls.size(); next2++) {
1986 Node *m = controls.at(next2);
1987 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
1988 Node* u = m->fast_out(i);
1989 if (u->is_CFG() && !u->is_Root() &&
1990 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
1991 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
1992 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
1993 controls.push(u);
1994 }
1995 }
1996 }
1997 memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
1998 for (uint next2 = 0; next2 < memories.size(); next2++) {
1999 Node *m = memories.at(next2);
2000 assert(m->bottom_type() == Type::MEMORY, "");
2001 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2002 Node* u = m->fast_out(i);
|