2413 ctrl = new IfFalseNode(in_cset_fast_test_iff);
2414 phase->register_control(ctrl, loop, in_cset_fast_test_iff);
2415 }
2416
2417 void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem,
2418 Node* raw_mem, Node* wb_mem,
2419 int alias,
2420 PhaseIdealLoop* phase) {
2421 IdealLoopTree*loop = phase->get_loop(ctrl);
2422 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
2423
2424 // The slow path stub consumes and produces raw memory in addition
2425 // to the existing memory edges
2426 Node* base = find_bottom_mem(ctrl, phase);
2427
2428 MergeMemNode* mm = MergeMemNode::make(base);
2429 mm->set_memory_at(alias, wb_mem);
2430 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
2431 phase->register_new_node(mm, ctrl);
2432
2433 Node* call = new CallLeafNoFPNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), ShenandoahBarrierSetAssembler::shenandoah_wb_C(), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
2434 call->init_req(TypeFunc::Control, ctrl);
2435 call->init_req(TypeFunc::I_O, phase->C->top());
2436 call->init_req(TypeFunc::Memory, mm);
2437 call->init_req(TypeFunc::FramePtr, phase->C->top());
2438 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
2439 call->init_req(TypeFunc::Parms, val);
2440 phase->register_control(call, loop, ctrl);
2441 ctrl = new ProjNode(call, TypeFunc::Control);
2442 phase->register_control(ctrl, loop, call);
2443 result_mem = new ProjNode(call, TypeFunc::Memory);
2444 phase->register_new_node(result_mem, call);
2445 val = new ProjNode(call, TypeFunc::Parms);
2446 phase->register_new_node(val, call);
2447 val = new CheckCastPPNode(ctrl, val, obj_type);
2448 phase->register_new_node(val, ctrl);
2449 }
2450
2451 void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
2452 Node* ctrl = phase->get_ctrl(barrier);
2453 Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
3138 phase->igvn().replace_input_of(iff, 1, bol);
3139 }
3140 }
3141 }
3142 }
3143 }
3144 }
3145 }
3146
3147 #ifdef ASSERT
3148 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) {
3149 const bool trace = false;
3150 ResourceMark rm;
3151 Unique_Node_List nodes;
3152 Unique_Node_List controls;
3153 Unique_Node_List memories;
3154
3155 nodes.push(root);
3156 for (uint next = 0; next < nodes.size(); next++) {
3157 Node *n = nodes.at(next);
3158 if (n->Opcode() == Op_CallLeafNoFP &&
3159 ShenandoahBarrierSetAssembler::is_shenandoah_wb_C_call(n->as_Call()->entry_point())) {
3160 controls.push(n);
3161 if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
3162 for (uint next2 = 0; next2 < controls.size(); next2++) {
3163 Node *m = controls.at(next2);
3164 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
3165 Node* u = m->fast_out(i);
3166 if (u->is_CFG() && !u->is_Root() &&
3167 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
3168 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
3169 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
3170 controls.push(u);
3171 }
3172 }
3173 }
3174 memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
3175 for (uint next2 = 0; next2 < memories.size(); next2++) {
3176 Node *m = memories.at(next2);
3177 assert(m->bottom_type() == Type::MEMORY, "");
3178 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
3179 Node* u = m->fast_out(i);
|
2413 ctrl = new IfFalseNode(in_cset_fast_test_iff);
2414 phase->register_control(ctrl, loop, in_cset_fast_test_iff);
2415 }
2416
2417 void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem,
2418 Node* raw_mem, Node* wb_mem,
2419 int alias,
2420 PhaseIdealLoop* phase) {
2421 IdealLoopTree*loop = phase->get_loop(ctrl);
2422 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
2423
2424 // The slow path stub consumes and produces raw memory in addition
2425 // to the existing memory edges
2426 Node* base = find_bottom_mem(ctrl, phase);
2427
2428 MergeMemNode* mm = MergeMemNode::make(base);
2429 mm->set_memory_at(alias, wb_mem);
2430 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
2431 phase->register_new_node(mm, ctrl);
2432
2433 Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
2434 call->init_req(TypeFunc::Control, ctrl);
2435 call->init_req(TypeFunc::I_O, phase->C->top());
2436 call->init_req(TypeFunc::Memory, mm);
2437 call->init_req(TypeFunc::FramePtr, phase->C->top());
2438 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
2439 call->init_req(TypeFunc::Parms, val);
2440 phase->register_control(call, loop, ctrl);
2441 ctrl = new ProjNode(call, TypeFunc::Control);
2442 phase->register_control(ctrl, loop, call);
2443 result_mem = new ProjNode(call, TypeFunc::Memory);
2444 phase->register_new_node(result_mem, call);
2445 val = new ProjNode(call, TypeFunc::Parms);
2446 phase->register_new_node(val, call);
2447 val = new CheckCastPPNode(ctrl, val, obj_type);
2448 phase->register_new_node(val, ctrl);
2449 }
2450
2451 void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) {
2452 Node* ctrl = phase->get_ctrl(barrier);
2453 Node* init_raw_mem = fixer.find_mem(ctrl, barrier);
3138 phase->igvn().replace_input_of(iff, 1, bol);
3139 }
3140 }
3141 }
3142 }
3143 }
3144 }
3145 }
3146
3147 #ifdef ASSERT
3148 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) {
3149 const bool trace = false;
3150 ResourceMark rm;
3151 Unique_Node_List nodes;
3152 Unique_Node_List controls;
3153 Unique_Node_List memories;
3154
3155 nodes.push(root);
3156 for (uint next = 0; next < nodes.size(); next++) {
3157 Node *n = nodes.at(next);
3158 if (ShenandoahBarrierSetC2::is_shenandoah_wb_call(n)) {
3159 controls.push(n);
3160 if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
3161 for (uint next2 = 0; next2 < controls.size(); next2++) {
3162 Node *m = controls.at(next2);
3163 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
3164 Node* u = m->fast_out(i);
3165 if (u->is_CFG() && !u->is_Root() &&
3166 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
3167 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
3168 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
3169 controls.push(u);
3170 }
3171 }
3172 }
3173 memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
3174 for (uint next2 = 0; next2 < memories.size(); next2++) {
3175 Node *m = memories.at(next2);
3176 assert(m->bottom_type() == Type::MEMORY, "");
3177 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
3178 Node* u = m->fast_out(i);
|