3740                                                         Node* raw_mem_phi, Node* unc_region, int alias, Unique_Node_List& uses,
3741                                                         PhaseIdealLoop* phase) {
3742   evacuation_in_progress_null_check(c, val, evacuation_iff, unc, unc_ctrl, unc_region, uses, phase);
3743 
3744   IdealLoopTree *loop = phase->get_loop(c);
3745   Node* rbtrue = new ShenandoahReadBarrierNode(c, wb_mem, val);
3746   phase->register_new_node(rbtrue, c);
3747 
3748   Node* in_cset_fast_test_failure = NULL;
3749   in_cset_fast_test(c, rbtrue, raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, phase);
3750 
3751   // The slow path stub consumes and produces raw memory in addition
3752   // to the existing memory edges
3753   Node* base = find_bottom_mem(c, phase);
3754 
3755   MergeMemNode* mm = MergeMemNode::make(base);
3756   mm->set_memory_at(alias, wb_mem);
3757   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
3758   phase->register_new_node(mm, c);
3759 
3760   Node* call = new CallLeafNoFPNode(OptoRuntime::shenandoah_write_barrier_Type(), StubRoutines::shenandoah_wb_C(), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
3761   call->init_req(TypeFunc::Control, c);
3762   call->init_req(TypeFunc::I_O, phase->C->top());
3763   call->init_req(TypeFunc::Memory, mm);
3764   call->init_req(TypeFunc::FramePtr, phase->C->top());
3765   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
3766   call->init_req(TypeFunc::Parms, rbtrue);
3767   phase->register_control(call, loop, c);
3768   Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
3769   phase->register_control(ctrl_proj, loop, call);
3770   Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
3771   phase->register_new_node(mem_proj, call);
3772   Node* res_proj = new ProjNode(call, TypeFunc::Parms);
3773   phase->register_new_node(res_proj, call);
3774   Node* res = new CheckCastPPNode(ctrl_proj, res_proj, phase->igvn().type(val)->is_oopptr()->cast_to_nonconst());
3775   phase->register_new_node(res, ctrl_proj);
3776   region->init_req(2, ctrl_proj);
3777   val_phi->init_req(2, res);
3778   mem_phi->init_req(2, mem_proj);
3779   raw_mem_phi->init_req(2, mem_proj);
3780   phase->register_control(region, loop, evacuation_iff);

4222             }
4223             phase->do_unswitching(loop, old_new);
4224           }
4225         }
4226       }
4227     }
4228   }
4229 }
4230 
4231 #ifdef ASSERT
4232 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) {
4233   const bool trace = false;
4234   ResourceMark rm;
4235   Unique_Node_List nodes;
4236   Unique_Node_List controls;
4237   Unique_Node_List memories;
4238 
4239   nodes.push(root);
4240   for (uint next = 0; next < nodes.size(); next++) {
4241     Node *n  = nodes.at(next);
4242     if (n->Opcode() == Op_CallLeafNoFP && n->as_Call()->_entry_point == StubRoutines::shenandoah_wb_C()) {
4243       controls.push(n);
4244       if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
4245       for (uint next2 = 0; next2 < controls.size(); next2++) {
4246         Node *m = controls.at(next2);
4247         if (!m->is_Loop() || controls.member(m->in(LoopNode::EntryControl)) || 1) {
4248           for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
4249             Node* u = m->fast_out(i);
4250             if (u->is_CFG() && !u->is_Root()) {
4251               if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
4252               controls.push(u);
4253             }
4254           }
4255         }
4256       }
4257       memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
4258       for (uint next2 = 0; next2 < memories.size(); next2++) {
4259         Node *m = memories.at(next2);
4260         assert(m->bottom_type() == Type::MEMORY, "");
4261         if (!m->is_Phi() || !m->in(0)->is_Loop() || controls.member(m->in(0)->in(LoopNode::EntryControl)) || 1) {
4262           for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {

3740                                                         Node* raw_mem_phi, Node* unc_region, int alias, Unique_Node_List& uses,
3741                                                         PhaseIdealLoop* phase) {
3742   evacuation_in_progress_null_check(c, val, evacuation_iff, unc, unc_ctrl, unc_region, uses, phase);
3743 
3744   IdealLoopTree *loop = phase->get_loop(c);
3745   Node* rbtrue = new ShenandoahReadBarrierNode(c, wb_mem, val);
3746   phase->register_new_node(rbtrue, c);
3747 
3748   Node* in_cset_fast_test_failure = NULL;
3749   in_cset_fast_test(c, rbtrue, raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, phase);
3750 
3751   // The slow path stub consumes and produces raw memory in addition
3752   // to the existing memory edges
3753   Node* base = find_bottom_mem(c, phase);
3754 
3755   MergeMemNode* mm = MergeMemNode::make(base);
3756   mm->set_memory_at(alias, wb_mem);
3757   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
3758   phase->register_new_node(mm, c);
3759 
3760   Node* call = new CallLeafNode(OptoRuntime::shenandoah_write_barrier_Type(), CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_barrier_JRT), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
3761   call->init_req(TypeFunc::Control, c);
3762   call->init_req(TypeFunc::I_O, phase->C->top());
3763   call->init_req(TypeFunc::Memory, mm);
3764   call->init_req(TypeFunc::FramePtr, phase->C->top());
3765   call->init_req(TypeFunc::ReturnAdr, phase->C->top());
3766   call->init_req(TypeFunc::Parms, rbtrue);
3767   phase->register_control(call, loop, c);
3768   Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
3769   phase->register_control(ctrl_proj, loop, call);
3770   Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
3771   phase->register_new_node(mem_proj, call);
3772   Node* res_proj = new ProjNode(call, TypeFunc::Parms);
3773   phase->register_new_node(res_proj, call);
3774   Node* res = new CheckCastPPNode(ctrl_proj, res_proj, phase->igvn().type(val)->is_oopptr()->cast_to_nonconst());
3775   phase->register_new_node(res, ctrl_proj);
3776   region->init_req(2, ctrl_proj);
3777   val_phi->init_req(2, res);
3778   mem_phi->init_req(2, mem_proj);
3779   raw_mem_phi->init_req(2, mem_proj);
3780   phase->register_control(region, loop, evacuation_iff);

4222             }
4223             phase->do_unswitching(loop, old_new);
4224           }
4225         }
4226       }
4227     }
4228   }
4229 }
4230 
4231 #ifdef ASSERT
4232 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) {
4233   const bool trace = false;
4234   ResourceMark rm;
4235   Unique_Node_List nodes;
4236   Unique_Node_List controls;
4237   Unique_Node_List memories;
4238 
4239   nodes.push(root);
4240   for (uint next = 0; next < nodes.size(); next++) {
4241     Node *n  = nodes.at(next);
4242     if (n->Opcode() == Op_CallLeafNoFP && n->as_Call()->_entry_point == CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_barrier_JRT)) {
4243       controls.push(n);
4244       if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
4245       for (uint next2 = 0; next2 < controls.size(); next2++) {
4246         Node *m = controls.at(next2);
4247         if (!m->is_Loop() || controls.member(m->in(LoopNode::EntryControl)) || 1) {
4248           for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
4249             Node* u = m->fast_out(i);
4250             if (u->is_CFG() && !u->is_Root()) {
4251               if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
4252               controls.push(u);
4253             }
4254           }
4255         }
4256       }
4257       memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
4258       for (uint next2 = 0; next2 < memories.size(); next2++) {
4259         Node *m = memories.at(next2);
4260         assert(m->bottom_type() == Type::MEMORY, "");
4261         if (!m->is_Phi() || !m->in(0)->is_Loop() || controls.member(m->in(0)->in(LoopNode::EntryControl)) || 1) {
4262           for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {