631 Node *m = wq.at(next);
632 if (m->is_SafePoint() && !m->is_CallLeaf()) {
633 return false;
634 }
635 if (m->is_Region()) {
636 for (uint i = 1; i < m->req(); i++) {
637 wq.push(m->in(i));
638 }
639 } else {
640 wq.push(m->in(0));
641 }
642 }
643 phase->igvn().replace_node(n, dominator);
644
645 return true;
646 }
647
648 Node* ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(Node* iff) {
649 assert(is_evacuation_in_progress_test(iff), "bad input");
650 Node* c = iff;
651 do {
652 assert(c->in(0)->is_Proj() && c->in(0)->in(0)->is_MemBar(), "where's the mem bar?");
653 c = c->in(0)->in(0);
654 } while (c->adr_type() != TypeRawPtr::BOTTOM);
655 return c->in(0);
656 }
657
658 bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase,
659 Node* b1,
660 Node* b2,
661 Node* current,
662 bool linear) {
663 ResourceMark rm;
664 VectorSet visited(Thread::current()->resource_area());
665 Node_Stack phis(0);
666
667
668 for(int i = 0; i < 10; i++) {
669 if (current == NULL) {
670 return false;
671 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
672 current = NULL;
673 while (phis.is_nonempty() && current == NULL) {
674 uint idx = phis.index();
3515 #endif
3516 }
3517
3518 void ShenandoahWriteBarrierNode::test_evacuation_in_progress(Node* ctrl, int alias, Node*& raw_mem, Node*& wb_mem,
3519 IfNode*& evacuation_iff, Node*& evac_in_progress,
3520 Node*& evac_not_in_progress, PhaseIdealLoop* phase) {
3521 IdealLoopTree *loop = phase->get_loop(ctrl);
3522 Node* thread = new ThreadLocalNode();
3523 phase->register_new_node(thread, ctrl);
3524 Node* offset = phase->igvn().MakeConX(in_bytes(JavaThread::gc_state_offset()));
3525 phase->set_ctrl(offset, phase->C->root());
3526 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
3527 phase->register_new_node(gc_state_addr, ctrl);
3528 uint gc_state_idx = Compile::AliasIdxRaw;
3529 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
3530 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
3531
3532 Node* gc_state = new LoadUBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
3533 phase->register_new_node(gc_state, ctrl);
3534
3535 Node* mb = MemBarNode::make(phase->C, Op_MemBarAcquire, Compile::AliasIdxRaw);
3536 mb->init_req(TypeFunc::Control, ctrl);
3537 mb->init_req(TypeFunc::Memory, raw_mem);
3538 phase->register_control(mb, loop, ctrl);
3539 Node* ctrl_proj = new ProjNode(mb,TypeFunc::Control);
3540 phase->register_control(ctrl_proj, loop, mb);
3541 raw_mem = new ProjNode(mb, TypeFunc::Memory);
3542 phase->register_new_node(raw_mem, mb);
3543
3544 mb = MemBarNode::make(phase->C, Op_MemBarAcquire, alias);
3545 mb->init_req(TypeFunc::Control, ctrl_proj);
3546 mb->init_req(TypeFunc::Memory, wb_mem);
3547 phase->register_control(mb, loop, ctrl_proj);
3548 ctrl_proj = new ProjNode(mb,TypeFunc::Control);
3549 phase->register_control(ctrl_proj, loop, mb);
3550 wb_mem = new ProjNode(mb,TypeFunc::Memory);
3551 phase->register_new_node(wb_mem, mb);
3552
3553 Node* evacuation_in_progress = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::EVACUATION));
3554 phase->register_new_node(evacuation_in_progress, ctrl_proj);
3555 Node* evacuation_in_progress_cmp = new CmpINode(evacuation_in_progress, phase->igvn().zerocon(T_INT));
3556 phase->register_new_node(evacuation_in_progress_cmp, ctrl_proj);
3557 Node* evacuation_in_progress_test = new BoolNode(evacuation_in_progress_cmp, BoolTest::ne);
3558 phase->register_new_node(evacuation_in_progress_test, ctrl_proj);
3559 evacuation_iff = new IfNode(ctrl_proj, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
3560 phase->register_control(evacuation_iff, loop, ctrl_proj);
3561
3562 assert(is_evacuation_in_progress_test(evacuation_iff), "Should match the shape");
3563 assert(is_gc_state_load(gc_state), "Should match the shape");
3564
3565 evac_not_in_progress = new IfFalseNode(evacuation_iff);
3566 phase->register_control(evac_not_in_progress, loop, evacuation_iff);
3567 evac_in_progress = new IfTrueNode(evacuation_iff);
3568 phase->register_control(evac_in_progress, loop, evacuation_iff);
3569 }
3570
3571 void ShenandoahWriteBarrierNode::evacuation_not_in_progress_null_check(Node*& c, Node*& val, Node* unc_ctrl, Node*& unc_region, PhaseIdealLoop* phase) {
3572 if (unc_ctrl != NULL) {
3573 // Clone the null check in this branch to allow implicit null check
3574 IdealLoopTree *loop = phase->get_loop(c);
3575 Node* iff = unc_ctrl->in(0);
3576 assert(iff->is_If(), "broken");
3577 Node* new_iff = iff->clone();
3578 new_iff->set_req(0, c);
3579 phase->register_control(new_iff, loop, c);
3580 Node* iffalse = new IfFalseNode(new_iff->as_If());
3954 follow_barrier_uses(n, ctrl, uses, phase);
3955 }
3956
3957 // The slow path call produces memory: hook the raw memory phi
3958 // from the expanded write barrier with the rest of the graph
3959 // which may require adding memory phis at every post dominated
3960 // region and at enclosing loop heads. Use the memory state
3961 // collected in memory_nodes to fix the memory graph. Update that
3962 // memory state as we go.
3963 fix_raw_mem(ctrl,region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, memory_nodes, uses, phase);
3964 assert(phase->C->shenandoah_barriers_count() == cnt - 1, "not replaced");
3965 }
3966
3967 assert(phase->C->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced");
3968 }
3969
3970 void ShenandoahWriteBarrierNode::move_evacuation_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
3971 // move test and its mem barriers out of the loop
3972 assert(is_evacuation_in_progress_test(iff), "inconsistent");
3973
3974 IdealLoopTree *loop = phase->get_loop(iff);
3975 Node* loop_head = loop->_head;
3976 Node* entry_c = loop_head->in(LoopNode::EntryControl);
3977 IdealLoopTree *entry_loop = phase->get_loop(entry_c);
3978
3979 GrowableArray<Node*> new_mbs;
3980 Node* c = iff->in(0);
3981 MemBarNode* mb = NULL;
3982 do {
3983 Node* proj_ctrl = c;
3984 assert(c->is_Proj(), "proj expected");
3985 mb = proj_ctrl->in(0)->as_MemBar();
3986 c = c->in(0)->in(0);
3987
3988 Node* proj_mem = mb->proj_out(TypeFunc::Memory);
3989
3990 MemBarNode* new_mb = mb->clone()->as_MemBar();;
3991 Node* new_proj_ctrl = new ProjNode(new_mb,TypeFunc::Control);
3992 Node* new_proj_mem = new ProjNode(new_mb,TypeFunc::Memory);
3993
4028 assert(c->is_Proj(), "proj expected");
4029 Node* proj_ctrl = c;
4030 MemBarNode* mb = proj_ctrl->in(0)->as_MemBar();
4031 Node* proj_mem = mb->proj_out(TypeFunc::Memory);
4032 Node* ctrl = mb->in(TypeFunc::Control);
4033 Node* mem = mb->in(TypeFunc::Memory);
4034
4035 phase->lazy_replace(proj_mem, mem);
4036 phase->lazy_replace(proj_ctrl, ctrl);
4037 phase->lazy_replace(mb, ctrl);
4038 loop->_body.yank(proj_ctrl);
4039 loop->_body.yank(proj_mem);
4040 loop->_body.yank(mb);
4041 if (mb->adr_type() == TypeRawPtr::BOTTOM) {
4042 break;
4043 }
4044 c = next;
4045 }
4046
4047 assert(phase->is_dominator(phase->get_ctrl(load->in(MemNode::Address)), entry_c), "address not out of loop?");
4048 }
4049
4050 void ShenandoahWriteBarrierNode::backtoback_evacs(IfNode* iff, IfNode* dom_if, PhaseIdealLoop* phase) {
4051 // move all mem barriers from this evac test to the dominating one,
4052 // removing duplicates in the process
4053 IdealLoopTree *loop = phase->get_loop(dom_if);
4054 Node* c1 = iff->in(0);
4055 Node* mb1 = NULL;
4056 GrowableArray<Node*> new_mbs;
4057 for(;;) {
4058 mb1 = c1->in(0);
4059 c1 = c1->in(0)->in(0);
4060 assert(mb1->Opcode() == Op_MemBarAcquire, "mem bar expected");
4061 if (mb1->adr_type() == TypeRawPtr::BOTTOM) {
4062 phase->lazy_replace(mb1->as_MemBar()->proj_out(TypeFunc::Memory), mb1->in(TypeFunc::Memory));
4063 break;
4064 }
4065 Node* c2 = dom_if->in(0);
4066 Node* mb2 = NULL;
4067 do {
4068 mb2 = c2->in(0);
4069 c2 = c2->in(0)->in(0);
4070 assert(mb2->Opcode() == Op_MemBarAcquire, "mem bar expected");
|
631 Node *m = wq.at(next);
632 if (m->is_SafePoint() && !m->is_CallLeaf()) {
633 return false;
634 }
635 if (m->is_Region()) {
636 for (uint i = 1; i < m->req(); i++) {
637 wq.push(m->in(i));
638 }
639 } else {
640 wq.push(m->in(0));
641 }
642 }
643 phase->igvn().replace_node(n, dominator);
644
645 return true;
646 }
647
648 Node* ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(Node* iff) {
649 assert(is_evacuation_in_progress_test(iff), "bad input");
650 Node* c = iff;
651 if (ShenandoahWriteBarrierMemBar) {
652 do {
653 assert(c->in(0)->is_Proj() && c->in(0)->in(0)->is_MemBar(), "where's the mem bar?");
654 c = c->in(0)->in(0);
655 } while (c->adr_type() != TypeRawPtr::BOTTOM);
656 }
657 return c->in(0);
658 }
659
660 bool ShenandoahBarrierNode::dominates_memory_impl(PhaseGVN* phase,
661 Node* b1,
662 Node* b2,
663 Node* current,
664 bool linear) {
665 ResourceMark rm;
666 VectorSet visited(Thread::current()->resource_area());
667 Node_Stack phis(0);
668
669
670 for(int i = 0; i < 10; i++) {
671 if (current == NULL) {
672 return false;
673 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
674 current = NULL;
675 while (phis.is_nonempty() && current == NULL) {
676 uint idx = phis.index();
3517 #endif
3518 }
3519
3520 void ShenandoahWriteBarrierNode::test_evacuation_in_progress(Node* ctrl, int alias, Node*& raw_mem, Node*& wb_mem,
3521 IfNode*& evacuation_iff, Node*& evac_in_progress,
3522 Node*& evac_not_in_progress, PhaseIdealLoop* phase) {
3523 IdealLoopTree *loop = phase->get_loop(ctrl);
3524 Node* thread = new ThreadLocalNode();
3525 phase->register_new_node(thread, ctrl);
3526 Node* offset = phase->igvn().MakeConX(in_bytes(JavaThread::gc_state_offset()));
3527 phase->set_ctrl(offset, phase->C->root());
3528 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
3529 phase->register_new_node(gc_state_addr, ctrl);
3530 uint gc_state_idx = Compile::AliasIdxRaw;
3531 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
3532 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
3533
3534 Node* gc_state = new LoadUBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
3535 phase->register_new_node(gc_state, ctrl);
3536
3537 if (ShenandoahWriteBarrierMemBar) {
3538 Node* mb = MemBarNode::make(phase->C, Op_MemBarAcquire, Compile::AliasIdxRaw);
3539 mb->init_req(TypeFunc::Control, ctrl);
3540 mb->init_req(TypeFunc::Memory, raw_mem);
3541 phase->register_control(mb, loop, ctrl);
3542 Node* ctrl_proj = new ProjNode(mb,TypeFunc::Control);
3543 phase->register_control(ctrl_proj, loop, mb);
3544 raw_mem = new ProjNode(mb, TypeFunc::Memory);
3545 phase->register_new_node(raw_mem, mb);
3546
3547 mb = MemBarNode::make(phase->C, Op_MemBarAcquire, alias);
3548 mb->init_req(TypeFunc::Control, ctrl_proj);
3549 mb->init_req(TypeFunc::Memory, wb_mem);
3550 phase->register_control(mb, loop, ctrl_proj);
3551 ctrl_proj = new ProjNode(mb,TypeFunc::Control);
3552 phase->register_control(ctrl_proj, loop, mb);
3553 wb_mem = new ProjNode(mb,TypeFunc::Memory);
3554 phase->register_new_node(wb_mem, mb);
3555
3556 ctrl = ctrl_proj;
3557 }
3558
3559 Node* evacuation_in_progress = gc_state;
3560 evacuation_in_progress = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::EVACUATION));
3561 phase->register_new_node(evacuation_in_progress, ctrl);
3562 Node* evacuation_in_progress_cmp = new CmpINode(evacuation_in_progress, phase->igvn().zerocon(T_INT));
3563 phase->register_new_node(evacuation_in_progress_cmp, ctrl);
3564 Node* evacuation_in_progress_test = new BoolNode(evacuation_in_progress_cmp, BoolTest::ne);
3565 phase->register_new_node(evacuation_in_progress_test, ctrl);
3566 evacuation_iff = new IfNode(ctrl, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
3567 phase->register_control(evacuation_iff, loop, ctrl);
3568
3569 assert(is_evacuation_in_progress_test(evacuation_iff), "Should match the shape");
3570 assert(is_gc_state_load(gc_state), "Should match the shape");
3571
3572 evac_not_in_progress = new IfFalseNode(evacuation_iff);
3573 phase->register_control(evac_not_in_progress, loop, evacuation_iff);
3574 evac_in_progress = new IfTrueNode(evacuation_iff);
3575 phase->register_control(evac_in_progress, loop, evacuation_iff);
3576 }
3577
3578 void ShenandoahWriteBarrierNode::evacuation_not_in_progress_null_check(Node*& c, Node*& val, Node* unc_ctrl, Node*& unc_region, PhaseIdealLoop* phase) {
3579 if (unc_ctrl != NULL) {
3580 // Clone the null check in this branch to allow implicit null check
3581 IdealLoopTree *loop = phase->get_loop(c);
3582 Node* iff = unc_ctrl->in(0);
3583 assert(iff->is_If(), "broken");
3584 Node* new_iff = iff->clone();
3585 new_iff->set_req(0, c);
3586 phase->register_control(new_iff, loop, c);
3587 Node* iffalse = new IfFalseNode(new_iff->as_If());
3961 follow_barrier_uses(n, ctrl, uses, phase);
3962 }
3963
3964 // The slow path call produces memory: hook the raw memory phi
3965 // from the expanded write barrier with the rest of the graph
3966 // which may require adding memory phis at every post dominated
3967 // region and at enclosing loop heads. Use the memory state
3968 // collected in memory_nodes to fix the memory graph. Update that
3969 // memory state as we go.
3970 fix_raw_mem(ctrl,region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, memory_nodes, uses, phase);
3971 assert(phase->C->shenandoah_barriers_count() == cnt - 1, "not replaced");
3972 }
3973
3974 assert(phase->C->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced");
3975 }
3976
3977 void ShenandoahWriteBarrierNode::move_evacuation_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
3978 // move test and its mem barriers out of the loop
3979 assert(is_evacuation_in_progress_test(iff), "inconsistent");
3980
3981 if (ShenandoahWriteBarrierMemBar) {
3982 IdealLoopTree *loop = phase->get_loop(iff);
3983 Node* loop_head = loop->_head;
3984 Node* entry_c = loop_head->in(LoopNode::EntryControl);
3985 IdealLoopTree *entry_loop = phase->get_loop(entry_c);
3986
3987 GrowableArray<Node*> new_mbs;
3988 Node* c = iff->in(0);
3989 MemBarNode* mb = NULL;
3990 do {
3991 Node* proj_ctrl = c;
3992 assert(c->is_Proj(), "proj expected");
3993 mb = proj_ctrl->in(0)->as_MemBar();
3994 c = c->in(0)->in(0);
3995
3996 Node* proj_mem = mb->proj_out(TypeFunc::Memory);
3997
3998 MemBarNode* new_mb = mb->clone()->as_MemBar();;
3999 Node* new_proj_ctrl = new ProjNode(new_mb,TypeFunc::Control);
4000 Node* new_proj_mem = new ProjNode(new_mb,TypeFunc::Memory);
4001
4036 assert(c->is_Proj(), "proj expected");
4037 Node* proj_ctrl = c;
4038 MemBarNode* mb = proj_ctrl->in(0)->as_MemBar();
4039 Node* proj_mem = mb->proj_out(TypeFunc::Memory);
4040 Node* ctrl = mb->in(TypeFunc::Control);
4041 Node* mem = mb->in(TypeFunc::Memory);
4042
4043 phase->lazy_replace(proj_mem, mem);
4044 phase->lazy_replace(proj_ctrl, ctrl);
4045 phase->lazy_replace(mb, ctrl);
4046 loop->_body.yank(proj_ctrl);
4047 loop->_body.yank(proj_mem);
4048 loop->_body.yank(mb);
4049 if (mb->adr_type() == TypeRawPtr::BOTTOM) {
4050 break;
4051 }
4052 c = next;
4053 }
4054
4055 assert(phase->is_dominator(phase->get_ctrl(load->in(MemNode::Address)), entry_c), "address not out of loop?");
4056 } else {
4057 IdealLoopTree *loop = phase->get_loop(iff);
4058 Node* loop_head = loop->_head;
4059 Node* entry_c = loop_head->in(LoopNode::EntryControl);
4060
4061 Node* load = iff->in(1)->in(1)->in(1);
4062 assert(load->Opcode() == Op_LoadUB, "inconsistent");
4063 Node* mem_ctrl = NULL;
4064 }
4065 }
4066
4067 void ShenandoahWriteBarrierNode::backtoback_evacs(IfNode* iff, IfNode* dom_if, PhaseIdealLoop* phase) {
4068 if (!ShenandoahWriteBarrierMemBar) {
4069 return;
4070 }
4071 // move all mem barriers from this evac test to the dominating one,
4072 // removing duplicates in the process
4073 IdealLoopTree *loop = phase->get_loop(dom_if);
4074 Node* c1 = iff->in(0);
4075 Node* mb1 = NULL;
4076 GrowableArray<Node*> new_mbs;
4077 for(;;) {
4078 mb1 = c1->in(0);
4079 c1 = c1->in(0)->in(0);
4080 assert(mb1->Opcode() == Op_MemBarAcquire, "mem bar expected");
4081 if (mb1->adr_type() == TypeRawPtr::BOTTOM) {
4082 phase->lazy_replace(mb1->as_MemBar()->proj_out(TypeFunc::Memory), mb1->in(TypeFunc::Memory));
4083 break;
4084 }
4085 Node* c2 = dom_if->in(0);
4086 Node* mb2 = NULL;
4087 do {
4088 mb2 = c2->in(0);
4089 c2 = c2->in(0)->in(0);
4090 assert(mb2->Opcode() == Op_MemBarAcquire, "mem bar expected");
|