564 if (iff->Opcode() != Op_If) {
565 return false;
566 }
567 Node* bol = iff->in(1);
568 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
569 return false;
570 }
571 Node* cmp = bol->in(1);
572 if (cmp->Opcode() != Op_CmpI) {
573 return false;
574 }
575 Node* in1 = cmp->in(1);
576 Node* in2 = cmp->in(2);
577 if (in2->find_int_con(-1) != 0) {
578 return false;
579 }
580 if (in1->Opcode() != Op_AndI) {
581 return false;
582 }
583 in2 = in1->in(2);
584 if (in2->find_int_con(-1) != (ShenandoahHeap::EVACUATION | ShenandoahHeap::PARTIAL | ShenandoahHeap::TRAVERSAL)) {
585 return false;
586 }
587 in1 = in1->in(1);
588
589 return is_gc_state_load(in1);
590 }
591
592 bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) {
593 if (!UseShenandoahGC) {
594 return false;
595 }
596 if (n->Opcode() != Op_LoadUB && n->Opcode() != Op_LoadB) {
597 return false;
598 }
599 Node* addp = n->in(MemNode::Address);
600 if (!addp->is_AddP()) {
601 return false;
602 }
603 Node* base = addp->in(AddPNode::Address);
604 Node* off = addp->in(AddPNode::Offset);
3591 #endif
3592 }
3593
3594 void ShenandoahWriteBarrierNode::test_evacuation_in_progress(Node* ctrl, int alias, Node*& raw_mem, Node*& wb_mem,
3595 IfNode*& evacuation_iff, Node*& evac_in_progress,
3596 Node*& evac_not_in_progress, PhaseIdealLoop* phase) {
3597 IdealLoopTree *loop = phase->get_loop(ctrl);
3598 Node* thread = new ThreadLocalNode();
3599 phase->register_new_node(thread, ctrl);
3600 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
3601 phase->set_ctrl(offset, phase->C->root());
3602 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
3603 phase->register_new_node(gc_state_addr, ctrl);
3604 uint gc_state_idx = Compile::AliasIdxRaw;
3605 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
3606 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
3607
3608 Node* gc_state = new LoadUBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
3609 phase->register_new_node(gc_state, ctrl);
3610
3611 Node* evacuation_in_progress = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::EVACUATION | ShenandoahHeap::PARTIAL | ShenandoahHeap::TRAVERSAL));
3612 phase->register_new_node(evacuation_in_progress, ctrl);
3613 Node* evacuation_in_progress_cmp = new CmpINode(evacuation_in_progress, phase->igvn().zerocon(T_INT));
3614 phase->register_new_node(evacuation_in_progress_cmp, ctrl);
3615 Node* evacuation_in_progress_test = new BoolNode(evacuation_in_progress_cmp, BoolTest::ne);
3616 phase->register_new_node(evacuation_in_progress_test, ctrl);
3617 evacuation_iff = new IfNode(ctrl, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
3618 phase->register_control(evacuation_iff, loop, ctrl);
3619
3620 assert(is_evacuation_in_progress_test(evacuation_iff), "Should match the shape");
3621 assert(is_gc_state_load(gc_state), "Should match the shape");
3622
3623 evac_not_in_progress = new IfFalseNode(evacuation_iff);
3624 phase->register_control(evac_not_in_progress, loop, evacuation_iff);
3625 evac_in_progress = new IfTrueNode(evacuation_iff);
3626 phase->register_control(evac_in_progress, loop, evacuation_iff);
3627 }
3628
3629 void ShenandoahWriteBarrierNode::evacuation_not_in_progress_null_check(Node*& c, Node*& val, Node* unc_ctrl, Node*& unc_region, PhaseIdealLoop* phase) {
3630 if (unc_ctrl != NULL) {
3631 // Clone the null check in this branch to allow implicit null check
|
564 if (iff->Opcode() != Op_If) {
565 return false;
566 }
567 Node* bol = iff->in(1);
568 if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {
569 return false;
570 }
571 Node* cmp = bol->in(1);
572 if (cmp->Opcode() != Op_CmpI) {
573 return false;
574 }
575 Node* in1 = cmp->in(1);
576 Node* in2 = cmp->in(2);
577 if (in2->find_int_con(-1) != 0) {
578 return false;
579 }
580 if (in1->Opcode() != Op_AndI) {
581 return false;
582 }
583 in2 = in1->in(2);
584 if (in2->find_int_con(-1) != (ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL)) {
585 return false;
586 }
587 in1 = in1->in(1);
588
589 return is_gc_state_load(in1);
590 }
591
592 bool ShenandoahWriteBarrierNode::is_gc_state_load(Node *n) {
593 if (!UseShenandoahGC) {
594 return false;
595 }
596 if (n->Opcode() != Op_LoadUB && n->Opcode() != Op_LoadB) {
597 return false;
598 }
599 Node* addp = n->in(MemNode::Address);
600 if (!addp->is_AddP()) {
601 return false;
602 }
603 Node* base = addp->in(AddPNode::Address);
604 Node* off = addp->in(AddPNode::Offset);
3591 #endif
3592 }
3593
3594 void ShenandoahWriteBarrierNode::test_evacuation_in_progress(Node* ctrl, int alias, Node*& raw_mem, Node*& wb_mem,
3595 IfNode*& evacuation_iff, Node*& evac_in_progress,
3596 Node*& evac_not_in_progress, PhaseIdealLoop* phase) {
3597 IdealLoopTree *loop = phase->get_loop(ctrl);
3598 Node* thread = new ThreadLocalNode();
3599 phase->register_new_node(thread, ctrl);
3600 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
3601 phase->set_ctrl(offset, phase->C->root());
3602 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
3603 phase->register_new_node(gc_state_addr, ctrl);
3604 uint gc_state_idx = Compile::AliasIdxRaw;
3605 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
3606 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
3607
3608 Node* gc_state = new LoadUBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
3609 phase->register_new_node(gc_state, ctrl);
3610
3611 Node* evacuation_in_progress = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL));
3612 phase->register_new_node(evacuation_in_progress, ctrl);
3613 Node* evacuation_in_progress_cmp = new CmpINode(evacuation_in_progress, phase->igvn().zerocon(T_INT));
3614 phase->register_new_node(evacuation_in_progress_cmp, ctrl);
3615 Node* evacuation_in_progress_test = new BoolNode(evacuation_in_progress_cmp, BoolTest::ne);
3616 phase->register_new_node(evacuation_in_progress_test, ctrl);
3617 evacuation_iff = new IfNode(ctrl, evacuation_in_progress_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
3618 phase->register_control(evacuation_iff, loop, ctrl);
3619
3620 assert(is_evacuation_in_progress_test(evacuation_iff), "Should match the shape");
3621 assert(is_gc_state_load(gc_state), "Should match the shape");
3622
3623 evac_not_in_progress = new IfFalseNode(evacuation_iff);
3624 phase->register_control(evac_not_in_progress, loop, evacuation_iff);
3625 evac_in_progress = new IfTrueNode(evacuation_iff);
3626 phase->register_control(evac_in_progress, loop, evacuation_iff);
3627 }
3628
3629 void ShenandoahWriteBarrierNode::evacuation_not_in_progress_null_check(Node*& c, Node*& val, Node* unc_ctrl, Node*& unc_region, PhaseIdealLoop* phase) {
3630 if (unc_ctrl != NULL) {
3631 // Clone the null check in this branch to allow implicit null check
|