843 void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
844 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
845 Node* u = n->fast_out(i);
846 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) {
847 uses.push(u);
848 }
849 }
850 }
851
852 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) {
853 OuterStripMinedLoopEndNode* le = inner->outer_loop_end();
854 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl));
855 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl));
856 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt);
857 phase->register_control(new_le, phase->get_loop(le), le->in(0));
858 phase->lazy_replace(outer, new_outer);
859 phase->lazy_replace(le, new_le);
860 inner->clear_strip_mined();
861 }
862
863 void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
864 PhaseIdealLoop* phase) {
865 IdealLoopTree* loop = phase->get_loop(ctrl);
866 Node* thread = new ThreadLocalNode();
867 phase->register_new_node(thread, ctrl);
868 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
869 phase->set_ctrl(offset, phase->C->root());
870 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
871 phase->register_new_node(gc_state_addr, ctrl);
872 uint gc_state_idx = Compile::AliasIdxRaw;
873 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
874 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
875
876 Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
877 phase->register_new_node(gc_state, ctrl);
878 Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED));
879 phase->register_new_node(heap_stable_and, ctrl);
880 Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
881 phase->register_new_node(heap_stable_cmp, ctrl);
882 Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne);
883 phase->register_new_node(heap_stable_test, ctrl);
884 IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
885 phase->register_control(heap_stable_iff, loop, ctrl);
886
887 heap_stable_ctrl = new IfFalseNode(heap_stable_iff);
888 phase->register_control(heap_stable_ctrl, loop, heap_stable_iff);
889 ctrl = new IfTrueNode(heap_stable_iff);
890 phase->register_control(ctrl, loop, heap_stable_iff);
891
892 assert(is_heap_stable_test(heap_stable_iff), "Should match the shape");
893 }
894
895 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
896 const Type* val_t = phase->igvn().type(val);
897 if (val_t->meet(TypePtr::NULL_PTR) == val_t) {
898 IdealLoopTree* loop = phase->get_loop(ctrl);
899 Node* null_cmp = new CmpPNode(val, phase->igvn().zerocon(T_OBJECT));
900 phase->register_new_node(null_cmp, ctrl);
901 Node* null_test = new BoolNode(null_cmp, BoolTest::ne);
902 phase->register_new_node(null_test, ctrl);
903 IfNode* null_iff = new IfNode(ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
904 phase->register_control(null_iff, loop, ctrl);
905 ctrl = new IfTrueNode(null_iff);
906 phase->register_control(ctrl, loop, null_iff);
907 null_ctrl = new IfFalseNode(null_iff);
908 phase->register_control(null_ctrl, loop, null_iff);
909 }
910 }
911
912 Node* ShenandoahBarrierC2Support::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {
1420 }
1421 }
1422
1423 Node* uncasted_val = val;
1424 if (unc != NULL) {
1425 uncasted_val = val->in(1);
1426 }
1427
1428 Node* heap_stable_ctrl = NULL;
1429 Node* null_ctrl = NULL;
1430
1431 assert(val->bottom_type()->make_oopptr(), "need oop");
1432 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1433
1434 enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT };
1435 Node* region = new RegionNode(PATH_LIMIT);
1436 Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1437 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1438
1439 // Stable path.
1440 test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1441 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1442
1443 // Heap stable case
1444 region->init_req(_heap_stable, heap_stable_ctrl);
1445 val_phi->init_req(_heap_stable, uncasted_val);
1446 raw_mem_phi->init_req(_heap_stable, raw_mem);
1447
1448 Node* reg2_ctrl = NULL;
1449 // Null case
1450 test_null(ctrl, val, null_ctrl, phase);
1451 if (null_ctrl != NULL) {
1452 reg2_ctrl = null_ctrl->in(0);
1453 region->init_req(_null_path, null_ctrl);
1454 val_phi->init_req(_null_path, uncasted_val);
1455 raw_mem_phi->init_req(_null_path, raw_mem);
1456 } else {
1457 region->del_req(_null_path);
1458 val_phi->del_req(_null_path);
1459 raw_mem_phi->del_req(_null_path);
1460 }
1591 }
1592
1593 Node* init_ctrl = ctrl;
1594 IdealLoopTree* loop = phase->get_loop(ctrl);
1595 Node* raw_mem = fixer.find_mem(ctrl, barrier);
1596 Node* init_raw_mem = raw_mem;
1597 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
1598 Node* heap_stable_ctrl = NULL;
1599 Node* null_ctrl = NULL;
1600 uint last = phase->C->unique();
1601
1602 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
1603 Node* region = new RegionNode(PATH_LIMIT);
1604 Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1605
1606 enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
1607 Node* region2 = new RegionNode(PATH_LIMIT2);
1608 Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1609
1610 // Stable path.
1611 test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
1612 region->init_req(_heap_stable, heap_stable_ctrl);
1613 phi->init_req(_heap_stable, raw_mem);
1614
1615 // Null path
1616 Node* reg2_ctrl = NULL;
1617 test_null(ctrl, pre_val, null_ctrl, phase);
1618 if (null_ctrl != NULL) {
1619 reg2_ctrl = null_ctrl->in(0);
1620 region2->init_req(_null_path, null_ctrl);
1621 phi2->init_req(_null_path, raw_mem);
1622 } else {
1623 region2->del_req(_null_path);
1624 phi2->del_req(_null_path);
1625 }
1626
1627 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1628 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1629 Node* thread = new ThreadLocalNode();
1630 phase->register_new_node(thread, ctrl);
1631 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
|
843 void ShenandoahBarrierC2Support::follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase) {
844 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
845 Node* u = n->fast_out(i);
846 if (!u->is_CFG() && phase->get_ctrl(u) == ctrl && (!u->is_Phi() || !u->in(0)->is_Loop() || u->in(LoopNode::LoopBackControl) != n)) {
847 uses.push(u);
848 }
849 }
850 }
851
852 static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNode* inner, PhaseIdealLoop* phase) {
853 OuterStripMinedLoopEndNode* le = inner->outer_loop_end();
854 Node* new_outer = new LoopNode(outer->in(LoopNode::EntryControl), outer->in(LoopNode::LoopBackControl));
855 phase->register_control(new_outer, phase->get_loop(outer), outer->in(LoopNode::EntryControl));
856 Node* new_le = new IfNode(le->in(0), le->in(1), le->_prob, le->_fcnt);
857 phase->register_control(new_le, phase->get_loop(le), le->in(0));
858 phase->lazy_replace(outer, new_outer);
859 phase->lazy_replace(le, new_le);
860 inner->clear_strip_mined();
861 }
862
863 void ShenandoahBarrierC2Support::test_heap_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
864 PhaseIdealLoop* phase, int flags) {
865 IdealLoopTree* loop = phase->get_loop(ctrl);
866 Node* thread = new ThreadLocalNode();
867 phase->register_new_node(thread, ctrl);
868 Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
869 phase->set_ctrl(offset, phase->C->root());
870 Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset);
871 phase->register_new_node(gc_state_addr, ctrl);
872 uint gc_state_idx = Compile::AliasIdxRaw;
873 const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
874 debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
875
876 Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
877 phase->register_new_node(gc_state, ctrl);
878 Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(flags));
879 phase->register_new_node(heap_stable_and, ctrl);
880 Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
881 phase->register_new_node(heap_stable_cmp, ctrl);
882 Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne);
883 phase->register_new_node(heap_stable_test, ctrl);
884 IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
885 phase->register_control(heap_stable_iff, loop, ctrl);
886
887 heap_stable_ctrl = new IfFalseNode(heap_stable_iff);
888 phase->register_control(heap_stable_ctrl, loop, heap_stable_iff);
889 ctrl = new IfTrueNode(heap_stable_iff);
890 phase->register_control(ctrl, loop, heap_stable_iff);
891
892 assert(is_heap_state_test(heap_stable_iff, flags), "Should match the shape");
893 }
894
895 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
896 const Type* val_t = phase->igvn().type(val);
897 if (val_t->meet(TypePtr::NULL_PTR) == val_t) {
898 IdealLoopTree* loop = phase->get_loop(ctrl);
899 Node* null_cmp = new CmpPNode(val, phase->igvn().zerocon(T_OBJECT));
900 phase->register_new_node(null_cmp, ctrl);
901 Node* null_test = new BoolNode(null_cmp, BoolTest::ne);
902 phase->register_new_node(null_test, ctrl);
903 IfNode* null_iff = new IfNode(ctrl, null_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
904 phase->register_control(null_iff, loop, ctrl);
905 ctrl = new IfTrueNode(null_iff);
906 phase->register_control(ctrl, loop, null_iff);
907 null_ctrl = new IfFalseNode(null_iff);
908 phase->register_control(null_ctrl, loop, null_iff);
909 }
910 }
911
912 Node* ShenandoahBarrierC2Support::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase) {
1420 }
1421 }
1422
1423 Node* uncasted_val = val;
1424 if (unc != NULL) {
1425 uncasted_val = val->in(1);
1426 }
1427
1428 Node* heap_stable_ctrl = NULL;
1429 Node* null_ctrl = NULL;
1430
1431 assert(val->bottom_type()->make_oopptr(), "need oop");
1432 assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant");
1433
1434 enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT };
1435 Node* region = new RegionNode(PATH_LIMIT);
1436 Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr());
1437 Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1438
1439 // Stable path.
1440 test_heap_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::HAS_FORWARDED);
1441 IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
1442
1443 // Heap stable case
1444 region->init_req(_heap_stable, heap_stable_ctrl);
1445 val_phi->init_req(_heap_stable, uncasted_val);
1446 raw_mem_phi->init_req(_heap_stable, raw_mem);
1447
1448 Node* reg2_ctrl = NULL;
1449 // Null case
1450 test_null(ctrl, val, null_ctrl, phase);
1451 if (null_ctrl != NULL) {
1452 reg2_ctrl = null_ctrl->in(0);
1453 region->init_req(_null_path, null_ctrl);
1454 val_phi->init_req(_null_path, uncasted_val);
1455 raw_mem_phi->init_req(_null_path, raw_mem);
1456 } else {
1457 region->del_req(_null_path);
1458 val_phi->del_req(_null_path);
1459 raw_mem_phi->del_req(_null_path);
1460 }
1591 }
1592
1593 Node* init_ctrl = ctrl;
1594 IdealLoopTree* loop = phase->get_loop(ctrl);
1595 Node* raw_mem = fixer.find_mem(ctrl, barrier);
1596 Node* init_raw_mem = raw_mem;
1597 Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL);
1598 Node* heap_stable_ctrl = NULL;
1599 Node* null_ctrl = NULL;
1600 uint last = phase->C->unique();
1601
1602 enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
1603 Node* region = new RegionNode(PATH_LIMIT);
1604 Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1605
1606 enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
1607 Node* region2 = new RegionNode(PATH_LIMIT2);
1608 Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
1609
1610 // Stable path.
1611 test_heap_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING);
1612 region->init_req(_heap_stable, heap_stable_ctrl);
1613 phi->init_req(_heap_stable, raw_mem);
1614
1615 // Null path
1616 Node* reg2_ctrl = NULL;
1617 test_null(ctrl, pre_val, null_ctrl, phase);
1618 if (null_ctrl != NULL) {
1619 reg2_ctrl = null_ctrl->in(0);
1620 region2->init_req(_null_path, null_ctrl);
1621 phi2->init_req(_null_path, raw_mem);
1622 } else {
1623 region2->del_req(_null_path);
1624 phi2->del_req(_null_path);
1625 }
1626
1627 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1628 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1629 Node* thread = new ThreadLocalNode();
1630 phase->register_new_node(thread, ctrl);
1631 Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
|