656 sibling->in(Control) == in(Control) &&
657 dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) {
658 return sibling;
659 }
660 }
661 }
662 return id;
663 }
664
665 const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const {
666 // Either input is TOP ==> the result is TOP
667 const Type *t1 = phase->type(in(Memory));
668 if (t1 == Type::TOP) return Type::TOP;
669 const Type *t2 = phase->type(in(ValueIn));
670 if( t2 == Type::TOP ) return Type::TOP;
671
672 if (t2 == TypePtr::NULL_PTR) {
673 return _type;
674 }
675
676 const Type* type = t2->is_oopptr()->cast_to_nonconst();
677 return type;
678 }
679
680 uint ShenandoahBarrierNode::hash() const {
681 return TypeNode::hash() + _allow_fromspace;
682 }
683
684 uint ShenandoahBarrierNode::cmp(const Node& n) const {
685 return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace
686 && TypeNode::cmp(n);
687 }
688
689 uint ShenandoahBarrierNode::size_of() const {
690 return sizeof(*this);
691 }
692
693 Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) {
694 Node* wb = in(WriteBarrier);
695 if (wb->is_top()) return phase->C->top(); // Dead path.
696
697 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
2404 Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
2405 phase->register_new_node(in_cset_fast_test_cmp, ctrl);
2406 Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
2407 phase->register_new_node(in_cset_fast_test_test, ctrl);
2408 IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
2409 phase->register_control(in_cset_fast_test_iff, loop, ctrl);
2410
2411 not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
2412 phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
2413
2414 ctrl = new IfFalseNode(in_cset_fast_test_iff);
2415 phase->register_control(ctrl, loop, in_cset_fast_test_iff);
2416 }
2417 }
2418
2419 void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem,
2420 Node* raw_mem, Node* wb_mem,
2421 int alias,
2422 PhaseIdealLoop* phase) {
2423 IdealLoopTree*loop = phase->get_loop(ctrl);
2424 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
2425
2426 // The slow path stub consumes and produces raw memory in addition
2427 // to the existing memory edges
2428 Node* base = find_bottom_mem(ctrl, phase);
2429
2430 MergeMemNode* mm = MergeMemNode::make(base);
2431 mm->set_memory_at(alias, wb_mem);
2432 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
2433 phase->register_new_node(mm, ctrl);
2434
2435 Node* call = new CallLeafNoFPNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), ShenandoahBarrierSetAssembler::shenandoah_wb_C(), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
2436 call->init_req(TypeFunc::Control, ctrl);
2437 call->init_req(TypeFunc::I_O, phase->C->top());
2438 call->init_req(TypeFunc::Memory, mm);
2439 call->init_req(TypeFunc::FramePtr, phase->C->top());
2440 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
2441 call->init_req(TypeFunc::Parms, val);
2442 phase->register_control(call, loop, ctrl);
2443 ctrl = new ProjNode(call, TypeFunc::Control);
2444 phase->register_control(ctrl, loop, call);
3131 memories.clear();
3132 }
3133 for( uint i = 0; i < n->len(); ++i ) {
3134 Node *m = n->in(i);
3135 if (m != NULL) {
3136 nodes.push(m);
3137 }
3138 }
3139 }
3140 }
3141 #endif
3142
3143 const Type* ShenandoahEnqueueBarrierNode::bottom_type() const {
3144 if (in(1) == NULL || in(1)->is_top()) {
3145 return Type::TOP;
3146 }
3147 const Type* t = in(1)->bottom_type();
3148 if (t == TypePtr::NULL_PTR) {
3149 return t;
3150 }
3151 return t->is_oopptr()->cast_to_nonconst();
3152 }
3153
3154 const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const {
3155 if (in(1) == NULL) {
3156 return Type::TOP;
3157 }
3158 const Type* t = phase->type(in(1));
3159 if (t == Type::TOP) {
3160 return Type::TOP;
3161 }
3162 if (t == TypePtr::NULL_PTR) {
3163 return t;
3164 }
3165 return t->is_oopptr()->cast_to_nonconst();
3166 }
3167
3168 int ShenandoahEnqueueBarrierNode::needed(Node* n) {
3169 if (n == NULL ||
3170 n->is_Allocate() ||
3171 n->bottom_type() == TypePtr::NULL_PTR ||
3172 (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL)) {
3173 return NotNeeded;
3174 }
3175 if (n->is_Phi() ||
3176 n->is_CMove()) {
3177 return MaybeNeeded;
3178 }
3179 return Needed;
3180 }
3181
3182 Node* ShenandoahEnqueueBarrierNode::next(Node* n) {
3183 for (;;) {
3184 if (n == NULL) {
3185 return n;
|
656 sibling->in(Control) == in(Control) &&
657 dominates_memory_rb(phase, sibling, this, phase->is_IterGVN() == NULL)) {
658 return sibling;
659 }
660 }
661 }
662 return id;
663 }
664
665 const Type* ShenandoahBarrierNode::Value(PhaseGVN* phase) const {
666 // Either input is TOP ==> the result is TOP
667 const Type *t1 = phase->type(in(Memory));
668 if (t1 == Type::TOP) return Type::TOP;
669 const Type *t2 = phase->type(in(ValueIn));
670 if( t2 == Type::TOP ) return Type::TOP;
671
672 if (t2 == TypePtr::NULL_PTR) {
673 return _type;
674 }
675
676 return t2;
677 }
678
679 uint ShenandoahBarrierNode::hash() const {
680 return TypeNode::hash() + _allow_fromspace;
681 }
682
683 uint ShenandoahBarrierNode::cmp(const Node& n) const {
684 return _allow_fromspace == ((ShenandoahBarrierNode&) n)._allow_fromspace
685 && TypeNode::cmp(n);
686 }
687
688 uint ShenandoahBarrierNode::size_of() const {
689 return sizeof(*this);
690 }
691
692 Node* ShenandoahWBMemProjNode::Identity(PhaseGVN* phase) {
693 Node* wb = in(WriteBarrier);
694 if (wb->is_top()) return phase->C->top(); // Dead path.
695
696 assert(wb->Opcode() == Op_ShenandoahWriteBarrier, "expect write barrier");
2403 Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
2404 phase->register_new_node(in_cset_fast_test_cmp, ctrl);
2405 Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
2406 phase->register_new_node(in_cset_fast_test_test, ctrl);
2407 IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
2408 phase->register_control(in_cset_fast_test_iff, loop, ctrl);
2409
2410 not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
2411 phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
2412
2413 ctrl = new IfFalseNode(in_cset_fast_test_iff);
2414 phase->register_control(ctrl, loop, in_cset_fast_test_iff);
2415 }
2416 }
2417
2418 void ShenandoahWriteBarrierNode::call_wb_stub(Node*& ctrl, Node*& val, Node*& result_mem,
2419 Node* raw_mem, Node* wb_mem,
2420 int alias,
2421 PhaseIdealLoop* phase) {
2422 IdealLoopTree*loop = phase->get_loop(ctrl);
2423 const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr();
2424
2425 // The slow path stub consumes and produces raw memory in addition
2426 // to the existing memory edges
2427 Node* base = find_bottom_mem(ctrl, phase);
2428
2429 MergeMemNode* mm = MergeMemNode::make(base);
2430 mm->set_memory_at(alias, wb_mem);
2431 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
2432 phase->register_new_node(mm, ctrl);
2433
2434 Node* call = new CallLeafNoFPNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), ShenandoahBarrierSetAssembler::shenandoah_wb_C(), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
2435 call->init_req(TypeFunc::Control, ctrl);
2436 call->init_req(TypeFunc::I_O, phase->C->top());
2437 call->init_req(TypeFunc::Memory, mm);
2438 call->init_req(TypeFunc::FramePtr, phase->C->top());
2439 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
2440 call->init_req(TypeFunc::Parms, val);
2441 phase->register_control(call, loop, ctrl);
2442 ctrl = new ProjNode(call, TypeFunc::Control);
2443 phase->register_control(ctrl, loop, call);
3130 memories.clear();
3131 }
3132 for( uint i = 0; i < n->len(); ++i ) {
3133 Node *m = n->in(i);
3134 if (m != NULL) {
3135 nodes.push(m);
3136 }
3137 }
3138 }
3139 }
3140 #endif
3141
3142 const Type* ShenandoahEnqueueBarrierNode::bottom_type() const {
3143 if (in(1) == NULL || in(1)->is_top()) {
3144 return Type::TOP;
3145 }
3146 const Type* t = in(1)->bottom_type();
3147 if (t == TypePtr::NULL_PTR) {
3148 return t;
3149 }
3150 return t;
3151 }
3152
3153 const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const {
3154 if (in(1) == NULL) {
3155 return Type::TOP;
3156 }
3157 const Type* t = phase->type(in(1));
3158 if (t == Type::TOP) {
3159 return Type::TOP;
3160 }
3161 if (t == TypePtr::NULL_PTR) {
3162 return t;
3163 }
3164 return t;
3165 }
3166
3167 int ShenandoahEnqueueBarrierNode::needed(Node* n) {
3168 if (n == NULL ||
3169 n->is_Allocate() ||
3170 n->bottom_type() == TypePtr::NULL_PTR ||
3171 (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL)) {
3172 return NotNeeded;
3173 }
3174 if (n->is_Phi() ||
3175 n->is_CMove()) {
3176 return MaybeNeeded;
3177 }
3178 return Needed;
3179 }
3180
3181 Node* ShenandoahEnqueueBarrierNode::next(Node* n) {
3182 for (;;) {
3183 if (n == NULL) {
3184 return n;
|