1001   const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument
1002   debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx));
1003   Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered);
1004   phase->register_new_node(in_cset_fast_test_load, ctrl);
1005   Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
1006   phase->register_new_node(in_cset_fast_test_cmp, ctrl);
1007   Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
1008   phase->register_new_node(in_cset_fast_test_test, ctrl);
1009   IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1010   phase->register_control(in_cset_fast_test_iff, loop, ctrl);
1011 
1012   not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
1013   phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
1014 
1015   ctrl = new IfFalseNode(in_cset_fast_test_iff);
1016   phase->register_control(ctrl, loop, in_cset_fast_test_iff);
1017 }
1018 
1019 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) {
1020   IdealLoopTree*loop = phase->get_loop(ctrl);
1021   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst();
1022 
1023   // The slow path stub consumes and produces raw memory in addition
1024   // to the existing memory edges
1025   Node* base = find_bottom_mem(ctrl, phase);
1026   MergeMemNode* mm = MergeMemNode::make(base);
1027   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1028   phase->register_new_node(mm, ctrl);
1029 
1030   address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
1031           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow) :
1032           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier);
1033 
1034   address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native)
1035                                : target;
1036   const char* name = is_native ? "load_reference_barrier_native" : "load_reference_barrier";
1037   Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM);
1038 
1039   call->init_req(TypeFunc::Control, ctrl);
1040   call->init_req(TypeFunc::I_O, phase->C->top());
1041   call->init_req(TypeFunc::Memory, mm);

2121       if (m != NULL) {
2122         nodes.push(m);
2123       }
2124     }
2125   }
2126 }
2127 #endif
2128 
2129 ShenandoahEnqueueBarrierNode::ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) {
2130   ShenandoahBarrierSetC2::bsc2()->state()->add_enqueue_barrier(this);
2131 }
2132 
2133 const Type* ShenandoahEnqueueBarrierNode::bottom_type() const {
2134   if (in(1) == NULL || in(1)->is_top()) {
2135     return Type::TOP;
2136   }
2137   const Type* t = in(1)->bottom_type();
2138   if (t == TypePtr::NULL_PTR) {
2139     return t;
2140   }
2141   return t->is_oopptr()->cast_to_nonconst();
2142 }
2143 
2144 const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const {
2145   if (in(1) == NULL) {
2146     return Type::TOP;
2147   }
2148   const Type* t = phase->type(in(1));
2149   if (t == Type::TOP) {
2150     return Type::TOP;
2151   }
2152   if (t == TypePtr::NULL_PTR) {
2153     return t;
2154   }
2155   return t->is_oopptr()->cast_to_nonconst();
2156 }
2157 
2158 int ShenandoahEnqueueBarrierNode::needed(Node* n) {
2159   if (n == NULL ||
2160       n->is_Allocate() ||
2161       n->Opcode() == Op_ShenandoahEnqueueBarrier ||
2162       n->bottom_type() == TypePtr::NULL_PTR ||
2163       (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL)) {
2164     return NotNeeded;
2165   }
2166   if (n->is_Phi() ||
2167       n->is_CMove()) {
2168     return MaybeNeeded;
2169   }
2170   return Needed;
2171 }
2172 
2173 Node* ShenandoahEnqueueBarrierNode::next(Node* n) {
2174   for (;;) {
2175     if (n == NULL) {

3062 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {
3063   if (in(ValueIn) == NULL || in(ValueIn)->is_top()) {
3064     return Type::TOP;
3065   }
3066   const Type* t = in(ValueIn)->bottom_type();
3067   if (t == TypePtr::NULL_PTR) {
3068     return t;
3069   }
3070   return t->is_oopptr();
3071 }
3072 
3073 const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const {
3074   // Either input is TOP ==> the result is TOP
3075   const Type *t2 = phase->type(in(ValueIn));
3076   if( t2 == Type::TOP ) return Type::TOP;
3077 
3078   if (t2 == TypePtr::NULL_PTR) {
3079     return t2;
3080   }
3081 
3082   const Type* type = t2->is_oopptr()/*->cast_to_nonconst()*/;
3083   return type;
3084 }
3085 
3086 Node* ShenandoahLoadReferenceBarrierNode::Identity(PhaseGVN* phase) {
3087   Node* value = in(ValueIn);
3088   if (!needs_barrier(phase, value)) {
3089     return value;
3090   }
3091   return this;
3092 }
3093 
3094 bool ShenandoahLoadReferenceBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) {
3095   Unique_Node_List visited;
3096   return needs_barrier_impl(phase, n, visited);
3097 }
3098 
3099 bool ShenandoahLoadReferenceBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) {
3100   if (n == NULL) return false;
3101   if (visited.member(n)) {
3102     return false; // Been there.

1001   const TypePtr* in_cset_fast_test_adr_type = NULL; // debug-mode-only argument
1002   debug_only(in_cset_fast_test_adr_type = phase->C->get_adr_type(in_cset_fast_test_idx));
1003   Node* in_cset_fast_test_load = new LoadBNode(ctrl, raw_mem, in_cset_fast_test_adr, in_cset_fast_test_adr_type, TypeInt::BYTE, MemNode::unordered);
1004   phase->register_new_node(in_cset_fast_test_load, ctrl);
1005   Node* in_cset_fast_test_cmp = new CmpINode(in_cset_fast_test_load, phase->igvn().zerocon(T_INT));
1006   phase->register_new_node(in_cset_fast_test_cmp, ctrl);
1007   Node* in_cset_fast_test_test = new BoolNode(in_cset_fast_test_cmp, BoolTest::eq);
1008   phase->register_new_node(in_cset_fast_test_test, ctrl);
1009   IfNode* in_cset_fast_test_iff = new IfNode(ctrl, in_cset_fast_test_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN);
1010   phase->register_control(in_cset_fast_test_iff, loop, ctrl);
1011 
1012   not_cset_ctrl = new IfTrueNode(in_cset_fast_test_iff);
1013   phase->register_control(not_cset_ctrl, loop, in_cset_fast_test_iff);
1014 
1015   ctrl = new IfFalseNode(in_cset_fast_test_iff);
1016   phase->register_control(ctrl, loop, in_cset_fast_test_iff);
1017 }
1018 
1019 void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) {
1020   IdealLoopTree*loop = phase->get_loop(ctrl);
1021   const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr();
1022 
1023   // The slow path stub consumes and produces raw memory in addition
1024   // to the existing memory edges
1025   Node* base = find_bottom_mem(ctrl, phase);
1026   MergeMemNode* mm = MergeMemNode::make(base);
1027   mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
1028   phase->register_new_node(mm, ctrl);
1029 
1030   address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
1031           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow) :
1032           CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier);
1033 
1034   address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native)
1035                                : target;
1036   const char* name = is_native ? "load_reference_barrier_native" : "load_reference_barrier";
1037   Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM);
1038 
1039   call->init_req(TypeFunc::Control, ctrl);
1040   call->init_req(TypeFunc::I_O, phase->C->top());
1041   call->init_req(TypeFunc::Memory, mm);

2121       if (m != NULL) {
2122         nodes.push(m);
2123       }
2124     }
2125   }
2126 }
2127 #endif
2128 
2129 ShenandoahEnqueueBarrierNode::ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) {
2130   ShenandoahBarrierSetC2::bsc2()->state()->add_enqueue_barrier(this);
2131 }
2132 
2133 const Type* ShenandoahEnqueueBarrierNode::bottom_type() const {
2134   if (in(1) == NULL || in(1)->is_top()) {
2135     return Type::TOP;
2136   }
2137   const Type* t = in(1)->bottom_type();
2138   if (t == TypePtr::NULL_PTR) {
2139     return t;
2140   }
2141   return t->is_oopptr();
2142 }
2143 
2144 const Type* ShenandoahEnqueueBarrierNode::Value(PhaseGVN* phase) const {
2145   if (in(1) == NULL) {
2146     return Type::TOP;
2147   }
2148   const Type* t = phase->type(in(1));
2149   if (t == Type::TOP) {
2150     return Type::TOP;
2151   }
2152   if (t == TypePtr::NULL_PTR) {
2153     return t;
2154   }
2155   return t->is_oopptr();
2156 }
2157 
2158 int ShenandoahEnqueueBarrierNode::needed(Node* n) {
2159   if (n == NULL ||
2160       n->is_Allocate() ||
2161       n->Opcode() == Op_ShenandoahEnqueueBarrier ||
2162       n->bottom_type() == TypePtr::NULL_PTR ||
2163       (n->bottom_type()->make_oopptr() != NULL && n->bottom_type()->make_oopptr()->const_oop() != NULL)) {
2164     return NotNeeded;
2165   }
2166   if (n->is_Phi() ||
2167       n->is_CMove()) {
2168     return MaybeNeeded;
2169   }
2170   return Needed;
2171 }
2172 
2173 Node* ShenandoahEnqueueBarrierNode::next(Node* n) {
2174   for (;;) {
2175     if (n == NULL) {

3062 const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {
3063   if (in(ValueIn) == NULL || in(ValueIn)->is_top()) {
3064     return Type::TOP;
3065   }
3066   const Type* t = in(ValueIn)->bottom_type();
3067   if (t == TypePtr::NULL_PTR) {
3068     return t;
3069   }
3070   return t->is_oopptr();
3071 }
3072 
3073 const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const {
3074   // Either input is TOP ==> the result is TOP
3075   const Type *t2 = phase->type(in(ValueIn));
3076   if( t2 == Type::TOP ) return Type::TOP;
3077 
3078   if (t2 == TypePtr::NULL_PTR) {
3079     return t2;
3080   }
3081 
3082   const Type* type = t2->is_oopptr();
3083   return type;
3084 }
3085 
3086 Node* ShenandoahLoadReferenceBarrierNode::Identity(PhaseGVN* phase) {
3087   Node* value = in(ValueIn);
3088   if (!needs_barrier(phase, value)) {
3089     return value;
3090   }
3091   return this;
3092 }
3093 
3094 bool ShenandoahLoadReferenceBarrierNode::needs_barrier(PhaseGVN* phase, Node* n) {
3095   Unique_Node_List visited;
3096   return needs_barrier_impl(phase, n, visited);
3097 }
3098 
3099 bool ShenandoahLoadReferenceBarrierNode::needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited) {
3100   if (n == NULL) return false;
3101   if (visited.member(n)) {
3102     return false; // Been there.