49 50 #ifndef PRODUCT 51 extern int explicit_null_checks_inserted, 52 explicit_null_checks_elided; 53 #endif 54 55 //---------------------------------array_load---------------------------------- 56 void Parse::array_load(BasicType bt) { 57 const Type* elemtype = Type::TOP; 58 Node* adr = array_addressing(bt, 0, &elemtype); 59 if (stopped()) return; // guaranteed null or range check 60 61 Node* idx = pop(); 62 Node* ary = pop(); 63 64 // Handle value type arrays 65 const TypeOopPtr* elemptr = elemtype->make_oopptr(); 66 const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr(); 67 if (elemtype->isa_valuetype() != NULL) { 68 // Load from flattened value type array 69 ciValueKlass* vk = elemtype->is_valuetype()->value_klass(); 70 Node* vt = ValueTypeNode::make_from_flattened(this, vk, ary, adr); 71 push(vt); 72 return; 73 } else if (elemptr != NULL && elemptr->is_valuetypeptr() && !elemptr->maybe_null()) { 74 // Load from non-flattened value type array (elements can never be null) 75 bt = T_VALUETYPE; 76 } else if (ValueArrayFlatten && elemptr != NULL && elemptr->can_be_value_type() && !ary_t->klass_is_exact()) { 77 // Cannot statically determine if array is flattened, emit runtime check 78 assert(!elemptr->is_valuetypeptr(), "we know the exact type"); 79 IdealKit ideal(this); 80 IdealVariable res(ideal); 81 ideal.declarations_done(); 82 Node* kls = load_object_klass(ary); 83 Node* tag = load_lh_array_tag(kls); 84 ideal.if_then(tag, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); { 85 // non flattened 86 sync_kit(ideal); 87 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 88 elemtype = ary_t->elem()->make_oopptr(); 89 Node* ld = access_load_at(ary, adr, adr_type, elemtype, bt, 90 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD); 91 ideal.sync_kit(this); 92 ideal.set(res, ld); 93 } ideal.else_(); { 94 // flattened 95 sync_kit(ideal); 96 Node* k_adr = basic_plus_adr(kls, in_bytes(ArrayKlass::element_klass_offset())); 97 Node* elem_klass = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS)); 98 Node* obj_size = NULL; 99 kill_dead_locals(); 100 inc_sp(2); 101 Node* alloc_obj = new_instance(elem_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true); 102 dec_sp(2); 103 104 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); 105 assert(alloc->maybe_set_complete(&_gvn), ""); 106 alloc->initialization()->set_complete_with_arraycopy(); 107 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 108 // Unknown value type so might have reference fields 109 if (!bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Parsing)) { 110 int base_off = sizeof(instanceOopDesc); 111 Node* dst_base = basic_plus_adr(alloc_obj, base_off); 112 Node* countx = obj_size; 113 countx = _gvn.transform(new SubXNode(countx, MakeConX(base_off))); 114 countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong))); 115 116 assert(Klass::_lh_log2_element_size_shift == 0, "use shift in place"); 117 Node* lhp = basic_plus_adr(kls, in_bytes(Klass::layout_helper_offset())); 118 Node* elem_shift = make_load(NULL, lhp, TypeInt::INT, T_INT, MemNode::unordered); 119 uint header = arrayOopDesc::base_offset_in_bytes(T_VALUETYPE); 120 Node* base = basic_plus_adr(ary, header); 121 idx = Compile::conv_I2X_index(&_gvn, idx, TypeInt::POS, control()); 122 Node* scale = _gvn.transform(new LShiftXNode(idx, elem_shift)); 123 Node* adr = basic_plus_adr(ary, base, scale); 124 125 access_clone(adr, dst_base, countx, false); 126 } else { 127 ideal.sync_kit(this); 128 ideal.make_leaf_call(OptoRuntime::load_unknown_value_Type(), 129 CAST_FROM_FN_PTR(address, OptoRuntime::load_unknown_value), 130 "load_unknown_value", 131 ary, idx, alloc_obj); 132 sync_kit(ideal); 133 } 134 135 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 136 137 ideal.sync_kit(this); 138 ideal.set(res, alloc_obj); 139 } ideal.end_if(); 140 sync_kit(ideal); 141 push_node(bt, ideal.value(res)); 142 return; 143 } 144 145 if (elemtype == TypeInt::BOOL) { 146 bt = T_BOOLEAN; 147 } else if (bt == T_OBJECT) { 148 elemtype = ary_t->elem()->make_oopptr(); 149 } 150 151 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 152 Node* ld = access_load_at(ary, adr, adr_type, elemtype, bt, 153 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD); 154 if (bt == T_VALUETYPE) { 155 // Loading a non-flattened (but flattenable) value type from an array 156 assert(!gvn().type(ld)->maybe_null(), "value type array elements should never be null"); 157 if (elemptr->value_klass()->is_scalarizable()) { 158 ld = ValueTypeNode::make_from_oop(this, ld, elemptr->value_klass()); 159 } 160 } 161 171 Node* cast_val = NULL; 172 if (bt == T_OBJECT) { 173 cast_val = array_store_check(); 174 if (stopped()) return; 175 } 176 Node* val = pop_node(bt); // Value to store 177 Node* idx = pop(); // Index in the array 178 Node* ary = pop(); // The array itself 179 180 const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr(); 181 if (bt == T_OBJECT) { 182 const TypeOopPtr* elemptr = elemtype->make_oopptr(); 183 const Type* val_t = _gvn.type(val); 184 if (elemtype->isa_valuetype() != NULL) { 185 // Store to flattened value type array 186 if (!cast_val->is_ValueType()) { 187 inc_sp(3); 188 cast_val = null_check(cast_val); 189 if (stopped()) return; 190 dec_sp(3); 191 cast_val = ValueTypeNode::make_from_oop(this, cast_val, elemtype->is_valuetype()->value_klass()); 192 } 193 cast_val->as_ValueType()->store_flattened(this, ary, adr); 194 return; 195 } else if (elemptr->is_valuetypeptr() && !elemptr->maybe_null()) { 196 // Store to non-flattened value type array 197 if (!cast_val->is_ValueType()) { 198 // Can not store null into a value type array 199 inc_sp(3); 200 cast_val = null_check(cast_val); 201 if (stopped()) return; 202 dec_sp(3); 203 } 204 } else if (elemptr->can_be_value_type() && !ary_t->klass_is_exact() && 205 (val->is_ValueType() || val_t == TypePtr::NULL_PTR || val_t->is_oopptr()->can_be_value_type())) { 206 if (ValueArrayFlatten) { 207 IdealKit ideal(this); 208 Node* kls = load_object_klass(ary); 209 Node* layout_val = load_lh_array_tag(kls); 210 ideal.if_then(layout_val, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); { 211 // non flattened 212 sync_kit(ideal); 213 214 if (!val->is_ValueType() && TypePtr::NULL_PTR->higher_equal(val_t)) { 215 gen_value_type_array_guard(ary, val, 3); 216 } 217 218 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 219 elemtype = ary_t->elem()->make_oopptr(); 220 access_store_at(ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY); 221 ideal.sync_kit(this); 222 } ideal.else_(); { 223 // flattened 224 // Object/interface array must be flattened, cast it 225 if (val->is_ValueType()) { 226 sync_kit(ideal); 227 const TypeValueType* vt = _gvn.type(val)->is_valuetype(); 228 ciArrayKlass* array_klass = ciArrayKlass::make(vt->value_klass(), true); 229 const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); 230 ary = _gvn.transform(new CheckCastPPNode(control(), ary, arytype)); 231 adr = array_element_address(ary, idx, T_OBJECT, arytype->size(), control()); 232 val->as_ValueType()->store_flattened(this, ary, adr); 233 ideal.sync_kit(this); 234 } else { 235 if (TypePtr::NULL_PTR->higher_equal(val_t)) { 236 sync_kit(ideal); 237 Node* null_ctl = top(); 238 val = null_check_oop(val, &null_ctl); 239 if (null_ctl != top()) { 240 PreserveJVMState pjvms(this); 241 inc_sp(3); 242 set_control(null_ctl); 243 uncommon_trap(Deoptimization::Reason_null_check, Deoptimization::Action_none); 244 dec_sp(3); 245 } 246 ideal.sync_kit(this); 247 } 248 if (!ideal.ctrl()->is_top()) { 249 ideal.make_leaf_call(OptoRuntime::store_unknown_value_Type(), 250 CAST_FROM_FN_PTR(address, OptoRuntime::store_unknown_value), 251 "store_unknown_value", 252 val, ary, idx); 253 } 254 } 255 } ideal.end_if(); 256 sync_kit(ideal); 257 return; 258 } else { 259 if (!val->is_ValueType() && TypePtr::NULL_PTR->higher_equal(val_t)) { 260 gen_value_type_array_guard(ary, val, 3); 261 } 262 } 263 } 264 } 265 266 if (elemtype == TypeInt::BOOL) { 267 bt = T_BOOLEAN; 268 } else if (bt == T_OBJECT) { 269 elemtype = ary_t->elem()->make_oopptr(); 270 } 271 272 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 273 274 access_store_at(ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY); 275 } 276 277 278 //------------------------------array_addressing------------------------------- 279 // Pull array and index from the stack. Compute pointer-to-element. 280 Node* Parse::array_addressing(BasicType type, int vals, const Type* *result2) { 281 Node *idx = peek(0+vals); // Get from stack without popping | 49 50 #ifndef PRODUCT 51 extern int explicit_null_checks_inserted, 52 explicit_null_checks_elided; 53 #endif 54 55 //---------------------------------array_load---------------------------------- 56 void Parse::array_load(BasicType bt) { 57 const Type* elemtype = Type::TOP; 58 Node* adr = array_addressing(bt, 0, &elemtype); 59 if (stopped()) return; // guaranteed null or range check 60 61 Node* idx = pop(); 62 Node* ary = pop(); 63 64 // Handle value type arrays 65 const TypeOopPtr* elemptr = elemtype->make_oopptr(); 66 const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr(); 67 if (elemtype->isa_valuetype() != NULL) { 68 // Load from flattened value type array 69 Node* vt = ValueTypeNode::make_from_flattened(this, elemtype->value_klass(), ary, adr); 70 push(vt); 71 return; 72 } else if (elemptr != NULL && elemptr->is_valuetypeptr() && !elemptr->maybe_null()) { 73 // Load from non-flattened but flattenable value type array (elements can never be null) 74 bt = T_VALUETYPE; 75 } else if (ValueArrayFlatten && elemptr != NULL && elemptr->can_be_value_type() && 76 (!ary_t->klass_is_exact() || (elemptr->is_valuetypeptr() && elemptr->value_klass()->flatten_array()))) { 77 // Cannot statically determine if array is flattened, emit runtime check 78 IdealKit ideal(this); 79 IdealVariable res(ideal); 80 ideal.declarations_done(); 81 Node* kls = load_object_klass(ary); 82 Node* tag = load_lh_array_tag(kls); 83 ideal.if_then(tag, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); { 84 // non-flattened 85 sync_kit(ideal); 86 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 87 Node* ld = access_load_at(ary, adr, adr_type, elemptr, bt, 88 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD); 89 ideal.sync_kit(this); 90 ideal.set(res, ld); 91 } ideal.else_(); { 92 // flattened 93 sync_kit(ideal); 94 if (elemptr->is_valuetypeptr()) { 95 // Element type is known, cast and load from flattened representation 96 assert(elemptr->maybe_null(), "must be nullable"); 97 ciValueKlass* vk = elemptr->value_klass(); 98 assert(vk->flatten_array(), "must be flattenable"); 99 ciArrayKlass* array_klass = ciArrayKlass::make(vk, /* never_null */ true); 100 const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); 101 Node* cast = _gvn.transform(new CheckCastPPNode(control(), ary, arytype)); 102 adr = array_element_address(cast, idx, T_VALUETYPE, ary_t->size(), control()); 103 Node* vt = ValueTypeNode::make_from_flattened(this, vk, cast, adr)->allocate(this, false, false)->get_oop(); 104 ideal.set(res, vt); 105 } else { 106 // Element type is unknown, emit runtime call 107 assert(!ary_t->klass_is_exact(), "should not have exact type here"); 108 Node* k_adr = basic_plus_adr(kls, in_bytes(ArrayKlass::element_klass_offset())); 109 Node* elem_klass = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS)); 110 Node* obj_size = NULL; 111 kill_dead_locals(); 112 inc_sp(2); 113 Node* alloc_obj = new_instance(elem_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true); 114 dec_sp(2); 115 116 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); 117 assert(alloc->maybe_set_complete(&_gvn), ""); 118 alloc->initialization()->set_complete_with_arraycopy(); 119 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 120 // Unknown value type might contain reference fields 121 if (!bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Parsing)) { 122 int base_off = sizeof(instanceOopDesc); 123 Node* dst_base = basic_plus_adr(alloc_obj, base_off); 124 Node* countx = obj_size; 125 countx = _gvn.transform(new SubXNode(countx, MakeConX(base_off))); 126 countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong))); 127 128 assert(Klass::_lh_log2_element_size_shift == 0, "use shift in place"); 129 Node* lhp = basic_plus_adr(kls, in_bytes(Klass::layout_helper_offset())); 130 Node* elem_shift = make_load(NULL, lhp, TypeInt::INT, T_INT, MemNode::unordered); 131 uint header = arrayOopDesc::base_offset_in_bytes(T_VALUETYPE); 132 Node* base = basic_plus_adr(ary, header); 133 idx = Compile::conv_I2X_index(&_gvn, idx, TypeInt::POS, control()); 134 Node* scale = _gvn.transform(new LShiftXNode(idx, elem_shift)); 135 Node* adr = basic_plus_adr(ary, base, scale); 136 137 access_clone(adr, dst_base, countx, false); 138 } else { 139 ideal.sync_kit(this); 140 ideal.make_leaf_call(OptoRuntime::load_unknown_value_Type(), 141 CAST_FROM_FN_PTR(address, OptoRuntime::load_unknown_value), 142 "load_unknown_value", 143 ary, idx, alloc_obj); 144 sync_kit(ideal); 145 } 146 147 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 148 149 ideal.set(res, alloc_obj); 150 } 151 ideal.sync_kit(this); 152 } ideal.end_if(); 153 sync_kit(ideal); 154 push_node(bt, _gvn.transform(ideal.value(res))); 155 return; 156 } 157 158 if (elemtype == TypeInt::BOOL) { 159 bt = T_BOOLEAN; 160 } else if (bt == T_OBJECT) { 161 elemtype = ary_t->elem()->make_oopptr(); 162 } 163 164 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 165 Node* ld = access_load_at(ary, adr, adr_type, elemtype, bt, 166 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD); 167 if (bt == T_VALUETYPE) { 168 // Loading a non-flattened (but flattenable) value type from an array 169 assert(!gvn().type(ld)->maybe_null(), "value type array elements should never be null"); 170 if (elemptr->value_klass()->is_scalarizable()) { 171 ld = ValueTypeNode::make_from_oop(this, ld, elemptr->value_klass()); 172 } 173 } 174 184 Node* cast_val = NULL; 185 if (bt == T_OBJECT) { 186 cast_val = array_store_check(); 187 if (stopped()) return; 188 } 189 Node* val = pop_node(bt); // Value to store 190 Node* idx = pop(); // Index in the array 191 Node* ary = pop(); // The array itself 192 193 const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr(); 194 if (bt == T_OBJECT) { 195 const TypeOopPtr* elemptr = elemtype->make_oopptr(); 196 const Type* val_t = _gvn.type(val); 197 if (elemtype->isa_valuetype() != NULL) { 198 // Store to flattened value type array 199 if (!cast_val->is_ValueType()) { 200 inc_sp(3); 201 cast_val = null_check(cast_val); 202 if (stopped()) return; 203 dec_sp(3); 204 cast_val = ValueTypeNode::make_from_oop(this, cast_val, elemtype->value_klass()); 205 } 206 cast_val->as_ValueType()->store_flattened(this, ary, adr); 207 return; 208 } else if (elemptr->is_valuetypeptr() && !elemptr->maybe_null()) { 209 // Store to non-flattened but flattenable value type array (elements can never be null) 210 if (!cast_val->is_ValueType()) { 211 inc_sp(3); 212 cast_val = null_check(cast_val); 213 if (stopped()) return; 214 dec_sp(3); 215 } 216 } else if (elemptr->can_be_value_type() && (!ary_t->klass_is_exact() || elemptr->is_valuetypeptr()) && 217 (val->is_ValueType() || val_t == TypePtr::NULL_PTR || val_t->is_oopptr()->can_be_value_type())) { 218 // Cannot statically determine if array is flattened, emit runtime check 219 ciValueKlass* vk = NULL; 220 // Try to determine the value klass 221 if (val->is_ValueType()) { 222 vk = val_t->value_klass(); 223 } else if (elemptr->is_valuetypeptr()) { 224 vk = elemptr->value_klass(); 225 } 226 if (ValueArrayFlatten && (vk == NULL || vk->flatten_array())) { 227 IdealKit ideal(this); 228 Node* kls = load_object_klass(ary); 229 Node* layout_val = load_lh_array_tag(kls); 230 ideal.if_then(layout_val, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); { 231 // non-flattened 232 sync_kit(ideal); 233 gen_value_array_null_guard(ary, val, 3); 234 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 235 elemtype = ary_t->elem()->make_oopptr(); 236 access_store_at(ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY, false, false); 237 ideal.sync_kit(this); 238 } ideal.else_(); { 239 // flattened 240 if (!val->is_ValueType() && TypePtr::NULL_PTR->higher_equal(val_t)) { 241 // Add null check 242 sync_kit(ideal); 243 Node* null_ctl = top(); 244 val = null_check_oop(val, &null_ctl); 245 if (null_ctl != top()) { 246 PreserveJVMState pjvms(this); 247 inc_sp(3); 248 set_control(null_ctl); 249 uncommon_trap(Deoptimization::Reason_null_check, Deoptimization::Action_none); 250 dec_sp(3); 251 } 252 ideal.sync_kit(this); 253 } 254 if (vk != NULL && !stopped()) { 255 // Element type is known, cast and store to flattened representation 256 sync_kit(ideal); 257 assert(vk->flatten_array(), "must be flattenable"); 258 assert(elemptr->maybe_null(), "must be nullable"); 259 ciArrayKlass* array_klass = ciArrayKlass::make(vk, /* never_null */ true); 260 const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); 261 ary = _gvn.transform(new CheckCastPPNode(control(), ary, arytype)); 262 adr = array_element_address(ary, idx, T_OBJECT, arytype->size(), control()); 263 if (!val->is_ValueType()) { 264 assert(!gvn().type(val)->maybe_null(), "value type array elements should never be null"); 265 val = ValueTypeNode::make_from_oop(this, val, vk); 266 } 267 val->as_ValueType()->store_flattened(this, ary, adr); 268 ideal.sync_kit(this); 269 } else if (!ideal.ctrl()->is_top()) { 270 // Element type is unknown, emit runtime call 271 assert(!ary_t->klass_is_exact(), "should not have exact type here"); 272 ideal.make_leaf_call(OptoRuntime::store_unknown_value_Type(), 273 CAST_FROM_FN_PTR(address, OptoRuntime::store_unknown_value), 274 "store_unknown_value", 275 val, ary, idx); 276 } 277 } ideal.end_if(); 278 sync_kit(ideal); 279 return; 280 } else { 281 gen_value_array_null_guard(ary, val, 3); 282 } 283 } 284 } 285 286 if (elemtype == TypeInt::BOOL) { 287 bt = T_BOOLEAN; 288 } else if (bt == T_OBJECT) { 289 elemtype = ary_t->elem()->make_oopptr(); 290 } 291 292 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 293 294 access_store_at(ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY); 295 } 296 297 298 //------------------------------array_addressing------------------------------- 299 // Pull array and index from the stack. Compute pointer-to-element. 300 Node* Parse::array_addressing(BasicType type, int vals, const Type* *result2) { 301 Node *idx = peek(0+vals); // Get from stack without popping |