64 // Handle value type arrays 65 const TypeOopPtr* elemptr = elemtype->make_oopptr(); 66 const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr(); 67 if (elemtype->isa_valuetype() != NULL) { 68 C->set_flattened_accesses(); 69 // Load from flattened value type array 70 Node* vt = ValueTypeNode::make_from_flattened(this, elemtype->value_klass(), ary, adr); 71 push(vt); 72 return; 73 } else if (elemptr != NULL && elemptr->is_valuetypeptr() && !elemptr->maybe_null()) { 74 // Load from non-flattened but flattenable value type array (elements can never be null) 75 bt = T_VALUETYPE; 76 } else if (!ary_t->is_not_flat()) { 77 // Cannot statically determine if array is flattened, emit runtime check 78 assert(ValueArrayFlatten && elemptr->can_be_value_type() && !ary_t->klass_is_exact() && !ary_t->is_not_null_free() && 79 (!elemptr->is_valuetypeptr() || elemptr->value_klass()->flatten_array()), "array can't be flattened"); 80 Node* ctl = control(); 81 IdealKit ideal(this); 82 IdealVariable res(ideal); 83 ideal.declarations_done(); 84 Node* kls = load_object_klass(ary); 85 Node* tag = load_lh_array_tag(kls); 86 ideal.if_then(tag, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); { 87 // non-flattened 88 sync_kit(ideal); 89 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 90 Node* ld = access_load_at(ary, adr, adr_type, elemptr, bt, 91 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD, ctl); 92 ideal.sync_kit(this); 93 ideal.set(res, ld); 94 } ideal.else_(); { 95 // flattened 96 sync_kit(ideal); 97 if (elemptr->is_valuetypeptr()) { 98 // Element type is known, cast and load from flattened representation 99 ciValueKlass* vk = elemptr->value_klass(); 100 assert(vk->flatten_array() && elemptr->maybe_null(), "must be a flattenable and nullable array"); 101 ciArrayKlass* array_klass = ciArrayKlass::make(vk, /* never_null */ true); 102 const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); 103 Node* cast = _gvn.transform(new CheckCastPPNode(control(), ary, arytype)); 104 adr = array_element_address(cast, idx, T_VALUETYPE, ary_t->size(), control()); 105 Node* vt = ValueTypeNode::make_from_flattened(this, vk, cast, adr)->allocate(this, false, false)->get_oop(); 106 ideal.set(res, vt); 107 ideal.sync_kit(this); 108 } else { 109 // Element type is unknown, emit runtime call 110 Node* k_adr = basic_plus_adr(kls, in_bytes(ArrayKlass::element_klass_offset())); 111 Node* elem_klass = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS)); 112 Node* obj_size = NULL; 113 kill_dead_locals(); 114 inc_sp(2); 115 Node* alloc_obj = new_instance(elem_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true); 116 dec_sp(2); 117 118 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); 119 assert(alloc->maybe_set_complete(&_gvn), ""); 120 alloc->initialization()->set_complete_with_arraycopy(); 121 122 // This membar keeps this access to an unknown flattened array 123 // correctly ordered with other unknown and known flattened 124 // array accesses. 125 insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::VALUES)); 126 127 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 128 // Unknown value type might contain reference fields 149 CAST_FROM_FN_PTR(address, OptoRuntime::load_unknown_value), 150 "load_unknown_value", 151 ary, idx, alloc_obj); 152 sync_kit(ideal); 153 } 154 155 // This makes sure no other thread sees a partially initialized buffered value 156 insert_mem_bar_volatile(Op_MemBarStoreStore, Compile::AliasIdxRaw, alloc->proj_out_or_null(AllocateNode::RawAddress)); 157 158 // Same as MemBarCPUOrder above: keep this unknown flattened 159 // array access correctly ordered with other flattened array 160 // access 161 insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::VALUES)); 162 163 // Prevent any use of the newly allocated value before it is 164 // fully initialized 165 alloc_obj = new CastPPNode(alloc_obj, _gvn.type(alloc_obj), true); 166 alloc_obj->set_req(0, control()); 167 alloc_obj = _gvn.transform(alloc_obj); 168 169 ideal.sync_kit(this); 170 171 ideal.set(res, alloc_obj); 172 } 173 } ideal.end_if(); 174 sync_kit(ideal); 175 push_node(bt, _gvn.transform(ideal.value(res))); 176 return; 177 } 178 179 if (elemtype == TypeInt::BOOL) { 180 bt = T_BOOLEAN; 181 } else if (bt == T_OBJECT) { 182 elemtype = ary_t->elem()->make_oopptr(); 183 } 184 185 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 186 Node* ld = access_load_at(ary, adr, adr_type, elemtype, bt, 187 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD); 188 if (bt == T_VALUETYPE) { 189 // Loading a non-flattened (but flattenable) value type from an array 190 assert(!gvn().type(ld)->maybe_null(), "value type array elements should never be null"); 191 if (elemptr->value_klass()->is_scalarizable()) { 192 ld = ValueTypeNode::make_from_oop(this, ld, elemptr->value_klass()); 252 cast_val = null_check(cast_val); 253 if (stopped()) return; 254 dec_sp(3); 255 cast_val = ValueTypeNode::make_from_oop(this, cast_val, ary_t->elem()->value_klass()); 256 } 257 cast_val->as_ValueType()->store_flattened(this, ary, adr); 258 return; 259 } else if (elemtype->is_valuetypeptr() && !elemtype->maybe_null()) { 260 // Store to non-flattened but flattenable value type array (elements can never be null) 261 if (!cast_val->is_ValueType() && tval->maybe_null()) { 262 inc_sp(3); 263 cast_val = null_check(cast_val); 264 if (stopped()) return; 265 dec_sp(3); 266 } 267 } else if (!ary_t->is_not_flat()) { 268 // Array might be flattened, emit runtime checks 269 assert(ValueArrayFlatten && !not_flattenable && elemtype->is_oopptr()->can_be_value_type() && 270 !ary_t->klass_is_exact() && !ary_t->is_not_null_free(), "array can't be flattened"); 271 IdealKit ideal(this); 272 Node* kls = load_object_klass(ary); 273 Node* layout_val = load_lh_array_tag(kls); 274 ideal.if_then(layout_val, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); 275 { 276 // non-flattened 277 sync_kit(ideal); 278 gen_value_array_null_guard(ary, cast_val, 3); 279 access_store_at(ary, adr, adr_type, cast_val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY, false, false); 280 ideal.sync_kit(this); 281 } 282 ideal.else_(); 283 { 284 // flattened 285 if (!cast_val->is_ValueType() && tval->maybe_null()) { 286 // Add null check 287 sync_kit(ideal); 288 Node* null_ctl = top(); 289 cast_val = null_check_oop(cast_val, &null_ctl); 290 if (null_ctl != top()) { 291 PreserveJVMState pjvms(this); 292 inc_sp(3); 293 set_control(null_ctl); 294 uncommon_trap(Deoptimization::Reason_null_check, Deoptimization::Action_none); 295 dec_sp(3); 296 } 297 ideal.sync_kit(this); 298 } 299 // Try to determine the value klass 300 ciValueKlass* vk = NULL; 301 if (tval->isa_valuetype() || tval->is_valuetypeptr()) { 302 vk = tval->value_klass(); 303 } else if (tval_init->isa_valuetype() || tval_init->is_valuetypeptr()) { 304 vk = tval_init->value_klass(); 305 } else if (elemtype->is_valuetypeptr()) { 306 vk = elemtype->value_klass(); 307 } 308 if (vk != NULL && !stopped()) { 309 // Element type is known, cast and store to flattened representation 310 sync_kit(ideal); 311 assert(vk->flatten_array() && elemtype->maybe_null(), "must be a flattenable and nullable array"); 312 ciArrayKlass* array_klass = ciArrayKlass::make(vk, /* never_null */ true); 313 const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); 314 ary = _gvn.transform(new CheckCastPPNode(control(), ary, arytype)); 315 adr = array_element_address(ary, idx, T_OBJECT, arytype->size(), control()); 316 if (!cast_val->is_ValueType()) { 317 assert(!gvn().type(cast_val)->maybe_null(), "value type array elements should never be null"); 318 cast_val = ValueTypeNode::make_from_oop(this, cast_val, vk); 319 } 320 cast_val->as_ValueType()->store_flattened(this, ary, adr); 321 ideal.sync_kit(this); 322 } else if (!ideal.ctrl()->is_top()) { 323 // Element type is unknown, emit runtime call 324 sync_kit(ideal); 325 326 // This membar keeps this access to an unknown flattened 327 // array correctly ordered with other unknown and known 328 // flattened array accesses. 329 insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::VALUES)); 330 ideal.sync_kit(this); 331 332 ideal.make_leaf_call(OptoRuntime::store_unknown_value_Type(), 333 CAST_FROM_FN_PTR(address, OptoRuntime::store_unknown_value), 334 "store_unknown_value", 335 cast_val, ary, idx); 336 337 sync_kit(ideal); 338 // Same as MemBarCPUOrder above: keep this unknown 339 // flattened array access correctly ordered with other 340 // flattened array access 341 insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::VALUES)); 342 ideal.sync_kit(this); 343 } 344 } 345 ideal.end_if(); 346 sync_kit(ideal); 347 return; 348 } else if (!ary_t->is_not_null_free()) { 349 // Array is not flattened but may be null free 350 assert(elemtype->is_oopptr()->can_be_value_type() && !ary_t->klass_is_exact(), "array can't be null free"); 351 ary = gen_value_array_null_guard(ary, cast_val, 3, true); 352 } 353 } 354 355 access_store_at(ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY); 356 } 357 358 359 //------------------------------array_addressing------------------------------- 360 // Pull array and index from the stack. Compute pointer-to-element. 361 Node* Parse::array_addressing(BasicType type, int vals, const Type* *result2) { 362 Node *idx = peek(0+vals); // Get from stack without popping 363 Node *ary = peek(1+vals); // in case of exception 364 | 64 // Handle value type arrays 65 const TypeOopPtr* elemptr = elemtype->make_oopptr(); 66 const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr(); 67 if (elemtype->isa_valuetype() != NULL) { 68 C->set_flattened_accesses(); 69 // Load from flattened value type array 70 Node* vt = ValueTypeNode::make_from_flattened(this, elemtype->value_klass(), ary, adr); 71 push(vt); 72 return; 73 } else if (elemptr != NULL && elemptr->is_valuetypeptr() && !elemptr->maybe_null()) { 74 // Load from non-flattened but flattenable value type array (elements can never be null) 75 bt = T_VALUETYPE; 76 } else if (!ary_t->is_not_flat()) { 77 // Cannot statically determine if array is flattened, emit runtime check 78 assert(ValueArrayFlatten && elemptr->can_be_value_type() && !ary_t->klass_is_exact() && !ary_t->is_not_null_free() && 79 (!elemptr->is_valuetypeptr() || elemptr->value_klass()->flatten_array()), "array can't be flattened"); 80 Node* ctl = control(); 81 IdealKit ideal(this); 82 IdealVariable res(ideal); 83 ideal.declarations_done(); 84 Node* flattened = gen_flattened_array_test(ary); 85 ideal.if_then(flattened, BoolTest::ne, zerocon(flattened->bottom_type()->basic_type())); { 86 // flattened 87 sync_kit(ideal); 88 if (elemptr->is_valuetypeptr()) { 89 // Element type is known, cast and load from flattened representation 90 ciValueKlass* vk = elemptr->value_klass(); 91 assert(vk->flatten_array() && elemptr->maybe_null(), "must be a flattenable and nullable array"); 92 ciArrayKlass* array_klass = ciArrayKlass::make(vk, /* never_null */ true); 93 const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); 94 Node* cast = _gvn.transform(new CheckCastPPNode(control(), ary, arytype)); 95 Node* casted_adr = array_element_address(cast, idx, T_VALUETYPE, ary_t->size(), control()); 96 Node* vt = ValueTypeNode::make_from_flattened(this, vk, cast, casted_adr)->allocate(this, false, false)->get_oop(); 97 ideal.set(res, vt); 98 ideal.sync_kit(this); 99 } else { 100 Node* kls = load_object_klass(ary); 101 // Element type is unknown, emit runtime call 102 Node* k_adr = basic_plus_adr(kls, in_bytes(ArrayKlass::element_klass_offset())); 103 Node* elem_klass = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS)); 104 Node* obj_size = NULL; 105 kill_dead_locals(); 106 inc_sp(2); 107 Node* alloc_obj = new_instance(elem_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true); 108 dec_sp(2); 109 110 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); 111 assert(alloc->maybe_set_complete(&_gvn), ""); 112 alloc->initialization()->set_complete_with_arraycopy(); 113 114 // This membar keeps this access to an unknown flattened array 115 // correctly ordered with other unknown and known flattened 116 // array accesses. 117 insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::VALUES)); 118 119 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 120 // Unknown value type might contain reference fields 141 CAST_FROM_FN_PTR(address, OptoRuntime::load_unknown_value), 142 "load_unknown_value", 143 ary, idx, alloc_obj); 144 sync_kit(ideal); 145 } 146 147 // This makes sure no other thread sees a partially initialized buffered value 148 insert_mem_bar_volatile(Op_MemBarStoreStore, Compile::AliasIdxRaw, alloc->proj_out_or_null(AllocateNode::RawAddress)); 149 150 // Same as MemBarCPUOrder above: keep this unknown flattened 151 // array access correctly ordered with other flattened array 152 // access 153 insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::VALUES)); 154 155 // Prevent any use of the newly allocated value before it is 156 // fully initialized 157 alloc_obj = new CastPPNode(alloc_obj, _gvn.type(alloc_obj), true); 158 alloc_obj->set_req(0, control()); 159 alloc_obj = _gvn.transform(alloc_obj); 160 161 const Type* unknown_value = TypeInstPtr::BOTTOM->cast_to_flatten_array(); 162 163 alloc_obj = _gvn.transform(new CheckCastPPNode(control(), alloc_obj, unknown_value)); 164 165 ideal.sync_kit(this); 166 167 ideal.set(res, alloc_obj); 168 } 169 } ideal.else_(); { 170 // non-flattened 171 sync_kit(ideal); 172 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 173 Node* ld = access_load_at(ary, adr, adr_type, elemptr, bt, 174 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD, ctl); 175 ideal.sync_kit(this); 176 ideal.set(res, ld); 177 } ideal.end_if(); 178 sync_kit(ideal); 179 push_node(bt, _gvn.transform(ideal.value(res))); 180 return; 181 } 182 183 if (elemtype == TypeInt::BOOL) { 184 bt = T_BOOLEAN; 185 } else if (bt == T_OBJECT) { 186 elemtype = ary_t->elem()->make_oopptr(); 187 } 188 189 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 190 Node* ld = access_load_at(ary, adr, adr_type, elemtype, bt, 191 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD); 192 if (bt == T_VALUETYPE) { 193 // Loading a non-flattened (but flattenable) value type from an array 194 assert(!gvn().type(ld)->maybe_null(), "value type array elements should never be null"); 195 if (elemptr->value_klass()->is_scalarizable()) { 196 ld = ValueTypeNode::make_from_oop(this, ld, elemptr->value_klass()); 256 cast_val = null_check(cast_val); 257 if (stopped()) return; 258 dec_sp(3); 259 cast_val = ValueTypeNode::make_from_oop(this, cast_val, ary_t->elem()->value_klass()); 260 } 261 cast_val->as_ValueType()->store_flattened(this, ary, adr); 262 return; 263 } else if (elemtype->is_valuetypeptr() && !elemtype->maybe_null()) { 264 // Store to non-flattened but flattenable value type array (elements can never be null) 265 if (!cast_val->is_ValueType() && tval->maybe_null()) { 266 inc_sp(3); 267 cast_val = null_check(cast_val); 268 if (stopped()) return; 269 dec_sp(3); 270 } 271 } else if (!ary_t->is_not_flat()) { 272 // Array might be flattened, emit runtime checks 273 assert(ValueArrayFlatten && !not_flattenable && elemtype->is_oopptr()->can_be_value_type() && 274 !ary_t->klass_is_exact() && !ary_t->is_not_null_free(), "array can't be flattened"); 275 IdealKit ideal(this); 276 Node* flattened = gen_flattened_array_test(ary); 277 ideal.if_then(flattened, BoolTest::ne, zerocon(flattened->bottom_type()->basic_type())); { 278 Node* val = cast_val; 279 // flattened 280 if (!val->is_ValueType() && tval->maybe_null()) { 281 // Add null check 282 sync_kit(ideal); 283 Node* null_ctl = top(); 284 val = null_check_oop(val, &null_ctl); 285 if (null_ctl != top()) { 286 PreserveJVMState pjvms(this); 287 inc_sp(3); 288 set_control(null_ctl); 289 uncommon_trap(Deoptimization::Reason_null_check, Deoptimization::Action_none); 290 dec_sp(3); 291 } 292 ideal.sync_kit(this); 293 } 294 // Try to determine the value klass 295 ciValueKlass* vk = NULL; 296 if (tval->isa_valuetype() || tval->is_valuetypeptr()) { 297 vk = tval->value_klass(); 298 } else if (tval_init->isa_valuetype() || tval_init->is_valuetypeptr()) { 299 vk = tval_init->value_klass(); 300 } else if (elemtype->is_valuetypeptr()) { 301 vk = elemtype->value_klass(); 302 } 303 Node* casted_ary = ary; 304 if (vk != NULL && !stopped()) { 305 // Element type is known, cast and store to flattened representation 306 sync_kit(ideal); 307 assert(vk->flatten_array() && elemtype->maybe_null(), "must be a flattenable and nullable array"); 308 ciArrayKlass* array_klass = ciArrayKlass::make(vk, /* never_null */ true); 309 const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); 310 casted_ary = _gvn.transform(new CheckCastPPNode(control(), casted_ary, arytype)); 311 Node* casted_adr = array_element_address(casted_ary, idx, T_OBJECT, arytype->size(), control()); 312 if (!val->is_ValueType()) { 313 assert(!gvn().type(val)->maybe_null(), "value type array elements should never be null"); 314 val = ValueTypeNode::make_from_oop(this, val, vk); 315 } 316 val->as_ValueType()->store_flattened(this, casted_ary, casted_adr); 317 ideal.sync_kit(this); 318 } else if (!ideal.ctrl()->is_top()) { 319 // Element type is unknown, emit runtime call 320 sync_kit(ideal); 321 322 // This membar keeps this access to an unknown flattened 323 // array correctly ordered with other unknown and known 324 // flattened array accesses. 325 insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::VALUES)); 326 ideal.sync_kit(this); 327 328 ideal.make_leaf_call(OptoRuntime::store_unknown_value_Type(), 329 CAST_FROM_FN_PTR(address, OptoRuntime::store_unknown_value), 330 "store_unknown_value", 331 val, casted_ary, idx); 332 333 sync_kit(ideal); 334 // Same as MemBarCPUOrder above: keep this unknown 335 // flattened array access correctly ordered with other 336 // flattened array accesses. 337 insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::VALUES)); 338 ideal.sync_kit(this); 339 } 340 } 341 ideal.else_(); 342 { 343 // non-flattened 344 sync_kit(ideal); 345 gen_value_array_null_guard(ary, cast_val, 3); 346 access_store_at(ary, adr, adr_type, cast_val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY, false, false); 347 ideal.sync_kit(this); 348 } 349 ideal.end_if(); 350 sync_kit(ideal); 351 return; 352 } else if (!ary_t->is_not_null_free()) { 353 // Array is not flattened but may be null free 354 assert(elemtype->is_oopptr()->can_be_value_type() && !ary_t->klass_is_exact(), "array can't be null free"); 355 ary = gen_value_array_null_guard(ary, cast_val, 3, true); 356 } 357 } 358 359 access_store_at(ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY); 360 } 361 362 363 //------------------------------array_addressing------------------------------- 364 // Pull array and index from the stack. Compute pointer-to-element. 365 Node* Parse::array_addressing(BasicType type, int vals, const Type* *result2) { 366 Node *idx = peek(0+vals); // Get from stack without popping 367 Node *ary = peek(1+vals); // in case of exception 368 |