46 #include "opto/valuetypenode.hpp" 47 #include "runtime/deoptimization.hpp" 48 #include "runtime/sharedRuntime.hpp" 49 50 #ifndef PRODUCT 51 extern int explicit_null_checks_inserted, 52 explicit_null_checks_elided; 53 #endif 54 55 //---------------------------------array_load---------------------------------- 56 void Parse::array_load(BasicType bt) { 57 const Type* elemtype = Type::TOP; 58 Node* adr = array_addressing(bt, 0, &elemtype); 59 if (stopped()) return; // guaranteed null or range check 60 61 Node* idx = pop(); 62 Node* ary = pop(); 63 64 // Handle value type arrays 65 const TypeOopPtr* elemptr = elemtype->make_oopptr(); 66 if (elemtype->isa_valuetype() != NULL) { 67 // Load from flattened value type array 68 ciValueKlass* vk = elemtype->is_valuetype()->value_klass(); 69 ValueTypeNode* vt = ValueTypeNode::make_from_flattened(this, vk, ary, adr); 70 push(vt); 71 return; 72 } else if (elemptr != NULL && elemptr->is_valuetypeptr()) { 73 // Load from non-flattened value type array (elements can never be null) 74 bt = T_VALUETYPE; 75 assert(elemptr->meet(TypePtr::NULL_PTR) != elemptr, "value type array elements should never be null"); 76 } else if (ValueArrayFlatten && elemptr != NULL && elemptr->can_be_value_type()) { 77 // Cannot statically determine if array is flattened, emit runtime check 78 gen_flattened_array_guard(ary, 2); 79 } 80 81 if (elemtype == TypeInt::BOOL) { 82 bt = T_BOOLEAN; 83 } else if (bt == T_OBJECT) { 84 elemtype = _gvn.type(ary)->is_aryptr()->elem()->make_oopptr(); 85 } 86 87 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 88 Node* ld = access_load_at(ary, adr, adr_type, elemtype, bt, 89 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD); 90 if (bt == T_VALUETYPE) { 91 // Loading a non-flattened (but flattenable) value type from an array 92 assert(!gvn().type(ld)->is_ptr()->maybe_null(), "value type array elements should never be null"); 93 ld = ValueTypeNode::make_from_oop(this, ld, elemptr->value_klass()); 94 } 95 96 push_node(bt, ld); 97 } 98 99 100 //--------------------------------array_store---------------------------------- 101 void Parse::array_store(BasicType bt) { 102 const Type* elemtype = Type::TOP; 103 Node* adr = array_addressing(bt, type2size[bt], &elemtype); 104 if (stopped()) return; // guaranteed null or range check 105 Node* cast_val = NULL; 106 if (bt == T_OBJECT) { 107 cast_val = array_store_check(); 108 if (stopped()) return; 109 } 110 Node* val = pop_node(bt); // Value to store 111 Node* idx = pop(); // Index in the array 112 Node* ary = pop(); // The array itself 113 114 if (bt == T_OBJECT) { 115 const TypeOopPtr* elemptr = elemtype->make_oopptr(); 116 if (elemtype->isa_valuetype() != NULL) { 117 // Store to flattened value type array 118 cast_val->as_ValueType()->store_flattened(this, ary, adr); 119 return; 120 } else if (elemptr->is_valuetypeptr()) { 121 // Store to non-flattened value type array 122 } else if (ValueArrayFlatten && elemptr->can_be_value_type() && val->is_ValueType()) { 123 IdealKit ideal(this); 124 Node* kls = load_object_klass(ary); 125 Node* lhp = basic_plus_adr(kls, in_bytes(Klass::layout_helper_offset())); 126 Node* layout_val = make_load(NULL, lhp, TypeInt::INT, T_INT, MemNode::unordered); 127 layout_val = _gvn.transform(new RShiftINode(layout_val, intcon(Klass::_lh_array_tag_shift))); 128 ideal.if_then(layout_val, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); { 129 // non flattened 130 sync_kit(ideal); 131 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 132 elemtype = _gvn.type(ary)->is_aryptr()->elem()->make_oopptr(); 133 access_store_at(control(), ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY); 134 ideal.sync_kit(this); 135 } ideal.else_(); { 136 // flattened 137 sync_kit(ideal); 138 // Object/interface array must be flattened, cast it 139 const TypeValueType* vt = _gvn.type(val)->is_valuetype(); 140 ciArrayKlass* array_klass = ciArrayKlass::make(vt->value_klass()); 141 const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); 142 ary = _gvn.transform(new CheckCastPPNode(control(), ary, arytype)); 143 adr = array_element_address(ary, idx, T_OBJECT, arytype->size(), control()); 144 val->as_ValueType()->store_flattened(this, ary, adr); 145 ideal.sync_kit(this); 146 } ideal.end_if(); 147 sync_kit(ideal); 148 return; 149 } 150 } 151 152 if (elemtype == TypeInt::BOOL) { 153 bt = T_BOOLEAN; 154 } else if (bt == T_OBJECT) { 155 elemtype = _gvn.type(ary)->is_aryptr()->elem()->make_oopptr(); 156 } 157 158 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 159 160 access_store_at(control(), ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY); 161 } 162 163 164 //------------------------------array_addressing------------------------------- 165 // Pull array and index from the stack. Compute pointer-to-element. 166 Node* Parse::array_addressing(BasicType type, int vals, const Type* *result2) { 167 Node *idx = peek(0+vals); // Get from stack without popping 168 Node *ary = peek(1+vals); // in case of exception 169 170 // Null check the array base, with correct stack contents 171 ary = null_check(ary, T_ARRAY); 172 // Compile-time detect of null-exception? 173 if (stopped()) return top(); 174 175 const TypeAryPtr* arytype = _gvn.type(ary)->is_aryptr(); | 46 #include "opto/valuetypenode.hpp" 47 #include "runtime/deoptimization.hpp" 48 #include "runtime/sharedRuntime.hpp" 49 50 #ifndef PRODUCT 51 extern int explicit_null_checks_inserted, 52 explicit_null_checks_elided; 53 #endif 54 55 //---------------------------------array_load---------------------------------- 56 void Parse::array_load(BasicType bt) { 57 const Type* elemtype = Type::TOP; 58 Node* adr = array_addressing(bt, 0, &elemtype); 59 if (stopped()) return; // guaranteed null or range check 60 61 Node* idx = pop(); 62 Node* ary = pop(); 63 64 // Handle value type arrays 65 const TypeOopPtr* elemptr = elemtype->make_oopptr(); 66 const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr(); 67 if (elemtype->isa_valuetype() != NULL) { 68 // Load from flattened value type array 69 ciValueKlass* vk = elemtype->is_valuetype()->value_klass(); 70 ValueTypeNode* vt = ValueTypeNode::make_from_flattened(this, vk, ary, adr); 71 push(vt); 72 return; 73 } else if (elemptr != NULL && elemptr->is_valuetypeptr()) { 74 // Load from non-flattened value type array (elements can never be null) 75 bt = T_VALUETYPE; 76 assert(elemptr->meet(TypePtr::NULL_PTR) != elemptr, "value type array elements should never be null"); 77 } else if (ValueArrayFlatten && elemptr != NULL && elemptr->can_be_value_type() && 78 !ary_t->klass_is_exact()) { 79 // Cannot statically determine if array is flattened, emit runtime check 80 IdealKit ideal(this); 81 IdealVariable res(ideal); 82 ideal.declarations_done(); 83 Node* kls = load_object_klass(ary); 84 Node* lhp = basic_plus_adr(kls, in_bytes(Klass::layout_helper_offset())); 85 Node* layout_val = make_load(NULL, lhp, TypeInt::INT, T_INT, MemNode::unordered); 86 Node* tag = _gvn.transform(new RShiftINode(layout_val, intcon(Klass::_lh_array_tag_shift))); 87 ideal.if_then(tag, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); { 88 // non flattened 89 sync_kit(ideal); 90 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 91 elemtype = ary_t->elem()->make_oopptr(); 92 Node* ld = access_load_at(ary, adr, adr_type, elemtype, bt, 93 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD); 94 ideal.sync_kit(this); 95 ideal.set(res, ld); 96 } ideal.else_(); { 97 // flattened 98 sync_kit(ideal); 99 Node* k_adr = basic_plus_adr(kls, in_bytes(ArrayKlass::element_klass_offset())); 100 Node* elem_klass = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), k_adr, TypeInstPtr::KLASS)); 101 Node* obj_size = NULL; 102 kill_dead_locals(); 103 inc_sp(2); 104 Node* alloc_obj = new_instance(elem_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true); 105 dec_sp(2); 106 107 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); 108 assert(alloc->maybe_set_complete(&_gvn), ""); 109 alloc->initialization()->set_complete_with_arraycopy(); 110 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 111 // Unknown value type so might have reference fields 112 if (!bs->array_copy_requires_gc_barriers(T_OBJECT)) { 113 int base_off = sizeof(instanceOopDesc); 114 Node* dst_base = basic_plus_adr(alloc_obj, base_off); 115 Node* countx = obj_size; 116 countx = _gvn.transform(new SubXNode(countx, MakeConX(base_off))); 117 countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong))); 118 119 assert(Klass::_lh_log2_element_size_shift == 0, "use shift in place"); 120 Node* elem_shift = layout_val; 121 uint header = arrayOopDesc::base_offset_in_bytes(T_VALUETYPE); 122 Node* base = basic_plus_adr(ary, header); 123 idx = Compile::conv_I2X_index(&_gvn, idx, TypeInt::POS, control()); 124 Node* scale = _gvn.transform(new LShiftXNode(idx, elem_shift)); 125 Node* adr = basic_plus_adr(ary, base, scale); 126 127 access_clone(control(), adr, dst_base, countx, false); 128 } else { 129 ideal.sync_kit(this); 130 ideal.make_leaf_call(OptoRuntime::load_unknown_value_Type(), 131 CAST_FROM_FN_PTR(address, OptoRuntime::load_unknown_value), 132 "load_unknown_value", 133 ary, idx, alloc_obj); 134 sync_kit(ideal); 135 } 136 137 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 138 139 ideal.sync_kit(this); 140 ideal.set(res, alloc_obj); 141 } ideal.end_if(); 142 sync_kit(ideal); 143 push_node(bt, ideal.value(res)); 144 return; 145 } 146 147 if (elemtype == TypeInt::BOOL) { 148 bt = T_BOOLEAN; 149 } else if (bt == T_OBJECT) { 150 elemtype = ary_t->elem()->make_oopptr(); 151 } 152 153 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 154 Node* ld = access_load_at(ary, adr, adr_type, elemtype, bt, 155 IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD); 156 if (bt == T_VALUETYPE) { 157 // Loading a non-flattened (but flattenable) value type from an array 158 assert(!gvn().type(ld)->is_ptr()->maybe_null(), "value type array elements should never be null"); 159 ld = ValueTypeNode::make_from_oop(this, ld, elemptr->value_klass()); 160 } 161 162 push_node(bt, ld); 163 } 164 165 166 //--------------------------------array_store---------------------------------- 167 void Parse::array_store(BasicType bt) { 168 const Type* elemtype = Type::TOP; 169 Node* adr = array_addressing(bt, type2size[bt], &elemtype); 170 if (stopped()) return; // guaranteed null or range check 171 Node* cast_val = NULL; 172 if (bt == T_OBJECT) { 173 cast_val = array_store_check(); 174 if (stopped()) return; 175 } 176 Node* val = pop_node(bt); // Value to store 177 Node* idx = pop(); // Index in the array 178 Node* ary = pop(); // The array itself 179 180 const TypeAryPtr* ary_t = _gvn.type(ary)->is_aryptr(); 181 if (bt == T_OBJECT) { 182 const TypeOopPtr* elemptr = elemtype->make_oopptr(); 183 const Type* val_t = _gvn.type(val); 184 if (elemtype->isa_valuetype() != NULL) { 185 // Store to flattened value type array 186 if (!val->is_ValueType() && val_t == TypePtr::NULL_PTR) { 187 // Can not store null into a value type array 188 inc_sp(3); 189 uncommon_trap(Deoptimization::Reason_null_check, Deoptimization::Action_none); 190 return; 191 } 192 cast_val->as_ValueType()->store_flattened(this, ary, adr); 193 return; 194 } else if (elemptr->is_valuetypeptr()) { 195 // Store to non-flattened value type array 196 if (!val->is_ValueType() && val_t == TypePtr::NULL_PTR) { 197 // Can not store null into a value type array 198 inc_sp(3); 199 uncommon_trap(Deoptimization::Reason_null_check, Deoptimization::Action_none); 200 return; 201 } 202 } else if (elemptr->can_be_value_type() && !ary_t->klass_is_exact() && 203 (val->is_ValueType() || val_t == TypePtr::NULL_PTR || val_t->is_oopptr()->can_be_value_type())) { 204 if (ValueArrayFlatten) { 205 IdealKit ideal(this); 206 Node* kls = load_object_klass(ary); 207 Node* lhp = basic_plus_adr(kls, in_bytes(Klass::layout_helper_offset())); 208 Node* layout_val = make_load(NULL, lhp, TypeInt::INT, T_INT, MemNode::unordered); 209 layout_val = _gvn.transform(new RShiftINode(layout_val, intcon(Klass::_lh_array_tag_shift))); 210 ideal.if_then(layout_val, BoolTest::ne, intcon(Klass::_lh_array_tag_vt_value)); { 211 // non flattened 212 sync_kit(ideal); 213 if (!val->is_ValueType() && TypePtr::NULL_PTR->higher_equal(val_t)) { 214 Node* null_ctl = top(); 215 Node* not_null_oop = null_check_oop(val, &null_ctl); 216 { 217 assert(null_ctl != top(), "expected to possibly be null") ; 218 PreserveJVMState pjvms(this); 219 set_control(null_ctl); 220 inc_sp(3); 221 uncommon_trap(Deoptimization::Reason_null_check, Deoptimization::Action_none); 222 } 223 } 224 225 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 226 elemtype = ary_t->elem()->make_oopptr(); 227 access_store_at(control(), ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY); 228 ideal.sync_kit(this); 229 } ideal.else_(); { 230 // flattened 231 // Object/interface array must be flattened, cast it 232 if (val->is_ValueType()) { 233 sync_kit(ideal); 234 const TypeValueType* vt = _gvn.type(val)->is_valuetype(); 235 ciArrayKlass* array_klass = ciArrayKlass::make(vt->value_klass()); 236 const TypeAryPtr* arytype = TypeOopPtr::make_from_klass(array_klass)->isa_aryptr(); 237 ary = _gvn.transform(new CheckCastPPNode(control(), ary, arytype)); 238 adr = array_element_address(ary, idx, T_OBJECT, arytype->size(), control()); 239 val->as_ValueType()->store_flattened(this, ary, adr); 240 ideal.sync_kit(this); 241 } else { 242 if (TypePtr::NULL_PTR->higher_equal(val_t)) { 243 sync_kit(ideal); 244 gen_value_type_array_guard(ary, val, NULL, 3); 245 ideal.sync_kit(this); 246 } 247 ideal.make_leaf_call(OptoRuntime::store_unknown_value_Type(), 248 CAST_FROM_FN_PTR(address, OptoRuntime::store_unknown_value), 249 "store_unknown_value", 250 val, ary, idx); 251 } 252 } ideal.end_if(); 253 sync_kit(ideal); 254 return; 255 } else { 256 if (!val->is_ValueType() && TypePtr::NULL_PTR->higher_equal(val_t)) { 257 gen_value_type_array_guard(ary, val, NULL, 3); 258 } 259 } 260 } 261 } 262 263 if (elemtype == TypeInt::BOOL) { 264 bt = T_BOOLEAN; 265 } else if (bt == T_OBJECT) { 266 elemtype = ary_t->elem()->make_oopptr(); 267 } 268 269 const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); 270 271 access_store_at(control(), ary, adr, adr_type, val, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY); 272 } 273 274 275 //------------------------------array_addressing------------------------------- 276 // Pull array and index from the stack. Compute pointer-to-element. 277 Node* Parse::array_addressing(BasicType type, int vals, const Type* *result2) { 278 Node *idx = peek(0+vals); // Get from stack without popping 279 Node *ary = peek(1+vals); // in case of exception 280 281 // Null check the array base, with correct stack contents 282 ary = null_check(ary, T_ARRAY); 283 // Compile-time detect of null-exception? 284 if (stopped()) return top(); 285 286 const TypeAryPtr* arytype = _gvn.type(ary)->is_aryptr(); |