172 transform_later(bol_lt);
173 generate_guard(ctrl, bol_lt, region, PROB_MIN);
174 }
175
176 Node* PhaseMacroExpand::generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative) {
177 if ((*ctrl)->is_top()) return NULL;
178
179 if (_igvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
180 return NULL; // index is already adequately typed
181 Node* cmp_le = new CmpINode(index, intcon(0));
182 transform_later(cmp_le);
183 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
184 Node* bol_le = new BoolNode(cmp_le, le_or_eq);
185 transform_later(bol_le);
186 Node* is_notp = generate_guard(ctrl, bol_le, NULL, PROB_MIN);
187
188 return is_notp;
189 }
190
191 Node* PhaseMacroExpand::generate_flattened_array_guard(Node** ctrl, Node* mem, Node* obj_or_klass, RegionNode* region) {
192 return generate_array_guard(ctrl, mem, obj_or_klass, region, Klass::_lh_array_tag_vt_value);
193 }
194
195 Node* PhaseMacroExpand::generate_object_array_guard(Node** ctrl, Node* mem, Node* obj_or_klass, RegionNode* region) {
196 return generate_array_guard(ctrl, mem, obj_or_klass, region, Klass::_lh_array_tag_obj_value);
197 }
198
199 Node* PhaseMacroExpand::generate_array_guard(Node** ctrl, Node* mem, Node* obj_or_klass, RegionNode* region, jint lh_con) {
200 if ((*ctrl)->is_top()) return NULL;
201
202 Node* kls = NULL;
203 if (_igvn.type(obj_or_klass)->isa_oopptr()) {
204 Node* k_adr = basic_plus_adr(obj_or_klass, oopDesc::klass_offset_in_bytes());
205 kls = transform_later(LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), k_adr, TypeInstPtr::KLASS));
206 } else {
207 assert(_igvn.type(obj_or_klass)->isa_klassptr(), "what else?");
208 kls = obj_or_klass;
209 }
210 Node* layout_val = make_load(NULL, mem, kls, in_bytes(Klass::layout_helper_offset()), TypeInt::INT, T_INT);
211
1306 if (ac->is_arraycopy_validated() &&
1307 dest_elem != T_CONFLICT &&
1308 src_elem == T_CONFLICT) {
1309 src_elem = dest_elem;
1310 }
1311
1312 if (src_elem == T_CONFLICT || dest_elem == T_CONFLICT) {
1313 // Conservatively insert a memory barrier on all memory slices.
1314 // Do not let writes into the source float below the arraycopy.
1315 {
1316 Node* mem = ac->in(TypeFunc::Memory);
1317 insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
1318
1319 merge_mem = MergeMemNode::make(mem);
1320 transform_later(merge_mem);
1321 }
1322
1323 RegionNode* slow_region = new RegionNode(1);
1324 transform_later(slow_region);
1325
1326 generate_flattened_array_guard(&ctrl, merge_mem, dest, slow_region);
1327
1328 // Call StubRoutines::generic_arraycopy stub.
1329 Node* mem = generate_arraycopy(ac, NULL, &ctrl, merge_mem, &io,
1330 TypeRawPtr::BOTTOM, T_CONFLICT,
1331 src, src_offset, dest, dest_offset, length,
1332 NULL,
1333 // If a negative length guard was generated for the ArrayCopyNode,
1334 // the length of the array can never be negative.
1335 false, ac->has_negative_length_guard(),
1336 slow_region);
1337
1338 return;
1339 }
1340
1341 assert(!ac->is_arraycopy_validated() || (src_elem == dest_elem && dest_elem != T_VOID), "validated but different basic types");
1342
1343 // (2) src and dest arrays must have elements of the same BasicType
1344 // Figure out the size and type of the elements we will be copying.
1345 //
1346 // We have no stub to copy flattened value type arrays with oop
1406
1407 // (7) src_offset + length must not exceed length of src.
1408 Node* alen = ac->in(ArrayCopyNode::SrcLen);
1409 assert(alen != NULL, "need src len");
1410 generate_limit_guard(&ctrl,
1411 src_offset, length,
1412 alen,
1413 slow_region);
1414
1415 // (8) dest_offset + length must not exceed length of dest.
1416 alen = ac->in(ArrayCopyNode::DestLen);
1417 assert(alen != NULL, "need dest len");
1418 generate_limit_guard(&ctrl,
1419 dest_offset, length,
1420 alen,
1421 slow_region);
1422
1423 // (9) each element of an oop array must be assignable
1424 // The generate_arraycopy subroutine checks this.
1425
1426 if (dest_elem == T_OBJECT &&
1427 ValueArrayFlatten &&
1428 top_dest->elem()->make_oopptr()->can_be_value_type()) {
1429 generate_flattened_array_guard(&ctrl, merge_mem, dest, slow_region);
1430 }
1431
1432 if (src_elem == T_OBJECT &&
1433 ValueArrayFlatten &&
1434 top_src->elem()->make_oopptr()->can_be_value_type()) {
1435 generate_flattened_array_guard(&ctrl, merge_mem, src, slow_region);
1436 }
1437 }
1438
1439 // This is where the memory effects are placed:
1440 const TypePtr* adr_type = NULL;
1441
1442 Node* dest_length = alloc != NULL ? alloc->in(AllocateNode::ALength) : NULL;
1443
1444 if (dest_elem == T_VALUETYPE) {
1445 adr_type = adjust_parameters_for_vt(top_dest, src_offset, dest_offset, length, dest_elem, dest_length);
1446 } else if (ac->_dest_type != TypeOopPtr::BOTTOM) {
1447 adr_type = ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr();
1448 } else {
1449 adr_type = TypeAryPtr::get_array_body_type(dest_elem);
1450 }
1451
1452 generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
1453 adr_type, dest_elem,
1454 src, src_offset, dest, dest_offset, length,
|
172 transform_later(bol_lt);
173 generate_guard(ctrl, bol_lt, region, PROB_MIN);
174 }
175
176 Node* PhaseMacroExpand::generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative) {
177 if ((*ctrl)->is_top()) return NULL;
178
179 if (_igvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
180 return NULL; // index is already adequately typed
181 Node* cmp_le = new CmpINode(index, intcon(0));
182 transform_later(cmp_le);
183 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
184 Node* bol_le = new BoolNode(cmp_le, le_or_eq);
185 transform_later(bol_le);
186 Node* is_notp = generate_guard(ctrl, bol_le, NULL, PROB_MIN);
187
188 return is_notp;
189 }
190
191 Node* PhaseMacroExpand::generate_flattened_array_guard(Node** ctrl, Node* mem, Node* obj_or_klass, RegionNode* region) {
192 assert(ValueArrayFlatten, "can never be flattened");
193 return generate_array_guard(ctrl, mem, obj_or_klass, region, Klass::_lh_array_tag_vt_value);
194 }
195
196 Node* PhaseMacroExpand::generate_object_array_guard(Node** ctrl, Node* mem, Node* obj_or_klass, RegionNode* region) {
197 return generate_array_guard(ctrl, mem, obj_or_klass, region, Klass::_lh_array_tag_obj_value);
198 }
199
200 Node* PhaseMacroExpand::generate_array_guard(Node** ctrl, Node* mem, Node* obj_or_klass, RegionNode* region, jint lh_con) {
201 if ((*ctrl)->is_top()) return NULL;
202
203 Node* kls = NULL;
204 if (_igvn.type(obj_or_klass)->isa_oopptr()) {
205 Node* k_adr = basic_plus_adr(obj_or_klass, oopDesc::klass_offset_in_bytes());
206 kls = transform_later(LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), k_adr, TypeInstPtr::KLASS));
207 } else {
208 assert(_igvn.type(obj_or_klass)->isa_klassptr(), "what else?");
209 kls = obj_or_klass;
210 }
211 Node* layout_val = make_load(NULL, mem, kls, in_bytes(Klass::layout_helper_offset()), TypeInt::INT, T_INT);
212
1307 if (ac->is_arraycopy_validated() &&
1308 dest_elem != T_CONFLICT &&
1309 src_elem == T_CONFLICT) {
1310 src_elem = dest_elem;
1311 }
1312
1313 if (src_elem == T_CONFLICT || dest_elem == T_CONFLICT) {
1314 // Conservatively insert a memory barrier on all memory slices.
1315 // Do not let writes into the source float below the arraycopy.
1316 {
1317 Node* mem = ac->in(TypeFunc::Memory);
1318 insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
1319
1320 merge_mem = MergeMemNode::make(mem);
1321 transform_later(merge_mem);
1322 }
1323
1324 RegionNode* slow_region = new RegionNode(1);
1325 transform_later(slow_region);
1326
1327 if (ValueArrayFlatten && (top_dest == NULL || !top_dest->is_not_flat())) {
1328 generate_flattened_array_guard(&ctrl, merge_mem, dest, slow_region);
1329 }
1330
1331 // Call StubRoutines::generic_arraycopy stub.
1332 Node* mem = generate_arraycopy(ac, NULL, &ctrl, merge_mem, &io,
1333 TypeRawPtr::BOTTOM, T_CONFLICT,
1334 src, src_offset, dest, dest_offset, length,
1335 NULL,
1336 // If a negative length guard was generated for the ArrayCopyNode,
1337 // the length of the array can never be negative.
1338 false, ac->has_negative_length_guard(),
1339 slow_region);
1340
1341 return;
1342 }
1343
1344 assert(!ac->is_arraycopy_validated() || (src_elem == dest_elem && dest_elem != T_VOID), "validated but different basic types");
1345
1346 // (2) src and dest arrays must have elements of the same BasicType
1347 // Figure out the size and type of the elements we will be copying.
1348 //
1349 // We have no stub to copy flattened value type arrays with oop
1409
1410 // (7) src_offset + length must not exceed length of src.
1411 Node* alen = ac->in(ArrayCopyNode::SrcLen);
1412 assert(alen != NULL, "need src len");
1413 generate_limit_guard(&ctrl,
1414 src_offset, length,
1415 alen,
1416 slow_region);
1417
1418 // (8) dest_offset + length must not exceed length of dest.
1419 alen = ac->in(ArrayCopyNode::DestLen);
1420 assert(alen != NULL, "need dest len");
1421 generate_limit_guard(&ctrl,
1422 dest_offset, length,
1423 alen,
1424 slow_region);
1425
1426 // (9) each element of an oop array must be assignable
1427 // The generate_arraycopy subroutine checks this.
1428
1429 if (dest_elem == T_OBJECT && !top_dest->elem()->isa_valuetype() && !top_dest->is_not_flat()) {
1430 generate_flattened_array_guard(&ctrl, merge_mem, dest, slow_region);
1431 }
1432
1433 if (src_elem == T_OBJECT && !top_src->elem()->isa_valuetype() && !top_src->is_not_flat()) {
1434 generate_flattened_array_guard(&ctrl, merge_mem, src, slow_region);
1435 }
1436 }
1437
1438 // This is where the memory effects are placed:
1439 const TypePtr* adr_type = NULL;
1440
1441 Node* dest_length = alloc != NULL ? alloc->in(AllocateNode::ALength) : NULL;
1442
1443 if (dest_elem == T_VALUETYPE) {
1444 adr_type = adjust_parameters_for_vt(top_dest, src_offset, dest_offset, length, dest_elem, dest_length);
1445 } else if (ac->_dest_type != TypeOopPtr::BOTTOM) {
1446 adr_type = ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr();
1447 } else {
1448 adr_type = TypeAryPtr::get_array_body_type(dest_elem);
1449 }
1450
1451 generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
1452 adr_type, dest_elem,
1453 src, src_offset, dest, dest_offset, length,
|