165 Node* base_src = src->in(AddPNode::Base);
166 Node* base_dest = dest->in(AddPNode::Base);
167
168 MergeMemNode* mem = MergeMemNode::make(in_mem);
169
170 const TypeInstPtr* inst_src = src_type->isa_instptr();
171
172 if (inst_src == NULL) {
173 return NULL;
174 }
175
176 if (!inst_src->klass_is_exact()) {
177 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
178 assert(!ik->is_interface() && !ik->has_subklass(), "inconsistent klass hierarchy");
179 phase->C->dependencies()->assert_leaf_type(ik);
180 }
181
182 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
183 assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields");
184
185 for (int i = 0; i < count; i++) {
186 ciField* field = ik->nonstatic_field_at(i);
187 int fieldidx = phase->C->alias_type(field)->index();
188 const TypePtr* adr_type = phase->C->alias_type(field)->adr_type();
189 Node* off = phase->MakeConX(field->offset());
190 Node* next_src = phase->transform(new AddPNode(base_src,base_src,off));
191 Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off));
192 BasicType bt = field->layout_type();
193
194 const Type *type;
195 if (bt == T_OBJECT) {
196 if (!field->type()->is_loaded()) {
197 type = TypeInstPtr::BOTTOM;
198 } else {
199 ciType* field_klass = field->type();
200 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
201 }
202 } else {
203 type = Type::get_const_basic_type(bt);
204 }
205
206 Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered);
207 v = phase->transform(v);
208 Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered);
209 s = phase->transform(s);
210 mem->set_memory_at(fieldidx, s);
211 }
212
213 if (!finish_transform(phase, can_reshape, ctl, mem)) {
214 // Return NodeSentinel to indicate that the transform failed
215 return NodeSentinel;
216 }
217
218 return mem;
219 }
220
221 bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape,
222 Node*& adr_src,
223 Node*& base_src,
224 Node*& adr_dest,
225 Node*& base_dest,
226 BasicType& copy_type,
227 const Type*& value_type,
359 const TypePtr* atp_src,
360 const TypePtr* atp_dest,
361 Node* adr_src,
362 Node* base_src,
363 Node* adr_dest,
364 Node* base_dest,
365 BasicType copy_type,
366 const Type* value_type,
367 int count) {
368 if (!forward_ctl->is_top()) {
369 // copy forward
370 mm = mm->clone()->as_MergeMem();
371 uint alias_idx_src = phase->C->get_alias_index(atp_src);
372 uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
373 Node *start_mem_src = mm->memory_at(alias_idx_src);
374 Node *start_mem_dest = mm->memory_at(alias_idx_dest);
375 Node* mem = start_mem_dest;
376 bool same_alias = (alias_idx_src == alias_idx_dest);
377
378 if (count > 0) {
379 Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
380 v = phase->transform(v);
381 mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
382 mem = phase->transform(mem);
383 for (int i = 1; i < count; i++) {
384 Node* off = phase->MakeConX(type2aelembytes(copy_type) * i);
385 Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
386 Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
387 v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
388 v = phase->transform(v);
389 mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
390 mem = phase->transform(mem);
391 }
392 mm->set_memory_at(alias_idx_dest, mem);
393 } else if(can_reshape) {
394 PhaseIterGVN* igvn = phase->is_IterGVN();
395 igvn->_worklist.push(adr_src);
396 igvn->_worklist.push(adr_dest);
397 }
398 return mm;
399 }
400 return phase->C->top();
401 }
402
403 Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase,
404 bool can_reshape,
405 Node*& backward_ctl,
406 MergeMemNode* mm,
407 const TypePtr* atp_src,
408 const TypePtr* atp_dest,
416 if (!backward_ctl->is_top()) {
417 // copy backward
418 mm = mm->clone()->as_MergeMem();
419 uint alias_idx_src = phase->C->get_alias_index(atp_src);
420 uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
421 Node *start_mem_src = mm->memory_at(alias_idx_src);
422 Node *start_mem_dest = mm->memory_at(alias_idx_dest);
423 Node* mem = start_mem_dest;
424
425 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
426 assert(copy_type != T_OBJECT || !bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization), "only tightly coupled allocations for object arrays");
427 bool same_alias = (alias_idx_src == alias_idx_dest);
428
429 if (count > 0) {
430 for (int i = count-1; i >= 1; i--) {
431 Node* off = phase->MakeConX(type2aelembytes(copy_type) * i);
432 Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
433 Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
434 Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
435 v = phase->transform(v);
436 mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
437 mem = phase->transform(mem);
438 }
439 Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
440 v = phase->transform(v);
441 mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
442 mem = phase->transform(mem);
443 mm->set_memory_at(alias_idx_dest, mem);
444 } else if(can_reshape) {
445 PhaseIterGVN* igvn = phase->is_IterGVN();
446 igvn->_worklist.push(adr_src);
447 igvn->_worklist.push(adr_dest);
448 }
449 return phase->transform(mm);
450 }
451 return phase->C->top();
452 }
453
454 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
455 Node* ctl, Node *mem) {
456 if (can_reshape) {
457 PhaseIterGVN* igvn = phase->is_IterGVN();
458 igvn->set_delay_transform(false);
459 if (is_clonebasic()) {
460 Node* out_mem = proj_out(TypeFunc::Memory);
|
165 Node* base_src = src->in(AddPNode::Base);
166 Node* base_dest = dest->in(AddPNode::Base);
167
168 MergeMemNode* mem = MergeMemNode::make(in_mem);
169
170 const TypeInstPtr* inst_src = src_type->isa_instptr();
171
172 if (inst_src == NULL) {
173 return NULL;
174 }
175
176 if (!inst_src->klass_is_exact()) {
177 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
178 assert(!ik->is_interface() && !ik->has_subklass(), "inconsistent klass hierarchy");
179 phase->C->dependencies()->assert_leaf_type(ik);
180 }
181
182 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
183 assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields");
184
185 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
186 for (int i = 0; i < count; i++) {
187 ciField* field = ik->nonstatic_field_at(i);
188 int fieldidx = phase->C->alias_type(field)->index();
189 const TypePtr* adr_type = phase->C->alias_type(field)->adr_type();
190 Node* off = phase->MakeConX(field->offset());
191 Node* next_src = phase->transform(new AddPNode(base_src,base_src,off));
192 Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off));
193 BasicType bt = field->layout_type();
194
195 const Type *type;
196 if (bt == T_OBJECT) {
197 if (!field->type()->is_loaded()) {
198 type = TypeInstPtr::BOTTOM;
199 } else {
200 ciType* field_klass = field->type();
201 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
202 }
203 } else {
204 type = Type::get_const_basic_type(bt);
205 }
206
207 Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered);
208 v = phase->transform(v);
209 if (bt == T_OBJECT) {
210 v = bs->array_copy_load_store_barrier(phase, can_reshape, v, mem, ctl);
211 }
212 Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered);
213 s = phase->transform(s);
214 mem->set_memory_at(fieldidx, s);
215 }
216
217 if (!finish_transform(phase, can_reshape, ctl, mem)) {
218 // Return NodeSentinel to indicate that the transform failed
219 return NodeSentinel;
220 }
221
222 return mem;
223 }
224
225 bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape,
226 Node*& adr_src,
227 Node*& base_src,
228 Node*& adr_dest,
229 Node*& base_dest,
230 BasicType& copy_type,
231 const Type*& value_type,
363 const TypePtr* atp_src,
364 const TypePtr* atp_dest,
365 Node* adr_src,
366 Node* base_src,
367 Node* adr_dest,
368 Node* base_dest,
369 BasicType copy_type,
370 const Type* value_type,
371 int count) {
372 if (!forward_ctl->is_top()) {
373 // copy forward
374 mm = mm->clone()->as_MergeMem();
375 uint alias_idx_src = phase->C->get_alias_index(atp_src);
376 uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
377 Node *start_mem_src = mm->memory_at(alias_idx_src);
378 Node *start_mem_dest = mm->memory_at(alias_idx_dest);
379 Node* mem = start_mem_dest;
380 bool same_alias = (alias_idx_src == alias_idx_dest);
381
382 if (count > 0) {
383 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
384 Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
385 v = phase->transform(v);
386 if (copy_type == T_OBJECT) {
387 v = bs->array_copy_load_store_barrier(phase, can_reshape, v, mm, forward_ctl);
388 }
389 mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
390 mem = phase->transform(mem);
391 for (int i = 1; i < count; i++) {
392 Node* off = phase->MakeConX(type2aelembytes(copy_type) * i);
393 Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
394 Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
395 v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
396 v = phase->transform(v);
397 if (copy_type == T_OBJECT) {
398 v = bs->array_copy_load_store_barrier(phase, can_reshape, v, mm, forward_ctl);
399 }
400 mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
401 mem = phase->transform(mem);
402 }
403 mm->set_memory_at(alias_idx_dest, mem);
404 } else if(can_reshape) {
405 PhaseIterGVN* igvn = phase->is_IterGVN();
406 igvn->_worklist.push(adr_src);
407 igvn->_worklist.push(adr_dest);
408 }
409 return mm;
410 }
411 return phase->C->top();
412 }
413
414 Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase,
415 bool can_reshape,
416 Node*& backward_ctl,
417 MergeMemNode* mm,
418 const TypePtr* atp_src,
419 const TypePtr* atp_dest,
427 if (!backward_ctl->is_top()) {
428 // copy backward
429 mm = mm->clone()->as_MergeMem();
430 uint alias_idx_src = phase->C->get_alias_index(atp_src);
431 uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
432 Node *start_mem_src = mm->memory_at(alias_idx_src);
433 Node *start_mem_dest = mm->memory_at(alias_idx_dest);
434 Node* mem = start_mem_dest;
435
436 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
437 assert(copy_type != T_OBJECT || !bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization), "only tightly coupled allocations for object arrays");
438 bool same_alias = (alias_idx_src == alias_idx_dest);
439
440 if (count > 0) {
441 for (int i = count-1; i >= 1; i--) {
442 Node* off = phase->MakeConX(type2aelembytes(copy_type) * i);
443 Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
444 Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
445 Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
446 v = phase->transform(v);
447 if (copy_type == T_OBJECT) {
448 v = bs->array_copy_load_store_barrier(phase, can_reshape, v, mm, backward_ctl);
449 }
450 mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
451 mem = phase->transform(mem);
452 }
453 Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
454 v = phase->transform(v);
455 if (copy_type == T_OBJECT) {
456 v = bs->array_copy_load_store_barrier(phase, can_reshape, v, mm, backward_ctl);
457 }
458 mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
459 mem = phase->transform(mem);
460 mm->set_memory_at(alias_idx_dest, mem);
461 } else if(can_reshape) {
462 PhaseIterGVN* igvn = phase->is_IterGVN();
463 igvn->_worklist.push(adr_src);
464 igvn->_worklist.push(adr_dest);
465 }
466 return phase->transform(mm);
467 }
468 return phase->C->top();
469 }
470
471 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
472 Node* ctl, Node *mem) {
473 if (can_reshape) {
474 PhaseIterGVN* igvn = phase->is_IterGVN();
475 igvn->set_delay_transform(false);
476 if (is_clonebasic()) {
477 Node* out_mem = proj_out(TypeFunc::Memory);
|