131 const TypeAryPtr* ary_src = src_type->isa_aryptr(); 132 assert (ary_src != NULL, "not an array or instance?"); 133 // clone passes a length as a rounded number of longs. If we're 134 // cloning an array we'll do it element by element. If the 135 // length input to ArrayCopyNode is constant, length of input 136 // array must be too. 137 138 assert((get_length_if_constant(phase) == -1) == !ary_src->size()->is_con() || 139 phase->is_IterGVN(), "inconsistent"); 140 141 if (ary_src->size()->is_con()) { 142 return ary_src->size()->get_con(); 143 } 144 return -1; 145 } 146 } 147 148 return get_length_if_constant(phase); 149 } 150 151 Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) { 152 if (!is_clonebasic()) { 153 return NULL; 154 } 155 156 Node* src = in(ArrayCopyNode::Src); 157 Node* dest = in(ArrayCopyNode::Dest); 158 Node* ctl = in(TypeFunc::Control); 159 Node* in_mem = in(TypeFunc::Memory); 160 161 const Type* src_type = phase->type(src); 162 163 assert(src->is_AddP(), "should be base + off"); 164 assert(dest->is_AddP(), "should be base + off"); 165 Node* base_src = src->in(AddPNode::Base); 166 Node* base_dest = dest->in(AddPNode::Base); 167 168 MergeMemNode* mem = MergeMemNode::make(in_mem); 169 170 const TypeInstPtr* inst_src = src_type->isa_instptr(); 171 172 if (inst_src == NULL) { 173 return NULL; 174 } 175 176 if (!inst_src->klass_is_exact()) { 177 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass(); 178 assert(!ik->is_interface() && !ik->has_subklass(), "inconsistent klass hierarchy"); 179 phase->C->dependencies()->assert_leaf_type(ik); 180 } 181 182 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass(); 183 assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields"); 184 185 for (int i = 0; i < count; i++) { 186 ciField* field = ik->nonstatic_field_at(i); 187 int fieldidx = phase->C->alias_type(field)->index(); 188 const TypePtr* adr_type = phase->C->alias_type(field)->adr_type(); 189 Node* off = phase->MakeConX(field->offset()); 190 Node* next_src = phase->transform(new AddPNode(base_src,base_src,off)); 191 Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off)); 192 BasicType bt = field->layout_type(); 193 194 const Type *type; 195 if (bt == T_OBJECT) { 196 if (!field->type()->is_loaded()) { 197 type = TypeInstPtr::BOTTOM; 198 } else { 199 ciType* field_klass = field->type(); 200 type = TypeOopPtr::make_from_klass(field_klass->as_klass()); 201 } 202 } else { 203 type = Type::get_const_basic_type(bt); 204 } 205 206 Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered); 207 v = phase->transform(v); 208 Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered); 209 s = phase->transform(s); 210 mem->set_memory_at(fieldidx, s); 211 } 212 213 if (!finish_transform(phase, can_reshape, ctl, mem)) { 214 // Return NodeSentinel to indicate that the transform failed 215 return NodeSentinel; 216 } 217 218 return mem; 219 } 220 221 bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape, 222 Node*& adr_src, 223 Node*& base_src, 224 Node*& adr_dest, 225 Node*& base_dest, 226 BasicType& copy_type, 227 const Type*& value_type, 228 bool& disjoint_bases) { 229 Node* src = in(ArrayCopyNode::Src); 230 Node* dest = in(ArrayCopyNode::Dest); 351 forward_ctl = ctl; 352 } 353 } 354 355 Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase, 356 bool can_reshape, 357 Node*& forward_ctl, 358 MergeMemNode* mm, 359 const TypePtr* atp_src, 360 const TypePtr* atp_dest, 361 Node* adr_src, 362 Node* base_src, 363 Node* adr_dest, 364 Node* base_dest, 365 BasicType copy_type, 366 const Type* value_type, 367 int count) { 368 if (!forward_ctl->is_top()) { 369 // copy forward 370 mm = mm->clone()->as_MergeMem(); 371 uint alias_idx_src = phase->C->get_alias_index(atp_src); 372 uint alias_idx_dest = phase->C->get_alias_index(atp_dest); 373 Node *start_mem_src = mm->memory_at(alias_idx_src); 374 Node *start_mem_dest = mm->memory_at(alias_idx_dest); 375 Node* mem = start_mem_dest; 376 bool same_alias = (alias_idx_src == alias_idx_dest); 377 378 if (count > 0) { 379 Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered); 380 v = phase->transform(v); 381 mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered); 382 mem = phase->transform(mem); 383 for (int i = 1; i < count; i++) { 384 Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); 385 Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); 386 Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); 387 v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered); 388 v = phase->transform(v); 389 mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered); 390 mem = phase->transform(mem); 391 } 392 mm->set_memory_at(alias_idx_dest, mem); 393 } else if(can_reshape) { 394 PhaseIterGVN* igvn = phase->is_IterGVN(); 395 igvn->_worklist.push(adr_src); 396 igvn->_worklist.push(adr_dest); 397 } 398 return mm; 399 } 400 return phase->C->top(); 401 } 402 403 Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase, 404 bool can_reshape, 405 Node*& backward_ctl, 406 MergeMemNode* mm, 407 const TypePtr* atp_src, 408 const TypePtr* atp_dest, 409 Node* adr_src, 410 Node* base_src, 411 Node* adr_dest, 412 Node* base_dest, 413 BasicType copy_type, 414 const Type* value_type, 415 int count) { 416 if (!backward_ctl->is_top()) { 417 // copy backward 418 mm = mm->clone()->as_MergeMem(); 419 uint alias_idx_src = phase->C->get_alias_index(atp_src); 420 uint alias_idx_dest = phase->C->get_alias_index(atp_dest); 421 Node *start_mem_src = mm->memory_at(alias_idx_src); 422 Node *start_mem_dest = mm->memory_at(alias_idx_dest); 423 Node* mem = start_mem_dest; 424 425 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 426 assert(copy_type != T_OBJECT || !bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization), "only tightly coupled allocations for object arrays"); 427 bool same_alias = (alias_idx_src == alias_idx_dest); 428 429 if (count > 0) { 430 for (int i = count-1; i >= 1; i--) { 431 Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); 432 Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); 433 Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); 434 Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered); 435 v = phase->transform(v); 436 mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered); 437 mem = phase->transform(mem); 438 } 439 Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered); 440 v = phase->transform(v); 441 mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered); 442 mem = phase->transform(mem); 443 mm->set_memory_at(alias_idx_dest, mem); 444 } else if(can_reshape) { 445 PhaseIterGVN* igvn = phase->is_IterGVN(); 446 igvn->_worklist.push(adr_src); 447 igvn->_worklist.push(adr_dest); 448 } 449 return phase->transform(mm); 450 } 451 return phase->C->top(); 452 } 453 454 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape, 455 Node* ctl, Node *mem) { 456 if (can_reshape) { 457 PhaseIterGVN* igvn = phase->is_IterGVN(); 458 igvn->set_delay_transform(false); 459 if (is_clonebasic()) { 460 Node* out_mem = proj_out(TypeFunc::Memory); 461 462 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 463 if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() || | 131 const TypeAryPtr* ary_src = src_type->isa_aryptr(); 132 assert (ary_src != NULL, "not an array or instance?"); 133 // clone passes a length as a rounded number of longs. If we're 134 // cloning an array we'll do it element by element. If the 135 // length input to ArrayCopyNode is constant, length of input 136 // array must be too. 137 138 assert((get_length_if_constant(phase) == -1) == !ary_src->size()->is_con() || 139 phase->is_IterGVN(), "inconsistent"); 140 141 if (ary_src->size()->is_con()) { 142 return ary_src->size()->get_con(); 143 } 144 return -1; 145 } 146 } 147 148 return get_length_if_constant(phase); 149 } 150 151 Node* ArrayCopyNode::load(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, const Type *type, BasicType bt) { 152 DecoratorSet decorators = C2_READ_ACCESS | C2_CONTROL_DEPENDENT_LOAD | IN_HEAP | C2_ARRAY_COPY; 153 C2AccessValuePtr addr(adr, adr_type); 154 C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr); 155 Node* res = bs->load_at(access, type); 156 ctl = access.ctl(); 157 return res; 158 } 159 160 void ArrayCopyNode::store(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, Node* val, const Type *type, BasicType bt) { 161 DecoratorSet decorators = C2_WRITE_ACCESS | IN_HEAP | C2_ARRAY_COPY; 162 if (is_alloc_tightly_coupled()) { 163 decorators |= C2_TIGHLY_COUPLED_ALLOC; 164 } 165 C2AccessValuePtr addr(adr, adr_type); 166 C2AccessValue value(val, type); 167 C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr); 168 bs->store_at(access, value); 169 ctl = access.ctl(); 170 } 171 172 173 Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) { 174 if (!is_clonebasic()) { 175 return NULL; 176 } 177 178 Node* src = in(ArrayCopyNode::Src); 179 Node* dest = in(ArrayCopyNode::Dest); 180 Node* ctl = in(TypeFunc::Control); 181 Node* in_mem = in(TypeFunc::Memory); 182 183 const Type* src_type = phase->type(src); 184 185 assert(src->is_AddP(), "should be base + off"); 186 assert(dest->is_AddP(), "should be base + off"); 187 Node* base_src = src->in(AddPNode::Base); 188 Node* base_dest = dest->in(AddPNode::Base); 189 190 MergeMemNode* mem = MergeMemNode::make(in_mem); 191 192 const TypeInstPtr* inst_src = src_type->isa_instptr(); 193 194 if (inst_src == NULL) { 195 return NULL; 196 } 197 198 if (!inst_src->klass_is_exact()) { 199 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass(); 200 assert(!ik->is_interface() && !ik->has_subklass(), "inconsistent klass hierarchy"); 201 phase->C->dependencies()->assert_leaf_type(ik); 202 } 203 204 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass(); 205 assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields"); 206 207 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 208 for (int i = 0; i < count; i++) { 209 ciField* field = ik->nonstatic_field_at(i); 210 int fieldidx = phase->C->alias_type(field)->index(); 211 const TypePtr* adr_type = phase->C->alias_type(field)->adr_type(); 212 Node* off = phase->MakeConX(field->offset()); 213 Node* next_src = phase->transform(new AddPNode(base_src,base_src,off)); 214 Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off)); 215 BasicType bt = field->layout_type(); 216 217 const Type *type; 218 if (bt == T_OBJECT) { 219 if (!field->type()->is_loaded()) { 220 type = TypeInstPtr::BOTTOM; 221 } else { 222 ciType* field_klass = field->type(); 223 type = TypeOopPtr::make_from_klass(field_klass->as_klass()); 224 } 225 } else { 226 type = Type::get_const_basic_type(bt); 227 } 228 229 Node* v = load(bs, phase, ctl, mem, next_src, adr_type, type, bt); 230 store(bs, phase, ctl, mem, next_dest, adr_type, v, type, bt); 231 } 232 233 if (!finish_transform(phase, can_reshape, ctl, mem)) { 234 // Return NodeSentinel to indicate that the transform failed 235 return NodeSentinel; 236 } 237 238 return mem; 239 } 240 241 bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape, 242 Node*& adr_src, 243 Node*& base_src, 244 Node*& adr_dest, 245 Node*& base_dest, 246 BasicType& copy_type, 247 const Type*& value_type, 248 bool& disjoint_bases) { 249 Node* src = in(ArrayCopyNode::Src); 250 Node* dest = in(ArrayCopyNode::Dest); 371 forward_ctl = ctl; 372 } 373 } 374 375 Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase, 376 bool can_reshape, 377 Node*& forward_ctl, 378 MergeMemNode* mm, 379 const TypePtr* atp_src, 380 const TypePtr* atp_dest, 381 Node* adr_src, 382 Node* base_src, 383 Node* adr_dest, 384 Node* base_dest, 385 BasicType copy_type, 386 const Type* value_type, 387 int count) { 388 if (!forward_ctl->is_top()) { 389 // copy forward 390 mm = mm->clone()->as_MergeMem(); 391 392 if (count > 0) { 393 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 394 Node* v = load(bs, phase, forward_ctl, mm, adr_src, atp_src, value_type, copy_type); 395 store(bs, phase, forward_ctl, mm, adr_dest, atp_dest, v, value_type, copy_type); 396 for (int i = 1; i < count; i++) { 397 Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); 398 Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); 399 Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); 400 v = load(bs, phase, forward_ctl, mm, next_src, atp_src, value_type, copy_type); 401 store(bs, phase, forward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type); 402 } 403 } else if(can_reshape) { 404 PhaseIterGVN* igvn = phase->is_IterGVN(); 405 igvn->_worklist.push(adr_src); 406 igvn->_worklist.push(adr_dest); 407 } 408 return mm; 409 } 410 return phase->C->top(); 411 } 412 413 Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase, 414 bool can_reshape, 415 Node*& backward_ctl, 416 MergeMemNode* mm, 417 const TypePtr* atp_src, 418 const TypePtr* atp_dest, 419 Node* adr_src, 420 Node* base_src, 421 Node* adr_dest, 422 Node* base_dest, 423 BasicType copy_type, 424 const Type* value_type, 425 int count) { 426 if (!backward_ctl->is_top()) { 427 // copy backward 428 mm = mm->clone()->as_MergeMem(); 429 430 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 431 assert(copy_type != T_OBJECT || !bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization), "only tightly coupled allocations for object arrays"); 432 433 if (count > 0) { 434 for (int i = count-1; i >= 1; i--) { 435 Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); 436 Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); 437 Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); 438 Node* v = load(bs, phase, backward_ctl, mm, next_src, atp_src, value_type, copy_type); 439 store(bs, phase, backward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type); 440 } 441 Node* v = load(bs, phase, backward_ctl, mm, adr_src, atp_src, value_type, copy_type); 442 store(bs, phase, backward_ctl, mm, adr_dest, atp_dest, v, value_type, copy_type); 443 } else if(can_reshape) { 444 PhaseIterGVN* igvn = phase->is_IterGVN(); 445 igvn->_worklist.push(adr_src); 446 igvn->_worklist.push(adr_dest); 447 } 448 return phase->transform(mm); 449 } 450 return phase->C->top(); 451 } 452 453 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape, 454 Node* ctl, Node *mem) { 455 if (can_reshape) { 456 PhaseIterGVN* igvn = phase->is_IterGVN(); 457 igvn->set_delay_transform(false); 458 if (is_clonebasic()) { 459 Node* out_mem = proj_out(TypeFunc::Memory); 460 461 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 462 if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() || |