< prev index next >

src/hotspot/share/opto/arraycopynode.cpp

Print this page




 131       const TypeAryPtr* ary_src = src_type->isa_aryptr();
 132       assert (ary_src != NULL, "not an array or instance?");
 133       // clone passes a length as a rounded number of longs. If we're
 134       // cloning an array we'll do it element by element. If the
 135       // length input to ArrayCopyNode is constant, length of input
 136       // array must be too.
 137 
 138       assert((get_length_if_constant(phase) == -1) == !ary_src->size()->is_con() ||
 139              phase->is_IterGVN(), "inconsistent");
 140 
 141       if (ary_src->size()->is_con()) {
 142         return ary_src->size()->get_con();
 143       }
 144       return -1;
 145     }
 146   }
 147 
 148   return get_length_if_constant(phase);
 149 }
 150 






















 151 Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) {
 152   if (!is_clonebasic()) {
 153     return NULL;
 154   }
 155 
 156   Node* src = in(ArrayCopyNode::Src);
 157   Node* dest = in(ArrayCopyNode::Dest);
 158   Node* ctl = in(TypeFunc::Control);
 159   Node* in_mem = in(TypeFunc::Memory);
 160 
 161   const Type* src_type = phase->type(src);
 162 
 163   assert(src->is_AddP(), "should be base + off");
 164   assert(dest->is_AddP(), "should be base + off");
 165   Node* base_src = src->in(AddPNode::Base);
 166   Node* base_dest = dest->in(AddPNode::Base);
 167 
 168   MergeMemNode* mem = MergeMemNode::make(in_mem);
 169 
 170   const TypeInstPtr* inst_src = src_type->isa_instptr();
 171 
 172   if (inst_src == NULL) {
 173     return NULL;
 174   }
 175 
 176   if (!inst_src->klass_is_exact()) {
 177     ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
 178     assert(!ik->is_interface() && !ik->has_subklass(), "inconsistent klass hierarchy");
 179     phase->C->dependencies()->assert_leaf_type(ik);
 180   }
 181 
 182   ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
 183   assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields");
 184 

 185   for (int i = 0; i < count; i++) {
 186     ciField* field = ik->nonstatic_field_at(i);
 187     int fieldidx = phase->C->alias_type(field)->index();
 188     const TypePtr* adr_type = phase->C->alias_type(field)->adr_type();
 189     Node* off = phase->MakeConX(field->offset());
 190     Node* next_src = phase->transform(new AddPNode(base_src,base_src,off));
 191     Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off));
 192     BasicType bt = field->layout_type();
 193 
 194     const Type *type;
 195     if (bt == T_OBJECT) {
 196       if (!field->type()->is_loaded()) {
 197         type = TypeInstPtr::BOTTOM;
 198       } else {
 199         ciType* field_klass = field->type();
 200         type = TypeOopPtr::make_from_klass(field_klass->as_klass());
 201       }
 202     } else {
 203       type = Type::get_const_basic_type(bt);
 204     }
 205 
 206     Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered);
 207     v = phase->transform(v);
 208     Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered);
 209     s = phase->transform(s);
 210     mem->set_memory_at(fieldidx, s);
 211   }
 212 
 213   if (!finish_transform(phase, can_reshape, ctl, mem)) {
 214     // Return NodeSentinel to indicate that the transform failed
 215     return NodeSentinel;
 216   }
 217 
 218   return mem;
 219 }
 220 
 221 bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape,
 222                                        Node*& adr_src,
 223                                        Node*& base_src,
 224                                        Node*& adr_dest,
 225                                        Node*& base_dest,
 226                                        BasicType& copy_type,
 227                                        const Type*& value_type,
 228                                        bool& disjoint_bases) {
 229   Node* src = in(ArrayCopyNode::Src);
 230   Node* dest = in(ArrayCopyNode::Dest);


 351     forward_ctl = ctl;
 352   }
 353 }
 354 
 355 Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase,
 356                                         bool can_reshape,
 357                                         Node*& forward_ctl,
 358                                         MergeMemNode* mm,
 359                                         const TypePtr* atp_src,
 360                                         const TypePtr* atp_dest,
 361                                         Node* adr_src,
 362                                         Node* base_src,
 363                                         Node* adr_dest,
 364                                         Node* base_dest,
 365                                         BasicType copy_type,
 366                                         const Type* value_type,
 367                                         int count) {
 368   if (!forward_ctl->is_top()) {
 369     // copy forward
 370     mm = mm->clone()->as_MergeMem();
 371     uint alias_idx_src = phase->C->get_alias_index(atp_src);
 372     uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
 373     Node *start_mem_src = mm->memory_at(alias_idx_src);
 374     Node *start_mem_dest = mm->memory_at(alias_idx_dest);
 375     Node* mem = start_mem_dest;
 376     bool same_alias = (alias_idx_src == alias_idx_dest);
 377 
 378     if (count > 0) {
 379       Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
 380       v = phase->transform(v);
 381       mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
 382       mem = phase->transform(mem);
 383       for (int i = 1; i < count; i++) {
 384         Node* off  = phase->MakeConX(type2aelembytes(copy_type) * i);
 385         Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
 386         Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
 387         v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
 388         v = phase->transform(v);
 389         mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
 390         mem = phase->transform(mem);
 391       }
 392       mm->set_memory_at(alias_idx_dest, mem);
 393     } else if(can_reshape) {
 394       PhaseIterGVN* igvn = phase->is_IterGVN();
 395       igvn->_worklist.push(adr_src);
 396       igvn->_worklist.push(adr_dest);
 397     }
 398     return mm;
 399   }
 400   return phase->C->top();
 401 }
 402 
 403 Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase,
 404                                          bool can_reshape,
 405                                          Node*& backward_ctl,
 406                                          MergeMemNode* mm,
 407                                          const TypePtr* atp_src,
 408                                          const TypePtr* atp_dest,
 409                                          Node* adr_src,
 410                                          Node* base_src,
 411                                          Node* adr_dest,
 412                                          Node* base_dest,
 413                                          BasicType copy_type,
 414                                          const Type* value_type,
 415                                          int count) {
 416   if (!backward_ctl->is_top()) {
 417     // copy backward
 418     mm = mm->clone()->as_MergeMem();
 419     uint alias_idx_src = phase->C->get_alias_index(atp_src);
 420     uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
 421     Node *start_mem_src = mm->memory_at(alias_idx_src);
 422     Node *start_mem_dest = mm->memory_at(alias_idx_dest);
 423     Node* mem = start_mem_dest;
 424 
 425     BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 426     assert(copy_type != T_OBJECT || !bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization), "only tightly coupled allocations for object arrays");
 427     bool same_alias = (alias_idx_src == alias_idx_dest);
 428 
 429     if (count > 0) {
 430       for (int i = count-1; i >= 1; i--) {
 431         Node* off  = phase->MakeConX(type2aelembytes(copy_type) * i);
 432         Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
 433         Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
 434         Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
 435         v = phase->transform(v);
 436         mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
 437         mem = phase->transform(mem);
 438       }
 439       Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
 440       v = phase->transform(v);
 441       mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
 442       mem = phase->transform(mem);
 443       mm->set_memory_at(alias_idx_dest, mem);
 444     } else if(can_reshape) {
 445       PhaseIterGVN* igvn = phase->is_IterGVN();
 446       igvn->_worklist.push(adr_src);
 447       igvn->_worklist.push(adr_dest);
 448     }
 449     return phase->transform(mm);
 450   }
 451   return phase->C->top();
 452 }
 453 
 454 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
 455                                      Node* ctl, Node *mem) {
 456   if (can_reshape) {
 457     PhaseIterGVN* igvn = phase->is_IterGVN();
 458     igvn->set_delay_transform(false);
 459     if (is_clonebasic()) {
 460       Node* out_mem = proj_out(TypeFunc::Memory);
 461 
 462       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 463       if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() ||




 131       const TypeAryPtr* ary_src = src_type->isa_aryptr();
 132       assert (ary_src != NULL, "not an array or instance?");
 133       // clone passes a length as a rounded number of longs. If we're
 134       // cloning an array we'll do it element by element. If the
 135       // length input to ArrayCopyNode is constant, length of input
 136       // array must be too.
 137 
 138       assert((get_length_if_constant(phase) == -1) == !ary_src->size()->is_con() ||
 139              phase->is_IterGVN(), "inconsistent");
 140 
 141       if (ary_src->size()->is_con()) {
 142         return ary_src->size()->get_con();
 143       }
 144       return -1;
 145     }
 146   }
 147 
 148   return get_length_if_constant(phase);
 149 }
 150 
 151 Node* ArrayCopyNode::load(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, const Type *type, BasicType bt) {
 152   DecoratorSet decorators = C2_READ_ACCESS | C2_CONTROL_DEPENDENT_LOAD | IN_HEAP | C2_ARRAY_COPY;
 153   C2AccessValuePtr addr(adr, adr_type);
 154   C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr);
 155   Node* res = bs->load_at(access, type);
 156   ctl = access.ctl();
 157   return res;
 158 }
 159 
 160 void ArrayCopyNode::store(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, Node* val, const Type *type, BasicType bt) {
 161   DecoratorSet decorators = C2_WRITE_ACCESS | IN_HEAP | C2_ARRAY_COPY;
 162   if (is_alloc_tightly_coupled()) {
 163     decorators |= C2_TIGHLY_COUPLED_ALLOC;
 164   }
 165   C2AccessValuePtr addr(adr, adr_type);
 166   C2AccessValue value(val, type);
 167   C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr);
 168   bs->store_at(access, value);
 169   ctl = access.ctl();
 170 }
 171 
 172 
 173 Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) {
 174   if (!is_clonebasic()) {
 175     return NULL;
 176   }
 177 
 178   Node* src = in(ArrayCopyNode::Src);
 179   Node* dest = in(ArrayCopyNode::Dest);
 180   Node* ctl = in(TypeFunc::Control);
 181   Node* in_mem = in(TypeFunc::Memory);
 182 
 183   const Type* src_type = phase->type(src);
 184 
 185   assert(src->is_AddP(), "should be base + off");
 186   assert(dest->is_AddP(), "should be base + off");
 187   Node* base_src = src->in(AddPNode::Base);
 188   Node* base_dest = dest->in(AddPNode::Base);
 189 
 190   MergeMemNode* mem = MergeMemNode::make(in_mem);
 191 
 192   const TypeInstPtr* inst_src = src_type->isa_instptr();
 193 
 194   if (inst_src == NULL) {
 195     return NULL;
 196   }
 197 
 198   if (!inst_src->klass_is_exact()) {
 199     ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
 200     assert(!ik->is_interface() && !ik->has_subklass(), "inconsistent klass hierarchy");
 201     phase->C->dependencies()->assert_leaf_type(ik);
 202   }
 203 
 204   ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
 205   assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields");
 206 
 207   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 208   for (int i = 0; i < count; i++) {
 209     ciField* field = ik->nonstatic_field_at(i);
 210     int fieldidx = phase->C->alias_type(field)->index();
 211     const TypePtr* adr_type = phase->C->alias_type(field)->adr_type();
 212     Node* off = phase->MakeConX(field->offset());
 213     Node* next_src = phase->transform(new AddPNode(base_src,base_src,off));
 214     Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off));
 215     BasicType bt = field->layout_type();
 216 
 217     const Type *type;
 218     if (bt == T_OBJECT) {
 219       if (!field->type()->is_loaded()) {
 220         type = TypeInstPtr::BOTTOM;
 221       } else {
 222         ciType* field_klass = field->type();
 223         type = TypeOopPtr::make_from_klass(field_klass->as_klass());
 224       }
 225     } else {
 226       type = Type::get_const_basic_type(bt);
 227     }
 228 
 229     Node* v = load(bs, phase, ctl, mem, next_src, adr_type, type, bt);
 230     store(bs, phase, ctl, mem, next_dest, adr_type, v, type, bt);



 231   }
 232 
 233   if (!finish_transform(phase, can_reshape, ctl, mem)) {
 234     // Return NodeSentinel to indicate that the transform failed
 235     return NodeSentinel;
 236   }
 237 
 238   return mem;
 239 }
 240 
 241 bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape,
 242                                        Node*& adr_src,
 243                                        Node*& base_src,
 244                                        Node*& adr_dest,
 245                                        Node*& base_dest,
 246                                        BasicType& copy_type,
 247                                        const Type*& value_type,
 248                                        bool& disjoint_bases) {
 249   Node* src = in(ArrayCopyNode::Src);
 250   Node* dest = in(ArrayCopyNode::Dest);


 371     forward_ctl = ctl;
 372   }
 373 }
 374 
 375 Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase,
 376                                         bool can_reshape,
 377                                         Node*& forward_ctl,
 378                                         MergeMemNode* mm,
 379                                         const TypePtr* atp_src,
 380                                         const TypePtr* atp_dest,
 381                                         Node* adr_src,
 382                                         Node* base_src,
 383                                         Node* adr_dest,
 384                                         Node* base_dest,
 385                                         BasicType copy_type,
 386                                         const Type* value_type,
 387                                         int count) {
 388   if (!forward_ctl->is_top()) {
 389     // copy forward
 390     mm = mm->clone()->as_MergeMem();






 391 
 392     if (count > 0) {
 393       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 394       Node* v = load(bs, phase, forward_ctl, mm, adr_src, atp_src, value_type, copy_type);
 395       store(bs, phase, forward_ctl, mm, adr_dest, atp_dest, v, value_type, copy_type);

 396       for (int i = 1; i < count; i++) {
 397         Node* off  = phase->MakeConX(type2aelembytes(copy_type) * i);
 398         Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
 399         Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
 400         v = load(bs, phase, forward_ctl, mm, next_src, atp_src, value_type, copy_type);
 401         store(bs, phase, forward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type);


 402       }

 403     } else if(can_reshape) {
 404       PhaseIterGVN* igvn = phase->is_IterGVN();
 405       igvn->_worklist.push(adr_src);
 406       igvn->_worklist.push(adr_dest);
 407     }
 408     return mm;
 409   }
 410   return phase->C->top();
 411 }
 412 
 413 Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase,
 414                                          bool can_reshape,
 415                                          Node*& backward_ctl,
 416                                          MergeMemNode* mm,
 417                                          const TypePtr* atp_src,
 418                                          const TypePtr* atp_dest,
 419                                          Node* adr_src,
 420                                          Node* base_src,
 421                                          Node* adr_dest,
 422                                          Node* base_dest,
 423                                          BasicType copy_type,
 424                                          const Type* value_type,
 425                                          int count) {
 426   if (!backward_ctl->is_top()) {
 427     // copy backward
 428     mm = mm->clone()->as_MergeMem();





 429 
 430     BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 431     assert(copy_type != T_OBJECT || !bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization), "only tightly coupled allocations for object arrays");

 432 
 433     if (count > 0) {
 434       for (int i = count-1; i >= 1; i--) {
 435         Node* off  = phase->MakeConX(type2aelembytes(copy_type) * i);
 436         Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
 437         Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
 438         Node* v = load(bs, phase, backward_ctl, mm, next_src, atp_src, value_type, copy_type);
 439         store(bs, phase, backward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type);
 440       }
 441       Node* v = load(bs, phase, backward_ctl, mm, adr_src, atp_src, value_type, copy_type);
 442       store(bs, phase, backward_ctl, mm, adr_dest, atp_dest, v, value_type, copy_type);





 443     } else if(can_reshape) {
 444       PhaseIterGVN* igvn = phase->is_IterGVN();
 445       igvn->_worklist.push(adr_src);
 446       igvn->_worklist.push(adr_dest);
 447     }
 448     return phase->transform(mm);
 449   }
 450   return phase->C->top();
 451 }
 452 
 453 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
 454                                      Node* ctl, Node *mem) {
 455   if (can_reshape) {
 456     PhaseIterGVN* igvn = phase->is_IterGVN();
 457     igvn->set_delay_transform(false);
 458     if (is_clonebasic()) {
 459       Node* out_mem = proj_out(TypeFunc::Memory);
 460 
 461       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 462       if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() ||


< prev index next >