< prev index next >
src/hotspot/share/opto/arraycopynode.cpp
Print this page
*** 30,40 ****
#include "opto/graphKit.hpp"
#include "runtime/sharedRuntime.hpp"
#include "utilities/macros.hpp"
ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard)
! : CallNode(arraycopy_type(), NULL, TypeRawPtr::BOTTOM),
_kind(None),
_alloc_tightly_coupled(alloc_tightly_coupled),
_has_negative_length_guard(has_negative_length_guard),
_arguments_validated(false),
_src_type(TypeOopPtr::BOTTOM),
--- 30,40 ----
#include "opto/graphKit.hpp"
#include "runtime/sharedRuntime.hpp"
#include "utilities/macros.hpp"
ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard)
! : CallNode(arraycopy_type(), NULL, TypePtr::BOTTOM),
_kind(None),
_alloc_tightly_coupled(alloc_tightly_coupled),
_has_negative_length_guard(has_negative_length_guard),
_arguments_validated(false),
_src_type(TypeOopPtr::BOTTOM),
*** 255,266 ****
// We don't know if arguments are arrays of the same type
return false;
}
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
! if (dest_elem == T_OBJECT && (!is_alloc_tightly_coupled() ||
! bs->array_copy_requires_gc_barriers(T_OBJECT))) {
// It's an object array copy but we can't emit the card marking
// that is needed
return false;
}
--- 255,265 ----
// We don't know if arguments are arrays of the same type
return false;
}
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
! if (bs->array_copy_requires_gc_barriers(is_alloc_tightly_coupled(), dest_elem, false, BarrierSetC2::Optimization)) {
// It's an object array copy but we can't emit the card marking
// that is needed
return false;
}
*** 305,314 ****
--- 304,318 ----
assert(phase->type(src->in(AddPNode::Offset))->is_intptr_t()->get_con() == phase->type(dest->in(AddPNode::Offset))->is_intptr_t()->get_con(), "same start offset?");
BasicType elem = ary_src->klass()->as_array_klass()->element_type()->basic_type();
if (elem == T_ARRAY) elem = T_OBJECT;
+ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+ if (bs->array_copy_requires_gc_barriers(true, elem, true, BarrierSetC2::Optimization)) {
+ return false;
+ }
+
int diff = arrayOopDesc::base_offset_in_bytes(elem) - phase->type(src->in(AddPNode::Offset))->is_intptr_t()->get_con();
assert(diff >= 0, "clone should not start after 1st array element");
if (diff > 0) {
adr_src = phase->transform(new AddPNode(base_src, adr_src, phase->MakeConX(diff)));
adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, phase->MakeConX(diff)));
*** 348,375 ****
}
}
Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase,
bool can_reshape,
! Node* forward_ctl,
! Node* start_mem_src,
! Node* start_mem_dest,
const TypePtr* atp_src,
const TypePtr* atp_dest,
Node* adr_src,
Node* base_src,
Node* adr_dest,
Node* base_dest,
BasicType copy_type,
const Type* value_type,
int count) {
- Node* mem = phase->C->top();
if (!forward_ctl->is_top()) {
// copy forward
! mem = start_mem_dest;
uint alias_idx_src = phase->C->get_alias_index(atp_src);
uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
bool same_alias = (alias_idx_src == alias_idx_dest);
if (count > 0) {
Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
v = phase->transform(v);
--- 352,380 ----
}
}
Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase,
bool can_reshape,
! Node*& forward_ctl,
! MergeMemNode* mm,
const TypePtr* atp_src,
const TypePtr* atp_dest,
Node* adr_src,
Node* base_src,
Node* adr_dest,
Node* base_dest,
BasicType copy_type,
const Type* value_type,
int count) {
if (!forward_ctl->is_top()) {
// copy forward
! mm = mm->clone()->as_MergeMem();
uint alias_idx_src = phase->C->get_alias_index(atp_src);
uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
+ Node *start_mem_src = mm->memory_at(alias_idx_src);
+ Node *start_mem_dest = mm->memory_at(alias_idx_dest);
+ Node* mem = start_mem_dest;
bool same_alias = (alias_idx_src == alias_idx_dest);
if (count > 0) {
Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
v = phase->transform(v);
*** 382,420 ****
v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
v = phase->transform(v);
mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
mem = phase->transform(mem);
}
} else if(can_reshape) {
PhaseIterGVN* igvn = phase->is_IterGVN();
igvn->_worklist.push(adr_src);
igvn->_worklist.push(adr_dest);
}
}
! return mem;
}
Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase,
bool can_reshape,
! Node* backward_ctl,
! Node* start_mem_src,
! Node* start_mem_dest,
const TypePtr* atp_src,
const TypePtr* atp_dest,
Node* adr_src,
Node* base_src,
Node* adr_dest,
Node* base_dest,
BasicType copy_type,
const Type* value_type,
int count) {
- Node* mem = phase->C->top();
if (!backward_ctl->is_top()) {
// copy backward
! mem = start_mem_dest;
uint alias_idx_src = phase->C->get_alias_index(atp_src);
uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
bool same_alias = (alias_idx_src == alias_idx_dest);
if (count > 0) {
for (int i = count-1; i >= 1; i--) {
Node* off = phase->MakeConX(type2aelembytes(copy_type) * i);
--- 387,431 ----
v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered);
v = phase->transform(v);
mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
mem = phase->transform(mem);
}
+ mm->set_memory_at(alias_idx_dest, mem);
} else if(can_reshape) {
PhaseIterGVN* igvn = phase->is_IterGVN();
igvn->_worklist.push(adr_src);
igvn->_worklist.push(adr_dest);
}
+ return mm;
}
! return phase->C->top();
}
Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase,
bool can_reshape,
! Node*& backward_ctl,
! MergeMemNode* mm,
const TypePtr* atp_src,
const TypePtr* atp_dest,
Node* adr_src,
Node* base_src,
Node* adr_dest,
Node* base_dest,
BasicType copy_type,
const Type* value_type,
int count) {
if (!backward_ctl->is_top()) {
// copy backward
! mm = mm->clone()->as_MergeMem();
uint alias_idx_src = phase->C->get_alias_index(atp_src);
uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
+ Node *start_mem_src = mm->memory_at(alias_idx_src);
+ Node *start_mem_dest = mm->memory_at(alias_idx_dest);
+ Node* mem = start_mem_dest;
+
+ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+ assert(copy_type != T_OBJECT || !bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization), "only tightly coupled allocations for object arrays");
bool same_alias = (alias_idx_src == alias_idx_dest);
if (count > 0) {
for (int i = count-1; i >= 1; i--) {
Node* off = phase->MakeConX(type2aelembytes(copy_type) * i);
*** 427,443 ****
}
Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
v = phase->transform(v);
mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
mem = phase->transform(mem);
} else if(can_reshape) {
PhaseIterGVN* igvn = phase->is_IterGVN();
igvn->_worklist.push(adr_src);
igvn->_worklist.push(adr_dest);
}
}
! return mem;
}
bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
Node* ctl, Node *mem) {
if (can_reshape) {
--- 438,456 ----
}
Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
v = phase->transform(v);
mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
mem = phase->transform(mem);
+ mm->set_memory_at(alias_idx_dest, mem);
} else if(can_reshape) {
PhaseIterGVN* igvn = phase->is_IterGVN();
igvn->_worklist.push(adr_src);
igvn->_worklist.push(adr_dest);
}
+ return phase->transform(mm);
}
! return phase->C->top();
}
bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
Node* ctl, Node *mem) {
if (can_reshape) {
*** 447,457 ****
Node* out_mem = proj_out(TypeFunc::Memory);
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() ||
out_mem->raw_out(0)->outcnt() != 1 || !out_mem->raw_out(0)->raw_out(0)->is_MemBar()) {
! assert(bs->array_copy_requires_gc_barriers(T_OBJECT), "can only happen with card marking");
return false;
}
igvn->replace_node(out_mem->raw_out(0), mem);
--- 460,470 ----
Node* out_mem = proj_out(TypeFunc::Memory);
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() ||
out_mem->raw_out(0)->outcnt() != 1 || !out_mem->raw_out(0)->raw_out(0)->is_MemBar()) {
! assert(bs->array_copy_requires_gc_barriers(true, T_OBJECT, true, BarrierSetC2::Optimization), "can only happen with card marking");
return false;
}
igvn->replace_node(out_mem->raw_out(0), mem);
*** 484,493 ****
--- 497,507 ----
}
} else {
if (in(TypeFunc::Control) != ctl) {
// we can't return new memory and control from Ideal at parse time
assert(!is_clonebasic(), "added control for clone?");
+ phase->record_for_igvn(this);
return false;
}
}
return true;
}
*** 557,575 ****
Node* src = in(ArrayCopyNode::Src);
Node* dest = in(ArrayCopyNode::Dest);
const TypePtr* atp_src = get_address_type(phase, src);
const TypePtr* atp_dest = get_address_type(phase, dest);
- uint alias_idx_src = phase->C->get_alias_index(atp_src);
- uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
Node *in_mem = in(TypeFunc::Memory);
! Node *start_mem_src = in_mem;
! Node *start_mem_dest = in_mem;
! if (in_mem->is_MergeMem()) {
! start_mem_src = in_mem->as_MergeMem()->memory_at(alias_idx_src);
! start_mem_dest = in_mem->as_MergeMem()->memory_at(alias_idx_dest);
}
if (can_reshape) {
assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms");
--- 571,584 ----
Node* src = in(ArrayCopyNode::Src);
Node* dest = in(ArrayCopyNode::Dest);
const TypePtr* atp_src = get_address_type(phase, src);
const TypePtr* atp_dest = get_address_type(phase, dest);
Node *in_mem = in(TypeFunc::Memory);
! if (!in_mem->is_MergeMem()) {
! in_mem = MergeMemNode::make(in_mem);
}
if (can_reshape) {
assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms");
*** 579,609 ****
Node* backward_ctl = phase->C->top();
Node* forward_ctl = phase->C->top();
array_copy_test_overlap(phase, can_reshape, disjoint_bases, count, forward_ctl, backward_ctl);
Node* forward_mem = array_copy_forward(phase, can_reshape, forward_ctl,
! start_mem_src, start_mem_dest,
atp_src, atp_dest,
adr_src, base_src, adr_dest, base_dest,
copy_type, value_type, count);
Node* backward_mem = array_copy_backward(phase, can_reshape, backward_ctl,
! start_mem_src, start_mem_dest,
atp_src, atp_dest,
adr_src, base_src, adr_dest, base_dest,
copy_type, value_type, count);
Node* ctl = NULL;
if (!forward_ctl->is_top() && !backward_ctl->is_top()) {
ctl = new RegionNode(3);
- mem = new PhiNode(ctl, Type::MEMORY, atp_dest);
ctl->init_req(1, forward_ctl);
- mem->init_req(1, forward_mem);
ctl->init_req(2, backward_ctl);
- mem->init_req(2, backward_mem);
ctl = phase->transform(ctl);
! mem = phase->transform(mem);
} else if (!forward_ctl->is_top()) {
ctl = forward_ctl;
mem = forward_mem;
} else {
assert(!backward_ctl->is_top(), "no copy?");
--- 588,626 ----
Node* backward_ctl = phase->C->top();
Node* forward_ctl = phase->C->top();
array_copy_test_overlap(phase, can_reshape, disjoint_bases, count, forward_ctl, backward_ctl);
Node* forward_mem = array_copy_forward(phase, can_reshape, forward_ctl,
! in_mem->as_MergeMem(),
atp_src, atp_dest,
adr_src, base_src, adr_dest, base_dest,
copy_type, value_type, count);
Node* backward_mem = array_copy_backward(phase, can_reshape, backward_ctl,
! in_mem->as_MergeMem(),
atp_src, atp_dest,
adr_src, base_src, adr_dest, base_dest,
copy_type, value_type, count);
Node* ctl = NULL;
if (!forward_ctl->is_top() && !backward_ctl->is_top()) {
ctl = new RegionNode(3);
ctl->init_req(1, forward_ctl);
ctl->init_req(2, backward_ctl);
ctl = phase->transform(ctl);
! MergeMemNode* forward_mm = forward_mem->as_MergeMem();
! MergeMemNode* backward_mm = backward_mem->as_MergeMem();
! for (MergeMemStream mms(forward_mm, backward_mm); mms.next_non_empty2(); ) {
! if (mms.memory() != mms.memory2()) {
! Node* phi = new PhiNode(ctl, Type::MEMORY, phase->C->get_adr_type(mms.alias_idx()));
! phi->init_req(1, mms.memory());
! phi->init_req(2, mms.memory2());
! phi = phase->transform(phi);
! mms.set_memory(phi);
! }
! }
! mem = forward_mem;
} else if (!forward_ctl->is_top()) {
ctl = forward_ctl;
mem = forward_mem;
} else {
assert(!backward_ctl->is_top(), "no copy?");
*** 614,627 ****
if (can_reshape) {
assert(phase->is_IterGVN()->delay_transform(), "should be delaying transforms");
phase->is_IterGVN()->set_delay_transform(false);
}
- MergeMemNode* out_mem = MergeMemNode::make(in_mem);
- out_mem->set_memory_at(alias_idx_dest, mem);
- mem = out_mem;
-
if (!finish_transform(phase, can_reshape, ctl, mem)) {
return NULL;
}
return mem;
--- 631,640 ----
< prev index next >