< prev index next >
src/share/vm/opto/macro.cpp
Print this page
*** 223,241 ****
}
// Eliminate a card mark sequence. p2x is a ConvP2XNode
void PhaseMacroExpand::eliminate_card_mark(Node* p2x) {
! assert(p2x->Opcode() == Op_CastP2X, "ConvP2XNode required");
if (!UseG1GC) {
// vanilla/CMS post barrier
Node *shift = p2x->unique_out();
Node *addp = shift->unique_out();
for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
Node *mem = addp->last_out(j);
if (UseCondCardMark && mem->is_Load()) {
! assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
// The load is checking if the card has been written so
// replace it with zero to fold the test.
_igvn.replace_node(mem, intcon(0));
continue;
}
--- 223,241 ----
}
// Eliminate a card mark sequence. p2x is a ConvP2XNode
void PhaseMacroExpand::eliminate_card_mark(Node* p2x) {
! assert(p2x->Opcode() == Opcodes::Op_CastP2X, "ConvP2XNode required");
if (!UseG1GC) {
// vanilla/CMS post barrier
Node *shift = p2x->unique_out();
Node *addp = shift->unique_out();
for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
Node *mem = addp->last_out(j);
if (UseCondCardMark && mem->is_Load()) {
! assert(mem->Opcode() == Opcodes::Op_LoadB, "unexpected code shape");
// The load is checking if the card has been written so
// replace it with zero to fold the test.
_igvn.replace_node(mem, intcon(0));
continue;
}
*** 261,271 ****
// Remove G1 post barrier.
// Search for CastP2X->Xor->URShift->Cmp path which
// checks if the store done to a different from the value's region.
// And replace Cmp with #0 (false) to collapse G1 post barrier.
! Node* xorx = p2x->find_out_with(Op_XorX);
if (xorx != NULL) {
Node* shift = xorx->unique_out();
Node* cmpx = shift->unique_out();
assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
--- 261,271 ----
// Remove G1 post barrier.
// Search for CastP2X->Xor->URShift->Cmp path which
// checks if the store done to a different from the value's region.
// And replace Cmp with #0 (false) to collapse G1 post barrier.
! Node* xorx = p2x->find_out_with(Opcodes::Op_XorX);
if (xorx != NULL) {
Node* shift = xorx->unique_out();
Node* cmpx = shift->unique_out();
assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
*** 291,301 ****
cmpx->in(1)->is_Load()) {
Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() +
SATBMarkQueue::byte_offset_of_active());
if (adr->is_AddP() && adr->in(AddPNode::Base) == top() &&
! adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&
adr->in(AddPNode::Offset) == MakeConX(marking_offset)) {
_igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ));
}
}
}
--- 291,301 ----
cmpx->in(1)->is_Load()) {
Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address);
const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() +
SATBMarkQueue::byte_offset_of_active());
if (adr->is_AddP() && adr->in(AddPNode::Base) == top() &&
! adr->in(AddPNode::Address)->Opcode() == Opcodes::Op_ThreadLocal &&
adr->in(AddPNode::Offset) == MakeConX(marking_offset)) {
_igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ));
}
}
}
*** 303,327 ****
} else {
assert(!GraphKit::use_ReduceInitialCardMarks(), "can only happen with card marking");
// This is a G1 post barrier emitted by the Object.clone() intrinsic.
// Search for the CastP2X->URShiftX->AddP->LoadB->Cmp path which checks if the card
// is marked as young_gen and replace the Cmp with 0 (false) to collapse the barrier.
! Node* shift = p2x->find_out_with(Op_URShiftX);
assert(shift != NULL, "missing G1 post barrier");
Node* addp = shift->unique_out();
! Node* load = addp->find_out_with(Op_LoadB);
assert(load != NULL, "missing G1 post barrier");
Node* cmpx = load->unique_out();
assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
"missing card value check in G1 post barrier");
_igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ));
// There is no G1 pre barrier in this case
}
// Now CastP2X can be removed since it is used only on dead path
// which currently still alive until igvn optimize it.
! assert(p2x->outcnt() == 0 || p2x->unique_out()->Opcode() == Op_URShiftX, "");
_igvn.replace_node(p2x, top());
}
}
// Search for a memory operation for the specified memory slice.
--- 303,327 ----
} else {
assert(!GraphKit::use_ReduceInitialCardMarks(), "can only happen with card marking");
// This is a G1 post barrier emitted by the Object.clone() intrinsic.
// Search for the CastP2X->URShiftX->AddP->LoadB->Cmp path which checks if the card
// is marked as young_gen and replace the Cmp with 0 (false) to collapse the barrier.
! Node* shift = p2x->find_out_with(Opcodes::Op_URShiftX);
assert(shift != NULL, "missing G1 post barrier");
Node* addp = shift->unique_out();
! Node* load = addp->find_out_with(Opcodes::Op_LoadB);
assert(load != NULL, "missing G1 post barrier");
Node* cmpx = load->unique_out();
assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() &&
cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne,
"missing card value check in G1 post barrier");
_igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ));
// There is no G1 pre barrier in this case
}
// Now CastP2X can be removed since it is used only on dead path
// which currently still alive until igvn optimize it.
! assert(p2x->outcnt() == 0 || p2x->unique_out()->Opcode() == Opcodes::Op_URShiftX, "");
_igvn.replace_node(p2x, top());
}
}
// Search for a memory operation for the specified memory slice.
*** 387,415 ****
return init;
else
return alloc->in(TypeFunc::Memory); // It will produce zero value (see callers).
}
// Otherwise skip it (the call updated 'mem' value).
! } else if (mem->Opcode() == Op_SCMemProj) {
mem = mem->in(0);
Node* adr = NULL;
if (mem->is_LoadStore()) {
adr = mem->in(MemNode::Address);
} else {
! assert(mem->Opcode() == Op_EncodeISOArray ||
! mem->Opcode() == Op_StrCompressedCopy, "sanity");
adr = mem->in(3); // Destination array
}
const TypePtr* atype = adr->bottom_type()->is_ptr();
int adr_idx = phase->C->get_alias_index(atype);
if (adr_idx == alias_idx) {
DEBUG_ONLY(mem->dump();)
assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field");
return NULL;
}
mem = mem->in(MemNode::Memory);
! } else if (mem->Opcode() == Op_StrInflatedCopy) {
Node* adr = mem->in(3); // Destination array
const TypePtr* atype = adr->bottom_type()->is_ptr();
int adr_idx = phase->C->get_alias_index(atype);
if (adr_idx == alias_idx) {
DEBUG_ONLY(mem->dump();)
--- 387,415 ----
return init;
else
return alloc->in(TypeFunc::Memory); // It will produce zero value (see callers).
}
// Otherwise skip it (the call updated 'mem' value).
! } else if (mem->Opcode() == Opcodes::Op_SCMemProj) {
mem = mem->in(0);
Node* adr = NULL;
if (mem->is_LoadStore()) {
adr = mem->in(MemNode::Address);
} else {
! assert(mem->Opcode() == Opcodes::Op_EncodeISOArray ||
! mem->Opcode() == Opcodes::Op_StrCompressedCopy, "sanity");
adr = mem->in(3); // Destination array
}
const TypePtr* atype = adr->bottom_type()->is_ptr();
int adr_idx = phase->C->get_alias_index(atype);
if (adr_idx == alias_idx) {
DEBUG_ONLY(mem->dump();)
assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field");
return NULL;
}
mem = mem->in(MemNode::Memory);
! } else if (mem->Opcode() == Opcodes::Op_StrInflatedCopy) {
Node* adr = mem->in(3); // Destination array
const TypePtr* atype = adr->bottom_type()->is_ptr();
int adr_idx = phase->C->get_alias_index(atype);
if (adr_idx == alias_idx) {
DEBUG_ONLY(mem->dump();)
*** 541,554 ****
val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1);
if (val == NULL) {
return NULL;
}
values.at_put(j, val);
! } else if (val->Opcode() == Op_SCMemProj) {
assert(val->in(0)->is_LoadStore() ||
! val->in(0)->Opcode() == Op_EncodeISOArray ||
! val->in(0)->Opcode() == Op_StrCompressedCopy, "sanity");
assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field");
return NULL;
} else if (val->is_ArrayCopy()) {
Node* res = make_arraycopy_load(val->as_ArrayCopy(), offset, val->in(0), ft, phi_type, alloc);
if (res == NULL) {
--- 541,554 ----
val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1);
if (val == NULL) {
return NULL;
}
values.at_put(j, val);
! } else if (val->Opcode() == Opcodes::Op_SCMemProj) {
assert(val->in(0)->is_LoadStore() ||
! val->in(0)->Opcode() == Opcodes::Op_EncodeISOArray ||
! val->in(0)->Opcode() == Opcodes::Op_StrCompressedCopy, "sanity");
assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field");
return NULL;
} else if (val->is_ArrayCopy()) {
Node* res = make_arraycopy_load(val->as_ArrayCopy(), offset, val->in(0), ft, phi_type, alloc);
if (res == NULL) {
*** 718,728 ****
break;
}
for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
k < kmax && can_eliminate; k++) {
Node* n = use->fast_out(k);
! if (!n->is_Store() && n->Opcode() != Op_CastP2X &&
!(n->is_ArrayCopy() &&
n->as_ArrayCopy()->is_clonebasic() &&
n->in(ArrayCopyNode::Dest) == use)) {
DEBUG_ONLY(disq_node = n;)
if (n->is_Load() || n->is_LoadStore()) {
--- 718,728 ----
break;
}
for (DUIterator_Fast kmax, k = use->fast_outs(kmax);
k < kmax && can_eliminate; k++) {
Node* n = use->fast_out(k);
! if (!n->is_Store() && n->Opcode() != Opcodes::Op_CastP2X &&
!(n->is_ArrayCopy() &&
n->as_ArrayCopy()->is_clonebasic() &&
n->in(ArrayCopyNode::Dest) == use)) {
DEBUG_ONLY(disq_node = n;)
if (n->is_Load() || n->is_LoadStore()) {
*** 753,772 ****
NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";)
can_eliminate = false;
} else {
safepoints.append_if_missing(sfpt);
}
! } else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark
if (use->is_Phi()) {
! if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) {
NOT_PRODUCT(fail_eliminate = "Object is return value";)
} else {
NOT_PRODUCT(fail_eliminate = "Object is referenced by Phi";)
}
DEBUG_ONLY(disq_node = use;)
} else {
! if (use->Opcode() == Op_Return) {
NOT_PRODUCT(fail_eliminate = "Object is return value";)
}else {
NOT_PRODUCT(fail_eliminate = "Object is referenced by node";)
}
DEBUG_ONLY(disq_node = use;)
--- 753,772 ----
NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";)
can_eliminate = false;
} else {
safepoints.append_if_missing(sfpt);
}
! } else if (use->Opcode() != Opcodes::Op_CastP2X) { // CastP2X is used by card mark
if (use->is_Phi()) {
! if (use->outcnt() == 1 && use->unique_out()->Opcode() == Opcodes::Op_Return) {
NOT_PRODUCT(fail_eliminate = "Object is return value";)
} else {
NOT_PRODUCT(fail_eliminate = "Object is referenced by Phi";)
}
DEBUG_ONLY(disq_node = use;)
} else {
! if (use->Opcode() == Opcodes::Op_Return) {
NOT_PRODUCT(fail_eliminate = "Object is return value";)
}else {
NOT_PRODUCT(fail_eliminate = "Object is referenced by node";)
}
DEBUG_ONLY(disq_node = use;)
*** 1555,1565 ****
(init == NULL || !init->is_complete_with_arraycopy())) {
if (init == NULL || init->req() < InitializeNode::RawStores) {
// No InitializeNode or no stores captured by zeroing
// elimination. Simply add the MemBarStoreStore after object
// initialization.
! MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
transform_later(mb);
mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
mb->init_req(TypeFunc::Control, fast_oop_ctrl);
fast_oop_ctrl = new ProjNode(mb,TypeFunc::Control);
--- 1555,1565 ----
(init == NULL || !init->is_complete_with_arraycopy())) {
if (init == NULL || init->req() < InitializeNode::RawStores) {
// No InitializeNode or no stores captured by zeroing
// elimination. Simply add the MemBarStoreStore after object
// initialization.
! MemBarNode* mb = MemBarNode::make(C, Opcodes::Op_MemBarStoreStore, Compile::AliasIdxBot);
transform_later(mb);
mb->init_req(TypeFunc::Memory, fast_oop_rawmem);
mb->init_req(TypeFunc::Control, fast_oop_ctrl);
fast_oop_ctrl = new ProjNode(mb,TypeFunc::Control);
*** 1573,1583 ****
// barrier.
Node* init_ctrl = init->proj_out(TypeFunc::Control);
Node* init_mem = init->proj_out(TypeFunc::Memory);
! MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot);
transform_later(mb);
Node* ctrl = new ProjNode(init,TypeFunc::Control);
transform_later(ctrl);
Node* mem = new ProjNode(init,TypeFunc::Memory);
--- 1573,1583 ----
// barrier.
Node* init_ctrl = init->proj_out(TypeFunc::Control);
Node* init_mem = init->proj_out(TypeFunc::Memory);
! MemBarNode* mb = MemBarNode::make(C, Opcodes::Op_MemBarStoreStore, Compile::AliasIdxBot);
transform_later(mb);
Node* ctrl = new ProjNode(init,TypeFunc::Control);
transform_later(ctrl);
Node* mem = new ProjNode(init,TypeFunc::Memory);
*** 2254,2264 ****
// The input to a Lock is merged memory, so extract its RawMem input
// (unless the MergeMem has been optimized away.)
if (alock->is_Lock()) {
// Seach for MemBarAcquireLock node and delete it also.
MemBarNode* membar = fallthroughproj->unique_ctrl_out()->as_MemBar();
! assert(membar != NULL && membar->Opcode() == Op_MemBarAcquireLock, "");
Node* ctrlproj = membar->proj_out(TypeFunc::Control);
Node* memproj = membar->proj_out(TypeFunc::Memory);
_igvn.replace_node(ctrlproj, fallthroughproj);
_igvn.replace_node(memproj, memproj_fallthrough);
--- 2254,2264 ----
// The input to a Lock is merged memory, so extract its RawMem input
// (unless the MergeMem has been optimized away.)
if (alock->is_Lock()) {
// Seach for MemBarAcquireLock node and delete it also.
MemBarNode* membar = fallthroughproj->unique_ctrl_out()->as_MemBar();
! assert(membar != NULL && membar->Opcode() == Opcodes::Op_MemBarAcquireLock, "");
Node* ctrlproj = membar->proj_out(TypeFunc::Control);
Node* memproj = membar->proj_out(TypeFunc::Memory);
_igvn.replace_node(ctrlproj, fallthroughproj);
_igvn.replace_node(memproj, memproj_fallthrough);
*** 2273,2283 ****
// Seach for MemBarReleaseLock node and delete it also.
if (alock->is_Unlock() && ctrl != NULL && ctrl->is_Proj() &&
ctrl->in(0)->is_MemBar()) {
MemBarNode* membar = ctrl->in(0)->as_MemBar();
! assert(membar->Opcode() == Op_MemBarReleaseLock &&
mem->is_Proj() && membar == mem->in(0), "");
_igvn.replace_node(fallthroughproj, ctrl);
_igvn.replace_node(memproj_fallthrough, mem);
fallthroughproj = ctrl;
memproj_fallthrough = mem;
--- 2273,2283 ----
// Seach for MemBarReleaseLock node and delete it also.
if (alock->is_Unlock() && ctrl != NULL && ctrl->is_Proj() &&
ctrl->in(0)->is_MemBar()) {
MemBarNode* membar = ctrl->in(0)->as_MemBar();
! assert(membar->Opcode() == Opcodes::Op_MemBarReleaseLock &&
mem->is_Proj() && membar == mem->in(0), "");
_igvn.replace_node(fallthroughproj, ctrl);
_igvn.replace_node(memproj_fallthrough, mem);
fallthroughproj = ctrl;
memproj_fallthrough = mem;
*** 2378,2388 ****
if (klass_node == NULL) {
Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
klass_node = transform_later(LoadKlassNode::make(_igvn, NULL, mem, k_adr, _igvn.type(k_adr)->is_ptr()));
#ifdef _LP64
if (UseCompressedClassPointers && klass_node->is_DecodeNKlass()) {
! assert(klass_node->in(1)->Opcode() == Op_LoadNKlass, "sanity");
klass_node->in(1)->init_req(0, ctrl);
} else
#endif
klass_node->init_req(0, ctrl);
}
--- 2378,2388 ----
if (klass_node == NULL) {
Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
klass_node = transform_later(LoadKlassNode::make(_igvn, NULL, mem, k_adr, _igvn.type(k_adr)->is_ptr()));
#ifdef _LP64
if (UseCompressedClassPointers && klass_node->is_DecodeNKlass()) {
! assert(klass_node->in(1)->Opcode() == Opcodes::Op_LoadNKlass, "sanity");
klass_node->in(1)->init_req(0, ctrl);
} else
#endif
klass_node->init_req(0, ctrl);
}
*** 2651,2664 ****
_has_locks = true;
break;
case Node::Class_ArrayCopy:
break;
default:
! assert(n->Opcode() == Op_LoopLimit ||
! n->Opcode() == Op_Opaque1 ||
! n->Opcode() == Op_Opaque2 ||
! n->Opcode() == Op_Opaque3, "unknown node type in macro list");
}
assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
progress = progress || success;
}
}
--- 2651,2664 ----
_has_locks = true;
break;
case Node::Class_ArrayCopy:
break;
default:
! assert(n->Opcode() == Opcodes::Op_LoopLimit ||
! n->Opcode() == Opcodes::Op_Opaque1 ||
! n->Opcode() == Opcodes::Op_Opaque2 ||
! n->Opcode() == Opcodes::Op_Opaque3, "unknown node type in macro list");
}
assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
progress = progress || success;
}
}
*** 2682,2706 ****
progress = false;
for (int i = C->macro_count(); i > 0; i--) {
Node * n = C->macro_node(i-1);
bool success = false;
debug_only(int old_macro_count = C->macro_count(););
! if (n->Opcode() == Op_LoopLimit) {
// Remove it from macro list and put on IGVN worklist to optimize.
C->remove_macro_node(n);
_igvn._worklist.push(n);
success = true;
! } else if (n->Opcode() == Op_CallStaticJava) {
// Remove it from macro list and put on IGVN worklist to optimize.
C->remove_macro_node(n);
_igvn._worklist.push(n);
success = true;
! } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) {
_igvn.replace_node(n, n->in(1));
success = true;
#if INCLUDE_RTM_OPT
! } else if ((n->Opcode() == Op_Opaque3) && ((Opaque3Node*)n)->rtm_opt()) {
assert(C->profile_rtm(), "should be used only in rtm deoptimization code");
assert((n->outcnt() == 1) && n->unique_out()->is_Cmp(), "");
Node* cmp = n->unique_out();
#ifdef ASSERT
// Validate graph.
--- 2682,2706 ----
progress = false;
for (int i = C->macro_count(); i > 0; i--) {
Node * n = C->macro_node(i-1);
bool success = false;
debug_only(int old_macro_count = C->macro_count(););
! if (n->Opcode() == Opcodes::Op_LoopLimit) {
// Remove it from macro list and put on IGVN worklist to optimize.
C->remove_macro_node(n);
_igvn._worklist.push(n);
success = true;
! } else if (n->Opcode() == Opcodes::Op_CallStaticJava) {
// Remove it from macro list and put on IGVN worklist to optimize.
C->remove_macro_node(n);
_igvn._worklist.push(n);
success = true;
! } else if (n->Opcode() == Opcodes::Op_Opaque1 || n->Opcode() == Opcodes::Op_Opaque2) {
_igvn.replace_node(n, n->in(1));
success = true;
#if INCLUDE_RTM_OPT
! } else if ((n->Opcode() == Opcodes::Op_Opaque3) && ((Opaque3Node*)n)->rtm_opt()) {
assert(C->profile_rtm(), "should be used only in rtm deoptimization code");
assert((n->outcnt() == 1) && n->unique_out()->is_Cmp(), "");
Node* cmp = n->unique_out();
#ifdef ASSERT
// Validate graph.
< prev index next >