< prev index next >
src/hotspot/share/opto/macro.cpp
Print this page
@@ -44,10 +44,11 @@
#include "opto/phaseX.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
#include "opto/subnode.hpp"
#include "opto/type.hpp"
+#include "opto/valuetypenode.hpp"
#include "runtime/sharedRuntime.hpp"
#include "utilities/macros.hpp"
#if INCLUDE_G1GC
#include "gc/g1/g1ThreadLocalData.hpp"
#endif // INCLUDE_G1GC
@@ -78,12 +79,12 @@
return nreplacements;
}
void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcall) {
// Copy debug information and adjust JVMState information
- uint old_dbg_start = oldcall->tf()->domain()->cnt();
- uint new_dbg_start = newcall->tf()->domain()->cnt();
+ uint old_dbg_start = oldcall->tf()->domain_sig()->cnt();
+ uint new_dbg_start = newcall->tf()->domain_sig()->cnt();
int jvms_adj = new_dbg_start - old_dbg_start;
assert (new_dbg_start == newcall->req(), "argument count mismatch");
// SafePointScalarObject node could be referenced several times in debug info.
// Use Dict to record cloned nodes.
@@ -274,11 +275,11 @@
} else if (mem->is_Store()) {
const TypePtr* atype = mem->as_Store()->adr_type();
int adr_idx = phase->C->get_alias_index(atype);
if (adr_idx == alias_idx) {
assert(atype->isa_oopptr(), "address type must be oopptr");
- int adr_offset = atype->offset();
+ int adr_offset = atype->flattened_offset();
uint adr_iid = atype->is_oopptr()->instance_id();
// Array elements references have the same alias_idx
// but different offset and different instance_id.
if (adr_offset == offset && adr_iid == alloc->_idx)
return mem;
@@ -385,11 +386,11 @@
// Note: this function is recursive, its depth is limited by the "level" argument
// Returns the computed Phi, or NULL if it cannot compute it.
Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level) {
assert(mem->is_Phi(), "sanity");
int alias_idx = C->get_alias_index(adr_t);
- int offset = adr_t->offset();
+ int offset = adr_t->flattened_offset();
int instance_id = adr_t->instance_id();
// Check if an appropriate value phi already exists.
Node* region = mem->in(0);
for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) {
@@ -487,18 +488,17 @@
assert(adr_t->is_known_instance_field(), "instance required");
int instance_id = adr_t->instance_id();
assert((uint)instance_id == alloc->_idx, "wrong allocation");
int alias_idx = C->get_alias_index(adr_t);
- int offset = adr_t->offset();
+ int offset = adr_t->flattened_offset();
Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory);
Node *alloc_ctrl = alloc->in(TypeFunc::Control);
Node *alloc_mem = alloc->in(TypeFunc::Memory);
Arena *a = Thread::current()->resource_area();
VectorSet visited(a);
-
bool done = sfpt_mem == alloc_mem;
Node *mem = sfpt_mem;
while (!done) {
if (visited.test_set(mem->_idx)) {
return NULL; // found a loop, give up
@@ -507,21 +507,21 @@
if (mem == start_mem || mem == alloc_mem) {
done = true; // hit a sentinel, return appropriate 0 value
} else if (mem->is_Initialize()) {
mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn);
if (mem == NULL) {
- done = true; // Something go wrong.
+ done = true; // Something went wrong.
} else if (mem->is_Store()) {
const TypePtr* atype = mem->as_Store()->adr_type();
assert(C->get_alias_index(atype) == Compile::AliasIdxRaw, "store is correct memory slice");
done = true;
}
} else if (mem->is_Store()) {
const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr();
assert(atype != NULL, "address type must be oopptr");
assert(C->get_alias_index(atype) == alias_idx &&
- atype->is_known_instance_field() && atype->offset() == offset &&
+ atype->is_known_instance_field() && atype->flattened_offset() == offset &&
atype->instance_id() == instance_id, "store is correct memory slice");
done = true;
} else if (mem->is_Phi()) {
// try to find a phi's unique input
Node *unique_input = NULL;
@@ -549,10 +549,15 @@
}
}
if (mem != NULL) {
if (mem == start_mem || mem == alloc_mem) {
// hit a sentinel, return appropriate 0 value
+ Node* default_value = alloc->in(AllocateNode::DefaultValue);
+ if (default_value != NULL) {
+ return default_value;
+ }
+ assert(alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
return _igvn.zerocon(ft);
} else if (mem->is_Store()) {
Node* n = mem->in(MemNode::ValueIn);
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
n = bs->step_over_gc_barrier(n);
@@ -580,14 +585,51 @@
m = sfpt_mem;
}
return make_arraycopy_load(mem->as_ArrayCopy(), offset, ctl, m, ft, ftype, alloc);
}
}
- // Something go wrong.
+ // Something went wrong.
return NULL;
}
+// Search the last value stored into the value type's fields.
+Node* PhaseMacroExpand::value_type_from_mem(Node* mem, Node* ctl, ciValueKlass* vk, const TypeAryPtr* adr_type, int offset, AllocateNode* alloc) {
+ // Subtract the offset of the first field to account for the missing oop header
+ offset -= vk->first_field_offset();
+ // Create a new ValueTypeNode and retrieve the field values from memory
+ ValueTypeNode* vt = ValueTypeNode::make_uninitialized(_igvn, vk)->as_ValueType();
+ for (int i = 0; i < vk->nof_declared_nonstatic_fields(); ++i) {
+ ciType* field_type = vt->field_type(i);
+ int field_offset = offset + vt->field_offset(i);
+ // Each value type field has its own memory slice
+ adr_type = adr_type->with_field_offset(field_offset);
+ Node* value = NULL;
+ if (vt->field_is_flattened(i)) {
+ value = value_type_from_mem(mem, ctl, field_type->as_value_klass(), adr_type, field_offset, alloc);
+ } else {
+ const Type* ft = Type::get_const_type(field_type);
+ BasicType bt = field_type->basic_type();
+ if (UseCompressedOops && !is_java_primitive(bt)) {
+ ft = ft->make_narrowoop();
+ bt = T_NARROWOOP;
+ }
+ value = value_from_mem(mem, ctl, bt, ft, adr_type, alloc);
+ if (value != NULL && ft->isa_narrowoop()) {
+ assert(UseCompressedOops, "unexpected narrow oop");
+ value = transform_later(new DecodeNNode(value, value->get_ptr_type()));
+ }
+ }
+ if (value != NULL) {
+ vt->set_field_value(i, value);
+ } else {
+ // We might have reached the TrackedInitializationLimit
+ return NULL;
+ }
+ }
+ return vt;
+}
+
// Check the possibility of scalar replacement.
bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) {
// Scan the uses of the allocation to check for anything that would
// prevent us from eliminating it.
NOT_PRODUCT( const char* fail_eliminate = NULL; )
@@ -639,11 +681,11 @@
n->in(ArrayCopyNode::Dest) == use)) {
DEBUG_ONLY(disq_node = n;)
if (n->is_Load() || n->is_LoadStore()) {
NOT_PRODUCT(fail_eliminate = "Field load";)
} else {
- NOT_PRODUCT(fail_eliminate = "Not store field referrence";)
+ NOT_PRODUCT(fail_eliminate = "Not store field reference";)
}
can_eliminate = false;
}
}
} else if (use->is_ArrayCopy() &&
@@ -666,10 +708,14 @@
NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";)
can_eliminate = false;
} else {
safepoints.append_if_missing(sfpt);
}
+ } else if (use->is_ValueType() && use->isa_ValueType()->get_oop() == res) {
+ // ok to eliminate
+ } else if (use->is_Store()) {
+ // store to mark work
} else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark
if (use->is_Phi()) {
if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) {
NOT_PRODUCT(fail_eliminate = "Object is return value";)
} else {
@@ -677,16 +723,19 @@
}
DEBUG_ONLY(disq_node = use;)
} else {
if (use->Opcode() == Op_Return) {
NOT_PRODUCT(fail_eliminate = "Object is return value";)
- }else {
+ } else {
NOT_PRODUCT(fail_eliminate = "Object is referenced by node";)
}
DEBUG_ONLY(disq_node = use;)
}
can_eliminate = false;
+ } else {
+ assert(use->Opcode() == Op_CastP2X, "should be");
+ assert(!use->has_out_with(Op_OrL), "should have been removed because oop is never null");
}
}
}
#ifndef PRODUCT
@@ -745,17 +794,29 @@
// find the array's elements which will be needed for safepoint debug information
nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1);
assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.");
elem_type = klass->as_array_klass()->element_type();
basic_elem_type = elem_type->basic_type();
+ if (elem_type->is_valuetype()) {
+ ciValueKlass* vk = elem_type->as_value_klass();
+ if (!vk->flatten_array()) {
+ assert(basic_elem_type == T_VALUETYPE, "unexpected element basic type");
+ basic_elem_type = T_OBJECT;
+ }
+ }
array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
element_size = type2aelembytes(basic_elem_type);
+ if (klass->is_value_array_klass()) {
+ // Flattened value type array
+ element_size = klass->as_value_array_klass()->element_byte_size();
+ }
}
}
//
// Process the safepoint uses
//
+ Unique_Node_List value_worklist;
while (safepoints.length() > 0) {
SafePointNode* sfpt = safepoints.pop();
Node* mem = sfpt->memory();
Node* ctl = sfpt->control();
assert(sfpt->jvms() != NULL, "missed JVMS");
@@ -778,10 +839,11 @@
if (iklass != NULL) {
field = iklass->nonstatic_field_at(j);
offset = field->offset();
elem_type = field->type();
basic_elem_type = field->layout_type();
+ assert(!field->is_flattened(), "flattened value type fields should not have safepoint uses");
} else {
offset = array_base + j * (intptr_t)element_size;
}
const Type *field_type;
@@ -805,13 +867,19 @@
}
} else {
field_type = Type::get_const_basic_type(basic_elem_type);
}
- const TypeOopPtr *field_addr_type = res_type->add_offset(offset)->isa_oopptr();
-
- Node *field_val = value_from_mem(mem, ctl, basic_elem_type, field_type, field_addr_type, alloc);
+ Node* field_val = NULL;
+ const TypeOopPtr* field_addr_type = res_type->add_offset(offset)->isa_oopptr();
+ if (klass->is_value_array_klass()) {
+ ciValueKlass* vk = elem_type->as_value_klass();
+ assert(vk->flatten_array(), "must be flattened");
+ field_val = value_type_from_mem(mem, ctl, vk, field_addr_type->isa_aryptr(), 0, alloc);
+ } else {
+ field_val = value_from_mem(mem, ctl, basic_elem_type, field_type, field_addr_type, alloc);
+ }
if (field_val == NULL) {
// We weren't able to find a value for this field,
// give up on eliminating this allocation.
// Remove any extra entries we added to the safepoint.
@@ -873,10 +941,13 @@
if (field_val->is_EncodeP()) {
field_val = field_val->in(1);
} else {
field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type()));
}
+ } else if (field_val->is_ValueType()) {
+ // Keep track of value types to scalarize them later
+ value_worklist.push(field_val);
}
sfpt->add_req(field_val);
}
JVMState *jvms = sfpt->jvms();
jvms->set_endoff(sfpt->req());
@@ -886,10 +957,15 @@
int end = jvms->debug_end();
sfpt->replace_edges_in_range(res, sobj, start, end);
_igvn._worklist.push(sfpt);
safepoints_done.append_if_missing(sfpt); // keep it for rollback
}
+ // Scalarize value types that were added to the safepoint
+ for (uint i = 0; i < value_worklist.size(); ++i) {
+ Node* vt = value_worklist.at(i);
+ vt->as_ValueType()->make_scalar_in_safepoints(&_igvn);
+ }
return true;
}
static void disconnect_projections(MultiNode* n, PhaseIterGVN& igvn) {
Node* ctl_proj = n->proj_out_or_null(TypeFunc::Control);
@@ -950,16 +1026,15 @@
// Disconnect ArrayCopy node
ArrayCopyNode* ac = use->as_ArrayCopy();
assert(ac->is_arraycopy_validated() ||
ac->is_copyof_validated() ||
ac->is_copyofrange_validated(), "unsupported");
- CallProjections callprojs;
- ac->extract_projections(&callprojs, true);
+ CallProjections* callprojs = ac->extract_projections(true);
- _igvn.replace_node(callprojs.fallthrough_ioproj, ac->in(TypeFunc::I_O));
- _igvn.replace_node(callprojs.fallthrough_memproj, ac->in(TypeFunc::Memory));
- _igvn.replace_node(callprojs.fallthrough_catchproj, ac->in(TypeFunc::Control));
+ _igvn.replace_node(callprojs->fallthrough_ioproj, ac->in(TypeFunc::I_O));
+ _igvn.replace_node(callprojs->fallthrough_memproj, ac->in(TypeFunc::Memory));
+ _igvn.replace_node(callprojs->fallthrough_catchproj, ac->in(TypeFunc::Control));
// Set control to top. IGVN will remove the remaining projections
ac->set_req(0, top());
ac->replace_edge(res, top());
@@ -972,10 +1047,16 @@
if (src->outcnt() == 0 && !src->is_top()) {
_igvn.remove_dead_node(src);
}
_igvn._worklist.push(ac);
+ } else if (use->is_ValueType()) {
+ assert(use->isa_ValueType()->get_oop() == res, "unexpected value type use");
+ _igvn.rehash_node_delayed(use);
+ use->isa_ValueType()->set_oop(_igvn.zerocon(T_VALUETYPE));
+ } else if (use->is_Store()) {
+ _igvn.replace_node(use, use->in(MemNode::Memory));
} else {
eliminate_gc_barrier(use);
}
j -= (oc1 - res->outcnt());
}
@@ -1123,11 +1204,11 @@
assert(boxing->result_cast() == NULL, "unexpected boxing node result");
extract_call_projections(boxing);
- const TypeTuple* r = boxing->tf()->range();
+ const TypeTuple* r = boxing->tf()->range_sig();
assert(r->cnt() > TypeFunc::Parms, "sanity");
const TypeInstPtr* t = r->field_at(TypeFunc::Parms)->isa_instptr();
assert(t != NULL, "sanity");
CompileLog* log = C->log();
@@ -1283,34 +1364,32 @@
// Force slow-path allocation
always_slow = true;
initial_slow_test = NULL;
}
-
- enum { too_big_or_final_path = 1, need_gc_path = 2 };
Node *slow_region = NULL;
Node *toobig_false = ctrl;
assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent");
// generate the initial test if necessary
if (initial_slow_test != NULL ) {
- slow_region = new RegionNode(3);
-
+ if (slow_region == NULL) {
+ slow_region = new RegionNode(1);
+ }
// Now make the initial failure test. Usually a too-big test but
// might be a TRUE for finalizers or a fancy class check for
// newInstance0.
- IfNode *toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
+ IfNode* toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN);
transform_later(toobig_iff);
// Plug the failing-too-big test into the slow-path region
- Node *toobig_true = new IfTrueNode( toobig_iff );
+ Node* toobig_true = new IfTrueNode(toobig_iff);
transform_later(toobig_true);
- slow_region ->init_req( too_big_or_final_path, toobig_true );
- toobig_false = new IfFalseNode( toobig_iff );
+ slow_region ->add_req(toobig_true);
+ toobig_false = new IfFalseNode(toobig_iff);
transform_later(toobig_false);
} else { // No initial test, just fall into next case
toobig_false = ctrl;
- debug_only(slow_region = NodeSentinel);
}
Node *slow_mem = mem; // save the current memory state for slow path
// generate the fast allocation code unless we know that the initial test will always go slow
if (!always_slow) {
@@ -1339,15 +1418,15 @@
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
Node* fast_oop = bs->obj_allocate(this, ctrl, mem, toobig_false, size_in_bytes, i_o, needgc_ctrl,
fast_oop_ctrl, fast_oop_rawmem,
prefetch_lines);
- if (initial_slow_test) {
- slow_region->init_req(need_gc_path, needgc_ctrl);
+ if (slow_region != NULL) {
+ slow_region->add_req(needgc_ctrl);
// This completes all paths into the slow merge point
transform_later(slow_region);
- } else { // No initial slow path needed!
+ } else {
// Just fall from the need-GC path straight into the VM call.
slow_region = needgc_ctrl;
}
InitializeNode* init = alloc->initialization();
@@ -1610,25 +1689,21 @@
}
// Helper for PhaseMacroExpand::expand_allocate_common.
// Initializes the newly-allocated storage.
-Node*
-PhaseMacroExpand::initialize_object(AllocateNode* alloc,
+Node* PhaseMacroExpand::initialize_object(AllocateNode* alloc,
Node* control, Node* rawmem, Node* object,
Node* klass_node, Node* length,
Node* size_in_bytes) {
InitializeNode* init = alloc->initialization();
// Store the klass & mark bits
- Node* mark_node = NULL;
- // For now only enable fast locking for non-array types
- if (UseBiasedLocking && (length == NULL)) {
- mark_node = make_load(control, rawmem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
- } else {
- mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype()));
+ Node* mark_node = alloc->make_ideal_mark(&_igvn, object, control, rawmem, klass_node);
+ if (!mark_node->is_Con()) {
+ transform_later(mark_node);
}
- rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
+ rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, TypeX_X->basic_type());
rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
int header_size = alloc->minimum_header_size(); // conservatively small
// Array length
@@ -1652,10 +1727,12 @@
// there can be two Allocates to one Initialize. The answer in all these
// edge cases is safety first. It is always safe to clear immediately
// within an Allocate, and then (maybe or maybe not) clear some more later.
if (!(UseTLAB && ZeroTLAB)) {
rawmem = ClearArrayNode::clear_memory(control, rawmem, object,
+ alloc->in(AllocateNode::DefaultValue),
+ alloc->in(AllocateNode::RawDefaultValue),
header_size, size_in_bytes,
&_igvn);
}
} else {
if (!init->is_complete()) {
@@ -2413,10 +2490,215 @@
mem_phi->init_req(2, mem);
transform_later(mem_phi);
_igvn.replace_node(_memproj_fallthrough, mem_phi);
}
+// A value type might be returned from the call but we don't know its
+// type. Either we get a buffered value (and nothing needs to be done)
+// or one of the values being returned is the klass of the value type
+// and we need to allocate a value type instance of that type and
+// initialize it with other values being returned. In that case, we
+// first try a fast path allocation and initialize the value with the
+// value klass's pack handler or we fall back to a runtime call.
+void PhaseMacroExpand::expand_mh_intrinsic_return(CallStaticJavaNode* call) {
+ assert(call->method()->is_method_handle_intrinsic(), "must be a method handle intrinsic call");
+ Node* ret = call->proj_out_or_null(TypeFunc::Parms);
+ if (ret == NULL) {
+ return;
+ }
+ const TypeFunc* tf = call->_tf;
+ const TypeTuple* domain = OptoRuntime::store_value_type_fields_Type()->domain_cc();
+ const TypeFunc* new_tf = TypeFunc::make(tf->domain_sig(), tf->domain_cc(), tf->range_sig(), domain);
+ call->_tf = new_tf;
+ // Make sure the change of type is applied before projections are processed by igvn
+ _igvn.set_type(call, call->Value(&_igvn));
+ _igvn.set_type(ret, ret->Value(&_igvn));
+
+ // Before any new projection is added:
+ CallProjections* projs = call->extract_projections(true, true);
+
+ Node* ctl = new Node(1);
+ Node* mem = new Node(1);
+ Node* io = new Node(1);
+ Node* ex_ctl = new Node(1);
+ Node* ex_mem = new Node(1);
+ Node* ex_io = new Node(1);
+ Node* res = new Node(1);
+
+ Node* cast = transform_later(new CastP2XNode(ctl, res));
+ Node* mask = MakeConX(0x1);
+ Node* masked = transform_later(new AndXNode(cast, mask));
+ Node* cmp = transform_later(new CmpXNode(masked, mask));
+ Node* bol = transform_later(new BoolNode(cmp, BoolTest::eq));
+ IfNode* allocation_iff = new IfNode(ctl, bol, PROB_MAX, COUNT_UNKNOWN);
+ transform_later(allocation_iff);
+ Node* allocation_ctl = transform_later(new IfTrueNode(allocation_iff));
+ Node* no_allocation_ctl = transform_later(new IfFalseNode(allocation_iff));
+
+ Node* no_allocation_res = transform_later(new CheckCastPPNode(no_allocation_ctl, res, TypeInstPtr::BOTTOM));
+
+ Node* mask2 = MakeConX(-2);
+ Node* masked2 = transform_later(new AndXNode(cast, mask2));
+ Node* rawklassptr = transform_later(new CastX2PNode(masked2));
+ Node* klass_node = transform_later(new CheckCastPPNode(allocation_ctl, rawklassptr, TypeKlassPtr::OBJECT_OR_NULL));
+
+ Node* slowpath_bol = NULL;
+ Node* top_adr = NULL;
+ Node* old_top = NULL;
+ Node* new_top = NULL;
+ if (UseTLAB) {
+ Node* end_adr = NULL;
+ set_eden_pointers(top_adr, end_adr);
+ Node* end = make_load(ctl, mem, end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS);
+ old_top = new LoadPNode(ctl, mem, top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered);
+ transform_later(old_top);
+ Node* layout_val = make_load(NULL, mem, klass_node, in_bytes(Klass::layout_helper_offset()), TypeInt::INT, T_INT);
+ Node* size_in_bytes = ConvI2X(layout_val);
+ new_top = new AddPNode(top(), old_top, size_in_bytes);
+ transform_later(new_top);
+ Node* slowpath_cmp = new CmpPNode(new_top, end);
+ transform_later(slowpath_cmp);
+ slowpath_bol = new BoolNode(slowpath_cmp, BoolTest::ge);
+ transform_later(slowpath_bol);
+ } else {
+ slowpath_bol = intcon(1);
+ top_adr = top();
+ old_top = top();
+ new_top = top();
+ }
+ IfNode* slowpath_iff = new IfNode(allocation_ctl, slowpath_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
+ transform_later(slowpath_iff);
+
+ Node* slowpath_true = new IfTrueNode(slowpath_iff);
+ transform_later(slowpath_true);
+
+ CallStaticJavaNode* slow_call = new CallStaticJavaNode(OptoRuntime::store_value_type_fields_Type(),
+ StubRoutines::store_value_type_fields_to_buf(),
+ "store_value_type_fields",
+ call->jvms()->bci(),
+ TypePtr::BOTTOM);
+ slow_call->init_req(TypeFunc::Control, slowpath_true);
+ slow_call->init_req(TypeFunc::Memory, mem);
+ slow_call->init_req(TypeFunc::I_O, io);
+ slow_call->init_req(TypeFunc::FramePtr, call->in(TypeFunc::FramePtr));
+ slow_call->init_req(TypeFunc::ReturnAdr, call->in(TypeFunc::ReturnAdr));
+ slow_call->init_req(TypeFunc::Parms, res);
+
+ Node* slow_ctl = transform_later(new ProjNode(slow_call, TypeFunc::Control));
+ Node* slow_mem = transform_later(new ProjNode(slow_call, TypeFunc::Memory));
+ Node* slow_io = transform_later(new ProjNode(slow_call, TypeFunc::I_O));
+ Node* slow_res = transform_later(new ProjNode(slow_call, TypeFunc::Parms));
+ Node* slow_catc = transform_later(new CatchNode(slow_ctl, slow_io, 2));
+ Node* slow_norm = transform_later(new CatchProjNode(slow_catc, CatchProjNode::fall_through_index, CatchProjNode::no_handler_bci));
+ Node* slow_excp = transform_later(new CatchProjNode(slow_catc, CatchProjNode::catch_all_index, CatchProjNode::no_handler_bci));
+
+ Node* ex_r = new RegionNode(3);
+ Node* ex_mem_phi = new PhiNode(ex_r, Type::MEMORY, TypePtr::BOTTOM);
+ Node* ex_io_phi = new PhiNode(ex_r, Type::ABIO);
+ ex_r->init_req(1, slow_excp);
+ ex_mem_phi->init_req(1, slow_mem);
+ ex_io_phi->init_req(1, slow_io);
+ ex_r->init_req(2, ex_ctl);
+ ex_mem_phi->init_req(2, ex_mem);
+ ex_io_phi->init_req(2, ex_io);
+
+ transform_later(ex_r);
+ transform_later(ex_mem_phi);
+ transform_later(ex_io_phi);
+
+ Node* slowpath_false = new IfFalseNode(slowpath_iff);
+ transform_later(slowpath_false);
+ Node* rawmem = new StorePNode(slowpath_false, mem, top_adr, TypeRawPtr::BOTTOM, new_top, MemNode::unordered);
+ transform_later(rawmem);
+ Node* mark_node = makecon(TypeRawPtr::make((address)markOopDesc::always_locked_prototype()));
+ rawmem = make_store(slowpath_false, rawmem, old_top, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS);
+ rawmem = make_store(slowpath_false, rawmem, old_top, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
+ if (UseCompressedClassPointers) {
+ rawmem = make_store(slowpath_false, rawmem, old_top, oopDesc::klass_gap_offset_in_bytes(), intcon(0), T_INT);
+ }
+ Node* fixed_block = make_load(slowpath_false, rawmem, klass_node, in_bytes(InstanceKlass::adr_valueklass_fixed_block_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
+ Node* pack_handler = make_load(slowpath_false, rawmem, fixed_block, in_bytes(ValueKlass::pack_handler_offset()), TypeRawPtr::BOTTOM, T_ADDRESS);
+
+ CallLeafNoFPNode* handler_call = new CallLeafNoFPNode(OptoRuntime::pack_value_type_Type(),
+ NULL,
+ "pack handler",
+ TypeRawPtr::BOTTOM);
+ handler_call->init_req(TypeFunc::Control, slowpath_false);
+ handler_call->init_req(TypeFunc::Memory, rawmem);
+ handler_call->init_req(TypeFunc::I_O, top());
+ handler_call->init_req(TypeFunc::FramePtr, call->in(TypeFunc::FramePtr));
+ handler_call->init_req(TypeFunc::ReturnAdr, top());
+ handler_call->init_req(TypeFunc::Parms, pack_handler);
+ handler_call->init_req(TypeFunc::Parms+1, old_top);
+
+ // We don't know how many values are returned. This assumes the
+ // worst case, that all available registers are used.
+ for (uint i = TypeFunc::Parms+1; i < domain->cnt(); i++) {
+ if (domain->field_at(i) == Type::HALF) {
+ slow_call->init_req(i, top());
+ handler_call->init_req(i+1, top());
+ continue;
+ }
+ Node* proj = transform_later(new ProjNode(call, i));
+ slow_call->init_req(i, proj);
+ handler_call->init_req(i+1, proj);
+ }
+
+ // We can safepoint at that new call
+ copy_call_debug_info(call, slow_call);
+ transform_later(slow_call);
+ transform_later(handler_call);
+
+ Node* handler_ctl = transform_later(new ProjNode(handler_call, TypeFunc::Control));
+ rawmem = transform_later(new ProjNode(handler_call, TypeFunc::Memory));
+ Node* slowpath_false_res = transform_later(new ProjNode(handler_call, TypeFunc::Parms));
+
+ MergeMemNode* slowpath_false_mem = MergeMemNode::make(mem);
+ slowpath_false_mem->set_memory_at(Compile::AliasIdxRaw, rawmem);
+ transform_later(slowpath_false_mem);
+
+ Node* r = new RegionNode(4);
+ Node* mem_phi = new PhiNode(r, Type::MEMORY, TypePtr::BOTTOM);
+ Node* io_phi = new PhiNode(r, Type::ABIO);
+ Node* res_phi = new PhiNode(r, TypeInstPtr::BOTTOM);
+
+ r->init_req(1, no_allocation_ctl);
+ mem_phi->init_req(1, mem);
+ io_phi->init_req(1, io);
+ res_phi->init_req(1, no_allocation_res);
+ r->init_req(2, slow_norm);
+ mem_phi->init_req(2, slow_mem);
+ io_phi->init_req(2, slow_io);
+ res_phi->init_req(2, slow_res);
+ r->init_req(3, handler_ctl);
+ mem_phi->init_req(3, slowpath_false_mem);
+ io_phi->init_req(3, io);
+ res_phi->init_req(3, slowpath_false_res);
+
+ transform_later(r);
+ transform_later(mem_phi);
+ transform_later(io_phi);
+ transform_later(res_phi);
+
+ assert(projs->nb_resproj == 1, "unexpected number of results");
+ _igvn.replace_in_uses(projs->fallthrough_catchproj, r);
+ _igvn.replace_in_uses(projs->fallthrough_memproj, mem_phi);
+ _igvn.replace_in_uses(projs->fallthrough_ioproj, io_phi);
+ _igvn.replace_in_uses(projs->resproj[0], res_phi);
+ _igvn.replace_in_uses(projs->catchall_catchproj, ex_r);
+ _igvn.replace_in_uses(projs->catchall_memproj, ex_mem_phi);
+ _igvn.replace_in_uses(projs->catchall_ioproj, ex_io_phi);
+
+ _igvn.replace_node(ctl, projs->fallthrough_catchproj);
+ _igvn.replace_node(mem, projs->fallthrough_memproj);
+ _igvn.replace_node(io, projs->fallthrough_ioproj);
+ _igvn.replace_node(res, projs->resproj[0]);
+ _igvn.replace_node(ex_ctl, projs->catchall_catchproj);
+ _igvn.replace_node(ex_mem, projs->catchall_memproj);
+ _igvn.replace_node(ex_io, projs->catchall_ioproj);
+ }
+
//---------------------------eliminate_macro_nodes----------------------
// Eliminate scalar replaced allocations and associated locks.
void PhaseMacroExpand::eliminate_macro_nodes() {
if (C->macro_count() == 0)
return;
@@ -2457,13 +2739,17 @@
switch (n->class_id()) {
case Node::Class_Allocate:
case Node::Class_AllocateArray:
success = eliminate_allocate_node(n->as_Allocate());
break;
- case Node::Class_CallStaticJava:
+ case Node::Class_CallStaticJava: {
+ CallStaticJavaNode* call = n->as_CallStaticJava();
+ if (!call->method()->is_method_handle_intrinsic()) {
success = eliminate_boxing_node(n->as_CallStaticJava());
+ }
break;
+ }
case Node::Class_Lock:
case Node::Class_Unlock:
assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
_has_locks = true;
break;
@@ -2509,14 +2795,17 @@
// Remove it from macro list and put on IGVN worklist to optimize.
C->remove_macro_node(n);
_igvn._worklist.push(n);
success = true;
} else if (n->Opcode() == Op_CallStaticJava) {
+ CallStaticJavaNode* call = n->as_CallStaticJava();
+ if (!call->method()->is_method_handle_intrinsic()) {
// Remove it from macro list and put on IGVN worklist to optimize.
C->remove_macro_node(n);
_igvn._worklist.push(n);
success = true;
+ }
} else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) {
_igvn.replace_node(n, n->in(1));
success = true;
#if INCLUDE_RTM_OPT
} else if ((n->Opcode() == Op_Opaque3) && ((Opaque3Node*)n)->rtm_opt()) {
@@ -2593,10 +2882,14 @@
expand_lock_node(n->as_Lock());
break;
case Node::Class_Unlock:
expand_unlock_node(n->as_Unlock());
break;
+ case Node::Class_CallStaticJava:
+ expand_mh_intrinsic_return(n->as_CallStaticJava());
+ C->remove_macro_node(n);
+ break;
default:
assert(false, "unknown node type in macro list");
}
assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
if (C->failing()) return true;
< prev index next >