--- old/src/hotspot/share/opto/macro.cpp 2019-03-11 14:26:47.630354582 +0100 +++ new/src/hotspot/share/opto/macro.cpp 2019-03-11 14:26:47.414354585 +0100 @@ -46,6 +46,7 @@ #include "opto/runtime.hpp" #include "opto/subnode.hpp" #include "opto/type.hpp" +#include "opto/valuetypenode.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/macros.hpp" #if INCLUDE_G1GC @@ -80,8 +81,8 @@ void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcall) { // Copy debug information and adjust JVMState information - uint old_dbg_start = oldcall->tf()->domain()->cnt(); - uint new_dbg_start = newcall->tf()->domain()->cnt(); + uint old_dbg_start = oldcall->tf()->domain_sig()->cnt(); + uint new_dbg_start = newcall->tf()->domain_sig()->cnt(); int jvms_adj = new_dbg_start - old_dbg_start; assert (new_dbg_start == newcall->req(), "argument count mismatch"); @@ -276,7 +277,7 @@ int adr_idx = phase->C->get_alias_index(atype); if (adr_idx == alias_idx) { assert(atype->isa_oopptr(), "address type must be oopptr"); - int adr_offset = atype->offset(); + int adr_offset = atype->flattened_offset(); uint adr_iid = atype->is_oopptr()->instance_id(); // Array elements references have the same alias_idx // but different offset and different instance_id. @@ -319,7 +320,7 @@ return NULL; } mem = mem->in(MemNode::Memory); - } else if (mem->Opcode() == Op_StrInflatedCopy) { + } else if (mem->Opcode() == Op_StrInflatedCopy) { Node* adr = mem->in(3); // Destination array const TypePtr* atype = adr->bottom_type()->is_ptr(); int adr_idx = phase->C->get_alias_index(atype); @@ -387,7 +388,7 @@ Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level) { assert(mem->is_Phi(), "sanity"); int alias_idx = C->get_alias_index(adr_t); - int offset = adr_t->offset(); + int offset = adr_t->flattened_offset(); int instance_id = adr_t->instance_id(); // Check if an appropriate value phi already exists. @@ -489,14 +490,13 @@ assert((uint)instance_id == alloc->_idx, "wrong allocation"); int alias_idx = C->get_alias_index(adr_t); - int offset = adr_t->offset(); + int offset = adr_t->flattened_offset(); Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory); Node *alloc_ctrl = alloc->in(TypeFunc::Control); Node *alloc_mem = alloc->in(TypeFunc::Memory); Arena *a = Thread::current()->resource_area(); VectorSet visited(a); - bool done = sfpt_mem == alloc_mem; Node *mem = sfpt_mem; while (!done) { @@ -509,7 +509,7 @@ } else if (mem->is_Initialize()) { mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn); if (mem == NULL) { - done = true; // Something go wrong. + done = true; // Something went wrong. } else if (mem->is_Store()) { const TypePtr* atype = mem->as_Store()->adr_type(); assert(C->get_alias_index(atype) == Compile::AliasIdxRaw, "store is correct memory slice"); @@ -519,7 +519,7 @@ const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr(); assert(atype != NULL, "address type must be oopptr"); assert(C->get_alias_index(atype) == alias_idx && - atype->is_known_instance_field() && atype->offset() == offset && + atype->is_known_instance_field() && atype->flattened_offset() == offset && atype->instance_id() == instance_id, "store is correct memory slice"); done = true; } else if (mem->is_Phi()) { @@ -551,6 +551,11 @@ if (mem != NULL) { if (mem == start_mem || mem == alloc_mem) { // hit a sentinel, return appropriate 0 value + Node* default_value = alloc->in(AllocateNode::DefaultValue); + if (default_value != NULL) { + return default_value; + } + assert(alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null"); return _igvn.zerocon(ft); } else if (mem->is_Store()) { Node* n = mem->in(MemNode::ValueIn); @@ -582,10 +587,47 @@ return make_arraycopy_load(mem->as_ArrayCopy(), offset, ctl, m, ft, ftype, alloc); } } - // Something go wrong. + // Something went wrong. return NULL; } +// Search the last value stored into the value type's fields. +Node* PhaseMacroExpand::value_type_from_mem(Node* mem, Node* ctl, ciValueKlass* vk, const TypeAryPtr* adr_type, int offset, AllocateNode* alloc) { + // Subtract the offset of the first field to account for the missing oop header + offset -= vk->first_field_offset(); + // Create a new ValueTypeNode and retrieve the field values from memory + ValueTypeNode* vt = ValueTypeNode::make_uninitialized(_igvn, vk)->as_ValueType(); + for (int i = 0; i < vk->nof_declared_nonstatic_fields(); ++i) { + ciType* field_type = vt->field_type(i); + int field_offset = offset + vt->field_offset(i); + // Each value type field has its own memory slice + adr_type = adr_type->with_field_offset(field_offset); + Node* value = NULL; + if (vt->field_is_flattened(i)) { + value = value_type_from_mem(mem, ctl, field_type->as_value_klass(), adr_type, field_offset, alloc); + } else { + const Type* ft = Type::get_const_type(field_type); + BasicType bt = field_type->basic_type(); + if (UseCompressedOops && !is_java_primitive(bt)) { + ft = ft->make_narrowoop(); + bt = T_NARROWOOP; + } + value = value_from_mem(mem, ctl, bt, ft, adr_type, alloc); + if (value != NULL && ft->isa_narrowoop()) { + assert(UseCompressedOops, "unexpected narrow oop"); + value = transform_later(new DecodeNNode(value, value->get_ptr_type())); + } + } + if (value != NULL) { + vt->set_field_value(i, value); + } else { + // We might have reached the TrackedInitializationLimit + return NULL; + } + } + return vt; +} + // Check the possibility of scalar replacement. bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray & safepoints) { // Scan the uses of the allocation to check for anything that would @@ -641,7 +683,7 @@ if (n->is_Load() || n->is_LoadStore()) { NOT_PRODUCT(fail_eliminate = "Field load";) } else { - NOT_PRODUCT(fail_eliminate = "Not store field referrence";) + NOT_PRODUCT(fail_eliminate = "Not store field reference";) } can_eliminate = false; } @@ -668,6 +710,10 @@ } else { safepoints.append_if_missing(sfpt); } + } else if (use->is_ValueType() && use->isa_ValueType()->get_oop() == res) { + // ok to eliminate + } else if (use->is_Store()) { + // store to mark work } else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark if (use->is_Phi()) { if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) { @@ -679,12 +725,15 @@ } else { if (use->Opcode() == Op_Return) { NOT_PRODUCT(fail_eliminate = "Object is return value";) - }else { + } else { NOT_PRODUCT(fail_eliminate = "Object is referenced by node";) } DEBUG_ONLY(disq_node = use;) } can_eliminate = false; + } else { + assert(use->Opcode() == Op_CastP2X, "should be"); + assert(!use->has_out_with(Op_OrL), "should have been removed because oop is never null"); } } } @@ -747,13 +796,25 @@ assert(klass->is_array_klass() && nfields >= 0, "must be an array klass."); elem_type = klass->as_array_klass()->element_type(); basic_elem_type = elem_type->basic_type(); + if (elem_type->is_valuetype()) { + ciValueKlass* vk = elem_type->as_value_klass(); + if (!vk->flatten_array()) { + assert(basic_elem_type == T_VALUETYPE, "unexpected element basic type"); + basic_elem_type = T_OBJECT; + } + } array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type); element_size = type2aelembytes(basic_elem_type); + if (klass->is_value_array_klass()) { + // Flattened value type array + element_size = klass->as_value_array_klass()->element_byte_size(); + } } } // // Process the safepoint uses // + Unique_Node_List value_worklist; while (safepoints.length() > 0) { SafePointNode* sfpt = safepoints.pop(); Node* mem = sfpt->memory(); @@ -780,6 +841,7 @@ offset = field->offset(); elem_type = field->type(); basic_elem_type = field->layout_type(); + assert(!field->is_flattened(), "flattened value type fields should not have safepoint uses"); } else { offset = array_base + j * (intptr_t)element_size; } @@ -807,9 +869,15 @@ field_type = Type::get_const_basic_type(basic_elem_type); } - const TypeOopPtr *field_addr_type = res_type->add_offset(offset)->isa_oopptr(); - - Node *field_val = value_from_mem(mem, ctl, basic_elem_type, field_type, field_addr_type, alloc); + Node* field_val = NULL; + const TypeOopPtr* field_addr_type = res_type->add_offset(offset)->isa_oopptr(); + if (klass->is_value_array_klass()) { + ciValueKlass* vk = elem_type->as_value_klass(); + assert(vk->flatten_array(), "must be flattened"); + field_val = value_type_from_mem(mem, ctl, vk, field_addr_type->isa_aryptr(), 0, alloc); + } else { + field_val = value_from_mem(mem, ctl, basic_elem_type, field_type, field_addr_type, alloc); + } if (field_val == NULL) { // We weren't able to find a value for this field, // give up on eliminating this allocation. @@ -875,6 +943,9 @@ } else { field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type())); } + } else if (field_val->is_ValueType()) { + // Keep track of value types to scalarize them later + value_worklist.push(field_val); } sfpt->add_req(field_val); } @@ -888,6 +959,11 @@ _igvn._worklist.push(sfpt); safepoints_done.append_if_missing(sfpt); // keep it for rollback } + // Scalarize value types that were added to the safepoint + for (uint i = 0; i < value_worklist.size(); ++i) { + Node* vt = value_worklist.at(i); + vt->as_ValueType()->make_scalar_in_safepoints(&_igvn); + } return true; } @@ -952,12 +1028,11 @@ assert(ac->is_arraycopy_validated() || ac->is_copyof_validated() || ac->is_copyofrange_validated(), "unsupported"); - CallProjections callprojs; - ac->extract_projections(&callprojs, true); + CallProjections* callprojs = ac->extract_projections(true); - _igvn.replace_node(callprojs.fallthrough_ioproj, ac->in(TypeFunc::I_O)); - _igvn.replace_node(callprojs.fallthrough_memproj, ac->in(TypeFunc::Memory)); - _igvn.replace_node(callprojs.fallthrough_catchproj, ac->in(TypeFunc::Control)); + _igvn.replace_node(callprojs->fallthrough_ioproj, ac->in(TypeFunc::I_O)); + _igvn.replace_node(callprojs->fallthrough_memproj, ac->in(TypeFunc::Memory)); + _igvn.replace_node(callprojs->fallthrough_catchproj, ac->in(TypeFunc::Control)); // Set control to top. IGVN will remove the remaining projections ac->set_req(0, top()); @@ -974,6 +1049,12 @@ } _igvn._worklist.push(ac); + } else if (use->is_ValueType()) { + assert(use->isa_ValueType()->get_oop() == res, "unexpected value type use"); + _igvn.rehash_node_delayed(use); + use->isa_ValueType()->set_oop(_igvn.zerocon(T_VALUETYPE)); + } else if (use->is_Store()) { + _igvn.replace_node(use, use->in(MemNode::Memory)); } else { eliminate_gc_barrier(use); } @@ -1125,7 +1206,7 @@ extract_call_projections(boxing); - const TypeTuple* r = boxing->tf()->range(); + const TypeTuple* r = boxing->tf()->range_sig(); assert(r->cnt() > TypeFunc::Parms, "sanity"); const TypeInstPtr* t = r->field_at(TypeFunc::Parms)->isa_instptr(); assert(t != NULL, "sanity"); @@ -1285,30 +1366,28 @@ initial_slow_test = NULL; } - - enum { too_big_or_final_path = 1, need_gc_path = 2 }; Node *slow_region = NULL; Node *toobig_false = ctrl; assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent"); // generate the initial test if necessary if (initial_slow_test != NULL ) { - slow_region = new RegionNode(3); - + if (slow_region == NULL) { + slow_region = new RegionNode(1); + } // Now make the initial failure test. Usually a too-big test but // might be a TRUE for finalizers or a fancy class check for // newInstance0. - IfNode *toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN); + IfNode* toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN); transform_later(toobig_iff); // Plug the failing-too-big test into the slow-path region - Node *toobig_true = new IfTrueNode( toobig_iff ); + Node* toobig_true = new IfTrueNode(toobig_iff); transform_later(toobig_true); - slow_region ->init_req( too_big_or_final_path, toobig_true ); - toobig_false = new IfFalseNode( toobig_iff ); + slow_region ->add_req(toobig_true); + toobig_false = new IfFalseNode(toobig_iff); transform_later(toobig_false); } else { // No initial test, just fall into next case toobig_false = ctrl; - debug_only(slow_region = NodeSentinel); } Node *slow_mem = mem; // save the current memory state for slow path @@ -1341,11 +1420,11 @@ fast_oop_ctrl, fast_oop_rawmem, prefetch_lines); - if (initial_slow_test) { - slow_region->init_req(need_gc_path, needgc_ctrl); + if (slow_region != NULL) { + slow_region->add_req(needgc_ctrl); // This completes all paths into the slow merge point transform_later(slow_region); - } else { // No initial slow path needed! + } else { // Just fall from the need-GC path straight into the VM call. slow_region = needgc_ctrl; } @@ -1612,21 +1691,17 @@ // Helper for PhaseMacroExpand::expand_allocate_common. // Initializes the newly-allocated storage. -Node* -PhaseMacroExpand::initialize_object(AllocateNode* alloc, - Node* control, Node* rawmem, Node* object, - Node* klass_node, Node* length, - Node* size_in_bytes) { +Node* PhaseMacroExpand::initialize_object(AllocateNode* alloc, + Node* control, Node* rawmem, Node* object, + Node* klass_node, Node* length, + Node* size_in_bytes) { InitializeNode* init = alloc->initialization(); // Store the klass & mark bits - Node* mark_node = NULL; - // For now only enable fast locking for non-array types - if (UseBiasedLocking && (length == NULL)) { - mark_node = make_load(control, rawmem, klass_node, in_bytes(Klass::prototype_header_offset()), TypeRawPtr::BOTTOM, T_ADDRESS); - } else { - mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype())); + Node* mark_node = alloc->make_ideal_mark(&_igvn, object, control, rawmem, klass_node); + if (!mark_node->is_Con()) { + transform_later(mark_node); } - rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS); + rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, TypeX_X->basic_type()); rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA); int header_size = alloc->minimum_header_size(); // conservatively small @@ -1654,6 +1729,8 @@ // within an Allocate, and then (maybe or maybe not) clear some more later. if (!(UseTLAB && ZeroTLAB)) { rawmem = ClearArrayNode::clear_memory(control, rawmem, object, + alloc->in(AllocateNode::DefaultValue), + alloc->in(AllocateNode::RawDefaultValue), header_size, size_in_bytes, &_igvn); } @@ -2415,6 +2492,211 @@ _igvn.replace_node(_memproj_fallthrough, mem_phi); } +// A value type might be returned from the call but we don't know its +// type. Either we get a buffered value (and nothing needs to be done) +// or one of the values being returned is the klass of the value type +// and we need to allocate a value type instance of that type and +// initialize it with other values being returned. In that case, we +// first try a fast path allocation and initialize the value with the +// value klass's pack handler or we fall back to a runtime call. +void PhaseMacroExpand::expand_mh_intrinsic_return(CallStaticJavaNode* call) { + assert(call->method()->is_method_handle_intrinsic(), "must be a method handle intrinsic call"); + Node* ret = call->proj_out_or_null(TypeFunc::Parms); + if (ret == NULL) { + return; + } + const TypeFunc* tf = call->_tf; + const TypeTuple* domain = OptoRuntime::store_value_type_fields_Type()->domain_cc(); + const TypeFunc* new_tf = TypeFunc::make(tf->domain_sig(), tf->domain_cc(), tf->range_sig(), domain); + call->_tf = new_tf; + // Make sure the change of type is applied before projections are processed by igvn + _igvn.set_type(call, call->Value(&_igvn)); + _igvn.set_type(ret, ret->Value(&_igvn)); + + // Before any new projection is added: + CallProjections* projs = call->extract_projections(true, true); + + Node* ctl = new Node(1); + Node* mem = new Node(1); + Node* io = new Node(1); + Node* ex_ctl = new Node(1); + Node* ex_mem = new Node(1); + Node* ex_io = new Node(1); + Node* res = new Node(1); + + Node* cast = transform_later(new CastP2XNode(ctl, res)); + Node* mask = MakeConX(0x1); + Node* masked = transform_later(new AndXNode(cast, mask)); + Node* cmp = transform_later(new CmpXNode(masked, mask)); + Node* bol = transform_later(new BoolNode(cmp, BoolTest::eq)); + IfNode* allocation_iff = new IfNode(ctl, bol, PROB_MAX, COUNT_UNKNOWN); + transform_later(allocation_iff); + Node* allocation_ctl = transform_later(new IfTrueNode(allocation_iff)); + Node* no_allocation_ctl = transform_later(new IfFalseNode(allocation_iff)); + + Node* no_allocation_res = transform_later(new CheckCastPPNode(no_allocation_ctl, res, TypeInstPtr::BOTTOM)); + + Node* mask2 = MakeConX(-2); + Node* masked2 = transform_later(new AndXNode(cast, mask2)); + Node* rawklassptr = transform_later(new CastX2PNode(masked2)); + Node* klass_node = transform_later(new CheckCastPPNode(allocation_ctl, rawklassptr, TypeKlassPtr::OBJECT_OR_NULL)); + + Node* slowpath_bol = NULL; + Node* top_adr = NULL; + Node* old_top = NULL; + Node* new_top = NULL; + if (UseTLAB) { + Node* end_adr = NULL; + set_eden_pointers(top_adr, end_adr); + Node* end = make_load(ctl, mem, end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS); + old_top = new LoadPNode(ctl, mem, top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered); + transform_later(old_top); + Node* layout_val = make_load(NULL, mem, klass_node, in_bytes(Klass::layout_helper_offset()), TypeInt::INT, T_INT); + Node* size_in_bytes = ConvI2X(layout_val); + new_top = new AddPNode(top(), old_top, size_in_bytes); + transform_later(new_top); + Node* slowpath_cmp = new CmpPNode(new_top, end); + transform_later(slowpath_cmp); + slowpath_bol = new BoolNode(slowpath_cmp, BoolTest::ge); + transform_later(slowpath_bol); + } else { + slowpath_bol = intcon(1); + top_adr = top(); + old_top = top(); + new_top = top(); + } + IfNode* slowpath_iff = new IfNode(allocation_ctl, slowpath_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN); + transform_later(slowpath_iff); + + Node* slowpath_true = new IfTrueNode(slowpath_iff); + transform_later(slowpath_true); + + CallStaticJavaNode* slow_call = new CallStaticJavaNode(OptoRuntime::store_value_type_fields_Type(), + StubRoutines::store_value_type_fields_to_buf(), + "store_value_type_fields", + call->jvms()->bci(), + TypePtr::BOTTOM); + slow_call->init_req(TypeFunc::Control, slowpath_true); + slow_call->init_req(TypeFunc::Memory, mem); + slow_call->init_req(TypeFunc::I_O, io); + slow_call->init_req(TypeFunc::FramePtr, call->in(TypeFunc::FramePtr)); + slow_call->init_req(TypeFunc::ReturnAdr, call->in(TypeFunc::ReturnAdr)); + slow_call->init_req(TypeFunc::Parms, res); + + Node* slow_ctl = transform_later(new ProjNode(slow_call, TypeFunc::Control)); + Node* slow_mem = transform_later(new ProjNode(slow_call, TypeFunc::Memory)); + Node* slow_io = transform_later(new ProjNode(slow_call, TypeFunc::I_O)); + Node* slow_res = transform_later(new ProjNode(slow_call, TypeFunc::Parms)); + Node* slow_catc = transform_later(new CatchNode(slow_ctl, slow_io, 2)); + Node* slow_norm = transform_later(new CatchProjNode(slow_catc, CatchProjNode::fall_through_index, CatchProjNode::no_handler_bci)); + Node* slow_excp = transform_later(new CatchProjNode(slow_catc, CatchProjNode::catch_all_index, CatchProjNode::no_handler_bci)); + + Node* ex_r = new RegionNode(3); + Node* ex_mem_phi = new PhiNode(ex_r, Type::MEMORY, TypePtr::BOTTOM); + Node* ex_io_phi = new PhiNode(ex_r, Type::ABIO); + ex_r->init_req(1, slow_excp); + ex_mem_phi->init_req(1, slow_mem); + ex_io_phi->init_req(1, slow_io); + ex_r->init_req(2, ex_ctl); + ex_mem_phi->init_req(2, ex_mem); + ex_io_phi->init_req(2, ex_io); + + transform_later(ex_r); + transform_later(ex_mem_phi); + transform_later(ex_io_phi); + + Node* slowpath_false = new IfFalseNode(slowpath_iff); + transform_later(slowpath_false); + Node* rawmem = new StorePNode(slowpath_false, mem, top_adr, TypeRawPtr::BOTTOM, new_top, MemNode::unordered); + transform_later(rawmem); + Node* mark_node = makecon(TypeRawPtr::make((address)markOopDesc::always_locked_prototype())); + rawmem = make_store(slowpath_false, rawmem, old_top, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS); + rawmem = make_store(slowpath_false, rawmem, old_top, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA); + if (UseCompressedClassPointers) { + rawmem = make_store(slowpath_false, rawmem, old_top, oopDesc::klass_gap_offset_in_bytes(), intcon(0), T_INT); + } + Node* fixed_block = make_load(slowpath_false, rawmem, klass_node, in_bytes(InstanceKlass::adr_valueklass_fixed_block_offset()), TypeRawPtr::BOTTOM, T_ADDRESS); + Node* pack_handler = make_load(slowpath_false, rawmem, fixed_block, in_bytes(ValueKlass::pack_handler_offset()), TypeRawPtr::BOTTOM, T_ADDRESS); + + CallLeafNoFPNode* handler_call = new CallLeafNoFPNode(OptoRuntime::pack_value_type_Type(), + NULL, + "pack handler", + TypeRawPtr::BOTTOM); + handler_call->init_req(TypeFunc::Control, slowpath_false); + handler_call->init_req(TypeFunc::Memory, rawmem); + handler_call->init_req(TypeFunc::I_O, top()); + handler_call->init_req(TypeFunc::FramePtr, call->in(TypeFunc::FramePtr)); + handler_call->init_req(TypeFunc::ReturnAdr, top()); + handler_call->init_req(TypeFunc::Parms, pack_handler); + handler_call->init_req(TypeFunc::Parms+1, old_top); + + // We don't know how many values are returned. This assumes the + // worst case, that all available registers are used. + for (uint i = TypeFunc::Parms+1; i < domain->cnt(); i++) { + if (domain->field_at(i) == Type::HALF) { + slow_call->init_req(i, top()); + handler_call->init_req(i+1, top()); + continue; + } + Node* proj = transform_later(new ProjNode(call, i)); + slow_call->init_req(i, proj); + handler_call->init_req(i+1, proj); + } + + // We can safepoint at that new call + copy_call_debug_info(call, slow_call); + transform_later(slow_call); + transform_later(handler_call); + + Node* handler_ctl = transform_later(new ProjNode(handler_call, TypeFunc::Control)); + rawmem = transform_later(new ProjNode(handler_call, TypeFunc::Memory)); + Node* slowpath_false_res = transform_later(new ProjNode(handler_call, TypeFunc::Parms)); + + MergeMemNode* slowpath_false_mem = MergeMemNode::make(mem); + slowpath_false_mem->set_memory_at(Compile::AliasIdxRaw, rawmem); + transform_later(slowpath_false_mem); + + Node* r = new RegionNode(4); + Node* mem_phi = new PhiNode(r, Type::MEMORY, TypePtr::BOTTOM); + Node* io_phi = new PhiNode(r, Type::ABIO); + Node* res_phi = new PhiNode(r, TypeInstPtr::BOTTOM); + + r->init_req(1, no_allocation_ctl); + mem_phi->init_req(1, mem); + io_phi->init_req(1, io); + res_phi->init_req(1, no_allocation_res); + r->init_req(2, slow_norm); + mem_phi->init_req(2, slow_mem); + io_phi->init_req(2, slow_io); + res_phi->init_req(2, slow_res); + r->init_req(3, handler_ctl); + mem_phi->init_req(3, slowpath_false_mem); + io_phi->init_req(3, io); + res_phi->init_req(3, slowpath_false_res); + + transform_later(r); + transform_later(mem_phi); + transform_later(io_phi); + transform_later(res_phi); + + assert(projs->nb_resproj == 1, "unexpected number of results"); + _igvn.replace_in_uses(projs->fallthrough_catchproj, r); + _igvn.replace_in_uses(projs->fallthrough_memproj, mem_phi); + _igvn.replace_in_uses(projs->fallthrough_ioproj, io_phi); + _igvn.replace_in_uses(projs->resproj[0], res_phi); + _igvn.replace_in_uses(projs->catchall_catchproj, ex_r); + _igvn.replace_in_uses(projs->catchall_memproj, ex_mem_phi); + _igvn.replace_in_uses(projs->catchall_ioproj, ex_io_phi); + + _igvn.replace_node(ctl, projs->fallthrough_catchproj); + _igvn.replace_node(mem, projs->fallthrough_memproj); + _igvn.replace_node(io, projs->fallthrough_ioproj); + _igvn.replace_node(res, projs->resproj[0]); + _igvn.replace_node(ex_ctl, projs->catchall_catchproj); + _igvn.replace_node(ex_mem, projs->catchall_memproj); + _igvn.replace_node(ex_io, projs->catchall_ioproj); + } + //---------------------------eliminate_macro_nodes---------------------- // Eliminate scalar replaced allocations and associated locks. void PhaseMacroExpand::eliminate_macro_nodes() { @@ -2459,9 +2741,13 @@ case Node::Class_AllocateArray: success = eliminate_allocate_node(n->as_Allocate()); break; - case Node::Class_CallStaticJava: - success = eliminate_boxing_node(n->as_CallStaticJava()); + case Node::Class_CallStaticJava: { + CallStaticJavaNode* call = n->as_CallStaticJava(); + if (!call->method()->is_method_handle_intrinsic()) { + success = eliminate_boxing_node(n->as_CallStaticJava()); + } break; + } case Node::Class_Lock: case Node::Class_Unlock: assert(!n->as_AbstractLock()->is_eliminated(), "sanity"); @@ -2511,10 +2797,13 @@ _igvn._worklist.push(n); success = true; } else if (n->Opcode() == Op_CallStaticJava) { - // Remove it from macro list and put on IGVN worklist to optimize. - C->remove_macro_node(n); - _igvn._worklist.push(n); - success = true; + CallStaticJavaNode* call = n->as_CallStaticJava(); + if (!call->method()->is_method_handle_intrinsic()) { + // Remove it from macro list and put on IGVN worklist to optimize. + C->remove_macro_node(n); + _igvn._worklist.push(n); + success = true; + } } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) { _igvn.replace_node(n, n->in(1)); success = true; @@ -2595,6 +2884,10 @@ case Node::Class_Unlock: expand_unlock_node(n->as_Unlock()); break; + case Node::Class_CallStaticJava: + expand_mh_intrinsic_return(n->as_CallStaticJava()); + C->remove_macro_node(n); + break; default: assert(false, "unknown node type in macro list"); }