--- old/src/hotspot/share/runtime/sharedRuntime.cpp 2019-03-11 14:27:18.622354153 +0100 +++ new/src/hotspot/share/runtime/sharedRuntime.cpp 2019-03-11 14:27:18.378354157 +0100 @@ -44,12 +44,17 @@ #include "jfr/jfrEvents.hpp" #include "logging/log.hpp" #include "memory/metaspaceShared.hpp" +#include "memory/oopFactory.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" +#include "oops/access.hpp" +#include "oops/fieldStreams.hpp" #include "oops/klass.hpp" #include "oops/method.inline.hpp" #include "oops/objArrayKlass.hpp" +#include "oops/objArrayOop.inline.hpp" #include "oops/oop.inline.hpp" +#include "oops/valueKlass.hpp" #include "prims/forte.hpp" #include "prims/jvmtiExport.hpp" #include "prims/methodHandles.hpp" @@ -1116,6 +1121,12 @@ default: break; } + } else { + assert(attached_method->has_scalarized_args(), "invalid use of attached method"); + if (!attached_method->method_holder()->is_value()) { + // Ignore the attached method in this case to not confuse below code + attached_method = NULL; + } } } @@ -1134,18 +1145,33 @@ // Caller-frame is a compiled frame frame callerFrame = stubFrame.sender(®_map2); - if (attached_method.is_null()) { - methodHandle callee = bytecode.static_target(CHECK_NH); + methodHandle callee = attached_method; + if (callee.is_null()) { + callee = bytecode.static_target(CHECK_NH); if (callee.is_null()) { THROW_(vmSymbols::java_lang_NoSuchMethodException(), nullHandle); } } + if (callee->has_scalarized_args() && callee->method_holder()->is_value()) { + // If the receiver is a value type that is passed as fields, no oop is available. + // Resolve the call without receiver null checking. + assert(!attached_method.is_null(), "must have attached method"); + if (bc == Bytecodes::_invokevirtual) { + LinkInfo link_info(attached_method->method_holder(), attached_method->name(), attached_method->signature()); + LinkResolver::resolve_virtual_call(callinfo, receiver, callee->method_holder(), link_info, /*check_null_and_abstract=*/ false, CHECK_NH); + } else { + assert(bc == Bytecodes::_invokeinterface, "anything else?"); + LinkInfo link_info(constantPoolHandle(THREAD, caller->constants()), bytecode_index, CHECK_NH); + LinkResolver::resolve_interface_call(callinfo, receiver, callee->method_holder(), link_info, /*check_null_and_abstract=*/ false, CHECK_NH); + } + return receiver; // is null + } else { + // Retrieve from a compiled argument list + receiver = Handle(THREAD, callerFrame.retrieve_receiver(®_map2)); - // Retrieve from a compiled argument list - receiver = Handle(THREAD, callerFrame.retrieve_receiver(®_map2)); - - if (receiver.is_null()) { - THROW_(vmSymbols::java_lang_NullPointerException(), nullHandle); + if (receiver.is_null()) { + THROW_(vmSymbols::java_lang_NullPointerException(), nullHandle); + } } } @@ -1273,10 +1299,16 @@ bool is_nmethod = caller_nm->is_nmethod(); if (is_virtual) { - assert(receiver.not_null() || invoke_code == Bytecodes::_invokehandle, "sanity check"); + Klass* receiver_klass = NULL; + if (ValueTypePassFieldsAsArgs && callee_method->method_holder()->is_value()) { + // If the receiver is a value type that is passed as fields, no oop is available + receiver_klass = callee_method->method_holder(); + } else { + assert(receiver.not_null() || invoke_code == Bytecodes::_invokehandle, "sanity check"); + receiver_klass = invoke_code == Bytecodes::_invokehandle ? NULL : receiver->klass(); + } bool static_bound = call_info.resolved_method()->can_be_statically_bound(); - Klass* klass = invoke_code == Bytecodes::_invokehandle ? NULL : receiver->klass(); - CompiledIC::compute_monomorphic_entry(callee_method, klass, + CompiledIC::compute_monomorphic_entry(callee_method, receiver_klass, is_optimized, static_bound, is_nmethod, virtual_call_info, CHECK_false); } else { @@ -1341,6 +1373,16 @@ // CLEANUP - with lazy deopt shouldn't need this lock nmethodLocker caller_lock(caller_nm); + if (!is_virtual && !is_optimized) { + SimpleScopeDesc ssd(caller_nm, caller_frame.pc()); + Bytecode bc(ssd.method(), ssd.method()->bcp_from(ssd.bci())); + // Substitutability test implementation piggy backs on static call resolution + if (bc.code() == Bytecodes::_if_acmpeq || bc.code() == Bytecodes::_if_acmpne) { + SystemDictionary::ValueBootstrapMethods_klass()->initialize(CHECK_NULL); + return SystemDictionary::ValueBootstrapMethods_klass()->find_method(vmSymbols::isSubstitutable_name(), vmSymbols::object_object_boolean_signature()); + } + } + // determine call info & receiver // note: a) receiver is NULL for static calls // b) an exception is thrown if receiver is NULL for non-static calls @@ -1428,14 +1470,16 @@ #endif /* ASSERT */ methodHandle callee_method; + bool is_optimized = false; JRT_BLOCK - callee_method = SharedRuntime::handle_ic_miss_helper(thread, CHECK_NULL); + callee_method = SharedRuntime::handle_ic_miss_helper(thread, is_optimized, CHECK_NULL); // Return Method* through TLS thread->set_vm_result_2(callee_method()); JRT_BLOCK_END // return compiled code entry point after potential safepoints - assert(callee_method->verified_code_entry() != NULL, " Jump to zero!"); - return callee_method->verified_code_entry(); + assert(callee_method->verified_code_entry() != NULL, "Jump to zero!"); + assert(callee_method->verified_value_ro_code_entry() != NULL, "Jump to zero!"); + return is_optimized ? callee_method->verified_code_entry() : callee_method->verified_value_ro_code_entry(); JRT_END @@ -1466,14 +1510,16 @@ // Must be compiled to compiled path which is safe to stackwalk methodHandle callee_method; + bool is_optimized = false; JRT_BLOCK // Force resolving of caller (if we called from compiled frame) - callee_method = SharedRuntime::reresolve_call_site(thread, CHECK_NULL); + callee_method = SharedRuntime::reresolve_call_site(thread, is_optimized, CHECK_NULL); thread->set_vm_result_2(callee_method()); JRT_BLOCK_END // return compiled code entry point after potential safepoints - assert(callee_method->verified_code_entry() != NULL, " Jump to zero!"); - return callee_method->verified_code_entry(); + assert(callee_method->verified_code_entry() != NULL, "Jump to zero!"); + assert(callee_method->verified_value_ro_code_entry() != NULL, "Jump to zero!"); + return is_optimized ? callee_method->verified_code_entry() : callee_method->verified_value_ro_code_entry(); JRT_END // Handle abstract method call @@ -1516,7 +1562,7 @@ thread->set_vm_result_2(callee_method()); JRT_BLOCK_END // return compiled code entry point after potential safepoints - assert(callee_method->verified_code_entry() != NULL, " Jump to zero!"); + assert(callee_method->verified_code_entry() != NULL, "Jump to zero!"); return callee_method->verified_code_entry(); JRT_END @@ -1529,8 +1575,8 @@ thread->set_vm_result_2(callee_method()); JRT_BLOCK_END // return compiled code entry point after potential safepoints - assert(callee_method->verified_code_entry() != NULL, " Jump to zero!"); - return callee_method->verified_code_entry(); + assert(callee_method->verified_value_ro_code_entry() != NULL, "Jump to zero!"); + return callee_method->verified_value_ro_code_entry(); JRT_END @@ -1543,7 +1589,7 @@ thread->set_vm_result_2(callee_method()); JRT_BLOCK_END // return compiled code entry point after potential safepoints - assert(callee_method->verified_code_entry() != NULL, " Jump to zero!"); + assert(callee_method->verified_code_entry() != NULL, "Jump to zero!"); return callee_method->verified_code_entry(); JRT_END @@ -1555,7 +1601,7 @@ bool SharedRuntime::handle_ic_miss_helper_internal(Handle receiver, CompiledMethod* caller_nm, const frame& caller_frame, methodHandle callee_method, Bytecodes::Code bc, CallInfo& call_info, - bool& needs_ic_stub_refill, TRAPS) { + bool& needs_ic_stub_refill, bool& is_optimized, TRAPS) { CompiledICLocker ml(caller_nm); CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc()); bool should_be_mono = false; @@ -1566,6 +1612,7 @@ callee_method->print_short_name(tty); tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code())); } + is_optimized = true; should_be_mono = true; } else if (inline_cache->is_icholder_call()) { CompiledICHolder* ic_oop = inline_cache->cached_icholder(); @@ -1627,7 +1674,7 @@ return true; } -methodHandle SharedRuntime::handle_ic_miss_helper(JavaThread *thread, TRAPS) { +methodHandle SharedRuntime::handle_ic_miss_helper(JavaThread *thread, bool& is_optimized, TRAPS) { ResourceMark rm(thread); CallInfo call_info; Bytecodes::Code bc; @@ -1647,7 +1694,7 @@ // did this would still be the correct thing to do for it too, hence no ifdef. // if (call_info.resolved_method()->can_be_statically_bound()) { - methodHandle callee_method = SharedRuntime::reresolve_call_site(thread, CHECK_(methodHandle())); + methodHandle callee_method = SharedRuntime::reresolve_call_site(thread, is_optimized, CHECK_(methodHandle())); if (TraceCallFixup) { RegisterMap reg_map(thread, false); frame caller_frame = thread->last_frame().sender(®_map); @@ -1702,7 +1749,7 @@ ICRefillVerifier ic_refill_verifier; bool needs_ic_stub_refill = false; bool successful = handle_ic_miss_helper_internal(receiver, caller_nm, caller_frame, callee_method, - bc, call_info, needs_ic_stub_refill, CHECK_(methodHandle())); + bc, call_info, needs_ic_stub_refill, is_optimized, CHECK_(methodHandle())); if (successful || !needs_ic_stub_refill) { return callee_method; } else { @@ -1734,7 +1781,7 @@ // sites, and static call sites. Typically used to change a call sites // destination from compiled to interpreted. // -methodHandle SharedRuntime::reresolve_call_site(JavaThread *thread, TRAPS) { +methodHandle SharedRuntime::reresolve_call_site(JavaThread *thread, bool& is_optimized, TRAPS) { ResourceMark rm(thread); RegisterMap reg_map(thread, false); frame stub_frame = thread->last_frame(); @@ -1795,6 +1842,7 @@ assert(iter.type() == relocInfo::virtual_call_type || iter.type() == relocInfo::opt_virtual_call_type , "unexpected relocInfo. type"); + is_optimized = (iter.type() == relocInfo::opt_virtual_call_type); } } else { assert(!UseInlineCaches, "relocation info. must exist for this address"); @@ -1820,7 +1868,6 @@ methodHandle callee_method = find_callee_method(thread, CHECK_(methodHandle())); - #ifndef PRODUCT Atomic::inc(&_wrong_method_ctr); @@ -2316,14 +2363,27 @@ // Remap BasicTypes that are handled equivalently by the adapters. // These are correct for the current system but someday it might be // necessary to make this mapping platform dependent. - static int adapter_encoding(BasicType in) { + static int adapter_encoding(BasicType in, bool is_valuetype) { switch (in) { case T_BOOLEAN: case T_BYTE: case T_SHORT: - case T_CHAR: - // There are all promoted to T_INT in the calling convention - return T_INT; + case T_CHAR: { + if (is_valuetype) { + // Do not widen value type field types + assert(ValueTypePassFieldsAsArgs, "must be enabled"); + return in; + } else { + // They are all promoted to T_INT in the calling convention + return T_INT; + } + } + + case T_VALUETYPE: { + // If value types are passed as fields, return 'in' to differentiate + // between a T_VALUETYPE and a T_OBJECT in the signature. + return ValueTypePassFieldsAsArgs ? in : adapter_encoding(T_OBJECT, false); + } case T_OBJECT: case T_ARRAY: @@ -2349,9 +2409,10 @@ } public: - AdapterFingerPrint(int total_args_passed, BasicType* sig_bt) { + AdapterFingerPrint(const GrowableArray* sig, bool has_ro_adapter = false) { // The fingerprint is based on the BasicType signature encoded // into an array of ints with eight entries per int. + int total_args_passed = (sig != NULL) ? sig->length() : 0; int* ptr; int len = (total_args_passed + (_basic_types_per_int-1)) / _basic_types_per_int; if (len <= _compact_int_count) { @@ -2369,17 +2430,37 @@ // Now pack the BasicTypes with 8 per int int sig_index = 0; + BasicType prev_sbt = T_ILLEGAL; + int vt_count = 0; for (int index = 0; index < len; index++) { int value = 0; for (int byte = 0; byte < _basic_types_per_int; byte++) { - int bt = ((sig_index < total_args_passed) - ? adapter_encoding(sig_bt[sig_index++]) - : 0); + int bt = 0; + if (sig_index < total_args_passed) { + BasicType sbt = sig->at(sig_index++)._bt; + if (ValueTypePassFieldsAsArgs && sbt == T_VALUETYPE) { + // Found start of value type in signature + vt_count++; + if (sig_index == 1 && has_ro_adapter) { + // With a ro_adapter, replace receiver value type delimiter by T_VOID to prevent matching + // with other adapters that have the same value type as first argument and no receiver. + sbt = T_VOID; + } + } else if (ValueTypePassFieldsAsArgs && sbt == T_VOID && + prev_sbt != T_LONG && prev_sbt != T_DOUBLE) { + // Found end of value type in signature + vt_count--; + assert(vt_count >= 0, "invalid vt_count"); + } + bt = adapter_encoding(sbt, vt_count > 0); + prev_sbt = sbt; + } assert((bt & _basic_type_mask) == bt, "must fit in 4 bits"); value = (value << _basic_type_bits) | bt; } ptr[index] = value; } + assert(vt_count == 0, "invalid vt_count"); } ~AdapterFingerPrint() { @@ -2465,9 +2546,9 @@ : BasicHashtable(293, (DumpSharedSpaces ? sizeof(CDSAdapterHandlerEntry) : sizeof(AdapterHandlerEntry))) { } // Create a new entry suitable for insertion in the table - AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry, address c2i_unverified_entry) { + AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry, address c2i_value_entry, address c2i_value_ro_entry, address c2i_unverified_entry) { AdapterHandlerEntry* entry = (AdapterHandlerEntry*)BasicHashtable::new_entry(fingerprint->compute_hash()); - entry->init(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry); + entry->init(fingerprint, i2c_entry, c2i_entry, c2i_value_entry, c2i_value_ro_entry, c2i_unverified_entry); if (DumpSharedSpaces) { ((CDSAdapterHandlerEntry*)entry)->init(); } @@ -2486,9 +2567,9 @@ } // Find a entry with the same fingerprint if it exists - AdapterHandlerEntry* lookup(int total_args_passed, BasicType* sig_bt) { + AdapterHandlerEntry* lookup(const GrowableArray* sig, bool has_ro_adapter = false) { NOT_PRODUCT(_lookups++); - AdapterFingerPrint fp(total_args_passed, sig_bt); + AdapterFingerPrint fp(sig, has_ro_adapter); unsigned int hash = fp.compute_hash(); int index = hash_to_index(hash); for (AdapterHandlerEntry* e = bucket(index); e != NULL; e = e->next()) { @@ -2608,16 +2689,18 @@ // Pass wrong_method_abstract for the c2i transitions to return // AbstractMethodError for invalid invocations. address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub(); - _abstract_method_handler = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(0, NULL), + _abstract_method_handler = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(NULL), StubRoutines::throw_AbstractMethodError_entry(), - wrong_method_abstract, wrong_method_abstract); + wrong_method_abstract, wrong_method_abstract, wrong_method_abstract, wrong_method_abstract); } AdapterHandlerEntry* AdapterHandlerLibrary::new_entry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry, + address c2i_value_entry, + address c2i_value_ro_entry, address c2i_unverified_entry) { - return _adapters->new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry); + return _adapters->new_entry(fingerprint, i2c_entry, c2i_entry, c2i_value_entry, c2i_value_ro_entry, c2i_unverified_entry); } AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter(const methodHandle& method) { @@ -2644,6 +2727,53 @@ return entry; } +static int compute_scalarized_cc(const methodHandle& method, GrowableArray& sig_cc, VMRegPair*& regs_cc, bool scalar_receiver) { + InstanceKlass* holder = method->method_holder(); + sig_cc = GrowableArray(method->size_of_parameters()); + if (!method->is_static()) { + if (holder->is_value() && scalar_receiver) { + sig_cc.appendAll(ValueKlass::cast(holder)->extended_sig()); + } else { + SigEntry::add_entry(&sig_cc, T_OBJECT); + } + } + Thread* THREAD = Thread::current(); + for (SignatureStream ss(method->signature()); !ss.at_return_type(); ss.next()) { + if (ss.type() == T_VALUETYPE) { + Klass* k = ss.as_klass(Handle(THREAD, holder->class_loader()), + Handle(THREAD, holder->protection_domain()), + SignatureStream::ReturnNull, THREAD); + assert(k != NULL && !HAS_PENDING_EXCEPTION, "value klass should have been pre-loaded"); + sig_cc.appendAll(ValueKlass::cast(k)->extended_sig()); + } else { + SigEntry::add_entry(&sig_cc, ss.type()); + } + } + regs_cc = NEW_RESOURCE_ARRAY(VMRegPair, sig_cc.length() + 2); + return SharedRuntime::java_calling_convention(&sig_cc, regs_cc); +} + +static int insert_reserved_entry(GrowableArray& sig_cc, VMRegPair*& regs_cc, int ret_off) { + // Find index in signature that belongs to return address slot + BasicType bt = T_ILLEGAL; + int i = 0; + for (uint off = 0; i < sig_cc.length(); ++i) { + if (SigEntry::skip_value_delimiters(&sig_cc, i)) { + VMReg first = regs_cc[off++].first(); + if (first->is_valid() && first->is_stack()) { + // Select a type for the reserved entry that will end up on the stack + bt = sig_cc.at(i)._bt; + if (((int)first->reg2stack() + VMRegImpl::slots_per_word) == ret_off) { + break; // Index of the return address found + } + } + } + } + // Insert reserved entry and re-compute calling convention + SigEntry::insert_reserved_entry(&sig_cc, i, bt); + return SharedRuntime::java_calling_convention(&sig_cc, regs_cc); +} + AdapterHandlerEntry* AdapterHandlerLibrary::get_adapter0(const methodHandle& method) { // Use customized signature handler. Need to lock around updates to // the AdapterHandlerTable (it is not safe for concurrent readers @@ -2652,36 +2782,123 @@ ResourceMark rm; - NOT_PRODUCT(int insts_size); + NOT_PRODUCT(int insts_size = 0); AdapterBlob* new_adapter = NULL; AdapterHandlerEntry* entry = NULL; AdapterFingerPrint* fingerprint = NULL; + { MutexLocker mu(AdapterHandlerLibrary_lock); // make sure data structure is initialized initialize(); - if (method->is_abstract()) { + bool has_value_arg = false; + bool has_value_recv = false; + GrowableArray sig(method->size_of_parameters()); + if (!method->is_static()) { + has_value_recv = method->method_holder()->is_value(); + has_value_arg = has_value_recv; + SigEntry::add_entry(&sig, T_OBJECT); + } + for (SignatureStream ss(method->signature()); !ss.at_return_type(); ss.next()) { + BasicType bt = ss.type(); + if (bt == T_VALUETYPE) { + has_value_arg = true; + bt = T_OBJECT; + } + SigEntry::add_entry(&sig, bt); + } + + // Process abstract method if it has value type args to set has_scalarized_args accordingly + if (method->is_abstract() && !(ValueTypePassFieldsAsArgs && has_value_arg)) { return _abstract_method_handler; } - // Fill in the signature array, for the calling-convention call. - int total_args_passed = method->size_of_parameters(); // All args on stack + // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage + VMRegPair* regs = NEW_RESOURCE_ARRAY(VMRegPair, sig.length()); + int args_on_stack = SharedRuntime::java_calling_convention(&sig, regs); - BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_args_passed); - VMRegPair* regs = NEW_RESOURCE_ARRAY(VMRegPair, total_args_passed); - int i = 0; - if (!method->is_static()) // Pass in receiver first - sig_bt[i++] = T_OBJECT; - for (SignatureStream ss(method->signature()); !ss.at_return_type(); ss.next()) { - sig_bt[i++] = ss.type(); // Collect remaining bits of signature - if (ss.type() == T_LONG || ss.type() == T_DOUBLE) - sig_bt[i++] = T_VOID; // Longs & doubles take 2 Java slots + // Now compute the scalarized calling convention if there are value types in the signature + GrowableArray sig_cc = sig; + GrowableArray sig_cc_ro = sig; + VMRegPair* regs_cc = regs; + VMRegPair* regs_cc_ro = regs; + int args_on_stack_cc = args_on_stack; + int args_on_stack_cc_ro = args_on_stack; + + if (ValueTypePassFieldsAsArgs && has_value_arg && !method->is_native()) { + MutexUnlocker mul(AdapterHandlerLibrary_lock); + args_on_stack_cc = compute_scalarized_cc(method, sig_cc, regs_cc, /* scalar_receiver = */ true); + + sig_cc_ro = sig_cc; + regs_cc_ro = regs_cc; + args_on_stack_cc_ro = args_on_stack_cc; + if (has_value_recv || args_on_stack_cc > args_on_stack) { + // For interface calls, we need another entry point / adapter to unpack the receiver + args_on_stack_cc_ro = compute_scalarized_cc(method, sig_cc_ro, regs_cc_ro, /* scalar_receiver = */ false); + } + + // Compute the stack extension that is required to convert between the calling conventions. + // The stack slots at these offsets are occupied by the return address with the unscalarized + // calling convention. Don't use them for arguments with the scalarized calling convention. + int ret_off = args_on_stack_cc - args_on_stack; + int ret_off_ro = args_on_stack_cc - args_on_stack_cc_ro; + assert(ret_off_ro <= 0 || ret_off > 0, "receiver unpacking requires more stack space than expected"); + + if (ret_off > 0) { + // Make sure the stack of the scalarized calling convention with the reserved + // entries (2 slots each) remains 16-byte (4 slots) aligned after stack extension. + int alignment = StackAlignmentInBytes / VMRegImpl::stack_slot_size; + if (ret_off_ro != ret_off && ret_off_ro >= 0) { + ret_off += 4; // Account for two reserved entries (4 slots) + ret_off_ro += 4; + ret_off = align_up(ret_off, alignment); + ret_off_ro = align_up(ret_off_ro, alignment); + // TODO can we avoid wasting a stack slot here? + //assert(ret_off != ret_off_ro, "fail"); + if (ret_off > ret_off_ro) { + swap(ret_off, ret_off_ro); // Sort by offset + } + args_on_stack_cc = insert_reserved_entry(sig_cc, regs_cc, ret_off); + args_on_stack_cc = insert_reserved_entry(sig_cc, regs_cc, ret_off_ro); + } else { + ret_off += 2; // Account for one reserved entry (2 slots) + ret_off = align_up(ret_off, alignment); + args_on_stack_cc = insert_reserved_entry(sig_cc, regs_cc, ret_off); + } + } + + // Upper bound on stack arguments to avoid hitting the argument limit and + // bailing out of compilation ("unsupported incoming calling sequence"). + // TODO we need a reasonable limit (flag?) here + if (args_on_stack_cc > 50) { + // Don't scalarize value type arguments + sig_cc = sig; + sig_cc_ro = sig; + regs_cc = regs; + regs_cc_ro = regs; + args_on_stack_cc = args_on_stack; + } else { + method->set_has_scalarized_args(true); + method->set_needs_stack_repair(args_on_stack_cc > args_on_stack); + } + } + + if (method->is_abstract()) { + // Save a C heap allocated version of the signature for abstract methods with scalarized value type arguments + assert(ValueTypePassFieldsAsArgs && has_value_arg, "must have scalarized value type args"); + address wrong_method_abstract = SharedRuntime::get_handle_wrong_method_abstract_stub(); + entry = AdapterHandlerLibrary::new_entry(new AdapterFingerPrint(NULL), + StubRoutines::throw_AbstractMethodError_entry(), + wrong_method_abstract, wrong_method_abstract, wrong_method_abstract, wrong_method_abstract); + GrowableArray* heap_sig = new (ResourceObj::C_HEAP, mtInternal) GrowableArray(sig_cc_ro.length(), true); + heap_sig->appendAll(&sig_cc_ro); + entry->set_sig_cc(heap_sig); + return entry; } - assert(i == total_args_passed, ""); // Lookup method signature's fingerprint - entry = _adapters->lookup(total_args_passed, sig_bt); + entry = _adapters->lookup(&sig_cc, regs_cc != regs_cc_ro); #ifdef ASSERT AdapterHandlerEntry* shared_entry = NULL; @@ -2696,11 +2913,8 @@ return entry; } - // Get a description of the compiled java calling convention and the largest used (VMReg) stack slot usage - int comp_args_on_stack = SharedRuntime::java_calling_convention(sig_bt, regs, total_args_passed, false); - // Make a C heap allocated version of the fingerprint to store in the adapter - fingerprint = new AdapterFingerPrint(total_args_passed, sig_bt); + fingerprint = new AdapterFingerPrint(&sig_cc, regs_cc != regs_cc_ro); // StubRoutines::code2() is initialized after this function can be called. As a result, // VerifyAdapterCalls and VerifyAdapterSharing can fail if we re-use code that generated @@ -2718,14 +2932,30 @@ MacroAssembler _masm(&buffer); entry = SharedRuntime::generate_i2c2i_adapters(&_masm, - total_args_passed, - comp_args_on_stack, - sig_bt, + args_on_stack, + args_on_stack_cc, + &sig, regs, - fingerprint); + &sig_cc, + regs_cc, + &sig_cc_ro, + regs_cc_ro, + fingerprint, + new_adapter); + + if (regs != regs_cc) { + // Save a C heap allocated version of the scalarized signature and store it in the adapter + GrowableArray* heap_sig = new (ResourceObj::C_HEAP, mtInternal) GrowableArray(sig_cc.length(), true); + heap_sig->appendAll(&sig_cc); + entry->set_sig_cc(heap_sig); + } + #ifdef ASSERT if (VerifyAdapterSharing) { if (shared_entry != NULL) { + if (!shared_entry->compare_code(buf->code_begin(), buffer.insts_size())) { + method->print(); + } assert(shared_entry->compare_code(buf->code_begin(), buffer.insts_size()), "code must match"); // Release the one just created and return the original _adapters->free_entry(entry); @@ -2736,7 +2966,6 @@ } #endif - new_adapter = AdapterBlob::create(&buffer); NOT_PRODUCT(insts_size = buffer.insts_size()); } if (new_adapter == NULL) { @@ -2792,6 +3021,8 @@ address base = _i2c_entry; if (base == NULL) base = _c2i_entry; assert(base <= _c2i_entry || _c2i_entry == NULL, ""); + assert(base <= _c2i_value_entry || _c2i_value_entry == NULL, ""); + assert(base <= _c2i_value_ro_entry || _c2i_value_ro_entry == NULL, ""); assert(base <= _c2i_unverified_entry || _c2i_unverified_entry == NULL, ""); return base; } @@ -2804,6 +3035,10 @@ _i2c_entry += delta; if (_c2i_entry != NULL) _c2i_entry += delta; + if (_c2i_value_entry != NULL) + _c2i_value_entry += delta; + if (_c2i_value_ro_entry != NULL) + _c2i_value_ro_entry += delta; if (_c2i_unverified_entry != NULL) _c2i_unverified_entry += delta; assert(base_address() == new_base, ""); @@ -2812,6 +3047,9 @@ void AdapterHandlerEntry::deallocate() { delete _fingerprint; + if (_sig_cc != NULL) { + delete _sig_cc; + } #ifdef ASSERT if (_saved_code) FREE_C_HEAP_ARRAY(unsigned char, _saved_code); #endif @@ -2883,7 +3121,8 @@ sig_bt[i++] = T_OBJECT; SignatureStream ss(method->signature()); for (; !ss.at_return_type(); ss.next()) { - sig_bt[i++] = ss.type(); // Collect remaining bits of signature + BasicType bt = ss.type(); + sig_bt[i++] = bt; // Collect remaining bits of signature if (ss.type() == T_LONG || ss.type() == T_DOUBLE) sig_bt[i++] = T_VOID; // Longs & doubles take 2 Java slots } @@ -2998,12 +3237,16 @@ while (*s++ != ';'); // Skip signature sig_bt[cnt++] = T_OBJECT; break; + case 'Q': // Value type + while (*s++ != ';'); // Skip signature + sig_bt[cnt++] = T_VALUETYPE; + break; case '[': { // Array do { // Skip optional size while (*s >= '0' && *s <= '9') s++; } while (*s++ == '['); // Nested arrays? // Skip element type - if (s[-1] == 'L') + if (s[-1] == 'L' || s[-1] == 'Q') while (*s++ != ';'); // Skip signature sig_bt[cnt++] = T_ARRAY; break; @@ -3144,9 +3387,9 @@ } void AdapterHandlerEntry::print_adapter_on(outputStream* st) const { - st->print_cr("AHE@" INTPTR_FORMAT ": %s i2c: " INTPTR_FORMAT " c2i: " INTPTR_FORMAT " c2iUV: " INTPTR_FORMAT, + st->print_cr("AHE@" INTPTR_FORMAT ": %s i2c: " INTPTR_FORMAT " c2i: " INTPTR_FORMAT " c2iVE: " INTPTR_FORMAT " c2iVROE: " INTPTR_FORMAT " c2iUE: " INTPTR_FORMAT, p2i(this), fingerprint()->as_string(), - p2i(get_i2c_entry()), p2i(get_c2i_entry()), p2i(get_c2i_unverified_entry())); + p2i(get_i2c_entry()), p2i(get_c2i_entry()), p2i(get_c2i_value_entry()), p2i(get_c2i_value_ro_entry()), p2i(get_c2i_unverified_entry())); } @@ -3240,3 +3483,210 @@ BarrierSet *bs = BarrierSet::barrier_set(); bs->on_slowpath_allocation_exit(thread, new_obj); } + +// We are at a compiled code to interpreter call. We need backing +// buffers for all value type arguments. Allocate an object array to +// hold them (convenient because once we're done with it we don't have +// to worry about freeing it). +JRT_ENTRY(void, SharedRuntime::allocate_value_types(JavaThread* thread, Method* callee_method, bool allocate_receiver)) +{ + assert(ValueTypePassFieldsAsArgs, "no reason to call this"); + ResourceMark rm; + JavaThread* THREAD = thread; + methodHandle callee(callee_method); + + int nb_slots = 0; + InstanceKlass* holder = callee->method_holder(); + allocate_receiver &= !callee->is_static() && holder->is_value(); + if (allocate_receiver) { + nb_slots++; + } + Handle class_loader(THREAD, holder->class_loader()); + Handle protection_domain(THREAD, holder->protection_domain()); + for (SignatureStream ss(callee->signature()); !ss.at_return_type(); ss.next()) { + if (ss.type() == T_VALUETYPE) { + nb_slots++; + } + } + objArrayOop array_oop = oopFactory::new_objectArray(nb_slots, CHECK); + objArrayHandle array(THREAD, array_oop); + int i = 0; + if (allocate_receiver) { + ValueKlass* vk = ValueKlass::cast(holder); + oop res = vk->allocate_instance(CHECK); + array->obj_at_put(i, res); + i++; + } + for (SignatureStream ss(callee->signature()); !ss.at_return_type(); ss.next()) { + if (ss.type() == T_VALUETYPE) { + Klass* k = ss.as_klass(class_loader, protection_domain, SignatureStream::ReturnNull, THREAD); + assert(k != NULL && !HAS_PENDING_EXCEPTION, "can't resolve klass"); + ValueKlass* vk = ValueKlass::cast(k); + oop res = vk->allocate_instance(CHECK); + array->obj_at_put(i, res); + i++; + } + } + thread->set_vm_result(array()); + thread->set_vm_result_2(callee()); // TODO: required to keep callee live? +} +JRT_END + +// Iterate of the array of heap allocated value types and apply the GC post barrier to all reference fields. +// This is called from the C2I adapter after value type arguments are heap allocated and initialized. +JRT_LEAF(void, SharedRuntime::apply_post_barriers(JavaThread* thread, objArrayOopDesc* array)) +{ + assert(ValueTypePassFieldsAsArgs, "no reason to call this"); + assert(oopDesc::is_oop(array), "should be oop"); + for (int i = 0; i < array->length(); ++i) { + instanceOop valueOop = (instanceOop)array->obj_at(i); + ValueKlass* vk = ValueKlass::cast(valueOop->klass()); + if (vk->contains_oops()) { + const address dst_oop_addr = ((address) (void*) valueOop); + OopMapBlock* map = vk->start_of_nonstatic_oop_maps(); + OopMapBlock* const end = map + vk->nonstatic_oop_map_count(); + while (map != end) { + address doop_address = dst_oop_addr + map->offset(); + barrier_set_cast(BarrierSet::barrier_set())-> + write_ref_array((HeapWord*) doop_address, map->count()); + map++; + } + } + } +} +JRT_END + +// We're returning from an interpreted method: load each field into a +// register following the calling convention +JRT_LEAF(void, SharedRuntime::load_value_type_fields_in_regs(JavaThread* thread, oopDesc* res)) +{ + assert(res->klass()->is_value(), "only value types here"); + ResourceMark rm; + RegisterMap reg_map(thread); + frame stubFrame = thread->last_frame(); + frame callerFrame = stubFrame.sender(®_map); + assert(callerFrame.is_interpreted_frame(), "should be coming from interpreter"); + + ValueKlass* vk = ValueKlass::cast(res->klass()); + + const Array* sig_vk = vk->extended_sig(); + const Array* regs = vk->return_regs(); + + if (regs == NULL) { + // The fields of the value klass don't fit in registers, bail out + return; + } + + int j = 1; + for (int i = 0; i < sig_vk->length(); i++) { + BasicType bt = sig_vk->at(i)._bt; + if (bt == T_VALUETYPE) { + continue; + } + if (bt == T_VOID) { + if (sig_vk->at(i-1)._bt == T_LONG || + sig_vk->at(i-1)._bt == T_DOUBLE) { + j++; + } + continue; + } + int off = sig_vk->at(i)._offset; + assert(off > 0, "offset in object should be positive"); + VMRegPair pair = regs->at(j); + address loc = reg_map.location(pair.first()); + switch(bt) { + case T_BOOLEAN: + *(jboolean*)loc = res->bool_field(off); + break; + case T_CHAR: + *(jchar*)loc = res->char_field(off); + break; + case T_BYTE: + *(jbyte*)loc = res->byte_field(off); + break; + case T_SHORT: + *(jshort*)loc = res->short_field(off); + break; + case T_INT: { + *(jint*)loc = res->int_field(off); + break; + } + case T_LONG: +#ifdef _LP64 + *(intptr_t*)loc = res->long_field(off); +#else + Unimplemented(); +#endif + break; + case T_OBJECT: + case T_ARRAY: { + *(oop*)loc = res->obj_field(off); + break; + } + case T_FLOAT: + *(jfloat*)loc = res->float_field(off); + break; + case T_DOUBLE: + *(jdouble*)loc = res->double_field(off); + break; + default: + ShouldNotReachHere(); + } + j++; + } + assert(j == regs->length(), "missed a field?"); + +#ifdef ASSERT + VMRegPair pair = regs->at(0); + address loc = reg_map.location(pair.first()); + assert(*(oopDesc**)loc == res, "overwritten object"); +#endif + + thread->set_vm_result(res); +} +JRT_END + +// We've returned to an interpreted method, the interpreter needs a +// reference to a value type instance. Allocate it and initialize it +// from field's values in registers. +JRT_BLOCK_ENTRY(void, SharedRuntime::store_value_type_fields_to_buf(JavaThread* thread, intptr_t res)) +{ + ResourceMark rm; + RegisterMap reg_map(thread); + frame stubFrame = thread->last_frame(); + frame callerFrame = stubFrame.sender(®_map); + +#ifdef ASSERT + ValueKlass* verif_vk = ValueKlass::returned_value_klass(reg_map); +#endif + + if (!is_set_nth_bit(res, 0)) { + // We're not returning with value type fields in registers (the + // calling convention didn't allow it for this value klass) + assert(!Metaspace::contains((void*)res), "should be oop or pointer in buffer area"); + thread->set_vm_result((oopDesc*)res); + assert(verif_vk == NULL, "broken calling convention"); + return; + } + + clear_nth_bit(res, 0); + ValueKlass* vk = (ValueKlass*)res; + assert(verif_vk == vk, "broken calling convention"); + assert(Metaspace::contains((void*)res), "should be klass"); + + // Allocate handles for every oop field so they are safe in case of + // a safepoint when allocating + GrowableArray handles; + vk->save_oop_fields(reg_map, handles); + + // It's unsafe to safepoint until we are here + JRT_BLOCK; + { + Thread* THREAD = thread; + oop vt = vk->realloc_result(reg_map, handles, CHECK); + thread->set_vm_result(vt); + } + JRT_BLOCK_END; +} +JRT_END +