--- old/src/hotspot/share/code/nmethod.cpp 2019-03-28 11:22:22.000000000 -0700 +++ new/src/hotspot/share/code/nmethod.cpp 2019-03-28 11:22:22.000000000 -0700 @@ -65,6 +65,7 @@ #include "utilities/xmlstream.hpp" #if INCLUDE_JVMCI #include "jvmci/jvmciJavaClasses.hpp" +#include "jvmci/jvmciRuntime.hpp" #endif #ifdef DTRACE_ENABLED @@ -112,6 +113,10 @@ int dependencies_size; int handler_table_size; int nul_chk_table_size; +#if INCLUDE_JVMCI + int speculations_size; + int jvmci_data_size; +#endif int oops_size; int metadata_size; @@ -129,6 +134,10 @@ dependencies_size += nm->dependencies_size(); handler_table_size += nm->handler_table_size(); nul_chk_table_size += nm->nul_chk_table_size(); +#if INCLUDE_JVMCI + speculations_size += nm->speculations_size(); + jvmci_data_size += nm->jvmci_data_size(); +#endif } void print_nmethod_stats(const char* name) { if (nmethod_count == 0) return; @@ -146,6 +155,10 @@ if (dependencies_size != 0) tty->print_cr(" dependencies = %d", dependencies_size); if (handler_table_size != 0) tty->print_cr(" handler table = %d", handler_table_size); if (nul_chk_table_size != 0) tty->print_cr(" nul chk table = %d", nul_chk_table_size); +#if INCLUDE_JVMCI + if (speculations_size != 0) tty->print_cr(" speculations = %d", speculations_size); + if (jvmci_data_size != 0) tty->print_cr(" JVMCI data = %d", jvmci_data_size); +#endif } }; @@ -426,11 +439,6 @@ #if INCLUDE_RTM_OPT _rtm_state = NoRTM; #endif -#if INCLUDE_JVMCI - _jvmci_installed_code = NULL; - _speculation_log = NULL; - _jvmci_installed_code_triggers_invalidation = false; -#endif } nmethod* nmethod::new_native_nmethod(const methodHandle& method, @@ -483,8 +491,11 @@ AbstractCompiler* compiler, int comp_level #if INCLUDE_JVMCI - , jweak installed_code, - jweak speculationLog + , char* speculations, + int speculations_len, + int nmethod_mirror_index, + const char* nmethod_mirror_name, + FailedSpeculation** failed_speculations #endif ) { @@ -493,12 +504,19 @@ // create nmethod nmethod* nm = NULL; { MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); +#if INCLUDE_JVMCI + int jvmci_data_size = !compiler->is_jvmci() ? 0 : JVMCINMethodData::compute_size(nmethod_mirror_name); +#endif int nmethod_size = CodeBlob::allocation_size(code_buffer, sizeof(nmethod)) + adjust_pcs_size(debug_info->pcs_size()) + align_up((int)dependencies->size_in_bytes(), oopSize) + align_up(handler_table->size_in_bytes() , oopSize) + align_up(nul_chk_table->size_in_bytes() , oopSize) +#if INCLUDE_JVMCI + + align_up(speculations_len , oopSize) + + align_up(jvmci_data_size , oopSize) +#endif + align_up(debug_info->data_size() , oopSize); nm = new (nmethod_size, comp_level) @@ -510,12 +528,19 @@ compiler, comp_level #if INCLUDE_JVMCI - , installed_code, - speculationLog + , speculations, + speculations_len, + jvmci_data_size #endif ); if (nm != NULL) { +#if INCLUDE_JVMCI + if (compiler->is_jvmci()) { + // Initialize the JVMCINMethodData object inlined into nm + nm->jvmci_nmethod_data()->initialize(nmethod_mirror_index, nmethod_mirror_name, failed_speculations); + } +#endif // To make dependency checking during class loading fast, record // the nmethod dependencies in the classes it is dependent on. // This allows the dependency checking code to simply walk the @@ -591,7 +616,13 @@ _dependencies_offset = _scopes_pcs_offset; _handler_table_offset = _dependencies_offset; _nul_chk_table_offset = _handler_table_offset; +#if INCLUDE_JVMCI + _speculations_offset = _nul_chk_table_offset; + _jvmci_data_offset = _speculations_offset; + _nmethod_end_offset = _jvmci_data_offset; +#else _nmethod_end_offset = _nul_chk_table_offset; +#endif _compile_id = compile_id; _comp_level = CompLevel_none; _entry_point = code_begin() + offsets->value(CodeOffsets::Entry); @@ -667,8 +698,9 @@ AbstractCompiler* compiler, int comp_level #if INCLUDE_JVMCI - , jweak installed_code, - jweak speculation_log + , char* speculations, + int speculations_len, + int jvmci_data_size #endif ) : CompiledMethod(method, "nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false), @@ -697,15 +729,6 @@ set_ctable_begin(header_begin() + _consts_offset); #if INCLUDE_JVMCI - _jvmci_installed_code = installed_code; - _speculation_log = speculation_log; - oop obj = JNIHandles::resolve(installed_code); - if (obj == NULL || (obj->is_a(HotSpotNmethod::klass()) && HotSpotNmethod::isDefault(obj))) { - _jvmci_installed_code_triggers_invalidation = false; - } else { - _jvmci_installed_code_triggers_invalidation = true; - } - if (compiler->is_jvmci()) { // JVMCI might not produce any stub sections if (offsets->value(CodeOffsets::Exceptions) != -1) { @@ -735,10 +758,10 @@ _deopt_mh_handler_begin = (address) this + _stub_offset + offsets->value(CodeOffsets::DeoptMH); } else { _deopt_mh_handler_begin = NULL; + } #if INCLUDE_JVMCI } #endif - } if (offsets->value(CodeOffsets::UnwindHandler) != -1) { _unwind_handler_offset = code_offset() + offsets->value(CodeOffsets::UnwindHandler); } else { @@ -753,7 +776,13 @@ _dependencies_offset = _scopes_pcs_offset + adjust_pcs_size(debug_info->pcs_size()); _handler_table_offset = _dependencies_offset + align_up((int)dependencies->size_in_bytes (), oopSize); _nul_chk_table_offset = _handler_table_offset + align_up(handler_table->size_in_bytes(), oopSize); +#if INCLUDE_JVMCI + _speculations_offset = _nul_chk_table_offset + align_up(nul_chk_table->size_in_bytes(), oopSize); + _jvmci_data_offset = _speculations_offset + align_up(speculations_len, oopSize); + _nmethod_end_offset = _jvmci_data_offset + align_up(jvmci_data_size, oopSize); +#else _nmethod_end_offset = _nul_chk_table_offset + align_up(nul_chk_table->size_in_bytes(), oopSize); +#endif _entry_point = code_begin() + offsets->value(CodeOffsets::Entry); _verified_entry_point = code_begin() + offsets->value(CodeOffsets::Verified_Entry); _osr_entry_point = code_begin() + offsets->value(CodeOffsets::OSR_Entry); @@ -779,6 +808,13 @@ handler_table->copy_to(this); nul_chk_table->copy_to(this); +#if INCLUDE_JVMCI + // Copy speculations to nmethod + if (speculations_size() != 0) { + memcpy(speculations_begin(), speculations, speculations_len); + } +#endif + // we use the information of entry points to find out if a method is // static or non static assert(compiler->is_c2() || compiler->is_jvmci() || @@ -798,13 +834,14 @@ log->print(" level='%d'", comp_level()); } #if INCLUDE_JVMCI - char buffer[O_BUFLEN]; - char* jvmci_name = jvmci_installed_code_name(buffer, O_BUFLEN); + if (jvmci_nmethod_data() != NULL) { + const char* jvmci_name = jvmci_nmethod_data()->name(); if (jvmci_name != NULL) { - log->print(" jvmci_installed_code_name='"); + log->print(" jvmci_mirror_name='"); log->text("%s", jvmci_name); log->print("'"); } + } #endif } @@ -1114,13 +1151,6 @@ // Log the unloading. log_state_change(); -#if INCLUDE_JVMCI - // The method can only be unloaded after the pointer to the installed code - // Java wrapper is no longer alive. Here we need to clear out this weak - // reference to the dead object. - maybe_invalidate_installed_code(); -#endif - // The Method* is gone at this point assert(_method == NULL, "Tautology"); @@ -1133,6 +1163,15 @@ // concurrent nmethod unloading. Therefore, there is no need for // acquire on the loader side. OrderAccess::release_store(&_state, (signed char)unloaded); + +#if INCLUDE_JVMCI + // Clear the link between this nmethod and a HotSpotNmethod mirror + JVMCINMethodData* nmethod_data = jvmci_nmethod_data(); + if (nmethod_data != NULL) { + nmethod_data->invalidate_nmethod_mirror(this); + nmethod_data->clear_nmethod_mirror(this); + } +#endif } void nmethod::invalidate_osr_method() { @@ -1264,13 +1303,18 @@ // Log the transition once log_state_change(); - // Invalidate while holding the patching lock - JVMCI_ONLY(maybe_invalidate_installed_code()); - // Remove nmethod from method. unlink_from_method(false /* already owns Patching_lock */); } // leave critical region under Patching_lock +#if INCLUDE_JVMCI + // Invalidate can't occur while holding the Patching lock + JVMCINMethodData* nmethod_data = jvmci_nmethod_data(); + if (nmethod_data != NULL) { + nmethod_data->invalidate_nmethod_mirror(this); + } +#endif + #ifdef ASSERT if (is_osr_method() && method() != NULL) { // Make sure osr nmethod is invalidated, i.e. not on the list @@ -1295,6 +1339,14 @@ flush_dependencies(/*delete_immediately*/true); } +#if INCLUDE_JVMCI + // Now that the nmethod has been unregistered, it's + // safe to clear the HotSpotNmethod mirror oop. + if (nmethod_data != NULL) { + nmethod_data->clear_nmethod_mirror(this); + } +#endif + // Clear ICStubs to prevent back patching stubs of zombie or flushed // nmethods during the next safepoint (see ICStub::finalize), as well // as to free up CompiledICHolder resources. @@ -1322,7 +1374,7 @@ assert(state == not_entrant, "other cases may need to be handled differently"); } - if (TraceCreateZombies) { + if (TraceCreateZombies && state == zombie) { ResourceMark m; tty->print_cr("nmethod <" INTPTR_FORMAT "> %s code made %s", p2i(this), this->method() ? this->method()->name_and_sig_as_C_string() : "null", (state == not_entrant) ? "not entrant" : "zombie"); } @@ -1360,11 +1412,6 @@ ec = next; } -#if INCLUDE_JVMCI - assert(_jvmci_installed_code == NULL, "should have been nulled out when transitioned to zombie"); - assert(_speculation_log == NULL, "should have been nulled out when transitioned to zombie"); -#endif - Universe::heap()->flush_nmethod(this); CodeBlob::flush(); @@ -1658,17 +1705,6 @@ if (is_unloading()) { make_unloaded(); } else { -#if INCLUDE_JVMCI - if (_jvmci_installed_code != NULL) { - if (JNIHandles::is_global_weak_cleared(_jvmci_installed_code)) { - if (_jvmci_installed_code_triggers_invalidation) { - make_not_entrant(); - } - clear_jvmci_installed_code(); - } - } -#endif - guarantee(unload_nmethod_caches(unloading_occurred), "Should not need transition stubs"); } @@ -2090,7 +2126,7 @@ if (cm->is_aot()) return; // FIXME: Revisit once _lock_count is added to aot_method nmethod* nm = cm->as_nmethod(); Atomic::inc(&nm->_lock_count); - assert(zombie_ok || !nm->is_zombie(), "cannot lock a zombie method"); + assert(zombie_ok || !nm->is_zombie(), "cannot lock a zombie method: %p", nm); } void nmethodLocker::unlock_nmethod(CompiledMethod* cm) { @@ -2299,6 +2335,16 @@ p2i(nul_chk_table_begin()), p2i(nul_chk_table_end()), nul_chk_table_size()); +#if INCLUDE_JVMCI + if (speculations_size () > 0) tty->print_cr(" speculations [" INTPTR_FORMAT "," INTPTR_FORMAT "] = %d", + p2i(speculations_begin()), + p2i(speculations_end()), + speculations_size()); + if (jvmci_data_size () > 0) tty->print_cr(" JVMCI data [" INTPTR_FORMAT "," INTPTR_FORMAT "] = %d", + p2i(jvmci_data_begin()), + p2i(jvmci_data_end()), + jvmci_data_size()); +#endif } #ifndef PRODUCT @@ -2881,115 +2927,18 @@ #endif // !PRODUCT #if INCLUDE_JVMCI -void nmethod::clear_jvmci_installed_code() { - assert_locked_or_safepoint(Patching_lock); - if (_jvmci_installed_code != NULL) { - JNIHandles::destroy_weak_global(_jvmci_installed_code); - _jvmci_installed_code = NULL; - } -} - -void nmethod::clear_speculation_log() { - assert_locked_or_safepoint(Patching_lock); - if (_speculation_log != NULL) { - JNIHandles::destroy_weak_global(_speculation_log); - _speculation_log = NULL; - } -} - -void nmethod::maybe_invalidate_installed_code() { - if (!is_compiled_by_jvmci()) { - return; - } - - assert(Patching_lock->is_locked() || - SafepointSynchronize::is_at_safepoint(), "should be performed under a lock for consistency"); - oop installed_code = JNIHandles::resolve(_jvmci_installed_code); - if (installed_code != NULL) { - // Update the values in the InstalledCode instance if it still refers to this nmethod - nmethod* nm = (nmethod*)InstalledCode::address(installed_code); - if (nm == this) { - if (!is_alive() || is_unloading()) { - // Break the link between nmethod and InstalledCode such that the nmethod - // can subsequently be flushed safely. The link must be maintained while - // the method could have live activations since invalidateInstalledCode - // might want to invalidate all existing activations. - InstalledCode::set_address(installed_code, 0); - InstalledCode::set_entryPoint(installed_code, 0); - } else if (is_not_entrant()) { - // Remove the entry point so any invocation will fail but keep - // the address link around that so that existing activations can - // be invalidated. - InstalledCode::set_entryPoint(installed_code, 0); - } - } - } - if (!is_alive() || is_unloading()) { - // Clear these out after the nmethod has been unregistered and any - // updates to the InstalledCode instance have been performed. - clear_jvmci_installed_code(); - clear_speculation_log(); +void nmethod::update_speculation(JavaThread* thread) { + jlong speculation = thread->pending_failed_speculation(); + if (speculation != 0) { + guarantee(jvmci_nmethod_data() != NULL, "failed speculation in nmethod without failed speculation list"); + jvmci_nmethod_data()->add_failed_speculation(this, speculation); + thread->set_pending_failed_speculation(0); } } -void nmethod::invalidate_installed_code(Handle installedCode, TRAPS) { - if (installedCode() == NULL) { - THROW(vmSymbols::java_lang_NullPointerException()); - } - jlong nativeMethod = InstalledCode::address(installedCode); - nmethod* nm = (nmethod*)nativeMethod; - if (nm == NULL) { - // Nothing to do - return; - } - - nmethodLocker nml(nm); -#ifdef ASSERT - { - MutexLockerEx pl(Patching_lock, Mutex::_no_safepoint_check_flag); - // This relationship can only be checked safely under a lock - assert(!nm->is_alive() || nm->is_unloading() || nm->jvmci_installed_code() == installedCode(), "sanity check"); - } -#endif - - if (nm->is_alive()) { - // Invalidating the InstalledCode means we want the nmethod - // to be deoptimized. - nm->mark_for_deoptimization(); - VM_Deoptimize op; - VMThread::execute(&op); - } - - // Multiple threads could reach this point so we now need to - // lock and re-check the link to the nmethod so that only one - // thread clears it. - MutexLockerEx pl(Patching_lock, Mutex::_no_safepoint_check_flag); - if (InstalledCode::address(installedCode) == nativeMethod) { - InstalledCode::set_address(installedCode, 0); - } -} - -oop nmethod::jvmci_installed_code() { - return JNIHandles::resolve(_jvmci_installed_code); -} - -oop nmethod::speculation_log() { - return JNIHandles::resolve(_speculation_log); -} - -char* nmethod::jvmci_installed_code_name(char* buf, size_t buflen) const { - if (!this->is_compiled_by_jvmci()) { - return NULL; - } - oop installed_code = JNIHandles::resolve(_jvmci_installed_code); - if (installed_code != NULL) { - oop installed_code_name = NULL; - if (installed_code->is_a(InstalledCode::klass())) { - installed_code_name = InstalledCode::name(installed_code); - } - if (installed_code_name != NULL) { - return java_lang_String::as_utf8_string(installed_code_name, buf, (int)buflen); - } +const char* nmethod::jvmci_name() { + if (jvmci_nmethod_data() != NULL) { + return jvmci_nmethod_data()->name(); } return NULL; }