diff --git a/src/hotspot/share/code/codeBehaviours.cpp b/src/hotspot/share/code/codeBehaviours.cpp index 91ac836..c0605c6 100644 --- a/src/hotspot/share/code/codeBehaviours.cpp +++ b/src/hotspot/share/code/codeBehaviours.cpp @@ -30,10 +30,10 @@ CompiledICProtectionBehaviour* CompiledICProtectionBehaviour::_current = NULL; bool DefaultICProtectionBehaviour::lock(CompiledMethod* method) { - if (CompiledIC_lock->owned_by_self()) { + if (is_safe(method)) { return false; } - CompiledIC_lock->lock(); + CompiledIC_lock->lock_without_safepoint_check(); return true; } diff --git a/src/hotspot/share/code/compiledIC.cpp b/src/hotspot/share/code/compiledIC.cpp index c5f15b3..794e21a 100644 --- a/src/hotspot/share/code/compiledIC.cpp +++ b/src/hotspot/share/code/compiledIC.cpp @@ -259,7 +259,10 @@ bool CompiledIC::set_to_megamorphic(CallInfo* call_info, Bytecodes::Code bytecod CompiledICHolder* holder = new CompiledICHolder(call_info->resolved_method()->method_holder(), call_info->resolved_klass(), false); holder->claim(); - InlineCacheBuffer::create_transition_stub(this, holder, entry); + if (!InlineCacheBuffer::create_transition_stub(this, holder, entry)) { + delete holder; + return false; + } } else { assert(call_info->call_kind() == CallInfo::vtable_call, "either itable or vtable"); // Can be different than selected_method->vtable_index(), due to package-private etc. @@ -269,7 +272,9 @@ bool CompiledIC::set_to_megamorphic(CallInfo* call_info, Bytecodes::Code bytecod if (entry == NULL) { return false; } - InlineCacheBuffer::create_transition_stub(this, NULL, entry); + if (!InlineCacheBuffer::create_transition_stub(this, NULL, entry)) { + return false; + } } if (TraceICs) { @@ -350,7 +355,7 @@ bool CompiledIC::is_call_to_interpreted() const { return is_call_to_interpreted; } -void CompiledIC::set_to_clean(bool in_use) { +bool CompiledIC::set_to_clean(bool in_use) { assert(CompiledICLocker::is_safe(_method), "mt unsafe call"); if (TraceInlineCacheClearing || TraceICs) { tty->print_cr("IC@" INTPTR_FORMAT ": set to clean", p2i(instruction_address())); @@ -373,7 +378,9 @@ void CompiledIC::set_to_clean(bool in_use) { } } else { // Unsafe transition - create stub. - InlineCacheBuffer::create_transition_stub(this, NULL, entry); + if (!InlineCacheBuffer::create_transition_stub(this, NULL, entry)) { + return false; + } } // We can't check this anymore. With lazy deopt we could have already // cleaned this IC entry before we even return. This is possible if @@ -382,6 +389,7 @@ void CompiledIC::set_to_clean(bool in_use) { // race because the IC entry was complete when we safepointed so // cleaning it immediately is harmless. // assert(is_clean(), "sanity check"); + return true; } bool CompiledIC::is_clean() const { @@ -393,7 +401,7 @@ bool CompiledIC::is_clean() const { return is_clean; } -void CompiledIC::set_to_monomorphic(CompiledICInfo& info) { +bool CompiledIC::set_to_monomorphic(CompiledICInfo& info) { assert(CompiledICLocker::is_safe(_method), "mt unsafe call"); // Updating a cache to the wrong entry can cause bugs that are very hard // to track down - if cache entry gets invalid - we just clean it. In @@ -430,7 +438,11 @@ void CompiledIC::set_to_monomorphic(CompiledICInfo& info) { } } else { // Call via method-klass-holder - InlineCacheBuffer::create_transition_stub(this, info.claim_cached_icholder(), info.entry()); + CompiledICHolder* holder = info.claim_cached_icholder(); + if (!InlineCacheBuffer::create_transition_stub(this, holder, info.entry())) { + delete holder; + return false; + } if (TraceICs) { ResourceMark rm(thread); tty->print_cr ("IC@" INTPTR_FORMAT ": monomorphic to interpreter via icholder ", p2i(instruction_address())); @@ -450,7 +462,9 @@ void CompiledIC::set_to_monomorphic(CompiledICInfo& info) { (!is_in_transition_state() && (info.is_optimized() || static_bound || is_clean())); if (!safe) { - InlineCacheBuffer::create_transition_stub(this, info.cached_metadata(), info.entry()); + if (!InlineCacheBuffer::create_transition_stub(this, info.cached_metadata(), info.entry())) { + return false; + } } else { if (is_optimized()) { set_ic_destination(info.entry()); @@ -475,6 +489,7 @@ void CompiledIC::set_to_monomorphic(CompiledICInfo& info) { // race because the IC entry was complete when we safepointed so // cleaning it immediately is harmless. // assert(is_call_to_compiled() || is_call_to_interpreted(), "sanity check"); + return true; } @@ -575,7 +590,7 @@ void CompiledIC::cleanup_call_site(virtual_call_Relocation* call_site, const Com // ---------------------------------------------------------------------------- -void CompiledStaticCall::set_to_clean(bool in_use) { +bool CompiledStaticCall::set_to_clean(bool in_use) { // in_use is unused but needed to match template function in CompiledMethod assert(CompiledICLocker::is_safe(instruction_address()), "mt unsafe call"); // Reset call site @@ -585,6 +600,7 @@ void CompiledStaticCall::set_to_clean(bool in_use) { // Do not reset stub here: It is too expensive to call find_stub. // Instead, rely on caller (nmethod::clear_inline_caches) to clear // both the call and its stub. + return true; } bool CompiledStaticCall::is_clean() const { diff --git a/src/hotspot/share/code/compiledIC.hpp b/src/hotspot/share/code/compiledIC.hpp index 3d5da1d..08b2a6e 100644 --- a/src/hotspot/share/code/compiledIC.hpp +++ b/src/hotspot/share/code/compiledIC.hpp @@ -272,8 +272,8 @@ class CompiledIC: public ResourceObj { // // They all takes a TRAP argument, since they can cause a GC if the inline-cache buffer is full. // - void set_to_clean(bool in_use = true); - void set_to_monomorphic(CompiledICInfo& info); + bool set_to_clean(bool in_use = true); + bool set_to_monomorphic(CompiledICInfo& info); void clear_ic_stub(); // Returns true if successful and false otherwise. The call can fail if memory @@ -372,7 +372,7 @@ public: virtual address destination() const = 0; // Clean static call (will force resolving on next use) - void set_to_clean(bool in_use = true); + bool set_to_clean(bool in_use = true); // Set state. The entry must be the same, as computed by compute_entry. // Computation and setting is split up, since the actions are separate during diff --git a/src/hotspot/share/code/compiledMethod.cpp b/src/hotspot/share/code/compiledMethod.cpp index ff22aec..9aed89f 100644 --- a/src/hotspot/share/code/compiledMethod.cpp +++ b/src/hotspot/share/code/compiledMethod.cpp @@ -27,6 +27,7 @@ #include "code/compiledMethod.inline.hpp" #include "code/scopeDesc.hpp" #include "code/codeCache.hpp" +#include "code/icBuffer.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/gcBehaviours.hpp" #include "interpreter/bytecode.inline.hpp" @@ -430,27 +431,27 @@ static void check_class(Metadata* md) { #endif // ASSERT -void CompiledMethod::clean_ic_if_metadata_is_dead(CompiledIC *ic) { +bool CompiledMethod::clean_ic_if_metadata_is_dead(CompiledIC *ic) { if (ic->is_icholder_call()) { // The only exception is compiledICHolder metdata which may // yet be marked below. (We check this further below). CompiledICHolder* cichk_metdata = ic->cached_icholder(); if (cichk_metdata->is_loader_alive()) { - return; + return true; } } else { Metadata* ic_metdata = ic->cached_metadata(); if (ic_metdata != NULL) { if (ic_metdata->is_klass()) { if (((Klass*)ic_metdata)->is_loader_alive()) { - return; + return true; } } else if (ic_metdata->is_method()) { Method* method = (Method*)ic_metdata; assert(!method->is_old(), "old method should have been cleaned"); if (method->method_holder()->is_loader_alive()) { - return; + return true; } } else { ShouldNotReachHere(); @@ -458,7 +459,10 @@ void CompiledMethod::clean_ic_if_metadata_is_dead(CompiledIC *ic) { } } - ic->set_to_clean(); + if (ic->is_clean()) { + return true; + } + return ic->set_to_clean(); } // static_stub_Relocations may have dangling references to @@ -496,7 +500,7 @@ void CompiledMethod::clean_ic_stubs() { // Clean references to unloaded nmethods at addr from this one, which is not unloaded. template -static void clean_if_nmethod_is_unloaded(CompiledICorStaticCall *ic, address addr, CompiledMethod* from, +static bool clean_if_nmethod_is_unloaded(CompiledICorStaticCall *ic, address addr, CompiledMethod* from, bool clean_all) { // Ok, to lookup references to zombies here CodeBlob *cb = CodeCache::find_blob_unsafe(addr); @@ -504,20 +508,23 @@ static void clean_if_nmethod_is_unloaded(CompiledICorStaticCall *ic, address add if (nm != NULL) { // Clean inline caches pointing to both zombie and not_entrant methods if (clean_all || !nm->is_in_use() || nm->is_unloading() || (nm->method()->code() != nm)) { - ic->set_to_clean(from->is_alive()); + if (!ic->set_to_clean(from->is_alive())) { + return false; + } assert(ic->is_clean(), "nmethod " PTR_FORMAT "not clean %s", p2i(from), from->method()->name_and_sig_as_C_string()); } } + return true; } -static void clean_if_nmethod_is_unloaded(CompiledIC *ic, CompiledMethod* from, +static bool clean_if_nmethod_is_unloaded(CompiledIC *ic, CompiledMethod* from, bool clean_all) { - clean_if_nmethod_is_unloaded(ic, ic->ic_destination(), from, clean_all); + return clean_if_nmethod_is_unloaded(ic, ic->ic_destination(), from, clean_all); } -static void clean_if_nmethod_is_unloaded(CompiledStaticCall *csc, CompiledMethod* from, +static bool clean_if_nmethod_is_unloaded(CompiledStaticCall *csc, CompiledMethod* from, bool clean_all) { - clean_if_nmethod_is_unloaded(csc, csc->destination(), from, clean_all); + return clean_if_nmethod_is_unloaded(csc, csc->destination(), from, clean_all); } // Cleans caches in nmethods that point to either classes that are unloaded @@ -527,7 +534,7 @@ static void clean_if_nmethod_is_unloaded(CompiledStaticCall *csc, CompiledMethod // nmethods are unloaded. Return postponed=true in the parallel case for // inline caches found that point to nmethods that are not yet visited during // the do_unloading walk. -void CompiledMethod::unload_nmethod_caches(bool unloading_occurred) { +bool CompiledMethod::unload_nmethod_caches(bool unloading_occurred) { ResourceMark rm; // Exception cache only needs to be called if unloading occurred @@ -535,18 +542,32 @@ void CompiledMethod::unload_nmethod_caches(bool unloading_occurred) { clean_exception_cache(); } - cleanup_inline_caches_impl(unloading_occurred, false); + if (!cleanup_inline_caches_impl(unloading_occurred, false)) { + return false; + } // All static stubs need to be cleaned. clean_ic_stubs(); // Check that the metadata embedded in the nmethod is alive DEBUG_ONLY(metadata_do(check_class)); + return true; +} + +void CompiledMethod::cleanup_inline_caches(bool clean_all) { + for (;;) { + { CompiledICLocker ic_locker(this); + if (cleanup_inline_caches_impl(false, clean_all)) { + return; + } + } + InlineCacheBuffer::refill_ic_stubs(); + } } // Called to clean up after class unloading for live nmethods and from the sweeper // for all methods. -void CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all) { +bool CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all) { assert(CompiledICLocker::is_safe(this), "mt unsafe call"); ResourceMark rm; @@ -561,30 +582,34 @@ void CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool cl if (unloading_occurred) { // If class unloading occurred we first clear ICs where the cached metadata // is referring to an unloaded klass or method. - clean_ic_if_metadata_is_dead(CompiledIC_at(&iter)); + if (!clean_ic_if_metadata_is_dead(CompiledIC_at(&iter))) { + return false; + } } - clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), this, clean_all); + if (!clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), this, clean_all)) { + return false; + } break; case relocInfo::opt_virtual_call_type: - clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), this, clean_all); + if (!clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), this, clean_all)) { + return false; + } break; case relocInfo::static_call_type: - clean_if_nmethod_is_unloaded(compiledStaticCall_at(iter.reloc()), this, clean_all); - break; - - case relocInfo::oop_type: + if (!clean_if_nmethod_is_unloaded(compiledStaticCall_at(iter.reloc()), this, clean_all)) { + return false; + } break; - case relocInfo::metadata_type: - break; // nothing to do. - default: break; } } + + return true; } // Iterating over all nmethods, e.g. with the help of CodeCache::nmethods_do(fun) was found diff --git a/src/hotspot/share/code/compiledMethod.hpp b/src/hotspot/share/code/compiledMethod.hpp index 4a8cfa5..29950a6 100644 --- a/src/hotspot/share/code/compiledMethod.hpp +++ b/src/hotspot/share/code/compiledMethod.hpp @@ -352,12 +352,11 @@ public: // Inline cache support for class unloading and nmethod unloading private: - void cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all); + bool cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all); + public: - void cleanup_inline_caches(bool clean_all) { - // Serial version used by sweeper and whitebox test - cleanup_inline_caches_impl(false, clean_all); - } + // Serial version used by sweeper and whitebox test + void cleanup_inline_caches(bool clean_all); virtual void clear_inline_caches(); void clear_ic_stubs(); @@ -390,7 +389,7 @@ public: address oops_reloc_begin() const; private: - void static clean_ic_if_metadata_is_dead(CompiledIC *ic); + bool static clean_ic_if_metadata_is_dead(CompiledIC *ic); void clean_ic_stubs(); @@ -400,7 +399,7 @@ public: virtual bool is_unloading() = 0; - void unload_nmethod_caches(bool class_unloading_occurred); + bool unload_nmethod_caches(bool class_unloading_occurred); virtual void do_unloading(bool unloading_occurred) { } private: diff --git a/src/hotspot/share/code/icBuffer.cpp b/src/hotspot/share/code/icBuffer.cpp index a26ec16..341ca85 100644 --- a/src/hotspot/share/code/icBuffer.cpp +++ b/src/hotspot/share/code/icBuffer.cpp @@ -42,7 +42,6 @@ DEF_STUB_INTERFACE(ICStub); StubQueue* InlineCacheBuffer::_buffer = NULL; -ICStub* InlineCacheBuffer::_next_stub = NULL; CompiledICHolder* InlineCacheBuffer::_pending_released = NULL; int InlineCacheBuffer::_pending_count = 0; @@ -103,52 +102,42 @@ void ICStub::print() { //----------------------------------------------------------------------------------------------- // Implementation of InlineCacheBuffer -void InlineCacheBuffer::init_next_stub() { - ICStub* ic_stub = (ICStub*)buffer()->request_committed (ic_stub_code_size()); - assert (ic_stub != NULL, "no room for a single stub"); - set_next_stub(ic_stub); -} void InlineCacheBuffer::initialize() { if (_buffer != NULL) return; // already initialized _buffer = new StubQueue(new ICStubInterface, 10*K, InlineCacheBuffer_lock, "InlineCacheBuffer"); assert (_buffer != NULL, "cannot allocate InlineCacheBuffer"); - init_next_stub(); } ICStub* InlineCacheBuffer::new_ic_stub() { - while (true) { - ICStub* ic_stub = (ICStub*)buffer()->request_committed(ic_stub_code_size()); - if (ic_stub != NULL) { - return ic_stub; - } - // we ran out of inline cache buffer space; must enter safepoint. - // We do this by forcing a safepoint - EXCEPTION_MARK; - - VM_ICBufferFull ibf; - VMThread::execute(&ibf); - // We could potential get an async. exception at this point. - // In that case we will rethrow it to ourselvs. - if (HAS_PENDING_EXCEPTION) { - oop exception = PENDING_EXCEPTION; - CLEAR_PENDING_EXCEPTION; - Thread::send_async_exception(JavaThread::current()->threadObj(), exception); - } + return (ICStub*)buffer()->request_committed(ic_stub_code_size()); +} + + +void InlineCacheBuffer::refill_ic_stubs() { + // we ran out of inline cache buffer space; must enter safepoint. + // We do this by forcing a safepoint + EXCEPTION_MARK; + + VM_ICBufferFull ibf; + VMThread::execute(&ibf); + // We could potential get an async. exception at this point. + // In that case we will rethrow it to ourselvs. + if (HAS_PENDING_EXCEPTION) { + oop exception = PENDING_EXCEPTION; + CLEAR_PENDING_EXCEPTION; + Thread::send_async_exception(JavaThread::current()->threadObj(), exception); } - ShouldNotReachHere(); - return NULL; } void InlineCacheBuffer::update_inline_caches() { - if (buffer()->number_of_stubs() > 1) { + if (buffer()->number_of_stubs() > 0) { if (TraceICBuffer) { tty->print_cr("[updating inline caches with %d stubs]", buffer()->number_of_stubs()); } buffer()->remove_all(); - init_next_stub(); } release_pending_icholders(); } @@ -160,7 +149,7 @@ bool InlineCacheBuffer::contains(address instruction_address) { bool InlineCacheBuffer::is_empty() { - return buffer()->number_of_stubs() == 1; // always has sentinel + return buffer()->number_of_stubs() == 0; } @@ -169,8 +158,7 @@ void InlineCacheBuffer_init() { } -void InlineCacheBuffer::create_transition_stub(CompiledIC *ic, void* cached_value, address entry) { - MutexLockerEx ml(CompiledIC_lock->owned_by_self() ? NULL : CompiledIC_lock); +bool InlineCacheBuffer::create_transition_stub(CompiledIC *ic, void* cached_value, address entry) { assert(!SafepointSynchronize::is_at_safepoint(), "should not be called during a safepoint"); assert(CompiledICLocker::is_safe(ic->instruction_address()), "mt unsafe call"); if (TraceICBuffer) { @@ -178,20 +166,23 @@ void InlineCacheBuffer::create_transition_stub(CompiledIC *ic, void* cached_valu p2i(ic->instruction_address()), p2i(entry), p2i(cached_value)); } + // allocate and initialize new "out-of-line" inline-cache + ICStub* ic_stub = new_ic_stub(); + if (ic_stub == NULL) { + return false; + } + // If an transition stub is already associate with the inline cache, then we remove the association. if (ic->is_in_transition_state()) { ICStub* old_stub = ICStub_from_destination_address(ic->stub_address()); old_stub->clear(); } - // allocate and initialize new "out-of-line" inline-cache - ICStub* ic_stub = get_next_stub(); ic_stub->set_stub(ic, cached_value, entry); // Update inline cache in nmethod to point to new "out-of-line" allocated inline cache ic->set_ic_destination(ic_stub); - - set_next_stub(new_ic_stub()); // can cause safepoint synchronization + return true; } @@ -225,9 +216,7 @@ void InlineCacheBuffer::release_pending_icholders() { // not safe to free them until them since they might be visible to // another thread. void InlineCacheBuffer::queue_for_release(CompiledICHolder* icholder) { - MutexLockerEx mex1((CompiledIC_lock->owned_by_self() || - SafepointSynchronize::is_at_safepoint()) ? NULL : CompiledIC_lock); - MutexLockerEx mex2(InlineCacheBuffer_lock); + MutexLockerEx mex(InlineCacheBuffer_lock, Mutex::_no_safepoint_check_flag); icholder->set_next(_pending_released); _pending_released = icholder; _pending_count++; diff --git a/src/hotspot/share/code/icBuffer.hpp b/src/hotspot/share/code/icBuffer.hpp index b60b620..4821e30 100644 --- a/src/hotspot/share/code/icBuffer.hpp +++ b/src/hotspot/share/code/icBuffer.hpp @@ -100,20 +100,14 @@ class InlineCacheBuffer: public AllStatic { static int ic_stub_code_size(); static StubQueue* _buffer; - static ICStub* _next_stub; static CompiledICHolder* _pending_released; static int _pending_count; static StubQueue* buffer() { return _buffer; } - static void set_next_stub(ICStub* next_stub) { _next_stub = next_stub; } - static ICStub* get_next_stub() { return _next_stub; } - - static void init_next_stub(); static ICStub* new_ic_stub(); - // Machine-dependent implementation of ICBuffer static void assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point); static address ic_buffer_entry_point (address code_begin); @@ -129,6 +123,7 @@ class InlineCacheBuffer: public AllStatic { // removes the ICStubs after backpatching static void update_inline_caches(); + static void refill_ic_stubs(); // for debugging static bool is_empty(); @@ -138,7 +133,7 @@ class InlineCacheBuffer: public AllStatic { static int pending_icholder_count() { return _pending_count; } // New interface - static void create_transition_stub(CompiledIC *ic, void* cached_value, address entry); + static bool create_transition_stub(CompiledIC *ic, void* cached_value, address entry); static address ic_destination_for(CompiledIC *ic); static void* cached_value_for(CompiledIC *ic); }; diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index 2719946..05f53a8 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -1624,7 +1624,8 @@ void nmethod::do_unloading(bool unloading_occurred) { } #endif - unload_nmethod_caches(unloading_occurred); + guarantee(unload_nmethod_caches(unloading_occurred), + "Should not need transition stubs"); } } diff --git a/src/hotspot/share/code/relocInfo.cpp b/src/hotspot/share/code/relocInfo.cpp index 0654df9..08afb8f 100644 --- a/src/hotspot/share/code/relocInfo.cpp +++ b/src/hotspot/share/code/relocInfo.cpp @@ -644,12 +644,12 @@ Method* virtual_call_Relocation::method_value() { return (Method*)m; } -void virtual_call_Relocation::clear_inline_cache() { +bool virtual_call_Relocation::clear_inline_cache() { // No stubs for ICs // Clean IC ResourceMark rm; CompiledIC* icache = CompiledIC_at(this); - icache->set_to_clean(); + return icache->set_to_clean(); } @@ -672,12 +672,14 @@ Method* opt_virtual_call_Relocation::method_value() { return (Method*)m; } -void opt_virtual_call_Relocation::clear_inline_cache() { +bool opt_virtual_call_Relocation::clear_inline_cache() { // No stubs for ICs // Clean IC ResourceMark rm; CompiledIC* icache = CompiledIC_at(this); - icache->set_to_clean(); + guarantee(icache->set_to_clean(), + "Should not need transition stubs"); + return true; } @@ -715,10 +717,12 @@ void static_call_Relocation::unpack_data() { _method_index = unpack_1_int(); } -void static_call_Relocation::clear_inline_cache() { +bool static_call_Relocation::clear_inline_cache() { // Safe call site info CompiledStaticCall* handler = this->code()->compiledStaticCall_at(this); - handler->set_to_clean(); + guarantee(handler->set_to_clean(), + "Should not need transition stubs"); + return true; } @@ -757,10 +761,11 @@ address trampoline_stub_Relocation::get_trampoline_for(address call, nmethod* co return NULL; } -void static_stub_Relocation::clear_inline_cache() { +bool static_stub_Relocation::clear_inline_cache() { // Call stub is only used when calling the interpreted code. // It does not really need to be cleared, except that we want to clean out the methodoop. CompiledDirectStaticCall::set_stub_to_clean(this); + return true; } diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp index 57931a1..69afef4 100644 --- a/src/hotspot/share/code/relocInfo.hpp +++ b/src/hotspot/share/code/relocInfo.hpp @@ -814,7 +814,7 @@ class Relocation { // all relocations are able to reassert their values virtual void set_value(address x); - virtual void clear_inline_cache() { } + virtual bool clear_inline_cache() { return true; } // This method assumes that all virtual/static (inline) caches are cleared (since for static_call_type and // ic_call_type is not always posisition dependent (depending on the state of the cache)). However, this is @@ -1052,7 +1052,7 @@ class virtual_call_Relocation : public CallRelocation { void pack_data_to(CodeSection* dest); void unpack_data(); - void clear_inline_cache(); + bool clear_inline_cache(); }; @@ -1083,7 +1083,7 @@ class opt_virtual_call_Relocation : public CallRelocation { void pack_data_to(CodeSection* dest); void unpack_data(); - void clear_inline_cache(); + bool clear_inline_cache(); // find the matching static_stub address static_stub(bool is_aot); @@ -1117,7 +1117,7 @@ class static_call_Relocation : public CallRelocation { void pack_data_to(CodeSection* dest); void unpack_data(); - void clear_inline_cache(); + bool clear_inline_cache(); // find the matching static_stub address static_stub(bool is_aot); @@ -1146,7 +1146,7 @@ class static_stub_Relocation : public Relocation { static_stub_Relocation() { } public: - void clear_inline_cache(); + bool clear_inline_cache(); address static_call() { return _static_call; } bool is_aot() { return _is_aot; } diff --git a/src/hotspot/share/code/stubs.cpp b/src/hotspot/share/code/stubs.cpp index 81717b9..d751f3d 100644 --- a/src/hotspot/share/code/stubs.cpp +++ b/src/hotspot/share/code/stubs.cpp @@ -117,7 +117,7 @@ Stub* StubQueue::request_committed(int code_size) { Stub* StubQueue::request(int requested_code_size) { assert(requested_code_size > 0, "requested_code_size must be > 0"); - if (_mutex != NULL) _mutex->lock(); + if (_mutex != NULL) _mutex->lock_without_safepoint_check(); Stub* s = current_stub(); int requested_size = align_up(stub_code_size_to_size(requested_code_size), CodeEntryAlignment); if (requested_size <= available_space()) { @@ -207,7 +207,7 @@ void StubQueue::remove_all(){ void StubQueue::verify() { // verify only if initialized if (_stub_buffer == NULL) return; - MutexLockerEx lock(_mutex); + MutexLockerEx lock(_mutex, Mutex::_no_safepoint_check_flag); // verify index boundaries guarantee(0 <= _buffer_size, "buffer size must be positive"); guarantee(0 <= _buffer_limit && _buffer_limit <= _buffer_size , "_buffer_limit out of bounds"); @@ -234,9 +234,8 @@ void StubQueue::verify() { void StubQueue::print() { - MutexLockerEx lock(_mutex); + MutexLockerEx lock(_mutex, Mutex::_no_safepoint_check_flag); for (Stub* s = first(); s != NULL; s = next(s)) { stub_print(s); } } - diff --git a/src/hotspot/share/code/vtableStubs.cpp b/src/hotspot/share/code/vtableStubs.cpp index 09a0e05..ae0787e 100644 --- a/src/hotspot/share/code/vtableStubs.cpp +++ b/src/hotspot/share/code/vtableStubs.cpp @@ -124,7 +124,7 @@ int VtableStubs::_itab_stub_size = 0; void VtableStubs::initialize() { VtableStub::_receiver_location = SharedRuntime::name_for_receiver(); { - MutexLocker ml(VtableStubs_lock); + MutexLockerEx ml(VtableStubs_lock, Mutex::_no_safepoint_check_flag); assert(_number_of_vtable_stubs == 0, "potential performance bug: VtableStubs initialized more than once"); assert(is_power_of_2(N), "N must be a power of 2"); for (int i = 0; i < N; i++) { @@ -247,7 +247,7 @@ inline uint VtableStubs::hash(bool is_vtable_stub, int vtable_index){ VtableStub* VtableStubs::lookup(bool is_vtable_stub, int vtable_index) { - MutexLocker ml(VtableStubs_lock); + MutexLockerEx ml(VtableStubs_lock, Mutex::_no_safepoint_check_flag); unsigned hash = VtableStubs::hash(is_vtable_stub, vtable_index); VtableStub* s = _table[hash]; while( s && !s->matches(is_vtable_stub, vtable_index)) s = s->next(); @@ -256,7 +256,7 @@ VtableStub* VtableStubs::lookup(bool is_vtable_stub, int vtable_index) { void VtableStubs::enter(bool is_vtable_stub, int vtable_index, VtableStub* s) { - MutexLocker ml(VtableStubs_lock); + MutexLockerEx ml(VtableStubs_lock, Mutex::_no_safepoint_check_flag); assert(s->matches(is_vtable_stub, vtable_index), "bad vtable stub"); unsigned int h = VtableStubs::hash(is_vtable_stub, vtable_index); // enter s at the beginning of the corresponding list @@ -266,7 +266,7 @@ void VtableStubs::enter(bool is_vtable_stub, int vtable_index, VtableStub* s) { } VtableStub* VtableStubs::entry_point(address pc) { - MutexLocker ml(VtableStubs_lock); + MutexLockerEx ml(VtableStubs_lock, Mutex::_no_safepoint_check_flag); VtableStub* stub = (VtableStub*)(pc - VtableStub::entry_offset()); uint hash = VtableStubs::hash(stub->is_vtable_stub(), stub->index()); VtableStub* s; diff --git a/src/hotspot/share/runtime/mutexLocker.cpp b/src/hotspot/share/runtime/mutexLocker.cpp index 06e1b3d..ea6deca 100644 --- a/src/hotspot/share/runtime/mutexLocker.cpp +++ b/src/hotspot/share/runtime/mutexLocker.cpp @@ -251,7 +251,7 @@ void mutex_init() { def(SystemDictionary_lock , PaddedMonitor, leaf, true, Monitor::_safepoint_check_always); // lookups done by VM thread def(SharedDictionary_lock , PaddedMutex , leaf, true, Monitor::_safepoint_check_always); // lookups done by VM thread def(Module_lock , PaddedMutex , leaf+2, true, Monitor::_safepoint_check_always); - def(InlineCacheBuffer_lock , PaddedMutex , leaf, true, Monitor::_safepoint_check_always); + def(InlineCacheBuffer_lock , PaddedMutex , leaf, true, Monitor::_safepoint_check_never); def(VMStatistic_lock , PaddedMutex , leaf, false, Monitor::_safepoint_check_always); def(ExpandHeap_lock , PaddedMutex , leaf, true, Monitor::_safepoint_check_always); // Used during compilation by VM thread def(JNIHandleBlockFreeList_lock , PaddedMutex , leaf-1, true, Monitor::_safepoint_check_never); // handles are used by VM thread @@ -281,7 +281,7 @@ void mutex_init() { def(VMOperationRequest_lock , PaddedMonitor, nonleaf, true, Monitor::_safepoint_check_sometimes); def(RetData_lock , PaddedMutex , nonleaf, false, Monitor::_safepoint_check_always); def(Terminator_lock , PaddedMonitor, nonleaf, true, Monitor::_safepoint_check_sometimes); - def(VtableStubs_lock , PaddedMutex , nonleaf, true, Monitor::_safepoint_check_always); + def(VtableStubs_lock , PaddedMutex , nonleaf, true, Monitor::_safepoint_check_never); def(Notify_lock , PaddedMonitor, nonleaf, true, Monitor::_safepoint_check_always); def(JNIGlobalAlloc_lock , PaddedMutex , nonleaf, true, Monitor::_safepoint_check_never); def(JNIGlobalActive_lock , PaddedMutex , nonleaf-1, true, Monitor::_safepoint_check_never); @@ -294,7 +294,7 @@ void mutex_init() { def(JfieldIdCreation_lock , PaddedMutex , nonleaf+1, true, Monitor::_safepoint_check_always); // jfieldID, Used in VM_Operation def(ResolvedMethodTable_lock , PaddedMutex , nonleaf+1, false, Monitor::_safepoint_check_always); // Used to protect ResolvedMethodTable - def(CompiledIC_lock , PaddedMutex , nonleaf+2, false, Monitor::_safepoint_check_always); // locks VtableStubs_lock, InlineCacheBuffer_lock + def(CompiledIC_lock , PaddedMutex , nonleaf+2, false, Monitor::_safepoint_check_never); // locks VtableStubs_lock, InlineCacheBuffer_lock def(CompileTaskAlloc_lock , PaddedMutex , nonleaf+2, true, Monitor::_safepoint_check_always); def(CompileStatistics_lock , PaddedMutex , nonleaf+2, false, Monitor::_safepoint_check_always); def(DirectivesStack_lock , PaddedMutex , special, true, Monitor::_safepoint_check_never); diff --git a/src/hotspot/share/runtime/safepoint.cpp b/src/hotspot/share/runtime/safepoint.cpp index e54cf9c..eeb504b 100644 --- a/src/hotspot/share/runtime/safepoint.cpp +++ b/src/hotspot/share/runtime/safepoint.cpp @@ -732,6 +732,7 @@ void SafepointSynchronize::do_cleanup_tasks() { // Finish monitor deflation. ObjectSynchronizer::finish_deflate_idle_monitors(&deflate_counters); + assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer"); } diff --git a/src/hotspot/share/runtime/sharedRuntime.cpp b/src/hotspot/share/runtime/sharedRuntime.cpp index da61a30..279b02e 100644 --- a/src/hotspot/share/runtime/sharedRuntime.cpp +++ b/src/hotspot/share/runtime/sharedRuntime.cpp @@ -25,12 +25,13 @@ #include "precompiled.hpp" #include "jvm.h" #include "aot/aotLoader.hpp" -#include "code/compiledMethod.inline.hpp" #include "classfile/stringTable.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" #include "code/codeCache.hpp" #include "code/compiledIC.hpp" +#include "code/icBuffer.hpp" +#include "code/compiledMethod.inline.hpp" #include "code/scopeDesc.hpp" #include "code/vtableStubs.hpp" #include "compiler/abstractCompiler.hpp" @@ -1315,74 +1316,86 @@ methodHandle SharedRuntime::resolve_sub_helper(JavaThread *thread, // (cached_oop, destination address) pair. For a static call/optimized // virtual this is just a destination address. - StaticCallInfo static_call_info; - CompiledICInfo virtual_call_info; + bool first_try = true; + for (;;) { + if (!first_try) { + // Patching IC caches may fail if we run out if transition stubs. + // We refill the ic stubs then. + InlineCacheBuffer::refill_ic_stubs(); + } + first_try = false; - // Make sure the callee nmethod does not get deoptimized and removed before - // we are done patching the code. - CompiledMethod* callee = callee_method->code(); + StaticCallInfo static_call_info; + CompiledICInfo virtual_call_info; - if (callee != NULL) { - assert(callee->is_compiled(), "must be nmethod for patching"); - } + // Make sure the callee nmethod does not get deoptimized and removed before + // we are done patching the code. + CompiledMethod* callee = callee_method->code(); - if (callee != NULL && !callee->is_in_use()) { - // Patch call site to C2I adapter if callee nmethod is deoptimized or unloaded. - callee = NULL; - } - nmethodLocker nl_callee(callee); + if (callee != NULL) { + assert(callee->is_compiled(), "must be nmethod for patching"); + } + + if (callee != NULL && !callee->is_in_use()) { + // Patch call site to C2I adapter if callee nmethod is deoptimized or unloaded. + callee = NULL; + } + nmethodLocker nl_callee(callee); #ifdef ASSERT - address dest_entry_point = callee == NULL ? 0 : callee->entry_point(); // used below + address dest_entry_point = callee == NULL ? 0 : callee->entry_point(); // used below #endif - bool is_nmethod = caller_nm->is_nmethod(); + bool is_nmethod = caller_nm->is_nmethod(); - if (is_virtual) { - assert(receiver.not_null() || invoke_code == Bytecodes::_invokehandle, "sanity check"); - bool static_bound = call_info.resolved_method()->can_be_statically_bound(); - Klass* klass = invoke_code == Bytecodes::_invokehandle ? NULL : receiver->klass(); - CompiledIC::compute_monomorphic_entry(callee_method, klass, - is_optimized, static_bound, is_nmethod, virtual_call_info, - CHECK_(methodHandle())); - } else { - // static call - CompiledStaticCall::compute_entry(callee_method, is_nmethod, static_call_info); - } + if (is_virtual) { + assert(receiver.not_null() || invoke_code == Bytecodes::_invokehandle, "sanity check"); + bool static_bound = call_info.resolved_method()->can_be_statically_bound(); + Klass* klass = invoke_code == Bytecodes::_invokehandle ? NULL : receiver->klass(); + CompiledIC::compute_monomorphic_entry(callee_method, klass, + is_optimized, static_bound, is_nmethod, virtual_call_info, + CHECK_(methodHandle())); + } else { + // static call + CompiledStaticCall::compute_entry(callee_method, is_nmethod, static_call_info); + } - // grab lock, check for deoptimization and potentially patch caller - { - CompiledICLocker ml(caller_nm); + // grab lock, check for deoptimization and potentially patch caller + { + CompiledICLocker ml(caller_nm); - // Lock blocks for safepoint during which both nmethods can change state. + // Lock blocks for safepoint during which both nmethods can change state. - // Now that we are ready to patch if the Method* was redefined then - // don't update call site and let the caller retry. - // Don't update call site if callee nmethod was unloaded or deoptimized. - // Don't update call site if callee nmethod was replaced by an other nmethod - // which may happen when multiply alive nmethod (tiered compilation) - // will be supported. - if (!callee_method->is_old() && - (callee == NULL || (callee->is_in_use() && callee_method->code() == callee))) { + // Now that we are ready to patch if the Method* was redefined then + // don't update call site and let the caller retry. + // Don't update call site if callee nmethod was unloaded or deoptimized. + // Don't update call site if callee nmethod was replaced by an other nmethod + // which may happen when multiply alive nmethod (tiered compilation) + // will be supported. + if (!callee_method->is_old() && + (callee == NULL || (callee->is_in_use() && callee_method->code() == callee))) { #ifdef ASSERT - // We must not try to patch to jump to an already unloaded method. - if (dest_entry_point != 0) { - CodeBlob* cb = CodeCache::find_blob(dest_entry_point); - assert((cb != NULL) && cb->is_compiled() && (((CompiledMethod*)cb) == callee), - "should not call unloaded nmethod"); - } + // We must not try to patch to jump to an already unloaded method. + if (dest_entry_point != 0) { + CodeBlob* cb = CodeCache::find_blob(dest_entry_point); + assert((cb != NULL) && cb->is_compiled() && (((CompiledMethod*)cb) == callee), + "should not call unloaded nmethod"); + } #endif - if (is_virtual) { - CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc()); - if (inline_cache->is_clean()) { - inline_cache->set_to_monomorphic(virtual_call_info); + if (is_virtual) { + CompiledIC* inline_cache = CompiledIC_before(caller_nm, caller_frame.pc()); + if (inline_cache->is_clean()) { + if (!inline_cache->set_to_monomorphic(virtual_call_info)) { + continue; + } + } + } else { + CompiledStaticCall* ssc = caller_nm->compiledStaticCall_before(caller_frame.pc()); + if (ssc->is_clean()) ssc->set(static_call_info); } - } else { - CompiledStaticCall* ssc = caller_nm->compiledStaticCall_before(caller_frame.pc()); - if (ssc->is_clean()) ssc->set(static_call_info); } - } - - } // unlock CompiledICLocker + } // unlock CompiledICLocker + break; + } return callee_method; } @@ -1555,8 +1568,6 @@ methodHandle SharedRuntime::handle_ic_miss_helper(JavaThread *thread, TRAPS) { methodHandle callee_method = call_info.selected_method(); - bool should_be_mono = false; - #ifndef PRODUCT Atomic::inc(&_ic_miss_ctr); @@ -1585,72 +1596,83 @@ methodHandle SharedRuntime::handle_ic_miss_helper(JavaThread *thread, TRAPS) { JvmtiDynamicCodeEventCollector event_collector; // Update inline cache to megamorphic. Skip update if we are called from interpreted. - { + bool first_try = true; + for (;;) { + if (!first_try) { + // Transitioning IC caches may require transition stubs. If we run out + // of transition stubs, we have to drop locks and perform a safepoint + // that refills them. + InlineCacheBuffer::refill_ic_stubs(); + } + first_try = false; RegisterMap reg_map(thread, false); frame caller_frame = thread->last_frame().sender(®_map); CodeBlob* cb = caller_frame.cb(); CompiledMethod* caller_nm = cb->as_compiled_method_or_null(); CompiledICLocker ml(caller_nm); - if (cb->is_compiled()) { - CompiledIC* inline_cache = CompiledIC_before(((CompiledMethod*)cb), caller_frame.pc()); - bool should_be_mono = false; - if (inline_cache->is_optimized()) { - if (TraceCallFixup) { - ResourceMark rm(thread); - tty->print("OPTIMIZED IC miss (%s) call to", Bytecodes::name(bc)); - callee_method->print_short_name(tty); - tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code())); - } - should_be_mono = true; - } else if (inline_cache->is_icholder_call()) { - CompiledICHolder* ic_oop = inline_cache->cached_icholder(); - if (ic_oop != NULL) { - if (!ic_oop->is_loader_alive()) { - // Deferred IC cleaning due to concurrent class unloading - inline_cache->set_to_clean(); - } else if (receiver()->klass() == ic_oop->holder_klass()) { - // This isn't a real miss. We must have seen that compiled code - // is now available and we want the call site converted to a - // monomorphic compiled call site. - // We can't assert for callee_method->code() != NULL because it - // could have been deoptimized in the meantime - if (TraceCallFixup) { - ResourceMark rm(thread); - tty->print("FALSE IC miss (%s) converting to compiled call to", Bytecodes::name(bc)); - callee_method->print_short_name(tty); - tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code())); - } - should_be_mono = true; + if (!cb->is_compiled()) { + Unimplemented(); + } + CompiledIC* inline_cache = CompiledIC_before(((CompiledMethod*)cb), caller_frame.pc()); + bool should_be_mono = false; + if (inline_cache->is_optimized()) { + if (TraceCallFixup) { + ResourceMark rm(thread); + tty->print("OPTIMIZED IC miss (%s) call to", Bytecodes::name(bc)); + callee_method->print_short_name(tty); + tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code())); + } + should_be_mono = true; + } else if (inline_cache->is_icholder_call()) { + CompiledICHolder* ic_oop = inline_cache->cached_icholder(); + if (ic_oop != NULL) { + if (!ic_oop->is_loader_alive()) { + // Deferred IC cleaning due to concurrent class unloading + inline_cache->set_to_clean(); + } else if (receiver()->klass() == ic_oop->holder_klass()) { + // This isn't a real miss. We must have seen that compiled code + // is now available and we want the call site converted to a + // monomorphic compiled call site. + // We can't assert for callee_method->code() != NULL because it + // could have been deoptimized in the meantime + if (TraceCallFixup) { + ResourceMark rm(thread); + tty->print("FALSE IC miss (%s) converting to compiled call to", Bytecodes::name(bc)); + callee_method->print_short_name(tty); + tty->print_cr(" code: " INTPTR_FORMAT, p2i(callee_method->code())); } + should_be_mono = true; } } + } - if (should_be_mono) { - - // We have a path that was monomorphic but was going interpreted - // and now we have (or had) a compiled entry. We correct the IC - // by using a new icBuffer. - CompiledICInfo info; - Klass* receiver_klass = receiver()->klass(); - inline_cache->compute_monomorphic_entry(callee_method, - receiver_klass, - inline_cache->is_optimized(), - false, caller_nm->is_nmethod(), - info, CHECK_(methodHandle())); - inline_cache->set_to_monomorphic(info); - } else if (!inline_cache->is_megamorphic() && !inline_cache->is_clean()) { - // Potential change to megamorphic - bool successful = inline_cache->set_to_megamorphic(&call_info, bc, CHECK_(methodHandle())); - if (!successful) { - inline_cache->set_to_clean(); + if (should_be_mono) { + // We have a path that was monomorphic but was going interpreted + // and now we have (or had) a compiled entry. We correct the IC + // by using a new icBuffer. + CompiledICInfo info; + Klass* receiver_klass = receiver()->klass(); + inline_cache->compute_monomorphic_entry(callee_method, + receiver_klass, + inline_cache->is_optimized(), + false, caller_nm->is_nmethod(), + info, CHECK_(methodHandle())); + if (!inline_cache->set_to_monomorphic(info)) { + continue; + } + } else if (!inline_cache->is_megamorphic() && !inline_cache->is_clean()) { + // Potential change to megamorphic + bool successful = inline_cache->set_to_megamorphic(&call_info, bc, CHECK_(methodHandle())); + if (!successful) { + if (!inline_cache->set_to_clean()) { + continue; } - } else { - // Either clean or megamorphic } } else { - fatal("Unimplemented"); + // Either clean or megamorphic } + break; } // Release CompiledICLocker return callee_method; @@ -1738,11 +1760,15 @@ methodHandle SharedRuntime::reresolve_call_site(JavaThread *thread, TRAPS) { CompiledICLocker ml(caller_nm); if (is_static_call) { CompiledStaticCall* ssc = caller_nm->compiledStaticCall_at(call_addr); - ssc->set_to_clean(); + if (!ssc->is_clean()) { + ssc->set_to_clean(); + } } else { // compiled, dispatched call (which used to call an interpreted method) CompiledIC* inline_cache = CompiledIC_at(caller_nm, call_addr); - inline_cache->set_to_clean(); + if (!inline_cache->is_clean()) { + inline_cache->set_to_clean(); + } } } } diff --git a/src/hotspot/share/runtime/sweeper.cpp b/src/hotspot/share/runtime/sweeper.cpp index c121f90..b3f864c 100644 --- a/src/hotspot/share/runtime/sweeper.cpp +++ b/src/hotspot/share/runtime/sweeper.cpp @@ -699,7 +699,6 @@ NMethodSweeper::MethodStateChange NMethodSweeper::process_compiled_method(Compil // But still remember to clean-up inline caches for alive nmethods if (cm->is_alive() && !cm->is_unloading()) { // Clean inline caches that point to zombie/non-entrant/unloaded nmethods - CompiledICLocker ml(cm); cm->cleanup_inline_caches(false); SWEEP(cm); } @@ -745,19 +744,16 @@ NMethodSweeper::MethodStateChange NMethodSweeper::process_compiled_method(Compil } } else { // Still alive, clean up its inline caches - CompiledICLocker ml(cm); cm->cleanup_inline_caches(false); SWEEP(cm); } } else if (cm->is_unloaded()) { // Code is unloaded, so there are no activations on the stack. // Convert the nmethod to zombie or flush it directly in the OSR case. - { - // Clean ICs of unloaded nmethods as well because they may reference other - // unloaded nmethods that may be flushed earlier in the sweeper cycle. - CompiledICLocker ml(cm); - cm->cleanup_inline_caches(false); - } + + // Clean ICs of unloaded nmethods as well because they may reference other + // unloaded nmethods that may be flushed earlier in the sweeper cycle. + cm->cleanup_inline_caches(false); if (cm->is_osr_method()) { SWEEP(cm); // No inline caches will ever point to osr methods, so we can just remove it @@ -776,7 +772,6 @@ NMethodSweeper::MethodStateChange NMethodSweeper::process_compiled_method(Compil possibly_flush((nmethod*)cm); } // Clean inline caches that point to zombie/non-entrant/unloaded nmethods - CompiledICLocker ml(cm); cm->cleanup_inline_caches(false); SWEEP(cm); }