--- old/src/hotspot/share/prims/jvm.cpp 2020-05-06 20:08:58.000000000 -0400 +++ new/src/hotspot/share/prims/jvm.cpp 2020-05-06 20:08:57.000000000 -0400 @@ -74,6 +74,7 @@ #include "runtime/os.inline.hpp" #include "runtime/perfData.hpp" #include "runtime/reflection.hpp" +#include "runtime/synchronizer.hpp" #include "runtime/thread.inline.hpp" #include "runtime/threadSMR.hpp" #include "runtime/vframe.inline.hpp" @@ -490,6 +491,11 @@ JVM_ENTRY_NO_ENV(void, JVM_GC(void)) JVMWrapper("JVM_GC"); if (!DisableExplicitGC) { + if (AsyncDeflateIdleMonitors) { + // AsyncDeflateIdleMonitors needs to know when System.gc() is + // called so any special deflation can be done at a safepoint. + ObjectSynchronizer::set_is_special_deflation_requested(true); + } Universe::heap()->collect(GCCause::_java_lang_system_gc); } JVM_END --- old/src/hotspot/share/prims/jvmtiEnvBase.cpp 2020-05-06 20:08:59.000000000 -0400 +++ new/src/hotspot/share/prims/jvmtiEnvBase.cpp 2020-05-06 20:08:59.000000000 -0400 @@ -653,6 +653,9 @@ current_jt == java_thread->active_handshaker(), "call by myself or at direct handshake"); oop obj = NULL; + // The ObjectMonitor* can't be async deflated since we are either + // at a safepoint or the calling thread is operating on itself so + // it cannot leave the underlying wait()/enter() call. ObjectMonitor *mon = java_thread->current_waiting_monitor(); if (mon == NULL) { // thread is not doing an Object.wait() call @@ -730,7 +733,10 @@ HandleMark hm; oop wait_obj = NULL; { - // save object of current wait() call (if any) for later comparison + // The ObjectMonitor* can't be async deflated since we are either + // at a safepoint or the calling thread is operating on itself so + // it cannot leave the underlying wait() call. + // Save object of current wait() call (if any) for later comparison. ObjectMonitor *mon = java_thread->current_waiting_monitor(); if (mon != NULL) { wait_obj = (oop)mon->object(); @@ -738,7 +744,10 @@ } oop pending_obj = NULL; { - // save object of current enter() call (if any) for later comparison + // The ObjectMonitor* can't be async deflated since we are either + // at a safepoint or the calling thread is operating on itself so + // it cannot leave the underlying enter() call. + // Save object of current enter() call (if any) for later comparison. ObjectMonitor *mon = java_thread->current_pending_monitor(); if (mon != NULL) { pending_obj = (oop)mon->object(); --- old/src/hotspot/share/prims/whitebox.cpp 2020-05-06 20:09:00.000000000 -0400 +++ new/src/hotspot/share/prims/whitebox.cpp 2020-05-06 20:09:00.000000000 -0400 @@ -72,6 +72,7 @@ #include "runtime/jniHandles.inline.hpp" #include "runtime/os.hpp" #include "runtime/sweeper.hpp" +#include "runtime/synchronizer.hpp" #include "runtime/thread.hpp" #include "runtime/threadSMR.hpp" #include "runtime/vm_version.hpp" @@ -476,6 +477,12 @@ WB_ENTRY(jboolean, WB_G1StartMarkCycle(JNIEnv* env, jobject o)) if (UseG1GC) { + if (AsyncDeflateIdleMonitors) { + // AsyncDeflateIdleMonitors needs to know when System.gc() or + // the equivalent is called so any special clean up can be done + // at a safepoint, e.g., TestHumongousClassLoader.java. + ObjectSynchronizer::set_is_special_deflation_requested(true); + } G1CollectedHeap* g1h = G1CollectedHeap::heap(); if (!g1h->concurrent_mark()->cm_thread()->during_cycle()) { g1h->collect(GCCause::_wb_conc_mark); @@ -1447,6 +1454,12 @@ WB_END WB_ENTRY(void, WB_FullGC(JNIEnv* env, jobject o)) + if (AsyncDeflateIdleMonitors) { + // AsyncDeflateIdleMonitors needs to know when System.gc() or + // the equivalent is called so any special clean up can be done + // at a safepoint, e.g., TestHumongousClassLoader.java. + ObjectSynchronizer::set_is_special_deflation_requested(true); + } Universe::heap()->soft_ref_policy()->set_should_clear_all_soft_refs(true); Universe::heap()->collect(GCCause::_wb_full_gc); #if INCLUDE_G1GC --- old/src/hotspot/share/runtime/basicLock.cpp 2020-05-06 20:09:01.000000000 -0400 +++ new/src/hotspot/share/runtime/basicLock.cpp 2020-05-06 20:09:01.000000000 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,24 +36,24 @@ void BasicLock::move_to(oop obj, BasicLock* dest) { // Check to see if we need to inflate the lock. This is only needed // if an object is locked using "this" lightweight monitor. In that - // case, the displaced_header() is unlocked, because the + // case, the displaced_header() is unlocked/is_neutral, because the // displaced_header() contains the header for the originally unlocked - // object. However the object could have already been inflated. But it - // does not matter, the inflation will just a no-op. For other cases, + // object. However the lock could have already been inflated. But it + // does not matter, this inflation will just a no-op. For other cases, // the displaced header will be either 0x0 or 0x3, which are location // independent, therefore the BasicLock is free to move. // // During OSR we may need to relocate a BasicLock (which contains a // displaced word) from a location in an interpreter frame to a // new location in a compiled frame. "this" refers to the source - // basiclock in the interpreter frame. "dest" refers to the destination - // basiclock in the new compiled frame. We *always* inflate in move_to(). - // The always-Inflate policy works properly, but in 1.5.0 it can sometimes - // cause performance problems in code that makes heavy use of a small # of - // uncontended locks. (We'd inflate during OSR, and then sync performance - // would subsequently plummet because the thread would be forced thru the slow-path). - // This problem has been made largely moot on IA32 by inlining the inflated fast-path - // operations in Fast_Lock and Fast_Unlock in i486.ad. + // BasicLock in the interpreter frame. "dest" refers to the destination + // BasicLock in the new compiled frame. We *always* inflate in move_to() + // when the object is locked using "this" lightweight monitor. + // + // The always-Inflate policy works properly, but it depends on the + // inflated fast-path operations in fast_lock and fast_unlock to avoid + // performance problems. See x86/macroAssembler_x86.cpp: fast_lock() + // and fast_unlock() for examples. // // Note that there is a way to safely swing the object's markword from // one stack location to another. This avoids inflation. Obviously, @@ -63,8 +63,10 @@ // we'll leave that optimization for another time. if (displaced_header().is_neutral()) { + // The object is locked and the resulting ObjectMonitor* will also be + // locked so it can't be async deflated until ownership is dropped. ObjectSynchronizer::inflate_helper(obj); - // WARNING: We can not put check here, because the inflation + // WARNING: We cannot put a check here, because the inflation // will not update the displaced header. Once BasicLock is inflated, // no one should ever look at its content. } else { --- old/src/hotspot/share/runtime/globals.hpp 2020-05-06 20:09:03.000000000 -0400 +++ new/src/hotspot/share/runtime/globals.hpp 2020-05-06 20:09:02.000000000 -0400 @@ -694,11 +694,21 @@ product(intx, MonitorBound, 0, "(Deprecated) Bound Monitor population") \ range(0, max_jint) \ \ + diagnostic(bool, AsyncDeflateIdleMonitors, true, \ + "Deflate idle monitors using the ServiceThread.") \ + \ + /* notice: the max range value here is max_jint, not max_intx */ \ + /* because of overflow issue */ \ + diagnostic(intx, AsyncDeflationInterval, 250, \ + "Async deflate idle monitors every so many milliseconds when " \ + "MonitorUsedDeflationThreshold is exceeded (0 is off).") \ + range(0, max_jint) \ + \ experimental(intx, MonitorUsedDeflationThreshold, 90, \ - "Percentage of used monitors before triggering cleanup " \ - "safepoint which deflates monitors (0 is off). " \ - "The check is performed on GuaranteedSafepointInterval.") \ - range(0, 100) \ + "Percentage of used monitors before triggering deflation (0 is " \ + "off). The check is performed on GuaranteedSafepointInterval " \ + "or AsyncDeflateInterval.") \ + range(0, 100) \ \ experimental(intx, hashCode, 5, \ "(Unstable) select hashCode generation algorithm") \ --- old/src/hotspot/share/runtime/init.cpp 2020-05-06 20:09:04.000000000 -0400 +++ new/src/hotspot/share/runtime/init.cpp 2020-05-06 20:09:04.000000000 -0400 @@ -176,8 +176,12 @@ if (log_is_enabled(Info, monitorinflation)) { // The ObjectMonitor subsystem uses perf counters so // do this before perfMemory_exit(). - // ObjectSynchronizer::finish_deflate_idle_monitors()'s call - // to audit_and_print_stats() is done at the Debug level. + // These other two audit_and_print_stats() calls are done at the + // Debug level at a safepoint: + // - for safepoint based deflation auditing: + // ObjectSynchronizer::finish_deflate_idle_monitors() + // - for async deflation auditing: + // ObjectSynchronizer::do_safepoint_work() ObjectSynchronizer::audit_and_print_stats(true /* on_exit */); } perfMemory_exit(); --- old/src/hotspot/share/runtime/objectMonitor.cpp 2020-05-06 20:09:05.000000000 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.cpp 2020-05-06 20:09:05.000000000 -0400 @@ -240,7 +240,7 @@ // ----------------------------------------------------------------------------- // Enter support -void ObjectMonitor::enter(TRAPS) { +bool ObjectMonitor::enter(TRAPS) { // The following code is ordered to check the most common cases first // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors. Thread * const Self = THREAD; @@ -248,20 +248,20 @@ void* cur = try_set_owner_from(NULL, Self); if (cur == NULL) { assert(_recursions == 0, "invariant"); - return; + return true; } if (cur == Self) { // TODO-FIXME: check for integer overflow! BUGID 6557169. _recursions++; - return; + return true; } if (Self->is_lock_owned((address)cur)) { assert(_recursions == 0, "internal state error"); _recursions = 1; set_owner_from_BasicLock(cur, Self); // Convert from BasicLock* to Thread*. - return; + return true; } // We've encountered genuine contention. @@ -281,7 +281,7 @@ ", encoded this=" INTPTR_FORMAT, ((oop)object())->mark().value(), markWord::encode(this).value()); Self->_Stalled = 0; - return; + return true; } assert(_owner != Self, "invariant"); @@ -290,12 +290,25 @@ JavaThread * jt = (JavaThread *) Self; assert(!SafepointSynchronize::is_at_safepoint(), "invariant"); assert(jt->thread_state() != _thread_blocked, "invariant"); - assert(this->object() != NULL, "invariant"); - assert(_contentions >= 0, "invariant"); + assert(AsyncDeflateIdleMonitors || this->object() != NULL, "invariant"); + assert(AsyncDeflateIdleMonitors || contentions() >= 0, "must not be negative: contentions=%d", contentions()); - // Prevent deflation at STW-time. See deflate_idle_monitors() and is_busy(). - // Ensure the object-monitor relationship remains stable while there's contention. + // Keep track of contention for JVM/TI and M&M queries. Atomic::inc(&_contentions); + if (AsyncDeflateIdleMonitors && is_being_async_deflated()) { + // Async deflation is in progress and our contentions increment + // above lost the race to async deflation. Undo the work and + // force the caller to retry. + const oop l_object = (oop)object(); + if (l_object != NULL) { + // Attempt to restore the header/dmw to the object's header so that + // we only retry once if the deflater thread happens to be slow. + install_displaced_markword_in_object(l_object); + } + Self->_Stalled = 0; + Atomic::dec(&_contentions); + return false; + } JFR_ONLY(JfrConditionalFlushWithStacktrace flush(jt);) EventJavaMonitorEnter event; @@ -357,7 +370,7 @@ } Atomic::dec(&_contentions); - assert(_contentions >= 0, "invariant"); + assert(contentions() >= 0, "must not be negative: contentions=%d", contentions()); Self->_Stalled = 0; // Must either set _recursions = 0 or ASSERT _recursions == 0. @@ -393,6 +406,7 @@ event.commit(); } OM_PERFDATA_OP(ContendedLockAttempts, inc()); + return true; } // Caveat: TryLock() is not necessarily serializing if it returns failure. @@ -412,12 +426,76 @@ return -1; } +// Install the displaced mark word (dmw) of a deflating ObjectMonitor +// into the header of the object associated with the monitor. This +// idempotent method is called by a thread that is deflating a +// monitor and by other threads that have detected a race with the +// deflation process. +void ObjectMonitor::install_displaced_markword_in_object(const oop obj) { + // This function must only be called when (owner == DEFLATER_MARKER + // && contentions <= 0), but we can't guarantee that here because + // those values could change when the ObjectMonitor gets moved from + // the global free list to a per-thread free list. + + guarantee(obj != NULL, "must be non-NULL"); + + const oop l_object = (oop)object(); + if (l_object == NULL) { + // ObjectMonitor's object ref has already been cleared by async + // deflation so we're done here. + return; + } + ADIM_guarantee(l_object == obj, "object=" INTPTR_FORMAT " must equal obj=" + INTPTR_FORMAT, p2i(l_object), p2i(obj)); + + markWord dmw = header(); + // The dmw has to be neutral (not NULL, not locked and not marked). + ADIM_guarantee(dmw.is_neutral(), "must be neutral: dmw=" INTPTR_FORMAT, dmw.value()); + + // Install displaced mark word if the object's header still points + // to this ObjectMonitor. All racing callers to this function will + // reach this point, but only one can win. + markWord res = obj->cas_set_mark(dmw, markWord::encode(this)); + if (res != markWord::encode(this)) { + // This should be rare so log at the Info level when it happens. + log_info(monitorinflation)("install_displaced_markword_in_object: " + "failed cas_set_mark: new_mark=" INTPTR_FORMAT + ", old_mark=" INTPTR_FORMAT ", res=" INTPTR_FORMAT, + dmw.value(), markWord::encode(this).value(), + res.value()); + } + + // Note: It does not matter which thread restored the header/dmw + // into the object's header. The thread deflating the monitor just + // wanted the object's header restored and it is. The threads that + // detected a race with the deflation process also wanted the + // object's header restored before they retry their operation and + // because it is restored they will only retry once. +} + // Convert the fields used by is_busy() to a string that can be // used for diagnostic output. const char* ObjectMonitor::is_busy_to_string(stringStream* ss) { - ss->print("is_busy: contentions=%d, waiters=%d, owner=" INTPTR_FORMAT - ", cxq=" INTPTR_FORMAT ", EntryList=" INTPTR_FORMAT, _contentions, - _waiters, p2i(_owner), p2i(_cxq), p2i(_EntryList)); + ss->print("is_busy: waiters=%d, ", _waiters); + if (!AsyncDeflateIdleMonitors) { + ss->print("contentions=%d, ", contentions()); + ss->print("owner=" INTPTR_FORMAT, p2i(_owner)); + } else { + if (contentions() > 0) { + ss->print("contentions=%d, ", contentions()); + } else { + ss->print("contentions=0"); + } + if (_owner != DEFLATER_MARKER) { + ss->print("owner=" INTPTR_FORMAT, p2i(_owner)); + } else { + // We report NULL instead of DEFLATER_MARKER here because is_busy() + // ignores DEFLATER_MARKER values. + ss->print("owner=" INTPTR_FORMAT, NULL); + } + } + ss->print(", cxq=" INTPTR_FORMAT ", EntryList=" INTPTR_FORMAT, p2i(_cxq), + p2i(_EntryList)); return ss->base(); } @@ -436,6 +514,16 @@ return; } + if (AsyncDeflateIdleMonitors && + try_set_owner_from(DEFLATER_MARKER, Self) == DEFLATER_MARKER) { + // The deflation protocol finished the first part (setting owner), + // but it failed the second part (making contentions negative) and + // bailed. Acquired the monitor. + assert(_succ != Self, "invariant"); + assert(_Responsible != Self, "invariant"); + return; + } + assert(InitDone, "Unexpectedly not initialized"); // We try one round of spinning *before* enqueueing Self. @@ -552,6 +640,14 @@ if (TryLock(Self) > 0) break; + if (AsyncDeflateIdleMonitors && + try_set_owner_from(DEFLATER_MARKER, Self) == DEFLATER_MARKER) { + // The deflation protocol finished the first part (setting owner), + // but it failed the second part (making contentions negative) and + // bailed. Acquired the monitor. + break; + } + // The lock is still contested. // Keep a tally of the # of futile wakeups. // Note that the counter is not protected by a lock or updated by atomics. @@ -816,7 +912,7 @@ // We'd like to assert that: (THREAD->thread_state() != _thread_blocked) ; // There's one exception to the claim above, however. EnterI() can call // exit() to drop a lock if the acquirer has been externally suspended. -// In that case exit() is called with _thread_state as _thread_blocked, +// In that case exit() is called with _thread_state == _thread_blocked, // but the monitor's _contentions field is > 0, which inhibits reclamation. // // 1-0 exit @@ -1091,7 +1187,7 @@ // out-of-scope (non-extant). Wakee = NULL; - // Drop the lock + // Drop the lock. // Uses a fence to separate release_store(owner) from the LD in unpark(). release_clear_owner(Self); OrderAccess::fence(); @@ -1139,16 +1235,19 @@ // reenter() enters a lock and sets recursion count // complete_exit/reenter operate as a wait without waiting -void ObjectMonitor::reenter(intx recursions, TRAPS) { +bool ObjectMonitor::reenter(intx recursions, TRAPS) { Thread * const Self = THREAD; assert(Self->is_Java_thread(), "Must be Java thread!"); JavaThread *jt = (JavaThread *)THREAD; guarantee(_owner != Self, "reenter already owner"); - enter(THREAD); // enter the monitor + if (!enter(THREAD)) { + return false; + } + // Entered the monitor. guarantee(_recursions == 0, "reenter recursion"); _recursions = recursions; - return; + return true; } // Checks that the current THREAD owns this monitor and causes an @@ -1962,14 +2061,20 @@ // (ObjectMonitor) 0x00007fdfb6012e40 = { // _header = 0x0000000000000001 // _object = 0x000000070ff45fd0 -// _next_om = 0x0000000000000000 +// _allocation_state = Old // _pad_buf0 = { // [0] = '\0' // ... -// [103] = '\0' +// [43] = '\0' // } // _owner = 0x0000000000000000 // _previous_owner_tid = 0 +// _pad_buf1 = { +// [0] = '\0' +// ... +// [47] = '\0' +// } +// _next_om = 0x0000000000000000 // _recursions = 0 // _EntryList = 0x0000000000000000 // _cxq = 0x0000000000000000 @@ -1987,7 +2092,17 @@ st->print_cr("(ObjectMonitor*) " INTPTR_FORMAT " = {", p2i(this)); st->print_cr(" _header = " INTPTR_FORMAT, header().value()); st->print_cr(" _object = " INTPTR_FORMAT, p2i(_object)); - st->print_cr(" _next_om = " INTPTR_FORMAT, p2i(next_om())); + st->print(" _allocation_state = "); + if (is_free()) { + st->print("Free"); + } else if (is_old()) { + st->print("Old"); + } else if (is_new()) { + st->print("New"); + } else { + st->print("unknown=%d", _allocation_state); + } + st->cr(); st->print_cr(" _pad_buf0 = {"); st->print_cr(" [0] = '\\0'"); st->print_cr(" ..."); @@ -1995,6 +2110,12 @@ st->print_cr(" }"); st->print_cr(" _owner = " INTPTR_FORMAT, p2i(_owner)); st->print_cr(" _previous_owner_tid = " JLONG_FORMAT, _previous_owner_tid); + st->print_cr(" _pad_buf1 = {"); + st->print_cr(" [0] = '\\0'"); + st->print_cr(" ..."); + st->print_cr(" [%d] = '\\0'", (int)sizeof(_pad_buf1) - 1); + st->print_cr(" }"); + st->print_cr(" _next_om = " INTPTR_FORMAT, p2i(next_om())); st->print_cr(" _recursions = " INTX_FORMAT, _recursions); st->print_cr(" _EntryList = " INTPTR_FORMAT, p2i(_EntryList)); st->print_cr(" _cxq = " INTPTR_FORMAT, p2i(_cxq)); @@ -2002,7 +2123,7 @@ st->print_cr(" _Responsible = " INTPTR_FORMAT, p2i(_Responsible)); st->print_cr(" _Spinner = %d", _Spinner); st->print_cr(" _SpinDuration = %d", _SpinDuration); - st->print_cr(" _contentions = %d", _contentions); + st->print_cr(" _contentions = %d", contentions()); st->print_cr(" _WaitSet = " INTPTR_FORMAT, p2i(_WaitSet)); st->print_cr(" _waiters = %d", _waiters); st->print_cr(" _WaitSetLock = %d", _WaitSetLock); --- old/src/hotspot/share/runtime/objectMonitor.hpp 2020-05-06 20:09:07.000000000 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.hpp 2020-05-06 20:09:06.000000000 -0400 @@ -136,13 +136,21 @@ // Enforced by the assert() in header_addr(). volatile markWord _header; // displaced object header word - mark void* volatile _object; // backward object pointer - strong root - private: + typedef enum { + Free = 0, // Free must be 0 for monitor to be free after memset(..,0,..). + New, + Old + } AllocationState; + AllocationState _allocation_state; // Separate _header and _owner on different cache lines since both can - // have busy multi-threaded access. _header and _object are set at - // initial inflation and _object doesn't change until deflation so - // _object is a good choice to share the cache line with _header. - DEFINE_PAD_MINUS_SIZE(0, OM_CACHE_LINE_SIZE, - sizeof(volatile markWord) + sizeof(void* volatile)); + // have busy multi-threaded access. _header, _object and _allocation_state + // are set at initial inflation. _object and _allocation_state don't + // change until deflation so _object and _allocation_state are good + // choices to share the cache line with _header. + DEFINE_PAD_MINUS_SIZE(0, OM_CACHE_LINE_SIZE, sizeof(volatile markWord) + + sizeof(void* volatile) + sizeof(AllocationState)); + // Used by async deflation as a marker in the _owner field: + #define DEFLATER_MARKER reinterpret_cast(-1) void* volatile _owner; // pointer to owning thread OR BasicLock volatile jlong _previous_owner_tid; // thread id of the previous owner of the monitor // Separate _owner and _next_om on different cache lines since @@ -164,9 +172,10 @@ volatile int _Spinner; // for exit->spinner handoff optimization volatile int _SpinDuration; - volatile jint _contentions; // Number of active contentions in enter(). It is used by is_busy() + jint _contentions; // Number of active contentions in enter(). It is used by is_busy() // along with other fields to determine if an ObjectMonitor can be - // deflated. See ObjectSynchronizer::deflate_monitor(). + // deflated. It is also used by the async deflation protocol. See + // ObjectSynchronizer::deflate_monitor() and deflate_monitor_using_JT(). protected: ObjectWaiter* volatile _WaitSet; // LL of threads wait()ing on the monitor volatile jint _waiters; // number of waiting threads @@ -233,17 +242,34 @@ intptr_t is_busy() const { // TODO-FIXME: assert _owner == null implies _recursions = 0 - return _contentions|_waiters|intptr_t(_owner)|intptr_t(_cxq)|intptr_t(_EntryList); + intptr_t ret_code = _waiters | intptr_t(_cxq) | intptr_t(_EntryList); + if (!AsyncDeflateIdleMonitors) { + ret_code |= contentions() | intptr_t(_owner); + } else { + if (contentions() > 0) { + ret_code |= contentions(); + } + if (_owner != DEFLATER_MARKER) { + ret_code |= intptr_t(_owner); + } + } + return ret_code; } const char* is_busy_to_string(stringStream* ss); intptr_t is_entered(Thread* current) const; - void* owner() const; + void* owner() const; // Returns NULL if DEFLATER_MARKER is observed. + // Returns true if owner field == DEFLATER_MARKER and false otherwise. + bool owner_is_DEFLATER_MARKER(); + // Returns true if 'this' is being async deflated and false otherwise. + bool is_being_async_deflated(); // Clear _owner field; current value must match old_value. void release_clear_owner(void* old_value); // Simply set _owner field to new_value; current value must match old_value. void set_owner_from(void* old_value, void* new_value); + // Simply set _owner field to new_value; current value must match old_value1 or old_value2. + void set_owner_from(void* old_value1, void* old_value2, void* new_value); // Simply set _owner field to self; current value must match basic_lock_p. void set_owner_from_BasicLock(void* basic_lock_p, Thread* self); // Try to set _owner field to new_value if the current value matches @@ -286,7 +312,9 @@ // _cxq == 0 _succ == NULL _owner == NULL _waiters == 0 // _contentions == 0 EntryList == NULL // _recursions == 0 _WaitSet == NULL - DEBUG_ONLY(stringStream ss;) +#ifdef ASSERT + stringStream ss; +#endif assert((is_busy() | _recursions) == 0, "freeing in-use monitor: %s, " "recursions=" INTX_FORMAT, is_busy_to_string(&ss), _recursions); _succ = NULL; @@ -301,13 +329,19 @@ void* object() const; void* object_addr(); void set_object(void* obj); + void set_allocation_state(AllocationState s); + AllocationState allocation_state() const; + bool is_free() const; + bool is_old() const; + bool is_new() const; // Returns true if the specified thread owns the ObjectMonitor. Otherwise // returns false and throws IllegalMonitorStateException (IMSE). bool check_owner(Thread* THREAD); void clear(); + void clear_using_JT(); - void enter(TRAPS); + bool enter(TRAPS); void exit(bool not_suspended, TRAPS); void wait(jlong millis, bool interruptable, TRAPS); void notify(TRAPS); @@ -321,7 +355,7 @@ // Use the following at your own risk intx complete_exit(TRAPS); - void reenter(intx recursions, TRAPS); + bool reenter(intx recursions, TRAPS); private: void AddWaiter(ObjectWaiter* waiter); @@ -332,10 +366,22 @@ void ReenterI(Thread* self, ObjectWaiter* self_node); void UnlinkAfterAcquire(Thread* self, ObjectWaiter* self_node); int TryLock(Thread* self); - int NotRunnable(Thread* self, Thread * Owner); + int NotRunnable(Thread* self, Thread* Owner); int TrySpin(Thread* self); void ExitEpilog(Thread* self, ObjectWaiter* Wakee); bool ExitSuspendEquivalent(JavaThread* self); + void install_displaced_markword_in_object(const oop obj); }; +// Macro to use guarantee() for more strict AsyncDeflateIdleMonitors +// checks and assert() otherwise. +#define ADIM_guarantee(p, ...) \ + do { \ + if (AsyncDeflateIdleMonitors) { \ + guarantee(p, __VA_ARGS__); \ + } else { \ + assert(p, __VA_ARGS__); \ + } \ + } while (0) + #endif // SHARE_RUNTIME_OBJECTMONITOR_HPP --- old/src/hotspot/share/runtime/objectMonitor.inline.hpp 2020-05-06 20:09:08.000000000 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.inline.hpp 2020-05-06 20:09:07.000000000 -0400 @@ -52,19 +52,55 @@ return _waiters; } +// Returns NULL if DEFLATER_MARKER is observed. inline void* ObjectMonitor::owner() const { - return _owner; + void* owner = _owner; + return owner != DEFLATER_MARKER ? owner : NULL; +} + +// Returns true if owner field == DEFLATER_MARKER and false otherwise. +// This accessor is called when we really need to know if the owner +// field == DEFLATER_MARKER and any non-NULL value won't do the trick. +inline bool ObjectMonitor::owner_is_DEFLATER_MARKER() { + return _owner == DEFLATER_MARKER; +} + +// Returns true if 'this' is being async deflated and false otherwise. +inline bool ObjectMonitor::is_being_async_deflated() { + return owner_is_DEFLATER_MARKER() && contentions() < 0; } inline void ObjectMonitor::clear() { assert(Atomic::load(&_header).value() != 0, "must be non-zero"); - assert(_contentions == 0, "must be 0: contentions=%d", _contentions); + assert(_owner == NULL, "must be NULL: owner=" INTPTR_FORMAT, p2i(_owner)); + + Atomic::store(&_header, markWord::zero()); + + clear_using_JT(); +} + +inline void ObjectMonitor::clear_using_JT() { + // Unlike other *_using_JT() functions, we cannot assert + // AsyncDeflateIdleMonitors or Thread::current()->is_Java_thread() + // because clear() calls this function for the rest of its checks. + + if (AsyncDeflateIdleMonitors) { + // Async deflation protocol uses the header, owner and contentions + // fields. While the ObjectMonitor being deflated is on the global free + // list, we leave those three fields alone; owner == DEFLATER_MARKER + // and contentions < 0 will force any racing threads to retry. The + // header field is used by install_displaced_markword_in_object() + // to restore the object's header so we cannot check its value here. + guarantee(_owner == NULL || _owner == DEFLATER_MARKER, + "must be NULL or DEFLATER_MARKER: owner=" INTPTR_FORMAT, + p2i(_owner)); + } + assert(contentions() <= 0, "must not be positive: contentions=%d", contentions()); assert(_waiters == 0, "must be 0: waiters=%d", _waiters); assert(_recursions == 0, "must be 0: recursions=" INTX_FORMAT, _recursions); assert(_object != NULL, "must be non-NULL"); - assert(_owner == NULL, "must be NULL: owner=" INTPTR_FORMAT, p2i(_owner)); - Atomic::store(&_header, markWord::zero()); + set_allocation_state(Free); _object = NULL; } @@ -80,16 +116,16 @@ _object = obj; } -// return number of threads contending for this monitor +// Return number of threads contending for this monitor. inline jint ObjectMonitor::contentions() const { - return _contentions; + return Atomic::load(&_contentions); } // Clear _owner field; current value must match old_value. inline void ObjectMonitor::release_clear_owner(void* old_value) { - DEBUG_ONLY(void* prev = Atomic::load(&_owner);) - assert(prev == old_value, "unexpected prev owner=" INTPTR_FORMAT - ", expected=" INTPTR_FORMAT, p2i(prev), p2i(old_value)); + void* prev = Atomic::load(&_owner); + ADIM_guarantee(prev == old_value, "unexpected prev owner=" INTPTR_FORMAT + ", expected=" INTPTR_FORMAT, p2i(prev), p2i(old_value)); Atomic::release_store(&_owner, (void*)NULL); log_trace(monitorinflation, owner)("release_clear_owner(): mid=" INTPTR_FORMAT ", old_value=" INTPTR_FORMAT, @@ -99,9 +135,9 @@ // Simply set _owner field to new_value; current value must match old_value. // (Simple means no memory sync needed.) inline void ObjectMonitor::set_owner_from(void* old_value, void* new_value) { - DEBUG_ONLY(void* prev = Atomic::load(&_owner);) - assert(prev == old_value, "unexpected prev owner=" INTPTR_FORMAT - ", expected=" INTPTR_FORMAT, p2i(prev), p2i(old_value)); + void* prev = Atomic::load(&_owner); + ADIM_guarantee(prev == old_value, "unexpected prev owner=" INTPTR_FORMAT + ", expected=" INTPTR_FORMAT, p2i(prev), p2i(old_value)); Atomic::store(&_owner, new_value); log_trace(monitorinflation, owner)("set_owner_from(): mid=" INTPTR_FORMAT ", old_value=" INTPTR_FORMAT @@ -109,11 +145,28 @@ p2i(old_value), p2i(new_value)); } +// Simply set _owner field to new_value; current value must match old_value1 or old_value2. +// (Simple means no memory sync needed.) +inline void ObjectMonitor::set_owner_from(void* old_value1, void* old_value2, void* new_value) { + void* prev = Atomic::load(&_owner); + ADIM_guarantee(prev == old_value1 || prev == old_value2, + "unexpected prev owner=" INTPTR_FORMAT ", expected1=" + INTPTR_FORMAT " or expected2=" INTPTR_FORMAT, p2i(prev), + p2i(old_value1), p2i(old_value2)); + _owner = new_value; + log_trace(monitorinflation, owner)("set_owner_from(old1=" INTPTR_FORMAT + ", old2=" INTPTR_FORMAT "): mid=" + INTPTR_FORMAT ", prev=" INTPTR_FORMAT + ", new=" INTPTR_FORMAT, p2i(old_value1), + p2i(old_value2), p2i(this), p2i(prev), + p2i(new_value)); +} + // Simply set _owner field to self; current value must match basic_lock_p. inline void ObjectMonitor::set_owner_from_BasicLock(void* basic_lock_p, Thread* self) { - DEBUG_ONLY(void* prev = Atomic::load(&_owner);) - assert(prev == basic_lock_p, "unexpected prev owner=" INTPTR_FORMAT - ", expected=" INTPTR_FORMAT, p2i(prev), p2i(basic_lock_p)); + void* prev = Atomic::load(&_owner); + ADIM_guarantee(prev == basic_lock_p, "unexpected prev owner=" INTPTR_FORMAT + ", expected=" INTPTR_FORMAT, p2i(prev), p2i(basic_lock_p)); // Non-null owner field to non-null owner field is safe without // cmpxchg() as long as all readers can tolerate either flavor. Atomic::store(&_owner, self); @@ -137,6 +190,26 @@ return prev; } +inline void ObjectMonitor::set_allocation_state(ObjectMonitor::AllocationState s) { + _allocation_state = s; +} + +inline ObjectMonitor::AllocationState ObjectMonitor::allocation_state() const { + return _allocation_state; +} + +inline bool ObjectMonitor::is_free() const { + return _allocation_state == Free; +} + +inline bool ObjectMonitor::is_old() const { + return _allocation_state == Old; +} + +inline bool ObjectMonitor::is_new() const { + return _allocation_state == New; +} + // The _next_om field can be concurrently read and modified so we // use Atomic operations to disable compiler optimizations that // might try to elide loading and/or storing this field. --- old/src/hotspot/share/runtime/safepoint.cpp 2020-05-06 20:09:09.000000000 -0400 +++ new/src/hotspot/share/runtime/safepoint.cpp 2020-05-06 20:09:09.000000000 -0400 @@ -490,8 +490,9 @@ } bool SafepointSynchronize::is_cleanup_needed() { - // Need a safepoint if there are many monitors to deflate. - if (ObjectSynchronizer::is_cleanup_needed()) return true; + // Need a cleanup safepoint if there are too many monitors in use + // and the monitor deflation needs to be done at a safepoint. + if (ObjectSynchronizer::is_safepoint_deflation_needed()) return true; // Need a safepoint if some inline cache buffers is non-empty if (!InlineCacheBuffer::is_empty()) return true; if (StringTable::needs_rehashing()) return true; @@ -514,6 +515,10 @@ _counters(counters) {} void do_thread(Thread* thread) { + // deflate_thread_local_monitors() handles or requests deflation of + // this thread's idle monitors. If !AsyncDeflateIdleMonitors or if + // there is a special cleanup request, deflation is handled now. + // Otherwise, async deflation is requested via a flag. ObjectSynchronizer::deflate_thread_local_monitors(thread, _counters); if (_nmethod_cl != NULL && thread->is_Java_thread() && ! thread->is_Code_cache_sweeper_thread()) { @@ -546,7 +551,11 @@ const char* name = "deflating global idle monitors"; EventSafepointCleanupTask event; TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); - ObjectSynchronizer::deflate_idle_monitors(_counters); + // AsyncDeflateIdleMonitors only uses DeflateMonitorCounters + // when a special cleanup has been requested. + // Note: This logging output will include global idle monitor + // elapsed times, but not global idle monitor deflation count. + ObjectSynchronizer::do_safepoint_work(_counters); post_safepoint_cleanup_task_event(event, safepoint_id, name); } --- old/src/hotspot/share/runtime/serviceThread.cpp 2020-05-06 20:09:10.000000000 -0400 +++ new/src/hotspot/share/runtime/serviceThread.cpp 2020-05-06 20:09:10.000000000 -0400 @@ -110,6 +110,7 @@ bool thread_id_table_work = false; bool protection_domain_table_work = false; bool oopstorage_work = false; + bool deflate_idle_monitors = false; JvmtiDeferredEvent jvmti_event; { // Need state transition ThreadBlockInVM so that this thread @@ -136,10 +137,14 @@ (resolved_method_table_work = ResolvedMethodTable::has_work()) | (thread_id_table_work = ThreadIdTable::has_work()) | (protection_domain_table_work = SystemDictionary::pd_cache_table()->has_work()) | - (oopstorage_work = OopStorage::has_cleanup_work_and_reset()) + (oopstorage_work = OopStorage::has_cleanup_work_and_reset()) | + (deflate_idle_monitors = ObjectSynchronizer::is_async_deflation_needed()) ) == 0) { // Wait until notified that there is some work to do. - ml.wait(); + // If AsyncDeflateIdleMonitors, then we wait for + // GuaranteedSafepointInterval so that is_async_deflation_needed() + // is checked at the same interval. + ml.wait(AsyncDeflateIdleMonitors ? GuaranteedSafepointInterval : 0); } if (has_jvmti_events) { @@ -191,6 +196,10 @@ if (oopstorage_work) { cleanup_oopstorages(); } + + if (deflate_idle_monitors) { + ObjectSynchronizer::deflate_idle_monitors_using_JT(); + } } } --- old/src/hotspot/share/runtime/sharedRuntime.cpp 2020-05-06 20:09:12.000000000 -0400 +++ new/src/hotspot/share/runtime/sharedRuntime.cpp 2020-05-06 20:09:11.000000000 -0400 @@ -65,6 +65,7 @@ #include "runtime/javaCalls.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" +#include "runtime/synchronizer.hpp" #include "runtime/vframe.inline.hpp" #include "runtime/vframeArray.hpp" #include "utilities/copy.hpp" @@ -3070,10 +3071,15 @@ kptr2 = fr.next_monitor_in_interpreter_frame(kptr2) ) { if (kptr2->obj() != NULL) { // Avoid 'holes' in the monitor array BasicLock *lock = kptr2->lock(); - // Inflate so the displaced header becomes position-independent - if (lock->displaced_header().is_unlocked()) + // Inflate so the object's header no longer refers to the BasicLock. + if (lock->displaced_header().is_unlocked()) { + // The object is locked and the resulting ObjectMonitor* will also be + // locked so it can't be async deflated until ownership is dropped. + // See the big comment in basicLock.cpp: BasicLock::move_to(). ObjectSynchronizer::inflate_helper(kptr2->obj()); - // Now the displaced header is free to move + } + // Now the displaced header is free to move because the + // object's header no longer refers to it. buf[i++] = (intptr_t)lock->displaced_header().value(); buf[i++] = cast_from_oop(kptr2->obj()); } --- old/src/hotspot/share/runtime/synchronizer.cpp 2020-05-06 20:09:13.000000000 -0400 +++ new/src/hotspot/share/runtime/synchronizer.cpp 2020-05-06 20:09:13.000000000 -0400 @@ -37,11 +37,13 @@ #include "runtime/atomic.hpp" #include "runtime/biasedLocking.hpp" #include "runtime/handles.inline.hpp" +#include "runtime/handshake.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/objectMonitor.hpp" #include "runtime/objectMonitor.inline.hpp" #include "runtime/osThread.hpp" +#include "runtime/safepointMechanism.inline.hpp" #include "runtime/safepointVerifiers.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" @@ -118,6 +120,9 @@ // global list of blocks of monitors PaddedObjectMonitor* ObjectSynchronizer::g_block_list = NULL; +bool volatile ObjectSynchronizer::_is_async_deflation_requested = false; +bool volatile ObjectSynchronizer::_is_special_deflation_requested = false; +jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0; struct ObjectMonitorListGlobals { char _pad_prefix[OM_CACHE_LINE_SIZE]; @@ -134,14 +139,24 @@ ObjectMonitor* _in_use_list; DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(ObjectMonitor*)); + // Global ObjectMonitor wait list. Deflated ObjectMonitors wait on + // this list until after a handshake or a safepoint for platforms + // that don't support handshakes. After the handshake or safepoint, + // the deflated ObjectMonitors are prepended to free_list. + ObjectMonitor* _wait_list; + DEFINE_PAD_MINUS_SIZE(3, OM_CACHE_LINE_SIZE, sizeof(ObjectMonitor*)); + int _free_count; // # on free_list - DEFINE_PAD_MINUS_SIZE(3, OM_CACHE_LINE_SIZE, sizeof(int)); + DEFINE_PAD_MINUS_SIZE(4, OM_CACHE_LINE_SIZE, sizeof(int)); int _in_use_count; // # on in_use_list - DEFINE_PAD_MINUS_SIZE(4, OM_CACHE_LINE_SIZE, sizeof(int)); + DEFINE_PAD_MINUS_SIZE(5, OM_CACHE_LINE_SIZE, sizeof(int)); int _population; // # Extant -- in circulation - DEFINE_PAD_MINUS_SIZE(5, OM_CACHE_LINE_SIZE, sizeof(int)); + DEFINE_PAD_MINUS_SIZE(6, OM_CACHE_LINE_SIZE, sizeof(int)); + + int _wait_count; // # on wait_list + DEFINE_PAD_MINUS_SIZE(7, OM_CACHE_LINE_SIZE, sizeof(int)); }; static ObjectMonitorListGlobals om_list_globals; @@ -299,6 +314,15 @@ &om_list_globals._free_count); } +// Prepend a list of ObjectMonitors to om_list_globals._wait_list. +// 'tail' is the last ObjectMonitor in the list and there are 'count' +// on the list. Also updates om_list_globals._wait_count. +static void prepend_list_to_global_wait_list(ObjectMonitor* list, + ObjectMonitor* tail, int count) { + prepend_list_to_common(list, tail, count, &om_list_globals._wait_list, + &om_list_globals._wait_count); +} + // Prepend a list of ObjectMonitors to om_list_globals._in_use_list. // 'tail' is the last ObjectMonitor in the list and there are 'count' // on the list. Also updates om_list_globals._in_use_list. @@ -316,7 +340,7 @@ om_lock(m); // Lock m so we can safely update its next field. ObjectMonitor* cur = NULL; // Lock the list head to guard against races with a list walker - // thread: + // or async deflater thread (which only races in om_in_use_list): if ((cur = get_list_head_locked(list_p)) != NULL) { // List head is now locked so we can safely switch it. m->set_next_om(cur); // m now points to cur (and unlocks m) @@ -354,7 +378,7 @@ int* count_p) { ObjectMonitor* take = NULL; // Lock the list head to guard against races with a list walker - // thread: + // or async deflater thread (which only races in om_list_globals._free_list): if ((take = get_list_head_locked(list_p)) == NULL) { return NULL; // None are available. } @@ -463,11 +487,21 @@ assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant"); NoSafepointVerifier nsv; if (obj == NULL) return false; // Need to throw NPE + const markWord mark = obj->mark(); if (mark.has_monitor()) { ObjectMonitor* const m = mark.monitor(); - assert(m->object() == obj, "invariant"); + if (AsyncDeflateIdleMonitors) { + // An async deflation can race us before we manage to make the + // ObjectMonitor busy by setting the owner below. If we detect + // that race we just bail out to the slow-path here. + if (m->object() == NULL) { + return false; + } + } else { + assert(m->object() == obj, "invariant"); + } Thread* const owner = (Thread *) m->_owner; // Lock contention and Transactional Lock Elision (TLE) diagnostics @@ -547,7 +581,15 @@ // must be non-zero to avoid looking like a re-entrant lock, // and must not look locked either. lock->set_displaced_header(markWord::unused_mark()); - inflate(THREAD, obj(), inflate_cause_monitor_enter)->enter(THREAD); + // An async deflation can race after the inflate() call and before + // enter() can make the ObjectMonitor busy. enter() returns false if + // we have lost the race to async deflation and we simply try again. + while (true) { + ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_monitor_enter); + if (monitor->enter(THREAD)) { + return; + } + } } void ObjectSynchronizer::exit(oop object, BasicLock* lock, TRAPS) { @@ -596,7 +638,10 @@ } // We have to take the slow-path of possible inflation and then exit. - inflate(THREAD, object, inflate_cause_vm_internal)->exit(true, THREAD); + // The ObjectMonitor* can't be async deflated until ownership is + // dropped inside exit() and the ObjectMonitor* must be !is_busy(). + ObjectMonitor* monitor = inflate(THREAD, object, inflate_cause_vm_internal); + monitor->exit(true, THREAD); } // ----------------------------------------------------------------------------- @@ -617,9 +662,11 @@ assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now"); } + // The ObjectMonitor* can't be async deflated until ownership is + // dropped inside exit() and the ObjectMonitor* must be !is_busy(). ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal); - - return monitor->complete_exit(THREAD); + intptr_t ret_code = monitor->complete_exit(THREAD); + return ret_code; } // NOTE: must use heavy weight monitor to handle complete_exit/reenter() @@ -629,10 +676,18 @@ assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now"); } - ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal); - - monitor->reenter(recursions, THREAD); + // An async deflation can race after the inflate() call and before + // reenter() -> enter() can make the ObjectMonitor busy. reenter() -> + // enter() returns false if we have lost the race to async deflation + // and we simply try again. + while (true) { + ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal); + if (monitor->reenter(recursions, THREAD)) { + return; + } + } } + // ----------------------------------------------------------------------------- // JNI locks on java objects // NOTE: must use heavy weight monitor to handle jni monitor enter @@ -643,7 +698,15 @@ assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now"); } THREAD->set_current_pending_monitor_is_from_java(false); - inflate(THREAD, obj(), inflate_cause_jni_enter)->enter(THREAD); + // An async deflation can race after the inflate() call and before + // enter() can make the ObjectMonitor busy. enter() returns false if + // we have lost the race to async deflation and we simply try again. + while (true) { + ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_jni_enter); + if (monitor->enter(THREAD)) { + break; + } + } THREAD->set_current_pending_monitor_is_from_java(true); } @@ -656,6 +719,8 @@ } assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now"); + // The ObjectMonitor* can't be async deflated until ownership is + // dropped inside exit() and the ObjectMonitor* must be !is_busy(). ObjectMonitor* monitor = inflate(THREAD, obj, inflate_cause_jni_exit); // If this thread has locked the object, exit the monitor. We // intentionally do not use CHECK here because we must exit the @@ -697,6 +762,9 @@ if (millis < 0) { THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative"); } + // The ObjectMonitor* can't be async deflated because the _waiters + // field is incremented before ownership is dropped and decremented + // after ownership is regained. ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_wait); DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis); @@ -706,7 +774,8 @@ // that's fixed we can uncomment the following line, remove the call // and change this function back into a "void" func. // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD); - return dtrace_waited_probe(monitor, obj, THREAD); + int ret_code = dtrace_waited_probe(monitor, obj, THREAD); + return ret_code; } void ObjectSynchronizer::wait_uninterruptibly(Handle obj, jlong millis, TRAPS) { @@ -717,7 +786,11 @@ if (millis < 0) { THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative"); } - inflate(THREAD, obj(), inflate_cause_wait)->wait(millis, false, THREAD); + // The ObjectMonitor* can't be async deflated because the _waiters + // field is incremented before ownership is dropped and decremented + // after ownership is regained. + ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_wait); + monitor->wait(millis, false, THREAD); } void ObjectSynchronizer::notify(Handle obj, TRAPS) { @@ -730,7 +803,10 @@ if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) { return; } - inflate(THREAD, obj(), inflate_cause_notify)->notify(THREAD); + // The ObjectMonitor* can't be async deflated until ownership is + // dropped by the calling thread. + ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_notify); + monitor->notify(THREAD); } // NOTE: see comment of notify() @@ -744,7 +820,10 @@ if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) { return; } - inflate(THREAD, obj(), inflate_cause_notify)->notifyAll(THREAD); + // The ObjectMonitor* can't be async deflated until ownership is + // dropped by the calling thread. + ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_notify); + monitor->notifyAll(THREAD); } // ----------------------------------------------------------------------------- @@ -937,84 +1016,98 @@ assert(Universe::verify_in_progress() || DumpSharedSpaces || ((JavaThread *)self)->thread_state() != _thread_blocked, "invariant"); - ObjectMonitor* monitor = NULL; - markWord temp, test; - intptr_t hash; - markWord mark = read_stable_mark(obj); + while (true) { + ObjectMonitor* monitor = NULL; + markWord temp, test; + intptr_t hash; + markWord mark = read_stable_mark(obj); - // object should remain ineligible for biased locking - assert(!mark.has_bias_pattern(), "invariant"); + // object should remain ineligible for biased locking + assert(!mark.has_bias_pattern(), "invariant"); - if (mark.is_neutral()) { // if this is a normal header + if (mark.is_neutral()) { // if this is a normal header + hash = mark.hash(); + if (hash != 0) { // if it has a hash, just return it + return hash; + } + hash = get_next_hash(self, obj); // get a new hash + temp = mark.copy_set_hash(hash); // merge the hash into header + // try to install the hash + test = obj->cas_set_mark(temp, mark); + if (test == mark) { // if the hash was installed, return it + return hash; + } + // Failed to install the hash. It could be that another thread + // installed the hash just before our attempt or inflation has + // occurred or... so we fall thru to inflate the monitor for + // stability and then install the hash. + } else if (mark.has_monitor()) { + // The first stage of a racing async deflation won't affect the + // hash value if this ObjectMonitor happens to already have one. + monitor = mark.monitor(); + temp = monitor->header(); + assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value()); + hash = temp.hash(); + if (hash != 0) { // if it has a hash, just return it + return hash; + } + // Fall thru so we only have one place that installs the hash in + // the ObjectMonitor. + } else if (self->is_lock_owned((address)mark.locker())) { + // This is a stack lock owned by the calling thread so fetch the + // displaced markWord from the BasicLock on the stack. + temp = mark.displaced_mark_helper(); + assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value()); + hash = temp.hash(); + if (hash != 0) { // if it has a hash, just return it + return hash; + } + // WARNING: + // The displaced header in the BasicLock on a thread's stack + // is strictly immutable. It CANNOT be changed in ANY cases. + // So we have to inflate the stack lock into an ObjectMonitor + // even if the current thread owns the lock. The BasicLock on + // a thread's stack can be asynchronously read by other threads + // during an inflate() call so any change to that stack memory + // may not propagate to other threads correctly. + } + + // Inflate the monitor to set the hash. + + // An async deflation can race after the inflate() call and before we + // can update the ObjectMonitor's header with the hash value below. + monitor = inflate(self, obj, inflate_cause_hash_code); + // Load ObjectMonitor's header/dmw field and see if it has a hash. + mark = monitor->header(); + assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value()); hash = mark.hash(); - if (hash != 0) { // if it has a hash, just return it - return hash; - } - hash = get_next_hash(self, obj); // get a new hash - temp = mark.copy_set_hash(hash); // merge the hash into header - // try to install the hash - test = obj->cas_set_mark(temp, mark); - if (test == mark) { // if the hash was installed, return it - return hash; - } - // Failed to install the hash. It could be that another thread - // installed the hash just before our attempt or inflation has - // occurred or... so we fall thru to inflate the monitor for - // stability and then install the hash. - } else if (mark.has_monitor()) { - monitor = mark.monitor(); - temp = monitor->header(); - assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value()); - hash = temp.hash(); - if (hash != 0) { // if it has a hash, just return it - return hash; - } - // Fall thru so we only have one place that installs the hash in - // the ObjectMonitor. - } else if (self->is_lock_owned((address)mark.locker())) { - // This is a stack lock owned by the calling thread so fetch the - // displaced markWord from the BasicLock on the stack. - temp = mark.displaced_mark_helper(); - assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value()); - hash = temp.hash(); - if (hash != 0) { // if it has a hash, just return it - return hash; - } - // WARNING: - // The displaced header in the BasicLock on a thread's stack - // is strictly immutable. It CANNOT be changed in ANY cases. - // So we have to inflate the stack lock into an ObjectMonitor - // even if the current thread owns the lock. The BasicLock on - // a thread's stack can be asynchronously read by other threads - // during an inflate() call so any change to that stack memory - // may not propagate to other threads correctly. - } - - // Inflate the monitor to set the hash. - monitor = inflate(self, obj, inflate_cause_hash_code); - // Load ObjectMonitor's header/dmw field and see if it has a hash. - mark = monitor->header(); - assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value()); - hash = mark.hash(); - if (hash == 0) { // if it does not have a hash - hash = get_next_hash(self, obj); // get a new hash - temp = mark.copy_set_hash(hash); // merge the hash into header - assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value()); - uintptr_t v = Atomic::cmpxchg((volatile uintptr_t*)monitor->header_addr(), mark.value(), temp.value()); - test = markWord(v); - if (test != mark) { - // The attempt to update the ObjectMonitor's header/dmw field - // did not work. This can happen if another thread managed to - // merge in the hash just before our cmpxchg(). - // If we add any new usages of the header/dmw field, this code - // will need to be updated. - hash = test.hash(); - assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value()); - assert(hash != 0, "should only have lost the race to a thread that set a non-zero hash"); + if (hash == 0) { // if it does not have a hash + hash = get_next_hash(self, obj); // get a new hash + temp = mark.copy_set_hash(hash); // merge the hash into header + assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value()); + uintptr_t v = Atomic::cmpxchg((volatile uintptr_t*)monitor->header_addr(), mark.value(), temp.value()); + test = markWord(v); + if (test != mark) { + // The attempt to update the ObjectMonitor's header/dmw field + // did not work. This can happen if another thread managed to + // merge in the hash just before our cmpxchg(). + // If we add any new usages of the header/dmw field, this code + // will need to be updated. + hash = test.hash(); + assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value()); + assert(hash != 0, "should only have lost the race to a thread that set a non-zero hash"); + } + if (monitor->is_being_async_deflated()) { + // If we detect that async deflation has occurred, then we + // simply retry so that the hash value can be stored in either + // the object's header or in the re-inflated ObjectMonitor's + // header as appropriate. + continue; + } } + // We finally get the hash. + return hash; } - // We finally get the hash. - return hash; } // Deprecated -- use FastHashCode() instead. @@ -1042,6 +1135,8 @@ } // Contended case, header points to ObjectMonitor (tagged pointer) if (mark.has_monitor()) { + // The first stage of async deflation does not affect any field + // used by this comparison so the ObjectMonitor* is usable here. ObjectMonitor* monitor = mark.monitor(); return monitor->is_entered(thread) != 0; } @@ -1083,9 +1178,12 @@ // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor. // The Object:ObjectMonitor relationship is stable as long as we're - // not at a safepoint. + // not at a safepoint and AsyncDeflateIdleMonitors is false. if (mark.has_monitor()) { - void* owner = mark.monitor()->_owner; + // The first stage of async deflation does not affect any field + // used by this comparison so the ObjectMonitor* is usable here. + ObjectMonitor* monitor = mark.monitor(); + void* owner = monitor->owner(); if (owner == NULL) return owner_none; return (owner == self || self->is_lock_owned((address)owner)) ? owner_self : owner_other; @@ -1119,6 +1217,8 @@ // Contended case, header points to ObjectMonitor (tagged pointer) else if (mark.has_monitor()) { + // The first stage of async deflation does not affect any field + // used by this comparison so the ObjectMonitor* is usable here. ObjectMonitor* monitor = mark.monitor(); assert(monitor != NULL, "monitor should be non-null"); owner = (address) monitor->owner(); @@ -1145,9 +1245,14 @@ assert(block->object() == CHAINMARKER, "must be a block header"); for (int i = _BLOCKSIZE - 1; i > 0; i--) { ObjectMonitor* mid = (ObjectMonitor *)(block + i); - oop object = (oop)mid->object(); - if (object != NULL) { + if (!mid->is_free() && mid->object() != NULL) { // Only process with closure if the object is set. + + // monitors_iterate() is only called at a safepoint or when the + // target thread is suspended or when the target thread is + // operating on itself. The closures are only interested in an + // owned ObjectMonitor and ownership cannot be dropped under the + // calling contexts so the ObjectMonitor cannot be async deflated. closure->do_monitor(mid); } } @@ -1163,7 +1268,8 @@ return false; } if (MonitorUsedDeflationThreshold > 0) { - int monitors_used = population - Atomic::load(&om_list_globals._free_count); + int monitors_used = population - Atomic::load(&om_list_globals._free_count) - + Atomic::load(&om_list_globals._wait_count); int monitor_usage = (monitors_used * 100LL) / population; return monitor_usage > MonitorUsedDeflationThreshold; } @@ -1177,12 +1283,32 @@ return mx > 0 && cnt > mx; } -bool ObjectSynchronizer::is_cleanup_needed() { - if (monitors_used_above_threshold()) { - // Too many monitors in use. +bool ObjectSynchronizer::is_async_deflation_needed() { + if (!AsyncDeflateIdleMonitors) { + return false; + } + if (is_async_deflation_requested()) { + // Async deflation request. + return true; + } + if (AsyncDeflationInterval > 0 && + time_since_last_async_deflation_ms() > AsyncDeflationInterval && + monitors_used_above_threshold()) { + // It's been longer than our specified deflate interval and there + // are too many monitors in use. We don't deflate more frequently + // than AsyncDeflationInterval (unless is_async_deflation_requested) + // in order to not swamp the ServiceThread. + _last_async_deflation_time_ns = os::javaTimeNanos(); return true; } - return needs_monitor_scavenge(); + int monitors_used = Atomic::load(&om_list_globals._population) - + Atomic::load(&om_list_globals._free_count) - + Atomic::load(&om_list_globals._wait_count); + if (is_MonitorBound_exceeded(monitors_used)) { + // Not enough ObjectMonitors on the global free list. + return true; + } + return false; } bool ObjectSynchronizer::needs_monitor_scavenge() { @@ -1193,6 +1319,26 @@ return false; } +bool ObjectSynchronizer::is_safepoint_deflation_needed() { + if (!AsyncDeflateIdleMonitors) { + if (monitors_used_above_threshold()) { + // Too many monitors in use. + return true; + } + return needs_monitor_scavenge(); + } + if (is_special_deflation_requested()) { + // For AsyncDeflateIdleMonitors only do a safepoint deflation + // if there is a special deflation request. + return true; + } + return false; +} + +jlong ObjectSynchronizer::time_since_last_async_deflation_ms() { + return (os::javaTimeNanos() - _last_async_deflation_time_ns) / (NANOUNITS / MILLIUNITS); +} + void ObjectSynchronizer::oops_do(OopClosure* f) { // We only scan the global used list here (for moribund threads), and // the thread-local monitors in Thread::oops_do(). @@ -1226,7 +1372,7 @@ // ----------------------- // Inflation unlinks monitors from om_list_globals._free_list or a per-thread // free list and associates them with objects. Deflation -- which occurs at -// STW-time -- disassociates idle monitors from objects. +// STW-time or asynchronously -- disassociates idle monitors from objects. // Such scavenged monitors are returned to the om_list_globals._free_list. // // ObjectMonitors reside in type-stable memory (TSM) and are immortal. @@ -1242,6 +1388,7 @@ // // If MonitorBound is not set (<= 0), MonitorBound checks are disabled. // +// When safepoint deflation is being used (!AsyncDeflateIdleMonitors): // The monitor pool is grow-only. We scavenge at STW safepoint-time, but the // the rate of scavenging is driven primarily by GC. As such, we can find // an inordinate number of monitors in circulation. @@ -1254,13 +1401,26 @@ // we'll incur more safepoints, which are harmful to performance. // See also: GuaranteedSafepointInterval // -// If MonitorBound is set, the boundry applies to +// When safepoint deflation is being used and MonitorBound is set, the +// boundry applies to // (om_list_globals._population - om_list_globals._free_count) // i.e., if there are not enough ObjectMonitors on the global free list, // then a safepoint deflation is induced. Picking a good MonitorBound value // is non-trivial. +// +// When async deflation is being used: +// The monitor pool is still grow-only. Async deflation is requested +// by a safepoint's cleanup phase or by the ServiceThread at periodic +// intervals when is_async_deflation_needed() returns true. In +// addition to other policies that are checked, if there are not +// enough ObjectMonitors on the global free list, then +// is_async_deflation_needed() will return true. The ServiceThread +// calls deflate_global_idle_monitors_using_JT() and also calls +// deflate_per_thread_idle_monitors_using_JT() as needed. static void InduceScavenge(Thread* self, const char * Whence) { + assert(!AsyncDeflateIdleMonitors, "is not used by async deflation"); + // Induce STW safepoint to trim monitors // Ultimately, this results in a call to deflate_idle_monitors() in the near future. // More precisely, trigger a cleanup safepoint as the number @@ -1293,6 +1453,7 @@ m = take_from_start_of_om_free_list(self); if (m != NULL) { guarantee(m->object() == NULL, "invariant"); + m->set_allocation_state(ObjectMonitor::New); prepend_to_om_in_use_list(self, m); return m; } @@ -1310,13 +1471,36 @@ break; // No more are available. } guarantee(take->object() == NULL, "invariant"); + if (AsyncDeflateIdleMonitors) { + // We allowed 3 field values to linger during async deflation. + // Clear or restore them as appropriate. + take->set_header(markWord::zero()); + // DEFLATER_MARKER is the only non-NULL value we should see here. + take->try_set_owner_from(DEFLATER_MARKER, NULL); + if (take->contentions() < 0) { + // Add back max_jint to restore the contentions field to its + // proper value. + Atomic::add(&take->_contentions, max_jint); + +#ifdef ASSERT + jint l_contentions = take->contentions(); +#endif + assert(l_contentions >= 0, "must not be negative: l_contentions=%d, contentions=%d", + l_contentions, take->contentions()); + } + } take->Recycle(); + // Since we're taking from the global free-list, take must be Free. + // om_release() also sets the allocation state to Free because it + // is called from other code paths. + assert(take->is_free(), "invariant"); om_release(self, take, false); } self->om_free_provision += 1 + (self->om_free_provision / 2); if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE; - if (is_MonitorBound_exceeded(Atomic::load(&om_list_globals._population) - + if (!AsyncDeflateIdleMonitors && + is_MonitorBound_exceeded(Atomic::load(&om_list_globals._population) - Atomic::load(&om_list_globals._free_count))) { // Not enough ObjectMonitors on the global free list. // We can't safely induce a STW safepoint from om_alloc() as our thread @@ -1353,6 +1537,7 @@ for (int i = 1; i < _BLOCKSIZE; i++) { temp[i].set_next_om((ObjectMonitor*)&temp[i + 1]); + assert(temp[i].is_free(), "invariant"); } // terminate the last monitor as the end of list @@ -1378,8 +1563,8 @@ // // Key constraint: all ObjectMonitors on a thread's free list and the global // free list must have their object field set to null. This prevents the -// scavenger -- deflate_monitor_list() -- from reclaiming them while we -// are trying to release them. +// scavenger -- deflate_monitor_list() or deflate_monitor_list_using_JT() +// -- from reclaiming them while we are trying to release them. void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m, bool from_per_thread_alloc) { @@ -1392,6 +1577,7 @@ fatal("freeing in-use monitor: %s, recursions=" INTX_FORMAT, m->is_busy_to_string(&ss), m->_recursions); } + m->set_allocation_state(ObjectMonitor::Free); // _next_om is used for both per-thread in-use and free lists so // we have to remove 'm' from the in-use list first (as needed). if (from_per_thread_alloc) { @@ -1399,12 +1585,12 @@ ObjectMonitor* mid = NULL; ObjectMonitor* next = NULL; - // This list walk can only race with another list walker since - // deflation can only happen at a safepoint so we don't have to - // worry about an ObjectMonitor being removed from this list - // while we are walking it. + // This list walk can race with another list walker or with async + // deflation so we have to worry about an ObjectMonitor being + // removed from this list while we are walking it. - // Lock the list head to avoid racing with another list walker. + // Lock the list head to avoid racing with another list walker + // or with async deflation. if ((mid = get_list_head_locked(&self->om_in_use_list)) == NULL) { fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self)); } @@ -1420,7 +1606,9 @@ // 'm' matches next after the list head and we already have the list // head locked so set mid to what we are extracting: mid = next; - // Lock mid to prevent races with a list walker: + // Lock mid to prevent races with a list walker or an async + // deflater thread that's ahead of us. The locked list head + // prevents races from behind us. om_lock(mid); // Update next to what follows mid (if anything): next = unmarked_next(mid); @@ -1429,16 +1617,19 @@ self->om_in_use_list->set_next_om(next); } else { // We have to search the list to find 'm'. - om_unlock(mid); // unlock the list head guarantee(next != NULL, "thread=" INTPTR_FORMAT ": om_in_use_list=" INTPTR_FORMAT " is too short.", p2i(self), p2i(self->om_in_use_list)); // Our starting anchor is next after the list head which is the // last ObjectMonitor we checked: ObjectMonitor* anchor = next; + // Lock anchor to prevent races with a list walker or an async + // deflater thread that's ahead of us. The locked list head + // prevents races from behind us. + om_lock(anchor); + om_unlock(mid); // Unlock the list head now that anchor is locked. while ((mid = unmarked_next(anchor)) != NULL) { if (m == mid) { // We found 'm' on the per-thread in-use list so extract it. - om_lock(anchor); // Lock the anchor so we can safely modify it. // Update next to what follows mid (if anything): next = unmarked_next(mid); // Switch next after the anchor to new next which unlocks the @@ -1446,7 +1637,13 @@ anchor->set_next_om(next); break; } else { - anchor = mid; + // Lock the next anchor to prevent races with a list walker + // or an async deflater thread that's ahead of us. The locked + // current anchor prevents races from behind us. + om_lock(mid); + // Unlock current anchor now that next anchor is locked: + om_unlock(anchor); + anchor = mid; // Advance to new anchor and try again. } } } @@ -1467,6 +1664,7 @@ } prepend_to_om_free_list(self, m); + guarantee(m->is_free(), "invariant"); } // Return ObjectMonitors on a moribund thread's free and in-use @@ -1481,6 +1679,11 @@ // scanned by a GC safepoint, either via Thread::oops_do() (before // om_flush() is called) or via ObjectSynchronizer::oops_do() (after // om_flush() is called). +// +// With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT() +// and deflate_per_thread_idle_monitors_using_JT() (in another thread) can +// run at the same time as om_flush() so we have to follow a careful +// protocol to prevent list corruption. void ObjectSynchronizer::om_flush(Thread* self) { // Process the per-thread in-use list first to be consistent. @@ -1489,8 +1692,11 @@ ObjectMonitor* in_use_tail = NULL; NoSafepointVerifier nsv; - // This function can race with a list walker thread so we lock the - // list head to prevent confusion. + // This function can race with a list walker or with an async + // deflater thread so we lock the list head to prevent confusion. + // An async deflater thread checks to see if the target thread + // is exiting, but if it has made it past that check before we + // started exiting, then it is racing to get to the in-use list. if ((in_use_list = get_list_head_locked(&self->om_in_use_list)) != NULL) { // At this point, we have locked the in-use list head so a racing // thread cannot come in after us. However, a racing thread could @@ -1505,21 +1711,33 @@ // already locked (by this thread): in_use_tail = in_use_list; in_use_count++; - for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL; cur_om = unmarked_next(cur_om)) { + for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL;) { if (is_locked(cur_om)) { - // cur_om is locked so there must be a racing walker thread ahead - // of us so we'll give it a chance to finish. + // cur_om is locked so there must be a racing walker or async + // deflater thread ahead of us so we'll give it a chance to finish. while (is_locked(cur_om)) { os::naked_short_sleep(1); } + // Refetch the possibly changed next field and try again. + cur_om = unmarked_next(in_use_tail); + continue; + } + if (cur_om->is_free()) { + // cur_om was deflated and the allocation state was changed + // to Free while it was locked. We happened to see it just + // after it was unlocked (and added to the free list). + // Refetch the possibly changed next field and try again. + cur_om = unmarked_next(in_use_tail); + continue; } in_use_tail = cur_om; in_use_count++; + cur_om = unmarked_next(cur_om); } guarantee(in_use_tail != NULL, "invariant"); int l_om_in_use_count = Atomic::load(&self->om_in_use_count); - assert(l_om_in_use_count == in_use_count, "in-use counts don't match: " - "l_om_in_use_count=%d, in_use_count=%d", l_om_in_use_count, in_use_count); + ADIM_guarantee(l_om_in_use_count == in_use_count, "in-use counts don't match: " + "l_om_in_use_count=%d, in_use_count=%d", l_om_in_use_count, in_use_count); Atomic::store(&self->om_in_use_count, 0); // Clear the in-use list head (which also unlocks it): Atomic::store(&self->om_in_use_list, (ObjectMonitor*)NULL); @@ -1561,8 +1779,8 @@ } guarantee(free_tail != NULL, "invariant"); int l_om_free_count = Atomic::load(&self->om_free_count); - assert(l_om_free_count == free_count, "free counts don't match: " - "l_om_free_count=%d, free_count=%d", l_om_free_count, free_count); + ADIM_guarantee(l_om_free_count == free_count, "free counts don't match: " + "l_om_free_count=%d, free_count=%d", l_om_free_count, free_count); Atomic::store(&self->om_free_count, 0); Atomic::store(&self->om_free_list, (ObjectMonitor*)NULL); om_unlock(free_list); @@ -1607,15 +1825,17 @@ void ObjectSynchronizer::inflate_helper(oop obj) { markWord mark = obj->mark(); if (mark.has_monitor()) { - assert(ObjectSynchronizer::verify_objmon_isinpool(mark.monitor()), "monitor is invalid"); - assert(mark.monitor()->header().is_neutral(), "monitor must record a good object header"); + ObjectMonitor* monitor = mark.monitor(); + assert(ObjectSynchronizer::verify_objmon_isinpool(monitor), "monitor=" INTPTR_FORMAT " is invalid", p2i(monitor)); + markWord dmw = monitor->header(); + assert(dmw.is_neutral(), "sanity check: header=" INTPTR_FORMAT, dmw.value()); return; } - inflate(Thread::current(), obj, inflate_cause_vm_internal); + (void)inflate(Thread::current(), obj, inflate_cause_vm_internal); } -ObjectMonitor* ObjectSynchronizer::inflate(Thread* self, - oop object, const InflateCause cause) { +ObjectMonitor* ObjectSynchronizer::inflate(Thread* self, oop object, + const InflateCause cause) { // Inflate mutates the heap ... // Relaxing assertion for bug 6320749. assert(Universe::verify_in_progress() || @@ -1639,7 +1859,7 @@ ObjectMonitor* inf = mark.monitor(); markWord dmw = inf->header(); assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value()); - assert(inf->object() == object, "invariant"); + assert(AsyncDeflateIdleMonitors || inf->object() == object, "invariant"); assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid"); return inf; } @@ -1687,6 +1907,7 @@ markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark); if (cmp != mark) { + // om_release() will reset the allocation state from New to Free. om_release(self, m, true); continue; // Interference -- just retry } @@ -1724,7 +1945,7 @@ markWord dmw = mark.displaced_mark_helper(); // Catch if the object's header is not neutral (not locked and // not marked is what we care about here). - assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value()); + ADIM_guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value()); // Setup monitor fields to proper values -- prepare the monitor m->set_header(dmw); @@ -1734,7 +1955,11 @@ // Note that a thread can inflate an object // that it has stack-locked -- as might happen in wait() -- directly // with CAS. That is, we can avoid the xchg-NULL .... ST idiom. - m->set_owner_from(NULL, mark.locker()); + if (AsyncDeflateIdleMonitors) { + m->set_owner_from(NULL, DEFLATER_MARKER, mark.locker()); + } else { + m->set_owner_from(NULL, mark.locker()); + } m->set_object(object); // TODO-FIXME: assert BasicLock->dhw != 0. @@ -1743,6 +1968,11 @@ guarantee(object->mark() == markWord::INFLATING(), "invariant"); object->release_set_mark(markWord::encode(m)); + // Once ObjectMonitor is configured and the object is associated + // with the ObjectMonitor, it is safe to allow async deflation: + assert(m->is_new(), "freshly allocated monitor must be new"); + m->set_allocation_state(ObjectMonitor::Old); + // Hopefully the performance counters are allocated on distinct cache lines // to avoid false sharing on MP systems ... OM_PERFDATA_OP(Inflations, inc()); @@ -1769,11 +1999,15 @@ // Catch if the object's header is not neutral (not locked and // not marked is what we care about here). - assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value()); + ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value()); ObjectMonitor* m = om_alloc(self); // prepare m for installation - set monitor to initial state m->Recycle(); m->set_header(mark); + if (AsyncDeflateIdleMonitors) { + // DEFLATER_MARKER is the only non-NULL value we should see here. + m->try_set_owner_from(DEFLATER_MARKER, NULL); + } m->set_object(object); m->_Responsible = NULL; m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class @@ -1782,6 +2016,7 @@ m->set_header(markWord::zero()); m->set_object(NULL); m->Recycle(); + // om_release() will reset the allocation state from New to Free. om_release(self, m, true); m = NULL; continue; @@ -1790,6 +2025,11 @@ // live-lock -- "Inflated" is an absorbing state. } + // Once the ObjectMonitor is configured and object is associated + // with the ObjectMonitor, it is safe to allow async deflation: + assert(m->is_new(), "freshly allocated monitor must be new"); + m->set_allocation_state(ObjectMonitor::Old); + // Hopefully the performance counters are allocated on distinct // cache lines to avoid false sharing on MP systems ... OM_PERFDATA_OP(Inflations, inc()); @@ -1809,6 +2049,7 @@ // We maintain a list of in-use monitors for each thread. // +// For safepoint based deflation: // deflate_thread_local_monitors() scans a single thread's in-use list, while // deflate_idle_monitors() scans only a global list of in-use monitors which // is populated only as a thread dies (see om_flush()). @@ -1827,6 +2068,40 @@ // typically drives the scavenge rate. Large heaps can mean infrequent GC, // which in turn can mean large(r) numbers of ObjectMonitors in circulation. // This is an unfortunate aspect of this design. +// +// For async deflation: +// If a special deflation request is made, then the safepoint based +// deflation mechanism is used. Otherwise, an async deflation request +// is registered with the ServiceThread and it is notified. + +void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* counters) { + assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); + + // The per-thread in-use lists are handled in + // ParallelSPCleanupThreadClosure::do_thread(). + + if (!AsyncDeflateIdleMonitors || is_special_deflation_requested()) { + // Use the older mechanism for the global in-use list or if a + // special deflation has been requested before the safepoint. + ObjectSynchronizer::deflate_idle_monitors(counters); + return; + } + + log_debug(monitorinflation)("requesting async deflation of idle monitors."); + // Request deflation of idle monitors by the ServiceThread: + set_is_async_deflation_requested(true); + MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag); + ml.notify_all(); + + if (log_is_enabled(Debug, monitorinflation)) { + // exit_globals()'s call to audit_and_print_stats() is done + // at the Info level and not at a safepoint. + // For safepoint based deflation, audit_and_print_stats() is called + // in ObjectSynchronizer::finish_deflate_idle_monitors() at the + // Debug level at a safepoint. + ObjectSynchronizer::audit_and_print_stats(false /* on_exit */); + } +} // Deflate a single monitor if not in-use // Return true if deflated, false if in-use @@ -1862,10 +2137,16 @@ // Restore the header back to obj obj->release_set_mark(dmw); + if (AsyncDeflateIdleMonitors) { + // clear() expects the owner field to be NULL. + // DEFLATER_MARKER is the only non-NULL value we should see here. + mid->try_set_owner_from(DEFLATER_MARKER, NULL); + } mid->clear(); assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT, p2i(mid->object())); + assert(mid->is_free(), "invariant"); // Move the deflated ObjectMonitor to the working free list // defined by free_head_p and free_tail_p. @@ -1894,6 +2175,143 @@ return deflated; } +// Deflate the specified ObjectMonitor if not in-use using a JavaThread. +// Returns true if it was deflated and false otherwise. +// +// The async deflation protocol sets owner to DEFLATER_MARKER and +// makes contentions negative as signals to contending threads that +// an async deflation is in progress. There are a number of checks +// as part of the protocol to make sure that the calling thread has +// not lost the race to a contending thread. +// +// The ObjectMonitor has been successfully async deflated when: +// (owner == DEFLATER_MARKER && contentions < 0) +// Contending threads that see those values know to retry their operation. +// +bool ObjectSynchronizer::deflate_monitor_using_JT(ObjectMonitor* mid, + ObjectMonitor** free_head_p, + ObjectMonitor** free_tail_p) { + assert(AsyncDeflateIdleMonitors, "sanity check"); + assert(Thread::current()->is_Java_thread(), "precondition"); + // A newly allocated ObjectMonitor should not be seen here so we + // avoid an endless inflate/deflate cycle. + assert(mid->is_old(), "must be old: allocation_state=%d", + (int) mid->allocation_state()); + + if (mid->is_busy()) { + // Easy checks are first - the ObjectMonitor is busy so no deflation. + return false; + } + + // Set a NULL owner to DEFLATER_MARKER to force any contending thread + // through the slow path. This is just the first part of the async + // deflation dance. + if (mid->try_set_owner_from(NULL, DEFLATER_MARKER) != NULL) { + // The owner field is no longer NULL so we lost the race since the + // ObjectMonitor is now busy. + return false; + } + + if (mid->contentions() > 0 || mid->_waiters != 0) { + // Another thread has raced to enter the ObjectMonitor after + // mid->is_busy() above or has already entered and waited on + // it which makes it busy so no deflation. Restore owner to + // NULL if it is still DEFLATER_MARKER. + mid->try_set_owner_from(DEFLATER_MARKER, NULL); + return false; + } + + // Make contentions negative to force any contending threads to + // retry. This is the second part of the async deflation dance. + if (Atomic::cmpxchg(&mid->_contentions, (jint)0, -max_jint) != 0) { + // The contentions was no longer 0 so we lost the race since the + // ObjectMonitor is now busy. Restore owner to NULL if it is + // still DEFLATER_MARKER: + mid->try_set_owner_from(DEFLATER_MARKER, NULL); + return false; + } + + // If owner is still DEFLATER_MARKER, then we have successfully + // signaled any contending threads to retry. + if (!mid->owner_is_DEFLATER_MARKER()) { + // If it is not, then we have lost the race to an entering thread + // and the ObjectMonitor is now busy. This is the third and final + // part of the async deflation dance. + // Note: This owner check solves the ABA problem with contentions + // where another thread acquired the ObjectMonitor, finished + // using it and restored contentions to zero. + + // Add back max_jint to restore the contentions field to its + // proper value (which may not be what we saw above): + Atomic::add(&mid->_contentions, max_jint); + +#ifdef ASSERT + jint l_contentions = mid->contentions(); +#endif + assert(l_contentions >= 0, "must not be negative: l_contentions=%d, contentions=%d", + l_contentions, mid->contentions()); + return false; + } + + // Sanity checks for the races: + guarantee(mid->contentions() < 0, "must be negative: contentions=%d", + mid->contentions()); + guarantee(mid->_waiters == 0, "must be 0: waiters=%d", mid->_waiters); + guarantee(mid->_cxq == NULL, "must be no contending threads: cxq=" + INTPTR_FORMAT, p2i(mid->_cxq)); + guarantee(mid->_EntryList == NULL, + "must be no entering threads: EntryList=" INTPTR_FORMAT, + p2i(mid->_EntryList)); + + const oop obj = (oop) mid->object(); + if (log_is_enabled(Trace, monitorinflation)) { + ResourceMark rm; + log_trace(monitorinflation)("deflate_monitor_using_JT: " + "object=" INTPTR_FORMAT ", mark=" + INTPTR_FORMAT ", type='%s'", + p2i(obj), obj->mark().value(), + obj->klass()->external_name()); + } + + // Install the old mark word if nobody else has already done it. + mid->install_displaced_markword_in_object(obj); + mid->clear_using_JT(); + + assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT, + p2i(mid->object())); + assert(mid->is_free(), "must be free: allocation_state=%d", + (int)mid->allocation_state()); + + // Move the deflated ObjectMonitor to the working free list + // defined by free_head_p and free_tail_p. + if (*free_head_p == NULL) { + // First one on the list. + *free_head_p = mid; + } + if (*free_tail_p != NULL) { + // We append to the list so the caller can use mid->_next_om + // to fix the linkages in its context. + ObjectMonitor* prevtail = *free_tail_p; + // prevtail should have been cleaned up by the caller: +#ifdef ASSERT + ObjectMonitor* l_next_om = unmarked_next(prevtail); +#endif + assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om)); + om_lock(prevtail); + prevtail->set_next_om(mid); // prevtail now points to mid (and is unlocked) + } + *free_tail_p = mid; + + // At this point, mid->_next_om still refers to its current + // value and another ObjectMonitor's _next_om field still + // refers to this ObjectMonitor. Those linkages have to be + // cleaned up by the caller who has the complete context. + + // We leave owner == DEFLATER_MARKER and contentions < 0 + // to force any racing threads to retry. + return true; // Success, ObjectMonitor has been deflated. +} + // Walk a given monitor list, and deflate idle monitors. // The given list could be a per-thread list or a global list. // @@ -1944,6 +2362,147 @@ return deflated_count; } +// Walk a given ObjectMonitor list and deflate idle ObjectMonitors using +// a JavaThread. Returns the number of deflated ObjectMonitors. The given +// list could be a per-thread in-use list or the global in-use list. +// If a safepoint has started, then we save state via saved_mid_in_use_p +// and return to the caller to honor the safepoint. +// +int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor** list_p, + int* count_p, + ObjectMonitor** free_head_p, + ObjectMonitor** free_tail_p, + ObjectMonitor** saved_mid_in_use_p) { + assert(AsyncDeflateIdleMonitors, "sanity check"); + JavaThread* self = JavaThread::current(); + + ObjectMonitor* cur_mid_in_use = NULL; + ObjectMonitor* mid = NULL; + ObjectMonitor* next = NULL; + ObjectMonitor* next_next = NULL; + int deflated_count = 0; + NoSafepointVerifier nsv; + + // We use the more complicated lock-cur_mid_in_use-and-mid-as-we-go + // protocol because om_release() can do list deletions in parallel; + // this also prevents races with a list walker thread. We also + // lock-next-next-as-we-go to prevent an om_flush() that is behind + // this thread from passing us. + if (*saved_mid_in_use_p == NULL) { + // No saved state so start at the beginning. + // Lock the list head so we can possibly deflate it: + if ((mid = get_list_head_locked(list_p)) == NULL) { + return 0; // The list is empty so nothing to deflate. + } + next = unmarked_next(mid); + } else { + // We're restarting after a safepoint so restore the necessary state + // before we resume. + cur_mid_in_use = *saved_mid_in_use_p; + // Lock cur_mid_in_use so we can possibly update its + // next field to extract a deflated ObjectMonitor. + om_lock(cur_mid_in_use); + mid = unmarked_next(cur_mid_in_use); + if (mid == NULL) { + om_unlock(cur_mid_in_use); + *saved_mid_in_use_p = NULL; + return 0; // The remainder is empty so nothing more to deflate. + } + // Lock mid so we can possibly deflate it: + om_lock(mid); + next = unmarked_next(mid); + } + + while (true) { + // The current mid is locked at this point. If we have a + // cur_mid_in_use, then it is also locked at this point. + + if (next != NULL) { + // We lock next so that an om_flush() thread that is behind us + // cannot pass us when we unlock the current mid. + om_lock(next); + next_next = unmarked_next(next); + } + + // Only try to deflate if there is an associated Java object and if + // mid is old (is not newly allocated and is not newly freed). + if (mid->object() != NULL && mid->is_old() && + deflate_monitor_using_JT(mid, free_head_p, free_tail_p)) { + // Deflation succeeded and already updated free_head_p and + // free_tail_p as needed. Finish the move to the local free list + // by unlinking mid from the global or per-thread in-use list. + if (cur_mid_in_use == NULL) { + // mid is the list head and it is locked. Switch the list head + // to next which is also locked (if not NULL) and also leave + // mid locked: + Atomic::store(list_p, next); + } else { + ObjectMonitor* locked_next = mark_om_ptr(next); + // mid and cur_mid_in_use are locked. Switch cur_mid_in_use's + // next field to locked_next and also leave mid locked: + cur_mid_in_use->set_next_om(locked_next); + } + // At this point mid is disconnected from the in-use list so + // its lock longer has any effects on in-use list. + deflated_count++; + Atomic::dec(count_p); + // mid is current tail in the free_head_p list so NULL terminate it + // (which also unlocks it): + mid->set_next_om(NULL); + + // All the list management is done so move on to the next one: + mid = next; // mid keeps non-NULL next's locked state + next = next_next; + } else { + // mid is considered in-use if it does not have an associated + // Java object or mid is not old or deflation did not succeed. + // A mid->is_new() node can be seen here when it is freshly + // returned by om_alloc() (and skips the deflation code path). + // A mid->is_old() node can be seen here when deflation failed. + // A mid->is_free() node can be seen here when a fresh node from + // om_alloc() is released by om_release() due to losing the race + // in inflate(). + + // All the list management is done so move on to the next one: + if (cur_mid_in_use != NULL) { + om_unlock(cur_mid_in_use); + } + // The next cur_mid_in_use keeps mid's lock state so + // that it is stable for a possible next field change. It + // cannot be modified by om_release() while it is locked. + cur_mid_in_use = mid; + mid = next; // mid keeps non-NULL next's locked state + next = next_next; + + if (SafepointMechanism::should_block(self) && + cur_mid_in_use != Atomic::load(list_p) && cur_mid_in_use->is_old()) { + // If a safepoint has started and cur_mid_in_use is not the list + // head and is old, then it is safe to use as saved state. Return + // to the caller before blocking. + *saved_mid_in_use_p = cur_mid_in_use; + om_unlock(cur_mid_in_use); + if (mid != NULL) { + om_unlock(mid); + } + return deflated_count; + } + } + if (mid == NULL) { + if (cur_mid_in_use != NULL) { + om_unlock(cur_mid_in_use); + } + break; // Reached end of the list so nothing more to deflate. + } + + // The current mid's next field is locked at this point. If we have + // a cur_mid_in_use, then it is also locked at this point. + } + // We finished the list without a safepoint starting so there's + // no need to save state. + *saved_mid_in_use_p = NULL; + return deflated_count; +} + void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) { counters->n_in_use = 0; // currently associated with objects counters->n_in_circulation = 0; // extant @@ -1954,6 +2513,15 @@ void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); + + if (AsyncDeflateIdleMonitors) { + // Nothing to do when global idle ObjectMonitors are deflated using + // a JavaThread unless a special deflation has been requested. + if (!is_special_deflation_requested()) { + return; + } + } + bool deflated = false; ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors @@ -2006,6 +2574,194 @@ } } +class HandshakeForDeflation : public HandshakeClosure { + public: + HandshakeForDeflation() : HandshakeClosure("HandshakeForDeflation") {} + + void do_thread(Thread* thread) { + log_trace(monitorinflation)("HandshakeForDeflation::do_thread: thread=" + INTPTR_FORMAT, p2i(thread)); + } +}; + +void ObjectSynchronizer::deflate_idle_monitors_using_JT() { + assert(AsyncDeflateIdleMonitors, "sanity check"); + + // Deflate any global idle monitors. + deflate_global_idle_monitors_using_JT(); + + int count = 0; + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { + if (Atomic::load(&jt->om_in_use_count) > 0 && !jt->is_exiting()) { + // This JavaThread is using ObjectMonitors so deflate any that + // are idle unless this JavaThread is exiting; do not race with + // ObjectSynchronizer::om_flush(). + deflate_per_thread_idle_monitors_using_JT(jt); + count++; + } + } + if (count > 0) { + log_debug(monitorinflation)("did async deflation of idle monitors for %d thread(s).", count); + } + + log_info(monitorinflation)("async global_population=%d, global_in_use_count=%d, " + "global_free_count=%d, global_wait_count=%d", + Atomic::load(&om_list_globals._population), + Atomic::load(&om_list_globals._in_use_count), + Atomic::load(&om_list_globals._free_count), + Atomic::load(&om_list_globals._wait_count)); + + // The ServiceThread's async deflation request has been processed. + set_is_async_deflation_requested(false); + + if (Atomic::load(&om_list_globals._wait_count) > 0) { + // There are deflated ObjectMonitors waiting for a handshake + // (or a safepoint) for safety. + + ObjectMonitor* list = Atomic::load(&om_list_globals._wait_list); + ADIM_guarantee(list != NULL, "om_list_globals._wait_list must not be NULL"); + int count = Atomic::load(&om_list_globals._wait_count); + Atomic::store(&om_list_globals._wait_count, 0); + Atomic::store(&om_list_globals._wait_list, (ObjectMonitor*)NULL); + + // Find the tail for prepend_list_to_common(). No need to mark + // ObjectMonitors for this list walk since only the deflater + // thread manages the wait list. + int l_count = 0; + ObjectMonitor* tail = NULL; + for (ObjectMonitor* n = list; n != NULL; n = unmarked_next(n)) { + tail = n; + l_count++; + } + ADIM_guarantee(count == l_count, "count=%d != l_count=%d", count, l_count); + + // Will execute a safepoint if !ThreadLocalHandshakes: + HandshakeForDeflation hfd_hc; + Handshake::execute(&hfd_hc); + + prepend_list_to_common(list, tail, count, &om_list_globals._free_list, + &om_list_globals._free_count); + + log_info(monitorinflation)("moved %d idle monitors from global waiting list to global free list", count); + } +} + +// Deflate global idle ObjectMonitors using a JavaThread. +// +void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() { + assert(AsyncDeflateIdleMonitors, "sanity check"); + assert(Thread::current()->is_Java_thread(), "precondition"); + JavaThread* self = JavaThread::current(); + + deflate_common_idle_monitors_using_JT(true /* is_global */, self); +} + +// Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread. +// +void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) { + assert(AsyncDeflateIdleMonitors, "sanity check"); + assert(Thread::current()->is_Java_thread(), "precondition"); + + deflate_common_idle_monitors_using_JT(false /* !is_global */, target); +} + +// Deflate global or per-thread idle ObjectMonitors using a JavaThread. +// +void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) { + JavaThread* self = JavaThread::current(); + + int deflated_count = 0; + ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors + ObjectMonitor* free_tail_p = NULL; + ObjectMonitor* saved_mid_in_use_p = NULL; + elapsedTimer timer; + + if (log_is_enabled(Info, monitorinflation)) { + timer.start(); + } + + if (is_global) { + OM_PERFDATA_OP(MonExtant, set_value(Atomic::load(&om_list_globals._in_use_count))); + } else { + OM_PERFDATA_OP(MonExtant, inc(Atomic::load(&target->om_in_use_count))); + } + + do { + int local_deflated_count; + if (is_global) { + local_deflated_count = + deflate_monitor_list_using_JT(&om_list_globals._in_use_list, + &om_list_globals._in_use_count, + &free_head_p, &free_tail_p, + &saved_mid_in_use_p); + } else { + local_deflated_count = + deflate_monitor_list_using_JT(&target->om_in_use_list, + &target->om_in_use_count, &free_head_p, + &free_tail_p, &saved_mid_in_use_p); + } + deflated_count += local_deflated_count; + + if (free_head_p != NULL) { + // Move the deflated ObjectMonitors to the global free list. + guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count); + // Note: The target thread can be doing an om_alloc() that + // is trying to prepend an ObjectMonitor on its in-use list + // at the same time that we have deflated the current in-use + // list head and put it on the local free list. prepend_to_common() + // will detect the race and retry which avoids list corruption, + // but the next field in free_tail_p can flicker to marked + // and then unmarked while prepend_to_common() is sorting it + // all out. +#ifdef ASSERT + ObjectMonitor* l_next_om = unmarked_next(free_tail_p); +#endif + assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om)); + + prepend_list_to_global_wait_list(free_head_p, free_tail_p, local_deflated_count); + + OM_PERFDATA_OP(Deflations, inc(local_deflated_count)); + } + + if (saved_mid_in_use_p != NULL) { + // deflate_monitor_list_using_JT() detected a safepoint starting. + timer.stop(); + { + if (is_global) { + log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint."); + } else { + log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target)); + } + assert(SafepointMechanism::should_block(self), "sanity check"); + ThreadBlockInVM blocker(self); + } + // Prepare for another loop after the safepoint. + free_head_p = NULL; + free_tail_p = NULL; + if (log_is_enabled(Info, monitorinflation)) { + timer.start(); + } + } + } while (saved_mid_in_use_p != NULL); + timer.stop(); + + LogStreamHandle(Debug, monitorinflation) lsh_debug; + LogStreamHandle(Info, monitorinflation) lsh_info; + LogStream* ls = NULL; + if (log_is_enabled(Debug, monitorinflation)) { + ls = &lsh_debug; + } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) { + ls = &lsh_info; + } + if (ls != NULL) { + if (is_global) { + ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count); + } else { + ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count); + } + } +} + void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) { // Report the cumulative time for deflating each thread's idle // monitors. Note: if the work is split among more than one @@ -2013,16 +2769,27 @@ // than a beginning to end measurement of the phase. log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", counters->per_thread_times, counters->per_thread_scavenged); + bool needs_special_deflation = is_special_deflation_requested(); + if (AsyncDeflateIdleMonitors && !needs_special_deflation) { + // Nothing to do when idle ObjectMonitors are deflated using + // a JavaThread unless a special deflation has been requested. + return; + } + if (log_is_enabled(Debug, monitorinflation)) { // exit_globals()'s call to audit_and_print_stats() is done // at the Info level and not at a safepoint. + // For async deflation, audit_and_print_stats() is called in + // ObjectSynchronizer::do_safepoint_work() at the Debug level + // at a safepoint. ObjectSynchronizer::audit_and_print_stats(false /* on_exit */); } else if (log_is_enabled(Info, monitorinflation)) { log_info(monitorinflation)("global_population=%d, global_in_use_count=%d, " - "global_free_count=%d", + "global_free_count=%d, global_wait_count=%d", Atomic::load(&om_list_globals._population), Atomic::load(&om_list_globals._in_use_count), - Atomic::load(&om_list_globals._free_count)); + Atomic::load(&om_list_globals._free_count), + Atomic::load(&om_list_globals._wait_count)); } Atomic::store(&_forceMonitorScavenge, 0); // Reset @@ -2032,11 +2799,20 @@ GVars.stw_random = os::random(); GVars.stw_cycle++; + + if (needs_special_deflation) { + set_is_special_deflation_requested(false); // special deflation is done + } } void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); + if (AsyncDeflateIdleMonitors && !is_special_deflation_requested()) { + // Nothing to do if a special deflation has NOT been requested. + return; + } + ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors ObjectMonitor* free_tail_p = NULL; elapsedTimer timer; @@ -2210,6 +2986,9 @@ // Check om_list_globals._free_list and om_list_globals._free_count: chk_global_free_list_and_count(ls, &error_cnt); + // Check om_list_globals._wait_list and om_list_globals._wait_count: + chk_global_wait_list_and_count(ls, &error_cnt); + ls->print_cr("Checking per-thread lists:"); for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { @@ -2260,12 +3039,13 @@ ": free per-thread monitor must have NULL _header " "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n), n->header().value()); - } else { + *error_cnt_p = *error_cnt_p + 1; + } else if (!AsyncDeflateIdleMonitors) { out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor " "must have NULL _header field: _header=" INTPTR_FORMAT, p2i(n), n->header().value()); + *error_cnt_p = *error_cnt_p + 1; } - *error_cnt_p = *error_cnt_p + 1; } if (n->object() != NULL) { if (jt != NULL) { @@ -2332,6 +3112,36 @@ } } +// Check the global wait list and count; log the results of the checks. +void ObjectSynchronizer::chk_global_wait_list_and_count(outputStream * out, + int *error_cnt_p) { + int chk_om_wait_count = 0; + ObjectMonitor* cur = NULL; + if ((cur = get_list_head_locked(&om_list_globals._wait_list)) != NULL) { + // Marked the global wait list head so process the list. + while (true) { + // Rules for om_list_globals._wait_list are the same as for + // om_list_globals._free_list: + chk_free_entry(NULL /* jt */, cur, out, error_cnt_p); + chk_om_wait_count++; + + cur = lock_next_for_traversal(cur); + if (cur == NULL) { + break; + } + } + } + if (Atomic::load(&om_list_globals._wait_count) == chk_om_wait_count) { + out->print_cr("global_wait_count=%d equals chk_om_wait_count=%d", + Atomic::load(&om_list_globals._wait_count), chk_om_wait_count); + } else { + out->print_cr("ERROR: global_wait_count=%d is not equal to " + "chk_om_wait_count=%d", + Atomic::load(&om_list_globals._wait_count), chk_om_wait_count); + *error_cnt_p = *error_cnt_p + 1; + } +} + // Check the global in-use list and count; log the results of the checks. void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out, int *error_cnt_p) { @@ -2555,14 +3365,16 @@ // the population count. int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) { int pop_count = 0; - out->print_cr("%18s %10s %10s %10s", - "Global Lists:", "InUse", "Free", "Total"); - out->print_cr("================== ========== ========== =========="); + out->print_cr("%18s %10s %10s %10s %10s", + "Global Lists:", "InUse", "Free", "Wait", "Total"); + out->print_cr("================== ========== ========== ========== =========="); int l_in_use_count = Atomic::load(&om_list_globals._in_use_count); int l_free_count = Atomic::load(&om_list_globals._free_count); - out->print_cr("%18s %10d %10d %10d", "", l_in_use_count, - l_free_count, Atomic::load(&om_list_globals._population)); - pop_count += l_in_use_count + l_free_count; + int l_wait_count = Atomic::load(&om_list_globals._wait_count); + out->print_cr("%18s %10d %10d %10d %10d", "", l_in_use_count, + l_free_count, l_wait_count, + Atomic::load(&om_list_globals._population)); + pop_count += l_in_use_count + l_free_count + l_wait_count; out->print_cr("%18s %10s %10s %10s", "Per-Thread Lists:", "InUse", "Free", "Provision"); --- old/src/hotspot/share/runtime/synchronizer.hpp 2020-05-06 20:09:14.000000000 -0400 +++ new/src/hotspot/share/runtime/synchronizer.hpp 2020-05-06 20:09:14.000000000 -0400 @@ -43,11 +43,11 @@ typedef PaddedEnd PaddedObjectMonitor; struct DeflateMonitorCounters { - int n_in_use; // currently associated with objects - int n_in_circulation; // extant - int n_scavenged; // reclaimed (global and per-thread) - int per_thread_scavenged; // per-thread scavenge total - double per_thread_times; // per-thread scavenge times + volatile int n_in_use; // currently associated with objects + volatile int n_in_circulation; // extant + volatile int n_scavenged; // reclaimed (global and per-thread) + volatile int per_thread_scavenged; // per-thread scavenge total + double per_thread_times; // per-thread scavenge times }; class ObjectSynchronizer : AllStatic { @@ -132,6 +132,10 @@ // Basically we deflate all monitors that are not busy. // An adaptive profile-based deflation policy could be used if needed static void deflate_idle_monitors(DeflateMonitorCounters* counters); + static void deflate_idle_monitors_using_JT(); + static void deflate_global_idle_monitors_using_JT(); + static void deflate_per_thread_idle_monitors_using_JT(JavaThread* target); + static void deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target); static void deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters); static void prepare_deflate_idle_monitors(DeflateMonitorCounters* counters); static void finish_deflate_idle_monitors(DeflateMonitorCounters* counters); @@ -141,11 +145,27 @@ int* count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p); + // For a given in-use monitor list: global or per-thread, deflate idle + // monitors using a JavaThread. + static int deflate_monitor_list_using_JT(ObjectMonitor** list_p, + int* count_p, + ObjectMonitor** free_head_p, + ObjectMonitor** free_tail_p, + ObjectMonitor** saved_mid_in_use_p); static bool deflate_monitor(ObjectMonitor* mid, oop obj, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p); - static bool is_cleanup_needed(); + static bool deflate_monitor_using_JT(ObjectMonitor* mid, + ObjectMonitor** free_head_p, + ObjectMonitor** free_tail_p); + static bool is_async_deflation_needed(); static bool needs_monitor_scavenge(); + static bool is_safepoint_deflation_needed(); + static bool is_async_deflation_requested() { return _is_async_deflation_requested; } + static bool is_special_deflation_requested() { return _is_special_deflation_requested; } + static void set_is_async_deflation_requested(bool new_value) { _is_async_deflation_requested = new_value; } + static void set_is_special_deflation_requested(bool new_value) { _is_special_deflation_requested = new_value; } + static jlong time_since_last_async_deflation_ms(); static void oops_do(OopClosure* f); // Process oops in thread local used monitors static void thread_local_used_oops_do(Thread* thread, OopClosure* f); @@ -156,6 +176,8 @@ outputStream * out, int *error_cnt_p); static void chk_global_free_list_and_count(outputStream * out, int *error_cnt_p); + static void chk_global_wait_list_and_count(outputStream * out, + int *error_cnt_p); static void chk_global_in_use_list_and_count(outputStream * out, int *error_cnt_p); static void chk_in_use_entry(JavaThread* jt, ObjectMonitor* n, @@ -170,12 +192,17 @@ static int log_monitor_list_counts(outputStream * out); static int verify_objmon_isinpool(ObjectMonitor *addr) PRODUCT_RETURN0; + static void do_safepoint_work(DeflateMonitorCounters* counters); + private: friend class SynchronizerTest; enum { _BLOCKSIZE = 128 }; // global list of blocks of monitors static PaddedObjectMonitor* g_block_list; + static volatile bool _is_async_deflation_requested; + static volatile bool _is_special_deflation_requested; + static jlong _last_async_deflation_time_ns; // Function to prepend new blocks to the appropriate lists: static void prepend_block_to_lists(PaddedObjectMonitor* new_blk); --- old/src/hotspot/share/runtime/thread.cpp 2020-05-06 20:09:15.000000000 -0400 +++ new/src/hotspot/share/runtime/thread.cpp 2020-05-06 20:09:15.000000000 -0400 @@ -4683,6 +4683,8 @@ DO_JAVA_THREADS(t_list, p) { if (!p->can_call_java()) continue; + // The first stage of async deflation does not affect any field + // used by this comparison so the ObjectMonitor* is usable here. address pending = (address)p->current_pending_monitor(); if (pending == monitor) { // found a match if (i < count) result->append(p); // save the first count matches --- old/src/hotspot/share/runtime/vframe.cpp 2020-05-06 20:09:18.000000000 -0400 +++ new/src/hotspot/share/runtime/vframe.cpp 2020-05-06 20:09:18.000000000 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -122,8 +122,14 @@ if (mons->is_empty()) return result; bool found_first_monitor = false; - ObjectMonitor *pending_monitor = thread()->current_pending_monitor(); + // The ObjectMonitor* can't be async deflated since we are either + // at a safepoint or the calling thread is operating on itself so + // it cannot leave the underlying wait()/enter() call. ObjectMonitor *waiting_monitor = thread()->current_waiting_monitor(); + ObjectMonitor *pending_monitor = NULL; + if (waiting_monitor == NULL) { + pending_monitor = thread()->current_pending_monitor(); + } oop pending_obj = (pending_monitor != NULL ? (oop) pending_monitor->object() : (oop) NULL); oop waiting_obj = (waiting_monitor != NULL ? (oop) waiting_monitor->object() : (oop) NULL); @@ -231,6 +237,8 @@ // an inflated monitor that is first on the monitor list in // the first frame can block us on a monitor enter. markWord mark = monitor->owner()->mark(); + // The first stage of async deflation does not affect any field + // used by this comparison so the ObjectMonitor* is usable here. if (mark.has_monitor() && ( // we have marked ourself as pending on this monitor mark.monitor() == thread()->current_pending_monitor() || --- old/src/hotspot/share/runtime/vmOperations.cpp 2020-05-06 20:09:19.000000000 -0400 +++ new/src/hotspot/share/runtime/vmOperations.cpp 2020-05-06 20:09:19.000000000 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -41,6 +41,7 @@ #include "runtime/frame.inline.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/sweeper.hpp" +#include "runtime/synchronizer.hpp" #include "runtime/thread.inline.hpp" #include "runtime/threadSMR.inline.hpp" #include "runtime/vmOperations.hpp" @@ -433,6 +434,17 @@ } } +bool VM_Exit::doit_prologue() { + if (AsyncDeflateIdleMonitors && log_is_enabled(Info, monitorinflation)) { + // AsyncDeflateIdleMonitors does a special deflation at the VM_Exit + // safepoint in order to reduce the in-use monitor population that + // is reported by ObjectSynchronizer::log_in_use_monitor_details() + // at VM exit. + ObjectSynchronizer::set_is_special_deflation_requested(true); + } + return true; +} + void VM_Exit::doit() { if (VerifyBeforeExit) { --- old/src/hotspot/share/runtime/vmOperations.hpp 2020-05-06 20:09:20.000000000 -0400 +++ new/src/hotspot/share/runtime/vmOperations.hpp 2020-05-06 20:09:20.000000000 -0400 @@ -420,6 +420,7 @@ } } VMOp_Type type() const { return VMOp_Exit; } + bool doit_prologue(); void doit(); }; --- old/src/hotspot/share/runtime/vmStructs.cpp 2020-05-06 20:09:22.000000000 -0400 +++ new/src/hotspot/share/runtime/vmStructs.cpp 2020-05-06 20:09:21.000000000 -0400 @@ -94,6 +94,7 @@ #include "runtime/serviceThread.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" +#include "runtime/synchronizer.hpp" #include "runtime/thread.inline.hpp" #include "runtime/threadSMR.hpp" #include "runtime/vframeArray.hpp" @@ -899,14 +900,14 @@ volatile_nonstatic_field(ObjectMonitor, _header, markWord) \ unchecked_nonstatic_field(ObjectMonitor, _object, sizeof(void *)) /* NOTE: no type */ \ unchecked_nonstatic_field(ObjectMonitor, _owner, sizeof(void *)) /* NOTE: no type */ \ - volatile_nonstatic_field(ObjectMonitor, _contentions, jint) \ + volatile_nonstatic_field(ObjectMonitor, _next_om, ObjectMonitor*) \ + volatile_nonstatic_field(BasicLock, _displaced_header, markWord) \ + nonstatic_field(ObjectMonitor, _contentions, jint) \ volatile_nonstatic_field(ObjectMonitor, _waiters, jint) \ volatile_nonstatic_field(ObjectMonitor, _recursions, intx) \ - nonstatic_field(ObjectMonitor, _next_om, ObjectMonitor*) \ - volatile_nonstatic_field(BasicLock, _displaced_header, markWord) \ nonstatic_field(BasicObjectLock, _lock, BasicLock) \ nonstatic_field(BasicObjectLock, _obj, oop) \ - static_ptr_volatile_field(ObjectSynchronizer, g_block_list, PaddedObjectMonitor*) \ + static_field(ObjectSynchronizer, g_block_list, PaddedObjectMonitor*) \ \ /*********************/ \ /* Matcher (C2 only) */ \ --- old/src/hotspot/share/runtime/vmThread.cpp 2020-05-06 20:09:23.000000000 -0400 +++ new/src/hotspot/share/runtime/vmThread.cpp 2020-05-06 20:09:23.000000000 -0400 @@ -41,6 +41,7 @@ #include "runtime/mutexLocker.hpp" #include "runtime/os.hpp" #include "runtime/safepoint.hpp" +#include "runtime/synchronizer.hpp" #include "runtime/thread.inline.hpp" #include "runtime/vmThread.hpp" #include "runtime/vmOperations.hpp" @@ -283,6 +284,14 @@ assert(should_terminate(), "termination flag must be set"); } + if (AsyncDeflateIdleMonitors && log_is_enabled(Info, monitorinflation)) { + // AsyncDeflateIdleMonitors does a special deflation at the final + // safepoint in order to reduce the in-use monitor population that + // is reported by ObjectSynchronizer::log_in_use_monitor_details() + // at VM exit. + ObjectSynchronizer::set_is_special_deflation_requested(true); + } + // 4526887 let VM thread exit at Safepoint _cur_vm_operation = &halt_op; SafepointSynchronize::begin(); --- old/src/hotspot/share/services/threadService.cpp 2020-05-06 20:09:24.000000000 -0400 +++ new/src/hotspot/share/services/threadService.cpp 2020-05-06 20:09:24.000000000 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -37,6 +37,7 @@ #include "runtime/handles.inline.hpp" #include "runtime/init.hpp" #include "runtime/objectMonitor.inline.hpp" +#include "runtime/safepointVerifiers.hpp" #include "runtime/thread.inline.hpp" #include "runtime/threadSMR.inline.hpp" #include "runtime/vframe.hpp" @@ -208,19 +209,27 @@ assert(thread != NULL, "should be non-NULL"); debug_only(Thread::check_for_dangling_thread_pointer(thread);) + // This function can be called on a target JavaThread that is not + // the caller and we are not at a safepoint. So it is possible for + // the waiting or pending condition to be over/stale and for the + // first stage of async deflation to clear the object field in + // the ObjectMonitor. It is also possible for the object to be + // inflated again and to be associated with a completely different + // ObjectMonitor by the time this object reference is processed + // by the caller. ObjectMonitor *wait_obj = thread->current_waiting_monitor(); oop obj = NULL; if (wait_obj != NULL) { // thread is doing an Object.wait() call obj = (oop) wait_obj->object(); - assert(obj != NULL, "Object.wait() should have an object"); + assert(AsyncDeflateIdleMonitors || obj != NULL, "Object.wait() should have an object"); } else { ObjectMonitor *enter_obj = thread->current_pending_monitor(); if (enter_obj != NULL) { // thread is trying to enter() an ObjectMonitor. obj = (oop) enter_obj->object(); - assert(obj != NULL, "ObjectMonitor should have an associated object!"); + assert(AsyncDeflateIdleMonitors || obj != NULL, "ObjectMonitor should have an associated object!"); } } @@ -391,6 +400,7 @@ cycle->reset(); + // The ObjectMonitor* can't be async deflated since we are at a safepoint. // When there is a deadlock, all the monitors involved in the dependency // cycle must be contended and heavyweight. So we only care about the // heavyweight monitor a thread is waiting to lock. @@ -967,13 +977,13 @@ st->print("============================="); JavaThread* currentThread; - ObjectMonitor* waitingToLockMonitor; JvmtiRawMonitor* waitingToLockRawMonitor; oop waitingToLockBlocker; int len = _threads->length(); for (int i = 0; i < len; i++) { currentThread = _threads->at(i); - waitingToLockMonitor = currentThread->current_pending_monitor(); + // The ObjectMonitor* can't be async deflated since we are at a safepoint. + ObjectMonitor* waitingToLockMonitor = currentThread->current_pending_monitor(); waitingToLockRawMonitor = currentThread->current_pending_raw_monitor(); waitingToLockBlocker = currentThread->current_park_blocker(); st->cr(); --- old/test/hotspot/gtest/oops/test_markWord.cpp 2020-05-06 20:09:26.000000000 -0400 +++ new/test/hotspot/gtest/oops/test_markWord.cpp 2020-05-06 20:09:25.000000000 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -123,6 +123,10 @@ // This is no longer biased, because ObjectLocker revokes the bias. assert_test_pattern(h_obj, "is_neutral no_hash"); + // Hash the object then print it. + intx hash = h_obj->identity_hash(); + assert_test_pattern(h_obj, "is_neutral hash=0x"); + // Wait gets the lock inflated. { ObjectLocker ol(h_obj, THREAD); @@ -137,14 +141,18 @@ done.wait_with_safepoint_check(THREAD); // wait till the thread is done. } - // Make the object older. Not all GCs use this field. - Universe::heap()->collect(GCCause::_java_lang_system_gc); - if (UseParallelGC) { - assert_test_pattern(h_obj, "is_neutral no_hash age 1"); + if (!AsyncDeflateIdleMonitors) { + // With AsyncDeflateIdleMonitors, the collect() call below + // does not guarantee monitor deflation. + // Make the object older. Not all GCs use this field. + Universe::heap()->collect(GCCause::_java_lang_system_gc); + if (UseParallelGC) { + assert_test_pattern(h_obj, "is_neutral no_hash age 1"); + } + + // Hash the object then print it. + intx hash = h_obj->identity_hash(); + assert_test_pattern(h_obj, "is_neutral hash=0x"); } - - // Hash the object then print it. - intx hash = h_obj->identity_hash(); - assert_test_pattern(h_obj, "is_neutral hash=0x"); } #endif // PRODUCT --- old/test/hotspot/jtreg/runtime/logging/SafepointCleanupTest.java 2020-05-06 20:09:27.000000000 -0400 +++ new/test/hotspot/jtreg/runtime/logging/SafepointCleanupTest.java 2020-05-06 20:09:27.000000000 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,12 +29,17 @@ * @modules java.base/jdk.internal.misc * java.management * @run driver SafepointCleanupTest + * @run driver SafepointCleanupTest -XX:+AsyncDeflateIdleMonitors */ import jdk.test.lib.process.OutputAnalyzer; import jdk.test.lib.process.ProcessTools; public class SafepointCleanupTest { + static final String ASYNC_DISABLE_OPTION = "-XX:-AsyncDeflateIdleMonitors"; + static final String ASYNC_ENABLE_OPTION = "-XX:+AsyncDeflateIdleMonitors"; + static final String UNLOCK_DIAG_OPTION = "-XX:+UnlockDiagnosticVMOptions"; + static void analyzeOutputOn(ProcessBuilder pb) throws Exception { OutputAnalyzer output = new OutputAnalyzer(pb.start()); output.shouldContain("[safepoint,cleanup]"); @@ -53,19 +58,40 @@ } public static void main(String[] args) throws Exception { + String async_option; + if (args.length == 0) { + // By default test deflating idle monitors at a safepoint. + async_option = ASYNC_DISABLE_OPTION; + } else { + async_option = args[0]; + } + if (!async_option.equals(ASYNC_DISABLE_OPTION) && + !async_option.equals(ASYNC_ENABLE_OPTION)) { + throw new RuntimeException("Unknown async_option value: '" + + async_option + "'"); + } + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder("-Xlog:safepoint+cleanup=info", + UNLOCK_DIAG_OPTION, + async_option, InnerClass.class.getName()); analyzeOutputOn(pb); pb = ProcessTools.createJavaProcessBuilder("-XX:+TraceSafepointCleanupTime", + UNLOCK_DIAG_OPTION, + async_option, InnerClass.class.getName()); analyzeOutputOn(pb); pb = ProcessTools.createJavaProcessBuilder("-Xlog:safepoint+cleanup=off", + UNLOCK_DIAG_OPTION, + async_option, InnerClass.class.getName()); analyzeOutputOff(pb); pb = ProcessTools.createJavaProcessBuilder("-XX:-TraceSafepointCleanupTime", + UNLOCK_DIAG_OPTION, + async_option, InnerClass.class.getName()); analyzeOutputOff(pb); }