--- old/src/hotspot/share/prims/jvm.cpp 2019-03-22 16:30:32.621896906 -0400 +++ new/src/hotspot/share/prims/jvm.cpp 2019-03-22 16:30:32.181896913 -0400 @@ -71,6 +71,7 @@ #include "runtime/os.inline.hpp" #include "runtime/perfData.hpp" #include "runtime/reflection.hpp" +#include "runtime/synchronizer.hpp" #include "runtime/thread.inline.hpp" #include "runtime/threadSMR.hpp" #include "runtime/vframe.inline.hpp" @@ -482,6 +483,11 @@ JVM_ENTRY_NO_ENV(void, JVM_GC(void)) JVMWrapper("JVM_GC"); if (!DisableExplicitGC) { + if (AsyncDeflateIdleMonitors) { + // AsyncDeflateIdleMonitors needs to know when System.gc() is + // called so any special clean up can be done at a safepoint. + ObjectSynchronizer::set_is_cleanup_requested(true); + } Universe::heap()->collect(GCCause::_java_lang_system_gc); } JVM_END --- old/src/hotspot/share/runtime/basicLock.cpp 2019-03-22 16:30:33.773896886 -0400 +++ new/src/hotspot/share/runtime/basicLock.cpp 2019-03-22 16:30:33.321896894 -0400 @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "runtime/basicLock.hpp" +#include "runtime/objectMonitor.hpp" #include "runtime/synchronizer.hpp" void BasicLock::print_on(outputStream* st) const { @@ -62,8 +63,11 @@ // is small (given the support for inflated fast-path locking in the fast_lock, etc) // we'll leave that optimization for another time. + // Disallow async deflation of the inflated monitor so the + // displaced header stays stable until we've copied it. + ObjectMonitorHandle omh; if (displaced_header()->is_neutral()) { - ObjectSynchronizer::inflate_helper(obj); + ObjectSynchronizer::inflate_helper(&omh, obj); // WARNING: We can not put check here, because the inflation // will not update the displaced header. Once BasicLock is inflated, // no one should ever look at its content. --- old/src/hotspot/share/runtime/globals.hpp 2019-03-22 16:30:34.869896867 -0400 +++ new/src/hotspot/share/runtime/globals.hpp 2019-03-22 16:30:34.441896874 -0400 @@ -819,6 +819,9 @@ product(intx, MonitorBound, 0, "Bound Monitor population") \ range(0, max_jint) \ \ + diagnostic(bool, AsyncDeflateIdleMonitors, true, \ + "Deflate idle monitors using JavaThreads and the ServiceThread.") \ + \ experimental(intx, MonitorUsedDeflationThreshold, 90, \ "Percentage of used monitors before triggering cleanup " \ "safepoint which deflates monitors (0 is off). " \ --- old/src/hotspot/share/runtime/objectMonitor.cpp 2019-03-22 16:30:35.769896851 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.cpp 2019-03-22 16:30:35.405896857 -0400 @@ -238,7 +238,7 @@ // ----------------------------------------------------------------------------- // Enter support -void ObjectMonitor::enter(TRAPS) { +bool ObjectMonitor::enter(TRAPS) { // The following code is ordered to check the most common cases first // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors. Thread * const Self = THREAD; @@ -248,13 +248,13 @@ // Either ASSERT _recursions == 0 or explicitly set _recursions = 0. assert(_recursions == 0, "invariant"); assert(_owner == Self, "invariant"); - return; + return true; } if (cur == Self) { // TODO-FIXME: check for integer overflow! BUGID 6557169. _recursions++; - return; + return true; } if (Self->is_lock_owned ((address)cur)) { @@ -263,7 +263,7 @@ // Commute owner from a thread-specific on-stack BasicLockObject address to // a full-fledged "Thread *". _owner = Self; - return; + return true; } // We've encountered genuine contention. @@ -280,7 +280,7 @@ assert(_recursions == 0, "invariant"); assert(((oop)(object()))->mark() == markOopDesc::encode(this), "invariant"); Self->_Stalled = 0; - return; + return true; } assert(_owner != Self, "invariant"); @@ -289,12 +289,20 @@ JavaThread * jt = (JavaThread *) Self; assert(!SafepointSynchronize::is_at_safepoint(), "invariant"); assert(jt->thread_state() != _thread_blocked, "invariant"); - assert(this->object() != NULL, "invariant"); - assert(_count >= 0, "invariant"); + assert(AsyncDeflateIdleMonitors || this->object() != NULL, "invariant"); + assert(AsyncDeflateIdleMonitors || _count >= 0, "invariant"); - // Prevent deflation at STW-time. See deflate_idle_monitors() and is_busy(). + // Prevent deflation. See ObjectSynchronizer::deflate_monitor() and is_busy(). // Ensure the object-monitor relationship remains stable while there's contention. - Atomic::inc(&_count); + const jint count = Atomic::add(1, &_count); + if (count <= 0 && _owner == DEFLATER_MARKER) { + // Async deflation in progress. Help deflater thread install + // the mark word (in case deflater thread is slow). + install_displaced_markword_in_object(); + Self->_Stalled = 0; + return false; // Caller should retry. Never mind about _count as this monitor has been deflated. + } + // The deflater thread will not deflate this monitor and the monitor is contended, continue. JFR_ONLY(JfrConditionalFlushWithStacktrace flush(jt);) EventJavaMonitorEnter event; @@ -356,7 +364,7 @@ } Atomic::dec(&_count); - assert(_count >= 0, "invariant"); + assert(AsyncDeflateIdleMonitors || _count >= 0, "invariant"); Self->_Stalled = 0; // Must either set _recursions = 0 or ASSERT _recursions == 0. @@ -392,6 +400,7 @@ event.commit(); } OM_PERFDATA_OP(ContendedLockAttempts, inc()); + return true; } // Caveat: TryLock() is not necessarily serializing if it returns failure. @@ -413,6 +422,67 @@ return -1; } +// Install the displaced markword of a deflated monitor into the object +// associated with the monitor. +// This method is idempotent and is executed by both mutators wanting to +// acquire a monitor for an object and the thread deflating monitors. +// A mutator trying to install a hash in the monitor's _header field can +// also run in parallel to this method. +void ObjectMonitor::install_displaced_markword_in_object() { + markOop dmw = header(); + if (dmw == NULL) { + // The thread deflating monitors has won the race so we + // have nothing to do. + return; + } + + // A non-NULL dmw has to be either neutral or is participating in + // this restoration protocol. + assert(dmw->is_neutral() || (dmw->is_marked() && dmw->hash() == 0), + "failed precondition: is_neutral=%d, is_marked=%d, hash=" + INTPTR_FORMAT, dmw->is_neutral(), dmw->is_marked(), dmw->hash()); + + if (!dmw->is_marked() && dmw->hash() == 0) { + // This dmw is neutral and has not yet started the restoration + // protocol so we mark a copy of the dmw to begin the protocol. + markOop marked_dmw = dmw->set_marked(); + assert(marked_dmw->is_marked() && marked_dmw->hash() == 0, + "sanity_check: is_marked=%d, hash=" INTPTR_FORMAT, + marked_dmw->is_marked(), marked_dmw->hash()); + + // There can be three different racers trying to update the _header + // field and the return dmw value will tell us what cleanup needs + // to be done (if any) after the race winner: + // 1) A mutator trying to install a hash in the object. + // Note: That mutator is not executing this code, but it is + // trying to update the _header field. + // If winner: dmw will contain the hash and be unmarked + // 2a) A mutator trying to acquire the monitor via enter(): + // If winner: dmw is marked and hash() == 0 + // 2b) The thread deflating the monitor via deflate_monitor_using_JT(): + // If winner: dmw is marked and hash() == 0 + dmw = (markOop) Atomic::cmpxchg(marked_dmw, &_header, dmw); + } + + if (dmw->is_marked()) { + // The dmw copy is marked which means a hash was not set by a racing + // thread. Clear the mark from the copy in preparation for possible + // restoration from this thread. + assert(dmw->hash() == 0, "must be 0: hash=" INTPTR_FORMAT, dmw->hash()); + dmw = dmw->set_unmarked(); + } + assert(dmw->is_neutral(), "must be a neutral markword"); + + oop const obj = (oop) object(); + // Install displaced markword if object markword still points to this + // monitor. Both the mutator trying to enter() and the thread deflating + // the monitor will reach this point, but only one can win. + // Note: If a mutator won the cmpxchg() race above and installed a hash + // in _header, then the updated dmw contains that hash and we'll install + // it in the object's markword here. + obj->cas_set_mark(dmw, markOopDesc::encode(this)); +} + #define MAX_RECHECK_INTERVAL 1000 void ObjectMonitor::EnterI(TRAPS) { @@ -428,6 +498,18 @@ return; } + if (_owner == DEFLATER_MARKER) { + guarantee(0 < _count, "_owner == DEFLATER_MARKER && _count <= 0 should have been handled by the caller"); + // Deflater thread tried to lock this monitor, but it failed to make _count negative and gave up. + // Try to acquire monitor. + if (Atomic::cmpxchg(Self, &_owner, DEFLATER_MARKER) == DEFLATER_MARKER) { + assert(_succ != Self, "invariant"); + assert(_owner == Self, "invariant"); + assert(_Responsible != Self, "invariant"); + return; + } + } + assert(InitDone, "Unexpectedly not initialized"); // We try one round of spinning *before* enqueueing Self. @@ -544,6 +626,15 @@ if (TryLock(Self) > 0) break; + if (_owner == DEFLATER_MARKER) { + guarantee(0 < _count, "_owner == DEFLATER_MARKER && _count <= 0 should have been handled by the caller"); + // Deflater thread tried to lock this monitor, but it failed to make _count negative and gave up. + if (Atomic::cmpxchg(Self, &_owner, DEFLATER_MARKER) == DEFLATER_MARKER) { + // Acquired the monitor. + break; + } + } + // The lock is still contested. // Keep a tally of the # of futile wakeups. // Note that the counter is not protected by a lock or updated by atomics. @@ -665,6 +756,14 @@ if (TryLock(Self) > 0) break; if (TrySpin(Self) > 0) break; + if (_owner == DEFLATER_MARKER) { + guarantee(0 <= _count, "Impossible: _owner == DEFLATER_MARKER && _count < 0, monitor must not be owned by deflater thread here"); + if (Atomic::cmpxchg(Self, &_owner, DEFLATER_MARKER) == DEFLATER_MARKER) { + // Acquired the monitor. + break; + } + } + // State transition wrappers around park() ... // ReenterI() wisely defers state transitions until // it's clear we must park the thread. @@ -1122,16 +1221,20 @@ // reenter() enters a lock and sets recursion count // complete_exit/reenter operate as a wait without waiting -void ObjectMonitor::reenter(intptr_t recursions, TRAPS) { +bool ObjectMonitor::reenter(intptr_t recursions, TRAPS) { Thread * const Self = THREAD; assert(Self->is_Java_thread(), "Must be Java thread!"); JavaThread *jt = (JavaThread *)THREAD; guarantee(_owner != Self, "reenter already owner"); - enter(THREAD); // enter the monitor + if (!enter(THREAD)) { + // Failed to enter the monitor so return for a retry. + return false; + } + // Entered the monitor. guarantee(_recursions == 0, "reenter recursion"); _recursions = recursions; - return; + return true; } @@ -1359,7 +1462,8 @@ assert(_owner != Self, "invariant"); ObjectWaiter::TStates v = node.TState; if (v == ObjectWaiter::TS_RUN) { - enter(Self); + const bool success = enter(Self); + guarantee(success, "enter signaled for a retry, but monitor should not have been deflated as waiters > 0"); } else { guarantee(v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant"); ReenterI(Self, &node); @@ -1922,3 +2026,70 @@ DEBUG_ONLY(InitDone = true;) } + +// For internal used by ObjectSynchronizer::monitors_iterate(). +ObjectMonitorHandle::ObjectMonitorHandle(ObjectMonitor * om_ptr) { + om_ptr->inc_ref_count(); + _om_ptr = om_ptr; +} + +ObjectMonitorHandle::~ObjectMonitorHandle() { + if (_om_ptr != NULL) { + _om_ptr->dec_ref_count(); + _om_ptr = NULL; + } +} + +// Save the ObjectMonitor* associated with the specified markOop and +// increment the ref_count. This function should only be called if +// the caller has verified mark->has_monitor() == true. The object +// parameter is needed to verify that ObjectMonitor* has not been +// deflated and reused for another object. +// +// This function returns true if the ObjectMonitor* has been safely +// saved. This function returns false if we have lost a race with +// async deflation; the caller should retry as appropriate. +// +bool ObjectMonitorHandle::save_om_ptr(oop object, markOop mark) { + guarantee(mark->has_monitor(), "sanity check: mark=" INTPTR_FORMAT, + p2i((address)mark)); + + ObjectMonitor * om_ptr = mark->monitor(); + om_ptr->inc_ref_count(); + + if (AsyncDeflateIdleMonitors) { + // Race here if monitor is not owned! The above ref_count bump + // will cause subsequent async deflation to skip it. However, + // previous or concurrent async deflation is a race. + if (om_ptr->_owner == DEFLATER_MARKER) { + // Async deflation won the race so we have to retry. + om_ptr->dec_ref_count(); + return false; + } + // The ObjectMonitor could have been deflated and reused for + // another object before we bumped the ref_count so make sure + // our object still refers to this ObjectMonitor. + const markOop tmp = object->mark(); + if (!tmp->has_monitor() || tmp->monitor() != om_ptr) { + // Async deflation and reuse won the race so we have to retry. + om_ptr->dec_ref_count(); + return false; + } + } + + guarantee(_om_ptr == NULL, "sanity check: _om_ptr=" INTPTR_FORMAT, + p2i(_om_ptr)); + _om_ptr = om_ptr; + return true; +} + +// For internal use by ObjectSynchronizer::inflate(). +void ObjectMonitorHandle::set_om_ptr(ObjectMonitor * om_ptr) { + // Cannot guarantee() is_new() here. As soon as the ObjectMonitor* + // is attached to the object in inflate(), it can be used by other + // JavaThreads. + // guarantee(om_ptr->is_new(), "sanity check: allocation_state=%d", + // int(om_ptr->allocation_state())); + om_ptr->inc_ref_count(); + _om_ptr = om_ptr; +} --- old/src/hotspot/share/runtime/objectMonitor.hpp 2019-03-22 16:30:36.805896833 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.hpp 2019-03-22 16:30:36.393896840 -0400 @@ -136,6 +136,7 @@ }; private: + friend class ObjectMonitorHandle; friend class ObjectSynchronizer; friend class ObjectWaiter; friend class VMStructs; @@ -149,6 +150,8 @@ sizeof(volatile markOop) + sizeof(void * volatile) + sizeof(ObjectMonitor *)); protected: // protected for JvmtiRawMonitor + // Used by async monitor deflation as a marker in the _owner field: + #define DEFLATER_MARKER reinterpret_cast(-1) void * volatile _owner; // pointer to owning thread OR BasicLock volatile jlong _previous_owner_tid; // thread id of the previous owner of the monitor volatile intptr_t _recursions; // recursion count, 0 for first entry @@ -171,6 +174,13 @@ volatile jint _waiters; // number of waiting threads private: volatile int _WaitSetLock; // protects Wait Queue - simple spinlock + volatile jint _ref_count; // ref count for ObjectMonitor* + typedef enum { + Free = 0, // Free must be 0 for monitor to be free after memset(..,0,..). + New, + Old + } AllocationState; + AllocationState _allocation_state; public: static void Initialize(); @@ -235,12 +245,33 @@ // TODO-FIXME: merge _count and _waiters. // TODO-FIXME: assert _owner == null implies _recursions = 0 // TODO-FIXME: assert _WaitSet != null implies _count > 0 + // We do not include _ref_count in the is_busy() check because + // _ref_count is for indicating that the ObjectMonitor* is in + // use which is orthogonal to whether the ObjectMonitor itself + // is in use for a locking operation. return _count|_waiters|intptr_t(_owner)|intptr_t(_cxq)|intptr_t(_EntryList); } + // Version of is_busy() that accounts for special values in + // _count and _owner when AsyncDeflateIdleMonitors is enabled. + intptr_t is_busy_async() const { + intptr_t ret_code = _waiters | intptr_t(_cxq) | intptr_t(_EntryList); + if (!AsyncDeflateIdleMonitors) { + ret_code |= _count | intptr_t(_owner); + } else { + if (_count > 0) { + ret_code |= _count; + } + if (_owner != DEFLATER_MARKER) { + ret_code |= intptr_t(_owner); + } + } + return ret_code; + } + intptr_t is_entered(Thread* current) const; - void* owner() const; + void* owner() const; // Returns NULL if DEFLATER_MARKER is observed. void set_owner(void* owner); jint waiters() const; @@ -285,12 +316,22 @@ void* object() const; void* object_addr(); void set_object(void* obj); + void set_allocation_state(AllocationState s); + AllocationState allocation_state() const; + bool is_free() const; + bool is_active() const; + bool is_old() const; + bool is_new() const; + void dec_ref_count(); + void inc_ref_count(); + jint ref_count() const; bool check(TRAPS); // true if the thread owns the monitor. void check_slow(TRAPS); void clear(); + void clear_using_JT(); - void enter(TRAPS); + bool enter(TRAPS); // Returns false if monitor is being async deflated and caller should retry locking the object. void exit(bool not_suspended, TRAPS); void wait(jlong millis, bool interruptable, TRAPS); void notify(TRAPS); @@ -298,7 +339,7 @@ // Use the following at your own risk intptr_t complete_exit(TRAPS); - void reenter(intptr_t recursions, TRAPS); + bool reenter(intptr_t recursions, TRAPS); // Returns false if monitor is being async deflated and caller should retry locking the object. private: void AddWaiter(ObjectWaiter * waiter); @@ -313,6 +354,27 @@ int TrySpin(Thread * Self); void ExitEpilog(Thread * Self, ObjectWaiter * Wakee); bool ExitSuspendEquivalent(JavaThread * Self); + void install_displaced_markword_in_object(); +}; + +// A helper object for managing an ObjectMonitor*'s ref_count. There +// are special safety considerations when async deflation is used. +class ObjectMonitorHandle : public StackObj { + private: + ObjectMonitor * _om_ptr; + public: + ObjectMonitorHandle() { _om_ptr = NULL; } + ~ObjectMonitorHandle(); + + ObjectMonitor * om_ptr() const { return _om_ptr; } + // Save the ObjectMonitor* associated with the specified markOop and + // increment the ref_count. + bool save_om_ptr(oop object, markOop mark); + + // For internal used by ObjectSynchronizer::monitors_iterate(). + ObjectMonitorHandle(ObjectMonitor * _om_ptr); + // For internal use by ObjectSynchronizer::inflate(). + void set_om_ptr(ObjectMonitor * om_ptr); }; #endif // SHARE_RUNTIME_OBJECTMONITOR_HPP --- old/src/hotspot/share/runtime/objectMonitor.inline.hpp 2019-03-22 16:30:37.789896816 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.inline.hpp 2019-03-22 16:30:37.385896823 -0400 @@ -53,20 +53,40 @@ return _waiters; } +// Returns NULL if DEFLATER_MARKER is observed. inline void* ObjectMonitor::owner() const { - return _owner; + void* owner = _owner; + return owner != DEFLATER_MARKER ? owner : NULL; } inline void ObjectMonitor::clear() { - assert(_header != NULL, "Fatal logic error in ObjectMonitor header!"); assert(_count == 0, "Fatal logic error in ObjectMonitor count!"); + assert(_owner == NULL, "Fatal logic error in ObjectMonitor owner!"); + + clear_using_JT(); +} + +inline void ObjectMonitor::clear_using_JT() { + // When clearing using a JavaThread, we leave _owner == DEFLATER_MARKER + // and _count < 0 to force any racing threads to retry. Unlike other + // *_using_JT() functions, we cannot assert AsyncDeflateIdleMonitors + // or Thread::current()->is_Java_thread() because clear() calls this + // function for the rest of its checks. + + assert(_header != NULL, "Fatal logic error in ObjectMonitor header!"); assert(_waiters == 0, "Fatal logic error in ObjectMonitor waiters!"); assert(_recursions == 0, "Fatal logic error in ObjectMonitor recursions!"); assert(_object != NULL, "Fatal logic error in ObjectMonitor object!"); - assert(_owner == NULL, "Fatal logic error in ObjectMonitor owner!"); + // Do not assert _ref_count == 0 here because a racing thread could + // increment _ref_count, observe _owner == DEFLATER_MARKER and then + // decrement _ref_count. + set_allocation_state(Free); _header = NULL; _object = NULL; + // Do not clear _ref_count here because _ref_count is for indicating + // that the ObjectMonitor* is in use which is orthogonal to whether + // the ObjectMonitor itself is in use for a locking operation. } inline void* ObjectMonitor::object() const { @@ -107,4 +127,59 @@ _recursions = 0; } +inline void ObjectMonitor::set_allocation_state(ObjectMonitor::AllocationState s) { + _allocation_state = s; +} + +inline ObjectMonitor::AllocationState ObjectMonitor::allocation_state() const { + return _allocation_state; +} + +inline bool ObjectMonitor::is_free() const { + return _allocation_state == Free; +} + +inline bool ObjectMonitor::is_active() const { + return !is_free(); +} + +inline bool ObjectMonitor::is_old() const { + return _allocation_state == Old; +} + +inline bool ObjectMonitor::is_new() const { + return _allocation_state == New; +} + +inline void ObjectMonitor::dec_ref_count() { + // The decrement needs to be MO_ACQ_REL. At the moment, the Atomic::dec + // backend on PPC does not yet conform to these requirements. Therefore + // the decrement is simulated with an Atomic::sub(1, &addr). Without + // this MO_ACQ_REL Atomic::dec simulation, AsyncDeflateIdleMonitors is + // not safe. + Atomic::sub((jint)1, &_ref_count); + guarantee(_ref_count >= 0, "sanity check: ref_count=%d", _ref_count); +} + +inline void ObjectMonitor::inc_ref_count() { + // The increment needs to be MO_SEQ_CST. At the moment, the Atomic::inc + // backend on PPC does not yet conform to these requirements. Therefore + // the increment is simulated with a load phi; cas phi + 1; loop. + // Without this MO_SEQ_CST Atomic::inc simulation, AsyncDeflateIdleMonitors + // is not safe. + for (;;) { + jint sample = OrderAccess::load_acquire(&_ref_count); + guarantee(sample >= 0, "sanity check: sample=%d", (int)sample); + if (Atomic::cmpxchg(sample + 1, &_ref_count, sample) == sample) { + // Incremented _ref_count without interference. + return; + } + // Implied else: Saw interference so loop and try again. + } +} + +inline jint ObjectMonitor::ref_count() const { + return OrderAccess::load_acquire(&_ref_count); +} + #endif // SHARE_RUNTIME_OBJECTMONITOR_INLINE_HPP --- old/src/hotspot/share/runtime/safepoint.cpp 2019-03-22 16:30:38.645896801 -0400 +++ new/src/hotspot/share/runtime/safepoint.cpp 2019-03-22 16:30:38.273896808 -0400 @@ -556,7 +556,11 @@ const char* name = "deflating global idle monitors"; EventSafepointCleanupTask event; TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); - ObjectSynchronizer::deflate_idle_monitors(_counters); + // AsyncDeflateIdleMonitors only uses DeflateMonitorCounters + // when a special cleanup has been requested. + // Note: This logging output will include global idle monitor + // elapsed times, but not global idle monitor deflation count. + ObjectSynchronizer::do_safepoint_work(!AsyncDeflateIdleMonitors ? _counters : NULL); post_safepoint_cleanup_task_event(event, safepoint_id, name); } --- old/src/hotspot/share/runtime/serviceThread.cpp 2019-03-22 16:30:39.577896785 -0400 +++ new/src/hotspot/share/runtime/serviceThread.cpp 2019-03-22 16:30:39.189896792 -0400 @@ -126,6 +126,7 @@ bool protection_domain_table_work = false; bool oopstorage_work = false; bool oopstorages_cleanup[oopstorage_count] = {}; // Zero (false) initialize. + bool deflate_idle_monitors = false; JvmtiDeferredEvent jvmti_event; { // Need state transition ThreadBlockInVM so that this thread @@ -153,9 +154,9 @@ (protection_domain_table_work = SystemDictionary::pd_cache_table()->has_work()) | (oopstorage_work = needs_oopstorage_cleanup(oopstorages, oopstorages_cleanup, - oopstorage_count))) - - == 0) { + oopstorage_count)) | + (deflate_idle_monitors = ObjectSynchronizer::gOmShouldDeflateIdleMonitors()) + ) == 0) { // Wait until notified that there is some work to do. ml.wait(Mutex::_no_safepoint_check_flag); } @@ -200,6 +201,13 @@ if (oopstorage_work) { cleanup_oopstorages(oopstorages, oopstorages_cleanup, oopstorage_count); } + + if (deflate_idle_monitors) { + // Deflate any global idle monitors. + // deflate_per_thread_idle_monitors_using_JT() is called by + // each JavaThread from ObjectSynchronizer::omAlloc() as needed. + ObjectSynchronizer::deflate_global_idle_monitors_using_JT(); + } } } --- old/src/hotspot/share/runtime/sharedRuntime.cpp 2019-03-22 16:30:40.557896768 -0400 +++ new/src/hotspot/share/runtime/sharedRuntime.cpp 2019-03-22 16:30:40.153896775 -0400 @@ -64,8 +64,10 @@ #include "runtime/interfaceSupport.inline.hpp" #include "runtime/java.hpp" #include "runtime/javaCalls.hpp" +#include "runtime/objectMonitor.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" +#include "runtime/synchronizer.hpp" #include "runtime/vframe.inline.hpp" #include "runtime/vframeArray.hpp" #include "utilities/copy.hpp" @@ -3110,9 +3112,13 @@ kptr2 = fr.next_monitor_in_interpreter_frame(kptr2) ) { if (kptr2->obj() != NULL) { // Avoid 'holes' in the monitor array BasicLock *lock = kptr2->lock(); + // Disallow async deflation of the inflated monitor so the + // displaced header stays stable until we've copied it. + ObjectMonitorHandle omh; // Inflate so the displaced header becomes position-independent - if (lock->displaced_header()->is_unlocked()) - ObjectSynchronizer::inflate_helper(kptr2->obj()); + if (lock->displaced_header()->is_unlocked()) { + ObjectSynchronizer::inflate_helper(&omh, kptr2->obj()); + } // Now the displaced header is free to move buf[i++] = (intptr_t)lock->displaced_header(); buf[i++] = cast_from_oop(kptr2->obj()); --- old/src/hotspot/share/runtime/synchronizer.cpp 2019-03-22 16:30:41.285896756 -0400 +++ new/src/hotspot/share/runtime/synchronizer.cpp 2019-03-22 16:30:40.961896761 -0400 @@ -124,6 +124,8 @@ ObjectMonitor * volatile ObjectSynchronizer::gOmInUseList = NULL; // count of entries in gOmInUseList int ObjectSynchronizer::gOmInUseCount = 0; +bool ObjectSynchronizer::_gOmShouldDeflateIdleMonitors = false; +bool volatile ObjectSynchronizer::_is_cleanup_requested = false; static volatile intptr_t gListLock = 0; // protects global monitor lists static volatile int gMonitorFreeCount = 0; // # on gFreeList @@ -210,40 +212,50 @@ assert(((JavaThread *) Self)->thread_state() == _thread_in_Java, "invariant"); NoSafepointVerifier nsv; if (obj == NULL) return false; // Need to throw NPE - const markOop mark = obj->mark(); - if (mark->has_monitor()) { - ObjectMonitor * const m = mark->monitor(); - assert(oopDesc::equals((oop) m->object(), obj), "invariant"); - Thread * const owner = (Thread *) m->_owner; - - // Lock contention and Transactional Lock Elision (TLE) diagnostics - // and observability - // Case: light contention possibly amenable to TLE - // Case: TLE inimical operations such as nested/recursive synchronization - - if (owner == Self) { - m->_recursions++; - return true; - } - - // This Java Monitor is inflated so obj's header will never be - // displaced to this thread's BasicLock. Make the displaced header - // non-NULL so this BasicLock is not seen as recursive nor as - // being locked. We do this unconditionally so that this thread's - // BasicLock cannot be mis-interpreted by any stack walkers. For - // performance reasons, stack walkers generally first check for - // Biased Locking in the object's header, the second check is for - // stack-locking in the object's header, the third check is for - // recursive stack-locking in the displaced header in the BasicLock, - // and last are the inflated Java Monitor (ObjectMonitor) checks. - lock->set_displaced_header(markOopDesc::unused_mark()); + while (true) { + const markOop mark = obj->mark(); + + if (mark->has_monitor()) { + ObjectMonitorHandle omh; + if (!omh.save_om_ptr(obj, mark)) { + // Lost a race with async deflation so try again. + assert(AsyncDeflateIdleMonitors, "sanity check"); + continue; + } + ObjectMonitor * const m = omh.om_ptr(); + assert(oopDesc::equals((oop) m->object(), obj), "invariant"); + Thread * const owner = (Thread *) m->_owner; + + // Lock contention and Transactional Lock Elision (TLE) diagnostics + // and observability + // Case: light contention possibly amenable to TLE + // Case: TLE inimical operations such as nested/recursive synchronization + + if (owner == Self) { + m->_recursions++; + return true; + } - if (owner == NULL && Atomic::replace_if_null(Self, &(m->_owner))) { - assert(m->_recursions == 0, "invariant"); - assert(m->_owner == Self, "invariant"); - return true; + // This Java Monitor is inflated so obj's header will never be + // displaced to this thread's BasicLock. Make the displaced header + // non-NULL so this BasicLock is not seen as recursive nor as + // being locked. We do this unconditionally so that this thread's + // BasicLock cannot be mis-interpreted by any stack walkers. For + // performance reasons, stack walkers generally first check for + // Biased Locking in the object's header, the second check is for + // stack-locking in the object's header, the third check is for + // recursive stack-locking in the displaced header in the BasicLock, + // and last are the inflated Java Monitor (ObjectMonitor) checks. + lock->set_displaced_header(markOopDesc::unused_mark()); + + if (owner == NULL && Atomic::replace_if_null(Self, &(m->_owner))) { + assert(m->_recursions == 0, "invariant"); + assert(m->_owner == Self, "invariant"); + return true; + } } + break; } // Note that we could inflate in quick_enter. @@ -327,7 +339,9 @@ } // We have to take the slow-path of possible inflation and then exit. - inflate(THREAD, object, inflate_cause_vm_internal)->exit(true, THREAD); + ObjectMonitorHandle omh; + inflate(&omh, THREAD, object, inflate_cause_vm_internal); + omh.om_ptr()->exit(true, THREAD); } // ----------------------------------------------------------------------------- @@ -336,31 +350,36 @@ // We don't need to use fast path here, because it must have been // failed in the interpreter/compiler code. void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) { - markOop mark = obj->mark(); - assert(!mark->has_bias_pattern(), "should not see bias pattern here"); + bool do_loop = true; + while (do_loop) { + markOop mark = obj->mark(); + assert(!mark->has_bias_pattern(), "should not see bias pattern here"); - if (mark->is_neutral()) { - // Anticipate successful CAS -- the ST of the displaced mark must - // be visible <= the ST performed by the CAS. - lock->set_displaced_header(mark); - if (mark == obj()->cas_set_mark((markOop) lock, mark)) { + if (mark->is_neutral()) { + // Anticipate successful CAS -- the ST of the displaced mark must + // be visible <= the ST performed by the CAS. + lock->set_displaced_header(mark); + if (mark == obj()->cas_set_mark((markOop) lock, mark)) { + return; + } + // Fall through to inflate() ... + } else if (mark->has_locker() && + THREAD->is_lock_owned((address)mark->locker())) { + assert(lock != mark->locker(), "must not re-lock the same lock"); + assert(lock != (BasicLock*)obj->mark(), "don't relock with same BasicLock"); + lock->set_displaced_header(NULL); return; } - // Fall through to inflate() ... - } else if (mark->has_locker() && - THREAD->is_lock_owned((address)mark->locker())) { - assert(lock != mark->locker(), "must not re-lock the same lock"); - assert(lock != (BasicLock*)obj->mark(), "don't relock with same BasicLock"); - lock->set_displaced_header(NULL); - return; - } - // The object header will never be displaced to this lock, - // so it does not matter what the value is, except that it - // must be non-zero to avoid looking like a re-entrant lock, - // and must not look locked either. - lock->set_displaced_header(markOopDesc::unused_mark()); - inflate(THREAD, obj(), inflate_cause_monitor_enter)->enter(THREAD); + // The object header will never be displaced to this lock, + // so it does not matter what the value is, except that it + // must be non-zero to avoid looking like a re-entrant lock, + // and must not look locked either. + lock->set_displaced_header(markOopDesc::unused_mark()); + ObjectMonitorHandle omh; + inflate(&omh, THREAD, obj(), inflate_cause_monitor_enter); + do_loop = !omh.om_ptr()->enter(THREAD); + } } // This routine is used to handle interpreter/compiler slow case @@ -389,9 +408,10 @@ assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); } - ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal); - - return monitor->complete_exit(THREAD); + ObjectMonitorHandle omh; + inflate(&omh, THREAD, obj(), inflate_cause_vm_internal); + intptr_t ret_code = omh.om_ptr()->complete_exit(THREAD); + return ret_code; } // NOTE: must use heavy weight monitor to handle complete_exit/reenter() @@ -401,9 +421,12 @@ assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); } - ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal); - - monitor->reenter(recursion, THREAD); + bool do_loop = true; + while (do_loop) { + ObjectMonitorHandle omh; + inflate(&omh, THREAD, obj(), inflate_cause_vm_internal); + do_loop = !omh.om_ptr()->reenter(recursion, THREAD); + } } // ----------------------------------------------------------------------------- // JNI locks on java objects @@ -415,7 +438,12 @@ assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); } THREAD->set_current_pending_monitor_is_from_java(false); - inflate(THREAD, obj(), inflate_cause_jni_enter)->enter(THREAD); + bool do_loop = true; + while (do_loop) { + ObjectMonitorHandle omh; + inflate(&omh, THREAD, obj(), inflate_cause_jni_enter); + do_loop = !omh.om_ptr()->enter(THREAD); + } THREAD->set_current_pending_monitor_is_from_java(true); } @@ -428,7 +456,9 @@ } assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now"); - ObjectMonitor* monitor = inflate(THREAD, obj, inflate_cause_jni_exit); + ObjectMonitorHandle omh; + inflate(&omh, THREAD, obj, inflate_cause_jni_exit); + ObjectMonitor * monitor = omh.om_ptr(); // If this thread has locked the object, exit the monitor. Note: can't use // monitor->check(CHECK); must exit even if an exception is pending. if (monitor->check(THREAD)) { @@ -468,7 +498,9 @@ if (millis < 0) { THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative"); } - ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_wait); + ObjectMonitorHandle omh; + inflate(&omh, THREAD, obj(), inflate_cause_wait); + ObjectMonitor * monitor = omh.om_ptr(); DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis); monitor->wait(millis, true, THREAD); @@ -477,7 +509,8 @@ // that's fixed we can uncomment the following line, remove the call // and change this function back into a "void" func. // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD); - return dtrace_waited_probe(monitor, obj, THREAD); + int ret_code = dtrace_waited_probe(monitor, obj, THREAD); + return ret_code; } void ObjectSynchronizer::waitUninterruptibly(Handle obj, jlong millis, TRAPS) { @@ -488,7 +521,9 @@ if (millis < 0) { THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative"); } - inflate(THREAD, obj(), inflate_cause_wait)->wait(millis, false, THREAD); + ObjectMonitorHandle omh; + inflate(&omh, THREAD, obj(), inflate_cause_wait); + omh.om_ptr()->wait(millis, false, THREAD); } void ObjectSynchronizer::notify(Handle obj, TRAPS) { @@ -501,7 +536,9 @@ if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) { return; } - inflate(THREAD, obj(), inflate_cause_notify)->notify(THREAD); + ObjectMonitorHandle omh; + inflate(&omh, THREAD, obj(), inflate_cause_notify); + omh.om_ptr()->notify(THREAD); } // NOTE: see comment of notify() @@ -515,7 +552,9 @@ if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) { return; } - inflate(THREAD, obj(), inflate_cause_notify)->notifyAll(THREAD); + ObjectMonitorHandle omh; + inflate(&omh, THREAD, obj(), inflate_cause_notify); + omh.om_ptr()->notifyAll(THREAD); } // ----------------------------------------------------------------------------- @@ -709,6 +748,7 @@ assert(Universe::verify_in_progress() || DumpSharedSpaces || ((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant"); + Retry: ObjectMonitor* monitor = NULL; markOop temp, test; intptr_t hash; @@ -733,7 +773,13 @@ // into heavy weight monitor. We could add more code here // for fast path, but it does not worth the complexity. } else if (mark->has_monitor()) { - monitor = mark->monitor(); + ObjectMonitorHandle omh; + if (!omh.save_om_ptr(obj, mark)) { + // Lost a race with async deflation so try again. + assert(AsyncDeflateIdleMonitors, "sanity check"); + goto Retry; + } + monitor = omh.om_ptr(); temp = monitor->header(); assert(temp->is_neutral(), "invariant: header=" INTPTR_FORMAT, p2i((address)temp)); hash = temp->hash(); @@ -760,7 +806,9 @@ } // Inflate the monitor to set hash code - monitor = inflate(Self, obj, inflate_cause_hash_code); + ObjectMonitorHandle omh; + inflate(&omh, Self, obj, inflate_cause_hash_code); + monitor = omh.om_ptr(); // Load displaced header and check it has hash code mark = monitor->header(); assert(mark->is_neutral(), "invariant: header=" INTPTR_FORMAT, p2i((address)mark)); @@ -800,20 +848,28 @@ assert(thread == JavaThread::current(), "Can only be called on current thread"); oop obj = h_obj(); - markOop mark = ReadStableMark(obj); + while (true) { + markOop mark = ReadStableMark(obj); - // Uncontended case, header points to stack - if (mark->has_locker()) { - return thread->is_lock_owned((address)mark->locker()); - } - // Contended case, header points to ObjectMonitor (tagged pointer) - if (mark->has_monitor()) { - ObjectMonitor* monitor = mark->monitor(); - return monitor->is_entered(thread) != 0; + // Uncontended case, header points to stack + if (mark->has_locker()) { + return thread->is_lock_owned((address)mark->locker()); + } + // Contended case, header points to ObjectMonitor (tagged pointer) + if (mark->has_monitor()) { + ObjectMonitorHandle omh; + if (!omh.save_om_ptr(obj, mark)) { + // Lost a race with async deflation so try again. + assert(AsyncDeflateIdleMonitors, "sanity check"); + continue; + } + bool ret_code = omh.om_ptr()->is_entered(thread) != 0; + return ret_code; + } + // Unlocked case, header in place + assert(mark->is_neutral(), "sanity check"); + return false; } - // Unlocked case, header in place - assert(mark->is_neutral(), "sanity check"); - return false; } // Be aware of this method could revoke bias of the lock object. @@ -839,27 +895,37 @@ assert(self == JavaThread::current(), "Can only be called on current thread"); oop obj = h_obj(); - markOop mark = ReadStableMark(obj); - // CASE: stack-locked. Mark points to a BasicLock on the owner's stack. - if (mark->has_locker()) { - return self->is_lock_owned((address)mark->locker()) ? - owner_self : owner_other; - } + while (true) { + markOop mark = ReadStableMark(obj); - // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor. - // The Object:ObjectMonitor relationship is stable as long as we're - // not at a safepoint. - if (mark->has_monitor()) { - void * owner = mark->monitor()->_owner; - if (owner == NULL) return owner_none; - return (owner == self || - self->is_lock_owned((address)owner)) ? owner_self : owner_other; - } + // CASE: stack-locked. Mark points to a BasicLock on the owner's stack. + if (mark->has_locker()) { + return self->is_lock_owned((address)mark->locker()) ? + owner_self : owner_other; + } + + // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor. + // The Object:ObjectMonitor relationship is stable as long as we're + // not at a safepoint and AsyncDeflateIdleMonitors is false. + if (mark->has_monitor()) { + ObjectMonitorHandle omh; + if (!omh.save_om_ptr(obj, mark)) { + // Lost a race with async deflation so try again. + assert(AsyncDeflateIdleMonitors, "sanity check"); + continue; + } + ObjectMonitor * monitor = omh.om_ptr(); + void * owner = monitor->_owner; + if (owner == NULL) return owner_none; + return (owner == self || + self->is_lock_owned((address)owner)) ? owner_self : owner_other; + } - // CASE: neutral - assert(mark->is_neutral(), "sanity check"); - return owner_none; // it's unlocked + // CASE: neutral + assert(mark->is_neutral(), "sanity check"); + return owner_none; // it's unlocked + } } // FIXME: jvmti should call this @@ -874,33 +940,41 @@ } oop obj = h_obj(); - address owner = NULL; - markOop mark = ReadStableMark(obj); + while (true) { + address owner = NULL; + markOop mark = ReadStableMark(obj); - // Uncontended case, header points to stack - if (mark->has_locker()) { - owner = (address) mark->locker(); - } + // Uncontended case, header points to stack + if (mark->has_locker()) { + owner = (address) mark->locker(); + } - // Contended case, header points to ObjectMonitor (tagged pointer) - else if (mark->has_monitor()) { - ObjectMonitor* monitor = mark->monitor(); - assert(monitor != NULL, "monitor should be non-null"); - owner = (address) monitor->owner(); - } + // Contended case, header points to ObjectMonitor (tagged pointer) + else if (mark->has_monitor()) { + ObjectMonitorHandle omh; + if (!omh.save_om_ptr(obj, mark)) { + // Lost a race with async deflation so try again. + assert(AsyncDeflateIdleMonitors, "sanity check"); + continue; + } + ObjectMonitor* monitor = omh.om_ptr(); + assert(monitor != NULL, "monitor should be non-null"); + owner = (address) monitor->owner(); + } - if (owner != NULL) { - // owning_thread_from_monitor_owner() may also return NULL here - return Threads::owning_thread_from_monitor_owner(t_list, owner); - } + if (owner != NULL) { + // owning_thread_from_monitor_owner() may also return NULL here + return Threads::owning_thread_from_monitor_owner(t_list, owner); + } - // Unlocked case, header in place - // Cannot have assertion since this object may have been - // locked by another thread when reaching here. - // assert(mark->is_neutral(), "sanity check"); + // Unlocked case, header in place + // Cannot have assertion since this object may have been + // locked by another thread when reaching here. + // assert(mark->is_neutral(), "sanity check"); - return NULL; + return NULL; + } } // Visitors ... @@ -911,8 +985,18 @@ assert(block->object() == CHAINMARKER, "must be a block header"); for (int i = _BLOCKSIZE - 1; i > 0; i--) { ObjectMonitor* mid = (ObjectMonitor *)(block + i); - oop object = (oop)mid->object(); - if (object != NULL) { + if (mid->is_active()) { + ObjectMonitorHandle omh(mid); + + if (mid->object() == NULL || + (AsyncDeflateIdleMonitors && mid->_owner == DEFLATER_MARKER)) { + // Only process with closure if the object is set. + // For async deflation, race here if monitor is not owned! + // The above ref_count bump (in ObjectMonitorHandle ctr) + // will cause subsequent async deflation to skip it. + // However, previous or concurrent async deflation is a race. + continue; + } closure->do_monitor(mid); } } @@ -1023,13 +1107,28 @@ } } -ObjectMonitor* ObjectSynchronizer::omAlloc(Thread * Self) { +ObjectMonitor* ObjectSynchronizer::omAlloc(Thread * Self, + const InflateCause cause) { // A large MAXPRIVATE value reduces both list lock contention // and list coherency traffic, but also tends to increase the // number of objectMonitors in circulation as well as the STW // scavenge costs. As usual, we lean toward time in space-time // tradeoffs. const int MAXPRIVATE = 1024; + + if (AsyncDeflateIdleMonitors) { + JavaThread * jt = (JavaThread *)Self; + if (jt->omShouldDeflateIdleMonitors && jt->omInUseCount > 0 && + cause != inflate_cause_vm_internal) { + // Deflate any per-thread idle monitors for this JavaThread if + // this is not an internal inflation. Clean up your own mess. + // (Gibbs Rule 45) Otherwise, skip this cleanup. + // deflate_global_idle_monitors_using_JT() is called by the ServiceThread. + debug_only(jt->check_for_valid_safepoint_state(false);) + ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(); + } + } + for (;;) { ObjectMonitor * m; @@ -1044,6 +1143,7 @@ Self->omFreeList = m->FreeNext; Self->omFreeCount--; guarantee(m->object() == NULL, "invariant"); + m->set_allocation_state(ObjectMonitor::New); m->FreeNext = Self->omInUseList; Self->omInUseList = m; Self->omInUseCount++; @@ -1065,8 +1165,13 @@ ObjectMonitor * take = gFreeList; gFreeList = take->FreeNext; guarantee(take->object() == NULL, "invariant"); + if (AsyncDeflateIdleMonitors) { + take->set_owner(NULL); + take->_count = 0; + } guarantee(!take->is_busy(), "invariant"); take->Recycle(); + assert(take->is_free(), "invariant"); omRelease(Self, take, false); } Thread::muxRelease(&gListLock); @@ -1119,6 +1224,7 @@ for (int i = 1; i < _BLOCKSIZE; i++) { temp[i].FreeNext = (ObjectMonitor *)&temp[i+1]; + assert(temp[i].is_free(), "invariant"); } // terminate the last monitor as the end of list @@ -1161,13 +1267,15 @@ // // Key constraint: all ObjectMonitors on a thread's free list and the global // free list must have their object field set to null. This prevents the -// scavenger -- deflate_monitor_list() -- from reclaiming them. +// scavenger -- deflate_monitor_list() or deflate_monitor_list_using_JT() +// -- from reclaiming them while we are trying to release them. void ObjectSynchronizer::omRelease(Thread * Self, ObjectMonitor * m, bool fromPerThreadAlloc) { guarantee(m->header() == NULL, "invariant"); guarantee(m->object() == NULL, "invariant"); guarantee(((m->is_busy()|m->_recursions) == 0), "freeing in-use monitor"); + m->set_allocation_state(ObjectMonitor::Free); // Remove from omInUseList if (fromPerThreadAlloc) { ObjectMonitor* cur_mid_in_use = NULL; @@ -1190,6 +1298,7 @@ // FreeNext is used for both omInUseList and omFreeList, so clear old before setting new m->FreeNext = Self->omFreeList; + guarantee(m->is_free(), "invariant"); Self->omFreeList = m; Self->omFreeCount++; } @@ -1213,6 +1322,10 @@ // either via Thread::oops_do() (if safepoint happens before omFlush()) or via // ObjectSynchronizer::oops_do() (if it happens after omFlush() and the thread's // monitors have been transferred to the global in-use list). +// +// With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT() +// and deflate_per_thread_idle_monitors_using_JT() (in another thread) can +// run at the same time as omFlush() so we have to be careful. void ObjectSynchronizer::omFlush(Thread * Self) { ObjectMonitor * list = Self->omFreeList; // Null-terminated SLL @@ -1232,7 +1345,7 @@ s->set_owner(NULL); // redundant but good hygiene } guarantee(tail != NULL, "invariant"); - assert(Self->omFreeCount == tally, "free-count off"); + guarantee(Self->omFreeCount == tally, "free-count off"); Self->omFreeList = NULL; Self->omFreeCount = 0; } @@ -1249,9 +1362,10 @@ for (cur_om = inUseList; cur_om != NULL; cur_om = cur_om->FreeNext) { inUseTail = cur_om; inUseTally++; + guarantee(cur_om->is_active(), "invariant"); } guarantee(inUseTail != NULL, "invariant"); - assert(Self->omInUseCount == inUseTally, "in-use count off"); + guarantee(Self->omInUseCount == inUseTally, "in-use count off"); Self->omInUseList = NULL; Self->omInUseCount = 0; } @@ -1299,19 +1413,28 @@ } // Fast path code shared by multiple functions -void ObjectSynchronizer::inflate_helper(oop obj) { - markOop mark = obj->mark(); - if (mark->has_monitor()) { - assert(ObjectSynchronizer::verify_objmon_isinpool(mark->monitor()), "monitor is invalid"); - assert(mark->monitor()->header()->is_neutral(), "monitor must record a good object header"); +void ObjectSynchronizer::inflate_helper(ObjectMonitorHandle * omh_p, oop obj) { + while (true) { + markOop mark = obj->mark(); + if (mark->has_monitor()) { + if (!omh_p->save_om_ptr(obj, mark)) { + // Lost a race with async deflation so try again. + assert(AsyncDeflateIdleMonitors, "sanity check"); + continue; + } + ObjectMonitor * monitor = omh_p->om_ptr(); + assert(ObjectSynchronizer::verify_objmon_isinpool(monitor), "monitor is invalid"); + markOop dmw = monitor->header(); + assert(dmw->is_neutral(), "sanity check: header=" INTPTR_FORMAT, p2i((address)dmw)); + return; + } + inflate(omh_p, Thread::current(), obj, inflate_cause_vm_internal); return; } - inflate(Thread::current(), obj, inflate_cause_vm_internal); } -ObjectMonitor* ObjectSynchronizer::inflate(Thread * Self, - oop object, - const InflateCause cause) { +void ObjectSynchronizer::inflate(ObjectMonitorHandle * omh_p, Thread * Self, + oop object, const InflateCause cause) { // Inflate mutates the heap ... // Relaxing assertion for bug 6320749. assert(Universe::verify_in_progress() || @@ -1332,12 +1455,17 @@ // CASE: inflated if (mark->has_monitor()) { - ObjectMonitor * inf = mark->monitor(); + if (!omh_p->save_om_ptr(object, mark)) { + // Lost a race with async deflation so try again. + assert(AsyncDeflateIdleMonitors, "sanity check"); + continue; + } + ObjectMonitor * inf = omh_p->om_ptr(); markOop dmw = inf->header(); assert(dmw->is_neutral(), "invariant: header=" INTPTR_FORMAT, p2i((address)dmw)); assert(oopDesc::equals((oop) inf->object(), object), "invariant"); assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid"); - return inf; + return; } // CASE: inflation in progress - inflating over a stack-lock. @@ -1373,7 +1501,18 @@ LogStreamHandle(Trace, monitorinflation) lsh; if (mark->has_locker()) { - ObjectMonitor * m = omAlloc(Self); + ObjectMonitor * m; + if (!AsyncDeflateIdleMonitors || cause == inflate_cause_vm_internal) { + // If !AsyncDeflateIdleMonitors or if an internal inflation, then + // we won't stop for a potential safepoint in omAlloc. + m = omAlloc(Self, cause); + } else { + // If AsyncDeflateIdleMonitors and not an internal inflation, then + // we may stop for a safepoint in omAlloc() so protect object. + Handle h_obj(Self, object); + m = omAlloc(Self, cause); + object = h_obj(); // Refresh object. + } // Optimistically prepare the objectmonitor - anticipate successful CAS // We do this before the CAS in order to minimize the length of time // in which INFLATING appears in the mark. @@ -1450,7 +1589,9 @@ if (event.should_commit()) { post_monitor_inflate_event(&event, object, cause); } - return m; + assert(!m->is_free(), "post-condition"); + omh_p->set_om_ptr(m); + return; } // CASE: neutral @@ -1464,7 +1605,18 @@ // would be useful. assert(mark->is_neutral(), "invariant"); - ObjectMonitor * m = omAlloc(Self); + ObjectMonitor * m; + if (!AsyncDeflateIdleMonitors || cause == inflate_cause_vm_internal) { + // If !AsyncDeflateIdleMonitors or if an internal inflation, then + // we won't stop for a potential safepoint in omAlloc. + m = omAlloc(Self, cause); + } else { + // If AsyncDeflateIdleMonitors and not an internal inflation, then + // we may stop for a safepoint in omAlloc() so protect object. + Handle h_obj(Self, object); + m = omAlloc(Self, cause); + object = h_obj(); // Refresh object. + } // prepare m for installation - set monitor to initial state m->Recycle(); m->set_header(mark); @@ -1498,7 +1650,8 @@ if (event.should_commit()) { post_monitor_inflate_event(&event, object, cause); } - return m; + omh_p->set_om_ptr(m); + return; } } @@ -1524,6 +1677,30 @@ // which in turn can mean large(r) numbers of objectmonitors in circulation. // This is an unfortunate aspect of this design. +void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* _counters) { + if (!AsyncDeflateIdleMonitors) { + // Use the older mechanism for the global in-use list. + ObjectSynchronizer::deflate_idle_monitors(_counters); + return; + } + + assert(_counters == NULL, "not used with AsyncDeflateIdleMonitors"); + + log_debug(monitorinflation)("requesting deflation of idle monitors."); + // Request deflation of global idle monitors by the ServiceThread: + _gOmShouldDeflateIdleMonitors = true; + MutexLockerEx ml(Service_lock, Mutex::_no_safepoint_check_flag); + Service_lock->notify_all(); + + // Request deflation of per-thread idle monitors by each JavaThread: + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { + if (jt->omInUseCount > 0) { + // This JavaThread is using monitors so check it. + jt->omShouldDeflateIdleMonitors = true; + } + } +} + // Deflate a single monitor if not in-use // Return true if deflated, false if in-use bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj, @@ -1554,6 +1731,7 @@ mid->clear(); assert(mid->object() == NULL, "invariant"); + assert(mid->is_free(), "invariant"); // Move the object to the working free list defined by freeHeadp, freeTailp if (*freeHeadp == NULL) *freeHeadp = mid; @@ -1568,6 +1746,132 @@ return deflated; } +// Deflate the specified ObjectMonitor if not in-use using a JavaThread. +// Returns true if it was deflated and false otherwise. +// +// The async deflation protocol sets _owner to DEFLATER_MARKER and +// makes _count negative as signals to contending threads that an +// async deflation is in progress. There are a number of checks as +// part of the protocol to make sure that the calling thread has +// not lost the race to a contending thread. +// +// The ObjectMonitor has been successfully async deflated when: +// (_owner == DEFLATER_MARKER && _count < 0). Contending threads that +// see those values know to retry their operation. +// +bool ObjectSynchronizer::deflate_monitor_using_JT(ObjectMonitor* mid, + ObjectMonitor** freeHeadp, + ObjectMonitor** freeTailp) { + assert(AsyncDeflateIdleMonitors, "sanity check"); + assert(Thread::current()->is_Java_thread(), "precondition"); + // A newly allocated ObjectMonitor should not be seen here so we + // avoid an endless inflate/deflate cycle. + assert(mid->is_old(), "precondition"); + + if (mid->is_busy() || mid->ref_count() != 0) { + // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor* + // is in use so no deflation. + return false; + } + + if (Atomic::cmpxchg(DEFLATER_MARKER, &mid->_owner, (void*)NULL) == NULL) { + // ObjectMonitor is not owned by another thread. Our setting + // _owner to DEFLATER_MARKER forces any contending thread through + // the slow path. This is just the first part of the async + // deflation dance. + + if (mid->_waiters != 0 || mid->ref_count() != 0) { + // Another thread has raced to enter the ObjectMonitor after + // mid->is_busy() above and has already waited on it which + // makes it busy so no deflation. Or the ObjectMonitor* is + // in use for some other operation like inflate(). Restore + // _owner to NULL if it is still DEFLATER_MARKER. + Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER); + return false; + } + + if (Atomic::cmpxchg(-max_jint, &mid->_count, (jint)0) == 0) { + // Make _count negative to force racing threads to retry. + // This is the second part of the async deflation dance. + + if (mid->_owner == DEFLATER_MARKER) { + // If _owner is still DEFLATER_MARKER, then we have successfully + // signaled any racing threads to retry. If it is not, then we + // have lost the race to another thread and the ObjectMonitor is + // now busy. This is the third and final part of the async + // deflation dance. + // Note: This _owner check solves the ABA problem with _count + // where another thread acquired the ObjectMonitor, finished + // using it and restored the _count to zero. + + // Sanity checks for the races: + guarantee(mid->_waiters == 0, "should be no waiters"); + guarantee(mid->_cxq == NULL, "should be no contending threads"); + guarantee(mid->_EntryList == NULL, "should be no entering threads"); + + if (log_is_enabled(Trace, monitorinflation)) { + oop obj = (oop) mid->object(); + assert(obj != NULL, "sanity check"); + if (obj->is_instance()) { + ResourceMark rm; + log_trace(monitorinflation)("deflate_monitor_using_JT: " + "object=" INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", type='%s'", + p2i(obj), p2i(obj->mark()), + obj->klass()->external_name()); + } + } + + // Install the old mark word if nobody else has already done it. + mid->install_displaced_markword_in_object(); + mid->clear_using_JT(); + + assert(mid->object() == NULL, "invariant"); + assert(mid->is_free(), "invariant"); + + // Move the deflated ObjectMonitor to the working free list + // defined by freeHeadp and freeTailp. + if (*freeHeadp == NULL) { + // First one on the list. + *freeHeadp = mid; + } + if (*freeTailp != NULL) { + // We append to the list so the caller can use mid->FreeNext + // to fix the linkages in its context. + ObjectMonitor * prevtail = *freeTailp; + assert(prevtail->FreeNext == NULL, "not cleaned up by the caller"); + prevtail->FreeNext = mid; + } + *freeTailp = mid; + + // At this point, mid->FreeNext still refers to its current + // value and another ObjectMonitor's FreeNext field still + // refers to this ObjectMonitor. Those linkages have to be + // cleaned up by the caller who has the complete context. + + // We leave _owner == DEFLATER_MARKER and _count < 0 to + // force any racing threads to retry. + return true; // Success, ObjectMonitor has been deflated. + } + + // The _owner was changed from DEFLATER_MARKER so we lost the + // race since the ObjectMonitor is now busy. Add back max_jint + // to restore the _count field to its proper value (which may + // not be what we saw above). + Atomic::add(max_jint, &mid->_count); + + assert(mid->_count >= 0, "_count should not be negative"); + } + + // The _count was no longer 0 so we lost the race since the + // ObjectMonitor is now busy. + assert(mid->_owner != DEFLATER_MARKER, "should no longer be set"); + } + + // The _owner field is no longer NULL so we lost the race since the + // ObjectMonitor is now busy. + return false; +} + // Walk a given monitor list, and deflate idle monitors // The given list could be a per-thread list or a global list // Caller acquires gListLock as needed. @@ -1611,6 +1915,87 @@ return deflated_count; } +// Walk a given ObjectMonitor list and deflate idle ObjectMonitors using +// a JavaThread. Returns the number of deflated ObjectMonitors. The given +// list could be a per-thread in-use list or the global in-use list. +// Caller acquires gListLock as appropriate. If a safepoint has started, +// then we save state via savedMidInUsep and return to the caller to +// honor the safepoint. +// +int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor** listHeadp, + ObjectMonitor** freeHeadp, + ObjectMonitor** freeTailp, + ObjectMonitor** savedMidInUsep) { + assert(AsyncDeflateIdleMonitors, "sanity check"); + assert(Thread::current()->is_Java_thread(), "precondition"); + + ObjectMonitor* mid; + ObjectMonitor* next; + ObjectMonitor* cur_mid_in_use = NULL; + int deflated_count = 0; + + if (*savedMidInUsep == NULL) { + // No saved state so start at the beginning. + mid = *listHeadp; + } else { + // We're restarting after a safepoint so restore the necessary state + // before we resume. + cur_mid_in_use = *savedMidInUsep; + mid = cur_mid_in_use->FreeNext; + } + while (mid != NULL) { + // Only try to deflate if there is an associated Java object and if + // mid is old (is not newly allocated and is not newly freed). + if (mid->object() != NULL && mid->is_old() && + deflate_monitor_using_JT(mid, freeHeadp, freeTailp)) { + // Deflation succeeded so update the in-use list. + if (mid == *listHeadp) { + *listHeadp = mid->FreeNext; + } else if (cur_mid_in_use != NULL) { + // Maintain the current in-use list. + cur_mid_in_use->FreeNext = mid->FreeNext; + } + next = mid->FreeNext; + mid->FreeNext = NULL; + // At this point mid is disconnected from the in-use list + // and is the current tail in the freeHeadp list. + mid = next; + deflated_count++; + } else { + // mid is considered in-use if it does not have an associated + // Java object or mid is not old or deflation did not succeed. + // A mid->is_new() node can be seen here when it is freshly returned + // by omAlloc() (and skips the deflation code path). + // A mid->is_old() node can be seen here when deflation failed. + // A mid->is_free() node can be seen here when a fresh node from + // omAlloc() is released by omRelease() due to losing the race + // in inflate(). + + if (mid->object() != NULL && mid->is_new()) { + // mid has an associated Java object and has now been seen + // as newly allocated so mark it as "old". + mid->set_allocation_state(ObjectMonitor::Old); + } + cur_mid_in_use = mid; + mid = mid->FreeNext; + + if (SafepointSynchronize::is_synchronizing() && + cur_mid_in_use != *listHeadp && cur_mid_in_use->is_old()) { + // If a safepoint has started and cur_mid_in_use is not the list + // head and is old, then it is safe to use as saved state. Return + // to the caller so gListLock can be dropped as appropriate + // before blocking. + *savedMidInUsep = cur_mid_in_use; + return deflated_count; + } + } + } + // We finished the list without a safepoint starting so there's + // no need to save state. + *savedMidInUsep = NULL; + return deflated_count; +} + void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) { counters->nInuse = 0; // currently associated with objects counters->nInCirculation = 0; // extant @@ -1620,6 +2005,7 @@ } void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) { + assert(!AsyncDeflateIdleMonitors, "sanity check"); assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); bool deflated = false; @@ -1673,14 +2059,170 @@ } } +// Deflate global idle ObjectMonitors using a JavaThread. +// +void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() { + assert(AsyncDeflateIdleMonitors, "sanity check"); + assert(Thread::current()->is_Java_thread(), "precondition"); + JavaThread * cur_jt = JavaThread::current(); + + _gOmShouldDeflateIdleMonitors = false; + + int deflated_count = 0; + ObjectMonitor * freeHeadp = NULL; // Local SLL of scavenged ObjectMonitors + ObjectMonitor * freeTailp = NULL; + ObjectMonitor * savedMidInUsep = NULL; + elapsedTimer timer; + + if (log_is_enabled(Info, monitorinflation)) { + timer.start(); + } + Thread::muxAcquire(&gListLock, "deflate_global_idle_monitors_using_JT(1)"); + OM_PERFDATA_OP(MonExtant, set_value(gOmInUseCount)); + + do { + int local_deflated_count = deflate_monitor_list_using_JT((ObjectMonitor **)&gOmInUseList, &freeHeadp, &freeTailp, &savedMidInUsep); + gOmInUseCount -= local_deflated_count; + deflated_count += local_deflated_count; + + if (freeHeadp != NULL) { + // Move the scavenged ObjectMonitors to the global free list. + guarantee(freeTailp != NULL && local_deflated_count > 0, "freeTailp=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(freeTailp), local_deflated_count); + assert(freeTailp->FreeNext == NULL, "invariant"); + + // Constant-time list splice - prepend scavenged segment to gFreeList. + freeTailp->FreeNext = gFreeList; + gFreeList = freeHeadp; + + gMonitorFreeCount += local_deflated_count; + OM_PERFDATA_OP(Deflations, inc(local_deflated_count)); + } + + if (savedMidInUsep != NULL) { + // deflate_monitor_list_using_JT() detected a safepoint starting. + Thread::muxRelease(&gListLock); + timer.stop(); + { + log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint."); + assert(SafepointSynchronize::is_synchronizing(), "sanity check"); + ThreadBlockInVM blocker(cur_jt); + } + // Prepare for another loop after the safepoint. + freeHeadp = NULL; + freeTailp = NULL; + if (log_is_enabled(Info, monitorinflation)) { + timer.start(); + } + Thread::muxAcquire(&gListLock, "deflate_global_idle_monitors_using_JT(2)"); + } + } while (savedMidInUsep != NULL); + Thread::muxRelease(&gListLock); + timer.stop(); + + LogStreamHandle(Debug, monitorinflation) lsh_debug; + LogStreamHandle(Info, monitorinflation) lsh_info; + LogStream * ls = NULL; + if (log_is_enabled(Debug, monitorinflation)) { + ls = &lsh_debug; + } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) { + ls = &lsh_info; + } + if (ls != NULL) { + ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count); + } +} + +// Deflate per-thread idle ObjectMonitors using a JavaThread. +// +void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT() { + assert(AsyncDeflateIdleMonitors, "sanity check"); + assert(Thread::current()->is_Java_thread(), "precondition"); + JavaThread * cur_jt = JavaThread::current(); + + cur_jt->omShouldDeflateIdleMonitors = false; + + int deflated_count = 0; + ObjectMonitor * freeHeadp = NULL; // Local SLL of scavenged ObjectMonitors + ObjectMonitor * freeTailp = NULL; + ObjectMonitor * savedMidInUsep = NULL; + elapsedTimer timer; + + if (log_is_enabled(Info, monitorinflation)) { + timer.start(); + } + + OM_PERFDATA_OP(MonExtant, inc(cur_jt->omInUseCount)); + do { + int local_deflated_count = deflate_monitor_list_using_JT(cur_jt->omInUseList_addr(), &freeHeadp, &freeTailp, &savedMidInUsep); + cur_jt->omInUseCount -= local_deflated_count; + deflated_count += local_deflated_count; + + if (freeHeadp != NULL) { + // Move the scavenged ObjectMonitors to the global free list. + Thread::muxAcquire(&gListLock, "deflate_per_thread_idle_monitors_using_JT"); + guarantee(freeTailp != NULL && local_deflated_count > 0, "freeTailp=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(freeTailp), local_deflated_count); + assert(freeTailp->FreeNext == NULL, "invariant"); + + // Constant-time list splice - prepend scavenged segment to gFreeList. + freeTailp->FreeNext = gFreeList; + gFreeList = freeHeadp; + + gMonitorFreeCount += local_deflated_count; + OM_PERFDATA_OP(Deflations, inc(local_deflated_count)); + Thread::muxRelease(&gListLock); + // Prepare for another loop on the current JavaThread. + freeHeadp = NULL; + freeTailp = NULL; + } + timer.stop(); + + if (savedMidInUsep != NULL) { + // deflate_monitor_list_using_JT() detected a safepoint starting. + { + log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(cur_jt)); + assert(SafepointSynchronize::is_synchronizing(), "sanity check"); + ThreadBlockInVM blocker(cur_jt); + } + // Prepare for another loop on the current JavaThread after + // the safepoint. + if (log_is_enabled(Info, monitorinflation)) { + timer.start(); + } + } + } while (savedMidInUsep != NULL); + + LogStreamHandle(Debug, monitorinflation) lsh_debug; + LogStreamHandle(Info, monitorinflation) lsh_info; + LogStream * ls = NULL; + if (log_is_enabled(Debug, monitorinflation)) { + ls = &lsh_debug; + } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) { + ls = &lsh_info; + } + if (ls != NULL) { + ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(cur_jt), timer.seconds(), deflated_count); + } +} + void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) { // Report the cumulative time for deflating each thread's idle // monitors. Note: if the work is split among more than one // worker thread, then the reported time will likely be more // than a beginning to end measurement of the phase. + // Note: AsyncDeflateIdleMonitors only deflates per-thread idle + // monitors at a safepoint when a special cleanup has been requested. log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", counters->perThreadTimes, counters->perThreadScavenged); - gMonitorFreeCount += counters->nScavenged; + bool needs_special_cleanup = is_cleanup_requested(); + if (!AsyncDeflateIdleMonitors || needs_special_cleanup) { + // AsyncDeflateIdleMonitors does not use these counters unless + // there is a special cleanup request. + + gMonitorFreeCount += counters->nScavenged; + + OM_PERFDATA_OP(Deflations, inc(counters->nScavenged)); + OM_PERFDATA_OP(MonExtant, set_value(counters->nInCirculation)); + } if (log_is_enabled(Debug, monitorinflation)) { // exit_globals()'s call to audit_and_print_stats() is done @@ -1695,17 +2237,24 @@ } ForceMonitorScavenge = 0; // Reset - - OM_PERFDATA_OP(Deflations, inc(counters->nScavenged)); - OM_PERFDATA_OP(MonExtant, set_value(counters->nInCirculation)); - GVars.stwRandom = os::random(); GVars.stwCycle++; + if (needs_special_cleanup) { + set_is_cleanup_requested(false); // special clean up is done + } } void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); + if (AsyncDeflateIdleMonitors) { + // Nothing to do when idle ObjectMonitors are deflated using a + // JavaThread unless a special cleanup has been requested. + if (!is_cleanup_requested()) { + return; + } + } + ObjectMonitor * freeHeadp = NULL; // Local SLL of scavenged monitors ObjectMonitor * freeTailp = NULL; elapsedTimer timer; @@ -1917,7 +2466,8 @@ // Check a free monitor entry; log any errors. void ObjectSynchronizer::chk_free_entry(JavaThread * jt, ObjectMonitor * n, outputStream * out, int *error_cnt_p) { - if (n->is_busy()) { + if ((!AsyncDeflateIdleMonitors && n->is_busy()) || + (AsyncDeflateIdleMonitors && n->is_busy_async())) { if (jt != NULL) { out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT ": free per-thread monitor must not be busy.", p2i(jt), @@ -2109,16 +2659,17 @@ if (gOmInUseCount > 0) { out->print_cr("In-use global monitor info:"); out->print_cr("(B -> is_busy, H -> has hashcode, L -> lock status)"); - out->print_cr("%18s %s %18s %18s", - "monitor", "BHL", "object", "object type"); - out->print_cr("================== === ================== =================="); + out->print_cr("%18s %s %7s %18s %18s", + "monitor", "BHL", "ref_cnt", "object", "object type"); + out->print_cr("================== === ======= ================== =================="); for (ObjectMonitor * n = gOmInUseList; n != NULL; n = n->FreeNext) { const oop obj = (oop) n->object(); const markOop mark = n->header(); ResourceMark rm; - out->print_cr(INTPTR_FORMAT " %d%d%d " INTPTR_FORMAT " %s", p2i(n), - n->is_busy() != 0, mark->hash() != 0, n->owner() != NULL, - p2i(obj), obj->klass()->external_name()); + out->print_cr(INTPTR_FORMAT " %d%d%d %7d " INTPTR_FORMAT " %s", + p2i(n), n->is_busy() != 0, mark->hash() != 0, + n->owner() != NULL, (int)n->ref_count(), p2i(obj), + obj->klass()->external_name()); } } @@ -2128,18 +2679,18 @@ out->print_cr("In-use per-thread monitor info:"); out->print_cr("(B -> is_busy, H -> has hashcode, L -> lock status)"); - out->print_cr("%18s %18s %s %18s %18s", - "jt", "monitor", "BHL", "object", "object type"); - out->print_cr("================== ================== === ================== =================="); + out->print_cr("%18s %18s %s %7s %18s %18s", + "jt", "monitor", "BHL", "ref_cnt", "object", "object type"); + out->print_cr("================== ================== === ======= ================== =================="); for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { for (ObjectMonitor * n = jt->omInUseList; n != NULL; n = n->FreeNext) { const oop obj = (oop) n->object(); const markOop mark = n->header(); ResourceMark rm; - out->print_cr(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d " INTPTR_FORMAT - " %s", p2i(jt), p2i(n), n->is_busy() != 0, - mark->hash() != 0, n->owner() != NULL, p2i(obj), - obj->klass()->external_name()); + out->print_cr(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d %7d " + INTPTR_FORMAT " %s", p2i(jt), p2i(n), n->is_busy() != 0, + mark->hash() != 0, n->owner() != NULL, (int)n->ref_count(), + p2i(obj), obj->klass()->external_name()); } } --- old/src/hotspot/share/runtime/synchronizer.hpp 2019-03-22 16:30:42.017896743 -0400 +++ new/src/hotspot/share/runtime/synchronizer.hpp 2019-03-22 16:30:41.697896748 -0400 @@ -32,6 +32,7 @@ #include "runtime/perfData.hpp" class ObjectMonitor; +class ObjectMonitorHandle; class ThreadsList; struct DeflateMonitorCounters { @@ -107,15 +108,16 @@ static void reenter (Handle obj, intptr_t recursion, TRAPS); // thread-specific and global objectMonitor free list accessors - static ObjectMonitor * omAlloc(Thread * Self); + static ObjectMonitor * omAlloc(Thread * Self, const InflateCause cause); static void omRelease(Thread * Self, ObjectMonitor * m, bool FromPerThreadAlloc); static void omFlush(Thread * Self); // Inflate light weight monitor to heavy weight monitor - static ObjectMonitor* inflate(Thread * Self, oop obj, const InflateCause cause); + static void inflate(ObjectMonitorHandle * omh_p, Thread * Self, oop obj, + const InflateCause cause); // This version is only for internal use - static void inflate_helper(oop obj); + static void inflate_helper(ObjectMonitorHandle * omh_p, oop obj); static const char* inflate_cause_name(const InflateCause cause); // Returns the identity hash value for an oop @@ -137,6 +139,8 @@ // Basically we deflate all monitors that are not busy. // An adaptive profile-based deflation policy could be used if needed static void deflate_idle_monitors(DeflateMonitorCounters* counters); + static void deflate_global_idle_monitors_using_JT(); + static void deflate_per_thread_idle_monitors_using_JT(); static void deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters); static void prepare_deflate_idle_monitors(DeflateMonitorCounters* counters); static void finish_deflate_idle_monitors(DeflateMonitorCounters* counters); @@ -145,10 +149,21 @@ static int deflate_monitor_list(ObjectMonitor** listheadp, ObjectMonitor** freeHeadp, ObjectMonitor** freeTailp); + // For a given in-use monitor list: global or per-thread, deflate idle + // monitors using a JavaThread. + static int deflate_monitor_list_using_JT(ObjectMonitor** listHeadp, + ObjectMonitor** freeHeadp, + ObjectMonitor** freeTailp, + ObjectMonitor** savedMidInUsep); static bool deflate_monitor(ObjectMonitor* mid, oop obj, ObjectMonitor** freeHeadp, ObjectMonitor** freeTailp); + static bool deflate_monitor_using_JT(ObjectMonitor* mid, + ObjectMonitor** freeHeadp, + ObjectMonitor** freeTailp); static bool is_cleanup_needed(); + static bool is_cleanup_requested() { return _is_cleanup_requested; } + static void set_is_cleanup_requested(bool new_value) { _is_cleanup_requested = new_value; } static void oops_do(OopClosure* f); // Process oops in thread local used monitors static void thread_local_used_oops_do(Thread* thread, OopClosure* f); @@ -173,6 +188,9 @@ static int log_monitor_list_counts(outputStream * out); static int verify_objmon_isinpool(ObjectMonitor *addr) PRODUCT_RETURN0; + static bool gOmShouldDeflateIdleMonitors() { return _gOmShouldDeflateIdleMonitors; } + static void do_safepoint_work(DeflateMonitorCounters* _counters); + private: friend class SynchronizerTest; @@ -186,6 +204,8 @@ static ObjectMonitor * volatile gOmInUseList; // count of entries in gOmInUseList static int gOmInUseCount; + static bool _gOmShouldDeflateIdleMonitors; + static volatile bool _is_cleanup_requested; // Process oops in all global used monitors (i.e. moribund thread's monitors) static void global_used_oops_do(OopClosure* f); --- old/src/hotspot/share/runtime/thread.cpp 2019-03-22 16:30:42.761896730 -0400 +++ new/src/hotspot/share/runtime/thread.cpp 2019-03-22 16:30:42.437896736 -0400 @@ -266,6 +266,7 @@ omFreeProvision = 32; omInUseList = NULL; omInUseCount = 0; + omShouldDeflateIdleMonitors = false; #ifdef ASSERT _visited_for_critical_count = false; --- old/src/hotspot/share/runtime/thread.hpp 2019-03-22 16:30:43.601896715 -0400 +++ new/src/hotspot/share/runtime/thread.hpp 2019-03-22 16:30:43.269896721 -0400 @@ -418,6 +418,7 @@ int omFreeProvision; // reload chunk size ObjectMonitor* omInUseList; // SLL to track monitors in circulation int omInUseCount; // length of omInUseList + volatile bool omShouldDeflateIdleMonitors; // should deflate idle monitors #ifdef ASSERT private: --- old/test/hotspot/jtreg/runtime/logging/SafepointCleanupTest.java 2019-03-22 16:30:44.589896698 -0400 +++ new/test/hotspot/jtreg/runtime/logging/SafepointCleanupTest.java 2019-03-22 16:30:44.153896706 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,12 +29,17 @@ * @modules java.base/jdk.internal.misc * java.management * @run driver SafepointCleanupTest + * @run driver SafepointCleanupTest -XX:+AsyncDeflateIdleMonitors */ import jdk.test.lib.process.OutputAnalyzer; import jdk.test.lib.process.ProcessTools; public class SafepointCleanupTest { + static final String ASYNC_DISABLE_OPTION = "-XX:-AsyncDeflateIdleMonitors"; + static final String ASYNC_ENABLE_OPTION = "-XX:+AsyncDeflateIdleMonitors"; + static final String UNLOCK_DIAG_OPTION = "-XX:+UnlockDiagnosticVMOptions"; + static void analyzeOutputOn(ProcessBuilder pb) throws Exception { OutputAnalyzer output = new OutputAnalyzer(pb.start()); output.shouldContain("[safepoint,cleanup]"); @@ -54,19 +59,40 @@ } public static void main(String[] args) throws Exception { + String async_option; + if (args.length == 0) { + // By default test deflating idle monitors at a safepoint. + async_option = ASYNC_DISABLE_OPTION; + } else { + async_option = args[0]; + } + if (!async_option.equals(ASYNC_DISABLE_OPTION) && + !async_option.equals(ASYNC_ENABLE_OPTION)) { + throw new RuntimeException("Unknown async_option value: '" + + async_option + "'"); + } + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder("-Xlog:safepoint+cleanup=info", + UNLOCK_DIAG_OPTION, + async_option, InnerClass.class.getName()); analyzeOutputOn(pb); pb = ProcessTools.createJavaProcessBuilder("-XX:+TraceSafepointCleanupTime", + UNLOCK_DIAG_OPTION, + async_option, InnerClass.class.getName()); analyzeOutputOn(pb); pb = ProcessTools.createJavaProcessBuilder("-Xlog:safepoint+cleanup=off", + UNLOCK_DIAG_OPTION, + async_option, InnerClass.class.getName()); analyzeOutputOff(pb); pb = ProcessTools.createJavaProcessBuilder("-XX:-TraceSafepointCleanupTime", + UNLOCK_DIAG_OPTION, + async_option, InnerClass.class.getName()); analyzeOutputOff(pb); } --- old/test/jdk/java/rmi/server/UnicastRemoteObject/unexportObject/UnexportLeak.java 2019-03-22 16:30:45.637896680 -0400 +++ new/test/jdk/java/rmi/server/UnicastRemoteObject/unexportObject/UnexportLeak.java 2019-03-22 16:30:45.201896688 -0400 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -74,7 +74,7 @@ } /** - * Force desparate garbage collection so that all WeakReference instances + * Force desperate garbage collection so that all WeakReference instances * will be cleared. */ private static void flushRefs() { @@ -85,6 +85,9 @@ chain.addElement(hungry); } } catch (OutOfMemoryError e) { + // An inflated Java monitor can keep 'obj' alive so request + // an explicit GC to make sure things are cleaned up. + System.gc(); } } } --- old/test/jdk/tools/jlink/multireleasejar/JLinkMultiReleaseJarTest.java 2019-03-22 16:30:46.549896664 -0400 +++ new/test/jdk/tools/jlink/multireleasejar/JLinkMultiReleaseJarTest.java 2019-03-22 16:30:46.141896671 -0400 @@ -193,6 +193,15 @@ int version = (int) getVersion.invoke(clazz.getConstructor().newInstance()); Assert.assertEquals(version, JarFile.runtimeVersion().major()); } + // Very rarely this test fails on Windows due to: + // Error. failed to clean up files after test + // and this mesg shows the problem file (variable jimage): + // Can't delete T:\\testOutput\\test-support\\jtreg_open_test_jdk_core_tools\\scratch\\0\\myimage\\lib\\modules + // The failure happens more with async monitor deflation + // so I think that an inflated monitor is keeping the above + // try-with-resources block from cleaning up in a timely + // fashion. Forcing a GC here appears to solve the problem. + System.gc(); } @Test