--- old/src/hotspot/share/runtime/objectMonitor.cpp 2020-02-03 17:39:20.000000000 -0500 +++ new/src/hotspot/share/runtime/objectMonitor.cpp 2020-02-03 17:39:20.000000000 -0500 @@ -241,6 +241,9 @@ // Enter support void ObjectMonitor::enter(TRAPS) { + jint l_ref_count = ref_count(); + ADIM_guarantee(l_ref_count > 0, "must be positive: l_ref_count=%d, ref_count=%d", l_ref_count, ref_count()); + // The following code is ordered to check the most common cases first // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors. Thread * const Self = THREAD; @@ -264,6 +267,16 @@ return; } + if (AsyncDeflateIdleMonitors && + try_set_owner_from(DEFLATER_MARKER, Self) == DEFLATER_MARKER) { + // The deflation protocol finished the first part (setting owner), + // but it failed the second part (making ref_count negative) and + // bailed. Or the ObjectMonitor was async deflated and reused. + // Acquired the monitor. + assert(_recursions == 0, "invariant"); + return; + } + // We've encountered genuine contention. assert(Self->_Stalled == 0, "invariant"); Self->_Stalled = intptr_t(this); @@ -291,10 +304,9 @@ assert(!SafepointSynchronize::is_at_safepoint(), "invariant"); assert(jt->thread_state() != _thread_blocked, "invariant"); assert(this->object() != NULL, "invariant"); - assert(_contentions >= 0, "invariant"); + assert(_contentions >= 0, "must not be negative: contentions=%d", _contentions); - // Prevent deflation at STW-time. See deflate_idle_monitors() and is_busy(). - // Ensure the object-monitor relationship remains stable while there's contention. + // Keep track of contention for JVM/TI and M&M queries. Atomic::inc(&_contentions); JFR_ONLY(JfrConditionalFlushWithStacktrace flush(jt);) @@ -357,7 +369,7 @@ } Atomic::dec(&_contentions); - assert(_contentions >= 0, "invariant"); + assert(_contentions >= 0, "must not be negative: contentions=%d", _contentions); Self->_Stalled = 0; // Must either set _recursions = 0 or ASSERT _recursions == 0. @@ -412,18 +424,103 @@ return -1; } +// Install the displaced mark word (dmw) of a deflating ObjectMonitor +// into the header of the object associated with the monitor. This +// idempotent method is called by a thread that is deflating a +// monitor and by other threads that have detected a race with the +// deflation process. +void ObjectMonitor::install_displaced_markword_in_object(const oop obj) { + // This function must only be called when (owner == DEFLATER_MARKER + // && ref_count <= 0), but we can't guarantee that here because + // those values could change when the ObjectMonitor gets moved from + // the global free list to a per-thread free list. + + guarantee(obj != NULL, "must be non-NULL"); + if (object() != obj) { + // ObjectMonitor's object ref no longer refers to the target object + // so the object's header has already been restored. + return; + } + + markWord dmw = header(); + if (dmw.value() == 0) { + // ObjectMonitor's header/dmw has been cleared so the ObjectMonitor + // has been deflated and taken off the global free list. + return; + } + + // A non-NULL dmw has to be either neutral (not locked and not marked) + // or is already participating in this restoration protocol. + assert(dmw.is_neutral() || (dmw.is_marked() && dmw.hash() == 0), + "failed precondition: dmw=" INTPTR_FORMAT, dmw.value()); + + markWord marked_dmw = markWord::zero(); + if (!dmw.is_marked() && dmw.hash() == 0) { + // This dmw has not yet started the restoration protocol so we + // mark a copy of the dmw to begin the protocol. + // Note: A dmw with a hashcode does not take this code path. + marked_dmw = dmw.set_marked(); + + // All of the callers to this function can be racing with each + // other trying to update the _header field. + dmw = (markWord) Atomic::cmpxchg(&_header, dmw, marked_dmw); + if (dmw.value() == 0) { + // ObjectMonitor's header/dmw has been cleared so the object's + // header has already been restored. + return; + } + // The _header field is now marked. The winner's 'dmw' variable + // contains the original, unmarked header/dmw value and any + // losers have a marked header/dmw value that will be cleaned + // up below. + } + + if (dmw.is_marked()) { + // Clear the mark from the header/dmw copy in preparation for + // possible restoration from this thread. + assert(dmw.hash() == 0, "hashcode must be 0: dmw=" INTPTR_FORMAT, + dmw.value()); + dmw = dmw.set_unmarked(); + } + assert(dmw.is_neutral(), "must be neutral: dmw=" INTPTR_FORMAT, dmw.value()); + + // Install displaced mark word if the object's header still points + // to this ObjectMonitor. All racing callers to this function will + // reach this point, but only one can win. + obj->cas_set_mark(dmw, markWord::encode(this)); + + // Note: It does not matter which thread restored the header/dmw + // into the object's header. The thread deflating the monitor just + // wanted the object's header restored and it is. The threads that + // detected a race with the deflation process also wanted the + // object's header restored before they retry their operation and + // because it is restored they will only retry once. +} + // Convert the fields used by is_busy() to a string that can be // used for diagnostic output. const char* ObjectMonitor::is_busy_to_string(stringStream* ss) { - ss->print("is_busy: contentions=%d, waiters=%d, owner=" INTPTR_FORMAT - ", cxq=" INTPTR_FORMAT ", EntryList=" INTPTR_FORMAT, _contentions, - _waiters, p2i(_owner), p2i(_cxq), p2i(_EntryList)); + ss->print("is_busy: contentions=%d, waiters=%d, ", _contentions, _waiters); + if (!AsyncDeflateIdleMonitors) { + ss->print("owner=" INTPTR_FORMAT, p2i(_owner)); + } else if (_owner != DEFLATER_MARKER) { + ss->print("owner=" INTPTR_FORMAT, p2i(_owner)); + } else { + // We report NULL instead of DEFLATER_MARKER here because is_busy() + // ignores DEFLATER_MARKER values. + ss->print("owner=" INTPTR_FORMAT, NULL); + } + ss->print(", cxq=" INTPTR_FORMAT ", EntryList=" INTPTR_FORMAT, p2i(_cxq), + p2i(_EntryList)); return ss->base(); } #define MAX_RECHECK_INTERVAL 1000 void ObjectMonitor::EnterI(TRAPS) { + jint l_ref_count = ref_count(); + ADIM_guarantee(l_ref_count > 0, "must be positive: l_ref_count=%d, ref_count=%d", l_ref_count, ref_count()); + Thread * const Self = THREAD; assert(Self->is_Java_thread(), "invariant"); assert(((JavaThread *) Self)->thread_state() == _thread_blocked, "invariant"); @@ -436,6 +533,17 @@ return; } + if (AsyncDeflateIdleMonitors && + try_set_owner_from(DEFLATER_MARKER, Self) == DEFLATER_MARKER) { + // The deflation protocol finished the first part (setting owner), + // but it failed the second part (making ref_count negative) and + // bailed. Or the ObjectMonitor was async deflated and reused. + // Acquired the monitor. + assert(_succ != Self, "invariant"); + assert(_Responsible != Self, "invariant"); + return; + } + assert(InitDone, "Unexpectedly not initialized"); // We try one round of spinning *before* enqueueing Self. @@ -552,6 +660,15 @@ if (TryLock(Self) > 0) break; + if (AsyncDeflateIdleMonitors && + try_set_owner_from(DEFLATER_MARKER, Self) == DEFLATER_MARKER) { + // The deflation protocol finished the first part (setting owner), + // but it failed the second part (making ref_count negative) and + // bailed. Or the ObjectMonitor was async deflated and reused. + // Acquired the monitor. + break; + } + // The lock is still contested. // Keep a tally of the # of futile wakeups. // Note that the counter is not protected by a lock or updated by atomics. @@ -656,6 +773,9 @@ // In the future we should reconcile EnterI() and ReenterI(). void ObjectMonitor::ReenterI(Thread * Self, ObjectWaiter * SelfNode) { + jint l_ref_count = ref_count(); + ADIM_guarantee(l_ref_count > 0, "must be positive: l_ref_count=%d, ref_count=%d", l_ref_count, ref_count()); + assert(Self != NULL, "invariant"); assert(SelfNode != NULL, "invariant"); assert(SelfNode->_thread == Self, "invariant"); @@ -673,6 +793,15 @@ if (TryLock(Self) > 0) break; if (TrySpin(Self) > 0) break; + if (AsyncDeflateIdleMonitors && + try_set_owner_from(DEFLATER_MARKER, Self) == DEFLATER_MARKER) { + // The deflation protocol finished the first part (setting owner), + // but it failed the second part (making ref_count negative) and + // bailed. Or the ObjectMonitor was async deflated and reused. + // Acquired the monitor. + break; + } + // State transition wrappers around park() ... // ReenterI() wisely defers state transitions until // it's clear we must park the thread. @@ -816,8 +945,8 @@ // We'd like to assert that: (THREAD->thread_state() != _thread_blocked) ; // There's one exception to the claim above, however. EnterI() can call // exit() to drop a lock if the acquirer has been externally suspended. -// In that case exit() is called with _thread_state as _thread_blocked, -// but the monitor's _contentions field is > 0, which inhibits reclamation. +// In that case exit() is called with _thread_state == _thread_blocked, +// but the monitor's ref_count is > 0, which inhibits reclamation. // // 1-0 exit // ~~~~~~~~ @@ -1091,7 +1220,7 @@ // out-of-scope (non-extant). Wakee = NULL; - // Drop the lock + // Drop the lock. // Uses a fence to separate release_store(owner) from the LD in unpark(). release_clear_owner(Self); OrderAccess::fence(); @@ -1145,10 +1274,10 @@ JavaThread *jt = (JavaThread *)THREAD; guarantee(_owner != Self, "reenter already owner"); - enter(THREAD); // enter the monitor + enter(THREAD); + // Entered the monitor. guarantee(_recursions == 0, "reenter recursion"); _recursions = recursions; - return; } // Checks that the current THREAD owns this monitor and causes an @@ -1947,6 +2076,131 @@ DEBUG_ONLY(InitDone = true;) } +ObjectMonitorHandle::~ObjectMonitorHandle() { + if (_om_ptr != NULL) { + _om_ptr->dec_ref_count(); + _om_ptr = NULL; + } +} + +// Save the ObjectMonitor* associated with the specified markWord and +// increment the ref_count. This function should only be called if +// the caller has verified mark.has_monitor() == true. The object +// parameter is needed to verify that ObjectMonitor* has not been +// deflated and reused for another object. +// +// This function returns true if the ObjectMonitor* has been safely +// saved. This function returns false if we have lost a race with +// async deflation; the caller should retry as appropriate. +// +bool ObjectMonitorHandle::save_om_ptr(oop object, markWord mark) { + // is_marked() is a superset of has_monitor() so make sure we + // are called with the proper markWord value. + guarantee(mark.has_monitor() && !mark.is_marked(), "sanity check: mark=" + INTPTR_FORMAT, mark.value()); + + ObjectMonitor* om_ptr = mark.monitor(); + om_ptr->inc_ref_count(); + + if (AsyncDeflateIdleMonitors) { + // Race here if monitor is not owned! The above ref_count bump + // will cause subsequent async deflation to skip it. However, + // previous or concurrent async deflation is a race. + if (om_ptr->owner_is_DEFLATER_MARKER() && om_ptr->ref_count() <= 0) { + // Async deflation is in progress and our ref_count increment + // above lost the race to async deflation. Attempt to restore + // the header/dmw to the object's header so that we only retry + // once if the deflater thread happens to be slow. + om_ptr->install_displaced_markword_in_object(object); + om_ptr->dec_ref_count(); + return false; + } + if (om_ptr->ref_count() <= 0) { + // Async deflation is in the process of bailing out, but has not + // yet restored the ref_count field so we return false to force + // a retry. We want a positive ref_count value for a true return. + om_ptr->dec_ref_count(); + return false; + } + // The ObjectMonitor could have been deflated and reused for + // another object before we bumped the ref_count so make sure + // our object still refers to this ObjectMonitor. + const markWord tmp = object->mark(); + if (!tmp.has_monitor() || tmp.monitor() != om_ptr) { + // Async deflation and reuse won the race so we have to retry. + // Skip object header restoration since that's already done. + om_ptr->dec_ref_count(); + return false; + } + } + + ADIM_guarantee(_om_ptr == NULL, "sanity check: _om_ptr=" INTPTR_FORMAT, + p2i(_om_ptr)); + _om_ptr = om_ptr; + return true; +} + +// For internal use by ObjectSynchronizer::inflate(). +// This function is only used when we don't have to worry about async +// deflation of the specified ObjectMonitor*. +void ObjectMonitorHandle::set_om_ptr(ObjectMonitor* om_ptr) { + if (_om_ptr == NULL) { + ADIM_guarantee(om_ptr != NULL, "cannot clear an unset om_ptr"); + om_ptr->inc_ref_count(); + _om_ptr = om_ptr; + } else { + ADIM_guarantee(om_ptr == NULL, "can only clear a set om_ptr"); + _om_ptr->dec_ref_count(); + _om_ptr = NULL; + } +} + +// Save the specified ObjectMonitor* if it is safe, i.e., not being +// async deflated. +// +// This function returns true if the ObjectMonitor* has been safely +// saved. This function returns false if the specified ObjectMonitor* +// is NULL or if we have lost a race with async deflation; the caller +// can retry as appropriate. +bool ObjectMonitorHandle::set_om_ptr_if_safe(ObjectMonitor* om_ptr) { + if (om_ptr == NULL) { + return false; // Nothing to save if input is NULL + } + + om_ptr->inc_ref_count(); + + if (AsyncDeflateIdleMonitors) { + if (om_ptr->owner_is_DEFLATER_MARKER() && om_ptr->ref_count() <= 0) { + // Async deflation is in progress and our ref_count increment + // above lost the race to async deflation. + om_ptr->dec_ref_count(); + return false; + } + if (om_ptr->ref_count() <= 0) { + // Async deflation is in the process of bailing out, but has not + // yet restored the ref_count field so we return false to force + // a retry. We want a positive ref_count value for a true return. + om_ptr->dec_ref_count(); + return false; + } + // Unlike save_om_ptr(), we don't have context to determine if + // the ObjectMonitor has been deflated and reused for another + // object. + } + + ADIM_guarantee(_om_ptr == NULL, "sanity check: _om_ptr=" INTPTR_FORMAT, + p2i(_om_ptr)); + _om_ptr = om_ptr; + return true; +} + +// Unset the _om_ptr field and decrement the ref_count field. +void ObjectMonitorHandle::unset_om_ptr() { + ADIM_guarantee(_om_ptr != NULL, "_om_ptr must not be NULL"); + _om_ptr->dec_ref_count(); + _om_ptr = NULL; +} + void ObjectMonitor::print_on(outputStream* st) const { // The minimal things to print for markWord printing, more can be added for debugging and logging. st->print("{contentions=0x%08x,waiters=0x%08x" @@ -1962,14 +2216,26 @@ // (ObjectMonitor) 0x00007fdfb6012e40 = { // _header = 0x0000000000000001 // _object = 0x000000070ff45fd0 -// _next_om = 0x0000000000000000 +// _allocation_state = Old // _pad_buf0 = { // [0] = '\0' // ... -// [103] = '\0' +// [43] = '\0' // } // _owner = 0x0000000000000000 // _previous_owner_tid = 0 +// _pad_buf1 = { +// [0] = '\0' +// ... +// [47] = '\0' +// } +// _ref_count = 1 +// _pad_buf2 = { +// [0] = '\0' +// ... +// [47] = '\0' +// } +// _next_om = 0x0000000000000000 // _recursions = 0 // _EntryList = 0x0000000000000000 // _cxq = 0x0000000000000000 @@ -1987,7 +2253,17 @@ st->print_cr("(ObjectMonitor*) " INTPTR_FORMAT " = {", p2i(this)); st->print_cr(" _header = " INTPTR_FORMAT, header().value()); st->print_cr(" _object = " INTPTR_FORMAT, p2i(_object)); - st->print_cr(" _next_om = " INTPTR_FORMAT, p2i(next_om())); + st->print(" _allocation_state = "); + if (is_free()) { + st->print("Free"); + } else if (is_old()) { + st->print("Old"); + } else if (is_new()) { + st->print("New"); + } else { + st->print("unknown=%d", _allocation_state); + } + st->cr(); st->print_cr(" _pad_buf0 = {"); st->print_cr(" [0] = '\\0'"); st->print_cr(" ..."); @@ -1995,6 +2271,18 @@ st->print_cr(" }"); st->print_cr(" _owner = " INTPTR_FORMAT, p2i(_owner)); st->print_cr(" _previous_owner_tid = " JLONG_FORMAT, _previous_owner_tid); + st->print_cr(" _pad_buf1 = {"); + st->print_cr(" [0] = '\\0'"); + st->print_cr(" ..."); + st->print_cr(" [%d] = '\\0'", (int)sizeof(_pad_buf1) - 1); + st->print_cr(" }"); + st->print_cr(" _ref_count = %d", ref_count()); + st->print_cr(" _pad_buf2 = {"); + st->print_cr(" [0] = '\\0'"); + st->print_cr(" ..."); + st->print_cr(" [%d] = '\\0'", (int)sizeof(_pad_buf1) - 1); + st->print_cr(" }"); + st->print_cr(" _next_om = " INTPTR_FORMAT, p2i(next_om())); st->print_cr(" _recursions = " INTX_FORMAT, _recursions); st->print_cr(" _EntryList = " INTPTR_FORMAT, p2i(_EntryList)); st->print_cr(" _cxq = " INTPTR_FORMAT, p2i(_cxq));