--- old/src/hotspot/share/runtime/objectMonitor.cpp 2019-10-17 17:29:00.000000000 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.cpp 2019-10-17 17:29:00.000000000 -0400 @@ -239,11 +239,13 @@ // Enter support void ObjectMonitor::enter(TRAPS) { + ADIM_guarantee(ref_count() > 0, "must be positive: ref_count=%d", ref_count()); + // The following code is ordered to check the most common cases first // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors. Thread * const Self = THREAD; - void * cur = Atomic::cmpxchg(Self, &_owner, (void*)NULL); + void* cur = try_set_owner_from(Self, NULL); if (cur == NULL) { assert(_recursions == 0, "invariant"); return; @@ -258,9 +260,17 @@ if (Self->is_lock_owned ((address)cur)) { assert(_recursions == 0, "internal state error"); _recursions = 1; - // Commute owner from a thread-specific on-stack BasicLockObject address to - // a full-fledged "Thread *". - _owner = Self; + set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. + return; + } + + if (AsyncDeflateIdleMonitors && + try_set_owner_from(Self, DEFLATER_MARKER) == DEFLATER_MARKER) { + // The deflation protocol finished the first part (setting owner), + // but it failed the second part (making ref_count negative) and + // bailed. Or the ObjectMonitor was async deflated and reused. + // Acquired the monitor. + assert(_recursions == 0, "invariant"); return; } @@ -275,8 +285,7 @@ // we forgo posting JVMTI events and firing DTRACE probes. if (TrySpin(Self) > 0) { assert(_owner == Self, "must be Self: owner=" INTPTR_FORMAT, p2i(_owner)); - assert(_recursions == 0, "must be 0: recursions=" INTPTR_FORMAT, - _recursions); + assert(_recursions == 0, "must be 0: recursions=" INTX_FORMAT, _recursions); assert(((oop)object())->mark() == markWord::encode(this), "object mark must match encoded this: mark=" INTPTR_FORMAT ", encoded this=" INTPTR_FORMAT, ((oop)object())->mark().value(), @@ -291,12 +300,14 @@ JavaThread * jt = (JavaThread *) Self; assert(!SafepointSynchronize::is_at_safepoint(), "invariant"); assert(jt->thread_state() != _thread_blocked, "invariant"); - assert(this->object() != NULL, "invariant"); - assert(_contentions >= 0, "invariant"); + assert(AsyncDeflateIdleMonitors || this->object() != NULL, "invariant"); + assert(_contentions >= 0, "must not be negative: contentions=%d", _contentions); - // Prevent deflation at STW-time. See deflate_idle_monitors() and is_busy(). - // Ensure the object-monitor relationship remains stable while there's contention. - Atomic::inc(&_contentions); + // Prevent deflation. See ObjectSynchronizer::deflate_monitor(), + // ObjectSynchronizer::deflate_monitor_using_JT() and is_busy(). + // Ensure the object <-> monitor relationship remains stable while + // there's contention. + Atomic::add(1, &_contentions); JFR_ONLY(JfrConditionalFlushWithStacktrace flush(jt);) EventJavaMonitorEnter event; @@ -358,7 +369,7 @@ } Atomic::dec(&_contentions); - assert(_contentions >= 0, "invariant"); + assert(_contentions >= 0, "must not be negative: contentions=%d", _contentions); Self->_Stalled = 0; // Must either set _recursions = 0 or ASSERT _recursions == 0. @@ -402,7 +413,7 @@ int ObjectMonitor::TryLock(Thread * Self) { void * own = _owner; if (own != NULL) return 0; - if (Atomic::replace_if_null(Self, &_owner)) { + if (try_set_owner_from(Self, NULL) == NULL) { assert(_recursions == 0, "invariant"); return 1; } @@ -413,18 +424,100 @@ return -1; } +// Install the displaced mark word (dmw) of a deflating ObjectMonitor +// into the header of the object associated with the monitor. This +// idempotent method is called by a thread that is deflating a +// monitor and by other threads that have detected a race with the +// deflation process. +void ObjectMonitor::install_displaced_markword_in_object(const oop obj) { + // This function must only be called when (owner == DEFLATER_MARKER + // && ref_count <= 0), but we can't guarantee that here because + // those values could change when the ObjectMonitor gets moved from + // the global free list to a per-thread free list. + + guarantee(obj != NULL, "must be non-NULL"); + if (object() != obj) { + // ObjectMonitor's object ref no longer refers to the target object + // so the object's header has already been restored. + return; + } + + markWord dmw = header(); + if (dmw.value() == 0) { + // ObjectMonitor's header/dmw has been cleared so the ObjectMonitor + // has been deflated and taken off the global free list. + return; + } + + // A non-NULL dmw has to be either neutral (not locked and not marked) + // or is already participating in this restoration protocol. + assert(dmw.is_neutral() || (dmw.is_marked() && dmw.hash() == 0), + "failed precondition: dmw=" INTPTR_FORMAT, dmw.value()); + + markWord marked_dmw = markWord::zero(); + if (!dmw.is_marked() && dmw.hash() == 0) { + // This dmw has not yet started the restoration protocol so we + // mark a copy of the dmw to begin the protocol. + // Note: A dmw with a hashcode does not take this code path. + marked_dmw = dmw.set_marked(); + + // All of the callers to this function can be racing with each + // other trying to update the _header field. + dmw = (markWord) Atomic::cmpxchg(marked_dmw, &_header, dmw); + if (dmw.value() == 0) { + // ObjectMonitor's header/dmw has been cleared so the object's + // header has already been restored. + return; + } + // The _header field is now marked. The winner's 'dmw' variable + // contains the original, unmarked header/dmw value and any + // losers have a marked header/dmw value that will be cleaned + // up below. + } + + if (dmw.is_marked()) { + // Clear the mark from the header/dmw copy in preparation for + // possible restoration from this thread. + assert(dmw.hash() == 0, "hashcode must be 0: dmw=" INTPTR_FORMAT, + dmw.value()); + dmw = dmw.set_unmarked(); + } + assert(dmw.is_neutral(), "must be neutral: dmw=" INTPTR_FORMAT, dmw.value()); + + // Install displaced mark word if the object's header still points + // to this ObjectMonitor. All racing callers to this function will + // reach this point, but only one can win. + obj->cas_set_mark(dmw, markWord::encode(this)); + + // Note: It does not matter which thread restored the header/dmw + // into the object's header. The thread deflating the monitor just + // wanted the object's header restored and it is. The threads that + // detected a race with the deflation process also wanted the + // object's header restored before they retry their operation and + // because it is restored they will only retry once. +} + // Convert the fields used by is_busy() to a string that can be // used for diagnostic output. const char* ObjectMonitor::is_busy_to_string(stringStream* ss) { - ss->print("is_busy: contentions=%d, waiters=%d, owner=" INTPTR_FORMAT - ", cxq=" INTPTR_FORMAT ", EntryList=" INTPTR_FORMAT, _contentions, - _waiters, p2i(_owner), p2i(_cxq), p2i(_EntryList)); + ss->print("is_busy: contentions=%d, waiters=%d, ", _contentions, _waiters); + if (!AsyncDeflateIdleMonitors) { + ss->print("owner=" INTPTR_FORMAT, p2i(_owner)); + } else if (_owner != DEFLATER_MARKER) { + ss->print("owner=" INTPTR_FORMAT, p2i(_owner)); + } else { + ss->print("owner=" INTPTR_FORMAT, NULL); + } + ss->print(", cxq=" INTPTR_FORMAT ", EntryList=" INTPTR_FORMAT, p2i(_cxq), + p2i(_EntryList)); return ss->base(); } #define MAX_RECHECK_INTERVAL 1000 void ObjectMonitor::EnterI(TRAPS) { + ADIM_guarantee(ref_count() > 0, "must be positive: ref_count=%d", ref_count()); + Thread * const Self = THREAD; assert(Self->is_Java_thread(), "invariant"); assert(((JavaThread *) Self)->thread_state() == _thread_blocked, "invariant"); @@ -437,6 +530,17 @@ return; } + if (AsyncDeflateIdleMonitors && + try_set_owner_from(Self, DEFLATER_MARKER) == DEFLATER_MARKER) { + // The deflation protocol finished the first part (setting owner), + // but it failed the second part (making ref_count negative) and + // bailed. Or the ObjectMonitor was async deflated and reused. + // Acquired the monitor. + assert(_succ != Self, "invariant"); + assert(_Responsible != Self, "invariant"); + return; + } + assert(InitDone, "Unexpectedly not initialized"); // We try one round of spinning *before* enqueueing Self. @@ -553,6 +657,15 @@ if (TryLock(Self) > 0) break; + if (AsyncDeflateIdleMonitors && + try_set_owner_from(Self, DEFLATER_MARKER) == DEFLATER_MARKER) { + // The deflation protocol finished the first part (setting owner), + // but it failed the second part (making ref_count negative) and + // bailed. Or the ObjectMonitor was async deflated and reused. + // Acquired the monitor. + break; + } + // The lock is still contested. // Keep a tally of the # of futile wakeups. // Note that the counter is not protected by a lock or updated by atomics. @@ -657,6 +770,8 @@ // In the future we should reconcile EnterI() and ReenterI(). void ObjectMonitor::ReenterI(Thread * Self, ObjectWaiter * SelfNode) { + ADIM_guarantee(ref_count() > 0, "must be positive: ref_count=%d", ref_count()); + assert(Self != NULL, "invariant"); assert(SelfNode != NULL, "invariant"); assert(SelfNode->_thread == Self, "invariant"); @@ -674,6 +789,15 @@ if (TryLock(Self) > 0) break; if (TrySpin(Self) > 0) break; + if (AsyncDeflateIdleMonitors && + try_set_owner_from(Self, DEFLATER_MARKER) == DEFLATER_MARKER) { + // The deflation protocol finished the first part (setting owner), + // but it failed the second part (making ref_count negative) and + // bailed. Or the ObjectMonitor was async deflated and reused. + // Acquired the monitor. + break; + } + // State transition wrappers around park() ... // ReenterI() wisely defers state transitions until // it's clear we must park the thread. @@ -863,13 +987,10 @@ void ObjectMonitor::exit(bool not_suspended, TRAPS) { Thread * const Self = THREAD; if (THREAD != _owner) { - if (THREAD->is_lock_owned((address) _owner)) { - // Transmute _owner from a BasicLock pointer to a Thread address. - // We don't need to hold _mutex for this transition. - // Non-null to Non-null is safe as long as all readers can - // tolerate either flavor. + void* cur = _owner; + if (THREAD->is_lock_owned((address)cur)) { assert(_recursions == 0, "invariant"); - _owner = THREAD; + set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. _recursions = 0; } else { // Apparent unbalanced locking ... @@ -881,7 +1002,14 @@ // way we should encounter this situation is in the presence of // unbalanced JNI locking. TODO: CheckJNICalls. // See also: CR4414101 - assert(false, "Non-balanced monitor enter/exit! Likely JNI locking"); + tty->print_cr("ERROR: ObjectMonitor::exit(): thread=" INTPTR_FORMAT + " is exiting an ObjectMonitor it does not own.", + p2i(THREAD)); + tty->print_cr("The imbalance is possibly caused by JNI locking."); + print_debug_style_on(tty); + // Changing this from an assert() to ADIM_guarantee() may run + // afoul of any test that is inducing non-balanced JNI locking. + ADIM_guarantee(false, "Non-balanced monitor enter/exit!"); return; } } @@ -910,8 +1038,12 @@ // must not float (reorder) past the following store that drops the lock. // On SPARC that requires MEMBAR #loadstore|#storestore. // But of course in TSO #loadstore|#storestore is not required. - OrderAccess::release_store(&_owner, (void*)NULL); // drop the lock - OrderAccess::storeload(); // See if we need to wake a successor + if (AsyncDeflateIdleMonitors) { + set_owner_from(NULL, Self); + } else { + OrderAccess::release_store(&_owner, (void*)NULL); // drop the lock + OrderAccess::storeload(); // See if we need to wake a successor + } if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) { return; } @@ -953,7 +1085,7 @@ // to reacquire the lock the responsibility for ensuring succession // falls to the new owner. // - if (!Atomic::replace_if_null(THREAD, &_owner)) { + if (try_set_owner_from(Self, NULL) != NULL) { return; } @@ -1086,8 +1218,12 @@ Wakee = NULL; // Drop the lock - OrderAccess::release_store(&_owner, (void*)NULL); - OrderAccess::fence(); // ST _owner vs LD in unpark() + if (AsyncDeflateIdleMonitors) { + set_owner_from(NULL, Self); + } else { + OrderAccess::release_store(&_owner, (void*)NULL); + OrderAccess::fence(); // ST _owner vs LD in unpark() + } DTRACE_MONITOR_PROBE(contended__exit, this, object(), Self); Trigger->unpark(); @@ -1114,9 +1250,10 @@ assert(InitDone, "Unexpectedly not initialized"); if (THREAD != _owner) { - if (THREAD->is_lock_owned ((address)_owner)) { + void* cur = _owner; + if (THREAD->is_lock_owned((address)cur)) { assert(_recursions == 0, "internal state error"); - _owner = THREAD; // Convert from basiclock addr to Thread addr + set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. _recursions = 0; } } @@ -1137,10 +1274,10 @@ JavaThread *jt = (JavaThread *)THREAD; guarantee(_owner != Self, "reenter already owner"); - enter(THREAD); // enter the monitor + enter(THREAD); + // Entered the monitor. guarantee(_recursions == 0, "reenter recursion"); _recursions = recursions; - return; } // Checks that the current THREAD owns this monitor and causes an @@ -1164,8 +1301,9 @@ if (_owner == THREAD) { return true; } - if (THREAD->is_lock_owned((address)_owner)) { - _owner = THREAD; // convert from BasicLock addr to Thread addr + void* cur = _owner; + if (THREAD->is_lock_owned((address)cur)) { + set_owner_from_BasicLock(THREAD, cur); // Convert from BasicLock* to Thread*. _recursions = 0; return true; } @@ -1670,7 +1808,7 @@ Thread * ox = (Thread *) _owner; if (ox == NULL) { - ox = (Thread*)Atomic::cmpxchg(Self, &_owner, (void*)NULL); + ox = (Thread*)try_set_owner_from(Self, NULL); if (ox == NULL) { // The CAS succeeded -- this thread acquired ownership // Take care of some bookkeeping to exit spin state. @@ -1934,11 +2072,178 @@ DEBUG_ONLY(InitDone = true;) } +// For internal use by ObjectSynchronizer::monitors_iterate(). +ObjectMonitorHandle::ObjectMonitorHandle(ObjectMonitor * om_ptr) { + om_ptr->inc_ref_count(); + _om_ptr = om_ptr; +} + +ObjectMonitorHandle::~ObjectMonitorHandle() { + if (_om_ptr != NULL) { + _om_ptr->dec_ref_count(); + _om_ptr = NULL; + } +} + +// Save the ObjectMonitor* associated with the specified markWord and +// increment the ref_count. This function should only be called if +// the caller has verified mark.has_monitor() == true. The object +// parameter is needed to verify that ObjectMonitor* has not been +// deflated and reused for another object. +// +// This function returns true if the ObjectMonitor* has been safely +// saved. This function returns false if we have lost a race with +// async deflation; the caller should retry as appropriate. +// +bool ObjectMonitorHandle::save_om_ptr(oop object, markWord mark) { + guarantee(mark.has_monitor(), "sanity check: mark=" INTPTR_FORMAT, + mark.value()); + + ObjectMonitor * om_ptr = mark.monitor(); + om_ptr->inc_ref_count(); + + if (AsyncDeflateIdleMonitors) { + // Race here if monitor is not owned! The above ref_count bump + // will cause subsequent async deflation to skip it. However, + // previous or concurrent async deflation is a race. + if (om_ptr->owner_is_DEFLATER_MARKER() && om_ptr->ref_count() <= 0) { + // Async deflation is in progress and our ref_count increment + // above lost the race to async deflation. Attempt to restore + // the header/dmw to the object's header so that we only retry + // once if the deflater thread happens to be slow. + om_ptr->install_displaced_markword_in_object(object); + om_ptr->dec_ref_count(); + return false; + } + if (om_ptr->ref_count() <= 0) { + // Async deflation is in the process of bailing out, but has not + // yet restored the ref_count field so we return false to force + // a retry. We want a positive ref_count value for a true return. + om_ptr->dec_ref_count(); + return false; + } + // The ObjectMonitor could have been deflated and reused for + // another object before we bumped the ref_count so make sure + // our object still refers to this ObjectMonitor. + const markWord tmp = object->mark(); + if (!tmp.has_monitor() || tmp.monitor() != om_ptr) { + // Async deflation and reuse won the race so we have to retry. + // Skip object header restoration since that's already done. + om_ptr->dec_ref_count(); + return false; + } + } + + ADIM_guarantee(_om_ptr == NULL, "sanity check: _om_ptr=" INTPTR_FORMAT, + p2i(_om_ptr)); + _om_ptr = om_ptr; + return true; +} + +// For internal use by ObjectSynchronizer::inflate(). +void ObjectMonitorHandle::set_om_ptr(ObjectMonitor * om_ptr) { + if (_om_ptr == NULL) { + ADIM_guarantee(om_ptr != NULL, "cannot clear an unset om_ptr"); + om_ptr->inc_ref_count(); + _om_ptr = om_ptr; + } else { + ADIM_guarantee(om_ptr == NULL, "can only clear a set om_ptr"); + _om_ptr->dec_ref_count(); + _om_ptr = NULL; + } +} + void ObjectMonitor::print_on(outputStream* st) const { // The minimal things to print for markWord printing, more can be added for debugging and logging. st->print("{contentions=0x%08x,waiters=0x%08x" - ",recursions=" INTPTR_FORMAT ",owner=" INTPTR_FORMAT "}", + ",recursions=" INTX_FORMAT ",owner=" INTPTR_FORMAT "}", contentions(), waiters(), recursions(), p2i(owner())); } void ObjectMonitor::print() const { print_on(tty); } + +// Print the ObjectMonitor like a debugger would: +// +// (ObjectMonitor) 0x00007fdfb6012e40 = { +// _header = (_value = 1) +// _object = 0x000000070ff45fd0 +// _allocation_state = Old +// _pad_buf0 = { +// [0] = '\0' +// ... +// [43] = '\0' +// } +// _owner = 0x0000000000000000 +// _previous_owner_tid = 0 +// _pad_buf1 = { +// [0] = '\0' +// ... +// [47] = '\0' +// } +// _ref_count = 1 +// _pad_buf2 = { +// [0] = '\0' +// ... +// [59] = '\0' +// } +// _next_om = 0x0000000000000000 +// _recursions = 0 +// _EntryList = 0x0000000000000000 +// _cxq = 0x0000000000000000 +// _succ = 0x0000000000000000 +// _Responsible = 0x0000000000000000 +// _Spinner = 0 +// _SpinDuration = 5000 +// _contentions = 0 +// _WaitSet = 0x0000700009756248 +// _waiters = 1 +// _WaitSetLock = 0 +// } +// +void ObjectMonitor::print_debug_style_on(outputStream* st) const { + st->print_cr("(ObjectMonitor *) " INTPTR_FORMAT " = {", p2i(this)); + st->print_cr(" _header = " INTPTR_FORMAT, header().value()); + st->print_cr(" _object = " INTPTR_FORMAT, p2i(_object)); + st->print(" _allocation_state = "); + if (is_free()) { + st->print("Free"); + } else if (is_old()) { + st->print("Old"); + } else if (is_new()) { + st->print("New"); + } else { + st->print("unknown=%d", _allocation_state); + } + st->cr(); + st->print_cr(" _pad_buf0 = {"); + st->print_cr(" [0] = '\\0'"); + st->print_cr(" ..."); + st->print_cr(" [%d] = '\\0'", (int)sizeof(_pad_buf0) - 1); + st->print_cr(" }"); + st->print_cr(" _owner = " INTPTR_FORMAT, p2i(_owner)); + st->print_cr(" _previous_owner_tid = " JLONG_FORMAT, _previous_owner_tid); + st->print_cr(" _pad_buf1 = {"); + st->print_cr(" [0] = '\\0'"); + st->print_cr(" ..."); + st->print_cr(" [%d] = '\\0'", (int)sizeof(_pad_buf1) - 1); + st->print_cr(" }"); + st->print_cr(" _ref_count = %d", ref_count()); + st->print_cr(" _pad_buf2 = {"); + st->print_cr(" [0] = '\\0'"); + st->print_cr(" ..."); + st->print_cr(" [%d] = '\\0'", (int)sizeof(_pad_buf1) - 1); + st->print_cr(" }"); + st->print_cr(" _next_om = " INTPTR_FORMAT, p2i(_next_om)); + st->print_cr(" _recursions = " INTX_FORMAT, _recursions); + st->print_cr(" _EntryList = " INTPTR_FORMAT, p2i(_EntryList)); + st->print_cr(" _cxq = " INTPTR_FORMAT, p2i(_cxq)); + st->print_cr(" _succ = " INTPTR_FORMAT, p2i(_succ)); + st->print_cr(" _Responsible = " INTPTR_FORMAT, p2i(_Responsible)); + st->print_cr(" _Spinner = %d", _Spinner); + st->print_cr(" _SpinDuration = %d", _SpinDuration); + st->print_cr(" _contentions = %d", _contentions); + st->print_cr(" _WaitSet = " INTPTR_FORMAT, p2i(_WaitSet)); + st->print_cr(" _waiters = %d", _waiters); + st->print_cr(" _WaitSetLock = %d", _WaitSetLock); + st->print_cr("}"); +}