--- old/src/hotspot/cpu/aarch64/globals_aarch64.hpp 2019-10-17 17:31:16.000000000 -0400 +++ new/src/hotspot/cpu/aarch64/globals_aarch64.hpp 2019-10-17 17:31:15.000000000 -0400 @@ -75,6 +75,9 @@ define_pd_global(intx, InitArrayShortSize, BytesPerLong); define_pd_global(bool, ThreadLocalHandshakes, true); +// ObjectMonitor ref_count not implemented in C2 fast_lock() or +// fast_unlock() so use a handshake for safety. +define_pd_global(bool, HandshakeAfterDeflateIdleMonitors, true); #if defined(COMPILER1) || defined(COMPILER2) define_pd_global(intx, InlineSmallCode, 1000); --- old/src/hotspot/cpu/arm/globals_arm.hpp 2019-10-17 17:31:18.000000000 -0400 +++ new/src/hotspot/cpu/arm/globals_arm.hpp 2019-10-17 17:31:17.000000000 -0400 @@ -74,6 +74,10 @@ define_pd_global(intx, InitArrayShortSize, 8*BytesPerLong); define_pd_global(bool, ThreadLocalHandshakes, false); +// ObjectMonitor ref_count not implemented in C2 fast_lock() or +// fast_unlock() so use a handshake for safety. +// Will use a safepoint instead of a handshake on this platform. +define_pd_global(bool, HandshakeAfterDeflateIdleMonitors, true); #define ARCH_FLAGS(develop, \ product, \ --- old/src/hotspot/cpu/ppc/globals_ppc.hpp 2019-10-17 17:31:20.000000000 -0400 +++ new/src/hotspot/cpu/ppc/globals_ppc.hpp 2019-10-17 17:31:19.000000000 -0400 @@ -78,6 +78,9 @@ define_pd_global(intx, InitArrayShortSize, 9*BytesPerLong); define_pd_global(bool, ThreadLocalHandshakes, true); +// ObjectMonitor ref_count not implemented in C2 fast_lock() or +// fast_unlock() so use a handshake for safety. +define_pd_global(bool, HandshakeAfterDeflateIdleMonitors, true); // Platform dependent flag handling: flags only defined on this platform. #define ARCH_FLAGS(develop, \ --- old/src/hotspot/cpu/s390/globals_s390.hpp 2019-10-17 17:31:22.000000000 -0400 +++ new/src/hotspot/cpu/s390/globals_s390.hpp 2019-10-17 17:31:21.000000000 -0400 @@ -80,6 +80,9 @@ define_pd_global(intx, InitArrayShortSize, 1*BytesPerLong); define_pd_global(bool, ThreadLocalHandshakes, true); +// ObjectMonitor ref_count not implemented in C2 fast_lock() or +// fast_unlock() so use a handshake for safety. +define_pd_global(bool, HandshakeAfterDeflateIdleMonitors, true); #define ARCH_FLAGS(develop, \ product, \ --- old/src/hotspot/cpu/sparc/globals_sparc.hpp 2019-10-17 17:31:24.000000000 -0400 +++ new/src/hotspot/cpu/sparc/globals_sparc.hpp 2019-10-17 17:31:23.000000000 -0400 @@ -84,6 +84,9 @@ define_pd_global(intx, InitArrayShortSize, 8*BytesPerLong); define_pd_global(bool, ThreadLocalHandshakes, true); +// ObjectMonitor ref_count not implemented in C2 fast_lock() or +// fast_unlock() so use a handshake for safety. +define_pd_global(bool, HandshakeAfterDeflateIdleMonitors, true); #define ARCH_FLAGS(develop, \ product, \ --- old/src/hotspot/cpu/x86/globals_x86.hpp 2019-10-17 17:31:26.000000000 -0400 +++ new/src/hotspot/cpu/x86/globals_x86.hpp 2019-10-17 17:31:25.000000000 -0400 @@ -94,9 +94,20 @@ #if defined(_LP64) || defined(_WINDOWS) define_pd_global(bool, ThreadLocalHandshakes, true); +// ObjectMonitor ref_count is implemented in LP64 C2 fast_lock() +// and fast_unlock() so we don't need the handshake by default. +#ifdef _LP64 +define_pd_global(bool, HandshakeAfterDeflateIdleMonitors, false); +#else +define_pd_global(bool, HandshakeAfterDeflateIdleMonitors, true); +#endif #else // get_thread() is slow on linux 32 bit, therefore off by default define_pd_global(bool, ThreadLocalHandshakes, false); +// ObjectMonitor ref_count not implemented in C2 fast_lock() or +// fast_unlock() so use a handshake for safety. +// Will use a safepoint instead of a handshake on this platform. +define_pd_global(bool, HandshakeAfterDeflateIdleMonitors, true); #endif #define ARCH_FLAGS(develop, \ --- old/src/hotspot/cpu/x86/macroAssembler_x86.cpp 2019-10-17 17:31:28.000000000 -0400 +++ new/src/hotspot/cpu/x86/macroAssembler_x86.cpp 2019-10-17 17:31:27.000000000 -0400 @@ -1296,6 +1296,58 @@ #ifdef COMPILER2 +// Increment the ObjectMonitor's ref_count for safety or force a branch +// to 'done' with ICC.ZF=0 to indicate failure/take the slow path. +void MacroAssembler::inc_om_ref_count(Register obj_reg, Register om_reg, Register tmp_reg, Label& done) { + atomic_incl(Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + + Label LGoSlowPath; + if (AsyncDeflateIdleMonitors) { + // Race here if monitor is not owned! The above ref_count bump + // will cause subsequent async deflation to skip it. However, + // previous or concurrent async deflation is a race. + + // First check: if the owner field == DEFLATER_MARKER: + movptr(tmp_reg, Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); + // DEFLATER_MARKER == reinterpret_cast(-1) so the compiler + // doesn't like to use the define here: + cmpptr(tmp_reg, -1); + // If marked for async deflation, then take the slow path. This is a + // simpler check than what ObjectMonitorHandle::save_om_ptr() does + // so ObjectMonitor::install_displaced_markword_in_object() doesn't + // have to be implemented in macro assembler. + jccb(Assembler::equal, LGoSlowPath); + + // Second check: if ref_count field <= 0: + movptr(tmp_reg, Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + cmpptr(tmp_reg, 0); + // If async deflation is in the process of bailing out, but has not + // yet restored the ref_count field, then we take the slow path. We + // want a stable ref_count value for the fast path. + jccb(Assembler::lessEqual, LGoSlowPath); + + // Final check: if object field == obj_reg: + cmpptr(obj_reg, Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(object))); + // If the ObjectMonitor has been deflated and recycled, then take + // the slow path. + jccb(Assembler::notEqual, LGoSlowPath); + } + + Label LRetToCaller; + // We leave the ref_count incremented to protect the caller's code + // paths against async deflation. + jmpb(LRetToCaller); + + bind(LGoSlowPath); + lock(); + decrementl(Address(om_reg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + // Jump to 'done' with ICC.ZF=0 to indicate failure/take the slow path. + orl(tmp_reg, 1); + jmp(done); + + bind(LRetToCaller); +} + #if INCLUDE_RTM_OPT // Update rtm_counters based on abort status @@ -1529,10 +1581,19 @@ assert(UseRTMLocking, "why call this otherwise?"); assert(tmpReg == rax, ""); assert(scrReg == rdx, ""); - Label L_rtm_retry, L_decrement_retry, L_on_abort; + Label L_rtm_retry, L_decrement_retry, L_on_abort, L_local_done; int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); - // Without cast to int32_t a movptr will destroy r10 which is typically obj + if (!HandshakeAfterDeflateIdleMonitors) { + // Increment the ObjectMonitor's ref_count for safety or force the + // enter slow path via DONE_LABEL. + // In rtm_inflated_locking(), initially tmpReg contains the object's + // mark word which, in this case, is the (ObjectMonitor* | monitor_value). + // Also this code uses scrReg as its temporary register. + inc_om_ref_count(objReg, tmpReg /* om_reg */, scrReg /* tmp_reg */, DONE_LABEL); + } + + // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); movptr(boxReg, tmpReg); // Save ObjectMonitor address @@ -1555,7 +1616,7 @@ movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); movptr(tmpReg, Address(tmpReg, owner_offset)); testptr(tmpReg, tmpReg); - jcc(Assembler::zero, DONE_LABEL); + jcc(Assembler::zero, L_local_done); if (UseRTMXendForLockBusy) { xend(); jmp(L_decrement_retry); @@ -1590,7 +1651,7 @@ if (RTMRetryCount > 0) { // success done else retry - jccb(Assembler::equal, DONE_LABEL) ; + jccb(Assembler::equal, L_local_done); bind(L_decrement_retry); // Spin and retry if lock is busy. rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry); @@ -1598,15 +1659,28 @@ else { bind(L_decrement_retry); } + + // rtm_inflated_locking() exit paths come here except for a failed + // inc_om_ref_count() which goes directly to DONE_LABEL. + bind(L_local_done); + if (!HandshakeAfterDeflateIdleMonitors) { + pushf(); // Preserve flags. + // Decrement the ObjectMonitor's ref_count. + lock(); + decrementl(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + popf(); // Restore flags so we have the proper ICC.ZF value. + } + + jmp(DONE_LABEL) ; } #endif // INCLUDE_RTM_OPT -// Fast_Lock and Fast_Unlock used by C2 +// fast_lock and fast_unlock used by C2 // Because the transitions from emitted code to the runtime // monitorenter/exit helper stubs are so slow it's critical that -// we inline both the stack-locking fast-path and the inflated fast path. +// we inline both the stack-locking fast path and the inflated fast path. // // See also: cmpFastLock and cmpFastUnlock. // @@ -1615,7 +1689,7 @@ // option would be to emit TrySlowEnter and TrySlowExit methods // at startup-time. These methods would accept arguments as // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure -// indications in the icc.ZFlag. Fast_Lock and Fast_Unlock would simply +// indications in the icc.ZFlag. fast_lock and fast_unlock would simply // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit. // In practice, however, the # of lock sites is bounded and is usually small. // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer @@ -1634,8 +1708,8 @@ // // TODO: // -// * Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr). -// This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals. +// * Arrange for C2 to pass "Self" into fast_lock and fast_unlock in one of the registers (scr). +// This avoids manifesting the Self pointer in the fast_lock and fast_unlock terminals. // Given TLAB allocation, Self is usually manifested in a register, so passing it into // the lock operators would typically be faster than reifying Self. // @@ -1661,14 +1735,14 @@ // * use jccb and jmpb instead of jcc and jmp to improve code density. // But beware of excessive branch density on AMD Opterons. // -// * Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success -// or failure of the fast-path. If the fast-path fails then we pass -// control to the slow-path, typically in C. In Fast_Lock and -// Fast_Unlock we often branch to DONE_LABEL, just to find that C2 +// * Both fast_lock and fast_unlock set the ICC.ZF to indicate success +// or failure of the fast path. If the fast path fails then we pass +// control to the slow path, typically in C. In fast_lock and +// fast_unlock we often branch to DONE_LABEL, just to find that C2 // will emit a conditional branch immediately after the node. // So we have branches to branches and lots of ICC.ZF games. // Instead, it might be better to have C2 pass a "FailureLabel" -// into Fast_Lock and Fast_Unlock. In the case of success, control +// into fast_lock and fast_unlock. In the case of success, control // will drop through the node. ICC.ZF is undefined at exit. // In the case of failure, the node will branch directly to the // FailureLabel @@ -1813,7 +1887,7 @@ movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg); xorptr(boxReg, boxReg); // set icc.ZFlag = 1 to indicate success - // If the CAS fails we can either retry or pass control to the slow-path. + // If the CAS fails we can either retry or pass control to the slow path. // We use the latter tactic. // Pass the CAS result in the icc.ZFlag into DONE_LABEL // If the CAS was successful ... @@ -1821,37 +1895,34 @@ // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. // Intentional fall-through into DONE_LABEL ... #else // _LP64 - // It's inflated + // It's inflated and we use scrReg for ObjectMonitor* in this section. movq(scrReg, tmpReg); - xorq(tmpReg, tmpReg); + if (!HandshakeAfterDeflateIdleMonitors) { + // Increment the ObjectMonitor's ref_count for safety or force the + // enter slow path via DONE_LABEL. + // In fast_lock(), scrReg contains the object's mark word which, + // in this case, is the (ObjectMonitor* | monitor_value). Also this + // code uses tmpReg as its temporary register. + inc_om_ref_count(objReg, scrReg /* om_reg */, tmpReg /* tmp_reg */, DONE_LABEL); + } + + xorq(tmpReg, tmpReg); lock(); cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); // Unconditionally set box->_displaced_header = markWord::unused_mark(). - // Without cast to int32_t movptr will destroy r10 which is typically obj. + // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); - // The following code to verify that the object field still refers - // to the object we are trying to lock is not needed with safepoint - // based deflation. It is also not needed with async deflation when - // the DEFLATER_MARKER is allowed to linger in the owner field in an - // async deflated ObjectMonitor until replaced by the next owner value. - // We keep this code as a sanity check against bugs in other parts - // of the async deflation mechanism. - // - // If we weren't able to swing _owner from NULL to r15_thread - // then take the slow path. - jccb(Assembler::notZero, DONE_LABEL); - // r15_thread is now the owner so verify that the ObjectMonitor - // still refers to the same object. - cmpptr(objReg, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(object))); - // The ObjectMonitor still refers to the same object so - // r15_thread's ownership is valid. - jccb(Assembler::zero, DONE_LABEL); - // The ObjectMonitor does not refer to the same object so - // drop ownership. - movptr(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD); // Intentional fall-through into DONE_LABEL ... - // Propagate ICC.ZF from cmpptr() above into DONE_LABEL. + // Propagate ICC.ZF from CAS above into DONE_LABEL. + + if (!HandshakeAfterDeflateIdleMonitors) { + pushf(); // Preserve flags. + // Decrement the ObjectMonitor's ref_count. + lock(); + decrementl(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + popf(); // Restore flags so we have the proper ICC.ZF value. + } #endif // _LP64 #if INCLUDE_RTM_OPT } // use_rtm() @@ -1864,9 +1935,9 @@ bind(DONE_LABEL); // At DONE_LABEL the icc ZFlag is set as follows ... - // Fast_Unlock uses the same protocol. + // fast_unlock uses the same protocol. // ZFlag == 1 -> Success - // ZFlag == 0 -> Failure - force control through the slow-path + // ZFlag == 0 -> Failure - force control through the slow path } // obj: object to unlock @@ -1875,7 +1946,7 @@ // // Some commentary on balanced locking: // -// Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites. +// fast_lock and fast_unlock are emitted only for provably balanced lock sites. // Methods that don't have provably balanced locking are forced to run in the // interpreter - such methods won't be compiled to use fast_lock and fast_unlock. // The interpreter provides two properties: @@ -1896,7 +1967,7 @@ // should not be unlocked by "normal" java-level locking and vice-versa. The specification // doesn't specify what will occur if a program engages in such mixed-mode locking, however. // Arguably given that the spec legislates the JNI case as undefined our implementation -// could reasonably *avoid* checking owner in Fast_Unlock(). +// could reasonably *avoid* checking owner in fast_unlock(). // In the interest of performance we elide m->Owner==Self check in unlock. // A perfectly viable alternative is to elide the owner check except when // Xcheck:jni is enabled. @@ -1931,7 +2002,7 @@ jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword testptr(tmpReg, markWord::monitor_value); // Inflated? - jccb (Assembler::zero, Stacked); + jcc (Assembler::zero, Stacked); // It's inflated. #if INCLUDE_RTM_OPT @@ -1942,7 +2013,7 @@ testptr(boxReg, boxReg); jccb(Assembler::notZero, L_regular_inflated_unlock); xend(); - jmpb(DONE_LABEL); + jmp(DONE_LABEL); bind(L_regular_inflated_unlock); } #endif @@ -1961,7 +2032,7 @@ // a costly MEMBAR or CAS. See synchronizer.cpp for details on how // we detect and recover from the race that the 1-0 exit admits. // - // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier + // Conceptually fast_unlock() must execute a STST|LDST "release" barrier // before it STs null into _owner, releasing the lock. Updates // to data protected by the critical section must be visible before // we drop the lock (and thus before any other thread could acquire @@ -2004,27 +2075,39 @@ bind (CheckSucc); #else // _LP64 // It's inflated + + if (!HandshakeAfterDeflateIdleMonitors) { + // Increment the ObjectMonitor's ref_count for safety or force the + // exit slow path via DONE_LABEL. + // In fast_unlock(), tmpReg contains the object's mark word which, + // in this case, is the (ObjectMonitor* | monitor_value). Also this + // code uses boxReg as its temporary register. + inc_om_ref_count(objReg, tmpReg /* om_reg */, boxReg /* tmp_reg */, DONE_LABEL); + } + + // Try to avoid passing control into the slow path ... + Label LSuccess, LGoSlowPath; xorptr(boxReg, boxReg); orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions))); - jccb (Assembler::notZero, DONE_LABEL); + jccb(Assembler::notZero, LGoSlowPath); movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq))); orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList))); jccb (Assembler::notZero, CheckSucc); + // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD); - jmpb (DONE_LABEL); + jmpb(LSuccess); - // Try to avoid passing control into the slow_path ... - Label LSuccess, LGoSlowPath ; bind (CheckSucc); // The following optional optimization can be elided if necessary - // Effectively: if (succ == null) goto SlowPath + // Effectively: if (succ == null) goto slow path // The code reduces the window for a race, however, // and thus benefits performance. cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD); jccb (Assembler::zero, LGoSlowPath); xorptr(boxReg, boxReg); + // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD); // Memory barrier/fence @@ -2059,13 +2142,21 @@ // If that didn't work, then another thread grabbed the // lock so we're done (and exit was a success). jccb (Assembler::notEqual, LSuccess); - // Intentional fall-through into slow-path + // Intentional fall-through into slow path bind (LGoSlowPath); + if (!HandshakeAfterDeflateIdleMonitors) { + lock(); + decrementl(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + } orl (boxReg, 1); // set ICC.ZF=0 to indicate failure jmpb (DONE_LABEL); bind (LSuccess); + if (!HandshakeAfterDeflateIdleMonitors) { + lock(); + decrementl(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(ref_count))); + } testl (boxReg, 0); // set ICC.ZF=1 to indicate success jmpb (DONE_LABEL); --- old/src/hotspot/cpu/x86/macroAssembler_x86.hpp 2019-10-17 17:31:30.000000000 -0400 +++ new/src/hotspot/cpu/x86/macroAssembler_x86.hpp 2019-10-17 17:31:30.000000000 -0400 @@ -681,6 +681,7 @@ BiasedLockingCounters* counters = NULL); void biased_locking_exit (Register obj_reg, Register temp_reg, Label& done); #ifdef COMPILER2 + void inc_om_ref_count(Register obj_reg, Register om_reg, Register temp_reg, Label& done); // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file. // See full desription in macroAssembler_x86.cpp. void fast_lock(Register obj, Register box, Register tmp, --- old/src/hotspot/cpu/zero/globals_zero.hpp 2019-10-17 17:31:32.000000000 -0400 +++ new/src/hotspot/cpu/zero/globals_zero.hpp 2019-10-17 17:31:32.000000000 -0400 @@ -77,6 +77,9 @@ define_pd_global(bool, CompactStrings, false); define_pd_global(bool, ThreadLocalHandshakes, true); +// No ObjectMonitor C2 fast_lock() or fast_unlock() so no need to +// use a handshake for safety. +define_pd_global(bool, HandshakeAfterDeflateIdleMonitors, false); #define ARCH_FLAGS(develop, \ product, \ --- old/src/hotspot/share/logging/logTag.hpp 2019-10-17 17:31:34.000000000 -0400 +++ new/src/hotspot/share/logging/logTag.hpp 2019-10-17 17:31:33.000000000 -0400 @@ -116,6 +116,7 @@ LOG_TAG(oops) \ LOG_TAG(oopstorage) \ LOG_TAG(os) \ + LOG_TAG(owner) \ LOG_TAG(pagesize) \ LOG_TAG(patch) \ LOG_TAG(path) \ --- old/src/hotspot/share/runtime/globals.hpp 2019-10-17 17:31:36.000000000 -0400 +++ new/src/hotspot/share/runtime/globals.hpp 2019-10-17 17:31:35.000000000 -0400 @@ -726,8 +726,9 @@ "MonitorUsedDeflationThreshold is exceeded (0 is off).") \ range(0, max_jint) \ \ - diagnostic(bool, CheckMonitorLists, false, \ - "Sanity check the lock free ObjectMonitor lists.") \ + diagnostic_pd(bool, HandshakeAfterDeflateIdleMonitors, \ + "Handshake with all JavaThreads after async deflating idle " \ + "monitors to force threads to leave C2 monitor code.") \ \ experimental(intx, MonitorUsedDeflationThreshold, 90, \ "Percentage of used monitors before triggering deflation (0 is " \ --- old/src/hotspot/share/runtime/objectMonitor.cpp 2019-10-17 17:31:38.000000000 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.cpp 2019-10-17 17:31:38.000000000 -0400 @@ -245,7 +245,7 @@ // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors. Thread * const Self = THREAD; - void * cur = Atomic::cmpxchg(Self, &_owner, (void*)NULL); + void* cur = try_set_owner_from(Self, NULL); if (cur == NULL) { assert(_recursions == 0, "invariant"); return; @@ -260,14 +260,12 @@ if (Self->is_lock_owned ((address)cur)) { assert(_recursions == 0, "internal state error"); _recursions = 1; - // Commute owner from a thread-specific on-stack BasicLockObject address to - // a full-fledged "Thread *". - _owner = Self; + set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. return; } if (AsyncDeflateIdleMonitors && - Atomic::cmpxchg(Self, &_owner, DEFLATER_MARKER) == DEFLATER_MARKER) { + try_set_owner_from(Self, DEFLATER_MARKER) == DEFLATER_MARKER) { // The deflation protocol finished the first part (setting owner), // but it failed the second part (making ref_count negative) and // bailed. Or the ObjectMonitor was async deflated and reused. @@ -415,7 +413,7 @@ int ObjectMonitor::TryLock(Thread * Self) { void * own = _owner; if (own != NULL) return 0; - if (Atomic::replace_if_null(Self, &_owner)) { + if (try_set_owner_from(Self, NULL) == NULL) { assert(_recursions == 0, "invariant"); return 1; } @@ -446,8 +444,8 @@ markWord dmw = header(); if (dmw.value() == 0) { - // ObjectMonitor's header/dmw has been cleared so the object's - // header has already been restored. + // ObjectMonitor's header/dmw has been cleared so the ObjectMonitor + // has been deflated and taken off the global free list. return; } @@ -497,12 +495,6 @@ // detected a race with the deflation process also wanted the // object's header restored before they retry their operation and // because it is restored they will only retry once. - - if (marked_dmw.value() != 0) { - // Clear _header to NULL if it is still marked_dmw so a racing - // install_displaced_markword_in_object() can bail out sooner. - Atomic::cmpxchg(markWord::zero(), &_header, marked_dmw); - } } // Convert the fields used by is_busy() to a string that can be @@ -539,7 +531,7 @@ } if (AsyncDeflateIdleMonitors && - Atomic::cmpxchg(Self, &_owner, DEFLATER_MARKER) == DEFLATER_MARKER) { + try_set_owner_from(Self, DEFLATER_MARKER) == DEFLATER_MARKER) { // The deflation protocol finished the first part (setting owner), // but it failed the second part (making ref_count negative) and // bailed. Or the ObjectMonitor was async deflated and reused. @@ -666,7 +658,7 @@ if (TryLock(Self) > 0) break; if (AsyncDeflateIdleMonitors && - Atomic::cmpxchg(Self, &_owner, DEFLATER_MARKER) == DEFLATER_MARKER) { + try_set_owner_from(Self, DEFLATER_MARKER) == DEFLATER_MARKER) { // The deflation protocol finished the first part (setting owner), // but it failed the second part (making ref_count negative) and // bailed. Or the ObjectMonitor was async deflated and reused. @@ -798,7 +790,7 @@ if (TrySpin(Self) > 0) break; if (AsyncDeflateIdleMonitors && - Atomic::cmpxchg(Self, &_owner, DEFLATER_MARKER) == DEFLATER_MARKER) { + try_set_owner_from(Self, DEFLATER_MARKER) == DEFLATER_MARKER) { // The deflation protocol finished the first part (setting owner), // but it failed the second part (making ref_count negative) and // bailed. Or the ObjectMonitor was async deflated and reused. @@ -995,13 +987,10 @@ void ObjectMonitor::exit(bool not_suspended, TRAPS) { Thread * const Self = THREAD; if (THREAD != _owner) { - if (THREAD->is_lock_owned((address) _owner)) { - // Transmute _owner from a BasicLock pointer to a Thread address. - // We don't need to hold _mutex for this transition. - // Non-null to Non-null is safe as long as all readers can - // tolerate either flavor. + void* cur = _owner; + if (THREAD->is_lock_owned((address)cur)) { assert(_recursions == 0, "invariant"); - _owner = THREAD; + set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. _recursions = 0; } else { // Apparent unbalanced locking ... @@ -1013,8 +1002,14 @@ // way we should encounter this situation is in the presence of // unbalanced JNI locking. TODO: CheckJNICalls. // See also: CR4414101 - assert(false, "Non-balanced monitor enter/exit! Likely JNI locking: " - "owner=" INTPTR_FORMAT, p2i(_owner)); + tty->print_cr("ERROR: ObjectMonitor::exit(): thread=" INTPTR_FORMAT + " is exiting an ObjectMonitor it does not own.", + p2i(THREAD)); + tty->print_cr("The imbalance is possibly caused by JNI locking."); + print_debug_style_on(tty); + // Changing this from an assert() to ADIM_guarantee() may run + // afoul of any test that is inducing non-balanced JNI locking. + ADIM_guarantee(false, "Non-balanced monitor enter/exit!"); return; } } @@ -1043,8 +1038,12 @@ // must not float (reorder) past the following store that drops the lock. // On SPARC that requires MEMBAR #loadstore|#storestore. // But of course in TSO #loadstore|#storestore is not required. - OrderAccess::release_store(&_owner, (void*)NULL); // drop the lock - OrderAccess::storeload(); // See if we need to wake a successor + if (AsyncDeflateIdleMonitors) { + set_owner_from(NULL, Self); + } else { + OrderAccess::release_store(&_owner, (void*)NULL); // drop the lock + OrderAccess::storeload(); // See if we need to wake a successor + } if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) { return; } @@ -1086,7 +1085,7 @@ // to reacquire the lock the responsibility for ensuring succession // falls to the new owner. // - if (!Atomic::replace_if_null(THREAD, &_owner)) { + if (try_set_owner_from(Self, NULL) != NULL) { return; } @@ -1219,8 +1218,12 @@ Wakee = NULL; // Drop the lock - OrderAccess::release_store(&_owner, (void*)NULL); - OrderAccess::fence(); // ST _owner vs LD in unpark() + if (AsyncDeflateIdleMonitors) { + set_owner_from(NULL, Self); + } else { + OrderAccess::release_store(&_owner, (void*)NULL); + OrderAccess::fence(); // ST _owner vs LD in unpark() + } DTRACE_MONITOR_PROBE(contended__exit, this, object(), Self); Trigger->unpark(); @@ -1247,9 +1250,10 @@ assert(InitDone, "Unexpectedly not initialized"); if (THREAD != _owner) { - if (THREAD->is_lock_owned ((address)_owner)) { + void* cur = _owner; + if (THREAD->is_lock_owned((address)cur)) { assert(_recursions == 0, "internal state error"); - _owner = THREAD; // Convert from basiclock addr to Thread addr + set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. _recursions = 0; } } @@ -1297,8 +1301,9 @@ if (_owner == THREAD) { return true; } - if (THREAD->is_lock_owned((address)_owner)) { - _owner = THREAD; // convert from BasicLock addr to Thread addr + void* cur = _owner; + if (THREAD->is_lock_owned((address)cur)) { + set_owner_from_BasicLock(THREAD, cur); // Convert from BasicLock* to Thread*. _recursions = 0; return true; } @@ -1803,7 +1808,7 @@ Thread * ox = (Thread *) _owner; if (ox == NULL) { - ox = (Thread*)Atomic::cmpxchg(Self, &_owner, (void*)NULL); + ox = (Thread*)try_set_owner_from(Self, NULL); if (ox == NULL) { // The CAS succeeded -- this thread acquired ownership // Take care of some bookkeeping to exit spin state. @@ -2156,3 +2161,89 @@ p2i(owner())); } void ObjectMonitor::print() const { print_on(tty); } + +// Print the ObjectMonitor like a debugger would: +// +// (ObjectMonitor) 0x00007fdfb6012e40 = { +// _header = (_value = 1) +// _object = 0x000000070ff45fd0 +// _allocation_state = Old +// _pad_buf0 = { +// [0] = '\0' +// ... +// [43] = '\0' +// } +// _owner = 0x0000000000000000 +// _previous_owner_tid = 0 +// _pad_buf1 = { +// [0] = '\0' +// ... +// [47] = '\0' +// } +// _ref_count = 1 +// _pad_buf2 = { +// [0] = '\0' +// ... +// [59] = '\0' +// } +// _next_om = 0x0000000000000000 +// _recursions = 0 +// _EntryList = 0x0000000000000000 +// _cxq = 0x0000000000000000 +// _succ = 0x0000000000000000 +// _Responsible = 0x0000000000000000 +// _Spinner = 0 +// _SpinDuration = 5000 +// _contentions = 0 +// _WaitSet = 0x0000700009756248 +// _waiters = 1 +// _WaitSetLock = 0 +// } +// +void ObjectMonitor::print_debug_style_on(outputStream* st) const { + st->print_cr("(ObjectMonitor *) " INTPTR_FORMAT " = {", p2i(this)); + st->print_cr(" _header = " INTPTR_FORMAT, header().value()); + st->print_cr(" _object = " INTPTR_FORMAT, p2i(_object)); + st->print(" _allocation_state = "); + if (is_free()) { + st->print("Free"); + } else if (is_old()) { + st->print("Old"); + } else if (is_new()) { + st->print("New"); + } else { + st->print("unknown=%d", _allocation_state); + } + st->cr(); + st->print_cr(" _pad_buf0 = {"); + st->print_cr(" [0] = '\\0'"); + st->print_cr(" ..."); + st->print_cr(" [%d] = '\\0'", (int)sizeof(_pad_buf0) - 1); + st->print_cr(" }"); + st->print_cr(" _owner = " INTPTR_FORMAT, p2i(_owner)); + st->print_cr(" _previous_owner_tid = " JLONG_FORMAT, _previous_owner_tid); + st->print_cr(" _pad_buf1 = {"); + st->print_cr(" [0] = '\\0'"); + st->print_cr(" ..."); + st->print_cr(" [%d] = '\\0'", (int)sizeof(_pad_buf1) - 1); + st->print_cr(" }"); + st->print_cr(" _ref_count = %d", ref_count()); + st->print_cr(" _pad_buf2 = {"); + st->print_cr(" [0] = '\\0'"); + st->print_cr(" ..."); + st->print_cr(" [%d] = '\\0'", (int)sizeof(_pad_buf1) - 1); + st->print_cr(" }"); + st->print_cr(" _next_om = " INTPTR_FORMAT, p2i(_next_om)); + st->print_cr(" _recursions = " INTX_FORMAT, _recursions); + st->print_cr(" _EntryList = " INTPTR_FORMAT, p2i(_EntryList)); + st->print_cr(" _cxq = " INTPTR_FORMAT, p2i(_cxq)); + st->print_cr(" _succ = " INTPTR_FORMAT, p2i(_succ)); + st->print_cr(" _Responsible = " INTPTR_FORMAT, p2i(_Responsible)); + st->print_cr(" _Spinner = %d", _Spinner); + st->print_cr(" _SpinDuration = %d", _SpinDuration); + st->print_cr(" _contentions = %d", _contentions); + st->print_cr(" _WaitSet = " INTPTR_FORMAT, p2i(_WaitSet)); + st->print_cr(" _waiters = %d", _waiters); + st->print_cr(" _WaitSetLock = %d", _WaitSetLock); + st->print_cr("}"); +} --- old/src/hotspot/share/runtime/objectMonitor.hpp 2019-10-17 17:31:40.000000000 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.hpp 2019-10-17 17:31:40.000000000 -0400 @@ -191,7 +191,6 @@ volatile int _WaitSetLock; // protects Wait Queue - simple spinlock public: - volatile int visit_marker; static void Initialize(); // Only perform a PerfData operation if the PerfData object has been @@ -226,6 +225,7 @@ static int header_offset_in_bytes() { return offset_of(ObjectMonitor, _header); } static int object_offset_in_bytes() { return offset_of(ObjectMonitor, _object); } static int owner_offset_in_bytes() { return offset_of(ObjectMonitor, _owner); } + static int ref_count_offset_in_bytes() { return offset_of(ObjectMonitor, _ref_count); } static int recursions_offset_in_bytes() { return offset_of(ObjectMonitor, _recursions); } static int cxq_offset_in_bytes() { return offset_of(ObjectMonitor, _cxq); } static int succ_offset_in_bytes() { return offset_of(ObjectMonitor, _succ); } @@ -272,7 +272,15 @@ void* owner() const; // Returns NULL if DEFLATER_MARKER is observed. // Returns true if owner field == DEFLATER_MARKER and false otherwise. bool owner_is_DEFLATER_MARKER(); - void set_owner(void* owner); + // Set _owner field to new_value; current value must match old_value. + void set_owner_from(void* new_value, void* old_value); + // Set _owner field to new_value; current value must match old_value1 or old_value2. + void set_owner_from(void* new_value, void* old_value1, void* old_value2); + // Set _owner field to self; current value must match basic_lock_p. + void set_owner_from_BasicLock(Thread* self, void* basic_lock_p); + // Try to set _owner field to new_value if the current value matches + // old_value. Otherwise, does not change the _owner field. + void* try_set_owner_from(void* new_value, void* old_value); jint waiters() const; @@ -339,6 +347,7 @@ void notifyAll(TRAPS); void print() const; + void print_debug_style_on(outputStream* st) const; void print_on(outputStream* st) const; // Use the following at your own risk --- old/src/hotspot/share/runtime/objectMonitor.inline.hpp 2019-10-17 17:31:42.000000000 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.inline.hpp 2019-10-17 17:31:42.000000000 -0400 @@ -25,6 +25,7 @@ #ifndef SHARE_RUNTIME_OBJECTMONITOR_INLINE_HPP #define SHARE_RUNTIME_OBJECTMONITOR_INLINE_HPP +#include "logging/log.hpp" #include "runtime/atomic.hpp" inline intptr_t ObjectMonitor::is_entered(TRAPS) const { @@ -118,8 +119,56 @@ return _contentions; } -inline void ObjectMonitor::set_owner(void* owner) { - _owner = owner; +// Set _owner field to new_value; current value must match old_value. +inline void ObjectMonitor::set_owner_from(void* new_value, void* old_value) { + void* prev = Atomic::cmpxchg(new_value, &_owner, old_value); + ADIM_guarantee(prev == old_value, "unexpected prev owner=" INTPTR_FORMAT + ", expected=" INTPTR_FORMAT, p2i(prev), p2i(old_value)); + log_trace(monitorinflation, owner)("mid=" INTPTR_FORMAT ", prev=" + INTPTR_FORMAT ", new=" INTPTR_FORMAT, + p2i(this), p2i(prev), p2i(new_value)); +} + +// Set _owner field to new_value; current value must match old_value1 or old_value2. +inline void ObjectMonitor::set_owner_from(void* new_value, void* old_value1, void* old_value2) { + void* prev = Atomic::cmpxchg(new_value, &_owner, old_value1); + if (prev != old_value1) { + prev = Atomic::cmpxchg(new_value, &_owner, old_value2); + } + ADIM_guarantee(prev == old_value1 || prev == old_value2, + "unexpected prev owner=" INTPTR_FORMAT ", expected1=" + INTPTR_FORMAT " or expected2=" INTPTR_FORMAT, p2i(prev), + p2i(old_value1), p2i(old_value2)); + log_trace(monitorinflation, owner)("mid=" INTPTR_FORMAT ", prev=" + INTPTR_FORMAT ", new=" INTPTR_FORMAT, + p2i(this), p2i(prev), p2i(new_value)); +} + +// Set _owner field to self; current value must match basic_lock_p. +inline void ObjectMonitor::set_owner_from_BasicLock(Thread* self, void* basic_lock_p) { + assert(self->is_lock_owned((address)basic_lock_p), "self=" INTPTR_FORMAT + " must own basic_lock_p=" INTPTR_FORMAT, p2i(self), p2i(basic_lock_p)); + void* prev = _owner; + ADIM_guarantee(prev == basic_lock_p, "unexpected prev owner=" INTPTR_FORMAT + ", expected=" INTPTR_FORMAT, p2i(prev), p2i(basic_lock_p)); + // Non-null owner field to non-null owner field is safe without + // cmpxchg() as long as all readers can tolerate either flavor. + _owner = self; + log_trace(monitorinflation, owner)("mid=" INTPTR_FORMAT ", prev=" + INTPTR_FORMAT ", new=" INTPTR_FORMAT, + p2i(this), p2i(prev), p2i(self)); +} + +// Try to set _owner field to new_value if the current value matches +// old_value. Otherwise, does not change the _owner field. +inline void* ObjectMonitor::try_set_owner_from(void* new_value, void* old_value) { + void* prev = Atomic::cmpxchg(new_value, &_owner, old_value); + if (prev == old_value) { + log_trace(monitorinflation, owner)("mid=" INTPTR_FORMAT ", prev=" + INTPTR_FORMAT ", new=" INTPTR_FORMAT, + p2i(this), p2i(prev), p2i(new_value)); + } + return prev; } inline void ObjectMonitor::set_allocation_state(ObjectMonitor::AllocationState s) { --- old/src/hotspot/share/runtime/serviceThread.cpp 2019-10-17 17:31:44.000000000 -0400 +++ new/src/hotspot/share/runtime/serviceThread.cpp 2019-10-17 17:31:44.000000000 -0400 @@ -190,24 +190,7 @@ } if (deflate_idle_monitors) { - // Deflate any global idle monitors. - ObjectSynchronizer::deflate_global_idle_monitors_using_JT(); - - int count = 0; - for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { - if (jt->om_in_use_count > 0 && !jt->is_exiting()) { - // This JavaThread is using ObjectMonitors so deflate any that - // are idle unless this JavaThread is exiting; do not race with - // ObjectSynchronizer::om_flush(). - ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(jt); - count++; - } - } - if (count > 0) { - log_debug(monitorinflation)("did async deflation of idle monitors for %d thread(s).", count); - } - // The ServiceThread's async deflation request has been processed. - ObjectSynchronizer::set_is_async_deflation_requested(false); + ObjectSynchronizer::deflate_idle_monitors_using_JT(); } } } --- old/src/hotspot/share/runtime/synchronizer.cpp 2019-10-17 17:31:46.000000000 -0400 +++ new/src/hotspot/share/runtime/synchronizer.cpp 2019-10-17 17:31:46.000000000 -0400 @@ -37,6 +37,7 @@ #include "runtime/atomic.hpp" #include "runtime/biasedLocking.hpp" #include "runtime/handles.inline.hpp" +#include "runtime/handshake.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/objectMonitor.hpp" @@ -128,10 +129,17 @@ // Global ObjectMonitor in-use list. When a JavaThread is exiting, // ObjectMonitors on its per-thread in-use list are prepended here. static ObjectMonitor* volatile g_om_in_use_list = NULL; +// Global ObjectMonitor wait list. If HandshakeAfterDeflateIdleMonitors +// is true, deflated ObjectMonitors wait on this list until after a +// handshake or a safepoint for platforms that don't support handshakes. +// After the handshake or safepoint, the deflated ObjectMonitors are +// prepended to g_free_list. +static ObjectMonitor* volatile g_wait_list = NULL; static volatile int g_om_free_count = 0; // # on g_free_list static volatile int g_om_in_use_count = 0; // # on g_om_in_use_list static volatile int g_om_population = 0; // # Extant -- in circulation +static volatile int g_om_wait_count = 0; // # on g_wait_list #define CHAINMARKER (cast_to_oop(-1)) @@ -212,86 +220,12 @@ return (ObjectMonitor*)((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1); } -#if 0 -// XXX - this is unused -// Unmark the next field in an ObjectMonitor. Requires that the next -// field be marked. -static void unmark_next(ObjectMonitor* om) { - ADIM_guarantee(is_next_marked(om), "next field must be marked: next=" INTPTR_FORMAT, p2i(om->_next_om)); - - ObjectMonitor* next = unmarked_next(om); - set_next(om, next); -} -#endif - -volatile int visit_counter = 42; -static void chk_for_list_loop(ObjectMonitor* list, int count) { - if (!CheckMonitorLists) { - return; - } - int l_visit_counter = Atomic::add(1, &visit_counter); - int l_count = 0; - ObjectMonitor* prev = NULL; - for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) { - if (mid->visit_marker == l_visit_counter) { - log_error(monitorinflation)("ERROR: prev=" INTPTR_FORMAT ", l_count=%d" - " refers to an ObjectMonitor that has" - " already been visited: mid=" INTPTR_FORMAT, - p2i(prev), l_count, p2i(mid)); - fatal("list=" INTPTR_FORMAT " of %d items has a loop.", p2i(list), count); - } - mid->visit_marker = l_visit_counter; - prev = mid; - if (++l_count > count + 1024 * 1024) { - fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d", - p2i(list), count, l_count); - } - } -} - -static void chk_om_not_on_list(ObjectMonitor* om, ObjectMonitor* list, int count) { - if (!CheckMonitorLists) { - return; - } - guarantee(list != om, "ERROR: om=" INTPTR_FORMAT " must not be head of the " - "list=" INTPTR_FORMAT ", count=%d", p2i(om), p2i(list), count); - int l_count = 0; - for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) { - if (unmarked_next(mid) == om) { - log_error(monitorinflation)("ERROR: mid=" INTPTR_FORMAT ", l_count=%d" - " next_om refers to om=" INTPTR_FORMAT, - p2i(mid), l_count, p2i(om)); - fatal("list=" INTPTR_FORMAT " of %d items has bad next_om value.", - p2i(list), count); - } - if (++l_count > count + 1024 * 1024) { - fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d", - p2i(list), count, l_count); - } - } -} - -static void chk_om_elems_not_on_list(ObjectMonitor* elems, int elems_count, - ObjectMonitor* list, int list_count) { - if (!CheckMonitorLists) { - return; - } - chk_for_list_loop(elems, elems_count); - for (ObjectMonitor* mid = elems; mid != NULL; mid = unmarked_next(mid)) { - chk_om_not_on_list(mid, list, list_count); - } -} - // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is // the last ObjectMonitor in the list and there are 'count' on the list. // Also updates the specified *count_p. static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail, int count, ObjectMonitor* volatile* list_p, volatile int* count_p) { - chk_for_list_loop(OrderAccess::load_acquire(list_p), - OrderAccess::load_acquire(count_p)); - chk_om_elems_not_on_list(list, count, OrderAccess::load_acquire(list_p), - OrderAccess::load_acquire(count_p)); while (true) { ObjectMonitor* cur = OrderAccess::load_acquire(list_p); // Prepend list to *list_p. @@ -332,10 +266,10 @@ void ObjectSynchronizer::prepend_block_to_lists(PaddedObjectMonitor* new_blk) { // First we handle g_block_list: while (true) { - PaddedObjectMonitor* cur = OrderAccess::load_acquire(&g_block_list); + PaddedObjectMonitor* cur = g_block_list; // Prepend new_blk to g_block_list. The first ObjectMonitor in // a block is reserved for use as linkage to the next block. - OrderAccess::release_store(&new_blk[0]._next_om, cur); + new_blk[0]._next_om = cur; if (Atomic::cmpxchg(new_blk, &g_block_list, cur) == cur) { // Successfully switched g_block_list to the new_blk value. Atomic::add(_BLOCKSIZE - 1, &g_om_population); @@ -357,6 +291,15 @@ prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count); } +// Prepend a list of ObjectMonitors to g_wait_list. 'tail' is the last +// ObjectMonitor in the list and there are 'count' on the list. Also +// updates g_om_wait_count. +static void prepend_list_to_g_wait_list(ObjectMonitor* list, + ObjectMonitor* tail, int count) { + assert(HandshakeAfterDeflateIdleMonitors, "sanity check"); + prepend_list_to_common(list, tail, count, &g_wait_list, &g_om_wait_count); +} + // Prepend a list of ObjectMonitors to g_om_in_use_list. 'tail' is the last // ObjectMonitor in the list and there are 'count' on the list. Also // updates g_om_in_use_list. @@ -369,43 +312,28 @@ // the specified counter. static void prepend_to_common(ObjectMonitor* m, ObjectMonitor* volatile * list_p, int volatile * count_p) { - chk_for_list_loop(OrderAccess::load_acquire(list_p), - OrderAccess::load_acquire(count_p)); - chk_om_not_on_list(m, OrderAccess::load_acquire(list_p), - OrderAccess::load_acquire(count_p)); - while (true) { - ObjectMonitor* cur = OrderAccess::load_acquire(list_p); - // Prepend ObjectMonitor to *list_p. + (void)mark_next_loop(m); // mark m so we can safely update its next field + ObjectMonitor* cur = NULL; ObjectMonitor* next = NULL; - if (!mark_next(m, &next)) { - continue; // failed to mark next field so try it all again + // Mark the list head to guard against A-B-A race: + if (mark_list_head(list_p, &cur, &next)) { + // List head is now marked so we can safely switch it. + set_next(m, cur); // m now points to cur (and unmarks m) + OrderAccess::release_store(list_p, m); // Switch list head to unmarked m. + set_next(cur, next); // Unmark the previous list head. + break; } - set_next(m, cur); // m now points to cur (and unmarks m) - if (cur == NULL) { - // No potential race with other prependers since *list_p is empty. - if (Atomic::cmpxchg(m, list_p, cur) == cur) { - // Successfully switched *list_p to 'm'. - Atomic::inc(count_p); - break; - } - // Implied else: try it all again - } else { - // Try to mark next field to guard against races: - if (!mark_next(cur, &next)) { - continue; // failed to mark next field so try it all again - } - // We marked the next field so try to switch *list_p to 'm'. - if (Atomic::cmpxchg(m, list_p, cur) != cur) { - // The list head has changed so unmark the next field and try again: - set_next(cur, next); - continue; - } - Atomic::inc(count_p); - set_next(cur, next); // unmark next field + // The list is empty so try to set the list head. + assert(cur == NULL, "cur must be NULL: cur=" INTPTR_FORMAT, p2i(cur)); + set_next(m, cur); // m now points to NULL (and unmarks m) + if (Atomic::cmpxchg(m, list_p, cur) == cur) { + // List head is now unmarked m. break; } + // Implied else: try it all again } + Atomic::inc(count_p); } // Prepend an ObjectMonitor to a per-thread om_free_list. @@ -424,9 +352,6 @@ // decrements the specified counter. Returns NULL if none are available. static ObjectMonitor* take_from_start_of_common(ObjectMonitor* volatile * list_p, int volatile * count_p) { - chk_for_list_loop(OrderAccess::load_acquire(list_p), - OrderAccess::load_acquire(count_p)); - ObjectMonitor* next = NULL; ObjectMonitor* take = NULL; // Mark the list head to guard against A-B-A race: @@ -572,13 +497,13 @@ // and last are the inflated Java Monitor (ObjectMonitor) checks. lock->set_displaced_header(markWord::unused_mark()); - if (owner == NULL && Atomic::replace_if_null(self, &(m->_owner))) { + if (owner == NULL && m->try_set_owner_from(self, NULL) == NULL) { assert(m->_recursions == 0, "invariant"); return true; } if (AsyncDeflateIdleMonitors && - Atomic::cmpxchg(self, &m->_owner, DEFLATER_MARKER) == DEFLATER_MARKER) { + m->try_set_owner_from(self, DEFLATER_MARKER) == DEFLATER_MARKER) { // The deflation protocol finished the first part (setting owner), // but it failed the second part (making ref_count negative) and // bailed. Or the ObjectMonitor was async deflated and reused. @@ -1319,6 +1244,9 @@ if (MonitorUsedDeflationThreshold > 0) { int monitors_used = OrderAccess::load_acquire(&g_om_population) - OrderAccess::load_acquire(&g_om_free_count); + if (HandshakeAfterDeflateIdleMonitors) { + monitors_used -= OrderAccess::load_acquire(&g_om_wait_count); + } int monitor_usage = (monitors_used * 100LL) / OrderAccess::load_acquire(&g_om_population); return monitor_usage > MonitorUsedDeflationThreshold; @@ -1351,8 +1279,12 @@ _last_async_deflation_time_ns = os::javaTimeNanos(); return true; } - if (is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) - - OrderAccess::load_acquire(&g_om_free_count))) { + int monitors_used = OrderAccess::load_acquire(&g_om_population) - + OrderAccess::load_acquire(&g_om_free_count); + if (HandshakeAfterDeflateIdleMonitors) { + monitors_used -= OrderAccess::load_acquire(&g_om_wait_count); + } + if (is_MonitorBound_exceeded(monitors_used)) { // Not enough ObjectMonitors on the global free list. return true; } @@ -1397,7 +1329,6 @@ void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, int count, OopClosure* f) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); - chk_for_list_loop(list, count); // The oops_do() phase does not overlap with monitor deflation // so no need to update the ObjectMonitor's ref_count for this // ObjectMonitor* use. @@ -1539,6 +1470,9 @@ } } take->Recycle(); + // Since we're taking from the global free-list, take must be Free. + // om_release() also sets the allocation state to Free because it + // is called from other code paths. assert(take->is_free(), "invariant"); om_release(self, take, false); } @@ -1638,24 +1572,15 @@ while (true) { if (m == mid) { // We found 'm' on the per-thread in-use list so try to extract it. - // First try the list head: - if (Atomic::cmpxchg(next, &self->om_in_use_list, mid) != mid) { - // We could not switch the list head to next. - ObjectMonitor* marked_mid = mark_om_ptr(mid); - // Switch cur_mid_in_use's next field to next (which also - // unmarks cur_mid_in_use): - ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL"); - if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, marked_mid) - != marked_mid) { - // We could not switch cur_mid_in_use's next field. This - // should not be possible since it was marked so we: - fatal("mid=" INTPTR_FORMAT " must be referred to by the list " - "head: &om_in_use_list=" INTPTR_FORMAT " or by " - "cur_mid_in_use's next field: cur_mid_in_use=" INTPTR_FORMAT - ", next_om=" INTPTR_FORMAT, p2i(mid), - p2i((ObjectMonitor**)&self->om_in_use_list), - p2i(cur_mid_in_use), p2i(cur_mid_in_use->_next_om)); - } + if (cur_mid_in_use == NULL) { + // mid is the list head and it is marked. Switch the list head + // to next which unmarks the list head, but leaves mid marked: + OrderAccess::release_store(&self->om_in_use_list, next); + } else { + // mid and cur_mid_in_use are marked. Switch cur_mid_in_use's + // next field to next which unmarks cur_mid_in_use, but leaves + // mid marked: + OrderAccess::release_store(&cur_mid_in_use->_next_om, next); } extracted = true; Atomic::dec(&self->om_in_use_count); @@ -1720,7 +1645,6 @@ // is exiting, but if it has made it past that check before we // started exiting, then it is racing to get to the in-use list. if (mark_list_head(&self->om_in_use_list, &in_use_list, &next)) { - chk_for_list_loop(in_use_list, OrderAccess::load_acquire(&self->om_in_use_count)); // At this point, we have marked the in-use list head so an // async deflation thread cannot come in after us. If an async // deflation thread is ahead of us, then we'll detect that and @@ -1776,7 +1700,6 @@ ObjectMonitor* free_list = OrderAccess::load_acquire(&self->om_free_list); ObjectMonitor* free_tail = NULL; if (free_list != NULL) { - chk_for_list_loop(free_list, OrderAccess::load_acquire(&self->om_free_count)); // The thread is going away. Set 'free_tail' to the last per-thread free // monitor which will be linked to g_free_list below. stringStream ss; @@ -1929,6 +1852,7 @@ markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark); if (cmp != mark) { + // om_release() will reset the allocation state from New to Free. om_release(self, m, true); continue; // Interference -- just retry } @@ -1976,19 +1900,26 @@ // Note that a thread can inflate an object // that it has stack-locked -- as might happen in wait() -- directly // with CAS. That is, we can avoid the xchg-NULL .... ST idiom. - m->set_owner(mark.locker()); + if (AsyncDeflateIdleMonitors) { + m->set_owner_from(mark.locker(), NULL, DEFLATER_MARKER); + } else { + m->set_owner_from(mark.locker(), NULL); + } m->set_object(object); // TODO-FIXME: assert BasicLock->dhw != 0. omh_p->set_om_ptr(m); - assert(m->is_new(), "freshly allocated monitor must be new"); - m->set_allocation_state(ObjectMonitor::Old); // Must preserve store ordering. The monitor state must // be stable at the time of publishing the monitor address. guarantee(object->mark() == markWord::INFLATING(), "invariant"); object->release_set_mark(markWord::encode(m)); + // Once ObjectMonitor is configured and the object is associated + // with the ObjectMonitor, it is safe to allow async deflation: + assert(m->is_new(), "freshly allocated monitor must be new"); + m->set_allocation_state(ObjectMonitor::Old); + // Hopefully the performance counters are allocated on distinct cache lines // to avoid false sharing on MP systems ... OM_PERFDATA_OP(Inflations, inc()); @@ -2029,19 +1960,13 @@ m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class omh_p->set_om_ptr(m); - assert(m->is_new(), "freshly allocated monitor must be new"); - m->set_allocation_state(ObjectMonitor::Old); if (object->cas_set_mark(markWord::encode(m), mark) != mark) { - guarantee(!m->owner_is_DEFLATER_MARKER() || m->ref_count() >= 0, - "race between deflation and om_release() with m=" INTPTR_FORMAT - ", _owner=" INTPTR_FORMAT ", ref_count=%d", p2i(m), - p2i(m->_owner), m->ref_count()); m->set_header(markWord::zero()); m->set_object(NULL); m->Recycle(); omh_p->set_om_ptr(NULL); - // om_release() will reset the allocation state + // om_release() will reset the allocation state from New to Free. om_release(self, m, true); m = NULL; continue; @@ -2050,6 +1975,11 @@ // live-lock -- "Inflated" is an absorbing state. } + // Once the ObjectMonitor is configured and object is associated + // with the ObjectMonitor, it is safe to allow async deflation: + assert(m->is_new(), "freshly allocated monitor must be new"); + m->set_allocation_state(ObjectMonitor::Old); + // Hopefully the performance counters are allocated on distinct // cache lines to avoid false sharing on MP systems ... OM_PERFDATA_OP(Inflations, inc()); @@ -2153,8 +2083,9 @@ if (AsyncDeflateIdleMonitors) { // clear() expects the owner field to be NULL and we won't race // with the simple C2 ObjectMonitor enter optimization since - // we're at a safepoint. - mid->set_owner(NULL); + // we're at a safepoint. DEFLATER_MARKER is the only non-NULL + // value we should see here. + mid->try_set_owner_from(NULL, DEFLATER_MARKER); } mid->clear(); @@ -2219,7 +2150,7 @@ return false; } - if (Atomic::replace_if_null(DEFLATER_MARKER, &(mid->_owner))) { + if (mid->try_set_owner_from(DEFLATER_MARKER, NULL) == NULL) { // ObjectMonitor is not owned by another thread. Our setting // owner to DEFLATER_MARKER forces any contending thread through // the slow path. This is just the first part of the async @@ -2230,7 +2161,7 @@ // mid->is_busy() above or has already entered and waited on // it which makes it busy so no deflation. Restore owner to // NULL if it is still DEFLATER_MARKER. - Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER); + mid->try_set_owner_from(NULL, DEFLATER_MARKER); return false; } @@ -2322,7 +2253,7 @@ // The ref_count was no longer 0 so we lost the race since the // ObjectMonitor is now busy or the ObjectMonitor* is now is use. // Restore owner to NULL if it is still DEFLATER_MARKER: - Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER); + mid->try_set_owner_from(NULL, DEFLATER_MARKER); } // The owner field is no longer NULL so we lost the race since the @@ -2363,28 +2294,20 @@ // Deflation succeeded and already updated free_head_p and // free_tail_p as needed. Finish the move to the local free list // by unlinking mid from the global or per-thread in-use list. - if (Atomic::cmpxchg(next, list_p, mid) != mid) { - // We could not switch the list head to next. - ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL"); - if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, mid) != mid) { - // deflate_monitor_list() is called at a safepoint so the - // global or per-thread in-use list should not be modified - // in parallel so we: - fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: " - "list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: " - "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT, - p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use), - p2i(cur_mid_in_use->_next_om)); - } + if (cur_mid_in_use == NULL) { + // mid is the list head and it is marked. Switch the list head + // to next which unmarks the list head, but leaves mid marked: + OrderAccess::release_store(list_p, next); + } else { + // mid is marked. Switch cur_mid_in_use's next field to next + // which is safe because we have no parallel list deletions, + // but we leave mid marked: + OrderAccess::release_store(&cur_mid_in_use->_next_om, next); } // At this point mid is disconnected from the in-use list so // its marked next field no longer has any effects. deflated_count++; Atomic::dec(count_p); - chk_for_list_loop(OrderAccess::load_acquire(list_p), - OrderAccess::load_acquire(count_p)); - chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p), - OrderAccess::load_acquire(count_p)); // mid is current tail in the free_head_p list so NULL terminate it // (which also unmarks it): set_next(mid, NULL); @@ -2458,7 +2381,7 @@ // a cur_mid_in_use, then its next field is also marked at this point. if (next != NULL) { - // We mark the next -> next field so that an om_flush() + // We mark next's next field so that an om_flush() // thread that is behind us cannot pass us when we // unmark the current mid's next field. next_next = mark_next_loop(next); @@ -2471,31 +2394,21 @@ // Deflation succeeded and already updated free_head_p and // free_tail_p as needed. Finish the move to the local free list // by unlinking mid from the global or per-thread in-use list. - if (Atomic::cmpxchg(next, list_p, mid) != mid) { - // We could not switch the list head to next. - ObjectMonitor* marked_mid = mark_om_ptr(mid); + if (cur_mid_in_use == NULL) { + // mid is the list head and it is marked. Switch the list head + // to next which is also marked (if not NULL) and also leave + // mid marked: + OrderAccess::release_store(list_p, next); + } else { ObjectMonitor* marked_next = mark_om_ptr(next); - // Switch cur_mid_in_use's next field to marked next: - ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL"); - if (Atomic::cmpxchg(marked_next, &cur_mid_in_use->_next_om, - marked_mid) != marked_mid) { - // We could not switch cur_mid_in_use's next field. This - // should not be possible since it was marked so we: - fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: " - "&list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: " - "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT, - p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use), - p2i(cur_mid_in_use->_next_om)); - } + // mid and cur_mid_in_use are marked. Switch cur_mid_in_use's + // next field to marked_next and also leave mid marked: + OrderAccess::release_store(&cur_mid_in_use->_next_om, marked_next); } // At this point mid is disconnected from the in-use list so // its marked next field no longer has any effects. deflated_count++; Atomic::dec(count_p); - chk_for_list_loop(OrderAccess::load_acquire(list_p), - OrderAccess::load_acquire(count_p)); - chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p), - OrderAccess::load_acquire(count_p)); // mid is current tail in the free_head_p list so NULL terminate it // (which also unmarks it): set_next(mid, NULL); @@ -2620,6 +2533,67 @@ } } +class HandshakeForDeflation : public ThreadClosure { + public: + void do_thread(Thread* thread) { + log_trace(monitorinflation)("HandshakeForDeflation::do_thread: thread=" + INTPTR_FORMAT, p2i(thread)); + } +}; + +void ObjectSynchronizer::deflate_idle_monitors_using_JT() { + assert(AsyncDeflateIdleMonitors, "sanity check"); + + // Deflate any global idle monitors. + deflate_global_idle_monitors_using_JT(); + + int count = 0; + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { + if (jt->om_in_use_count > 0 && !jt->is_exiting()) { + // This JavaThread is using ObjectMonitors so deflate any that + // are idle unless this JavaThread is exiting; do not race with + // ObjectSynchronizer::om_flush(). + deflate_per_thread_idle_monitors_using_JT(jt); + count++; + } + } + if (count > 0) { + log_debug(monitorinflation)("did async deflation of idle monitors for %d thread(s).", count); + } + // The ServiceThread's async deflation request has been processed. + set_is_async_deflation_requested(false); + + if (HandshakeAfterDeflateIdleMonitors && g_om_wait_count > 0) { + // There are deflated ObjectMonitors waiting for a handshake + // (or a safepoint) for safety. + + // g_wait_list and g_om_wait_count are only updated by the calling + // thread so no need for load_acquire() or release_store(). + ObjectMonitor* list = g_wait_list; + ADIM_guarantee(list != NULL, "g_wait_list must not be NULL"); + int count = g_om_wait_count; + g_wait_list = NULL; + g_om_wait_count = 0; + + // Find the tail for prepend_list_to_common(). + int l_count = 0; + ObjectMonitor* tail = NULL; + for (ObjectMonitor* n = list; n != NULL; n = unmarked_next(n)) { + tail = n; + l_count++; + } + ADIM_guarantee(count == l_count, "count=%d != l_count=%d", count, l_count); + + // Will execute a safepoint if !ThreadLocalHandshakes: + HandshakeForDeflation hfd_tc; + Handshake::execute(&hfd_tc); + + prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count); + + log_info(monitorinflation)("moved %d idle monitors from global waiting list to global free list", count); + } +} + // Deflate global idle ObjectMonitors using a JavaThread. // void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() { @@ -2684,7 +2658,11 @@ assert(unmarked_next(free_tail_p) == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(unmarked_next(free_tail_p))); - prepend_list_to_g_free_list(free_head_p, free_tail_p, local_deflated_count); + if (HandshakeAfterDeflateIdleMonitors) { + prepend_list_to_g_wait_list(free_head_p, free_tail_p, local_deflated_count); + } else { + prepend_list_to_g_free_list(free_head_p, free_tail_p, local_deflated_count); + } OM_PERFDATA_OP(Deflations, inc(local_deflated_count)); } @@ -2754,10 +2732,11 @@ ObjectSynchronizer::audit_and_print_stats(false /* on_exit */); } else if (log_is_enabled(Info, monitorinflation)) { log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, " - "g_om_free_count=%d", + "g_om_free_count=%d, g_om_wait_count=%d", OrderAccess::load_acquire(&g_om_population), OrderAccess::load_acquire(&g_om_in_use_count), - OrderAccess::load_acquire(&g_om_free_count)); + OrderAccess::load_acquire(&g_om_free_count), + OrderAccess::load_acquire(&g_om_wait_count)); } ForceMonitorScavenge = 0; // Reset @@ -2924,11 +2903,14 @@ OrderAccess::load_acquire(&g_om_population), chk_om_population); } else { - ls->print_cr("ERROR: g_om_population=%d is not equal to " + // With lock free access to the monitor lists, it is possible for + // log_monitor_list_counts() to return a value that doesn't match + // g_om_population. So far a higher value has been seen in testing + // so something is being double counted by log_monitor_list_counts(). + ls->print_cr("WARNING: g_om_population=%d is not equal to " "chk_om_population=%d", OrderAccess::load_acquire(&g_om_population), chk_om_population); - error_cnt++; } // Check g_om_in_use_list and g_om_in_use_count: @@ -2937,6 +2919,11 @@ // Check g_free_list and g_om_free_count: chk_global_free_list_and_count(ls, &error_cnt); + if (HandshakeAfterDeflateIdleMonitors) { + // Check g_wait_list and g_om_wait_count: + chk_global_wait_list_and_count(ls, &error_cnt); + } + ls->print_cr("Checking per-thread lists:"); for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { @@ -3034,6 +3021,28 @@ } } +// Check the global wait list and count; log the results of the checks. +void ObjectSynchronizer::chk_global_wait_list_and_count(outputStream * out, + int *error_cnt_p) { + int chk_om_wait_count = 0; + for (ObjectMonitor* n = OrderAccess::load_acquire(&g_wait_list); n != NULL; n = unmarked_next(n)) { + // Rules for g_wait_list are the same as of g_free_list: + chk_free_entry(NULL /* jt */, n, out, error_cnt_p); + chk_om_wait_count++; + } + if (OrderAccess::load_acquire(&g_om_wait_count) == chk_om_wait_count) { + out->print_cr("g_om_wait_count=%d equals chk_om_wait_count=%d", + OrderAccess::load_acquire(&g_om_wait_count), + chk_om_wait_count); + } else { + out->print_cr("ERROR: g_om_wait_count=%d is not equal to " + "chk_om_wait_count=%d", + OrderAccess::load_acquire(&g_om_wait_count), + chk_om_wait_count); + *error_cnt_p = *error_cnt_p + 1; + } +} + // Check the global in-use list and count; log the results of the checks. void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out, int *error_cnt_p) { @@ -3047,10 +3056,12 @@ OrderAccess::load_acquire(&g_om_in_use_count), chk_om_in_use_count); } else { - out->print_cr("ERROR: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d", + // With lock free access to the monitor lists, it is possible for + // an exiting JavaThread to put its in-use ObjectMonitors on the + // global in-use list after chk_om_in_use_count is calculated above. + out->print_cr("WARNING: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d", OrderAccess::load_acquire(&g_om_in_use_count), chk_om_in_use_count); - *error_cnt_p = *error_cnt_p + 1; } } @@ -3215,15 +3226,19 @@ // the population count. int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) { int pop_count = 0; - out->print_cr("%18s %10s %10s %10s", - "Global Lists:", "InUse", "Free", "Total"); - out->print_cr("================== ========== ========== =========="); - out->print_cr("%18s %10d %10d %10d", "", + out->print_cr("%18s %10s %10s %10s %10s", + "Global Lists:", "InUse", "Free", "Wait", "Total"); + out->print_cr("================== ========== ========== ========== =========="); + out->print_cr("%18s %10d %10d %10d %10d", "", OrderAccess::load_acquire(&g_om_in_use_count), OrderAccess::load_acquire(&g_om_free_count), + OrderAccess::load_acquire(&g_om_wait_count), OrderAccess::load_acquire(&g_om_population)); pop_count += OrderAccess::load_acquire(&g_om_in_use_count) + OrderAccess::load_acquire(&g_om_free_count); + if (HandshakeAfterDeflateIdleMonitors) { + pop_count += OrderAccess::load_acquire(&g_om_wait_count); + } out->print_cr("%18s %10s %10s %10s", "Per-Thread Lists:", "InUse", "Free", "Provision"); --- old/src/hotspot/share/runtime/synchronizer.hpp 2019-10-17 17:31:49.000000000 -0400 +++ new/src/hotspot/share/runtime/synchronizer.hpp 2019-10-17 17:31:48.000000000 -0400 @@ -134,6 +134,7 @@ // Basically we deflate all monitors that are not busy. // An adaptive profile-based deflation policy could be used if needed static void deflate_idle_monitors(DeflateMonitorCounters* counters); + static void deflate_idle_monitors_using_JT(); static void deflate_global_idle_monitors_using_JT(); static void deflate_per_thread_idle_monitors_using_JT(JavaThread* target); static void deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target); @@ -176,6 +177,8 @@ outputStream * out, int *error_cnt_p); static void chk_global_free_list_and_count(outputStream * out, int *error_cnt_p); + static void chk_global_wait_list_and_count(outputStream * out, + int *error_cnt_p); static void chk_global_in_use_list_and_count(outputStream * out, int *error_cnt_p); static void chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,