--- old/src/hotspot/cpu/sparc/globalDefinitions_sparc.hpp 2019-08-28 15:03:27.366958522 -0400 +++ new/src/hotspot/cpu/sparc/globalDefinitions_sparc.hpp 2019-08-28 15:03:27.178958528 -0400 @@ -40,12 +40,14 @@ #if defined(TIERED) // tiered, 64-bit, large machine #define DEFAULT_CACHE_LINE_SIZE 128 + #define OM_CACHE_LINE_SIZE 64 #elif defined(COMPILER1) // pure C1, 32-bit, small machine #define DEFAULT_CACHE_LINE_SIZE 16 #elif defined(COMPILER2) // pure C2, 64-bit, large machine #define DEFAULT_CACHE_LINE_SIZE 128 + #define OM_CACHE_LINE_SIZE 64 #endif #if defined(SOLARIS) --- old/src/hotspot/cpu/x86/globalDefinitions_x86.hpp 2019-08-28 15:03:28.058958498 -0400 +++ new/src/hotspot/cpu/x86/globalDefinitions_x86.hpp 2019-08-28 15:03:27.886958504 -0400 @@ -38,6 +38,7 @@ #ifdef _LP64 // tiered, 64-bit, large machine #define DEFAULT_CACHE_LINE_SIZE 128 + #define OM_CACHE_LINE_SIZE 64 #else // tiered, 32-bit, medium machine #define DEFAULT_CACHE_LINE_SIZE 64 @@ -50,6 +51,7 @@ #ifdef _LP64 // pure C2, 64-bit, large machine #define DEFAULT_CACHE_LINE_SIZE 128 + #define OM_CACHE_LINE_SIZE 64 #else // pure C2, 32-bit, medium machine #define DEFAULT_CACHE_LINE_SIZE 64 --- old/src/hotspot/share/prims/whitebox.cpp 2019-08-28 15:03:28.694958476 -0400 +++ new/src/hotspot/share/prims/whitebox.cpp 2019-08-28 15:03:28.514958482 -0400 @@ -70,6 +70,7 @@ #include "runtime/jniHandles.inline.hpp" #include "runtime/os.hpp" #include "runtime/sweeper.hpp" +#include "runtime/synchronizer.hpp" #include "runtime/thread.hpp" #include "runtime/threadSMR.hpp" #include "runtime/vm_version.hpp" @@ -462,6 +463,12 @@ WB_ENTRY(jboolean, WB_G1StartMarkCycle(JNIEnv* env, jobject o)) if (UseG1GC) { + if (AsyncDeflateIdleMonitors) { + // AsyncDeflateIdleMonitors needs to know when System.gc() or + // the equivalent is called so any special clean up can be done + // at a safepoint, e.g., TestHumongousClassLoader.java. + ObjectSynchronizer::set_is_special_deflation_requested(true); + } G1CollectedHeap* g1h = G1CollectedHeap::heap(); if (!g1h->concurrent_mark()->cm_thread()->during_cycle()) { g1h->collect(GCCause::_wb_conc_mark); @@ -1381,6 +1388,12 @@ WB_END WB_ENTRY(void, WB_FullGC(JNIEnv* env, jobject o)) + if (AsyncDeflateIdleMonitors) { + // AsyncDeflateIdleMonitors needs to know when System.gc() or + // the equivalent is called so any special clean up can be done + // at a safepoint, e.g., TestHumongousClassLoader.java. + ObjectSynchronizer::set_is_special_deflation_requested(true); + } Universe::heap()->soft_ref_policy()->set_should_clear_all_soft_refs(true); Universe::heap()->collect(GCCause::_wb_full_gc); #if INCLUDE_G1GC --- old/src/hotspot/share/runtime/objectMonitor.cpp 2019-08-28 15:03:29.370958452 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.cpp 2019-08-28 15:03:29.186958458 -0400 @@ -2111,6 +2111,13 @@ om_ptr->dec_ref_count(); return false; } + if (om_ptr->ref_count() <= 0) { + // Async deflation is in the process of bailing out, but has not + // yet restored the ref_count field so we return false to force + // a retry. We want a positive ref_count value for a true return. + om_ptr->dec_ref_count(); + return false; + } // The ObjectMonitor could have been deflated and reused for // another object before we bumped the ref_count so make sure // our object still refers to this ObjectMonitor. --- old/src/hotspot/share/runtime/objectMonitor.hpp 2019-08-28 15:03:30.106958426 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.hpp 2019-08-28 15:03:29.914958433 -0400 @@ -126,6 +126,12 @@ // intptr_t. There's no reason to use a 64-bit type for this field // in a 64-bit JVM. +#ifndef OM_CACHE_LINE_SIZE +// Use DEFAULT_CACHE_LINE_SIZE if not already specified for +// the current build platform. +#define OM_CACHE_LINE_SIZE DEFAULT_CACHE_LINE_SIZE +#endif + class ObjectMonitor { public: enum { @@ -147,24 +153,39 @@ // Enforced by the assert() in header_addr(). volatile markWord _header; // displaced object header word - mark void* volatile _object; // backward object pointer - strong root + typedef enum { + Free = 0, // Free must be 0 for monitor to be free after memset(..,0,..). + New, + Old + } AllocationState; + AllocationState _allocation_state; public: ObjectMonitor* _next_om; // Next ObjectMonitor* linkage private: // Separate _header and _owner on different cache lines since both can - // have busy multi-threaded access. _header and _object are set at - // initial inflation and _object doesn't change until deflation so - // _object is a good choice to share the cache line with _header. - // _next_om shares _header's cache line for pre-monitor list historical - // reasons. _next_om only changes if the next ObjectMonitor is deflated. - DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, - sizeof(volatile markWord) + sizeof(void* volatile) + - sizeof(ObjectMonitor *)); + // have busy multi-threaded access. _header, _object and _allocation_state + // are set at initial inflation. _object and _allocation_state don't + // change until deflation so _object and _allocation_state are good + // choices to share the cache line with _header. _next_om shares _header's + // cache line for pre-monitor list historical reasons. _next_om only + // changes if the next ObjectMonitor is deflated. + DEFINE_PAD_MINUS_SIZE(0, OM_CACHE_LINE_SIZE, sizeof(volatile markWord) + + sizeof(void* volatile) + sizeof(AllocationState) + + sizeof(ObjectMonitor*)); // Used by async deflation as a marker in the _owner field: #define DEFLATER_MARKER reinterpret_cast(-1) protected: // protected for JvmtiRawMonitor void* volatile _owner; // pointer to owning thread OR BasicLock private: volatile jlong _previous_owner_tid; // thread id of the previous owner of the monitor + // Separate _owner and _ref_count on different cache lines since both + // can have busy multi-threaded access. _previous_owner_tid is only + // changed by ObjectMonitor::exit() so it is a good choice to share the + // cache line with _owner. + DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(void* volatile) + + sizeof(volatile jlong)); + volatile jint _ref_count; // ref count for ObjectMonitor* and used by the async deflation + // protocol. See ObjectSynchronizer::deflate_monitor_using_JT(). protected: // protected for JvmtiRawMonitor volatile intptr_t _recursions; // recursion count, 0 for first entry ObjectWaiter* volatile _EntryList; // Threads blocked on entry or reentry. @@ -187,14 +208,6 @@ volatile jint _waiters; // number of waiting threads private: volatile int _WaitSetLock; // protects Wait Queue - simple spinlock - volatile jint _ref_count; // ref count for ObjectMonitor* and used by the async deflation - // protocol. See ObjectSynchronizer::deflate_monitor_using_JT(). - typedef enum { - Free = 0, // Free must be 0 for monitor to be free after memset(..,0,..). - New, - Old - } AllocationState; - AllocationState _allocation_state; public: static void Initialize(); --- old/src/hotspot/share/runtime/synchronizer.cpp 2019-08-28 15:03:30.774958403 -0400 +++ new/src/hotspot/share/runtime/synchronizer.cpp 2019-08-28 15:03:30.602958409 -0400 @@ -118,20 +118,21 @@ // global list of blocks of monitors PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL; +bool volatile ObjectSynchronizer::_is_async_deflation_requested = false; +bool volatile ObjectSynchronizer::_is_special_deflation_requested = false; +jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0; + // Global ObjectMonitor free list. Newly allocated and deflated // ObjectMonitors are prepended here. -ObjectMonitor* volatile ObjectSynchronizer::g_free_list = NULL; +static ObjectMonitor* volatile g_free_list = NULL; // Global ObjectMonitor in-use list. When a JavaThread is exiting, // ObjectMonitors on its per-thread in-use list are prepended here. -ObjectMonitor* volatile ObjectSynchronizer::g_om_in_use_list = NULL; -int ObjectSynchronizer::g_om_in_use_count = 0; // # on g_om_in_use_list -bool volatile ObjectSynchronizer::_is_async_deflation_requested = false; -bool volatile ObjectSynchronizer::_is_special_deflation_requested = false; -jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0; +static ObjectMonitor* volatile g_om_in_use_list = NULL; -static volatile intptr_t gListLock = 0; // protects global monitor lists -static volatile int g_om_free_count = 0; // # on g_free_list -static volatile int g_om_population = 0; // # Extant -- in circulation +static volatile intptr_t gListLock = 0; // protects global monitor lists +static volatile int g_om_free_count = 0; // # on g_free_list +static volatile int g_om_in_use_count = 0; // # on g_om_in_use_list +static volatile int g_om_population = 0; // # Extant -- in circulation #define CHAINMARKER (cast_to_oop(-1)) @@ -582,15 +583,15 @@ // performed by the CPU(s) or platform. struct SharedGlobals { - char _pad_prefix[DEFAULT_CACHE_LINE_SIZE]; + char _pad_prefix[OM_CACHE_LINE_SIZE]; // These are highly shared mostly-read variables. // To avoid false-sharing they need to be the sole occupants of a cache line. volatile int stw_random; volatile int stw_cycle; - DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int) * 2); + DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(volatile int) * 2); // Hot RW variable -- Sequester to avoid false-sharing volatile int hc_sequence; - DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int)); + DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile int)); }; static SharedGlobals GVars; @@ -998,12 +999,13 @@ ObjectMonitorHandle omh(mid); if (mid->object() == NULL || - (AsyncDeflateIdleMonitors && mid->_owner == DEFLATER_MARKER)) { + (AsyncDeflateIdleMonitors && mid->ref_count() < 0)) { // Only process with closure if the object is set. // For async deflation, race here if monitor is not owned! // The above ref_count bump (in ObjectMonitorHandle ctr) // will cause subsequent async deflation to skip it. - // However, previous or concurrent async deflation is a race. + // However, previous or concurrent async deflation is a race + // so skip this ObjectMonitor if it is being async deflated. continue; } closure->do_monitor(mid); @@ -1098,8 +1100,7 @@ // The oops_do() phase does not overlap with monitor deflation // so no need to update the ObjectMonitor's ref_count for this // ObjectMonitor* use. - ObjectMonitor* mid; - for (mid = list; mid != NULL; mid = mid->_next_om) { + for (ObjectMonitor* mid = list; mid != NULL; mid = mid->_next_om) { if (mid->object() != NULL) { f->do_oop((oop*)mid->object_addr()); } @@ -1202,7 +1203,7 @@ // by the ServiceThread. Per-thread async deflation is triggered // by the ServiceThread via om_request_deflation. debug_only(jt->check_for_valid_safepoint_state(false);) - ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(); + ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(jt); } } @@ -1289,10 +1290,10 @@ assert(_BLOCKSIZE > 1, "invariant"); size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE; PaddedObjectMonitor* temp; - size_t aligned_size = neededsize + (DEFAULT_CACHE_LINE_SIZE - 1); + size_t aligned_size = neededsize + (OM_CACHE_LINE_SIZE - 1); void* real_malloc_addr = (void*)NEW_C_HEAP_ARRAY(char, aligned_size, mtInternal); - temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, DEFAULT_CACHE_LINE_SIZE); + temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, OM_CACHE_LINE_SIZE); // NOTE: (almost) no way to recover if allocation failed. // We might be able to induce a STW safepoint and scavenge enough @@ -1414,36 +1415,15 @@ // run at the same time as om_flush() so we have to be careful. void ObjectSynchronizer::om_flush(Thread* self) { - ObjectMonitor* free_list = self->om_free_list; - ObjectMonitor* free_tail = NULL; - int free_count = 0; - if (free_list != NULL) { - ObjectMonitor* s; - // The thread is going away. Set 'free_tail' to the last per-thread free - // monitor which will be linked to g_free_list below under the gListLock. - stringStream ss; - for (s = free_list; s != NULL; s = s->_next_om) { - free_count++; - free_tail = s; - guarantee(s->object() == NULL, "invariant"); - guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss)); - } - guarantee(free_tail != NULL, "invariant"); - ADIM_guarantee(self->om_free_count == free_count, "free-count off"); - self->om_free_list = NULL; - self->om_free_count = 0; - } - + int in_use_count = 0; ObjectMonitor* in_use_list = self->om_in_use_list; ObjectMonitor* in_use_tail = NULL; - int in_use_count = 0; if (in_use_list != NULL) { // The thread is going away, however the ObjectMonitors on the // om_in_use_list may still be in-use by other threads. Link // them to in_use_tail, which will be linked into the global // in-use list g_om_in_use_list below, under the gListLock. - ObjectMonitor *cur_om; - for (cur_om = in_use_list; cur_om != NULL; cur_om = cur_om->_next_om) { + for (ObjectMonitor* cur_om = in_use_list; cur_om != NULL; cur_om = cur_om->_next_om) { in_use_tail = cur_om; in_use_count++; ADIM_guarantee(cur_om->is_active(), "invariant"); @@ -1454,6 +1434,25 @@ self->om_in_use_count = 0; } + int free_count = 0; + ObjectMonitor* free_list = self->om_free_list; + ObjectMonitor* free_tail = NULL; + if (free_list != NULL) { + // The thread is going away. Set 'free_tail' to the last per-thread free + // monitor which will be linked to g_free_list below under the gListLock. + stringStream ss; + for (ObjectMonitor* s = free_list; s != NULL; s = s->_next_om) { + free_count++; + free_tail = s; + guarantee(s->object() == NULL, "invariant"); + guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss)); + } + guarantee(free_tail != NULL, "invariant"); + ADIM_guarantee(self->om_free_count == free_count, "free-count off"); + self->om_free_list = NULL; + self->om_free_count = 0; + } + Thread::muxAcquire(&gListLock, "om_flush"); if (free_tail != NULL) { free_tail->_next_om = g_free_list; @@ -1855,7 +1854,8 @@ // to fix the linkages in its context. ObjectMonitor* prevtail = *free_tail_p; // Should have been cleaned up by the caller: - assert(prevtail->_next_om == NULL, "cleaned up deflated?"); + assert(prevtail->_next_om == NULL, "must be NULL: _next_om=" + INTPTR_FORMAT, p2i(prevtail->_next_om)); prevtail->_next_om = mid; } *free_tail_p = mid; @@ -2022,11 +2022,12 @@ // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and // Threads::parallel_java_threads_do() in thread.cpp. int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor** list_p, + int* count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p) { + ObjectMonitor* cur_mid_in_use = NULL; ObjectMonitor* mid; ObjectMonitor* next; - ObjectMonitor* cur_mid_in_use = NULL; int deflated_count = 0; for (mid = *list_p; mid != NULL;) { @@ -2044,6 +2045,7 @@ mid->_next_om = NULL; // This mid is current tail in the free_head_p list mid = next; deflated_count++; + *count_p = *count_p - 1; } else { cur_mid_in_use = mid; mid = mid->_next_om; @@ -2060,15 +2062,16 @@ // honor the safepoint. // int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor** list_p, + int* count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p, ObjectMonitor** saved_mid_in_use_p) { assert(AsyncDeflateIdleMonitors, "sanity check"); assert(Thread::current()->is_Java_thread(), "precondition"); + ObjectMonitor* cur_mid_in_use = NULL; ObjectMonitor* mid; ObjectMonitor* next; - ObjectMonitor* cur_mid_in_use = NULL; int deflated_count = 0; if (*saved_mid_in_use_p == NULL) { @@ -2100,6 +2103,7 @@ // and is the current tail in the free_head_p list. mid = next; deflated_count++; + *count_p = *count_p - 1; } else { // mid is considered in-use if it does not have an associated // Java object or mid is not old or deflation did not succeed. @@ -2169,21 +2173,22 @@ // For moribund threads, scan g_om_in_use_list int deflated_count = 0; - if (g_om_in_use_list) { + if (g_om_in_use_list != NULL) { + // Update n_in_circulation before g_om_in_use_count is updated by deflation. counters->n_in_circulation += g_om_in_use_count; - deflated_count = deflate_monitor_list((ObjectMonitor **)&g_om_in_use_list, &free_head_p, &free_tail_p); - g_om_in_use_count -= deflated_count; - counters->n_scavenged += deflated_count; + deflated_count = deflate_monitor_list((ObjectMonitor**)&g_om_in_use_list, (int*)&g_om_in_use_count, &free_head_p, &free_tail_p); counters->n_in_use += g_om_in_use_count; } if (free_head_p != NULL) { // Move the deflated ObjectMonitors back to the global free list. - guarantee(free_tail_p != NULL && counters->n_scavenged > 0, "invariant"); - assert(free_tail_p->_next_om == NULL, "invariant"); + guarantee(free_tail_p != NULL && deflated_count > 0, "invariant"); + assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om=" + INTPTR_FORMAT, p2i(free_tail_p->_next_om)); // constant-time list splice - prepend scavenged segment to g_free_list free_tail_p->_next_om = g_free_list; g_free_list = free_head_p; + counters->n_scavenged += deflated_count; } Thread::muxRelease(&gListLock); timer.stop(); @@ -2211,21 +2216,22 @@ deflate_common_idle_monitors_using_JT(true /* is_global */, self); } -// Deflate per-thread idle ObjectMonitors using a JavaThread. +// Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread. // -void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT() { +void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) { assert(AsyncDeflateIdleMonitors, "sanity check"); assert(Thread::current()->is_Java_thread(), "precondition"); - JavaThread* self = JavaThread::current(); - self->om_request_deflation = false; + target->om_request_deflation = false; - deflate_common_idle_monitors_using_JT(false /* !is_global */, self); + deflate_common_idle_monitors_using_JT(false /* !is_global */, target); } // Deflate global or per-thread idle ObjectMonitors using a JavaThread. // -void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* self) { +void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) { + JavaThread* self = JavaThread::current(); + int deflated_count = 0; ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors ObjectMonitor* free_tail_p = NULL; @@ -2240,24 +2246,23 @@ Thread::muxAcquire(&gListLock, "deflate_global_idle_monitors_using_JT(1)"); OM_PERFDATA_OP(MonExtant, set_value(g_om_in_use_count)); } else { - OM_PERFDATA_OP(MonExtant, inc(self->om_in_use_count)); + OM_PERFDATA_OP(MonExtant, inc(target->om_in_use_count)); } do { int local_deflated_count; if (is_global) { - local_deflated_count = deflate_monitor_list_using_JT((ObjectMonitor **)&g_om_in_use_list, &free_head_p, &free_tail_p, &saved_mid_in_use_p); - g_om_in_use_count -= local_deflated_count; + local_deflated_count = deflate_monitor_list_using_JT((ObjectMonitor**)&g_om_in_use_list, (int*)&g_om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p); } else { - local_deflated_count = deflate_monitor_list_using_JT(self->om_in_use_list_addr(), &free_head_p, &free_tail_p, &saved_mid_in_use_p); - self->om_in_use_count -= local_deflated_count; + local_deflated_count = deflate_monitor_list_using_JT(&target->om_in_use_list, &target->om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p); } deflated_count += local_deflated_count; if (free_head_p != NULL) { // Move the deflated ObjectMonitors to the global free list. guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count); - assert(free_tail_p->_next_om == NULL, "invariant"); + assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om=" + INTPTR_FORMAT, p2i(free_tail_p->_next_om)); if (!is_global) { Thread::muxAcquire(&gListLock, "deflate_per_thread_idle_monitors_using_JT(2)"); @@ -2266,7 +2271,6 @@ free_tail_p->_next_om = g_free_list; g_free_list = free_head_p; - g_om_free_count += local_deflated_count; OM_PERFDATA_OP(Deflations, inc(local_deflated_count)); if (!is_global) { Thread::muxRelease(&gListLock); @@ -2283,7 +2287,7 @@ if (is_global) { log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint."); } else { - log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(self)); + log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target)); } assert(SafepointSynchronize::is_synchronizing(), "sanity check"); ThreadBlockInVM blocker(self); @@ -2316,7 +2320,7 @@ if (is_global) { ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count); } else { - ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(self), timer.seconds(), deflated_count); + ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count); } } } @@ -2335,8 +2339,6 @@ // AsyncDeflateIdleMonitors does not use these counters unless // there is a special deflation request. - g_om_free_count += counters->n_scavenged; - OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged)); OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation)); } @@ -2385,25 +2387,25 @@ timer.start(); } - int deflated_count = deflate_monitor_list(thread->om_in_use_list_addr(), &free_head_p, &free_tail_p); - - Thread::muxAcquire(&gListLock, "deflate_thread_local_monitors"); - - // Adjust counters + // Update n_in_circulation before om_in_use_count is updated by deflation. counters->n_in_circulation += thread->om_in_use_count; - thread->om_in_use_count -= deflated_count; - counters->n_scavenged += deflated_count; + + int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p); counters->n_in_use += thread->om_in_use_count; - counters->per_thread_scavenged += deflated_count; + + Thread::muxAcquire(&gListLock, "deflate_thread_local_monitors"); if (free_head_p != NULL) { // Move the deflated ObjectMonitors back to the global free list. guarantee(free_tail_p != NULL && deflated_count > 0, "invariant"); - assert(free_tail_p->_next_om == NULL, "invariant"); + assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om=" + INTPTR_FORMAT, p2i(free_tail_p->_next_om)); // constant-time list splice - prepend scavenged segment to g_free_list free_tail_p->_next_om = g_free_list; g_free_list = free_head_p; + counters->n_scavenged += deflated_count; + counters->per_thread_scavenged += deflated_count; } timer.stop(); --- old/src/hotspot/share/runtime/synchronizer.hpp 2019-08-28 15:03:31.470958379 -0400 +++ new/src/hotspot/share/runtime/synchronizer.hpp 2019-08-28 15:03:31.282958385 -0400 @@ -35,7 +35,13 @@ class ObjectMonitorHandle; class ThreadsList; -typedef PaddedEnd PaddedObjectMonitor; +#ifndef OM_CACHE_LINE_SIZE +// Use DEFAULT_CACHE_LINE_SIZE if not already specified for +// the current build platform. +#define OM_CACHE_LINE_SIZE DEFAULT_CACHE_LINE_SIZE +#endif + +typedef PaddedEnd PaddedObjectMonitor; struct DeflateMonitorCounters { int n_in_use; // currently associated with objects @@ -142,19 +148,21 @@ // An adaptive profile-based deflation policy could be used if needed static void deflate_idle_monitors(DeflateMonitorCounters* counters); static void deflate_global_idle_monitors_using_JT(); - static void deflate_per_thread_idle_monitors_using_JT(); - static void deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* self); + static void deflate_per_thread_idle_monitors_using_JT(JavaThread* target); + static void deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target); static void deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters); static void prepare_deflate_idle_monitors(DeflateMonitorCounters* counters); static void finish_deflate_idle_monitors(DeflateMonitorCounters* counters); // For a given monitor list: global or per-thread, deflate idle monitors static int deflate_monitor_list(ObjectMonitor** list_p, + int* count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p); // For a given in-use monitor list: global or per-thread, deflate idle // monitors using a JavaThread. static int deflate_monitor_list_using_JT(ObjectMonitor** list_p, + int* count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p, ObjectMonitor** saved_mid_in_use_p); @@ -203,13 +211,6 @@ enum { _BLOCKSIZE = 128 }; // global list of blocks of monitors static PaddedObjectMonitor* volatile g_block_list; - // global monitor free list - static ObjectMonitor* volatile g_free_list; - // global monitor in-use list, for moribund threads, - // monitors they inflated need to be scanned for deflation - static ObjectMonitor* volatile g_om_in_use_list; - // count of entries in g_om_in_use_list - static int g_om_in_use_count; static volatile bool _is_async_deflation_requested; static volatile bool _is_special_deflation_requested; static jlong _last_async_deflation_time_ns; --- old/src/hotspot/share/runtime/thread.hpp 2019-08-28 15:03:32.134958356 -0400 +++ new/src/hotspot/share/runtime/thread.hpp 2019-08-28 15:03:31.942958362 -0400 @@ -523,7 +523,6 @@ os::set_native_thread_name(name); } - ObjectMonitor** om_in_use_list_addr() { return (ObjectMonitor **)&om_in_use_list; } Monitor* SR_lock() const { return _SR_lock; } bool has_async_exception() const { return (_suspend_flags & _has_async_exception) != 0; } --- old/src/hotspot/share/runtime/vmStructs.cpp 2019-08-28 15:03:32.866958330 -0400 +++ new/src/hotspot/share/runtime/vmStructs.cpp 2019-08-28 15:03:32.674958337 -0400 @@ -910,11 +910,11 @@ volatile_nonstatic_field(ObjectMonitor, _header, markWord) \ unchecked_nonstatic_field(ObjectMonitor, _object, sizeof(void *)) /* NOTE: no type */ \ unchecked_nonstatic_field(ObjectMonitor, _owner, sizeof(void *)) /* NOTE: no type */ \ + nonstatic_field(ObjectMonitor, _next_om, ObjectMonitor*) \ + volatile_nonstatic_field(BasicLock, _displaced_header, markWord) \ volatile_nonstatic_field(ObjectMonitor, _contentions, jint) \ volatile_nonstatic_field(ObjectMonitor, _waiters, jint) \ volatile_nonstatic_field(ObjectMonitor, _recursions, intptr_t) \ - nonstatic_field(ObjectMonitor, _next_om, ObjectMonitor*) \ - volatile_nonstatic_field(BasicLock, _displaced_header, markWord) \ nonstatic_field(BasicObjectLock, _lock, BasicLock) \ nonstatic_field(BasicObjectLock, _obj, oop) \ static_ptr_volatile_field(ObjectSynchronizer, g_block_list, PaddedObjectMonitor*) \