--- old/src/hotspot/share/runtime/synchronizer.cpp 2019-08-28 15:03:30.774958403 -0400 +++ new/src/hotspot/share/runtime/synchronizer.cpp 2019-08-28 15:03:30.602958409 -0400 @@ -118,20 +118,21 @@ // global list of blocks of monitors PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL; +bool volatile ObjectSynchronizer::_is_async_deflation_requested = false; +bool volatile ObjectSynchronizer::_is_special_deflation_requested = false; +jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0; + // Global ObjectMonitor free list. Newly allocated and deflated // ObjectMonitors are prepended here. -ObjectMonitor* volatile ObjectSynchronizer::g_free_list = NULL; +static ObjectMonitor* volatile g_free_list = NULL; // Global ObjectMonitor in-use list. When a JavaThread is exiting, // ObjectMonitors on its per-thread in-use list are prepended here. -ObjectMonitor* volatile ObjectSynchronizer::g_om_in_use_list = NULL; -int ObjectSynchronizer::g_om_in_use_count = 0; // # on g_om_in_use_list -bool volatile ObjectSynchronizer::_is_async_deflation_requested = false; -bool volatile ObjectSynchronizer::_is_special_deflation_requested = false; -jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0; +static ObjectMonitor* volatile g_om_in_use_list = NULL; -static volatile intptr_t gListLock = 0; // protects global monitor lists -static volatile int g_om_free_count = 0; // # on g_free_list -static volatile int g_om_population = 0; // # Extant -- in circulation +static volatile intptr_t gListLock = 0; // protects global monitor lists +static volatile int g_om_free_count = 0; // # on g_free_list +static volatile int g_om_in_use_count = 0; // # on g_om_in_use_list +static volatile int g_om_population = 0; // # Extant -- in circulation #define CHAINMARKER (cast_to_oop(-1)) @@ -582,15 +583,15 @@ // performed by the CPU(s) or platform. struct SharedGlobals { - char _pad_prefix[DEFAULT_CACHE_LINE_SIZE]; + char _pad_prefix[OM_CACHE_LINE_SIZE]; // These are highly shared mostly-read variables. // To avoid false-sharing they need to be the sole occupants of a cache line. volatile int stw_random; volatile int stw_cycle; - DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int) * 2); + DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(volatile int) * 2); // Hot RW variable -- Sequester to avoid false-sharing volatile int hc_sequence; - DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int)); + DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile int)); }; static SharedGlobals GVars; @@ -998,12 +999,13 @@ ObjectMonitorHandle omh(mid); if (mid->object() == NULL || - (AsyncDeflateIdleMonitors && mid->_owner == DEFLATER_MARKER)) { + (AsyncDeflateIdleMonitors && mid->ref_count() < 0)) { // Only process with closure if the object is set. // For async deflation, race here if monitor is not owned! // The above ref_count bump (in ObjectMonitorHandle ctr) // will cause subsequent async deflation to skip it. - // However, previous or concurrent async deflation is a race. + // However, previous or concurrent async deflation is a race + // so skip this ObjectMonitor if it is being async deflated. continue; } closure->do_monitor(mid); @@ -1098,8 +1100,7 @@ // The oops_do() phase does not overlap with monitor deflation // so no need to update the ObjectMonitor's ref_count for this // ObjectMonitor* use. - ObjectMonitor* mid; - for (mid = list; mid != NULL; mid = mid->_next_om) { + for (ObjectMonitor* mid = list; mid != NULL; mid = mid->_next_om) { if (mid->object() != NULL) { f->do_oop((oop*)mid->object_addr()); } @@ -1202,7 +1203,7 @@ // by the ServiceThread. Per-thread async deflation is triggered // by the ServiceThread via om_request_deflation. debug_only(jt->check_for_valid_safepoint_state(false);) - ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(); + ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(jt); } } @@ -1289,10 +1290,10 @@ assert(_BLOCKSIZE > 1, "invariant"); size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE; PaddedObjectMonitor* temp; - size_t aligned_size = neededsize + (DEFAULT_CACHE_LINE_SIZE - 1); + size_t aligned_size = neededsize + (OM_CACHE_LINE_SIZE - 1); void* real_malloc_addr = (void*)NEW_C_HEAP_ARRAY(char, aligned_size, mtInternal); - temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, DEFAULT_CACHE_LINE_SIZE); + temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, OM_CACHE_LINE_SIZE); // NOTE: (almost) no way to recover if allocation failed. // We might be able to induce a STW safepoint and scavenge enough @@ -1414,36 +1415,15 @@ // run at the same time as om_flush() so we have to be careful. void ObjectSynchronizer::om_flush(Thread* self) { - ObjectMonitor* free_list = self->om_free_list; - ObjectMonitor* free_tail = NULL; - int free_count = 0; - if (free_list != NULL) { - ObjectMonitor* s; - // The thread is going away. Set 'free_tail' to the last per-thread free - // monitor which will be linked to g_free_list below under the gListLock. - stringStream ss; - for (s = free_list; s != NULL; s = s->_next_om) { - free_count++; - free_tail = s; - guarantee(s->object() == NULL, "invariant"); - guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss)); - } - guarantee(free_tail != NULL, "invariant"); - ADIM_guarantee(self->om_free_count == free_count, "free-count off"); - self->om_free_list = NULL; - self->om_free_count = 0; - } - + int in_use_count = 0; ObjectMonitor* in_use_list = self->om_in_use_list; ObjectMonitor* in_use_tail = NULL; - int in_use_count = 0; if (in_use_list != NULL) { // The thread is going away, however the ObjectMonitors on the // om_in_use_list may still be in-use by other threads. Link // them to in_use_tail, which will be linked into the global // in-use list g_om_in_use_list below, under the gListLock. - ObjectMonitor *cur_om; - for (cur_om = in_use_list; cur_om != NULL; cur_om = cur_om->_next_om) { + for (ObjectMonitor* cur_om = in_use_list; cur_om != NULL; cur_om = cur_om->_next_om) { in_use_tail = cur_om; in_use_count++; ADIM_guarantee(cur_om->is_active(), "invariant"); @@ -1454,6 +1434,25 @@ self->om_in_use_count = 0; } + int free_count = 0; + ObjectMonitor* free_list = self->om_free_list; + ObjectMonitor* free_tail = NULL; + if (free_list != NULL) { + // The thread is going away. Set 'free_tail' to the last per-thread free + // monitor which will be linked to g_free_list below under the gListLock. + stringStream ss; + for (ObjectMonitor* s = free_list; s != NULL; s = s->_next_om) { + free_count++; + free_tail = s; + guarantee(s->object() == NULL, "invariant"); + guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss)); + } + guarantee(free_tail != NULL, "invariant"); + ADIM_guarantee(self->om_free_count == free_count, "free-count off"); + self->om_free_list = NULL; + self->om_free_count = 0; + } + Thread::muxAcquire(&gListLock, "om_flush"); if (free_tail != NULL) { free_tail->_next_om = g_free_list; @@ -1855,7 +1854,8 @@ // to fix the linkages in its context. ObjectMonitor* prevtail = *free_tail_p; // Should have been cleaned up by the caller: - assert(prevtail->_next_om == NULL, "cleaned up deflated?"); + assert(prevtail->_next_om == NULL, "must be NULL: _next_om=" + INTPTR_FORMAT, p2i(prevtail->_next_om)); prevtail->_next_om = mid; } *free_tail_p = mid; @@ -2022,11 +2022,12 @@ // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and // Threads::parallel_java_threads_do() in thread.cpp. int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor** list_p, + int* count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p) { + ObjectMonitor* cur_mid_in_use = NULL; ObjectMonitor* mid; ObjectMonitor* next; - ObjectMonitor* cur_mid_in_use = NULL; int deflated_count = 0; for (mid = *list_p; mid != NULL;) { @@ -2044,6 +2045,7 @@ mid->_next_om = NULL; // This mid is current tail in the free_head_p list mid = next; deflated_count++; + *count_p = *count_p - 1; } else { cur_mid_in_use = mid; mid = mid->_next_om; @@ -2060,15 +2062,16 @@ // honor the safepoint. // int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor** list_p, + int* count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p, ObjectMonitor** saved_mid_in_use_p) { assert(AsyncDeflateIdleMonitors, "sanity check"); assert(Thread::current()->is_Java_thread(), "precondition"); + ObjectMonitor* cur_mid_in_use = NULL; ObjectMonitor* mid; ObjectMonitor* next; - ObjectMonitor* cur_mid_in_use = NULL; int deflated_count = 0; if (*saved_mid_in_use_p == NULL) { @@ -2100,6 +2103,7 @@ // and is the current tail in the free_head_p list. mid = next; deflated_count++; + *count_p = *count_p - 1; } else { // mid is considered in-use if it does not have an associated // Java object or mid is not old or deflation did not succeed. @@ -2169,21 +2173,22 @@ // For moribund threads, scan g_om_in_use_list int deflated_count = 0; - if (g_om_in_use_list) { + if (g_om_in_use_list != NULL) { + // Update n_in_circulation before g_om_in_use_count is updated by deflation. counters->n_in_circulation += g_om_in_use_count; - deflated_count = deflate_monitor_list((ObjectMonitor **)&g_om_in_use_list, &free_head_p, &free_tail_p); - g_om_in_use_count -= deflated_count; - counters->n_scavenged += deflated_count; + deflated_count = deflate_monitor_list((ObjectMonitor**)&g_om_in_use_list, (int*)&g_om_in_use_count, &free_head_p, &free_tail_p); counters->n_in_use += g_om_in_use_count; } if (free_head_p != NULL) { // Move the deflated ObjectMonitors back to the global free list. - guarantee(free_tail_p != NULL && counters->n_scavenged > 0, "invariant"); - assert(free_tail_p->_next_om == NULL, "invariant"); + guarantee(free_tail_p != NULL && deflated_count > 0, "invariant"); + assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om=" + INTPTR_FORMAT, p2i(free_tail_p->_next_om)); // constant-time list splice - prepend scavenged segment to g_free_list free_tail_p->_next_om = g_free_list; g_free_list = free_head_p; + counters->n_scavenged += deflated_count; } Thread::muxRelease(&gListLock); timer.stop(); @@ -2211,21 +2216,22 @@ deflate_common_idle_monitors_using_JT(true /* is_global */, self); } -// Deflate per-thread idle ObjectMonitors using a JavaThread. +// Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread. // -void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT() { +void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) { assert(AsyncDeflateIdleMonitors, "sanity check"); assert(Thread::current()->is_Java_thread(), "precondition"); - JavaThread* self = JavaThread::current(); - self->om_request_deflation = false; + target->om_request_deflation = false; - deflate_common_idle_monitors_using_JT(false /* !is_global */, self); + deflate_common_idle_monitors_using_JT(false /* !is_global */, target); } // Deflate global or per-thread idle ObjectMonitors using a JavaThread. // -void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* self) { +void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) { + JavaThread* self = JavaThread::current(); + int deflated_count = 0; ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors ObjectMonitor* free_tail_p = NULL; @@ -2240,24 +2246,23 @@ Thread::muxAcquire(&gListLock, "deflate_global_idle_monitors_using_JT(1)"); OM_PERFDATA_OP(MonExtant, set_value(g_om_in_use_count)); } else { - OM_PERFDATA_OP(MonExtant, inc(self->om_in_use_count)); + OM_PERFDATA_OP(MonExtant, inc(target->om_in_use_count)); } do { int local_deflated_count; if (is_global) { - local_deflated_count = deflate_monitor_list_using_JT((ObjectMonitor **)&g_om_in_use_list, &free_head_p, &free_tail_p, &saved_mid_in_use_p); - g_om_in_use_count -= local_deflated_count; + local_deflated_count = deflate_monitor_list_using_JT((ObjectMonitor**)&g_om_in_use_list, (int*)&g_om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p); } else { - local_deflated_count = deflate_monitor_list_using_JT(self->om_in_use_list_addr(), &free_head_p, &free_tail_p, &saved_mid_in_use_p); - self->om_in_use_count -= local_deflated_count; + local_deflated_count = deflate_monitor_list_using_JT(&target->om_in_use_list, &target->om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p); } deflated_count += local_deflated_count; if (free_head_p != NULL) { // Move the deflated ObjectMonitors to the global free list. guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count); - assert(free_tail_p->_next_om == NULL, "invariant"); + assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om=" + INTPTR_FORMAT, p2i(free_tail_p->_next_om)); if (!is_global) { Thread::muxAcquire(&gListLock, "deflate_per_thread_idle_monitors_using_JT(2)"); @@ -2266,7 +2271,6 @@ free_tail_p->_next_om = g_free_list; g_free_list = free_head_p; - g_om_free_count += local_deflated_count; OM_PERFDATA_OP(Deflations, inc(local_deflated_count)); if (!is_global) { Thread::muxRelease(&gListLock); @@ -2283,7 +2287,7 @@ if (is_global) { log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint."); } else { - log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(self)); + log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target)); } assert(SafepointSynchronize::is_synchronizing(), "sanity check"); ThreadBlockInVM blocker(self); @@ -2316,7 +2320,7 @@ if (is_global) { ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count); } else { - ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(self), timer.seconds(), deflated_count); + ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count); } } } @@ -2335,8 +2339,6 @@ // AsyncDeflateIdleMonitors does not use these counters unless // there is a special deflation request. - g_om_free_count += counters->n_scavenged; - OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged)); OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation)); } @@ -2385,25 +2387,25 @@ timer.start(); } - int deflated_count = deflate_monitor_list(thread->om_in_use_list_addr(), &free_head_p, &free_tail_p); - - Thread::muxAcquire(&gListLock, "deflate_thread_local_monitors"); - - // Adjust counters + // Update n_in_circulation before om_in_use_count is updated by deflation. counters->n_in_circulation += thread->om_in_use_count; - thread->om_in_use_count -= deflated_count; - counters->n_scavenged += deflated_count; + + int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p); counters->n_in_use += thread->om_in_use_count; - counters->per_thread_scavenged += deflated_count; + + Thread::muxAcquire(&gListLock, "deflate_thread_local_monitors"); if (free_head_p != NULL) { // Move the deflated ObjectMonitors back to the global free list. guarantee(free_tail_p != NULL && deflated_count > 0, "invariant"); - assert(free_tail_p->_next_om == NULL, "invariant"); + assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om=" + INTPTR_FORMAT, p2i(free_tail_p->_next_om)); // constant-time list splice - prepend scavenged segment to g_free_list free_tail_p->_next_om = g_free_list; g_free_list = free_head_p; + counters->n_scavenged += deflated_count; + counters->per_thread_scavenged += deflated_count; } timer.stop();