--- old/src/hotspot/share/runtime/globals.hpp 2019-08-28 15:03:59.838957391 -0400 +++ new/src/hotspot/share/runtime/globals.hpp 2019-08-28 15:03:59.634957398 -0400 @@ -726,6 +726,9 @@ "MonitorUsedDeflationThreshold is exceeded (0 is off).") \ range(0, max_jint) \ \ + diagnostic(bool, CheckMonitorLists, false, \ + "Sanity check the lock free ObjectMonitor lists.") \ + \ experimental(intx, MonitorUsedDeflationThreshold, 90, \ "Percentage of used monitors before triggering deflation (0 is " \ "off). The check is performed on GuaranteedSafepointInterval " \ --- old/src/hotspot/share/runtime/objectMonitor.hpp 2019-08-28 15:04:00.598957364 -0400 +++ new/src/hotspot/share/runtime/objectMonitor.hpp 2019-08-28 15:04:00.410957371 -0400 @@ -159,19 +159,13 @@ Old } AllocationState; AllocationState _allocation_state; - public: - ObjectMonitor* _next_om; // Next ObjectMonitor* linkage - private: // Separate _header and _owner on different cache lines since both can // have busy multi-threaded access. _header, _object and _allocation_state // are set at initial inflation. _object and _allocation_state don't // change until deflation so _object and _allocation_state are good - // choices to share the cache line with _header. _next_om shares _header's - // cache line for pre-monitor list historical reasons. _next_om only - // changes if the next ObjectMonitor is deflated. + // choices to share the cache line with _header. DEFINE_PAD_MINUS_SIZE(0, OM_CACHE_LINE_SIZE, sizeof(volatile markWord) + - sizeof(void* volatile) + sizeof(AllocationState) + - sizeof(ObjectMonitor*)); + sizeof(void* volatile) + sizeof(AllocationState)); // Used by async deflation as a marker in the _owner field: #define DEFLATER_MARKER reinterpret_cast(-1) protected: // protected for JvmtiRawMonitor @@ -186,6 +180,12 @@ sizeof(volatile jlong)); volatile jint _ref_count; // ref count for ObjectMonitor* and used by the async deflation // protocol. See ObjectSynchronizer::deflate_monitor_using_JT(). + private: + // Separate _ref_count and _next_om on different cache lines since + // both can have busy multi-threaded access. + DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile jint)); + public: // for static synchronizer.cpp access: + ObjectMonitor* volatile _next_om; // Next ObjectMonitor* linkage protected: // protected for JvmtiRawMonitor volatile intptr_t _recursions; // recursion count, 0 for first entry ObjectWaiter* volatile _EntryList; // Threads blocked on entry or reentry. @@ -210,6 +210,7 @@ volatile int _WaitSetLock; // protects Wait Queue - simple spinlock public: + volatile int visit_marker; static void Initialize(); // Only perform a PerfData operation if the PerfData object has been --- old/src/hotspot/share/runtime/serviceThread.cpp 2019-08-28 15:04:01.314957339 -0400 +++ new/src/hotspot/share/runtime/serviceThread.cpp 2019-08-28 15:04:01.130957346 -0400 @@ -190,25 +190,21 @@ // Deflate any global idle monitors. ObjectSynchronizer::deflate_global_idle_monitors_using_JT(); - // deflate_per_thread_idle_monitors_using_JT() is called by - // each JavaThread from ObjectSynchronizer::omAlloc() as needed. int count = 0; for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { - if (jt->om_in_use_count > 0) { - // This JavaThread is using monitors so request deflation. - jt->om_request_deflation = true; + if (jt->om_in_use_count > 0 && !jt->is_exiting()) { + // This JavaThread is using ObjectMonitors so deflate any that + // are idle unless this JavaThread is exiting; do not race with + // ObjectSynchronizer::om_flush(). + ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(jt); count++; } } if (count > 0) { - log_debug(monitorinflation)("requesting async deflation of idle monitors for %d thread(s).", count); + log_debug(monitorinflation)("did async deflation of idle monitors for %d thread(s).", count); } // The ServiceThread's async deflation request has been processed. ObjectSynchronizer::set_is_async_deflation_requested(false); - - // The global in-use list was handled above, but the request won't - // be complete until the JavaThreads have handled their in-use - // lists. This is the nature of an async deflation request. } } } --- old/src/hotspot/share/runtime/synchronizer.cpp 2019-08-28 15:04:02.034957314 -0400 +++ new/src/hotspot/share/runtime/synchronizer.cpp 2019-08-28 15:04:01.850957321 -0400 @@ -129,7 +129,6 @@ // ObjectMonitors on its per-thread in-use list are prepended here. static ObjectMonitor* volatile g_om_in_use_list = NULL; -static volatile intptr_t gListLock = 0; // protects global monitor lists static volatile int g_om_free_count = 0; // # on g_free_list static volatile int g_om_in_use_count = 0; // # on g_om_in_use_list static volatile int g_om_population = 0; // # Extant -- in circulation @@ -137,6 +136,327 @@ #define CHAINMARKER (cast_to_oop(-1)) +// =====================> List Management functions + +// Return true if the ObjectMonitor's next field is marked. +// Otherwise returns false. +static bool is_next_marked(ObjectMonitor* om) { + return ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & 0x1) != 0; +} + +// Mark an ObjectMonitor* and return it. Note: the om parameter +// may or may not have been marked originally. +static ObjectMonitor* mark_om_ptr(ObjectMonitor* om) { + return (ObjectMonitor*)((intptr_t)om | 0x1); +} + +// Mark the next field in an ObjectMonitor. If marking was successful, +// then the unmarked next field is returned via parameter and true is +// returned. Otherwise false is returned. +static bool mark_next(ObjectMonitor* om, ObjectMonitor** next_p) { + // Get current next field without any marking value. + ObjectMonitor* next = (ObjectMonitor*) + ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1); + if (Atomic::cmpxchg(mark_om_ptr(next), &om->_next_om, next) != next) { + return false; // Could not mark the next field or it was already marked. + } + *next_p = next; + return true; +} + +// Loop until we mark the next field in an ObjectMonitor. The unmarked +// next field is returned. +static ObjectMonitor* mark_next_loop(ObjectMonitor* om) { + ObjectMonitor* next; + while (true) { + if (mark_next(om, &next)) { + // Marked om's next field so return the unmarked value. + return next; + } + } +} + +// Set the next field in an ObjectMonitor to the specified value. +// The caller of set_next() must be the same thread that marked the +// ObjectMonitor. +static void set_next(ObjectMonitor* om, ObjectMonitor* value) { + OrderAccess::release_store(&om->_next_om, value); +} + +// Mark the next field in the list head ObjectMonitor. If marking was +// successful, then the mid and the unmarked next field are returned +// via parameter and true is returned. Otherwise false is returned. +static bool mark_list_head(ObjectMonitor* volatile * list_p, + ObjectMonitor** mid_p, ObjectMonitor** next_p) { + while (true) { + ObjectMonitor* mid = OrderAccess::load_acquire(list_p); + if (mid == NULL) { + return false; // The list is empty so nothing to mark. + } + if (mark_next(mid, next_p)) { + if (OrderAccess::load_acquire(list_p) != mid) { + // The list head changed so we have to retry. + set_next(mid, *next_p); // unmark mid + continue; + } + // We marked next field to guard against races. + *mid_p = mid; + return true; + } + } +} + +// Return the unmarked next field in an ObjectMonitor. Note: the next +// field may or may not have been marked originally. +static ObjectMonitor* unmarked_next(ObjectMonitor* om) { + return (ObjectMonitor*)((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1); +} + +#if 0 +// XXX - this is unused +// Unmark the next field in an ObjectMonitor. Requires that the next +// field be marked. +static void unmark_next(ObjectMonitor* om) { + ADIM_guarantee(is_next_marked(om), "next field must be marked: next=" INTPTR_FORMAT, p2i(om->_next_om)); + + ObjectMonitor* next = unmarked_next(om); + set_next(om, next); +} +#endif + +volatile int visit_counter = 42; +static void chk_for_list_loop(ObjectMonitor* list, int count) { + if (!CheckMonitorLists) { + return; + } + int l_visit_counter = Atomic::add(1, &visit_counter); + int l_count = 0; + ObjectMonitor* prev = NULL; + for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) { + if (mid->visit_marker == l_visit_counter) { + log_error(monitorinflation)("ERROR: prev=" INTPTR_FORMAT ", l_count=%d" + " refers to an ObjectMonitor that has" + " already been visited: mid=" INTPTR_FORMAT, + p2i(prev), l_count, p2i(mid)); + fatal("list=" INTPTR_FORMAT " of %d items has a loop.", p2i(list), count); + } + mid->visit_marker = l_visit_counter; + prev = mid; + if (++l_count > count + 1024 * 1024) { + fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d", + p2i(list), count, l_count); + } + } +} + +static void chk_om_not_on_list(ObjectMonitor* om, ObjectMonitor* list, int count) { + if (!CheckMonitorLists) { + return; + } + guarantee(list != om, "ERROR: om=" INTPTR_FORMAT " must not be head of the " + "list=" INTPTR_FORMAT ", count=%d", p2i(om), p2i(list), count); + int l_count = 0; + for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) { + if (unmarked_next(mid) == om) { + log_error(monitorinflation)("ERROR: mid=" INTPTR_FORMAT ", l_count=%d" + " next_om refers to om=" INTPTR_FORMAT, + p2i(mid), l_count, p2i(om)); + fatal("list=" INTPTR_FORMAT " of %d items has bad next_om value.", + p2i(list), count); + } + if (++l_count > count + 1024 * 1024) { + fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d", + p2i(list), count, l_count); + } + } +} + +static void chk_om_elems_not_on_list(ObjectMonitor* elems, int elems_count, + ObjectMonitor* list, int list_count) { + if (!CheckMonitorLists) { + return; + } + chk_for_list_loop(elems, elems_count); + for (ObjectMonitor* mid = elems; mid != NULL; mid = unmarked_next(mid)) { + chk_om_not_on_list(mid, list, list_count); + } +} + +// Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is +// the last ObjectMonitor in the list and there are 'count' on the list. +// Also updates the specified *count_p. +static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail, + int count, ObjectMonitor* volatile* list_p, + volatile int* count_p) { + chk_for_list_loop(OrderAccess::load_acquire(list_p), + OrderAccess::load_acquire(count_p)); + chk_om_elems_not_on_list(list, count, OrderAccess::load_acquire(list_p), + OrderAccess::load_acquire(count_p)); + while (true) { + ObjectMonitor* cur = OrderAccess::load_acquire(list_p); + // Prepend list to *list_p. + ObjectMonitor* next = NULL; + if (!mark_next(tail, &next)) { + continue; // failed to mark next field so try it all again + } + set_next(tail, cur); // tail now points to cur (and unmarks tail) + if (cur == NULL) { + // No potential race with takers or other prependers since + // *list_p is empty. + if (Atomic::cmpxchg(list, list_p, cur) == cur) { + // Successfully switched *list_p to the list value. + Atomic::add(count, count_p); + break; + } + // Implied else: try it all again + } else { + // Try to mark next field to guard against races: + if (!mark_next(cur, &next)) { + continue; // failed to mark next field so try it all again + } + // We marked the next field so try to switch *list_p to the list value. + if (Atomic::cmpxchg(list, list_p, cur) != cur) { + // The list head has changed so unmark the next field and try again: + set_next(cur, next); + continue; + } + Atomic::add(count, count_p); + set_next(cur, next); // unmark next field + break; + } + } +} + +// Prepend a newly allocated block of ObjectMonitors to g_block_list and +// g_free_list. Also updates g_om_population and g_om_free_count. +void ObjectSynchronizer::prepend_block_to_lists(PaddedObjectMonitor* new_blk) { + // First we handle g_block_list: + while (true) { + PaddedObjectMonitor* cur = OrderAccess::load_acquire(&g_block_list); + // Prepend new_blk to g_block_list. The first ObjectMonitor in + // a block is reserved for use as linkage to the next block. + OrderAccess::release_store(&new_blk[0]._next_om, cur); + if (Atomic::cmpxchg(new_blk, &g_block_list, cur) == cur) { + // Successfully switched g_block_list to the new_blk value. + Atomic::add(_BLOCKSIZE - 1, &g_om_population); + break; + } + // Implied else: try it all again + } + + // Second we handle g_free_list: + prepend_list_to_common(new_blk + 1, &new_blk[_BLOCKSIZE - 1], _BLOCKSIZE - 1, + &g_free_list, &g_om_free_count); +} + +// Prepend a list of ObjectMonitors to g_free_list. 'tail' is the last +// ObjectMonitor in the list and there are 'count' on the list. Also +// updates g_om_free_count. +static void prepend_list_to_g_free_list(ObjectMonitor* list, + ObjectMonitor* tail, int count) { + prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count); +} + +// Prepend a list of ObjectMonitors to g_om_in_use_list. 'tail' is the last +// ObjectMonitor in the list and there are 'count' on the list. Also +// updates g_om_in_use_list. +static void prepend_list_to_g_om_in_use_list(ObjectMonitor* list, + ObjectMonitor* tail, int count) { + prepend_list_to_common(list, tail, count, &g_om_in_use_list, &g_om_in_use_count); +} + +// Prepend an ObjectMonitor to the specified list. Also updates +// the specified counter. +static void prepend_to_common(ObjectMonitor* m, ObjectMonitor* volatile * list_p, + int volatile * count_p) { + chk_for_list_loop(OrderAccess::load_acquire(list_p), + OrderAccess::load_acquire(count_p)); + chk_om_not_on_list(m, OrderAccess::load_acquire(list_p), + OrderAccess::load_acquire(count_p)); + + while (true) { + ObjectMonitor* cur = OrderAccess::load_acquire(list_p); + // Prepend ObjectMonitor to *list_p. + ObjectMonitor* next = NULL; + if (!mark_next(m, &next)) { + continue; // failed to mark next field so try it all again + } + set_next(m, cur); // m now points to cur (and unmarks m) + if (cur == NULL) { + // No potential race with other prependers since *list_p is empty. + if (Atomic::cmpxchg(m, list_p, cur) == cur) { + // Successfully switched *list_p to 'm'. + Atomic::inc(count_p); + break; + } + // Implied else: try it all again + } else { + // Try to mark next field to guard against races: + if (!mark_next(cur, &next)) { + continue; // failed to mark next field so try it all again + } + // We marked the next field so try to switch *list_p to 'm'. + if (Atomic::cmpxchg(m, list_p, cur) != cur) { + // The list head has changed so unmark the next field and try again: + set_next(cur, next); + continue; + } + Atomic::inc(count_p); + set_next(cur, next); // unmark next field + break; + } + } +} + +// Prepend an ObjectMonitor to a per-thread om_free_list. +// Also updates the per-thread om_free_count. +static void prepend_to_om_free_list(Thread* self, ObjectMonitor* m) { + prepend_to_common(m, &self->om_free_list, &self->om_free_count); +} + +// Prepend an ObjectMonitor to a per-thread om_in_use_list. +// Also updates the per-thread om_in_use_count. +static void prepend_to_om_in_use_list(Thread* self, ObjectMonitor* m) { + prepend_to_common(m, &self->om_in_use_list, &self->om_in_use_count); +} + +// Take an ObjectMonitor from the start of the specified list. Also +// decrements the specified counter. Returns NULL if none are available. +static ObjectMonitor* take_from_start_of_common(ObjectMonitor* volatile * list_p, + int volatile * count_p) { + chk_for_list_loop(OrderAccess::load_acquire(list_p), + OrderAccess::load_acquire(count_p)); + + ObjectMonitor* next = NULL; + ObjectMonitor* take = NULL; + // Mark the list head to guard against A-B-A race: + if (!mark_list_head(list_p, &take, &next)) { + return NULL; // None are available. + } + // Switch marked list head to next (which unmarks the list head, but + // leaves take marked): + OrderAccess::release_store(list_p, next); + Atomic::dec(count_p); + // Unmark take, but leave the next value for any lagging list + // walkers. It will get cleaned up when take is prepended to + // the in-use list: + set_next(take, next); + return take; +} + +// Take an ObjectMonitor from the start of the global free-list. Also +// updates g_om_free_count. Returns NULL if none are available. +static ObjectMonitor* take_from_start_of_g_free_list() { + return take_from_start_of_common(&g_free_list, &g_om_free_count); +} + +// Take an ObjectMonitor from the start of a per-thread free-list. +// Also updates om_free_count. Returns NULL if none are available. +static ObjectMonitor* take_from_start_of_om_free_list(Thread* self) { + return take_from_start_of_common(&self->om_free_list, &self->om_free_count); +} + + // =====================> Quick functions // The quick_* forms are special fast-path variants used to improve @@ -1011,17 +1331,20 @@ closure->do_monitor(mid); } } - block = (PaddedObjectMonitor*)block->_next_om; + // unmarked_next() is not needed with g_block_list (no next field marking). + block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om); } } static bool monitors_used_above_threshold() { - if (g_om_population == 0) { + if (OrderAccess::load_acquire(&g_om_population) == 0) { return false; } if (MonitorUsedDeflationThreshold > 0) { - int monitors_used = g_om_population - g_om_free_count; - int monitor_usage = (monitors_used * 100LL) / g_om_population; + int monitors_used = OrderAccess::load_acquire(&g_om_population) - + OrderAccess::load_acquire(&g_om_free_count); + int monitor_usage = (monitors_used * 100LL) / + OrderAccess::load_acquire(&g_om_population); return monitor_usage > MonitorUsedDeflationThreshold; } return false; @@ -1052,7 +1375,8 @@ _last_async_deflation_time_ns = os::javaTimeNanos(); return true; } - if (is_MonitorBound_exceeded(g_om_population - g_om_free_count)) { + if (is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) - + OrderAccess::load_acquire(&g_om_free_count))) { // Not enough ObjectMonitors on the global free list. return true; } @@ -1087,20 +1411,21 @@ void ObjectSynchronizer::global_used_oops_do(OopClosure* f) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); - list_oops_do(g_om_in_use_list, f); + list_oops_do(OrderAccess::load_acquire(&g_om_in_use_list), OrderAccess::load_acquire(&g_om_in_use_count), f); } void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); - list_oops_do(thread->om_in_use_list, f); + list_oops_do(OrderAccess::load_acquire(&thread->om_in_use_list), OrderAccess::load_acquire(&thread->om_in_use_count), f); } -void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, OopClosure* f) { +void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, int count, OopClosure* f) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); + chk_for_list_loop(list, count); // The oops_do() phase does not overlap with monitor deflation // so no need to update the ObjectMonitor's ref_count for this // ObjectMonitor* use. - for (ObjectMonitor* mid = list; mid != NULL; mid = mid->_next_om) { + for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) { if (mid->object() != NULL) { f->do_oop((oop*)mid->object_addr()); } @@ -1116,9 +1441,6 @@ // STW-time -- disassociates idle monitors from objects. Such // scavenged monitors are returned to the g_free_list. // -// The global list is protected by gListLock. All the critical sections -// are short and operate in constant-time. -// // ObjectMonitors reside in type-stable memory (TSM) and are immortal. // // Lifecycle: @@ -1161,8 +1483,8 @@ // addition to other policies that are checked, if there are not // enough ObjectMonitors on the global free list, then // is_async_deflation_needed() will return true. The ServiceThread -// calls deflate_global_idle_monitors_using_JT() and also sets the -// per-thread om_request_deflation flag as needed. +// calls deflate_global_idle_monitors_using_JT() and also calls +// deflate_per_thread_idle_monitors_using_JT() as needed. static void InduceScavenge(Thread* self, const char * Whence) { assert(!AsyncDeflateIdleMonitors, "is not used by async deflation"); @@ -1191,41 +1513,21 @@ // tradeoffs. const int MAXPRIVATE = 1024; - if (AsyncDeflateIdleMonitors) { - JavaThread* jt = (JavaThread *)self; - if (jt->om_request_deflation && jt->om_in_use_count > 0 && - cause != inflate_cause_vm_internal) { - // Deflate any per-thread idle monitors for this JavaThread if - // this is not an internal inflation; internal inflations can - // occur in places where it is not safe to pause for a safepoint. - // Clean up your own mess (Gibbs Rule 45). Otherwise, skip this - // deflation. deflate_global_idle_monitors_using_JT() is called - // by the ServiceThread. Per-thread async deflation is triggered - // by the ServiceThread via om_request_deflation. - debug_only(jt->check_for_valid_safepoint_state(false);) - ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(jt); - } - } - stringStream ss; for (;;) { ObjectMonitor* m; // 1: try to allocate from the thread's local om_free_list. // Threads will attempt to allocate first from their local list, then - // from the global list, and only after those attempts fail will the thread - // attempt to instantiate new monitors. Thread-local free lists take - // heat off the gListLock and improve allocation latency, as well as reducing - // coherency traffic on the shared global list. - m = self->om_free_list; + // from the global list, and only after those attempts fail will the + // thread attempt to instantiate new monitors. Thread-local free lists + // improve allocation latency, as well as reducing coherency traffic + // on the shared global list. + m = take_from_start_of_om_free_list(self); if (m != NULL) { - self->om_free_list = m->_next_om; - self->om_free_count--; guarantee(m->object() == NULL, "invariant"); m->set_allocation_state(ObjectMonitor::New); - m->_next_om = self->om_in_use_list; - self->om_in_use_list = m; - self->om_in_use_count++; + prepend_to_om_in_use_list(self, m); return m; } @@ -1234,15 +1536,15 @@ // If the muxTry() fails then drop immediately into case 3. // If we're using thread-local free lists then try // to reprovision the caller's free list. - if (g_free_list != NULL) { + if (OrderAccess::load_acquire(&g_free_list) != NULL) { // Reprovision the thread's om_free_list. // Use bulk transfers to reduce the allocation rate and heat // on various locks. - Thread::muxAcquire(&gListLock, "om_alloc(1)"); - for (int i = self->om_free_provision; --i >= 0 && g_free_list != NULL;) { - g_om_free_count--; - ObjectMonitor* take = g_free_list; - g_free_list = take->_next_om; + for (int i = self->om_free_provision; --i >= 0;) { + ObjectMonitor* take = take_from_start_of_g_free_list(); + if (take == NULL) { + break; // No more are available. + } guarantee(take->object() == NULL, "invariant"); if (AsyncDeflateIdleMonitors) { // We allowed 3 field values to linger during async deflation. @@ -1264,12 +1566,12 @@ assert(take->is_free(), "invariant"); om_release(self, take, false); } - Thread::muxRelease(&gListLock); self->om_free_provision += 1 + (self->om_free_provision/2); if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE; if (!AsyncDeflateIdleMonitors && - is_MonitorBound_exceeded(g_om_population - g_om_free_count)) { + is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) - + OrderAccess::load_acquire(&g_om_free_count))) { // Not enough ObjectMonitors on the global free list. // We can't safely induce a STW safepoint from om_alloc() as our thread // state may not be appropriate for such activities and callers may hold @@ -1313,12 +1615,12 @@ // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; } for (int i = 1; i < _BLOCKSIZE; i++) { - temp[i]._next_om = (ObjectMonitor *)&temp[i+1]; + OrderAccess::release_store(&temp[i]._next_om, (ObjectMonitor*)&temp[i+1]); assert(temp[i].is_free(), "invariant"); } // terminate the last monitor as the end of list - temp[_BLOCKSIZE - 1]._next_om = NULL; + OrderAccess::release_store(&temp[_BLOCKSIZE - 1]._next_om, (ObjectMonitor*)NULL); // Element [0] is reserved for global list linkage temp[0].set_object(CHAINMARKER); @@ -1327,24 +1629,7 @@ // block in hand. This avoids some lock traffic and redundant // list activity. - // Acquire the gListLock to manipulate g_block_list and g_free_list. - // An Oyama-Taura-Yonezawa scheme might be more efficient. - Thread::muxAcquire(&gListLock, "om_alloc(2)"); - g_om_population += _BLOCKSIZE-1; - g_om_free_count += _BLOCKSIZE-1; - - // Add the new block to the list of extant blocks (g_block_list). - // The very first ObjectMonitor in a block is reserved and dedicated. - // It serves as blocklist "next" linkage. - temp[0]._next_om = g_block_list; - // There are lock-free uses of g_block_list so make sure that - // the previous stores happen before we update g_block_list. - OrderAccess::release_store(&g_block_list, temp); - - // Add the new string of ObjectMonitors to the global free list - temp[_BLOCKSIZE - 1]._next_om = g_free_list; - g_free_list = temp + 1; - Thread::muxRelease(&gListLock); + prepend_block_to_lists(temp); } } @@ -1373,28 +1658,66 @@ // we have to remove 'm' from the in-use list first (as needed). if (from_per_thread_alloc) { // Need to remove 'm' from om_in_use_list. + // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go + // protocol because async deflation can do list deletions in parallel. ObjectMonitor* cur_mid_in_use = NULL; + ObjectMonitor* mid = NULL; + ObjectMonitor* next = NULL; bool extracted = false; - for (ObjectMonitor* mid = self->om_in_use_list; mid != NULL; cur_mid_in_use = mid, mid = mid->_next_om) { + + if (!mark_list_head(&self->om_in_use_list, &mid, &next)) { + fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self)); + } + while (true) { if (m == mid) { - // extract from per-thread in-use list - if (mid == self->om_in_use_list) { - self->om_in_use_list = mid->_next_om; - } else if (cur_mid_in_use != NULL) { - cur_mid_in_use->_next_om = mid->_next_om; // maintain the current thread in-use list + // We found 'm' on the per-thread in-use list so try to extract it. + // First try the list head: + if (Atomic::cmpxchg(next, &self->om_in_use_list, mid) != mid) { + // We could not switch the list head to next. + ObjectMonitor* marked_mid = mark_om_ptr(mid); + // Switch cur_mid_in_use's next field to next (which also + // unmarks cur_mid_in_use): + ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL"); + if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, marked_mid) + != marked_mid) { + // We could not switch cur_mid_in_use's next field. This + // should not be possible since it was marked so we: + fatal("mid=" INTPTR_FORMAT " must be referred to by the list " + "head: &om_in_use_list=" INTPTR_FORMAT " or by " + "cur_mid_in_use's next field: cur_mid_in_use=" INTPTR_FORMAT + ", next_om=" INTPTR_FORMAT, p2i(mid), + p2i((ObjectMonitor**)&self->om_in_use_list), + p2i(cur_mid_in_use), p2i(cur_mid_in_use->_next_om)); + } } extracted = true; - self->om_in_use_count--; + Atomic::dec(&self->om_in_use_count); + // Unmark mid, but leave the next value for any lagging list + // walkers. It will get cleaned up when mid is prepended to + // the thread's free list: + set_next(mid, next); break; } + if (cur_mid_in_use != NULL) { + set_next(cur_mid_in_use, mid); // umark cur_mid_in_use + } + // The next cur_mid_in_use keeps mid's marked next field so + // that it is stable for a possible next field change. It + // cannot be deflated while it is marked. + cur_mid_in_use = mid; + mid = next; + if (mid == NULL) { + // Reached end of the list and didn't find m so: + fatal("must find m=" INTPTR_FORMAT "on om_in_use_list=" INTPTR_FORMAT, + p2i(m), p2i(self->om_in_use_list)); + } + // Mark mid's next field so we can possibly extract it: + next = mark_next_loop(mid); } - assert(extracted, "Should have extracted from in-use list"); } - m->_next_om = self->om_free_list; + prepend_to_om_free_list(self, m); guarantee(m->is_free(), "invariant"); - self->om_free_list = m; - self->om_free_count++; } // Return ObjectMonitors on a moribund thread's free and in-use @@ -1412,62 +1735,107 @@ // // With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT() // and deflate_per_thread_idle_monitors_using_JT() (in another thread) can -// run at the same time as om_flush() so we have to be careful. +// run at the same time as om_flush() so we have to follow a careful +// protocol to prevent list corruption. void ObjectSynchronizer::om_flush(Thread* self) { + // This function can race with an async deflater thread. Since + // deflation has to process the per-thread in-use list before + // prepending the deflated ObjectMonitors to the global free list, + // we process the per-thread lists in the same order to prevent + // ordering races. int in_use_count = 0; - ObjectMonitor* in_use_list = self->om_in_use_list; + ObjectMonitor* in_use_list = NULL; ObjectMonitor* in_use_tail = NULL; - if (in_use_list != NULL) { + ObjectMonitor* next = NULL; + + // An async deflation thread checks to see if the target thread + // is exiting, but if it has made it past that check before we + // started exiting, then it is racing to get to the in-use list. + if (mark_list_head(&self->om_in_use_list, &in_use_list, &next)) { + chk_for_list_loop(in_use_list, OrderAccess::load_acquire(&self->om_in_use_count)); + // At this point, we have marked the in-use list head so an + // async deflation thread cannot come in after us. If an async + // deflation thread is ahead of us, then we'll detect that and + // wait for it to finish its work. + // // The thread is going away, however the ObjectMonitors on the // om_in_use_list may still be in-use by other threads. Link // them to in_use_tail, which will be linked into the global - // in-use list g_om_in_use_list below, under the gListLock. - for (ObjectMonitor* cur_om = in_use_list; cur_om != NULL; cur_om = cur_om->_next_om) { + // in-use list g_om_in_use_list below. + // + // Account for the in-use list head before the loop since it is + // already marked (by this thread): + in_use_tail = in_use_list; + in_use_count++; + for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL;) { + if (is_next_marked(cur_om)) { + // This next field is marked so there must be an async deflater + // thread ahead of us so we'll give it a chance to finish. + while (is_next_marked(cur_om)) { + os::naked_short_sleep(1); + } + // Refetch the possibly changed next field and try again. + cur_om = unmarked_next(in_use_tail); + continue; + } + if (!cur_om->is_active()) { + // cur_om was deflated and the allocation state was changed + // to Free while it was marked. We happened to see it just + // after it was unmarked (and added to the free list). + // Refetch the possibly changed next field and try again. + cur_om = unmarked_next(in_use_tail); + continue; + } in_use_tail = cur_om; in_use_count++; - ADIM_guarantee(cur_om->is_active(), "invariant"); + cur_om = unmarked_next(cur_om); } guarantee(in_use_tail != NULL, "invariant"); - ADIM_guarantee(self->om_in_use_count == in_use_count, "in-use count off"); - self->om_in_use_list = NULL; - self->om_in_use_count = 0; + int l_om_in_use_count = OrderAccess::load_acquire(&self->om_in_use_count); + ADIM_guarantee(l_om_in_use_count == in_use_count, "in-use counts don't " + "match: l_om_in_use_count=%d, in_use_count=%d", + l_om_in_use_count, in_use_count); + // Clear the in-use count before unmarking the in-use list head + // to avoid races: + OrderAccess::release_store(&self->om_in_use_count, 0); + // Clear the in-use list head (which also unmarks it): + OrderAccess::release_store(&self->om_in_use_list, (ObjectMonitor*)NULL); + // Unmark the disconnected list head: + set_next(in_use_list, next); } int free_count = 0; - ObjectMonitor* free_list = self->om_free_list; + ObjectMonitor* free_list = OrderAccess::load_acquire(&self->om_free_list); ObjectMonitor* free_tail = NULL; if (free_list != NULL) { + chk_for_list_loop(free_list, OrderAccess::load_acquire(&self->om_free_count)); // The thread is going away. Set 'free_tail' to the last per-thread free - // monitor which will be linked to g_free_list below under the gListLock. + // monitor which will be linked to g_free_list below. stringStream ss; - for (ObjectMonitor* s = free_list; s != NULL; s = s->_next_om) { + for (ObjectMonitor* s = free_list; s != NULL; s = unmarked_next(s)) { free_count++; free_tail = s; guarantee(s->object() == NULL, "invariant"); guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss)); } guarantee(free_tail != NULL, "invariant"); - ADIM_guarantee(self->om_free_count == free_count, "free-count off"); - self->om_free_list = NULL; - self->om_free_count = 0; + int l_om_free_count = OrderAccess::load_acquire(&self->om_free_count); + ADIM_guarantee(l_om_free_count == free_count, "free counts don't match: " + "l_om_free_count=%d, free_count=%d", l_om_free_count, + free_count); + OrderAccess::release_store(&self->om_free_list, (ObjectMonitor*)NULL); + OrderAccess::release_store(&self->om_free_count, 0); } - Thread::muxAcquire(&gListLock, "om_flush"); if (free_tail != NULL) { - free_tail->_next_om = g_free_list; - g_free_list = free_list; - g_om_free_count += free_count; + prepend_list_to_g_free_list(free_list, free_tail, free_count); } if (in_use_tail != NULL) { - in_use_tail->_next_om = g_om_in_use_list; - g_om_in_use_list = in_use_list; - g_om_in_use_count += in_use_count; + prepend_list_to_g_om_in_use_list(in_use_list, in_use_tail, in_use_count); } - Thread::muxRelease(&gListLock); - LogStreamHandle(Debug, monitorinflation) lsh_debug; LogStreamHandle(Info, monitorinflation) lsh_info; LogStream* ls = NULL; @@ -1584,18 +1952,7 @@ LogStreamHandle(Trace, monitorinflation) lsh; if (mark.has_locker()) { - ObjectMonitor* m; - if (!AsyncDeflateIdleMonitors || cause == inflate_cause_vm_internal) { - // If !AsyncDeflateIdleMonitors or if an internal inflation, then - // we won't stop for a potential safepoint in om_alloc. - m = om_alloc(self, cause); - } else { - // If AsyncDeflateIdleMonitors and not an internal inflation, then - // we may stop for a safepoint in om_alloc() so protect object. - Handle h_obj(self, object); - m = om_alloc(self, cause); - object = h_obj(); // Refresh object. - } + ObjectMonitor* m = om_alloc(self, cause); // Optimistically prepare the objectmonitor - anticipate successful CAS // We do this before the CAS in order to minimize the length of time // in which INFLATING appears in the mark. @@ -1693,19 +2050,8 @@ // Catch if the object's header is not neutral (not locked and // not marked is what we care about here). - ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value()); - ObjectMonitor* m; - if (!AsyncDeflateIdleMonitors || cause == inflate_cause_vm_internal) { - // If !AsyncDeflateIdleMonitors or if an internal inflation, then - // we won't stop for a potential safepoint in om_alloc. - m = om_alloc(self, cause); - } else { - // If AsyncDeflateIdleMonitors and not an internal inflation, then - // we may stop for a safepoint in om_alloc() so protect object. - Handle h_obj(self, object); - m = om_alloc(self, cause); - object = h_obj(); // Refresh object. - } + ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT,mark.value()); + ObjectMonitor* m = om_alloc(self, cause); // prepare m for installation - set monitor to initial state m->Recycle(); m->set_header(mark); @@ -1721,6 +2067,10 @@ m->set_allocation_state(ObjectMonitor::Old); if (object->cas_set_mark(markWord::encode(m), mark) != mark) { + guarantee(!m->owner_is_DEFLATER_MARKER() || m->ref_count() >= 0, + "race between deflation and om_release() with m=" INTPTR_FORMAT + ", _owner=" INTPTR_FORMAT ", ref_count=%d", p2i(m), + p2i(m->_owner), m->ref_count()); m->set_header(markWord::zero()); m->set_object(NULL); m->Recycle(); @@ -1779,7 +2129,7 @@ // deflation mechanism is used. Otherwise, an async deflation request // is registered with the ServiceThread and it is notified. -void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* _counters) { +void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* counters) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); // The per-thread in-use lists are handled in @@ -1788,7 +2138,7 @@ if (!AsyncDeflateIdleMonitors || is_special_deflation_requested()) { // Use the older mechanism for the global in-use list or if a // special deflation has been requested before the safepoint. - ObjectSynchronizer::deflate_idle_monitors(_counters); + ObjectSynchronizer::deflate_idle_monitors(counters); return; } @@ -1847,16 +2197,20 @@ assert(mid->is_free(), "invariant"); // Move the deflated ObjectMonitor to the working free list - // defined by free_head_p and free_tail_p. + // defined by free_head_p and free_tail_p. No races on this list + // so no need for load_acquire() or store_release(). if (*free_head_p == NULL) *free_head_p = mid; if (*free_tail_p != NULL) { // We append to the list so the caller can use mid->_next_om // to fix the linkages in its context. ObjectMonitor* prevtail = *free_tail_p; // Should have been cleaned up by the caller: + // Note: Should not have to mark prevtail here since we're at a + // safepoint and ObjectMonitors on the local free list should + // not be accessed in parallel. assert(prevtail->_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(prevtail->_next_om)); - prevtail->_next_om = mid; + set_next(prevtail, mid); } *free_tail_p = mid; // At this point, mid->_next_om still refers to its current @@ -1959,7 +2313,8 @@ (int) mid->allocation_state()); // Move the deflated ObjectMonitor to the working free list - // defined by free_head_p and free_tail_p. + // defined by free_head_p and free_tail_p. No races on this list + // so no need for load_acquire() or store_release(). if (*free_head_p == NULL) { // First one on the list. *free_head_p = mid; @@ -1969,9 +2324,10 @@ // to fix the linkages in its context. ObjectMonitor* prevtail = *free_tail_p; // Should have been cleaned up by the caller: - assert(prevtail->_next_om == NULL, "must be NULL: _next_om=" - INTPTR_FORMAT, p2i(prevtail->_next_om)); - prevtail->_next_om = mid; + ObjectMonitor* next = mark_next_loop(prevtail); + assert(unmarked_next(prevtail) == NULL, "must be NULL: _next_om=" + INTPTR_FORMAT, p2i(unmarked_next(prevtail))); + set_next(prevtail, mid); // prevtail now points to mid (and is unmarked) } *free_tail_p = mid; @@ -2008,9 +2364,8 @@ return false; } -// Walk a given monitor list, and deflate idle monitors -// The given list could be a per-thread list or a global list -// Caller acquires gListLock as needed. +// Walk a given monitor list, and deflate idle monitors. +// The given list could be a per-thread list or a global list. // // In the case of parallel processing of thread local monitor lists, // work is done by Threads::parallel_threads_do() which ensures that @@ -2021,35 +2376,67 @@ // See also ParallelSPCleanupTask and // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and // Threads::parallel_java_threads_do() in thread.cpp. -int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor** list_p, - int* count_p, +int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor* volatile * list_p, + int volatile * count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p) { ObjectMonitor* cur_mid_in_use = NULL; - ObjectMonitor* mid; - ObjectMonitor* next; + ObjectMonitor* mid = NULL; + ObjectMonitor* next = NULL; int deflated_count = 0; - for (mid = *list_p; mid != NULL;) { + // We use the simpler mark-mid-as-we-go protocol since there are no + // parallel list deletions since we are at a safepoint. + if (!mark_list_head(list_p, &mid, &next)) { + return 0; // The list is empty so nothing to deflate. + } + + while (true) { oop obj = (oop) mid->object(); if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) { // Deflation succeeded and already updated free_head_p and // free_tail_p as needed. Finish the move to the local free list // by unlinking mid from the global or per-thread in-use list. - if (mid == *list_p) { - *list_p = mid->_next_om; - } else if (cur_mid_in_use != NULL) { - cur_mid_in_use->_next_om = mid->_next_om; // maintain the current thread in-use list + if (Atomic::cmpxchg(next, list_p, mid) != mid) { + // We could not switch the list head to next. + ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL"); + if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, mid) != mid) { + // deflate_monitor_list() is called at a safepoint so the + // global or per-thread in-use list should not be modified + // in parallel so we: + fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: " + "list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: " + "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT, + p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use), + p2i(cur_mid_in_use->_next_om)); + } } - next = mid->_next_om; - mid->_next_om = NULL; // This mid is current tail in the free_head_p list - mid = next; + // At this point mid is disconnected from the in-use list so + // its marked next field no longer has any effects. deflated_count++; - *count_p = *count_p - 1; + Atomic::dec(count_p); + chk_for_list_loop(OrderAccess::load_acquire(list_p), + OrderAccess::load_acquire(count_p)); + chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p), + OrderAccess::load_acquire(count_p)); + // mid is current tail in the free_head_p list so NULL terminate it + // (which also unmarks it): + set_next(mid, NULL); + + // All the list management is done so move on to the next one: + mid = next; } else { + set_next(mid, next); // unmark next field + + // All the list management is done so move on to the next one: cur_mid_in_use = mid; - mid = mid->_next_om; + mid = next; + } + if (mid == NULL) { + break; // Reached end of the list so nothing more to deflate. } + // Mark mid's next field so we can possibly deflate it: + next = mark_next_loop(mid); } return deflated_count; } @@ -2057,12 +2444,11 @@ // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using // a JavaThread. Returns the number of deflated ObjectMonitors. The given // list could be a per-thread in-use list or the global in-use list. -// Caller acquires gListLock as appropriate. If a safepoint has started, -// then we save state via saved_mid_in_use_p and return to the caller to -// honor the safepoint. +// If a safepoint has started, then we save state via saved_mid_in_use_p +// and return to the caller to honor the safepoint. // -int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor** list_p, - int* count_p, +int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor* volatile * list_p, + int volatile * count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p, ObjectMonitor** saved_mid_in_use_p) { @@ -2070,20 +2456,48 @@ assert(Thread::current()->is_Java_thread(), "precondition"); ObjectMonitor* cur_mid_in_use = NULL; - ObjectMonitor* mid; - ObjectMonitor* next; + ObjectMonitor* mid = NULL; + ObjectMonitor* next = NULL; + ObjectMonitor* next_next = NULL; int deflated_count = 0; + // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go + // protocol because om_release() can do list deletions in parallel. + // We also mark-next-next-as-we-go to prevent an om_flush() that is + // behind this thread from passing us. if (*saved_mid_in_use_p == NULL) { // No saved state so start at the beginning. - mid = *list_p; + // Mark the list head's next field so we can possibly deflate it: + if (!mark_list_head(list_p, &mid, &next)) { + return 0; // The list is empty so nothing to deflate. + } } else { // We're restarting after a safepoint so restore the necessary state // before we resume. cur_mid_in_use = *saved_mid_in_use_p; - mid = cur_mid_in_use->_next_om; + // Mark cur_mid_in_use's next field so we can possibly update its + // next field to extract a deflated ObjectMonitor. + mid = mark_next_loop(cur_mid_in_use); + if (mid == NULL) { + set_next(cur_mid_in_use, NULL); // unmark next field + *saved_mid_in_use_p = NULL; + return 0; // The remainder is empty so nothing more to deflate. + } + // Mark mid's next field so we can possibly deflate it: + next = mark_next_loop(mid); } - while (mid != NULL) { + + while (true) { + // The current mid's next field is marked at this point. If we have + // a cur_mid_in_use, then its next field is also marked at this point. + + if (next != NULL) { + // We mark the next -> next field so that an om_flush() + // thread that is behind us cannot pass us when we + // unmark the current mid's next field. + next_next = mark_next_loop(next); + } + // Only try to deflate if there is an associated Java object and if // mid is old (is not newly allocated and is not newly freed). if (mid->object() != NULL && mid->is_old() && @@ -2091,19 +2505,38 @@ // Deflation succeeded and already updated free_head_p and // free_tail_p as needed. Finish the move to the local free list // by unlinking mid from the global or per-thread in-use list. - if (mid == *list_p) { - *list_p = mid->_next_om; - } else if (cur_mid_in_use != NULL) { - // Maintain the current in-use list. - cur_mid_in_use->_next_om = mid->_next_om; - } - next = mid->_next_om; - mid->_next_om = NULL; - // At this point mid is disconnected from the in-use list - // and is the current tail in the free_head_p list. - mid = next; + if (Atomic::cmpxchg(next, list_p, mid) != mid) { + // We could not switch the list head to next. + ObjectMonitor* marked_mid = mark_om_ptr(mid); + ObjectMonitor* marked_next = mark_om_ptr(next); + // Switch cur_mid_in_use's next field to marked next: + ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL"); + if (Atomic::cmpxchg(marked_next, &cur_mid_in_use->_next_om, + marked_mid) != marked_mid) { + // We could not switch cur_mid_in_use's next field. This + // should not be possible since it was marked so we: + fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: " + "&list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: " + "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT, + p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use), + p2i(cur_mid_in_use->_next_om)); + } + } + // At this point mid is disconnected from the in-use list so + // its marked next field no longer has any effects. deflated_count++; - *count_p = *count_p - 1; + Atomic::dec(count_p); + chk_for_list_loop(OrderAccess::load_acquire(list_p), + OrderAccess::load_acquire(count_p)); + chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p), + OrderAccess::load_acquire(count_p)); + // mid is current tail in the free_head_p list so NULL terminate it + // (which also unmarks it): + set_next(mid, NULL); + + // All the list management is done so move on to the next one: + mid = next; // mid keeps non-NULL next's marked next field + next = next_next; } else { // mid is considered in-use if it does not have an associated // Java object or mid is not old or deflation did not succeed. @@ -2114,19 +2547,40 @@ // om_alloc() is released by om_release() due to losing the race // in inflate(). + // All the list management is done so move on to the next one: + if (cur_mid_in_use != NULL) { + set_next(cur_mid_in_use, mid); // umark cur_mid_in_use + } + // The next cur_mid_in_use keeps mid's marked next field so + // that it is stable for a possible next field change. It + // cannot be modified by om_release() while it is marked. cur_mid_in_use = mid; - mid = mid->_next_om; + mid = next; // mid keeps non-NULL next's marked next field + next = next_next; if (SafepointSynchronize::is_synchronizing() && - cur_mid_in_use != *list_p && cur_mid_in_use->is_old()) { + cur_mid_in_use != OrderAccess::load_acquire(list_p) && + cur_mid_in_use->is_old()) { // If a safepoint has started and cur_mid_in_use is not the list // head and is old, then it is safe to use as saved state. Return - // to the caller so gListLock can be dropped as appropriate - // before blocking. + // to the caller before blocking. *saved_mid_in_use_p = cur_mid_in_use; + set_next(cur_mid_in_use, mid); // umark cur_mid_in_use + if (mid != NULL) { + set_next(mid, next); // umark mid + } return deflated_count; } } + if (mid == NULL) { + if (cur_mid_in_use != NULL) { + set_next(cur_mid_in_use, mid); // umark cur_mid_in_use + } + break; // Reached end of the list so nothing more to deflate. + } + + // The current mid's next field is marked at this point. If we have + // a cur_mid_in_use, then its next field is also marked at this point. } // We finished the list without a safepoint starting so there's // no need to save state. @@ -2135,11 +2589,11 @@ } void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) { - counters->n_in_use = 0; // currently associated with objects - counters->n_in_circulation = 0; // extant - counters->n_scavenged = 0; // reclaimed (global and per-thread) - counters->per_thread_scavenged = 0; // per-thread scavenge total - counters->per_thread_times = 0.0; // per-thread scavenge times + OrderAccess::release_store(&counters->n_in_use, 0); // currently associated with objects + OrderAccess::release_store(&counters->n_in_circulation, 0); // extant + OrderAccess::release_store(&counters->n_scavenged, 0); // reclaimed (global and per-thread) + OrderAccess::release_store(&counters->per_thread_scavenged, 0); // per-thread scavenge total + counters->per_thread_times = 0.0; // per-thread scavenge times } void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) { @@ -2163,34 +2617,28 @@ timer.start(); } - // Prevent om_flush from changing mids in Thread dtor's during deflation - // And in case the vm thread is acquiring a lock during a safepoint - // See e.g. 6320749 - Thread::muxAcquire(&gListLock, "deflate_idle_monitors"); - // Note: the thread-local monitors lists get deflated in // a separate pass. See deflate_thread_local_monitors(). // For moribund threads, scan g_om_in_use_list int deflated_count = 0; - if (g_om_in_use_list != NULL) { + if (OrderAccess::load_acquire(&g_om_in_use_list) != NULL) { // Update n_in_circulation before g_om_in_use_count is updated by deflation. - counters->n_in_circulation += g_om_in_use_count; - deflated_count = deflate_monitor_list((ObjectMonitor**)&g_om_in_use_list, (int*)&g_om_in_use_count, &free_head_p, &free_tail_p); - counters->n_in_use += g_om_in_use_count; + Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_circulation); + + deflated_count = deflate_monitor_list(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p); + Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_use); } if (free_head_p != NULL) { // Move the deflated ObjectMonitors back to the global free list. + // No races on the working free list so no need for load_acquire(). guarantee(free_tail_p != NULL && deflated_count > 0, "invariant"); assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(free_tail_p->_next_om)); - // constant-time list splice - prepend scavenged segment to g_free_list - free_tail_p->_next_om = g_free_list; - g_free_list = free_head_p; - counters->n_scavenged += deflated_count; + prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count); + Atomic::add(deflated_count, &counters->n_scavenged); } - Thread::muxRelease(&gListLock); timer.stop(); LogStreamHandle(Debug, monitorinflation) lsh_debug; @@ -2222,8 +2670,6 @@ assert(AsyncDeflateIdleMonitors, "sanity check"); assert(Thread::current()->is_Java_thread(), "precondition"); - target->om_request_deflation = false; - deflate_common_idle_monitors_using_JT(false /* !is_global */, target); } @@ -2243,16 +2689,15 @@ } if (is_global) { - Thread::muxAcquire(&gListLock, "deflate_global_idle_monitors_using_JT(1)"); - OM_PERFDATA_OP(MonExtant, set_value(g_om_in_use_count)); + OM_PERFDATA_OP(MonExtant, set_value(OrderAccess::load_acquire(&g_om_in_use_count))); } else { - OM_PERFDATA_OP(MonExtant, inc(target->om_in_use_count)); + OM_PERFDATA_OP(MonExtant, inc(OrderAccess::load_acquire(&target->om_in_use_count))); } do { int local_deflated_count; if (is_global) { - local_deflated_count = deflate_monitor_list_using_JT((ObjectMonitor**)&g_om_in_use_list, (int*)&g_om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p); + local_deflated_count = deflate_monitor_list_using_JT(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p); } else { local_deflated_count = deflate_monitor_list_using_JT(&target->om_in_use_list, &target->om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p); } @@ -2260,28 +2705,26 @@ if (free_head_p != NULL) { // Move the deflated ObjectMonitors to the global free list. + // No races on the working list so no need for load_acquire(). guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count); - assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om=" - INTPTR_FORMAT, p2i(free_tail_p->_next_om)); + // Note: The target thread can be doing an om_alloc() that + // is trying to prepend an ObjectMonitor on its in-use list + // at the same time that we have deflated the current in-use + // list head and put it on the local free list. prepend_to_common() + // will detect the race and retry which avoids list corruption, + // but the next field in free_tail_p can flicker to marked + // and then unmarked while prepend_to_common() is sorting it + // all out. + assert(unmarked_next(free_tail_p) == NULL, "must be NULL: _next_om=" + INTPTR_FORMAT, p2i(unmarked_next(free_tail_p))); - if (!is_global) { - Thread::muxAcquire(&gListLock, "deflate_per_thread_idle_monitors_using_JT(2)"); - } - // Constant-time list splice - prepend scavenged segment to g_free_list. - free_tail_p->_next_om = g_free_list; - g_free_list = free_head_p; + prepend_list_to_g_free_list(free_head_p, free_tail_p, local_deflated_count); OM_PERFDATA_OP(Deflations, inc(local_deflated_count)); - if (!is_global) { - Thread::muxRelease(&gListLock); - } } if (saved_mid_in_use_p != NULL) { // deflate_monitor_list_using_JT() detected a safepoint starting. - if (is_global) { - Thread::muxRelease(&gListLock); - } timer.stop(); { if (is_global) { @@ -2298,14 +2741,8 @@ if (log_is_enabled(Info, monitorinflation)) { timer.start(); } - if (is_global) { - Thread::muxAcquire(&gListLock, "deflate_global_idle_monitors_using_JT(3)"); - } } } while (saved_mid_in_use_p != NULL); - if (is_global) { - Thread::muxRelease(&gListLock); - } timer.stop(); LogStreamHandle(Debug, monitorinflation) lsh_debug; @@ -2332,7 +2769,9 @@ // than a beginning to end measurement of the phase. // Note: AsyncDeflateIdleMonitors only deflates per-thread idle // monitors at a safepoint when a special deflation has been requested. - log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", counters->per_thread_times, counters->per_thread_scavenged); + log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", + counters->per_thread_times, + OrderAccess::load_acquire(&counters->per_thread_scavenged)); bool needs_special_deflation = is_special_deflation_requested(); if (!AsyncDeflateIdleMonitors || needs_special_deflation) { @@ -2348,11 +2787,11 @@ // at the Info level. ObjectSynchronizer::audit_and_print_stats(false /* on_exit */); } else if (log_is_enabled(Info, monitorinflation)) { - Thread::muxAcquire(&gListLock, "finish_deflate_idle_monitors"); log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, " - "g_om_free_count=%d", g_om_population, - g_om_in_use_count, g_om_free_count); - Thread::muxRelease(&gListLock); + "g_om_free_count=%d", + OrderAccess::load_acquire(&g_om_population), + OrderAccess::load_acquire(&g_om_in_use_count), + OrderAccess::load_acquire(&g_om_free_count)); } ForceMonitorScavenge = 0; // Reset @@ -2366,16 +2805,9 @@ void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); - if (AsyncDeflateIdleMonitors) { - if (!is_special_deflation_requested()) { - // Mark the JavaThread for idle monitor deflation if a special - // deflation has NOT been requested. - if (thread->om_in_use_count > 0) { - // This JavaThread is using monitors so mark it. - thread->om_request_deflation = true; - } - return; - } + if (AsyncDeflateIdleMonitors && !is_special_deflation_requested()) { + // Nothing to do if a special deflation has NOT been requested. + return; } ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors @@ -2388,24 +2820,20 @@ } // Update n_in_circulation before om_in_use_count is updated by deflation. - counters->n_in_circulation += thread->om_in_use_count; + Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_circulation); int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p); - counters->n_in_use += thread->om_in_use_count; - - Thread::muxAcquire(&gListLock, "deflate_thread_local_monitors"); + Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_use); if (free_head_p != NULL) { // Move the deflated ObjectMonitors back to the global free list. + // No races on the working list so no need for load_acquire(). guarantee(free_tail_p != NULL && deflated_count > 0, "invariant"); assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(free_tail_p->_next_om)); - - // constant-time list splice - prepend scavenged segment to g_free_list - free_tail_p->_next_om = g_free_list; - g_free_list = free_head_p; - counters->n_scavenged += deflated_count; - counters->per_thread_scavenged += deflated_count; + prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count); + Atomic::add(deflated_count, &counters->n_scavenged); + Atomic::add(deflated_count, &counters->per_thread_scavenged); } timer.stop(); @@ -2414,8 +2842,6 @@ // should be cheap. counters->per_thread_times += timer.seconds(); - Thread::muxRelease(&gListLock); - LogStreamHandle(Debug, monitorinflation) lsh_debug; LogStreamHandle(Info, monitorinflation) lsh_info; LogStream* ls = NULL; @@ -2466,9 +2892,7 @@ assert(THREAD == JavaThread::current(), "must be current Java thread"); NoSafepointVerifier nsv; ReleaseJavaMonitorsClosure rjmc(THREAD); - Thread::muxAcquire(&gListLock, "release_monitors_owned_by_thread"); ObjectSynchronizer::monitors_iterate(&rjmc); - Thread::muxRelease(&gListLock); THREAD->clear_pending_exception(); } @@ -2522,11 +2946,6 @@ } assert(ls != NULL, "sanity check"); - if (!on_exit) { - // Not at VM exit so grab the global list lock. - Thread::muxAcquire(&gListLock, "audit_and_print_stats"); - } - // Log counts for the global and per-thread monitor lists: int chk_om_population = log_monitor_list_counts(ls); int error_cnt = 0; @@ -2534,12 +2953,14 @@ ls->print_cr("Checking global lists:"); // Check g_om_population: - if (g_om_population == chk_om_population) { + if (OrderAccess::load_acquire(&g_om_population) == chk_om_population) { ls->print_cr("g_om_population=%d equals chk_om_population=%d", - g_om_population, chk_om_population); + OrderAccess::load_acquire(&g_om_population), + chk_om_population); } else { ls->print_cr("ERROR: g_om_population=%d is not equal to " - "chk_om_population=%d", g_om_population, + "chk_om_population=%d", + OrderAccess::load_acquire(&g_om_population), chk_om_population); error_cnt++; } @@ -2550,10 +2971,6 @@ // Check g_free_list and g_om_free_count: chk_global_free_list_and_count(ls, &error_cnt); - if (!on_exit) { - Thread::muxRelease(&gListLock); - } - ls->print_cr("Checking per-thread lists:"); for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { @@ -2575,7 +2992,7 @@ // When exiting this log output is at the Info level. When called // at a safepoint, this log output is at the Trace level since // there can be a lot of it. - log_in_use_monitor_details(ls, on_exit); + log_in_use_monitor_details(ls); } ls->flush(); @@ -2631,18 +3048,23 @@ void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out, int *error_cnt_p) { int chk_om_free_count = 0; - for (ObjectMonitor* n = g_free_list; n != NULL; n = n->_next_om) { + for (ObjectMonitor* n = OrderAccess::load_acquire(&g_free_list); n != NULL; n = unmarked_next(n)) { chk_free_entry(NULL /* jt */, n, out, error_cnt_p); chk_om_free_count++; } - if (g_om_free_count == chk_om_free_count) { + if (OrderAccess::load_acquire(&g_om_free_count) == chk_om_free_count) { out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d", - g_om_free_count, chk_om_free_count); + OrderAccess::load_acquire(&g_om_free_count), + chk_om_free_count); } else { - out->print_cr("ERROR: g_om_free_count=%d is not equal to " - "chk_om_free_count=%d", g_om_free_count, + // With lock free access to g_free_list, it is possible for an + // ObjectMonitor to be prepended to g_free_list after we started + // calculating chk_om_free_count so g_om_free_count may not + // match anymore. + out->print_cr("WARNING: g_om_free_count=%d is not equal to " + "chk_om_free_count=%d", + OrderAccess::load_acquire(&g_om_free_count), chk_om_free_count); - *error_cnt_p = *error_cnt_p + 1; } } @@ -2650,16 +3072,18 @@ void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out, int *error_cnt_p) { int chk_om_in_use_count = 0; - for (ObjectMonitor* n = g_om_in_use_list; n != NULL; n = n->_next_om) { + for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) { chk_in_use_entry(NULL /* jt */, n, out, error_cnt_p); chk_om_in_use_count++; } - if (g_om_in_use_count == chk_om_in_use_count) { - out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d", g_om_in_use_count, + if (OrderAccess::load_acquire(&g_om_in_use_count) == chk_om_in_use_count) { + out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d", + OrderAccess::load_acquire(&g_om_in_use_count), chk_om_in_use_count); } else { out->print_cr("ERROR: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d", - g_om_in_use_count, chk_om_in_use_count); + OrderAccess::load_acquire(&g_om_in_use_count), + chk_om_in_use_count); *error_cnt_p = *error_cnt_p + 1; } } @@ -2728,16 +3152,19 @@ outputStream * out, int *error_cnt_p) { int chk_om_free_count = 0; - for (ObjectMonitor* n = jt->om_free_list; n != NULL; n = n->_next_om) { + for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_free_list); n != NULL; n = unmarked_next(n)) { chk_free_entry(jt, n, out, error_cnt_p); chk_om_free_count++; } - if (jt->om_free_count == chk_om_free_count) { + if (OrderAccess::load_acquire(&jt->om_free_count) == chk_om_free_count) { out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals " - "chk_om_free_count=%d", p2i(jt), jt->om_free_count, chk_om_free_count); + "chk_om_free_count=%d", p2i(jt), + OrderAccess::load_acquire(&jt->om_free_count), + chk_om_free_count); } else { out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not " - "equal to chk_om_free_count=%d", p2i(jt), jt->om_free_count, + "equal to chk_om_free_count=%d", p2i(jt), + OrderAccess::load_acquire(&jt->om_free_count), chk_om_free_count); *error_cnt_p = *error_cnt_p + 1; } @@ -2748,17 +3175,19 @@ outputStream * out, int *error_cnt_p) { int chk_om_in_use_count = 0; - for (ObjectMonitor* n = jt->om_in_use_list; n != NULL; n = n->_next_om) { + for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) { chk_in_use_entry(jt, n, out, error_cnt_p); chk_om_in_use_count++; } - if (jt->om_in_use_count == chk_om_in_use_count) { + if (OrderAccess::load_acquire(&jt->om_in_use_count) == chk_om_in_use_count) { out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals " - "chk_om_in_use_count=%d", p2i(jt), jt->om_in_use_count, + "chk_om_in_use_count=%d", p2i(jt), + OrderAccess::load_acquire(&jt->om_in_use_count), chk_om_in_use_count); } else { out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not " - "equal to chk_om_in_use_count=%d", p2i(jt), jt->om_in_use_count, + "equal to chk_om_in_use_count=%d", p2i(jt), + OrderAccess::load_acquire(&jt->om_in_use_count), chk_om_in_use_count); *error_cnt_p = *error_cnt_p + 1; } @@ -2767,21 +3196,15 @@ // Log details about ObjectMonitors on the in-use lists. The 'BHL' // flags indicate why the entry is in-use, 'object' and 'object type' // indicate the associated object and its type. -void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out, - bool on_exit) { - if (!on_exit) { - // Not at VM exit so grab the global list lock. - Thread::muxAcquire(&gListLock, "log_in_use_monitor_details"); - } - +void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out) { stringStream ss; - if (g_om_in_use_count > 0) { + if (OrderAccess::load_acquire(&g_om_in_use_count) > 0) { out->print_cr("In-use global monitor info:"); out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)"); out->print_cr("%18s %s %7s %18s %18s", "monitor", "BHL", "ref_cnt", "object", "object type"); out->print_cr("================== === ======= ================== =================="); - for (ObjectMonitor* n = g_om_in_use_list; n != NULL; n = n->_next_om) { + for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) { const oop obj = (oop) n->object(); const markWord mark = n->header(); ResourceMark rm; @@ -2797,17 +3220,13 @@ } } - if (!on_exit) { - Thread::muxRelease(&gListLock); - } - out->print_cr("In-use per-thread monitor info:"); out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)"); out->print_cr("%18s %18s %s %7s %18s %18s", "jt", "monitor", "BHL", "ref_cnt", "object", "object type"); out->print_cr("================== ================== === ======= ================== =================="); for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { - for (ObjectMonitor* n = jt->om_in_use_list; n != NULL; n = n->_next_om) { + for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) { const oop obj = (oop) n->object(); const markWord mark = n->header(); ResourceMark rm; @@ -2834,8 +3253,11 @@ "Global Lists:", "InUse", "Free", "Total"); out->print_cr("================== ========== ========== =========="); out->print_cr("%18s %10d %10d %10d", "", - g_om_in_use_count, g_om_free_count, g_om_population); - pop_count += g_om_in_use_count + g_om_free_count; + OrderAccess::load_acquire(&g_om_in_use_count), + OrderAccess::load_acquire(&g_om_free_count), + OrderAccess::load_acquire(&g_om_population)); + pop_count += OrderAccess::load_acquire(&g_om_in_use_count) + + OrderAccess::load_acquire(&g_om_free_count); out->print_cr("%18s %10s %10s %10s", "Per-Thread Lists:", "InUse", "Free", "Provision"); @@ -2843,8 +3265,11 @@ for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt), - jt->om_in_use_count, jt->om_free_count, jt->om_free_provision); - pop_count += jt->om_in_use_count + jt->om_free_count; + OrderAccess::load_acquire(&jt->om_in_use_count), + OrderAccess::load_acquire(&jt->om_free_count), + jt->om_free_provision); + pop_count += OrderAccess::load_acquire(&jt->om_in_use_count) + + OrderAccess::load_acquire(&jt->om_free_count); } return pop_count; } @@ -2866,7 +3291,8 @@ assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned"); return 1; } - block = (PaddedObjectMonitor*)block->_next_om; + // unmarked_next() is not needed with g_block_list (no next field marking). + block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om); } return 0; } --- old/src/hotspot/share/runtime/synchronizer.hpp 2019-08-28 15:04:02.778957288 -0400 +++ new/src/hotspot/share/runtime/synchronizer.hpp 2019-08-28 15:04:02.590957295 -0400 @@ -44,11 +44,11 @@ typedef PaddedEnd PaddedObjectMonitor; struct DeflateMonitorCounters { - int n_in_use; // currently associated with objects - int n_in_circulation; // extant - int n_scavenged; // reclaimed (global and per-thread) - int per_thread_scavenged; // per-thread scavenge total - double per_thread_times; // per-thread scavenge times + volatile int n_in_use; // currently associated with objects + volatile int n_in_circulation; // extant + volatile int n_scavenged; // reclaimed (global and per-thread) + volatile int per_thread_scavenged; // per-thread scavenge total + double per_thread_times; // per-thread scavenge times }; class ObjectSynchronizer : AllStatic { @@ -155,14 +155,14 @@ static void finish_deflate_idle_monitors(DeflateMonitorCounters* counters); // For a given monitor list: global or per-thread, deflate idle monitors - static int deflate_monitor_list(ObjectMonitor** list_p, - int* count_p, + static int deflate_monitor_list(ObjectMonitor* volatile * list_p, + int volatile * count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p); // For a given in-use monitor list: global or per-thread, deflate idle // monitors using a JavaThread. - static int deflate_monitor_list_using_JT(ObjectMonitor** list_p, - int* count_p, + static int deflate_monitor_list_using_JT(ObjectMonitor* volatile * list_p, + int volatile * count_p, ObjectMonitor** free_head_p, ObjectMonitor** free_tail_p, ObjectMonitor** saved_mid_in_use_p); @@ -199,11 +199,11 @@ static void chk_per_thread_free_list_and_count(JavaThread *jt, outputStream * out, int *error_cnt_p); - static void log_in_use_monitor_details(outputStream * out, bool on_exit); + static void log_in_use_monitor_details(outputStream * out); static int log_monitor_list_counts(outputStream * out); static int verify_objmon_isinpool(ObjectMonitor *addr) PRODUCT_RETURN0; - static void do_safepoint_work(DeflateMonitorCounters* _counters); + static void do_safepoint_work(DeflateMonitorCounters* counters); private: friend class SynchronizerTest; @@ -215,10 +215,13 @@ static volatile bool _is_special_deflation_requested; static jlong _last_async_deflation_time_ns; + // Function to prepend new blocks to the appropriate lists: + static void prepend_block_to_lists(PaddedObjectMonitor* new_blk); + // Process oops in all global used monitors (i.e. moribund thread's monitors) static void global_used_oops_do(OopClosure* f); // Process oops in monitors on the given list - static void list_oops_do(ObjectMonitor* list, OopClosure* f); + static void list_oops_do(ObjectMonitor* list, int count, OopClosure* f); // Support for SynchronizerTest access to GVars fields: static u_char* get_gvars_addr(); --- old/src/hotspot/share/runtime/thread.cpp 2019-08-28 15:04:03.458957265 -0400 +++ new/src/hotspot/share/runtime/thread.cpp 2019-08-28 15:04:03.278957271 -0400 @@ -264,7 +264,6 @@ om_free_provision = 32; om_in_use_list = NULL; om_in_use_count = 0; - om_request_deflation = false; #ifdef ASSERT _visited_for_critical_count = false; --- old/src/hotspot/share/runtime/thread.hpp 2019-08-28 15:04:04.222957238 -0400 +++ new/src/hotspot/share/runtime/thread.hpp 2019-08-28 15:04:04.026957245 -0400 @@ -410,12 +410,11 @@ // Per-thread ObjectMonitor lists: public: - ObjectMonitor* om_free_list; // SLL of free ObjectMonitors - int om_free_count; // # on om_free_list + ObjectMonitor* volatile om_free_list; // SLL of free ObjectMonitors + volatile int om_free_count; // # on om_free_list int om_free_provision; // # to try to allocate next - ObjectMonitor* om_in_use_list; // SLL of in-use ObjectMonitors - int om_in_use_count; // # on om_in_use_list - volatile bool om_request_deflation; // request deflation of idle monitors + ObjectMonitor* volatile om_in_use_list; // SLL of in-use ObjectMonitors + volatile int om_in_use_count; // # on om_in_use_list #ifdef ASSERT private: --- old/src/hotspot/share/runtime/vmStructs.cpp 2019-08-28 15:04:04.950957213 -0400 +++ new/src/hotspot/share/runtime/vmStructs.cpp 2019-08-28 15:04:04.770957219 -0400 @@ -910,7 +910,7 @@ volatile_nonstatic_field(ObjectMonitor, _header, markWord) \ unchecked_nonstatic_field(ObjectMonitor, _object, sizeof(void *)) /* NOTE: no type */ \ unchecked_nonstatic_field(ObjectMonitor, _owner, sizeof(void *)) /* NOTE: no type */ \ - nonstatic_field(ObjectMonitor, _next_om, ObjectMonitor*) \ + volatile_nonstatic_field(ObjectMonitor, _next_om, ObjectMonitor*) \ volatile_nonstatic_field(BasicLock, _displaced_header, markWord) \ volatile_nonstatic_field(ObjectMonitor, _contentions, jint) \ volatile_nonstatic_field(ObjectMonitor, _waiters, jint) \ --- old/test/jdk/tools/jlink/multireleasejar/JLinkMultiReleaseJarTest.java 2019-08-28 15:04:05.694957187 -0400 +++ new/test/jdk/tools/jlink/multireleasejar/JLinkMultiReleaseJarTest.java 2019-08-28 15:04:05.522957193 -0400 @@ -193,15 +193,6 @@ int version = (int) getVersion.invoke(clazz.getConstructor().newInstance()); Assert.assertEquals(version, JarFile.runtimeVersion().major()); } - // Very rarely this test fails on Windows due to: - // Error. failed to clean up files after test - // and this mesg shows the problem file (variable jimage): - // Can't delete T:\\testOutput\\test-support\\jtreg_open_test_jdk_core_tools\\scratch\\0\\myimage\\lib\\modules - // The failure happens more with async monitor deflation - // so I think that an inflated monitor is keeping the above - // try-with-resources block from cleaning up in a timely - // fashion. Forcing a GC here appears to solve the problem. - System.gc(); } @Test