--- old/src/hotspot/cpu/x86/macroAssembler_x86.cpp 2019-11-04 14:58:58.000000000 -0500 +++ new/src/hotspot/cpu/x86/macroAssembler_x86.cpp 2019-11-04 14:58:58.000000000 -0500 @@ -1584,6 +1584,9 @@ Label L_rtm_retry, L_decrement_retry, L_on_abort, L_local_done; int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner); + // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. + movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); + if (!HandshakeAfterDeflateIdleMonitors) { // Increment the ObjectMonitor's ref_count for safety or force the // enter slow path via DONE_LABEL. @@ -1593,8 +1596,6 @@ inc_om_ref_count(objReg, tmpReg /* om_reg */, scrReg /* tmp_reg */, DONE_LABEL); } - // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. - movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); movptr(boxReg, tmpReg); // Save ObjectMonitor address if (RTMRetryCount > 0) { @@ -1898,6 +1899,10 @@ // It's inflated and we use scrReg for ObjectMonitor* in this section. movq(scrReg, tmpReg); + // Unconditionally set box->_displaced_header = markWord::unused_mark(). + // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. + movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); + if (!HandshakeAfterDeflateIdleMonitors) { // Increment the ObjectMonitor's ref_count for safety or force the // enter slow path via DONE_LABEL. @@ -1910,9 +1915,6 @@ xorq(tmpReg, tmpReg); lock(); cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner))); - // Unconditionally set box->_displaced_header = markWord::unused_mark(). - // Without cast to int32_t this style of movptr will destroy r10 which is typically obj. - movptr(Address(boxReg, 0), (int32_t)intptr_t(markWord::unused_mark().value())); // Intentional fall-through into DONE_LABEL ... // Propagate ICC.ZF from CAS above into DONE_LABEL. --- old/src/hotspot/share/prims/jvmtiEnvBase.cpp 2019-11-04 14:58:59.000000000 -0500 +++ new/src/hotspot/share/prims/jvmtiEnvBase.cpp 2019-11-04 14:58:59.000000000 -0500 @@ -654,10 +654,11 @@ java_thread->is_thread_fully_suspended(false, &debug_bits)), "at safepoint or target thread is suspended"); oop obj = NULL; - ObjectMonitor *mon = java_thread->current_waiting_monitor(); + ObjectMonitorHandle omh; + ObjectMonitor *mon = java_thread->current_waiting_monitor(&omh); if (mon == NULL) { // thread is not doing an Object.wait() call - mon = java_thread->current_pending_monitor(); + mon = java_thread->current_pending_monitor(&omh); if (mon != NULL) { // The thread is trying to enter() an ObjectMonitor. obj = (oop)mon->object(); @@ -733,16 +734,18 @@ HandleMark hm; oop wait_obj = NULL; { + ObjectMonitorHandle omh; // save object of current wait() call (if any) for later comparison - ObjectMonitor *mon = java_thread->current_waiting_monitor(); + ObjectMonitor *mon = java_thread->current_waiting_monitor(&omh); if (mon != NULL) { wait_obj = (oop)mon->object(); } } oop pending_obj = NULL; { + ObjectMonitorHandle omh; // save object of current enter() call (if any) for later comparison - ObjectMonitor *mon = java_thread->current_pending_monitor(); + ObjectMonitor *mon = java_thread->current_pending_monitor(&omh); if (mon != NULL) { pending_obj = (oop)mon->object(); } --- old/src/hotspot/share/runtime/init.cpp 2019-11-04 14:59:00.000000000 -0500 +++ new/src/hotspot/share/runtime/init.cpp 2019-11-04 14:59:00.000000000 -0500 @@ -178,8 +178,12 @@ if (log_is_enabled(Info, monitorinflation)) { // The ObjectMonitor subsystem uses perf counters so // do this before perfMemory_exit(). - // ObjectSynchronizer::finish_deflate_idle_monitors()'s call - // to audit_and_print_stats() is done at the Debug level. + // These other two audit_and_print_stats() calls are done at the + // Debug level at a safepoint: + // - for safepoint based deflation auditing: + // ObjectSynchronizer::finish_deflate_idle_monitors() + // - for async deflation auditing: + // ObjectSynchronizer::do_safepoint_work() ObjectSynchronizer::audit_and_print_stats(true /* on_exit */); } perfMemory_exit(); --- old/src/hotspot/share/runtime/objectMonitor.cpp 2019-11-04 14:59:01.000000000 -0500 +++ new/src/hotspot/share/runtime/objectMonitor.cpp 2019-11-04 14:59:01.000000000 -0500 @@ -239,7 +239,8 @@ // Enter support void ObjectMonitor::enter(TRAPS) { - ADIM_guarantee(ref_count() > 0, "must be positive: ref_count=%d", ref_count()); + jint l_ref_count = ref_count(); + ADIM_guarantee(l_ref_count > 0, "must be positive: l_ref_count=%d, ref_count=%d", l_ref_count, ref_count()); // The following code is ordered to check the most common cases first // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors. @@ -257,10 +258,10 @@ return; } - if (Self->is_lock_owned ((address)cur)) { + if (Self->is_lock_owned((address)cur)) { assert(_recursions == 0, "internal state error"); _recursions = 1; - set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. + simply_set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. return; } @@ -300,7 +301,7 @@ JavaThread * jt = (JavaThread *) Self; assert(!SafepointSynchronize::is_at_safepoint(), "invariant"); assert(jt->thread_state() != _thread_blocked, "invariant"); - assert(AsyncDeflateIdleMonitors || this->object() != NULL, "invariant"); + assert(this->object() != NULL, "invariant"); assert(_contentions >= 0, "must not be negative: contentions=%d", _contentions); // Prevent deflation. See ObjectSynchronizer::deflate_monitor(), @@ -506,6 +507,8 @@ } else if (_owner != DEFLATER_MARKER) { ss->print("owner=" INTPTR_FORMAT, p2i(_owner)); } else { + // We report NULL instead of DEFLATER_MARKER here because is_busy() + // ignores DEFLATER_MARKER values. ss->print("owner=" INTPTR_FORMAT, NULL); } ss->print(", cxq=" INTPTR_FORMAT ", EntryList=" INTPTR_FORMAT, p2i(_cxq), @@ -516,7 +519,8 @@ #define MAX_RECHECK_INTERVAL 1000 void ObjectMonitor::EnterI(TRAPS) { - ADIM_guarantee(ref_count() > 0, "must be positive: ref_count=%d", ref_count()); + jint l_ref_count = ref_count(); + ADIM_guarantee(l_ref_count > 0, "must be positive: l_ref_count=%d, ref_count=%d", l_ref_count, ref_count()); Thread * const Self = THREAD; assert(Self->is_Java_thread(), "invariant"); @@ -770,7 +774,8 @@ // In the future we should reconcile EnterI() and ReenterI(). void ObjectMonitor::ReenterI(Thread * Self, ObjectWaiter * SelfNode) { - ADIM_guarantee(ref_count() > 0, "must be positive: ref_count=%d", ref_count()); + jint l_ref_count = ref_count(); + ADIM_guarantee(l_ref_count > 0, "must be positive: l_ref_count=%d, ref_count=%d", l_ref_count, ref_count()); assert(Self != NULL, "invariant"); assert(SelfNode != NULL, "invariant"); @@ -990,7 +995,7 @@ void* cur = _owner; if (THREAD->is_lock_owned((address)cur)) { assert(_recursions == 0, "invariant"); - set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. + simply_set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. _recursions = 0; } else { // Apparent unbalanced locking ... @@ -1036,8 +1041,6 @@ // release semantics: prior loads and stores from within the critical section // must not float (reorder) past the following store that drops the lock. - // On SPARC that requires MEMBAR #loadstore|#storestore. - // But of course in TSO #loadstore|#storestore is not required. if (AsyncDeflateIdleMonitors) { set_owner_from(NULL, Self); } else { @@ -1253,7 +1256,7 @@ void* cur = _owner; if (THREAD->is_lock_owned((address)cur)) { assert(_recursions == 0, "internal state error"); - set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. + simply_set_owner_from_BasicLock(Self, cur); // Convert from BasicLock* to Thread*. _recursions = 0; } } @@ -1303,7 +1306,7 @@ } void* cur = _owner; if (THREAD->is_lock_owned((address)cur)) { - set_owner_from_BasicLock(THREAD, cur); // Convert from BasicLock* to Thread*. + simply_set_owner_from_BasicLock(THREAD, cur); // Convert from BasicLock* to Thread*. _recursions = 0; return true; } @@ -2072,12 +2075,6 @@ DEBUG_ONLY(InitDone = true;) } -// For internal use by ObjectSynchronizer::monitors_iterate(). -ObjectMonitorHandle::ObjectMonitorHandle(ObjectMonitor * om_ptr) { - om_ptr->inc_ref_count(); - _om_ptr = om_ptr; -} - ObjectMonitorHandle::~ObjectMonitorHandle() { if (_om_ptr != NULL) { _om_ptr->dec_ref_count(); @@ -2099,7 +2096,7 @@ guarantee(mark.has_monitor(), "sanity check: mark=" INTPTR_FORMAT, mark.value()); - ObjectMonitor * om_ptr = mark.monitor(); + ObjectMonitor* om_ptr = mark.monitor(); om_ptr->inc_ref_count(); if (AsyncDeflateIdleMonitors) { @@ -2141,7 +2138,9 @@ } // For internal use by ObjectSynchronizer::inflate(). -void ObjectMonitorHandle::set_om_ptr(ObjectMonitor * om_ptr) { +// This function is only used when we don't have to worry about async +// deflation of the specified ObjectMonitor*. +void ObjectMonitorHandle::set_om_ptr(ObjectMonitor* om_ptr) { if (_om_ptr == NULL) { ADIM_guarantee(om_ptr != NULL, "cannot clear an unset om_ptr"); om_ptr->inc_ref_count(); @@ -2153,6 +2152,52 @@ } } +// Save the specified ObjectMonitor* if it is safe, i.e., not being +// async deflated. +// +// This function returns true if the ObjectMonitor* has been safely +// saved. This function returns false if the specified ObjectMonitor* +// is NULL or if we have lost a race with async deflation; the caller +// can retry as appropriate. +bool ObjectMonitorHandle::set_om_ptr_if_safe(ObjectMonitor* om_ptr) { + if (om_ptr == NULL) { + return false; // Nothing to save if input is NULL + } + + om_ptr->inc_ref_count(); + + if (AsyncDeflateIdleMonitors) { + if (om_ptr->owner_is_DEFLATER_MARKER() && om_ptr->ref_count() <= 0) { + // Async deflation is in progress and our ref_count increment + // above lost the race to async deflation. + om_ptr->dec_ref_count(); + return false; + } + if (om_ptr->ref_count() <= 0) { + // Async deflation is in the process of bailing out, but has not + // yet restored the ref_count field so we return false to force + // a retry. We want a positive ref_count value for a true return. + om_ptr->dec_ref_count(); + return false; + } + // Unlike save_om_ptr(), we don't have context to determine if + // the ObjectMonitor has been deflated and reused for another + // object. + } + + ADIM_guarantee(_om_ptr == NULL, "sanity check: _om_ptr=" INTPTR_FORMAT, + p2i(_om_ptr)); + _om_ptr = om_ptr; + return true; +} + +// Unset the _om_ptr field and decrement the ref_count field. +void ObjectMonitorHandle::unset_om_ptr() { + ADIM_guarantee(_om_ptr != NULL, "_om_ptr must not be NULL"); + _om_ptr->dec_ref_count(); + _om_ptr = NULL; +} + void ObjectMonitor::print_on(outputStream* st) const { // The minimal things to print for markWord printing, more can be added for debugging and logging. st->print("{contentions=0x%08x,waiters=0x%08x" @@ -2201,7 +2246,7 @@ // } // void ObjectMonitor::print_debug_style_on(outputStream* st) const { - st->print_cr("(ObjectMonitor *) " INTPTR_FORMAT " = {", p2i(this)); + st->print_cr("(ObjectMonitor*) " INTPTR_FORMAT " = {", p2i(this)); st->print_cr(" _header = " INTPTR_FORMAT, header().value()); st->print_cr(" _object = " INTPTR_FORMAT, p2i(_object)); st->print(" _allocation_state = "); --- old/src/hotspot/share/runtime/objectMonitor.hpp 2019-11-04 14:59:02.000000000 -0500 +++ new/src/hotspot/share/runtime/objectMonitor.hpp 2019-11-04 14:59:02.000000000 -0500 @@ -274,10 +274,12 @@ bool owner_is_DEFLATER_MARKER(); // Set _owner field to new_value; current value must match old_value. void set_owner_from(void* new_value, void* old_value); - // Set _owner field to new_value; current value must match old_value1 or old_value2. - void set_owner_from(void* new_value, void* old_value1, void* old_value2); - // Set _owner field to self; current value must match basic_lock_p. - void set_owner_from_BasicLock(Thread* self, void* basic_lock_p); + // Simply set _owner field to new_value; current value must match old_value. + void simply_set_owner_from(void* new_value, void* old_value); + // Simply set _owner field to new_value; current value must match old_value1 or old_value2. + void simply_set_owner_from(void* new_value, void* old_value1, void* old_value2); + // Simply set _owner field to self; current value must match basic_lock_p. + void simply_set_owner_from_BasicLock(Thread* self, void* basic_lock_p); // Try to set _owner field to new_value if the current value matches // old_value. Otherwise, does not change the _owner field. void* try_set_owner_from(void* new_value, void* old_value); @@ -327,7 +329,6 @@ void set_allocation_state(AllocationState s); AllocationState allocation_state() const; bool is_free() const; - bool is_active() const; bool is_old() const; bool is_new() const; void dec_ref_count(); @@ -383,9 +384,11 @@ // Save the ObjectMonitor* associated with the specified markWord and // increment the ref_count. bool save_om_ptr(oop object, markWord mark); + // Save the specified ObjectMonitor* if safe and increment the ref_count. + bool set_om_ptr_if_safe(ObjectMonitor* om_ptr); + // Unset the _om_ptr field and decrement the ref_count. + void unset_om_ptr(); - // For internal used by ObjectSynchronizer::monitors_iterate(). - ObjectMonitorHandle(ObjectMonitor* _om_ptr); // For internal use by ObjectSynchronizer::inflate(). void set_om_ptr(ObjectMonitor* om_ptr); }; --- old/src/hotspot/share/runtime/objectMonitor.inline.hpp 2019-11-04 14:59:03.000000000 -0500 +++ new/src/hotspot/share/runtime/objectMonitor.inline.hpp 2019-11-04 14:59:03.000000000 -0500 @@ -62,13 +62,14 @@ // This accessor is called when we really need to know if the owner // field == DEFLATER_MARKER and any non-NULL value won't do the trick. inline bool ObjectMonitor::owner_is_DEFLATER_MARKER() { - return OrderAccess::load_acquire(&_owner) == DEFLATER_MARKER; + return _owner == DEFLATER_MARKER; } inline void ObjectMonitor::clear() { assert(Atomic::load(&_header).value() != 0, "must be non-zero"); assert(_owner == NULL, "must be NULL: owner=" INTPTR_FORMAT, p2i(_owner)); - assert(ref_count() == 0, "must be 0: ref_count=%d", ref_count()); + DEBUG_ONLY(jint l_ref_count = ref_count();) + assert(l_ref_count == 0, "must be 0: l_ref_count=%d, ref_count=%d", l_ref_count, ref_count()); Atomic::store(markWord::zero(), &_header); @@ -91,13 +92,14 @@ guarantee(_owner == NULL || _owner == DEFLATER_MARKER, "must be NULL or DEFLATER_MARKER: owner=" INTPTR_FORMAT, p2i(_owner)); - guarantee(ref_count() <= 0, "must be <= 0: ref_count=%d", ref_count()); + jint l_ref_count = ref_count(); + guarantee(l_ref_count <= 0, "must be <= 0: l_ref_count=%d, ref_count=%d", l_ref_count, ref_count()); } assert(_contentions == 0, "must be 0: contentions=%d", _contentions); assert(_waiters == 0, "must be 0: waiters=%d", _waiters); assert(_recursions == 0, "must be 0: recursions=" INTX_FORMAT, _recursions); assert(_object != NULL, "must be non-NULL"); - + set_allocation_state(Free); _object = NULL; } @@ -124,39 +126,55 @@ void* prev = Atomic::cmpxchg(new_value, &_owner, old_value); ADIM_guarantee(prev == old_value, "unexpected prev owner=" INTPTR_FORMAT ", expected=" INTPTR_FORMAT, p2i(prev), p2i(old_value)); - log_trace(monitorinflation, owner)("mid=" INTPTR_FORMAT ", prev=" - INTPTR_FORMAT ", new=" INTPTR_FORMAT, - p2i(this), p2i(prev), p2i(new_value)); + log_trace(monitorinflation, owner)("set_owner_from(): mid=" INTPTR_FORMAT + ", prev=" INTPTR_FORMAT ", new=" + INTPTR_FORMAT, p2i(this), p2i(prev), + p2i(new_value)); } -// Set _owner field to new_value; current value must match old_value1 or old_value2. -inline void ObjectMonitor::set_owner_from(void* new_value, void* old_value1, void* old_value2) { - void* prev = Atomic::cmpxchg(new_value, &_owner, old_value1); - if (prev != old_value1) { - prev = Atomic::cmpxchg(new_value, &_owner, old_value2); - } +// Simply set _owner field to new_value; current value must match old_value. +// (Simple means no memory sync needed.) +inline void ObjectMonitor::simply_set_owner_from(void* new_value, void* old_value) { + void* prev = _owner; + ADIM_guarantee(prev == old_value, "unexpected prev owner=" INTPTR_FORMAT + ", expected=" INTPTR_FORMAT, p2i(prev), p2i(old_value)); + _owner = new_value; + log_trace(monitorinflation, owner)("simply_set_owner_from(): mid=" + INTPTR_FORMAT ", prev=" INTPTR_FORMAT + ", new=" INTPTR_FORMAT, p2i(this), + p2i(prev), p2i(new_value)); +} + +// Simply set _owner field to new_value; current value must match old_value1 or old_value2. +// (Simple means no memory sync needed.) +inline void ObjectMonitor::simply_set_owner_from(void* new_value, void* old_value1, void* old_value2) { + void* prev = _owner; ADIM_guarantee(prev == old_value1 || prev == old_value2, "unexpected prev owner=" INTPTR_FORMAT ", expected1=" INTPTR_FORMAT " or expected2=" INTPTR_FORMAT, p2i(prev), p2i(old_value1), p2i(old_value2)); - log_trace(monitorinflation, owner)("mid=" INTPTR_FORMAT ", prev=" - INTPTR_FORMAT ", new=" INTPTR_FORMAT, - p2i(this), p2i(prev), p2i(new_value)); + _owner = new_value; + log_trace(monitorinflation, owner)("simply_set_owner_from(old1=" INTPTR_FORMAT + ", old2=" INTPTR_FORMAT "): mid=" + INTPTR_FORMAT ", prev=" INTPTR_FORMAT + ", new=" INTPTR_FORMAT, p2i(old_value1), + p2i(old_value2), p2i(this), p2i(prev), + p2i(new_value)); } -// Set _owner field to self; current value must match basic_lock_p. -inline void ObjectMonitor::set_owner_from_BasicLock(Thread* self, void* basic_lock_p) { - assert(self->is_lock_owned((address)basic_lock_p), "self=" INTPTR_FORMAT - " must own basic_lock_p=" INTPTR_FORMAT, p2i(self), p2i(basic_lock_p)); +// Simply set _owner field to self; current value must match basic_lock_p. +inline void ObjectMonitor::simply_set_owner_from_BasicLock(Thread* self, void* basic_lock_p) { void* prev = _owner; ADIM_guarantee(prev == basic_lock_p, "unexpected prev owner=" INTPTR_FORMAT ", expected=" INTPTR_FORMAT, p2i(prev), p2i(basic_lock_p)); // Non-null owner field to non-null owner field is safe without // cmpxchg() as long as all readers can tolerate either flavor. _owner = self; - log_trace(monitorinflation, owner)("mid=" INTPTR_FORMAT ", prev=" - INTPTR_FORMAT ", new=" INTPTR_FORMAT, - p2i(this), p2i(prev), p2i(self)); + log_trace(monitorinflation, owner)("simply_set_owner_from_BasicLock(): mid=" + INTPTR_FORMAT ", prev=" INTPTR_FORMAT + ", new=" INTPTR_FORMAT ", basic_lock_p=" + INTPTR_FORMAT, p2i(this), p2i(prev), + p2i(self), p2i(basic_lock_p)); } // Try to set _owner field to new_value if the current value matches @@ -164,9 +182,10 @@ inline void* ObjectMonitor::try_set_owner_from(void* new_value, void* old_value) { void* prev = Atomic::cmpxchg(new_value, &_owner, old_value); if (prev == old_value) { - log_trace(monitorinflation, owner)("mid=" INTPTR_FORMAT ", prev=" - INTPTR_FORMAT ", new=" INTPTR_FORMAT, - p2i(this), p2i(prev), p2i(new_value)); + log_trace(monitorinflation, owner)("try_set_owner_from(): mid=" + INTPTR_FORMAT ", prev=" INTPTR_FORMAT + ", new=" INTPTR_FORMAT, p2i(this), + p2i(prev), p2i(new_value)); } return prev; } @@ -183,10 +202,6 @@ return _allocation_state == Free; } -inline bool ObjectMonitor::is_active() const { - return !is_free(); -} - inline bool ObjectMonitor::is_old() const { return _allocation_state == Old; } @@ -200,8 +215,9 @@ // counter is volatile. Atomic::dec(&_ref_count); // Can be negative as part of async deflation protocol. - ADIM_guarantee(AsyncDeflateIdleMonitors || ref_count() >= 0, - "sanity check: ref_count=%d", ref_count()); + jint l_ref_count = ref_count(); + ADIM_guarantee(AsyncDeflateIdleMonitors || l_ref_count >= 0, + "sanity check: l_ref_count=%d, ref_count=%d", l_ref_count, ref_count()); } inline void ObjectMonitor::inc_ref_count() { @@ -210,12 +226,13 @@ // async deflation protocol. Atomic::inc(&_ref_count); // Can be negative as part of async deflation protocol. - ADIM_guarantee(AsyncDeflateIdleMonitors || ref_count() > 0, - "sanity check: ref_count=%d", ref_count()); + jint l_ref_count = ref_count(); + ADIM_guarantee(AsyncDeflateIdleMonitors || l_ref_count > 0, + "sanity check: l_ref_count=%d, ref_count=%d", l_ref_count, ref_count()); } inline jint ObjectMonitor::ref_count() const { - return OrderAccess::load_acquire(&_ref_count); + return _ref_count; } #endif // SHARE_RUNTIME_OBJECTMONITOR_INLINE_HPP --- old/src/hotspot/share/runtime/synchronizer.cpp 2019-11-04 14:59:04.000000000 -0500 +++ new/src/hotspot/share/runtime/synchronizer.cpp 2019-11-04 14:59:04.000000000 -0500 @@ -149,6 +149,8 @@ // Return true if the ObjectMonitor's next field is marked. // Otherwise returns false. static bool is_next_marked(ObjectMonitor* om) { + // Use load_acquire() since _next_om fields are updated with a + // release_store(). return ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & 0x1) != 0; } @@ -163,8 +165,7 @@ // returned. Otherwise false is returned. static bool mark_next(ObjectMonitor* om, ObjectMonitor** next_p) { // Get current next field without any marking value. - ObjectMonitor* next = (ObjectMonitor*) - ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1); + ObjectMonitor* next = (ObjectMonitor*)((intptr_t)om->_next_om & ~0x1); if (Atomic::cmpxchg(mark_om_ptr(next), &om->_next_om, next) != next) { return false; // Could not mark the next field or it was already marked. } @@ -197,12 +198,12 @@ static bool mark_list_head(ObjectMonitor* volatile * list_p, ObjectMonitor** mid_p, ObjectMonitor** next_p) { while (true) { - ObjectMonitor* mid = OrderAccess::load_acquire(list_p); + ObjectMonitor* mid = *list_p; if (mid == NULL) { return false; // The list is empty so nothing to mark. } if (mark_next(mid, next_p)) { - if (OrderAccess::load_acquire(list_p) != mid) { + if (*list_p != mid) { // The list head changed so we have to retry. set_next(mid, *next_p); // unmark mid continue; @@ -217,9 +218,28 @@ // Return the unmarked next field in an ObjectMonitor. Note: the next // field may or may not have been marked originally. static ObjectMonitor* unmarked_next(ObjectMonitor* om) { + // Use load_acquire() since _next_om fields are updated with a + // release_store(). return (ObjectMonitor*)((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1); } +// Mark the next ObjectMonitor for traversal. The current ObjectMonitor +// is unmarked after the next ObjectMonitor is marked. *cur_p and *next_p +// are updated to their next values in the list traversal. *cur_p is set +// to NULL when the end of the list is reached. +static void mark_next_for_traversal(ObjectMonitor** cur_p, ObjectMonitor** next_p) { + ObjectMonitor* prev = *cur_p; // Save current for unmarking. + if (*next_p == NULL) { // Reached the end of the list. + set_next(prev, NULL); // Unmark previous. + *cur_p = NULL; // Tell the caller we are done. + return; + } + (void)mark_next_loop(*next_p); // Mark next. + set_next(prev, *next_p); // Unmark previous. + *cur_p = *next_p; // Update current. + *next_p = unmarked_next(*cur_p); // Update next. +} + // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is // the last ObjectMonitor in the list and there are 'count' on the list. // Also updates the specified *count_p. @@ -227,7 +247,7 @@ int count, ObjectMonitor* volatile* list_p, volatile int* count_p) { while (true) { - ObjectMonitor* cur = OrderAccess::load_acquire(list_p); + ObjectMonitor* cur = *list_p; // Prepend list to *list_p. ObjectMonitor* next = NULL; if (!mark_next(tail, &next)) { @@ -320,7 +340,9 @@ if (mark_list_head(list_p, &cur, &next)) { // List head is now marked so we can safely switch it. set_next(m, cur); // m now points to cur (and unmarks m) - OrderAccess::release_store(list_p, m); // Switch list head to unmarked m. + *list_p = m; // Switch list head to unmarked m. + // mark_list_head() used cmpxchg() above, switching list head can be lazier: + OrderAccess::storestore(); set_next(cur, next); // Unmark the previous list head. break; } @@ -360,8 +382,10 @@ } // Switch marked list head to next (which unmarks the list head, but // leaves take marked): - OrderAccess::release_store(list_p, next); + *list_p = next; Atomic::dec(count_p); + // mark_list_head() used cmpxchg() above, switching list head can be lazier: + OrderAccess::storestore(); // Unmark take, but leave the next value for any lagging list // walkers. It will get cleaned up when take is prepended to // the in-use list: @@ -1211,44 +1235,38 @@ // Visitors ... void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) { - PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list); + PaddedObjectMonitor* block = g_block_list; while (block != NULL) { assert(block->object() == CHAINMARKER, "must be a block header"); for (int i = _BLOCKSIZE - 1; i > 0; i--) { ObjectMonitor* mid = (ObjectMonitor *)(block + i); - if (mid->is_active()) { - ObjectMonitorHandle omh(mid); - - if (mid->object() == NULL || - (AsyncDeflateIdleMonitors && mid->ref_count() < 0)) { + ObjectMonitorHandle omh; + if (!mid->is_free() && omh.set_om_ptr_if_safe(mid)) { + // The ObjectMonitor* is not free and it has been made safe. + if (mid->object() == NULL) { // Only process with closure if the object is set. - // For async deflation, race here if monitor is not owned! - // The above ref_count bump (in ObjectMonitorHandle ctr) - // will cause subsequent async deflation to skip it. - // However, previous or concurrent async deflation is a race - // so skip this ObjectMonitor if it is being async deflated. continue; } closure->do_monitor(mid); } } - // unmarked_next() is not needed with g_block_list (no next field marking). - block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om); + // unmarked_next() is not needed with g_block_list (no next field + // marking) and no load_acquire() needed because _next_om is + // updated before g_block_list is changed with cmpxchg(). + block = (PaddedObjectMonitor*)block->_next_om; } } static bool monitors_used_above_threshold() { - if (OrderAccess::load_acquire(&g_om_population) == 0) { + if (g_om_population == 0) { return false; } if (MonitorUsedDeflationThreshold > 0) { - int monitors_used = OrderAccess::load_acquire(&g_om_population) - - OrderAccess::load_acquire(&g_om_free_count); + int monitors_used = g_om_population - g_om_free_count; if (HandshakeAfterDeflateIdleMonitors) { - monitors_used -= OrderAccess::load_acquire(&g_om_wait_count); + monitors_used -= g_om_wait_count; } - int monitor_usage = (monitors_used * 100LL) / - OrderAccess::load_acquire(&g_om_population); + int monitor_usage = (monitors_used * 100LL) / g_om_population; return monitor_usage > MonitorUsedDeflationThreshold; } return false; @@ -1279,10 +1297,9 @@ _last_async_deflation_time_ns = os::javaTimeNanos(); return true; } - int monitors_used = OrderAccess::load_acquire(&g_om_population) - - OrderAccess::load_acquire(&g_om_free_count); + int monitors_used = g_om_population - g_om_free_count; if (HandshakeAfterDeflateIdleMonitors) { - monitors_used -= OrderAccess::load_acquire(&g_om_wait_count); + monitors_used -= g_om_wait_count; } if (is_MonitorBound_exceeded(monitors_used)) { // Not enough ObjectMonitors on the global free list. @@ -1319,19 +1336,20 @@ void ObjectSynchronizer::global_used_oops_do(OopClosure* f) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); - list_oops_do(OrderAccess::load_acquire(&g_om_in_use_list), OrderAccess::load_acquire(&g_om_in_use_count), f); + list_oops_do(g_om_in_use_list, g_om_in_use_count, f); } void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); - list_oops_do(OrderAccess::load_acquire(&thread->om_in_use_list), OrderAccess::load_acquire(&thread->om_in_use_count), f); + list_oops_do(thread->om_in_use_list, thread->om_in_use_count, f); } void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, int count, OopClosure* f) { assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); // The oops_do() phase does not overlap with monitor deflation // so no need to update the ObjectMonitor's ref_count for this - // ObjectMonitor* use. + // ObjectMonitor* use and no need to mark ObjectMonitors for the + // list traversal. for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) { if (mid->object() != NULL) { f->do_oop((oop*)mid->object_addr()); @@ -1443,7 +1461,7 @@ // If the muxTry() fails then drop immediately into case 3. // If we're using thread-local free lists then try // to reprovision the caller's free list. - if (OrderAccess::load_acquire(&g_free_list) != NULL) { + if (g_free_list != NULL) { // Reprovision the thread's om_free_list. // Use bulk transfers to reduce the allocation rate and heat // on various locks. @@ -1465,8 +1483,9 @@ // proper value. Atomic::add(max_jint, &take->_ref_count); - assert(take->ref_count() >= 0, "must not be negative: ref_count=%d", - take->ref_count()); + DEBUG_ONLY(jint l_ref_count = take->ref_count();) + assert(l_ref_count >= 0, "must not be negative: l_ref_count=%d, ref_count=%d", + l_ref_count, take->ref_count()); } } take->Recycle(); @@ -1476,12 +1495,11 @@ assert(take->is_free(), "invariant"); om_release(self, take, false); } - self->om_free_provision += 1 + (self->om_free_provision/2); + self->om_free_provision += 1 + (self->om_free_provision / 2); if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE; if (!AsyncDeflateIdleMonitors && - is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) - - OrderAccess::load_acquire(&g_om_free_count))) { + is_MonitorBound_exceeded(g_om_population - g_om_free_count)) { // Not enough ObjectMonitors on the global free list. // We can't safely induce a STW safepoint from om_alloc() as our thread // state may not be appropriate for such activities and callers may hold @@ -1516,12 +1534,12 @@ // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; } for (int i = 1; i < _BLOCKSIZE; i++) { - OrderAccess::release_store(&temp[i]._next_om, (ObjectMonitor*)&temp[i+1]); + temp[i]._next_om = (ObjectMonitor*)&temp[i + 1]; assert(temp[i].is_free(), "invariant"); } // terminate the last monitor as the end of list - OrderAccess::release_store(&temp[_BLOCKSIZE - 1]._next_om, (ObjectMonitor*)NULL); + temp[_BLOCKSIZE - 1]._next_om = (ObjectMonitor*)NULL; // Element [0] is reserved for global list linkage temp[0].set_object(CHAINMARKER); @@ -1575,7 +1593,9 @@ if (cur_mid_in_use == NULL) { // mid is the list head and it is marked. Switch the list head // to next which unmarks the list head, but leaves mid marked: - OrderAccess::release_store(&self->om_in_use_list, next); + self->om_in_use_list = next; + // mark_list_head() used cmpxchg() above, switching list head can be lazier: + OrderAccess::storestore(); } else { // mid and cur_mid_in_use are marked. Switch cur_mid_in_use's // next field to next which unmarks cur_mid_in_use, but leaves @@ -1670,7 +1690,7 @@ cur_om = unmarked_next(in_use_tail); continue; } - if (!cur_om->is_active()) { + if (cur_om->is_free()) { // cur_om was deflated and the allocation state was changed // to Free while it was marked. We happened to see it just // after it was unmarked (and added to the free list). @@ -1683,21 +1703,20 @@ cur_om = unmarked_next(cur_om); } guarantee(in_use_tail != NULL, "invariant"); - int l_om_in_use_count = OrderAccess::load_acquire(&self->om_in_use_count); + int l_om_in_use_count = self->om_in_use_count; ADIM_guarantee(l_om_in_use_count == in_use_count, "in-use counts don't " "match: l_om_in_use_count=%d, in_use_count=%d", l_om_in_use_count, in_use_count); - // Clear the in-use count before unmarking the in-use list head - // to avoid races: - OrderAccess::release_store(&self->om_in_use_count, 0); + self->om_in_use_count = 0; // Clear the in-use list head (which also unmarks it): - OrderAccess::release_store(&self->om_in_use_list, (ObjectMonitor*)NULL); - // Unmark the disconnected list head: + self->om_in_use_list = (ObjectMonitor*)NULL; + // mark_list_head() used cmpxchg() above, clearing the disconnected list head can be lazier: + OrderAccess::storestore(); set_next(in_use_list, next); } int free_count = 0; - ObjectMonitor* free_list = OrderAccess::load_acquire(&self->om_free_list); + ObjectMonitor* free_list = self->om_free_list; ObjectMonitor* free_tail = NULL; if (free_list != NULL) { // The thread is going away. Set 'free_tail' to the last per-thread free @@ -1710,12 +1729,13 @@ guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss)); } guarantee(free_tail != NULL, "invariant"); - int l_om_free_count = OrderAccess::load_acquire(&self->om_free_count); + int l_om_free_count = self->om_free_count; ADIM_guarantee(l_om_free_count == free_count, "free counts don't match: " "l_om_free_count=%d, free_count=%d", l_om_free_count, free_count); - OrderAccess::release_store(&self->om_free_list, (ObjectMonitor*)NULL); - OrderAccess::release_store(&self->om_free_count, 0); + self->om_free_count = 0; + self->om_free_list = NULL; + OrderAccess::storestore(); // Lazier memory is okay for list walkers. } if (free_tail != NULL) { @@ -1901,9 +1921,9 @@ // that it has stack-locked -- as might happen in wait() -- directly // with CAS. That is, we can avoid the xchg-NULL .... ST idiom. if (AsyncDeflateIdleMonitors) { - m->set_owner_from(mark.locker(), NULL, DEFLATER_MARKER); + m->simply_set_owner_from(mark.locker(), NULL, DEFLATER_MARKER); } else { - m->set_owner_from(mark.locker(), NULL); + m->simply_set_owner_from(mark.locker(), NULL); } m->set_object(object); // TODO-FIXME: assert BasicLock->dhw != 0. @@ -2043,6 +2063,15 @@ set_is_async_deflation_requested(true); MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag); ml.notify_all(); + + if (log_is_enabled(Debug, monitorinflation)) { + // exit_globals()'s call to audit_and_print_stats() is done + // at the Info level and not at a safepoint. + // For safepoint based deflation, audit_and_print_stats() is called + // in ObjectSynchronizer::finish_deflate_idle_monitors() at the + // Debug level at a safepoint. + ObjectSynchronizer::audit_and_print_stats(false /* on_exit */); + } } // Deflate a single monitor if not in-use @@ -2245,8 +2274,9 @@ // proper value (which may not be what we saw above): Atomic::add(max_jint, &mid->_ref_count); - assert(mid->ref_count() >= 0, "must not be negative: ref_count=%d", - mid->ref_count()); + DEBUG_ONLY(jint l_ref_count = mid->ref_count();) + assert(l_ref_count >= 0, "must not be negative: l_ref_count=%d, ref_count=%d", + l_ref_count, mid->ref_count()); return false; } @@ -2297,7 +2327,9 @@ if (cur_mid_in_use == NULL) { // mid is the list head and it is marked. Switch the list head // to next which unmarks the list head, but leaves mid marked: - OrderAccess::release_store(list_p, next); + *list_p = next; + // mark_list_head() used cmpxchg() above, switching list head can be lazier: + OrderAccess::storestore(); } else { // mid is marked. Switch cur_mid_in_use's next field to next // which is safe because we have no parallel list deletions, @@ -2398,7 +2430,9 @@ // mid is the list head and it is marked. Switch the list head // to next which is also marked (if not NULL) and also leave // mid marked: - OrderAccess::release_store(list_p, next); + *list_p = next; + // mark_list_head() used cmpxchg() above, switching list head can be lazier: + OrderAccess::storestore(); } else { ObjectMonitor* marked_next = mark_om_ptr(next); // mid and cur_mid_in_use are marked. Switch cur_mid_in_use's @@ -2438,8 +2472,7 @@ next = next_next; if (SafepointSynchronize::is_synchronizing() && - cur_mid_in_use != OrderAccess::load_acquire(list_p) && - cur_mid_in_use->is_old()) { + cur_mid_in_use != *list_p && cur_mid_in_use->is_old()) { // If a safepoint has started and cur_mid_in_use is not the list // head and is old, then it is safe to use as saved state. Return // to the caller before blocking. @@ -2468,11 +2501,12 @@ } void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) { - OrderAccess::release_store(&counters->n_in_use, 0); // currently associated with objects - OrderAccess::release_store(&counters->n_in_circulation, 0); // extant - OrderAccess::release_store(&counters->n_scavenged, 0); // reclaimed (global and per-thread) - OrderAccess::release_store(&counters->per_thread_scavenged, 0); // per-thread scavenge total - counters->per_thread_times = 0.0; // per-thread scavenge times + counters->n_in_use = 0; // currently associated with objects + counters->n_in_circulation = 0; // extant + counters->n_scavenged = 0; // reclaimed (global and per-thread) + counters->per_thread_scavenged = 0; // per-thread scavenge total + counters->per_thread_times = 0.0; // per-thread scavenge times + OrderAccess::storestore(); // flush inits for worker threads } void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) { @@ -2501,12 +2535,12 @@ // For moribund threads, scan g_om_in_use_list int deflated_count = 0; - if (OrderAccess::load_acquire(&g_om_in_use_list) != NULL) { + if (g_om_in_use_list != NULL) { // Update n_in_circulation before g_om_in_use_count is updated by deflation. - Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_circulation); + Atomic::add(g_om_in_use_count, &counters->n_in_circulation); deflated_count = deflate_monitor_list(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p); - Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_use); + Atomic::add(g_om_in_use_count, &counters->n_in_use); } if (free_head_p != NULL) { @@ -2560,6 +2594,12 @@ if (count > 0) { log_debug(monitorinflation)("did async deflation of idle monitors for %d thread(s).", count); } + + log_info(monitorinflation)("async g_om_population=%d, g_om_in_use_count=%d, " + "g_om_free_count=%d, g_om_wait_count=%d", + g_om_population, g_om_in_use_count, + g_om_free_count, g_om_wait_count); + // The ServiceThread's async deflation request has been processed. set_is_async_deflation_requested(false); @@ -2572,10 +2612,13 @@ ObjectMonitor* list = g_wait_list; ADIM_guarantee(list != NULL, "g_wait_list must not be NULL"); int count = g_om_wait_count; - g_wait_list = NULL; g_om_wait_count = 0; + g_wait_list = NULL; + OrderAccess::storestore(); // Lazier memory sync is okay for list walkers. - // Find the tail for prepend_list_to_common(). + // Find the tail for prepend_list_to_common(). No need to mark + // ObjectMonitors for this list walk since only the deflater + // thread manages the wait list. int l_count = 0; ObjectMonitor* tail = NULL; for (ObjectMonitor* n = list; n != NULL; n = unmarked_next(n)) { @@ -2629,9 +2672,9 @@ } if (is_global) { - OM_PERFDATA_OP(MonExtant, set_value(OrderAccess::load_acquire(&g_om_in_use_count))); + OM_PERFDATA_OP(MonExtant, set_value(g_om_in_use_count)); } else { - OM_PERFDATA_OP(MonExtant, inc(OrderAccess::load_acquire(&target->om_in_use_count))); + OM_PERFDATA_OP(MonExtant, inc(target->om_in_use_count)); } do { @@ -2711,37 +2754,37 @@ // monitors. Note: if the work is split among more than one // worker thread, then the reported time will likely be more // than a beginning to end measurement of the phase. - // Note: AsyncDeflateIdleMonitors only deflates per-thread idle - // monitors at a safepoint when a special deflation has been requested. - log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", - counters->per_thread_times, - OrderAccess::load_acquire(&counters->per_thread_scavenged)); + log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", counters->per_thread_times, counters->per_thread_scavenged); bool needs_special_deflation = is_special_deflation_requested(); - if (!AsyncDeflateIdleMonitors || needs_special_deflation) { - // AsyncDeflateIdleMonitors does not use these counters unless - // there is a special deflation request. - - OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged)); - OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation)); + if (AsyncDeflateIdleMonitors && !needs_special_deflation) { + // Nothing to do when idle ObjectMonitors are deflated using + // a JavaThread unless a special deflation has been requested. + return; } if (log_is_enabled(Debug, monitorinflation)) { // exit_globals()'s call to audit_and_print_stats() is done - // at the Info level. + // at the Info level and not at a safepoint. + // For async deflation, audit_and_print_stats() is called in + // ObjectSynchronizer::do_safepoint_work() at the Debug level + // at a safepoint. ObjectSynchronizer::audit_and_print_stats(false /* on_exit */); } else if (log_is_enabled(Info, monitorinflation)) { log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, " "g_om_free_count=%d, g_om_wait_count=%d", - OrderAccess::load_acquire(&g_om_population), - OrderAccess::load_acquire(&g_om_in_use_count), - OrderAccess::load_acquire(&g_om_free_count), - OrderAccess::load_acquire(&g_om_wait_count)); + g_om_population, g_om_in_use_count, + g_om_free_count, g_om_wait_count); } ForceMonitorScavenge = 0; // Reset + + OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged)); + OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation)); + GVars.stw_random = os::random(); GVars.stw_cycle++; + if (needs_special_deflation) { set_is_special_deflation_requested(false); // special deflation is done } @@ -2765,10 +2808,10 @@ } // Update n_in_circulation before om_in_use_count is updated by deflation. - Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_circulation); + Atomic::add(thread->om_in_use_count, &counters->n_in_circulation); int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p); - Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_use); + Atomic::add(thread->om_in_use_count, &counters->n_in_use); if (free_head_p != NULL) { // Move the deflated ObjectMonitors back to the global free list. @@ -2898,19 +2941,16 @@ ls->print_cr("Checking global lists:"); // Check g_om_population: - if (OrderAccess::load_acquire(&g_om_population) == chk_om_population) { + if (g_om_population == chk_om_population) { ls->print_cr("g_om_population=%d equals chk_om_population=%d", - OrderAccess::load_acquire(&g_om_population), - chk_om_population); + g_om_population, chk_om_population); } else { // With lock free access to the monitor lists, it is possible for // log_monitor_list_counts() to return a value that doesn't match // g_om_population. So far a higher value has been seen in testing // so something is being double counted by log_monitor_list_counts(). ls->print_cr("WARNING: g_om_population=%d is not equal to " - "chk_om_population=%d", - OrderAccess::load_acquire(&g_om_population), - chk_om_population); + "chk_om_population=%d", g_om_population, chk_om_population); } // Check g_om_in_use_list and g_om_in_use_count: @@ -3001,23 +3041,30 @@ void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out, int *error_cnt_p) { int chk_om_free_count = 0; - for (ObjectMonitor* n = OrderAccess::load_acquire(&g_free_list); n != NULL; n = unmarked_next(n)) { - chk_free_entry(NULL /* jt */, n, out, error_cnt_p); - chk_om_free_count++; + ObjectMonitor* cur = NULL; + ObjectMonitor* next = NULL; + if (mark_list_head(&g_free_list, &cur, &next)) { + // Marked the global free list head so process the list. + while (true) { + chk_free_entry(NULL /* jt */, cur, out, error_cnt_p); + chk_om_free_count++; + + mark_next_for_traversal(&cur, &next); + if (cur == NULL) { + break; + } + } } - if (OrderAccess::load_acquire(&g_om_free_count) == chk_om_free_count) { + if (g_om_free_count == chk_om_free_count) { out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d", - OrderAccess::load_acquire(&g_om_free_count), - chk_om_free_count); + g_om_free_count, chk_om_free_count); } else { // With lock free access to g_free_list, it is possible for an // ObjectMonitor to be prepended to g_free_list after we started // calculating chk_om_free_count so g_om_free_count may not // match anymore. out->print_cr("WARNING: g_om_free_count=%d is not equal to " - "chk_om_free_count=%d", - OrderAccess::load_acquire(&g_om_free_count), - chk_om_free_count); + "chk_om_free_count=%d", g_om_free_count, chk_om_free_count); } } @@ -3025,20 +3072,27 @@ void ObjectSynchronizer::chk_global_wait_list_and_count(outputStream * out, int *error_cnt_p) { int chk_om_wait_count = 0; - for (ObjectMonitor* n = OrderAccess::load_acquire(&g_wait_list); n != NULL; n = unmarked_next(n)) { - // Rules for g_wait_list are the same as of g_free_list: - chk_free_entry(NULL /* jt */, n, out, error_cnt_p); - chk_om_wait_count++; + ObjectMonitor* cur = NULL; + ObjectMonitor* next = NULL; + if (mark_list_head(&g_wait_list, &cur, &next)) { + // Marked the global wait list head so process the list. + while (true) { + // Rules for g_wait_list are the same as of g_free_list: + chk_free_entry(NULL /* jt */, cur, out, error_cnt_p); + chk_om_wait_count++; + + mark_next_for_traversal(&cur, &next); + if (cur == NULL) { + break; + } + } } - if (OrderAccess::load_acquire(&g_om_wait_count) == chk_om_wait_count) { + if (g_om_wait_count == chk_om_wait_count) { out->print_cr("g_om_wait_count=%d equals chk_om_wait_count=%d", - OrderAccess::load_acquire(&g_om_wait_count), - chk_om_wait_count); + g_om_wait_count, chk_om_wait_count); } else { out->print_cr("ERROR: g_om_wait_count=%d is not equal to " - "chk_om_wait_count=%d", - OrderAccess::load_acquire(&g_om_wait_count), - chk_om_wait_count); + "chk_om_wait_count=%d", g_om_wait_count, chk_om_wait_count); *error_cnt_p = *error_cnt_p + 1; } } @@ -3047,21 +3101,29 @@ void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out, int *error_cnt_p) { int chk_om_in_use_count = 0; - for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) { - chk_in_use_entry(NULL /* jt */, n, out, error_cnt_p); - chk_om_in_use_count++; + ObjectMonitor* cur = NULL; + ObjectMonitor* next = NULL; + if (mark_list_head(&g_om_in_use_list, &cur, &next)) { + // Marked the global in-use list head so process the list. + while (true) { + chk_in_use_entry(NULL /* jt */, cur, out, error_cnt_p); + chk_om_in_use_count++; + + mark_next_for_traversal(&cur, &next); + if (cur == NULL) { + break; + } + } } - if (OrderAccess::load_acquire(&g_om_in_use_count) == chk_om_in_use_count) { + if (g_om_in_use_count == chk_om_in_use_count) { out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d", - OrderAccess::load_acquire(&g_om_in_use_count), - chk_om_in_use_count); + g_om_in_use_count, chk_om_in_use_count); } else { // With lock free access to the monitor lists, it is possible for // an exiting JavaThread to put its in-use ObjectMonitors on the // global in-use list after chk_om_in_use_count is calculated above. out->print_cr("WARNING: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d", - OrderAccess::load_acquire(&g_om_in_use_count), - chk_om_in_use_count); + g_om_in_use_count, chk_om_in_use_count); } } @@ -3129,19 +3191,27 @@ outputStream * out, int *error_cnt_p) { int chk_om_free_count = 0; - for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_free_list); n != NULL; n = unmarked_next(n)) { - chk_free_entry(jt, n, out, error_cnt_p); - chk_om_free_count++; + ObjectMonitor* cur = NULL; + ObjectMonitor* next = NULL; + if (mark_list_head(&jt->om_free_list, &cur, &next)) { + // Marked the per-thread free list head so process the list. + while (true) { + chk_free_entry(jt, cur, out, error_cnt_p); + chk_om_free_count++; + + mark_next_for_traversal(&cur, &next); + if (cur == NULL) { + break; + } + } } - if (OrderAccess::load_acquire(&jt->om_free_count) == chk_om_free_count) { + if (jt->om_free_count == chk_om_free_count) { out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals " - "chk_om_free_count=%d", p2i(jt), - OrderAccess::load_acquire(&jt->om_free_count), + "chk_om_free_count=%d", p2i(jt), jt->om_free_count, chk_om_free_count); } else { out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not " - "equal to chk_om_free_count=%d", p2i(jt), - OrderAccess::load_acquire(&jt->om_free_count), + "equal to chk_om_free_count=%d", p2i(jt), jt->om_free_count, chk_om_free_count); *error_cnt_p = *error_cnt_p + 1; } @@ -3152,20 +3222,28 @@ outputStream * out, int *error_cnt_p) { int chk_om_in_use_count = 0; - for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) { - chk_in_use_entry(jt, n, out, error_cnt_p); - chk_om_in_use_count++; + ObjectMonitor* cur = NULL; + ObjectMonitor* next = NULL; + if (mark_list_head(&jt->om_in_use_list, &cur, &next)) { + // Marked the per-thread in-use list head so process the list. + while (true) { + chk_in_use_entry(jt, cur, out, error_cnt_p); + chk_om_in_use_count++; + + mark_next_for_traversal(&cur, &next); + if (cur == NULL) { + break; + } + } } - if (OrderAccess::load_acquire(&jt->om_in_use_count) == chk_om_in_use_count) { + if (jt->om_in_use_count == chk_om_in_use_count) { out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals " "chk_om_in_use_count=%d", p2i(jt), - OrderAccess::load_acquire(&jt->om_in_use_count), - chk_om_in_use_count); + jt->om_in_use_count, chk_om_in_use_count); } else { out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not " "equal to chk_om_in_use_count=%d", p2i(jt), - OrderAccess::load_acquire(&jt->om_in_use_count), - chk_om_in_use_count); + jt->om_in_use_count, chk_om_in_use_count); *error_cnt_p = *error_cnt_p + 1; } } @@ -3175,25 +3253,35 @@ // indicate the associated object and its type. void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out) { stringStream ss; - if (OrderAccess::load_acquire(&g_om_in_use_count) > 0) { + if (g_om_in_use_count > 0) { out->print_cr("In-use global monitor info:"); out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)"); out->print_cr("%18s %s %7s %18s %18s", "monitor", "BHL", "ref_cnt", "object", "object type"); out->print_cr("================== === ======= ================== =================="); - for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) { - const oop obj = (oop) n->object(); - const markWord mark = n->header(); - ResourceMark rm; - out->print(INTPTR_FORMAT " %d%d%d %7d " INTPTR_FORMAT " %s", - p2i(n), n->is_busy() != 0, mark.hash() != 0, - n->owner() != NULL, (int)n->ref_count(), p2i(obj), - obj->klass()->external_name()); - if (n->is_busy() != 0) { - out->print(" (%s)", n->is_busy_to_string(&ss)); - ss.reset(); + ObjectMonitor* cur = NULL; + ObjectMonitor* next = NULL; + if (mark_list_head(&g_om_in_use_list, &cur, &next)) { + // Marked the global in-use list head so process the list. + while (true) { + const oop obj = (oop) cur->object(); + const markWord mark = cur->header(); + ResourceMark rm; + out->print(INTPTR_FORMAT " %d%d%d %7d " INTPTR_FORMAT " %s", + p2i(cur), cur->is_busy() != 0, mark.hash() != 0, + cur->owner() != NULL, (int)cur->ref_count(), p2i(obj), + obj->klass()->external_name()); + if (cur->is_busy() != 0) { + out->print(" (%s)", cur->is_busy_to_string(&ss)); + ss.reset(); + } + out->cr(); + + mark_next_for_traversal(&cur, &next); + if (cur == NULL) { + break; + } } - out->cr(); } } @@ -3203,19 +3291,29 @@ "jt", "monitor", "BHL", "ref_cnt", "object", "object type"); out->print_cr("================== ================== === ======= ================== =================="); for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { - for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) { - const oop obj = (oop) n->object(); - const markWord mark = n->header(); - ResourceMark rm; - out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d %7d " - INTPTR_FORMAT " %s", p2i(jt), p2i(n), n->is_busy() != 0, - mark.hash() != 0, n->owner() != NULL, (int)n->ref_count(), - p2i(obj), obj->klass()->external_name()); - if (n->is_busy() != 0) { - out->print(" (%s)", n->is_busy_to_string(&ss)); - ss.reset(); + ObjectMonitor* cur = NULL; + ObjectMonitor* next = NULL; + if (mark_list_head(&jt->om_in_use_list, &cur, &next)) { + // Marked the global in-use list head so process the list. + while (true) { + const oop obj = (oop) cur->object(); + const markWord mark = cur->header(); + ResourceMark rm; + out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d %7d " + INTPTR_FORMAT " %s", p2i(jt), p2i(cur), cur->is_busy() != 0, + mark.hash() != 0, cur->owner() != NULL, (int)cur->ref_count(), + p2i(obj), obj->klass()->external_name()); + if (cur->is_busy() != 0) { + out->print(" (%s)", cur->is_busy_to_string(&ss)); + ss.reset(); + } + out->cr(); + + mark_next_for_traversal(&cur, &next); + if (cur == NULL) { + break; + } } - out->cr(); } } @@ -3229,15 +3327,11 @@ out->print_cr("%18s %10s %10s %10s %10s", "Global Lists:", "InUse", "Free", "Wait", "Total"); out->print_cr("================== ========== ========== ========== =========="); - out->print_cr("%18s %10d %10d %10d %10d", "", - OrderAccess::load_acquire(&g_om_in_use_count), - OrderAccess::load_acquire(&g_om_free_count), - OrderAccess::load_acquire(&g_om_wait_count), - OrderAccess::load_acquire(&g_om_population)); - pop_count += OrderAccess::load_acquire(&g_om_in_use_count) + - OrderAccess::load_acquire(&g_om_free_count); + out->print_cr("%18s %10d %10d %10d %10d", "", g_om_in_use_count, + g_om_free_count, g_om_wait_count, g_om_population); + pop_count += g_om_in_use_count + g_om_free_count; if (HandshakeAfterDeflateIdleMonitors) { - pop_count += OrderAccess::load_acquire(&g_om_wait_count); + pop_count += g_om_wait_count; } out->print_cr("%18s %10s %10s %10s", @@ -3246,11 +3340,8 @@ for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) { out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt), - OrderAccess::load_acquire(&jt->om_in_use_count), - OrderAccess::load_acquire(&jt->om_free_count), - jt->om_free_provision); - pop_count += OrderAccess::load_acquire(&jt->om_in_use_count) + - OrderAccess::load_acquire(&jt->om_free_count); + jt->om_in_use_count, jt->om_free_count, jt->om_free_provision); + pop_count += jt->om_in_use_count + jt->om_free_count; } return pop_count; } @@ -3262,7 +3353,7 @@ // the list of extant blocks without taking a lock. int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) { - PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list); + PaddedObjectMonitor* block = g_block_list; while (block != NULL) { assert(block->object() == CHAINMARKER, "must be a block header"); if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) { @@ -3272,8 +3363,10 @@ assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned"); return 1; } - // unmarked_next() is not needed with g_block_list (no next field marking). - block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om); + // unmarked_next() is not needed with g_block_list (no next field + // marking) and no load_acquire() needed because _next_om is + // updated before g_block_list is changed with cmpxchg(). + block = (PaddedObjectMonitor*)block->_next_om; } return 0; } --- old/src/hotspot/share/runtime/thread.cpp 2019-11-04 14:59:05.000000000 -0500 +++ new/src/hotspot/share/runtime/thread.cpp 2019-11-04 14:59:05.000000000 -0500 @@ -4658,7 +4658,8 @@ DO_JAVA_THREADS(t_list, p) { if (!p->can_call_java()) continue; - address pending = (address)p->current_pending_monitor(); + ObjectMonitorHandle omh; + address pending = (address)p->current_pending_monitor(&omh); if (pending == monitor) { // found a match if (i < count) result->append(p); // save the first count matches i++; --- old/src/hotspot/share/runtime/thread.hpp 2019-11-04 14:59:06.000000000 -0500 +++ new/src/hotspot/share/runtime/thread.hpp 2019-11-04 14:59:06.000000000 -0500 @@ -622,11 +622,23 @@ void increment_vm_operation_completed_count() { _vm_operation_completed_count++; } // For tracking the heavyweight monitor the thread is pending on. - ObjectMonitor* current_pending_monitor() { - return _current_pending_monitor; + ObjectMonitor* current_pending_monitor(ObjectMonitorHandle* omh_p) { + if (omh_p->set_om_ptr_if_safe(_current_pending_monitor)) { + return omh_p->om_ptr(); // Return the safe ObjectMonitor*. + } + return NULL; } void set_current_pending_monitor(ObjectMonitor* monitor) { + ObjectMonitor* saved_cur = NULL; + if (monitor != NULL) { + monitor->inc_ref_count(); // Protect the ObjectMonitor* we're about to cache. + } else { + saved_cur = _current_pending_monitor; + } _current_pending_monitor = monitor; + if (saved_cur != NULL) { + saved_cur->dec_ref_count(); // Cleared the cached ObjectMonitor*. + } } void set_current_pending_monitor_is_from_java(bool from_java) { _current_pending_monitor_is_from_java = from_java; @@ -636,11 +648,23 @@ } // For tracking the ObjectMonitor on which this thread called Object.wait() - ObjectMonitor* current_waiting_monitor() { - return _current_waiting_monitor; + ObjectMonitor* current_waiting_monitor(ObjectMonitorHandle* omh_p) { + if (omh_p->set_om_ptr_if_safe(_current_waiting_monitor)) { + return omh_p->om_ptr(); // Return the safe ObjectMonitor*. + } + return NULL; } void set_current_waiting_monitor(ObjectMonitor* monitor) { + ObjectMonitor* saved_cur = NULL; + if (monitor != NULL) { + monitor->inc_ref_count(); // Protect the ObjectMonitor* we're about to cache. + } else { + saved_cur = _current_waiting_monitor; + } _current_waiting_monitor = monitor; + if (saved_cur != NULL) { + saved_cur->dec_ref_count(); // Cleared the cached ObjectMonitor*. + } } // For tracking the Jvmti raw monitor the thread is pending on. --- old/src/hotspot/share/runtime/vframe.cpp 2019-11-04 14:59:07.000000000 -0500 +++ new/src/hotspot/share/runtime/vframe.cpp 2019-11-04 14:59:07.000000000 -0500 @@ -122,8 +122,12 @@ if (mons->is_empty()) return result; bool found_first_monitor = false; - ObjectMonitor *pending_monitor = thread()->current_pending_monitor(); - ObjectMonitor *waiting_monitor = thread()->current_waiting_monitor(); + ObjectMonitorHandle omh; + ObjectMonitor *waiting_monitor = thread()->current_waiting_monitor(&omh); + ObjectMonitor *pending_monitor = NULL; + if (waiting_monitor == NULL) { + pending_monitor = thread()->current_pending_monitor(&omh); + } oop pending_obj = (pending_monitor != NULL ? (oop) pending_monitor->object() : (oop) NULL); oop waiting_obj = (waiting_monitor != NULL ? (oop) waiting_monitor->object() : (oop) NULL); @@ -231,9 +235,10 @@ // an inflated monitor that is first on the monitor list in // the first frame can block us on a monitor enter. markWord mark = monitor->owner()->mark(); + ObjectMonitorHandle omh; if (mark.has_monitor() && ( // we have marked ourself as pending on this monitor - mark.monitor() == thread()->current_pending_monitor() || + mark.monitor() == thread()->current_pending_monitor(&omh) || // we are not the owner of this monitor !mark.monitor()->is_entered(thread()) )) { --- old/src/hotspot/share/services/threadService.cpp 2019-11-04 14:59:08.000000000 -0500 +++ new/src/hotspot/share/services/threadService.cpp 2019-11-04 14:59:08.000000000 -0500 @@ -208,7 +208,8 @@ assert(thread != NULL, "should be non-NULL"); debug_only(Thread::check_for_dangling_thread_pointer(thread);) - ObjectMonitor *wait_obj = thread->current_waiting_monitor(); + ObjectMonitorHandle omh; + ObjectMonitor *wait_obj = thread->current_waiting_monitor(&omh); oop obj = NULL; if (wait_obj != NULL) { @@ -216,7 +217,8 @@ obj = (oop) wait_obj->object(); assert(obj != NULL, "Object.wait() should have an object"); } else { - ObjectMonitor *enter_obj = thread->current_pending_monitor(); + ObjectMonitorHandle omh; + ObjectMonitor *enter_obj = thread->current_pending_monitor(&omh); if (enter_obj != NULL) { // thread is trying to enter() an ObjectMonitor. obj = (oop) enter_obj->object(); @@ -362,6 +364,7 @@ // This code was modified from the original Threads::find_deadlocks code. int globalDfn = 0, thisDfn; + ObjectMonitorHandle omh; ObjectMonitor* waitingToLockMonitor = NULL; JvmtiRawMonitor* waitingToLockRawMonitor = NULL; oop waitingToLockBlocker = NULL; @@ -391,10 +394,15 @@ cycle->reset(); + if (waitingToLockMonitor != NULL) { + // Done with the current waitingToLockMonitor value so release + // the ObjectMonitorHandle manually before we use it again: + omh.unset_om_ptr(); + } // When there is a deadlock, all the monitors involved in the dependency // cycle must be contended and heavyweight. So we only care about the // heavyweight monitor a thread is waiting to lock. - waitingToLockMonitor = jt->current_pending_monitor(); + waitingToLockMonitor = jt->current_pending_monitor(&omh); // JVM TI raw monitors can also be involved in deadlocks, and we can be // waiting to lock both a raw monitor and ObjectMonitor at the same time. // It isn't clear how to make deadlock detection work correctly if that @@ -485,7 +493,12 @@ break; } previousThread = currentThread; - waitingToLockMonitor = (ObjectMonitor*)currentThread->current_pending_monitor(); + if (waitingToLockMonitor != NULL) { + // Done with the current waitingToLockMonitor value so release + // the ObjectMonitorHandle manually before we use it again: + omh.unset_om_ptr(); + } + waitingToLockMonitor = (ObjectMonitor*)currentThread->current_pending_monitor(&omh); if (concurrent_locks) { waitingToLockBlocker = currentThread->current_park_blocker(); } @@ -967,13 +980,13 @@ st->print("============================="); JavaThread* currentThread; - ObjectMonitor* waitingToLockMonitor; JvmtiRawMonitor* waitingToLockRawMonitor; oop waitingToLockBlocker; int len = _threads->length(); for (int i = 0; i < len; i++) { currentThread = _threads->at(i); - waitingToLockMonitor = currentThread->current_pending_monitor(); + ObjectMonitorHandle omh; + ObjectMonitor* waitingToLockMonitor = currentThread->current_pending_monitor(&omh); waitingToLockRawMonitor = currentThread->current_pending_raw_monitor(); waitingToLockBlocker = currentThread->current_park_blocker(); st->cr();