src/hotspot/share/runtime/objectMonitor.cpp

rev 59376 : 8153224.v2.10.patch merged with 8153224.v2.11.patch.
rev 59377 : CR1 changes from dcubed, dholmes, eosterlund and rehn.
rev 59378 : CR changes from dholmes, dcubed; fix is_being_async_deflated() race found by eosterlund; WB_ForceSafepoint() should request a special clean up with AsyncDeflateIdleMonitors; add a barrier in install_displaced_markword_in_object() to separate the header load from the preceding loads in is_being_async_deflated().
rev 59379 : eosterlund CR - Switch from three part async deflation protocol to a two part async deflation protocol where a negative contentions field is a linearization point.

@@ -238,32 +238,32 @@
 }
 
 // -----------------------------------------------------------------------------
 // Enter support
 
-void ObjectMonitor::enter(TRAPS) {
+bool ObjectMonitor::enter(TRAPS) {
   // The following code is ordered to check the most common cases first
   // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors.
   Thread * const Self = THREAD;
 
   void* cur = try_set_owner_from(NULL, Self);
   if (cur == NULL) {
     assert(_recursions == 0, "invariant");
-    return;
+    return true;
   }
 
   if (cur == Self) {
     // TODO-FIXME: check for integer overflow!  BUGID 6557169.
     _recursions++;
-    return;
+    return true;
   }
 
   if (Self->is_lock_owned((address)cur)) {
     assert(_recursions == 0, "internal state error");
     _recursions = 1;
     set_owner_from_BasicLock(cur, Self);  // Convert from BasicLock* to Thread*.
-    return;
+    return true;
   }
 
   // We've encountered genuine contention.
   assert(Self->_Stalled == 0, "invariant");
   Self->_Stalled = intptr_t(this);

@@ -279,25 +279,38 @@
     assert(((oop)object())->mark() == markWord::encode(this),
            "object mark must match encoded this: mark=" INTPTR_FORMAT
            ", encoded this=" INTPTR_FORMAT, ((oop)object())->mark().value(),
            markWord::encode(this).value());
     Self->_Stalled = 0;
-    return;
+    return true;
   }
 
   assert(_owner != Self, "invariant");
   assert(_succ != Self, "invariant");
   assert(Self->is_Java_thread(), "invariant");
   JavaThread * jt = (JavaThread *) Self;
   assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
   assert(jt->thread_state() != _thread_blocked, "invariant");
-  assert(this->object() != NULL, "invariant");
-  assert(_contentions >= 0, "invariant");
+  assert(AsyncDeflateIdleMonitors || this->object() != NULL, "invariant");
+  assert(AsyncDeflateIdleMonitors || contentions() >= 0, "must not be negative: contentions=%d", contentions());
 
-  // Prevent deflation at STW-time.  See deflate_idle_monitors() and is_busy().
-  // Ensure the object-monitor relationship remains stable while there's contention.
-  Atomic::inc(&_contentions);
+  // Keep track of contention for JVM/TI and M&M queries.
+  add_to_contentions(1);
+  if (is_being_async_deflated()) {
+    // Async deflation is in progress and our contentions increment
+    // above lost the race to async deflation. Undo the work and
+    // force the caller to retry.
+    const oop l_object = (oop)object();
+    if (l_object != NULL) {
+      // Attempt to restore the header/dmw to the object's header so that
+      // we only retry once if the deflater thread happens to be slow.
+      install_displaced_markword_in_object(l_object);
+    }
+    Self->_Stalled = 0;
+    add_to_contentions(-1);
+    return false;
+  }
 
   JFR_ONLY(JfrConditionalFlushWithStacktrace<EventJavaMonitorEnter> flush(jt);)
   EventJavaMonitorEnter event;
   if (event.should_commit()) {
     event.set_monitorClass(((oop)this->object())->klass());

@@ -354,12 +367,12 @@
     // as having "-locked" the monitor, but the OS and java.lang.Thread
     // states will still report that the thread is blocked trying to
     // acquire it.
   }
 
-  Atomic::dec(&_contentions);
-  assert(_contentions >= 0, "invariant");
+  add_to_contentions(-1);
+  assert(contentions() >= 0, "must not be negative: contentions=%d", contentions());
   Self->_Stalled = 0;
 
   // Must either set _recursions = 0 or ASSERT _recursions == 0.
   assert(_recursions == 0, "invariant");
   assert(_owner == Self, "invariant");

@@ -391,10 +404,11 @@
   if (event.should_commit()) {
     event.set_previousOwner((uintptr_t)_previous_owner_tid);
     event.commit();
   }
   OM_PERFDATA_OP(ContendedLockAttempts, inc());
+  return true;
 }
 
 // Caveat: TryLock() is not necessarily serializing if it returns failure.
 // Callers must compensate as needed.
 

@@ -410,16 +424,90 @@
   // We can either return -1 or retry.
   // Retry doesn't make as much sense because the lock was just acquired.
   return -1;
 }
 
+// Install the displaced mark word (dmw) of a deflating ObjectMonitor
+// into the header of the object associated with the monitor. This
+// idempotent method is called by a thread that is deflating a
+// monitor and by other threads that have detected a race with the
+// deflation process.
+void ObjectMonitor::install_displaced_markword_in_object(const oop obj) {
+  // This function must only be called when (owner == DEFLATER_MARKER
+  // && contentions <= 0), but we can't guarantee that here because
+  // those values could change when the ObjectMonitor gets moved from
+  // the global free list to a per-thread free list.
+
+  guarantee(obj != NULL, "must be non-NULL");
+
+  // Separate loads in is_being_async_deflated(), which is almost always
+  // called before this function, from the load of dmw/header below.
+  if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+    // A non-multiple copy atomic (nMCA) machine needs a bigger
+    // hammer to separate the loads before and the load below.
+    OrderAccess::fence();
+  } else {
+    OrderAccess::loadload();
+  }
+
+  const oop l_object = (oop)object();
+  if (l_object == NULL) {
+    // ObjectMonitor's object ref has already been cleared by async
+    // deflation so we're done here.
+    return;
+  }
+  ADIM_guarantee(l_object == obj, "object=" INTPTR_FORMAT " must equal obj="
+                 INTPTR_FORMAT, p2i(l_object), p2i(obj));
+
+  markWord dmw = header();
+  // The dmw has to be neutral (not NULL, not locked and not marked).
+  ADIM_guarantee(dmw.is_neutral(), "must be neutral: dmw=" INTPTR_FORMAT, dmw.value());
+
+  // Install displaced mark word if the object's header still points
+  // to this ObjectMonitor. More than one racing caller to this function
+  // can rarely reach this point, but only one can win.
+  markWord res = obj->cas_set_mark(dmw, markWord::encode(this));
+  if (res != markWord::encode(this)) {
+    // This should be rare so log at the Info level when it happens.
+    log_info(monitorinflation)("install_displaced_markword_in_object: "
+                               "failed cas_set_mark: new_mark=" INTPTR_FORMAT
+                               ", old_mark=" INTPTR_FORMAT ", res=" INTPTR_FORMAT,
+                               dmw.value(), markWord::encode(this).value(),
+                               res.value());
+  }
+
+  // Note: It does not matter which thread restored the header/dmw
+  // into the object's header. The thread deflating the monitor just
+  // wanted the object's header restored and it is. The threads that
+  // detected a race with the deflation process also wanted the
+  // object's header restored before they retry their operation and
+  // because it is restored they will only retry once.
+}
+
 // Convert the fields used by is_busy() to a string that can be
 // used for diagnostic output.
 const char* ObjectMonitor::is_busy_to_string(stringStream* ss) {
-  ss->print("is_busy: contentions=%d, waiters=%d, owner=" INTPTR_FORMAT
-            ", cxq=" INTPTR_FORMAT ", EntryList=" INTPTR_FORMAT, _contentions,
-            _waiters, p2i(_owner), p2i(_cxq), p2i(_EntryList));
+  ss->print("is_busy: waiters=%d, ", _waiters);
+  if (!AsyncDeflateIdleMonitors) {
+    ss->print("contentions=%d, ", contentions());
+    ss->print("owner=" INTPTR_FORMAT, p2i(_owner));
+  } else {
+    if (contentions() > 0) {
+      ss->print("contentions=%d, ", contentions());
+    } else {
+      ss->print("contentions=0");
+    }
+    if (_owner != DEFLATER_MARKER) {
+      ss->print("owner=" INTPTR_FORMAT, p2i(_owner));
+    } else {
+      // We report NULL instead of DEFLATER_MARKER here because is_busy()
+      // ignores DEFLATER_MARKER values.
+      ss->print("owner=" INTPTR_FORMAT, NULL);
+    }
+  }
+  ss->print(", cxq=" INTPTR_FORMAT ", EntryList=" INTPTR_FORMAT, p2i(_cxq),
+            p2i(_EntryList));
   return ss->base();
 }
 
 #define MAX_RECHECK_INTERVAL 1000
 

@@ -434,10 +522,24 @@
     assert(_owner == Self, "invariant");
     assert(_Responsible != Self, "invariant");
     return;
   }
 
+  if (AsyncDeflateIdleMonitors &&
+      try_set_owner_from(DEFLATER_MARKER, Self) == DEFLATER_MARKER) {
+    // Cancelled the in-progress async deflation. We bump contentions an
+    // extra time to prevent the async deflater thread from temporarily
+    // changing it to -max_jint and back to zero (no flicker to confuse
+    // is_being_async_deflated()). The async deflater thread will
+    // decrement contentions after it recognizes that the async
+    // deflation was cancelled.
+    add_to_contentions(1);
+    assert(_succ != Self, "invariant");
+    assert(_Responsible != Self, "invariant");
+    return;
+  }
+
   assert(InitDone, "Unexpectedly not initialized");
 
   // We try one round of spinning *before* enqueueing Self.
   //
   // If the _owner is ready but OFFPROC we could use a YieldTo()

@@ -550,10 +652,22 @@
       Self->_ParkEvent->park();
     }
 
     if (TryLock(Self) > 0) break;
 
+    if (AsyncDeflateIdleMonitors &&
+        try_set_owner_from(DEFLATER_MARKER, Self) == DEFLATER_MARKER) {
+      // Cancelled the in-progress async deflation. We bump contentions an
+      // extra time to prevent the async deflater thread from temporarily
+      // changing it to -max_jint and back to zero (no flicker to confuse
+      // is_being_async_deflated()). The async deflater thread will
+      // decrement contentions after it recognizes that the async
+      // deflation was cancelled.
+      add_to_contentions(1);
+      break;
+    }
+
     // The lock is still contested.
     // Keep a tally of the # of futile wakeups.
     // Note that the counter is not protected by a lock or updated by atomics.
     // That is by design - we trade "lossy" counters which are exposed to
     // races during updates for a lower probe effect.

@@ -814,11 +928,11 @@
 // inopportune) reclamation of "this".
 //
 // We'd like to assert that: (THREAD->thread_state() != _thread_blocked) ;
 // There's one exception to the claim above, however.  EnterI() can call
 // exit() to drop a lock if the acquirer has been externally suspended.
-// In that case exit() is called with _thread_state as _thread_blocked,
+// In that case exit() is called with _thread_state == _thread_blocked,
 // but the monitor's _contentions field is > 0, which inhibits reclamation.
 //
 // 1-0 exit
 // ~~~~~~~~
 // ::exit() uses a canonical 1-1 idiom with a MEMBAR although some of

@@ -1089,11 +1203,11 @@
   // Hygiene -- once we've set _owner = NULL we can't safely dereference Wakee again.
   // The thread associated with Wakee may have grabbed the lock and "Wakee" may be
   // out-of-scope (non-extant).
   Wakee  = NULL;
 
-  // Drop the lock
+  // Drop the lock.
   // Uses a fence to separate release_store(owner) from the LD in unpark().
   release_clear_owner(Self);
   OrderAccess::fence();
 
   DTRACE_MONITOR_PROBE(contended__exit, this, object(), Self);

@@ -1137,20 +1251,23 @@
   return save;
 }
 
 // reenter() enters a lock and sets recursion count
 // complete_exit/reenter operate as a wait without waiting
-void ObjectMonitor::reenter(intx recursions, TRAPS) {
+bool ObjectMonitor::reenter(intx recursions, TRAPS) {
   Thread * const Self = THREAD;
   assert(Self->is_Java_thread(), "Must be Java thread!");
   JavaThread *jt = (JavaThread *)THREAD;
 
   guarantee(_owner != Self, "reenter already owner");
-  enter(THREAD);       // enter the monitor
+  if (!enter(THREAD)) {
+    return false;
+  }
+  // Entered the monitor.
   guarantee(_recursions == 0, "reenter recursion");
   _recursions = recursions;
-  return;
+  return true;
 }
 
 // Checks that the current THREAD owns this monitor and causes an
 // immediate return if it doesn't. We don't use the CHECK macro
 // because we want the IMSE to be the only exception that is thrown

@@ -1960,18 +2077,24 @@
 // Print the ObjectMonitor like a debugger would:
 //
 // (ObjectMonitor) 0x00007fdfb6012e40 = {
 //   _header = 0x0000000000000001
 //   _object = 0x000000070ff45fd0
-//   _next_om = 0x0000000000000000
+//   _allocation_state = Old
 //   _pad_buf0 = {
 //     [0] = '\0'
 //     ...
-//     [103] = '\0'
+//     [43] = '\0'
 //   }
 //   _owner = 0x0000000000000000
 //   _previous_owner_tid = 0
+//   _pad_buf1 = {
+//     [0] = '\0'
+//     ...
+//     [47] = '\0'
+//   }
+//   _next_om = 0x0000000000000000
 //   _recursions = 0
 //   _EntryList = 0x0000000000000000
 //   _cxq = 0x0000000000000000
 //   _succ = 0x0000000000000000
 //   _Responsible = 0x0000000000000000

@@ -1985,26 +2108,42 @@
 //
 void ObjectMonitor::print_debug_style_on(outputStream* st) const {
   st->print_cr("(ObjectMonitor*) " INTPTR_FORMAT " = {", p2i(this));
   st->print_cr("  _header = " INTPTR_FORMAT, header().value());
   st->print_cr("  _object = " INTPTR_FORMAT, p2i(_object));
-  st->print_cr("  _next_om = " INTPTR_FORMAT, p2i(next_om()));
+  st->print("  _allocation_state = ");
+  if (is_free()) {
+    st->print("Free");
+  } else if (is_old()) {
+    st->print("Old");
+  } else if (is_new()) {
+    st->print("New");
+  } else {
+    st->print("unknown=%d", _allocation_state);
+  }
+  st->cr();
   st->print_cr("  _pad_buf0 = {");
   st->print_cr("    [0] = '\\0'");
   st->print_cr("    ...");
   st->print_cr("    [%d] = '\\0'", (int)sizeof(_pad_buf0) - 1);
   st->print_cr("  }");
   st->print_cr("  _owner = " INTPTR_FORMAT, p2i(_owner));
   st->print_cr("  _previous_owner_tid = " JLONG_FORMAT, _previous_owner_tid);
+  st->print_cr("  _pad_buf1 = {");
+  st->print_cr("    [0] = '\\0'");
+  st->print_cr("    ...");
+  st->print_cr("    [%d] = '\\0'", (int)sizeof(_pad_buf1) - 1);
+  st->print_cr("  }");
+  st->print_cr("  _next_om = " INTPTR_FORMAT, p2i(next_om()));
   st->print_cr("  _recursions = " INTX_FORMAT, _recursions);
   st->print_cr("  _EntryList = " INTPTR_FORMAT, p2i(_EntryList));
   st->print_cr("  _cxq = " INTPTR_FORMAT, p2i(_cxq));
   st->print_cr("  _succ = " INTPTR_FORMAT, p2i(_succ));
   st->print_cr("  _Responsible = " INTPTR_FORMAT, p2i(_Responsible));
   st->print_cr("  _Spinner = %d", _Spinner);
   st->print_cr("  _SpinDuration = %d", _SpinDuration);
-  st->print_cr("  _contentions = %d", _contentions);
+  st->print_cr("  _contentions = %d", contentions());
   st->print_cr("  _WaitSet = " INTPTR_FORMAT, p2i(_WaitSet));
   st->print_cr("  _waiters = %d", _waiters);
   st->print_cr("  _WaitSetLock = %d", _WaitSetLock);
   st->print_cr("}");
 }