1 /*
   2  * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/vmSymbols.hpp"
  27 #include "logging/log.hpp"
  28 #include "logging/logStream.hpp"
  29 #include "jfr/jfrEvents.hpp"
  30 #include "memory/allocation.inline.hpp"
  31 #include "memory/metaspaceShared.hpp"
  32 #include "memory/padded.hpp"
  33 #include "memory/resourceArea.hpp"
  34 #include "memory/universe.hpp"
  35 #include "oops/markWord.hpp"
  36 #include "oops/oop.inline.hpp"
  37 #include "runtime/atomic.hpp"
  38 #include "runtime/biasedLocking.hpp"
  39 #include "runtime/handles.inline.hpp"
  40 #include "runtime/interfaceSupport.inline.hpp"
  41 #include "runtime/mutexLocker.hpp"
  42 #include "runtime/objectMonitor.hpp"
  43 #include "runtime/objectMonitor.inline.hpp"
  44 #include "runtime/osThread.hpp"
  45 #include "runtime/safepointVerifiers.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubRoutines.hpp"
  48 #include "runtime/synchronizer.hpp"
  49 #include "runtime/thread.inline.hpp"
  50 #include "runtime/timer.hpp"
  51 #include "runtime/vframe.hpp"
  52 #include "runtime/vmThread.hpp"
  53 #include "utilities/align.hpp"
  54 #include "utilities/dtrace.hpp"
  55 #include "utilities/events.hpp"
  56 #include "utilities/preserveException.hpp"
  57 
  58 // The "core" versions of monitor enter and exit reside in this file.
  59 // The interpreter and compilers contain specialized transliterated
  60 // variants of the enter-exit fast-path operations.  See i486.ad fast_lock(),
  61 // for instance.  If you make changes here, make sure to modify the
  62 // interpreter, and both C1 and C2 fast-path inline locking code emission.
  63 //
  64 // -----------------------------------------------------------------------------
  65 
  66 #ifdef DTRACE_ENABLED
  67 
  68 // Only bother with this argument setup if dtrace is available
  69 // TODO-FIXME: probes should not fire when caller is _blocked.  assert() accordingly.
  70 
  71 #define DTRACE_MONITOR_PROBE_COMMON(obj, thread)                           \
  72   char* bytes = NULL;                                                      \
  73   int len = 0;                                                             \
  74   jlong jtid = SharedRuntime::get_java_tid(thread);                        \
  75   Symbol* klassname = ((oop)(obj))->klass()->name();                       \
  76   if (klassname != NULL) {                                                 \
  77     bytes = (char*)klassname->bytes();                                     \
  78     len = klassname->utf8_length();                                        \
  79   }
  80 
  81 #define DTRACE_MONITOR_WAIT_PROBE(monitor, obj, thread, millis)            \
  82   {                                                                        \
  83     if (DTraceMonitorProbes) {                                             \
  84       DTRACE_MONITOR_PROBE_COMMON(obj, thread);                            \
  85       HOTSPOT_MONITOR_WAIT(jtid,                                           \
  86                            (uintptr_t)(monitor), bytes, len, (millis));    \
  87     }                                                                      \
  88   }
  89 
  90 #define HOTSPOT_MONITOR_PROBE_notify HOTSPOT_MONITOR_NOTIFY
  91 #define HOTSPOT_MONITOR_PROBE_notifyAll HOTSPOT_MONITOR_NOTIFYALL
  92 #define HOTSPOT_MONITOR_PROBE_waited HOTSPOT_MONITOR_WAITED
  93 
  94 #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread)                  \
  95   {                                                                        \
  96     if (DTraceMonitorProbes) {                                             \
  97       DTRACE_MONITOR_PROBE_COMMON(obj, thread);                            \
  98       HOTSPOT_MONITOR_PROBE_##probe(jtid, /* probe = waited */             \
  99                                     (uintptr_t)(monitor), bytes, len);     \
 100     }                                                                      \
 101   }
 102 
 103 #else //  ndef DTRACE_ENABLED
 104 
 105 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon)    {;}
 106 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon)          {;}
 107 
 108 #endif // ndef DTRACE_ENABLED
 109 
 110 // This exists only as a workaround of dtrace bug 6254741
 111 int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
 112   DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
 113   return 0;
 114 }
 115 
 116 #define NINFLATIONLOCKS 256
 117 static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];
 118 
 119 // global list of blocks of monitors
 120 PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL;
 121 // Global ObjectMonitor free list. Newly allocated and deflated
 122 // ObjectMonitors are prepended here.
 123 ObjectMonitor* volatile ObjectSynchronizer::g_free_list = NULL;
 124 // Global ObjectMonitor in-use list. When a JavaThread is exiting,
 125 // ObjectMonitors on its per-thread in-use list are prepended here.
 126 ObjectMonitor* volatile ObjectSynchronizer::g_om_in_use_list = NULL;
 127 int ObjectSynchronizer::g_om_in_use_count = 0;  // # on g_om_in_use_list
 128 
 129 static volatile intptr_t gListLock = 0;   // protects global monitor lists
 130 static volatile int g_om_free_count = 0;  // # on g_free_list
 131 static volatile int g_om_population = 0;  // # Extant -- in circulation
 132 
 133 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
 134 
 135 
 136 // =====================> Quick functions
 137 
 138 // The quick_* forms are special fast-path variants used to improve
 139 // performance.  In the simplest case, a "quick_*" implementation could
 140 // simply return false, in which case the caller will perform the necessary
 141 // state transitions and call the slow-path form.
 142 // The fast-path is designed to handle frequently arising cases in an efficient
 143 // manner and is just a degenerate "optimistic" variant of the slow-path.
 144 // returns true  -- to indicate the call was satisfied.
 145 // returns false -- to indicate the call needs the services of the slow-path.
 146 // A no-loitering ordinance is in effect for code in the quick_* family
 147 // operators: safepoints or indefinite blocking (blocking that might span a
 148 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
 149 // entry.
 150 //
 151 // Consider: An interesting optimization is to have the JIT recognize the
 152 // following common idiom:
 153 //   synchronized (someobj) { .... ; notify(); }
 154 // That is, we find a notify() or notifyAll() call that immediately precedes
 155 // the monitorexit operation.  In that case the JIT could fuse the operations
 156 // into a single notifyAndExit() runtime primitive.
 157 
 158 bool ObjectSynchronizer::quick_notify(oopDesc* obj, Thread* self, bool all) {
 159   assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
 160   assert(self->is_Java_thread(), "invariant");
 161   assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
 162   NoSafepointVerifier nsv;
 163   if (obj == NULL) return false;  // slow-path for invalid obj
 164   const markWord mark = obj->mark();
 165 
 166   if (mark.has_locker() && self->is_lock_owned((address)mark.locker())) {
 167     // Degenerate notify
 168     // stack-locked by caller so by definition the implied waitset is empty.
 169     return true;
 170   }
 171 
 172   if (mark.has_monitor()) {
 173     ObjectMonitor* const mon = mark.monitor();
 174     assert(mon->object() == obj, "invariant");
 175     if (mon->owner() != self) return false;  // slow-path for IMS exception
 176 
 177     if (mon->first_waiter() != NULL) {
 178       // We have one or more waiters. Since this is an inflated monitor
 179       // that we own, we can transfer one or more threads from the waitset
 180       // to the entrylist here and now, avoiding the slow-path.
 181       if (all) {
 182         DTRACE_MONITOR_PROBE(notifyAll, mon, obj, self);
 183       } else {
 184         DTRACE_MONITOR_PROBE(notify, mon, obj, self);
 185       }
 186       int free_count = 0;
 187       do {
 188         mon->INotify(self);
 189         ++free_count;
 190       } while (mon->first_waiter() != NULL && all);
 191       OM_PERFDATA_OP(Notifications, inc(free_count));
 192     }
 193     return true;
 194   }
 195 
 196   // biased locking and any other IMS exception states take the slow-path
 197   return false;
 198 }
 199 
 200 
 201 // The LockNode emitted directly at the synchronization site would have
 202 // been too big if it were to have included support for the cases of inflated
 203 // recursive enter and exit, so they go here instead.
 204 // Note that we can't safely call AsyncPrintJavaStack() from within
 205 // quick_enter() as our thread state remains _in_Java.
 206 
 207 bool ObjectSynchronizer::quick_enter(oop obj, Thread* self,
 208                                      BasicLock * lock) {
 209   assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
 210   assert(self->is_Java_thread(), "invariant");
 211   assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
 212   NoSafepointVerifier nsv;
 213   if (obj == NULL) return false;       // Need to throw NPE
 214   const markWord mark = obj->mark();
 215 
 216   if (mark.has_monitor()) {
 217     ObjectMonitor* const m = mark.monitor();
 218     assert(m->object() == obj, "invariant");
 219     Thread* const owner = (Thread *) m->_owner;
 220 
 221     // Lock contention and Transactional Lock Elision (TLE) diagnostics
 222     // and observability
 223     // Case: light contention possibly amenable to TLE
 224     // Case: TLE inimical operations such as nested/recursive synchronization
 225 
 226     if (owner == self) {
 227       m->_recursions++;
 228       return true;
 229     }
 230 
 231     // This Java Monitor is inflated so obj's header will never be
 232     // displaced to this thread's BasicLock. Make the displaced header
 233     // non-NULL so this BasicLock is not seen as recursive nor as
 234     // being locked. We do this unconditionally so that this thread's
 235     // BasicLock cannot be mis-interpreted by any stack walkers. For
 236     // performance reasons, stack walkers generally first check for
 237     // Biased Locking in the object's header, the second check is for
 238     // stack-locking in the object's header, the third check is for
 239     // recursive stack-locking in the displaced header in the BasicLock,
 240     // and last are the inflated Java Monitor (ObjectMonitor) checks.
 241     lock->set_displaced_header(markWord::unused_mark());
 242 
 243     if (owner == NULL && Atomic::replace_if_null(self, &(m->_owner))) {
 244       assert(m->_recursions == 0, "invariant");
 245       return true;
 246     }
 247   }
 248 
 249   // Note that we could inflate in quick_enter.
 250   // This is likely a useful optimization
 251   // Critically, in quick_enter() we must not:
 252   // -- perform bias revocation, or
 253   // -- block indefinitely, or
 254   // -- reach a safepoint
 255 
 256   return false;        // revert to slow-path
 257 }
 258 
 259 // -----------------------------------------------------------------------------
 260 // Monitor Enter/Exit
 261 // The interpreter and compiler assembly code tries to lock using the fast path
 262 // of this algorithm. Make sure to update that code if the following function is
 263 // changed. The implementation is extremely sensitive to race condition. Be careful.
 264 
 265 void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, TRAPS) {
 266   if (UseBiasedLocking) {
 267     if (!SafepointSynchronize::is_at_safepoint()) {
 268       BiasedLocking::revoke(obj, THREAD);
 269     } else {
 270       BiasedLocking::revoke_at_safepoint(obj);
 271     }
 272   }
 273 
 274   markWord mark = obj->mark();
 275   assert(!mark.has_bias_pattern(), "should not see bias pattern here");
 276 
 277   if (mark.is_neutral()) {
 278     // Anticipate successful CAS -- the ST of the displaced mark must
 279     // be visible <= the ST performed by the CAS.
 280     lock->set_displaced_header(mark);
 281     if (mark == obj()->cas_set_mark(markWord::from_pointer(lock), mark)) {
 282       return;
 283     }
 284     // Fall through to inflate() ...
 285   } else if (mark.has_locker() &&
 286              THREAD->is_lock_owned((address)mark.locker())) {
 287     assert(lock != mark.locker(), "must not re-lock the same lock");
 288     assert(lock != (BasicLock*)obj->mark().value(), "don't relock with same BasicLock");
 289     lock->set_displaced_header(markWord::from_pointer(NULL));
 290     return;
 291   }
 292 
 293   // The object header will never be displaced to this lock,
 294   // so it does not matter what the value is, except that it
 295   // must be non-zero to avoid looking like a re-entrant lock,
 296   // and must not look locked either.
 297   lock->set_displaced_header(markWord::unused_mark());
 298   inflate(THREAD, obj(), inflate_cause_monitor_enter)->enter(THREAD);
 299 }
 300 
 301 void ObjectSynchronizer::exit(oop object, BasicLock* lock, TRAPS) {
 302   markWord mark = object->mark();
 303   // We cannot check for Biased Locking if we are racing an inflation.
 304   assert(mark == markWord::INFLATING() ||
 305          !mark.has_bias_pattern(), "should not see bias pattern here");
 306 
 307   markWord dhw = lock->displaced_header();
 308   if (dhw.value() == 0) {
 309     // If the displaced header is NULL, then this exit matches up with
 310     // a recursive enter. No real work to do here except for diagnostics.
 311 #ifndef PRODUCT
 312     if (mark != markWord::INFLATING()) {
 313       // Only do diagnostics if we are not racing an inflation. Simply
 314       // exiting a recursive enter of a Java Monitor that is being
 315       // inflated is safe; see the has_monitor() comment below.
 316       assert(!mark.is_neutral(), "invariant");
 317       assert(!mark.has_locker() ||
 318              THREAD->is_lock_owned((address)mark.locker()), "invariant");
 319       if (mark.has_monitor()) {
 320         // The BasicLock's displaced_header is marked as a recursive
 321         // enter and we have an inflated Java Monitor (ObjectMonitor).
 322         // This is a special case where the Java Monitor was inflated
 323         // after this thread entered the stack-lock recursively. When a
 324         // Java Monitor is inflated, we cannot safely walk the Java
 325         // Monitor owner's stack and update the BasicLocks because a
 326         // Java Monitor can be asynchronously inflated by a thread that
 327         // does not own the Java Monitor.
 328         ObjectMonitor* m = mark.monitor();
 329         assert(((oop)(m->object()))->mark() == mark, "invariant");
 330         assert(m->is_entered(THREAD), "invariant");
 331       }
 332     }
 333 #endif
 334     return;
 335   }
 336 
 337   if (mark == markWord::from_pointer(lock)) {
 338     // If the object is stack-locked by the current thread, try to
 339     // swing the displaced header from the BasicLock back to the mark.
 340     assert(dhw.is_neutral(), "invariant");
 341     if (object->cas_set_mark(dhw, mark) == mark) {
 342       return;
 343     }
 344   }
 345 
 346   // We have to take the slow-path of possible inflation and then exit.
 347   inflate(THREAD, object, inflate_cause_vm_internal)->exit(true, THREAD);
 348 }
 349 
 350 // -----------------------------------------------------------------------------
 351 // Class Loader  support to workaround deadlocks on the class loader lock objects
 352 // Also used by GC
 353 // complete_exit()/reenter() are used to wait on a nested lock
 354 // i.e. to give up an outer lock completely and then re-enter
 355 // Used when holding nested locks - lock acquisition order: lock1 then lock2
 356 //  1) complete_exit lock1 - saving recursion count
 357 //  2) wait on lock2
 358 //  3) when notified on lock2, unlock lock2
 359 //  4) reenter lock1 with original recursion count
 360 //  5) lock lock2
 361 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
 362 intx ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
 363   if (UseBiasedLocking) {
 364     BiasedLocking::revoke(obj, THREAD);
 365     assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
 366   }
 367 
 368   ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal);
 369 
 370   return monitor->complete_exit(THREAD);
 371 }
 372 
 373 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
 374 void ObjectSynchronizer::reenter(Handle obj, intx recursions, TRAPS) {
 375   if (UseBiasedLocking) {
 376     BiasedLocking::revoke(obj, THREAD);
 377     assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
 378   }
 379 
 380   ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal);
 381 
 382   monitor->reenter(recursions, THREAD);
 383 }
 384 // -----------------------------------------------------------------------------
 385 // JNI locks on java objects
 386 // NOTE: must use heavy weight monitor to handle jni monitor enter
 387 void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) {
 388   // the current locking is from JNI instead of Java code
 389   if (UseBiasedLocking) {
 390     BiasedLocking::revoke(obj, THREAD);
 391     assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
 392   }
 393   THREAD->set_current_pending_monitor_is_from_java(false);
 394   inflate(THREAD, obj(), inflate_cause_jni_enter)->enter(THREAD);
 395   THREAD->set_current_pending_monitor_is_from_java(true);
 396 }
 397 
 398 // NOTE: must use heavy weight monitor to handle jni monitor exit
 399 void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
 400   if (UseBiasedLocking) {
 401     Handle h_obj(THREAD, obj);
 402     BiasedLocking::revoke(h_obj, THREAD);
 403     obj = h_obj();
 404   }
 405   assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
 406 
 407   ObjectMonitor* monitor = inflate(THREAD, obj, inflate_cause_jni_exit);
 408   // If this thread has locked the object, exit the monitor. We
 409   // intentionally do not use CHECK here because we must exit the
 410   // monitor even if an exception is pending.
 411   if (monitor->check_owner(THREAD)) {
 412     monitor->exit(true, THREAD);
 413   }
 414 }
 415 
 416 // -----------------------------------------------------------------------------
 417 // Internal VM locks on java objects
 418 // standard constructor, allows locking failures
 419 ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool do_lock) {
 420   _dolock = do_lock;
 421   _thread = thread;
 422   _thread->check_for_valid_safepoint_state();
 423   _obj = obj;
 424 
 425   if (_dolock) {
 426     ObjectSynchronizer::enter(_obj, &_lock, _thread);
 427   }
 428 }
 429 
 430 ObjectLocker::~ObjectLocker() {
 431   if (_dolock) {
 432     ObjectSynchronizer::exit(_obj(), &_lock, _thread);
 433   }
 434 }
 435 
 436 
 437 // -----------------------------------------------------------------------------
 438 //  Wait/Notify/NotifyAll
 439 // NOTE: must use heavy weight monitor to handle wait()
 440 int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
 441   if (UseBiasedLocking) {
 442     BiasedLocking::revoke(obj, THREAD);
 443     assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
 444   }
 445   if (millis < 0) {
 446     THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
 447   }
 448   ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_wait);
 449 
 450   DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);
 451   monitor->wait(millis, true, THREAD);
 452 
 453   // This dummy call is in place to get around dtrace bug 6254741.  Once
 454   // that's fixed we can uncomment the following line, remove the call
 455   // and change this function back into a "void" func.
 456   // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
 457   return dtrace_waited_probe(monitor, obj, THREAD);
 458 }
 459 
 460 void ObjectSynchronizer::wait_uninterruptibly(Handle obj, jlong millis, TRAPS) {
 461   if (UseBiasedLocking) {
 462     BiasedLocking::revoke(obj, THREAD);
 463     assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
 464   }
 465   if (millis < 0) {
 466     THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
 467   }
 468   inflate(THREAD, obj(), inflate_cause_wait)->wait(millis, false, THREAD);
 469 }
 470 
 471 void ObjectSynchronizer::notify(Handle obj, TRAPS) {
 472   if (UseBiasedLocking) {
 473     BiasedLocking::revoke(obj, THREAD);
 474     assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
 475   }
 476 
 477   markWord mark = obj->mark();
 478   if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
 479     return;
 480   }
 481   inflate(THREAD, obj(), inflate_cause_notify)->notify(THREAD);
 482 }
 483 
 484 // NOTE: see comment of notify()
 485 void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
 486   if (UseBiasedLocking) {
 487     BiasedLocking::revoke(obj, THREAD);
 488     assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
 489   }
 490 
 491   markWord mark = obj->mark();
 492   if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
 493     return;
 494   }
 495   inflate(THREAD, obj(), inflate_cause_notify)->notifyAll(THREAD);
 496 }
 497 
 498 // -----------------------------------------------------------------------------
 499 // Hash Code handling
 500 //
 501 // Performance concern:
 502 // OrderAccess::storestore() calls release() which at one time stored 0
 503 // into the global volatile OrderAccess::dummy variable. This store was
 504 // unnecessary for correctness. Many threads storing into a common location
 505 // causes considerable cache migration or "sloshing" on large SMP systems.
 506 // As such, I avoided using OrderAccess::storestore(). In some cases
 507 // OrderAccess::fence() -- which incurs local latency on the executing
 508 // processor -- is a better choice as it scales on SMP systems.
 509 //
 510 // See http://blogs.oracle.com/dave/entry/biased_locking_in_hotspot for
 511 // a discussion of coherency costs. Note that all our current reference
 512 // platforms provide strong ST-ST order, so the issue is moot on IA32,
 513 // x64, and SPARC.
 514 //
 515 // As a general policy we use "volatile" to control compiler-based reordering
 516 // and explicit fences (barriers) to control for architectural reordering
 517 // performed by the CPU(s) or platform.
 518 
 519 struct SharedGlobals {
 520   char         _pad_prefix[DEFAULT_CACHE_LINE_SIZE];
 521   // These are highly shared mostly-read variables.
 522   // To avoid false-sharing they need to be the sole occupants of a cache line.
 523   volatile int stw_random;
 524   volatile int stw_cycle;
 525   DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int) * 2);
 526   // Hot RW variable -- Sequester to avoid false-sharing
 527   volatile int hc_sequence;
 528   DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int));
 529 };
 530 
 531 static SharedGlobals GVars;
 532 static int MonitorScavengeThreshold = 1000000;
 533 static volatile int ForceMonitorScavenge = 0; // Scavenge required and pending
 534 
 535 static markWord read_stable_mark(oop obj) {
 536   markWord mark = obj->mark();
 537   if (!mark.is_being_inflated()) {
 538     return mark;       // normal fast-path return
 539   }
 540 
 541   int its = 0;
 542   for (;;) {
 543     markWord mark = obj->mark();
 544     if (!mark.is_being_inflated()) {
 545       return mark;    // normal fast-path return
 546     }
 547 
 548     // The object is being inflated by some other thread.
 549     // The caller of read_stable_mark() must wait for inflation to complete.
 550     // Avoid live-lock
 551     // TODO: consider calling SafepointSynchronize::do_call_back() while
 552     // spinning to see if there's a safepoint pending.  If so, immediately
 553     // yielding or blocking would be appropriate.  Avoid spinning while
 554     // there is a safepoint pending.
 555     // TODO: add inflation contention performance counters.
 556     // TODO: restrict the aggregate number of spinners.
 557 
 558     ++its;
 559     if (its > 10000 || !os::is_MP()) {
 560       if (its & 1) {
 561         os::naked_yield();
 562       } else {
 563         // Note that the following code attenuates the livelock problem but is not
 564         // a complete remedy.  A more complete solution would require that the inflating
 565         // thread hold the associated inflation lock.  The following code simply restricts
 566         // the number of spinners to at most one.  We'll have N-2 threads blocked
 567         // on the inflationlock, 1 thread holding the inflation lock and using
 568         // a yield/park strategy, and 1 thread in the midst of inflation.
 569         // A more refined approach would be to change the encoding of INFLATING
 570         // to allow encapsulation of a native thread pointer.  Threads waiting for
 571         // inflation to complete would use CAS to push themselves onto a singly linked
 572         // list rooted at the markword.  Once enqueued, they'd loop, checking a per-thread flag
 573         // and calling park().  When inflation was complete the thread that accomplished inflation
 574         // would detach the list and set the markword to inflated with a single CAS and
 575         // then for each thread on the list, set the flag and unpark() the thread.
 576         // This is conceptually similar to muxAcquire-muxRelease, except that muxRelease
 577         // wakes at most one thread whereas we need to wake the entire list.
 578         int ix = (cast_from_oop<intptr_t>(obj) >> 5) & (NINFLATIONLOCKS-1);
 579         int YieldThenBlock = 0;
 580         assert(ix >= 0 && ix < NINFLATIONLOCKS, "invariant");
 581         assert((NINFLATIONLOCKS & (NINFLATIONLOCKS-1)) == 0, "invariant");
 582         Thread::muxAcquire(gInflationLocks + ix, "gInflationLock");
 583         while (obj->mark() == markWord::INFLATING()) {
 584           // Beware: NakedYield() is advisory and has almost no effect on some platforms
 585           // so we periodically call self->_ParkEvent->park(1).
 586           // We use a mixed spin/yield/block mechanism.
 587           if ((YieldThenBlock++) >= 16) {
 588             Thread::current()->_ParkEvent->park(1);
 589           } else {
 590             os::naked_yield();
 591           }
 592         }
 593         Thread::muxRelease(gInflationLocks + ix);
 594       }
 595     } else {
 596       SpinPause();       // SMP-polite spinning
 597     }
 598   }
 599 }
 600 
 601 // hashCode() generation :
 602 //
 603 // Possibilities:
 604 // * MD5Digest of {obj,stw_random}
 605 // * CRC32 of {obj,stw_random} or any linear-feedback shift register function.
 606 // * A DES- or AES-style SBox[] mechanism
 607 // * One of the Phi-based schemes, such as:
 608 //   2654435761 = 2^32 * Phi (golden ratio)
 609 //   HashCodeValue = ((uintptr_t(obj) >> 3) * 2654435761) ^ GVars.stw_random ;
 610 // * A variation of Marsaglia's shift-xor RNG scheme.
 611 // * (obj ^ stw_random) is appealing, but can result
 612 //   in undesirable regularity in the hashCode values of adjacent objects
 613 //   (objects allocated back-to-back, in particular).  This could potentially
 614 //   result in hashtable collisions and reduced hashtable efficiency.
 615 //   There are simple ways to "diffuse" the middle address bits over the
 616 //   generated hashCode values:
 617 
 618 static inline intptr_t get_next_hash(Thread* self, oop obj) {
 619   intptr_t value = 0;
 620   if (hashCode == 0) {
 621     // This form uses global Park-Miller RNG.
 622     // On MP system we'll have lots of RW access to a global, so the
 623     // mechanism induces lots of coherency traffic.
 624     value = os::random();
 625   } else if (hashCode == 1) {
 626     // This variation has the property of being stable (idempotent)
 627     // between STW operations.  This can be useful in some of the 1-0
 628     // synchronization schemes.
 629     intptr_t addr_bits = cast_from_oop<intptr_t>(obj) >> 3;
 630     value = addr_bits ^ (addr_bits >> 5) ^ GVars.stw_random;
 631   } else if (hashCode == 2) {
 632     value = 1;            // for sensitivity testing
 633   } else if (hashCode == 3) {
 634     value = ++GVars.hc_sequence;
 635   } else if (hashCode == 4) {
 636     value = cast_from_oop<intptr_t>(obj);
 637   } else {
 638     // Marsaglia's xor-shift scheme with thread-specific state
 639     // This is probably the best overall implementation -- we'll
 640     // likely make this the default in future releases.
 641     unsigned t = self->_hashStateX;
 642     t ^= (t << 11);
 643     self->_hashStateX = self->_hashStateY;
 644     self->_hashStateY = self->_hashStateZ;
 645     self->_hashStateZ = self->_hashStateW;
 646     unsigned v = self->_hashStateW;
 647     v = (v ^ (v >> 19)) ^ (t ^ (t >> 8));
 648     self->_hashStateW = v;
 649     value = v;
 650   }
 651 
 652   value &= markWord::hash_mask;
 653   if (value == 0) value = 0xBAD;
 654   assert(value != markWord::no_hash, "invariant");
 655   return value;
 656 }
 657 
 658 intptr_t ObjectSynchronizer::FastHashCode(Thread* self, oop obj) {
 659   if (UseBiasedLocking) {
 660     // NOTE: many places throughout the JVM do not expect a safepoint
 661     // to be taken here, in particular most operations on perm gen
 662     // objects. However, we only ever bias Java instances and all of
 663     // the call sites of identity_hash that might revoke biases have
 664     // been checked to make sure they can handle a safepoint. The
 665     // added check of the bias pattern is to avoid useless calls to
 666     // thread-local storage.
 667     if (obj->mark().has_bias_pattern()) {
 668       // Handle for oop obj in case of STW safepoint
 669       Handle hobj(self, obj);
 670       // Relaxing assertion for bug 6320749.
 671       assert(Universe::verify_in_progress() ||
 672              !SafepointSynchronize::is_at_safepoint(),
 673              "biases should not be seen by VM thread here");
 674       BiasedLocking::revoke(hobj, JavaThread::current());
 675       obj = hobj();
 676       assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
 677     }
 678   }
 679 
 680   // hashCode() is a heap mutator ...
 681   // Relaxing assertion for bug 6320749.
 682   assert(Universe::verify_in_progress() || DumpSharedSpaces ||
 683          !SafepointSynchronize::is_at_safepoint(), "invariant");
 684   assert(Universe::verify_in_progress() || DumpSharedSpaces ||
 685          self->is_Java_thread() , "invariant");
 686   assert(Universe::verify_in_progress() || DumpSharedSpaces ||
 687          ((JavaThread *)self)->thread_state() != _thread_blocked, "invariant");
 688 
 689   ObjectMonitor* monitor = NULL;
 690   markWord temp, test;
 691   intptr_t hash;
 692   markWord mark = read_stable_mark(obj);
 693 
 694   // object should remain ineligible for biased locking
 695   assert(!mark.has_bias_pattern(), "invariant");
 696 
 697   if (mark.is_neutral()) {
 698     hash = mark.hash();               // this is a normal header
 699     if (hash != 0) {                  // if it has hash, just return it
 700       return hash;
 701     }
 702     hash = get_next_hash(self, obj);  // allocate a new hash code
 703     temp = mark.copy_set_hash(hash);  // merge the hash code into header
 704     // use (machine word version) atomic operation to install the hash
 705     test = obj->cas_set_mark(temp, mark);
 706     if (test == mark) {
 707       return hash;
 708     }
 709     // If atomic operation failed, we must inflate the header
 710     // into heavy weight monitor. We could add more code here
 711     // for fast path, but it does not worth the complexity.
 712   } else if (mark.has_monitor()) {
 713     monitor = mark.monitor();
 714     temp = monitor->header();
 715     assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
 716     hash = temp.hash();
 717     if (hash != 0) {
 718       return hash;
 719     }
 720     // Skip to the following code to reduce code size
 721   } else if (self->is_lock_owned((address)mark.locker())) {
 722     temp = mark.displaced_mark_helper(); // this is a lightweight monitor owned
 723     assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
 724     hash = temp.hash();                  // by current thread, check if the displaced
 725     if (hash != 0) {                     // header contains hash code
 726       return hash;
 727     }
 728     // WARNING:
 729     // The displaced header in the BasicLock on a thread's stack
 730     // is strictly immutable. It CANNOT be changed in ANY cases.
 731     // So we have to inflate the stack lock into an ObjectMonitor
 732     // even if the current thread owns the lock. The BasicLock on
 733     // a thread's stack can be asynchronously read by other threads
 734     // during an inflate() call so any change to that stack memory
 735     // may not propagate to other threads correctly.
 736   }
 737 
 738   // Inflate the monitor to set hash code
 739   monitor = inflate(self, obj, inflate_cause_hash_code);
 740   // Load displaced header and check it has hash code
 741   mark = monitor->header();
 742   assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
 743   hash = mark.hash();
 744   if (hash == 0) {
 745     hash = get_next_hash(self, obj);
 746     temp = mark.copy_set_hash(hash); // merge hash code into header
 747     assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
 748     uintptr_t v = Atomic::cmpxchg(temp.value(), (volatile uintptr_t*)monitor->header_addr(), mark.value());
 749     test = markWord(v);
 750     if (test != mark) {
 751       // The only non-deflation update to the ObjectMonitor's
 752       // header/dmw field is to merge in the hash code. If someone
 753       // adds a new usage of the header/dmw field, please update
 754       // this code.
 755       hash = test.hash();
 756       assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value());
 757       assert(hash != 0, "Trivial unexpected object/monitor header usage.");
 758     }
 759   }
 760   // We finally get the hash
 761   return hash;
 762 }
 763 
 764 // Deprecated -- use FastHashCode() instead.
 765 
 766 intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) {
 767   return FastHashCode(Thread::current(), obj());
 768 }
 769 
 770 
 771 bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
 772                                                    Handle h_obj) {
 773   if (UseBiasedLocking) {
 774     BiasedLocking::revoke(h_obj, thread);
 775     assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
 776   }
 777 
 778   assert(thread == JavaThread::current(), "Can only be called on current thread");
 779   oop obj = h_obj();
 780 
 781   markWord mark = read_stable_mark(obj);
 782 
 783   // Uncontended case, header points to stack
 784   if (mark.has_locker()) {
 785     return thread->is_lock_owned((address)mark.locker());
 786   }
 787   // Contended case, header points to ObjectMonitor (tagged pointer)
 788   if (mark.has_monitor()) {
 789     ObjectMonitor* monitor = mark.monitor();
 790     return monitor->is_entered(thread) != 0;
 791   }
 792   // Unlocked case, header in place
 793   assert(mark.is_neutral(), "sanity check");
 794   return false;
 795 }
 796 
 797 // Be aware of this method could revoke bias of the lock object.
 798 // This method queries the ownership of the lock handle specified by 'h_obj'.
 799 // If the current thread owns the lock, it returns owner_self. If no
 800 // thread owns the lock, it returns owner_none. Otherwise, it will return
 801 // owner_other.
 802 ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership
 803 (JavaThread *self, Handle h_obj) {
 804   // The caller must beware this method can revoke bias, and
 805   // revocation can result in a safepoint.
 806   assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
 807   assert(self->thread_state() != _thread_blocked, "invariant");
 808 
 809   // Possible mark states: neutral, biased, stack-locked, inflated
 810 
 811   if (UseBiasedLocking && h_obj()->mark().has_bias_pattern()) {
 812     // CASE: biased
 813     BiasedLocking::revoke(h_obj, self);
 814     assert(!h_obj->mark().has_bias_pattern(),
 815            "biases should be revoked by now");
 816   }
 817 
 818   assert(self == JavaThread::current(), "Can only be called on current thread");
 819   oop obj = h_obj();
 820   markWord mark = read_stable_mark(obj);
 821 
 822   // CASE: stack-locked.  Mark points to a BasicLock on the owner's stack.
 823   if (mark.has_locker()) {
 824     return self->is_lock_owned((address)mark.locker()) ?
 825       owner_self : owner_other;
 826   }
 827 
 828   // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor.
 829   // The Object:ObjectMonitor relationship is stable as long as we're
 830   // not at a safepoint.
 831   if (mark.has_monitor()) {
 832     void* owner = mark.monitor()->_owner;
 833     if (owner == NULL) return owner_none;
 834     return (owner == self ||
 835             self->is_lock_owned((address)owner)) ? owner_self : owner_other;
 836   }
 837 
 838   // CASE: neutral
 839   assert(mark.is_neutral(), "sanity check");
 840   return owner_none;           // it's unlocked
 841 }
 842 
 843 // FIXME: jvmti should call this
 844 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
 845   if (UseBiasedLocking) {
 846     if (SafepointSynchronize::is_at_safepoint()) {
 847       BiasedLocking::revoke_at_safepoint(h_obj);
 848     } else {
 849       BiasedLocking::revoke(h_obj, JavaThread::current());
 850     }
 851     assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
 852   }
 853 
 854   oop obj = h_obj();
 855   address owner = NULL;
 856 
 857   markWord mark = read_stable_mark(obj);
 858 
 859   // Uncontended case, header points to stack
 860   if (mark.has_locker()) {
 861     owner = (address) mark.locker();
 862   }
 863 
 864   // Contended case, header points to ObjectMonitor (tagged pointer)
 865   else if (mark.has_monitor()) {
 866     ObjectMonitor* monitor = mark.monitor();
 867     assert(monitor != NULL, "monitor should be non-null");
 868     owner = (address) monitor->owner();
 869   }
 870 
 871   if (owner != NULL) {
 872     // owning_thread_from_monitor_owner() may also return NULL here
 873     return Threads::owning_thread_from_monitor_owner(t_list, owner);
 874   }
 875 
 876   // Unlocked case, header in place
 877   // Cannot have assertion since this object may have been
 878   // locked by another thread when reaching here.
 879   // assert(mark.is_neutral(), "sanity check");
 880 
 881   return NULL;
 882 }
 883 
 884 // Visitors ...
 885 
 886 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
 887   PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
 888   while (block != NULL) {
 889     assert(block->object() == CHAINMARKER, "must be a block header");
 890     for (int i = _BLOCKSIZE - 1; i > 0; i--) {
 891       ObjectMonitor* mid = (ObjectMonitor *)(block + i);
 892       oop object = (oop)mid->object();
 893       if (object != NULL) {
 894         // Only process with closure if the object is set.
 895         closure->do_monitor(mid);
 896       }
 897     }
 898     block = (PaddedObjectMonitor*)block->_next_om;
 899   }
 900 }
 901 
 902 static bool monitors_used_above_threshold() {
 903   if (g_om_population == 0) {
 904     return false;
 905   }
 906   int monitors_used = g_om_population - g_om_free_count;
 907   int monitor_usage = (monitors_used * 100LL) / g_om_population;
 908   return monitor_usage > MonitorUsedDeflationThreshold;
 909 }
 910 
 911 bool ObjectSynchronizer::is_cleanup_needed() {
 912   if (MonitorUsedDeflationThreshold > 0) {
 913     return monitors_used_above_threshold();
 914   }
 915   return false;
 916 }
 917 
 918 void ObjectSynchronizer::oops_do(OopClosure* f) {
 919   // We only scan the global used list here (for moribund threads), and
 920   // the thread-local monitors in Thread::oops_do().
 921   global_used_oops_do(f);
 922 }
 923 
 924 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
 925   assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
 926   list_oops_do(g_om_in_use_list, f);
 927 }
 928 
 929 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
 930   assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
 931   list_oops_do(thread->om_in_use_list, f);
 932 }
 933 
 934 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, OopClosure* f) {
 935   assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
 936   ObjectMonitor* mid;
 937   for (mid = list; mid != NULL; mid = mid->_next_om) {
 938     if (mid->object() != NULL) {
 939       f->do_oop((oop*)mid->object_addr());
 940     }
 941   }
 942 }
 943 
 944 
 945 // -----------------------------------------------------------------------------
 946 // ObjectMonitor Lifecycle
 947 // -----------------------
 948 // Inflation unlinks monitors from the global g_free_list and
 949 // associates them with objects.  Deflation -- which occurs at
 950 // STW-time -- disassociates idle monitors from objects.  Such
 951 // scavenged monitors are returned to the g_free_list.
 952 //
 953 // The global list is protected by gListLock.  All the critical sections
 954 // are short and operate in constant-time.
 955 //
 956 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
 957 //
 958 // Lifecycle:
 959 // --   unassigned and on the global free list
 960 // --   unassigned and on a thread's private om_free_list
 961 // --   assigned to an object.  The object is inflated and the mark refers
 962 //      to the objectmonitor.
 963 
 964 
 965 // Constraining monitor pool growth via MonitorBound ...
 966 //
 967 // If MonitorBound is not set (<= 0), MonitorBound checks are disabled.
 968 //
 969 // The monitor pool is grow-only.  We scavenge at STW safepoint-time, but the
 970 // the rate of scavenging is driven primarily by GC.  As such,  we can find
 971 // an inordinate number of monitors in circulation.
 972 // To avoid that scenario we can artificially induce a STW safepoint
 973 // if the pool appears to be growing past some reasonable bound.
 974 // Generally we favor time in space-time tradeoffs, but as there's no
 975 // natural back-pressure on the # of extant monitors we need to impose some
 976 // type of limit.  Beware that if MonitorBound is set to too low a value
 977 // we could just loop. In addition, if MonitorBound is set to a low value
 978 // we'll incur more safepoints, which are harmful to performance.
 979 // See also: GuaranteedSafepointInterval
 980 //
 981 // The current implementation uses asynchronous VM operations.
 982 //
 983 // If MonitorBound is set, the boundry applies to
 984 //     (g_om_population - g_om_free_count)
 985 // i.e., if there are not enough ObjectMonitors on the global free list,
 986 // then a safepoint deflation is induced. Picking a good MonitorBound value
 987 // is non-trivial.
 988 
 989 static void InduceScavenge(Thread* self, const char * Whence) {
 990   // Induce STW safepoint to trim monitors
 991   // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
 992   // More precisely, trigger an asynchronous STW safepoint as the number
 993   // of active monitors passes the specified threshold.
 994   // TODO: assert thread state is reasonable
 995 
 996   if (ForceMonitorScavenge == 0 && Atomic::xchg (1, &ForceMonitorScavenge) == 0) {
 997     // Induce a 'null' safepoint to scavenge monitors
 998     // Must VM_Operation instance be heap allocated as the op will be enqueue and posted
 999     // to the VMthread and have a lifespan longer than that of this activation record.
1000     // The VMThread will delete the op when completed.
1001     VMThread::execute(new VM_ScavengeMonitors());
1002   }
1003 }
1004 
1005 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self) {
1006   // A large MAXPRIVATE value reduces both list lock contention
1007   // and list coherency traffic, but also tends to increase the
1008   // number of ObjectMonitors in circulation as well as the STW
1009   // scavenge costs.  As usual, we lean toward time in space-time
1010   // tradeoffs.
1011   const int MAXPRIVATE = 1024;
1012   stringStream ss;
1013   for (;;) {
1014     ObjectMonitor* m;
1015 
1016     // 1: try to allocate from the thread's local om_free_list.
1017     // Threads will attempt to allocate first from their local list, then
1018     // from the global list, and only after those attempts fail will the thread
1019     // attempt to instantiate new monitors.   Thread-local free lists take
1020     // heat off the gListLock and improve allocation latency, as well as reducing
1021     // coherency traffic on the shared global list.
1022     m = self->om_free_list;
1023     if (m != NULL) {
1024       self->om_free_list = m->_next_om;
1025       self->om_free_count--;
1026       guarantee(m->object() == NULL, "invariant");
1027       m->_next_om = self->om_in_use_list;
1028       self->om_in_use_list = m;
1029       self->om_in_use_count++;
1030       return m;
1031     }
1032 
1033     // 2: try to allocate from the global g_free_list
1034     // CONSIDER: use muxTry() instead of muxAcquire().
1035     // If the muxTry() fails then drop immediately into case 3.
1036     // If we're using thread-local free lists then try
1037     // to reprovision the caller's free list.
1038     if (g_free_list != NULL) {
1039       // Reprovision the thread's om_free_list.
1040       // Use bulk transfers to reduce the allocation rate and heat
1041       // on various locks.
1042       Thread::muxAcquire(&gListLock, "om_alloc(1)");
1043       for (int i = self->om_free_provision; --i >= 0 && g_free_list != NULL;) {
1044         g_om_free_count--;
1045         ObjectMonitor* take = g_free_list;
1046         g_free_list = take->_next_om;
1047         guarantee(take->object() == NULL, "invariant");
1048         take->Recycle();
1049         om_release(self, take, false);
1050       }
1051       Thread::muxRelease(&gListLock);
1052       self->om_free_provision += 1 + (self->om_free_provision/2);
1053       if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1054 
1055       const int mx = MonitorBound;
1056       if (mx > 0 && (g_om_population-g_om_free_count) > mx) {
1057         // Not enough ObjectMonitors on the global free list.
1058         // We can't safely induce a STW safepoint from om_alloc() as our thread
1059         // state may not be appropriate for such activities and callers may hold
1060         // naked oops, so instead we defer the action.
1061         InduceScavenge(self, "om_alloc");
1062       }
1063       continue;
1064     }
1065 
1066     // 3: allocate a block of new ObjectMonitors
1067     // Both the local and global free lists are empty -- resort to malloc().
1068     // In the current implementation ObjectMonitors are TSM - immortal.
1069     // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1070     // each ObjectMonitor to start at the beginning of a cache line,
1071     // so we use align_up().
1072     // A better solution would be to use C++ placement-new.
1073     // BEWARE: As it stands currently, we don't run the ctors!
1074     assert(_BLOCKSIZE > 1, "invariant");
1075     size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE;
1076     PaddedObjectMonitor* temp;
1077     size_t aligned_size = neededsize + (DEFAULT_CACHE_LINE_SIZE - 1);
1078     void* real_malloc_addr = NEW_C_HEAP_ARRAY(char, aligned_size, mtInternal);
1079     temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, DEFAULT_CACHE_LINE_SIZE);
1080     (void)memset((void *) temp, 0, neededsize);
1081 
1082     // Format the block.
1083     // initialize the linked list, each monitor points to its next
1084     // forming the single linked free list, the very first monitor
1085     // will points to next block, which forms the block list.
1086     // The trick of using the 1st element in the block as g_block_list
1087     // linkage should be reconsidered.  A better implementation would
1088     // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; }
1089 
1090     for (int i = 1; i < _BLOCKSIZE; i++) {
1091       temp[i]._next_om = (ObjectMonitor *)&temp[i+1];
1092     }
1093 
1094     // terminate the last monitor as the end of list
1095     temp[_BLOCKSIZE - 1]._next_om = NULL;
1096 
1097     // Element [0] is reserved for global list linkage
1098     temp[0].set_object(CHAINMARKER);
1099 
1100     // Consider carving out this thread's current request from the
1101     // block in hand.  This avoids some lock traffic and redundant
1102     // list activity.
1103 
1104     // Acquire the gListLock to manipulate g_block_list and g_free_list.
1105     // An Oyama-Taura-Yonezawa scheme might be more efficient.
1106     Thread::muxAcquire(&gListLock, "om_alloc(2)");
1107     g_om_population += _BLOCKSIZE-1;
1108     g_om_free_count += _BLOCKSIZE-1;
1109 
1110     // Add the new block to the list of extant blocks (g_block_list).
1111     // The very first ObjectMonitor in a block is reserved and dedicated.
1112     // It serves as blocklist "next" linkage.
1113     temp[0]._next_om = g_block_list;
1114     // There are lock-free uses of g_block_list so make sure that
1115     // the previous stores happen before we update g_block_list.
1116     OrderAccess::release_store(&g_block_list, temp);
1117 
1118     // Add the new string of ObjectMonitors to the global free list
1119     temp[_BLOCKSIZE - 1]._next_om = g_free_list;
1120     g_free_list = temp + 1;
1121     Thread::muxRelease(&gListLock);
1122   }
1123 }
1124 
1125 // Place "m" on the caller's private per-thread om_free_list.
1126 // In practice there's no need to clamp or limit the number of
1127 // monitors on a thread's om_free_list as the only non-allocation time
1128 // we'll call om_release() is to return a monitor to the free list after
1129 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1130 // accumulate on a thread's free list.
1131 //
1132 // Key constraint: all ObjectMonitors on a thread's free list and the global
1133 // free list must have their object field set to null. This prevents the
1134 // scavenger -- deflate_monitor_list() -- from reclaiming them while we
1135 // are trying to release them.
1136 
1137 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1138                                     bool from_per_thread_alloc) {
1139   guarantee(m->header().value() == 0, "invariant");
1140   guarantee(m->object() == NULL, "invariant");
1141   stringStream ss;
1142   guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1143             "%s, recursions=" INTX_FORMAT, m->is_busy_to_string(&ss),
1144             m->_recursions);
1145   // _next_om is used for both per-thread in-use and free lists so
1146   // we have to remove 'm' from the in-use list first (as needed).
1147   if (from_per_thread_alloc) {
1148     // Need to remove 'm' from om_in_use_list.
1149     ObjectMonitor* cur_mid_in_use = NULL;
1150     bool extracted = false;
1151     for (ObjectMonitor* mid = self->om_in_use_list; mid != NULL; cur_mid_in_use = mid, mid = mid->_next_om) {
1152       if (m == mid) {
1153         // extract from per-thread in-use list
1154         if (mid == self->om_in_use_list) {
1155           self->om_in_use_list = mid->_next_om;
1156         } else if (cur_mid_in_use != NULL) {
1157           cur_mid_in_use->_next_om = mid->_next_om; // maintain the current thread in-use list
1158         }
1159         extracted = true;
1160         self->om_in_use_count--;
1161         break;
1162       }
1163     }
1164     assert(extracted, "Should have extracted from in-use list");
1165   }
1166 
1167   m->_next_om = self->om_free_list;
1168   self->om_free_list = m;
1169   self->om_free_count++;
1170 }
1171 
1172 // Return ObjectMonitors on a moribund thread's free and in-use
1173 // lists to the appropriate global lists. The ObjectMonitors on the
1174 // per-thread in-use list may still be in use by other threads.
1175 //
1176 // We currently call om_flush() from Threads::remove() before the
1177 // thread has been excised from the thread list and is no longer a
1178 // mutator. This means that om_flush() cannot run concurrently with
1179 // a safepoint and interleave with deflate_idle_monitors(). In
1180 // particular, this ensures that the thread's in-use monitors are
1181 // scanned by a GC safepoint, either via Thread::oops_do() (before
1182 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1183 // om_flush() is called).
1184 
1185 void ObjectSynchronizer::om_flush(Thread* self) {
1186   ObjectMonitor* free_list = self->om_free_list;
1187   ObjectMonitor* free_tail = NULL;
1188   int free_count = 0;
1189   if (free_list != NULL) {
1190     ObjectMonitor* s;
1191     // The thread is going away. Set 'free_tail' to the last per-thread free
1192     // monitor which will be linked to g_free_list below under the gListLock.
1193     stringStream ss;
1194     for (s = free_list; s != NULL; s = s->_next_om) {
1195       free_count++;
1196       free_tail = s;
1197       guarantee(s->object() == NULL, "invariant");
1198       guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss));
1199     }
1200     guarantee(free_tail != NULL, "invariant");
1201     assert(self->om_free_count == free_count, "free-count off");
1202     self->om_free_list = NULL;
1203     self->om_free_count = 0;
1204   }
1205 
1206   ObjectMonitor* in_use_list = self->om_in_use_list;
1207   ObjectMonitor* in_use_tail = NULL;
1208   int in_use_count = 0;
1209   if (in_use_list != NULL) {
1210     // The thread is going away, however the ObjectMonitors on the
1211     // om_in_use_list may still be in-use by other threads. Link
1212     // them to in_use_tail, which will be linked into the global
1213     // in-use list g_om_in_use_list below, under the gListLock.
1214     ObjectMonitor *cur_om;
1215     for (cur_om = in_use_list; cur_om != NULL; cur_om = cur_om->_next_om) {
1216       in_use_tail = cur_om;
1217       in_use_count++;
1218     }
1219     guarantee(in_use_tail != NULL, "invariant");
1220     assert(self->om_in_use_count == in_use_count, "in-use count off");
1221     self->om_in_use_list = NULL;
1222     self->om_in_use_count = 0;
1223   }
1224 
1225   Thread::muxAcquire(&gListLock, "om_flush");
1226   if (free_tail != NULL) {
1227     free_tail->_next_om = g_free_list;
1228     g_free_list = free_list;
1229     g_om_free_count += free_count;
1230   }
1231 
1232   if (in_use_tail != NULL) {
1233     in_use_tail->_next_om = g_om_in_use_list;
1234     g_om_in_use_list = in_use_list;
1235     g_om_in_use_count += in_use_count;
1236   }
1237 
1238   Thread::muxRelease(&gListLock);
1239 
1240   LogStreamHandle(Debug, monitorinflation) lsh_debug;
1241   LogStreamHandle(Info, monitorinflation) lsh_info;
1242   LogStream* ls = NULL;
1243   if (log_is_enabled(Debug, monitorinflation)) {
1244     ls = &lsh_debug;
1245   } else if ((free_count != 0 || in_use_count != 0) &&
1246              log_is_enabled(Info, monitorinflation)) {
1247     ls = &lsh_info;
1248   }
1249   if (ls != NULL) {
1250     ls->print_cr("om_flush: jt=" INTPTR_FORMAT ", free_count=%d"
1251                  ", in_use_count=%d" ", om_free_provision=%d",
1252                  p2i(self), free_count, in_use_count, self->om_free_provision);
1253   }
1254 }
1255 
1256 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1257                                        const oop obj,
1258                                        ObjectSynchronizer::InflateCause cause) {
1259   assert(event != NULL, "invariant");
1260   assert(event->should_commit(), "invariant");
1261   event->set_monitorClass(obj->klass());
1262   event->set_address((uintptr_t)(void*)obj);
1263   event->set_cause((u1)cause);
1264   event->commit();
1265 }
1266 
1267 // Fast path code shared by multiple functions
1268 void ObjectSynchronizer::inflate_helper(oop obj) {
1269   markWord mark = obj->mark();
1270   if (mark.has_monitor()) {
1271     assert(ObjectSynchronizer::verify_objmon_isinpool(mark.monitor()), "monitor is invalid");
1272     assert(mark.monitor()->header().is_neutral(), "monitor must record a good object header");
1273     return;
1274   }
1275   inflate(Thread::current(), obj, inflate_cause_vm_internal);
1276 }
1277 
1278 ObjectMonitor* ObjectSynchronizer::inflate(Thread* self,
1279                                            oop object,
1280                                            const InflateCause cause) {
1281   // Inflate mutates the heap ...
1282   // Relaxing assertion for bug 6320749.
1283   assert(Universe::verify_in_progress() ||
1284          !SafepointSynchronize::is_at_safepoint(), "invariant");
1285 
1286   EventJavaMonitorInflate event;
1287 
1288   for (;;) {
1289     const markWord mark = object->mark();
1290     assert(!mark.has_bias_pattern(), "invariant");
1291 
1292     // The mark can be in one of the following states:
1293     // *  Inflated     - just return
1294     // *  Stack-locked - coerce it to inflated
1295     // *  INFLATING    - busy wait for conversion to complete
1296     // *  Neutral      - aggressively inflate the object.
1297     // *  BIASED       - Illegal.  We should never see this
1298 
1299     // CASE: inflated
1300     if (mark.has_monitor()) {
1301       ObjectMonitor* inf = mark.monitor();
1302       markWord dmw = inf->header();
1303       assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1304       assert(inf->object() == object, "invariant");
1305       assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
1306       return inf;
1307     }
1308 
1309     // CASE: inflation in progress - inflating over a stack-lock.
1310     // Some other thread is converting from stack-locked to inflated.
1311     // Only that thread can complete inflation -- other threads must wait.
1312     // The INFLATING value is transient.
1313     // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1314     // We could always eliminate polling by parking the thread on some auxiliary list.
1315     if (mark == markWord::INFLATING()) {
1316       read_stable_mark(object);
1317       continue;
1318     }
1319 
1320     // CASE: stack-locked
1321     // Could be stack-locked either by this thread or by some other thread.
1322     //
1323     // Note that we allocate the objectmonitor speculatively, _before_ attempting
1324     // to install INFLATING into the mark word.  We originally installed INFLATING,
1325     // allocated the objectmonitor, and then finally STed the address of the
1326     // objectmonitor into the mark.  This was correct, but artificially lengthened
1327     // the interval in which INFLATED appeared in the mark, thus increasing
1328     // the odds of inflation contention.
1329     //
1330     // We now use per-thread private objectmonitor free lists.
1331     // These list are reprovisioned from the global free list outside the
1332     // critical INFLATING...ST interval.  A thread can transfer
1333     // multiple objectmonitors en-mass from the global free list to its local free list.
1334     // This reduces coherency traffic and lock contention on the global free list.
1335     // Using such local free lists, it doesn't matter if the om_alloc() call appears
1336     // before or after the CAS(INFLATING) operation.
1337     // See the comments in om_alloc().
1338 
1339     LogStreamHandle(Trace, monitorinflation) lsh;
1340 
1341     if (mark.has_locker()) {
1342       ObjectMonitor* m = om_alloc(self);
1343       // Optimistically prepare the objectmonitor - anticipate successful CAS
1344       // We do this before the CAS in order to minimize the length of time
1345       // in which INFLATING appears in the mark.
1346       m->Recycle();
1347       m->_Responsible  = NULL;
1348       m->_SpinDuration = ObjectMonitor::Knob_SpinLimit;   // Consider: maintain by type/class
1349 
1350       markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1351       if (cmp != mark) {
1352         om_release(self, m, true);
1353         continue;       // Interference -- just retry
1354       }
1355 
1356       // We've successfully installed INFLATING (0) into the mark-word.
1357       // This is the only case where 0 will appear in a mark-word.
1358       // Only the singular thread that successfully swings the mark-word
1359       // to 0 can perform (or more precisely, complete) inflation.
1360       //
1361       // Why do we CAS a 0 into the mark-word instead of just CASing the
1362       // mark-word from the stack-locked value directly to the new inflated state?
1363       // Consider what happens when a thread unlocks a stack-locked object.
1364       // It attempts to use CAS to swing the displaced header value from the
1365       // on-stack BasicLock back into the object header.  Recall also that the
1366       // header value (hash code, etc) can reside in (a) the object header, or
1367       // (b) a displaced header associated with the stack-lock, or (c) a displaced
1368       // header in an ObjectMonitor.  The inflate() routine must copy the header
1369       // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1370       // the while preserving the hashCode stability invariants.  If the owner
1371       // decides to release the lock while the value is 0, the unlock will fail
1372       // and control will eventually pass from slow_exit() to inflate.  The owner
1373       // will then spin, waiting for the 0 value to disappear.   Put another way,
1374       // the 0 causes the owner to stall if the owner happens to try to
1375       // drop the lock (restoring the header from the BasicLock to the object)
1376       // while inflation is in-progress.  This protocol avoids races that might
1377       // would otherwise permit hashCode values to change or "flicker" for an object.
1378       // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1379       // 0 serves as a "BUSY" inflate-in-progress indicator.
1380 
1381 
1382       // fetch the displaced mark from the owner's stack.
1383       // The owner can't die or unwind past the lock while our INFLATING
1384       // object is in the mark.  Furthermore the owner can't complete
1385       // an unlock on the object, either.
1386       markWord dmw = mark.displaced_mark_helper();
1387       // Catch if the object's header is not neutral (not locked and
1388       // not marked is what we care about here).
1389       assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1390 
1391       // Setup monitor fields to proper values -- prepare the monitor
1392       m->set_header(dmw);
1393 
1394       // Optimization: if the mark.locker stack address is associated
1395       // with this thread we could simply set m->_owner = self.
1396       // Note that a thread can inflate an object
1397       // that it has stack-locked -- as might happen in wait() -- directly
1398       // with CAS.  That is, we can avoid the xchg-NULL .... ST idiom.
1399       m->set_owner(mark.locker());
1400       m->set_object(object);
1401       // TODO-FIXME: assert BasicLock->dhw != 0.
1402 
1403       // Must preserve store ordering. The monitor state must
1404       // be stable at the time of publishing the monitor address.
1405       guarantee(object->mark() == markWord::INFLATING(), "invariant");
1406       object->release_set_mark(markWord::encode(m));
1407 
1408       // Hopefully the performance counters are allocated on distinct cache lines
1409       // to avoid false sharing on MP systems ...
1410       OM_PERFDATA_OP(Inflations, inc());
1411       if (log_is_enabled(Trace, monitorinflation)) {
1412         ResourceMark rm(self);
1413         lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
1414                      INTPTR_FORMAT ", type='%s'", p2i(object),
1415                      object->mark().value(), object->klass()->external_name());
1416       }
1417       if (event.should_commit()) {
1418         post_monitor_inflate_event(&event, object, cause);
1419       }
1420       return m;
1421     }
1422 
1423     // CASE: neutral
1424     // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
1425     // If we know we're inflating for entry it's better to inflate by swinging a
1426     // pre-locked ObjectMonitor pointer into the object header.   A successful
1427     // CAS inflates the object *and* confers ownership to the inflating thread.
1428     // In the current implementation we use a 2-step mechanism where we CAS()
1429     // to inflate and then CAS() again to try to swing _owner from NULL to self.
1430     // An inflateTry() method that we could call from enter() would be useful.
1431 
1432     // Catch if the object's header is not neutral (not locked and
1433     // not marked is what we care about here).
1434     assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1435     ObjectMonitor* m = om_alloc(self);
1436     // prepare m for installation - set monitor to initial state
1437     m->Recycle();
1438     m->set_header(mark);
1439     m->set_object(object);
1440     m->_Responsible  = NULL;
1441     m->_SpinDuration = ObjectMonitor::Knob_SpinLimit;       // consider: keep metastats by type/class
1442 
1443     if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
1444       m->set_header(markWord::zero());
1445       m->set_object(NULL);
1446       m->Recycle();
1447       om_release(self, m, true);
1448       m = NULL;
1449       continue;
1450       // interference - the markword changed - just retry.
1451       // The state-transitions are one-way, so there's no chance of
1452       // live-lock -- "Inflated" is an absorbing state.
1453     }
1454 
1455     // Hopefully the performance counters are allocated on distinct
1456     // cache lines to avoid false sharing on MP systems ...
1457     OM_PERFDATA_OP(Inflations, inc());
1458     if (log_is_enabled(Trace, monitorinflation)) {
1459       ResourceMark rm(self);
1460       lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
1461                    INTPTR_FORMAT ", type='%s'", p2i(object),
1462                    object->mark().value(), object->klass()->external_name());
1463     }
1464     if (event.should_commit()) {
1465       post_monitor_inflate_event(&event, object, cause);
1466     }
1467     return m;
1468   }
1469 }
1470 
1471 
1472 // We maintain a list of in-use monitors for each thread.
1473 //
1474 // deflate_thread_local_monitors() scans a single thread's in-use list, while
1475 // deflate_idle_monitors() scans only a global list of in-use monitors which
1476 // is populated only as a thread dies (see om_flush()).
1477 //
1478 // These operations are called at all safepoints, immediately after mutators
1479 // are stopped, but before any objects have moved. Collectively they traverse
1480 // the population of in-use monitors, deflating where possible. The scavenged
1481 // monitors are returned to the global monitor free list.
1482 //
1483 // Beware that we scavenge at *every* stop-the-world point. Having a large
1484 // number of monitors in-use could negatively impact performance. We also want
1485 // to minimize the total # of monitors in circulation, as they incur a small
1486 // footprint penalty.
1487 //
1488 // Perversely, the heap size -- and thus the STW safepoint rate --
1489 // typically drives the scavenge rate.  Large heaps can mean infrequent GC,
1490 // which in turn can mean large(r) numbers of ObjectMonitors in circulation.
1491 // This is an unfortunate aspect of this design.
1492 
1493 // Deflate a single monitor if not in-use
1494 // Return true if deflated, false if in-use
1495 bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj,
1496                                          ObjectMonitor** free_head_p,
1497                                          ObjectMonitor** free_tail_p) {
1498   bool deflated;
1499   // Normal case ... The monitor is associated with obj.
1500   const markWord mark = obj->mark();
1501   guarantee(mark == markWord::encode(mid), "should match: mark="
1502             INTPTR_FORMAT ", encoded mid=" INTPTR_FORMAT, mark.value(),
1503             markWord::encode(mid).value());
1504   // Make sure that mark.monitor() and markWord::encode() agree:
1505   guarantee(mark.monitor() == mid, "should match: monitor()=" INTPTR_FORMAT
1506             ", mid=" INTPTR_FORMAT, p2i(mark.monitor()), p2i(mid));
1507   const markWord dmw = mid->header();
1508   guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1509 
1510   if (mid->is_busy()) {
1511     deflated = false;
1512   } else {
1513     // Deflate the monitor if it is no longer being used
1514     // It's idle - scavenge and return to the global free list
1515     // plain old deflation ...
1516     if (log_is_enabled(Trace, monitorinflation)) {
1517       ResourceMark rm;
1518       log_trace(monitorinflation)("deflate_monitor: "
1519                                   "object=" INTPTR_FORMAT ", mark="
1520                                   INTPTR_FORMAT ", type='%s'", p2i(obj),
1521                                   mark.value(), obj->klass()->external_name());
1522     }
1523 
1524     // Restore the header back to obj
1525     obj->release_set_mark(dmw);
1526     mid->clear();
1527 
1528     assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
1529            p2i(mid->object()));
1530 
1531     // Move the deflated ObjectMonitor to the working free list
1532     // defined by free_head_p and free_tail_p.
1533     if (*free_head_p == NULL) *free_head_p = mid;
1534     if (*free_tail_p != NULL) {
1535       // We append to the list so the caller can use mid->_next_om
1536       // to fix the linkages in its context.
1537       ObjectMonitor* prevtail = *free_tail_p;
1538       // Should have been cleaned up by the caller:
1539       assert(prevtail->_next_om == NULL, "cleaned up deflated?");
1540       prevtail->_next_om = mid;
1541     }
1542     *free_tail_p = mid;
1543     // At this point, mid->_next_om still refers to its current
1544     // value and another ObjectMonitor's _next_om field still
1545     // refers to this ObjectMonitor. Those linkages have to be
1546     // cleaned up by the caller who has the complete context.
1547     deflated = true;
1548   }
1549   return deflated;
1550 }
1551 
1552 // Walk a given monitor list, and deflate idle monitors
1553 // The given list could be a per-thread list or a global list
1554 // Caller acquires gListLock as needed.
1555 //
1556 // In the case of parallel processing of thread local monitor lists,
1557 // work is done by Threads::parallel_threads_do() which ensures that
1558 // each Java thread is processed by exactly one worker thread, and
1559 // thus avoid conflicts that would arise when worker threads would
1560 // process the same monitor lists concurrently.
1561 //
1562 // See also ParallelSPCleanupTask and
1563 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
1564 // Threads::parallel_java_threads_do() in thread.cpp.
1565 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor** list_p,
1566                                              ObjectMonitor** free_head_p,
1567                                              ObjectMonitor** free_tail_p) {
1568   ObjectMonitor* mid;
1569   ObjectMonitor* next;
1570   ObjectMonitor* cur_mid_in_use = NULL;
1571   int deflated_count = 0;
1572 
1573   for (mid = *list_p; mid != NULL;) {
1574     oop obj = (oop) mid->object();
1575     if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
1576       // Deflation succeeded and already updated free_head_p and
1577       // free_tail_p as needed. Finish the move to the local free list
1578       // by unlinking mid from the global or per-thread in-use list.
1579       if (mid == *list_p) {
1580         *list_p = mid->_next_om;
1581       } else if (cur_mid_in_use != NULL) {
1582         cur_mid_in_use->_next_om = mid->_next_om; // maintain the current thread in-use list
1583       }
1584       next = mid->_next_om;
1585       mid->_next_om = NULL;  // This mid is current tail in the free_head_p list
1586       mid = next;
1587       deflated_count++;
1588     } else {
1589       cur_mid_in_use = mid;
1590       mid = mid->_next_om;
1591     }
1592   }
1593   return deflated_count;
1594 }
1595 
1596 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
1597   counters->n_in_use = 0;              // currently associated with objects
1598   counters->n_in_circulation = 0;      // extant
1599   counters->n_scavenged = 0;           // reclaimed (global and per-thread)
1600   counters->per_thread_scavenged = 0;  // per-thread scavenge total
1601   counters->per_thread_times = 0.0;    // per-thread scavenge times
1602 }
1603 
1604 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
1605   assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1606   bool deflated = false;
1607 
1608   ObjectMonitor* free_head_p = NULL;  // Local SLL of scavenged monitors
1609   ObjectMonitor* free_tail_p = NULL;
1610   elapsedTimer timer;
1611 
1612   if (log_is_enabled(Info, monitorinflation)) {
1613     timer.start();
1614   }
1615 
1616   // Prevent om_flush from changing mids in Thread dtor's during deflation
1617   // And in case the vm thread is acquiring a lock during a safepoint
1618   // See e.g. 6320749
1619   Thread::muxAcquire(&gListLock, "deflate_idle_monitors");
1620 
1621   // Note: the thread-local monitors lists get deflated in
1622   // a separate pass. See deflate_thread_local_monitors().
1623 
1624   // For moribund threads, scan g_om_in_use_list
1625   int deflated_count = 0;
1626   if (g_om_in_use_list) {
1627     counters->n_in_circulation += g_om_in_use_count;
1628     deflated_count = deflate_monitor_list((ObjectMonitor **)&g_om_in_use_list, &free_head_p, &free_tail_p);
1629     g_om_in_use_count -= deflated_count;
1630     counters->n_scavenged += deflated_count;
1631     counters->n_in_use += g_om_in_use_count;
1632   }
1633 
1634   if (free_head_p != NULL) {
1635     // Move the deflated ObjectMonitors back to the global free list.
1636     guarantee(free_tail_p != NULL && counters->n_scavenged > 0, "invariant");
1637     assert(free_tail_p->_next_om == NULL, "invariant");
1638     // constant-time list splice - prepend scavenged segment to g_free_list
1639     free_tail_p->_next_om = g_free_list;
1640     g_free_list = free_head_p;
1641   }
1642   Thread::muxRelease(&gListLock);
1643   timer.stop();
1644 
1645   LogStreamHandle(Debug, monitorinflation) lsh_debug;
1646   LogStreamHandle(Info, monitorinflation) lsh_info;
1647   LogStream* ls = NULL;
1648   if (log_is_enabled(Debug, monitorinflation)) {
1649     ls = &lsh_debug;
1650   } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
1651     ls = &lsh_info;
1652   }
1653   if (ls != NULL) {
1654     ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
1655   }
1656 }
1657 
1658 void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) {
1659   // Report the cumulative time for deflating each thread's idle
1660   // monitors. Note: if the work is split among more than one
1661   // worker thread, then the reported time will likely be more
1662   // than a beginning to end measurement of the phase.
1663   log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", counters->per_thread_times, counters->per_thread_scavenged);
1664 
1665   g_om_free_count += counters->n_scavenged;
1666 
1667   if (log_is_enabled(Debug, monitorinflation)) {
1668     // exit_globals()'s call to audit_and_print_stats() is done
1669     // at the Info level.
1670     ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
1671   } else if (log_is_enabled(Info, monitorinflation)) {
1672     Thread::muxAcquire(&gListLock, "finish_deflate_idle_monitors");
1673     log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, "
1674                                "g_om_free_count=%d", g_om_population,
1675                                g_om_in_use_count, g_om_free_count);
1676     Thread::muxRelease(&gListLock);
1677   }
1678 
1679   ForceMonitorScavenge = 0;    // Reset
1680 
1681   OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged));
1682   OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation));
1683 
1684   GVars.stw_random = os::random();
1685   GVars.stw_cycle++;
1686 }
1687 
1688 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
1689   assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1690 
1691   ObjectMonitor* free_head_p = NULL;  // Local SLL of scavenged monitors
1692   ObjectMonitor* free_tail_p = NULL;
1693   elapsedTimer timer;
1694 
1695   if (log_is_enabled(Info, safepoint, cleanup) ||
1696       log_is_enabled(Info, monitorinflation)) {
1697     timer.start();
1698   }
1699 
1700   int deflated_count = deflate_monitor_list(thread->om_in_use_list_addr(), &free_head_p, &free_tail_p);
1701 
1702   Thread::muxAcquire(&gListLock, "deflate_thread_local_monitors");
1703 
1704   // Adjust counters
1705   counters->n_in_circulation += thread->om_in_use_count;
1706   thread->om_in_use_count -= deflated_count;
1707   counters->n_scavenged += deflated_count;
1708   counters->n_in_use += thread->om_in_use_count;
1709   counters->per_thread_scavenged += deflated_count;
1710 
1711   if (free_head_p != NULL) {
1712     // Move the deflated ObjectMonitors back to the global free list.
1713     guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
1714     assert(free_tail_p->_next_om == NULL, "invariant");
1715 
1716     // constant-time list splice - prepend scavenged segment to g_free_list
1717     free_tail_p->_next_om = g_free_list;
1718     g_free_list = free_head_p;
1719   }
1720 
1721   timer.stop();
1722   // Safepoint logging cares about cumulative per_thread_times and
1723   // we'll capture most of the cost, but not the muxRelease() which
1724   // should be cheap.
1725   counters->per_thread_times += timer.seconds();
1726 
1727   Thread::muxRelease(&gListLock);
1728 
1729   LogStreamHandle(Debug, monitorinflation) lsh_debug;
1730   LogStreamHandle(Info, monitorinflation) lsh_info;
1731   LogStream* ls = NULL;
1732   if (log_is_enabled(Debug, monitorinflation)) {
1733     ls = &lsh_debug;
1734   } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
1735     ls = &lsh_info;
1736   }
1737   if (ls != NULL) {
1738     ls->print_cr("jt=" INTPTR_FORMAT ": deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(thread), timer.seconds(), deflated_count);
1739   }
1740 }
1741 
1742 // Monitor cleanup on JavaThread::exit
1743 
1744 // Iterate through monitor cache and attempt to release thread's monitors
1745 // Gives up on a particular monitor if an exception occurs, but continues
1746 // the overall iteration, swallowing the exception.
1747 class ReleaseJavaMonitorsClosure: public MonitorClosure {
1748  private:
1749   TRAPS;
1750 
1751  public:
1752   ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {}
1753   void do_monitor(ObjectMonitor* mid) {
1754     if (mid->owner() == THREAD) {
1755       (void)mid->complete_exit(CHECK);
1756     }
1757   }
1758 };
1759 
1760 // Release all inflated monitors owned by THREAD.  Lightweight monitors are
1761 // ignored.  This is meant to be called during JNI thread detach which assumes
1762 // all remaining monitors are heavyweight.  All exceptions are swallowed.
1763 // Scanning the extant monitor list can be time consuming.
1764 // A simple optimization is to add a per-thread flag that indicates a thread
1765 // called jni_monitorenter() during its lifetime.
1766 //
1767 // Instead of No_Savepoint_Verifier it might be cheaper to
1768 // use an idiom of the form:
1769 //   auto int tmp = SafepointSynchronize::_safepoint_counter ;
1770 //   <code that must not run at safepoint>
1771 //   guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
1772 // Since the tests are extremely cheap we could leave them enabled
1773 // for normal product builds.
1774 
1775 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
1776   assert(THREAD == JavaThread::current(), "must be current Java thread");
1777   NoSafepointVerifier nsv;
1778   ReleaseJavaMonitorsClosure rjmc(THREAD);
1779   Thread::muxAcquire(&gListLock, "release_monitors_owned_by_thread");
1780   ObjectSynchronizer::monitors_iterate(&rjmc);
1781   Thread::muxRelease(&gListLock);
1782   THREAD->clear_pending_exception();
1783 }
1784 
1785 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
1786   switch (cause) {
1787     case inflate_cause_vm_internal:    return "VM Internal";
1788     case inflate_cause_monitor_enter:  return "Monitor Enter";
1789     case inflate_cause_wait:           return "Monitor Wait";
1790     case inflate_cause_notify:         return "Monitor Notify";
1791     case inflate_cause_hash_code:      return "Monitor Hash Code";
1792     case inflate_cause_jni_enter:      return "JNI Monitor Enter";
1793     case inflate_cause_jni_exit:       return "JNI Monitor Exit";
1794     default:
1795       ShouldNotReachHere();
1796   }
1797   return "Unknown";
1798 }
1799 
1800 //------------------------------------------------------------------------------
1801 // Debugging code
1802 
1803 u_char* ObjectSynchronizer::get_gvars_addr() {
1804   return (u_char*)&GVars;
1805 }
1806 
1807 u_char* ObjectSynchronizer::get_gvars_hc_sequence_addr() {
1808   return (u_char*)&GVars.hc_sequence;
1809 }
1810 
1811 size_t ObjectSynchronizer::get_gvars_size() {
1812   return sizeof(SharedGlobals);
1813 }
1814 
1815 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
1816   return (u_char*)&GVars.stw_random;
1817 }
1818 
1819 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
1820   assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
1821 
1822   LogStreamHandle(Debug, monitorinflation) lsh_debug;
1823   LogStreamHandle(Info, monitorinflation) lsh_info;
1824   LogStreamHandle(Trace, monitorinflation) lsh_trace;
1825   LogStream* ls = NULL;
1826   if (log_is_enabled(Trace, monitorinflation)) {
1827     ls = &lsh_trace;
1828   } else if (log_is_enabled(Debug, monitorinflation)) {
1829     ls = &lsh_debug;
1830   } else if (log_is_enabled(Info, monitorinflation)) {
1831     ls = &lsh_info;
1832   }
1833   assert(ls != NULL, "sanity check");
1834 
1835   if (!on_exit) {
1836     // Not at VM exit so grab the global list lock.
1837     Thread::muxAcquire(&gListLock, "audit_and_print_stats");
1838   }
1839 
1840   // Log counts for the global and per-thread monitor lists:
1841   int chk_om_population = log_monitor_list_counts(ls);
1842   int error_cnt = 0;
1843 
1844   ls->print_cr("Checking global lists:");
1845 
1846   // Check g_om_population:
1847   if (g_om_population == chk_om_population) {
1848     ls->print_cr("g_om_population=%d equals chk_om_population=%d",
1849                  g_om_population, chk_om_population);
1850   } else {
1851     ls->print_cr("ERROR: g_om_population=%d is not equal to "
1852                  "chk_om_population=%d", g_om_population,
1853                  chk_om_population);
1854     error_cnt++;
1855   }
1856 
1857   // Check g_om_in_use_list and g_om_in_use_count:
1858   chk_global_in_use_list_and_count(ls, &error_cnt);
1859 
1860   // Check g_free_list and g_om_free_count:
1861   chk_global_free_list_and_count(ls, &error_cnt);
1862 
1863   if (!on_exit) {
1864     Thread::muxRelease(&gListLock);
1865   }
1866 
1867   ls->print_cr("Checking per-thread lists:");
1868 
1869   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
1870     // Check om_in_use_list and om_in_use_count:
1871     chk_per_thread_in_use_list_and_count(jt, ls, &error_cnt);
1872 
1873     // Check om_free_list and om_free_count:
1874     chk_per_thread_free_list_and_count(jt, ls, &error_cnt);
1875   }
1876 
1877   if (error_cnt == 0) {
1878     ls->print_cr("No errors found in monitor list checks.");
1879   } else {
1880     log_error(monitorinflation)("found monitor list errors: error_cnt=%d", error_cnt);
1881   }
1882 
1883   if ((on_exit && log_is_enabled(Info, monitorinflation)) ||
1884       (!on_exit && log_is_enabled(Trace, monitorinflation))) {
1885     // When exiting this log output is at the Info level. When called
1886     // at a safepoint, this log output is at the Trace level since
1887     // there can be a lot of it.
1888     log_in_use_monitor_details(ls, on_exit);
1889   }
1890 
1891   ls->flush();
1892 
1893   guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
1894 }
1895 
1896 // Check a free monitor entry; log any errors.
1897 void ObjectSynchronizer::chk_free_entry(JavaThread* jt, ObjectMonitor* n,
1898                                         outputStream * out, int *error_cnt_p) {
1899   stringStream ss;
1900   if (n->is_busy()) {
1901     if (jt != NULL) {
1902       out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
1903                     ": free per-thread monitor must not be busy: %s", p2i(jt),
1904                     p2i(n), n->is_busy_to_string(&ss));
1905     } else {
1906       out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
1907                     "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
1908     }
1909     *error_cnt_p = *error_cnt_p + 1;
1910   }
1911   if (n->header().value() != 0) {
1912     if (jt != NULL) {
1913       out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
1914                     ": free per-thread monitor must have NULL _header "
1915                     "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n),
1916                     n->header().value());
1917     } else {
1918       out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
1919                     "must have NULL _header field: _header=" INTPTR_FORMAT,
1920                     p2i(n), n->header().value());
1921     }
1922     *error_cnt_p = *error_cnt_p + 1;
1923   }
1924   if (n->object() != NULL) {
1925     if (jt != NULL) {
1926       out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
1927                     ": free per-thread monitor must have NULL _object "
1928                     "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
1929                     p2i(n->object()));
1930     } else {
1931       out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
1932                     "must have NULL _object field: _object=" INTPTR_FORMAT,
1933                     p2i(n), p2i(n->object()));
1934     }
1935     *error_cnt_p = *error_cnt_p + 1;
1936   }
1937 }
1938 
1939 // Check the global free list and count; log the results of the checks.
1940 void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out,
1941                                                         int *error_cnt_p) {
1942   int chk_om_free_count = 0;
1943   for (ObjectMonitor* n = g_free_list; n != NULL; n = n->_next_om) {
1944     chk_free_entry(NULL /* jt */, n, out, error_cnt_p);
1945     chk_om_free_count++;
1946   }
1947   if (g_om_free_count == chk_om_free_count) {
1948     out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d",
1949                   g_om_free_count, chk_om_free_count);
1950   } else {
1951     out->print_cr("ERROR: g_om_free_count=%d is not equal to "
1952                   "chk_om_free_count=%d", g_om_free_count,
1953                   chk_om_free_count);
1954     *error_cnt_p = *error_cnt_p + 1;
1955   }
1956 }
1957 
1958 // Check the global in-use list and count; log the results of the checks.
1959 void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out,
1960                                                           int *error_cnt_p) {
1961   int chk_om_in_use_count = 0;
1962   for (ObjectMonitor* n = g_om_in_use_list; n != NULL; n = n->_next_om) {
1963     chk_in_use_entry(NULL /* jt */, n, out, error_cnt_p);
1964     chk_om_in_use_count++;
1965   }
1966   if (g_om_in_use_count == chk_om_in_use_count) {
1967     out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d", g_om_in_use_count,
1968                   chk_om_in_use_count);
1969   } else {
1970     out->print_cr("ERROR: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d",
1971                   g_om_in_use_count, chk_om_in_use_count);
1972     *error_cnt_p = *error_cnt_p + 1;
1973   }
1974 }
1975 
1976 // Check an in-use monitor entry; log any errors.
1977 void ObjectSynchronizer::chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,
1978                                           outputStream * out, int *error_cnt_p) {
1979   if (n->header().value() == 0) {
1980     if (jt != NULL) {
1981       out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
1982                     ": in-use per-thread monitor must have non-NULL _header "
1983                     "field.", p2i(jt), p2i(n));
1984     } else {
1985       out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
1986                     "must have non-NULL _header field.", p2i(n));
1987     }
1988     *error_cnt_p = *error_cnt_p + 1;
1989   }
1990   if (n->object() == NULL) {
1991     if (jt != NULL) {
1992       out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
1993                     ": in-use per-thread monitor must have non-NULL _object "
1994                     "field.", p2i(jt), p2i(n));
1995     } else {
1996       out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
1997                     "must have non-NULL _object field.", p2i(n));
1998     }
1999     *error_cnt_p = *error_cnt_p + 1;
2000   }
2001   const oop obj = (oop)n->object();
2002   const markWord mark = obj->mark();
2003   if (!mark.has_monitor()) {
2004     if (jt != NULL) {
2005       out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2006                     ": in-use per-thread monitor's object does not think "
2007                     "it has a monitor: obj=" INTPTR_FORMAT ", mark="
2008                     INTPTR_FORMAT,  p2i(jt), p2i(n), p2i(obj), mark.value());
2009     } else {
2010       out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
2011                     "monitor's object does not think it has a monitor: obj="
2012                     INTPTR_FORMAT ", mark=" INTPTR_FORMAT, p2i(n),
2013                     p2i(obj), mark.value());
2014     }
2015     *error_cnt_p = *error_cnt_p + 1;
2016   }
2017   ObjectMonitor* const obj_mon = mark.monitor();
2018   if (n != obj_mon) {
2019     if (jt != NULL) {
2020       out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2021                     ": in-use per-thread monitor's object does not refer "
2022                     "to the same monitor: obj=" INTPTR_FORMAT ", mark="
2023                     INTPTR_FORMAT ", obj_mon=" INTPTR_FORMAT, p2i(jt),
2024                     p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
2025     } else {
2026       out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
2027                     "monitor's object does not refer to the same monitor: obj="
2028                     INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
2029                     INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
2030     }
2031     *error_cnt_p = *error_cnt_p + 1;
2032   }
2033 }
2034 
2035 // Check the thread's free list and count; log the results of the checks.
2036 void ObjectSynchronizer::chk_per_thread_free_list_and_count(JavaThread *jt,
2037                                                             outputStream * out,
2038                                                             int *error_cnt_p) {
2039   int chk_om_free_count = 0;
2040   for (ObjectMonitor* n = jt->om_free_list; n != NULL; n = n->_next_om) {
2041     chk_free_entry(jt, n, out, error_cnt_p);
2042     chk_om_free_count++;
2043   }
2044   if (jt->om_free_count == chk_om_free_count) {
2045     out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals "
2046                   "chk_om_free_count=%d", p2i(jt), jt->om_free_count, chk_om_free_count);
2047   } else {
2048     out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not "
2049                   "equal to chk_om_free_count=%d", p2i(jt), jt->om_free_count,
2050                   chk_om_free_count);
2051     *error_cnt_p = *error_cnt_p + 1;
2052   }
2053 }
2054 
2055 // Check the thread's in-use list and count; log the results of the checks.
2056 void ObjectSynchronizer::chk_per_thread_in_use_list_and_count(JavaThread *jt,
2057                                                               outputStream * out,
2058                                                               int *error_cnt_p) {
2059   int chk_om_in_use_count = 0;
2060   for (ObjectMonitor* n = jt->om_in_use_list; n != NULL; n = n->_next_om) {
2061     chk_in_use_entry(jt, n, out, error_cnt_p);
2062     chk_om_in_use_count++;
2063   }
2064   if (jt->om_in_use_count == chk_om_in_use_count) {
2065     out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals "
2066                   "chk_om_in_use_count=%d", p2i(jt), jt->om_in_use_count,
2067                   chk_om_in_use_count);
2068   } else {
2069     out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not "
2070                   "equal to chk_om_in_use_count=%d", p2i(jt), jt->om_in_use_count,
2071                   chk_om_in_use_count);
2072     *error_cnt_p = *error_cnt_p + 1;
2073   }
2074 }
2075 
2076 // Log details about ObjectMonitors on the in-use lists. The 'BHL'
2077 // flags indicate why the entry is in-use, 'object' and 'object type'
2078 // indicate the associated object and its type.
2079 void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out,
2080                                                     bool on_exit) {
2081   if (!on_exit) {
2082     // Not at VM exit so grab the global list lock.
2083     Thread::muxAcquire(&gListLock, "log_in_use_monitor_details");
2084   }
2085 
2086   stringStream ss;
2087   if (g_om_in_use_count > 0) {
2088     out->print_cr("In-use global monitor info:");
2089     out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
2090     out->print_cr("%18s  %s  %18s  %18s",
2091                   "monitor", "BHL", "object", "object type");
2092     out->print_cr("==================  ===  ==================  ==================");
2093     for (ObjectMonitor* n = g_om_in_use_list; n != NULL; n = n->_next_om) {
2094       const oop obj = (oop) n->object();
2095       const markWord mark = n->header();
2096       ResourceMark rm;
2097       out->print(INTPTR_FORMAT "  %d%d%d  " INTPTR_FORMAT "  %s", p2i(n),
2098                  n->is_busy() != 0, mark.hash() != 0, n->owner() != NULL,
2099                  p2i(obj), obj->klass()->external_name());
2100       if (n->is_busy() != 0) {
2101         out->print(" (%s)", n->is_busy_to_string(&ss));
2102         ss.reset();
2103       }
2104       out->cr();
2105     }
2106   }
2107 
2108   if (!on_exit) {
2109     Thread::muxRelease(&gListLock);
2110   }
2111 
2112   out->print_cr("In-use per-thread monitor info:");
2113   out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
2114   out->print_cr("%18s  %18s  %s  %18s  %18s",
2115                 "jt", "monitor", "BHL", "object", "object type");
2116   out->print_cr("==================  ==================  ===  ==================  ==================");
2117   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2118     for (ObjectMonitor* n = jt->om_in_use_list; n != NULL; n = n->_next_om) {
2119       const oop obj = (oop) n->object();
2120       const markWord mark = n->header();
2121       ResourceMark rm;
2122       out->print(INTPTR_FORMAT "  " INTPTR_FORMAT "  %d%d%d  " INTPTR_FORMAT
2123                  "  %s", p2i(jt), p2i(n), n->is_busy() != 0,
2124                  mark.hash() != 0, n->owner() != NULL, p2i(obj),
2125                  obj->klass()->external_name());
2126       if (n->is_busy() != 0) {
2127         out->print(" (%s)", n->is_busy_to_string(&ss));
2128         ss.reset();
2129       }
2130       out->cr();
2131     }
2132   }
2133 
2134   out->flush();
2135 }
2136 
2137 // Log counts for the global and per-thread monitor lists and return
2138 // the population count.
2139 int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) {
2140   int pop_count = 0;
2141   out->print_cr("%18s  %10s  %10s  %10s",
2142                 "Global Lists:", "InUse", "Free", "Total");
2143   out->print_cr("==================  ==========  ==========  ==========");
2144   out->print_cr("%18s  %10d  %10d  %10d", "",
2145                 g_om_in_use_count, g_om_free_count, g_om_population);
2146   pop_count += g_om_in_use_count + g_om_free_count;
2147 
2148   out->print_cr("%18s  %10s  %10s  %10s",
2149                 "Per-Thread Lists:", "InUse", "Free", "Provision");
2150   out->print_cr("==================  ==========  ==========  ==========");
2151 
2152   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2153     out->print_cr(INTPTR_FORMAT "  %10d  %10d  %10d", p2i(jt),
2154                   jt->om_in_use_count, jt->om_free_count, jt->om_free_provision);
2155     pop_count += jt->om_in_use_count + jt->om_free_count;
2156   }
2157   return pop_count;
2158 }
2159 
2160 #ifndef PRODUCT
2161 
2162 // Check if monitor belongs to the monitor cache
2163 // The list is grow-only so it's *relatively* safe to traverse
2164 // the list of extant blocks without taking a lock.
2165 
2166 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
2167   PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
2168   while (block != NULL) {
2169     assert(block->object() == CHAINMARKER, "must be a block header");
2170     if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
2171       address mon = (address)monitor;
2172       address blk = (address)block;
2173       size_t diff = mon - blk;
2174       assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
2175       return 1;
2176     }
2177     block = (PaddedObjectMonitor*)block->_next_om;
2178   }
2179   return 0;
2180 }
2181 
2182 #endif