474
475
476 // The LockNode emitted directly at the synchronization site would have
477 // been too big if it were to have included support for the cases of inflated
478 // recursive enter and exit, so they go here instead.
479 // Note that we can't safely call AsyncPrintJavaStack() from within
480 // quick_enter() as our thread state remains _in_Java.
481
482 bool ObjectSynchronizer::quick_enter(oop obj, Thread* self,
483 BasicLock * lock) {
484 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
485 assert(self->is_Java_thread(), "invariant");
486 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
487 NoSafepointVerifier nsv;
488 if (obj == NULL) return false; // Need to throw NPE
489
490 const markWord mark = obj->mark();
491
492 if (mark.has_monitor()) {
493 ObjectMonitor* const m = mark.monitor();
494 if (AsyncDeflateIdleMonitors) {
495 // An async deflation can race us before we manage to make the
496 // ObjectMonitor busy by setting the owner below. If we detect
497 // that race we just bail out to the slow-path here.
498 if (m->object() == NULL) {
499 return false;
500 }
501 } else {
502 assert(m->object() == obj, "invariant");
503 }
504 Thread* const owner = (Thread *) m->_owner;
505
506 // Lock contention and Transactional Lock Elision (TLE) diagnostics
507 // and observability
508 // Case: light contention possibly amenable to TLE
509 // Case: TLE inimical operations such as nested/recursive synchronization
510
511 if (owner == self) {
512 m->_recursions++;
513 return true;
514 }
515
516 // This Java Monitor is inflated so obj's header will never be
517 // displaced to this thread's BasicLock. Make the displaced header
518 // non-NULL so this BasicLock is not seen as recursive nor as
519 // being locked. We do this unconditionally so that this thread's
520 // BasicLock cannot be mis-interpreted by any stack walkers. For
521 // performance reasons, stack walkers generally first check for
522 // Biased Locking in the object's header, the second check is for
523 // stack-locking in the object's header, the third check is for
969 unsigned t = self->_hashStateX;
970 t ^= (t << 11);
971 self->_hashStateX = self->_hashStateY;
972 self->_hashStateY = self->_hashStateZ;
973 self->_hashStateZ = self->_hashStateW;
974 unsigned v = self->_hashStateW;
975 v = (v ^ (v >> 19)) ^ (t ^ (t >> 8));
976 self->_hashStateW = v;
977 value = v;
978 }
979
980 value &= markWord::hash_mask;
981 if (value == 0) value = 0xBAD;
982 assert(value != markWord::no_hash, "invariant");
983 return value;
984 }
985
986 intptr_t ObjectSynchronizer::FastHashCode(Thread* self, oop obj) {
987 if (UseBiasedLocking) {
988 // NOTE: many places throughout the JVM do not expect a safepoint
989 // to be taken here, in particular most operations on perm gen
990 // objects. However, we only ever bias Java instances and all of
991 // the call sites of identity_hash that might revoke biases have
992 // been checked to make sure they can handle a safepoint. The
993 // added check of the bias pattern is to avoid useless calls to
994 // thread-local storage.
995 if (obj->mark().has_bias_pattern()) {
996 // Handle for oop obj in case of STW safepoint
997 Handle hobj(self, obj);
998 // Relaxing assertion for bug 6320749.
999 assert(Universe::verify_in_progress() ||
1000 !SafepointSynchronize::is_at_safepoint(),
1001 "biases should not be seen by VM thread here");
1002 BiasedLocking::revoke(hobj, JavaThread::current());
1003 obj = hobj();
1004 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
1005 }
1006 }
1007
1008 // hashCode() is a heap mutator ...
1009 // Relaxing assertion for bug 6320749.
1010 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1011 !SafepointSynchronize::is_at_safepoint(), "invariant");
1176 // Possible mark states: neutral, biased, stack-locked, inflated
1177
1178 if (UseBiasedLocking && h_obj()->mark().has_bias_pattern()) {
1179 // CASE: biased
1180 BiasedLocking::revoke(h_obj, self);
1181 assert(!h_obj->mark().has_bias_pattern(),
1182 "biases should be revoked by now");
1183 }
1184
1185 assert(self == JavaThread::current(), "Can only be called on current thread");
1186 oop obj = h_obj();
1187 markWord mark = read_stable_mark(obj);
1188
1189 // CASE: stack-locked. Mark points to a BasicLock on the owner's stack.
1190 if (mark.has_locker()) {
1191 return self->is_lock_owned((address)mark.locker()) ?
1192 owner_self : owner_other;
1193 }
1194
1195 // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor.
1196 // The Object:ObjectMonitor relationship is stable as long as we're
1197 // not at a safepoint and AsyncDeflateIdleMonitors is false.
1198 if (mark.has_monitor()) {
1199 // The first stage of async deflation does not affect any field
1200 // used by this comparison so the ObjectMonitor* is usable here.
1201 ObjectMonitor* monitor = mark.monitor();
1202 void* owner = monitor->owner();
1203 if (owner == NULL) return owner_none;
1204 return (owner == self ||
1205 self->is_lock_owned((address)owner)) ? owner_self : owner_other;
1206 }
1207
1208 // CASE: neutral
1209 assert(mark.is_neutral(), "sanity check");
1210 return owner_none; // it's unlocked
1211 }
1212
1213 // FIXME: jvmti should call this
1214 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
1215 if (UseBiasedLocking) {
1216 if (SafepointSynchronize::is_at_safepoint()) {
1217 BiasedLocking::revoke_at_safepoint(h_obj);
1277 // used with block linkage _next_om fields).
1278 block = (PaddedObjectMonitor*)block->next_om();
1279 }
1280 }
1281
1282 static bool monitors_used_above_threshold() {
1283 int population = Atomic::load(&om_list_globals._population);
1284 if (population == 0) {
1285 return false;
1286 }
1287 if (MonitorUsedDeflationThreshold > 0) {
1288 int monitors_used = population - Atomic::load(&om_list_globals._free_count) -
1289 Atomic::load(&om_list_globals._wait_count);
1290 int monitor_usage = (monitors_used * 100LL) / population;
1291 return monitor_usage > MonitorUsedDeflationThreshold;
1292 }
1293 return false;
1294 }
1295
1296 bool ObjectSynchronizer::is_async_deflation_needed() {
1297 if (!AsyncDeflateIdleMonitors) {
1298 return false;
1299 }
1300 if (is_async_deflation_requested()) {
1301 // Async deflation request.
1302 return true;
1303 }
1304 if (AsyncDeflationInterval > 0 &&
1305 time_since_last_async_deflation_ms() > AsyncDeflationInterval &&
1306 monitors_used_above_threshold()) {
1307 // It's been longer than our specified deflate interval and there
1308 // are too many monitors in use. We don't deflate more frequently
1309 // than AsyncDeflationInterval (unless is_async_deflation_requested)
1310 // in order to not swamp the ServiceThread.
1311 return true;
1312 }
1313 return false;
1314 }
1315
1316 bool ObjectSynchronizer::is_safepoint_deflation_needed() {
1317 return !AsyncDeflateIdleMonitors &&
1318 monitors_used_above_threshold(); // Too many monitors in use.
1319 }
1320
1321 bool ObjectSynchronizer::request_deflate_idle_monitors() {
1322 bool is_JavaThread = Thread::current()->is_Java_thread();
1323 bool ret_code = false;
1324
1325 if (AsyncDeflateIdleMonitors) {
1326 jlong last_time = last_async_deflation_time_ns();
1327 set_is_async_deflation_requested(true);
1328 {
1329 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
1330 ml.notify_all();
1331 }
1332 const int N_CHECKS = 5;
1333 for (int i = 0; i < N_CHECKS; i++) { // sleep for at most 5 seconds
1334 if (last_async_deflation_time_ns() > last_time) {
1335 log_info(monitorinflation)("Async Deflation happened after %d check(s).", i);
1336 ret_code = true;
1337 break;
1338 }
1339 if (is_JavaThread) {
1340 // JavaThread has to honor the blocking protocol.
1341 ThreadBlockInVM tbivm(JavaThread::current());
1342 os::naked_short_sleep(999); // sleep for almost 1 second
1343 } else {
1344 os::naked_short_sleep(999); // sleep for almost 1 second
1345 }
1346 }
1347 if (!ret_code) {
1348 log_info(monitorinflation)("Async Deflation DID NOT happen after %d checks.", N_CHECKS);
1349 }
1350 } else {
1351 // Only need to force this safepoint if we are not using async
1352 // deflation. The VMThread won't call this function before the
1353 // final safepoint if we are not using async deflation so we
1354 // don't have to reason about the VMThread executing a VM-op here.
1355 VM_ForceSafepoint force_safepoint_op;
1356 VMThread::execute(&force_safepoint_op);
1357 ret_code = true;
1358 }
1359
1360 return ret_code;
1361 }
1362
1363 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1364 return (os::javaTimeNanos() - last_async_deflation_time_ns()) / (NANOUNITS / MILLIUNITS);
1365 }
1366
1367 void ObjectSynchronizer::oops_do(OopClosure* f) {
1368 // We only scan the global used list here (for moribund threads), and
1369 // the thread-local monitors in Thread::oops_do().
1370 global_used_oops_do(f);
1371 }
1372
1373 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
1374 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1375 list_oops_do(Atomic::load(&om_list_globals._in_use_list), f);
1376 }
1377
1378 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
1379 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1380 list_oops_do(thread->om_in_use_list, f);
1381 }
1382
1383 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, OopClosure* f) {
1384 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1385 // The oops_do() phase does not overlap with monitor deflation
1386 // so no need to lock ObjectMonitors for the list traversal.
1387 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
1388 if (mid->object() != NULL) {
1389 f->do_oop((oop*)mid->object_addr());
1390 }
1391 }
1392 }
1393
1394
1395 // -----------------------------------------------------------------------------
1396 // ObjectMonitor Lifecycle
1397 // -----------------------
1398 // Inflation unlinks monitors from om_list_globals._free_list or a per-thread
1399 // free list and associates them with objects. Deflation -- which occurs at
1400 // STW-time or asynchronously -- disassociates idle monitors from objects.
1401 // Such scavenged monitors are returned to the om_list_globals._free_list.
1402 //
1403 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
1404 //
1405 // Lifecycle:
1406 // -- unassigned and on the om_list_globals._free_list
1407 // -- unassigned and on a per-thread free list
1408 // -- assigned to an object. The object is inflated and the mark refers
1409 // to the ObjectMonitor.
1410
1411 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self) {
1412 // A large MAXPRIVATE value reduces both list lock contention
1413 // and list coherency traffic, but also tends to increase the
1414 // number of ObjectMonitors in circulation as well as the STW
1415 // scavenge costs. As usual, we lean toward time in space-time
1416 // tradeoffs.
1417 const int MAXPRIVATE = 1024;
1418 NoSafepointVerifier nsv;
1419
1420 for (;;) {
1421 ObjectMonitor* m;
1422
1423 // 1: try to allocate from the thread's local om_free_list.
1424 // Threads will attempt to allocate first from their local list, then
1425 // from the global list, and only after those attempts fail will the
1426 // thread attempt to instantiate new monitors. Thread-local free lists
1427 // improve allocation latency, as well as reducing coherency traffic
1428 // on the shared global list.
1429 m = take_from_start_of_om_free_list(self);
1430 if (m != NULL) {
1431 guarantee(m->object() == NULL, "invariant");
1432 m->set_allocation_state(ObjectMonitor::New);
1433 prepend_to_om_in_use_list(self, m);
1434 return m;
1435 }
1436
1437 // 2: try to allocate from the global om_list_globals._free_list
1438 // If we're using thread-local free lists then try
1439 // to reprovision the caller's free list.
1440 if (Atomic::load(&om_list_globals._free_list) != NULL) {
1441 // Reprovision the thread's om_free_list.
1442 // Use bulk transfers to reduce the allocation rate and heat
1443 // on various locks.
1444 for (int i = self->om_free_provision; --i >= 0;) {
1445 ObjectMonitor* take = take_from_start_of_global_free_list();
1446 if (take == NULL) {
1447 break; // No more are available.
1448 }
1449 guarantee(take->object() == NULL, "invariant");
1450 if (AsyncDeflateIdleMonitors) {
1451 // We allowed 3 field values to linger during async deflation.
1452 // Clear or restore them as appropriate.
1453 take->set_header(markWord::zero());
1454 // DEFLATER_MARKER is the only non-NULL value we should see here.
1455 take->try_set_owner_from(DEFLATER_MARKER, NULL);
1456 if (take->contentions() < 0) {
1457 // Add back max_jint to restore the contentions field to its
1458 // proper value.
1459 take->add_to_contentions(max_jint);
1460
1461 #ifdef ASSERT
1462 jint l_contentions = take->contentions();
1463 #endif
1464 assert(l_contentions >= 0, "must not be negative: l_contentions=%d, contentions=%d",
1465 l_contentions, take->contentions());
1466 }
1467 }
1468 take->Recycle();
1469 // Since we're taking from the global free-list, take must be Free.
1470 // om_release() also sets the allocation state to Free because it
1471 // is called from other code paths.
1472 assert(take->is_free(), "invariant");
1473 om_release(self, take, false);
1474 }
1475 self->om_free_provision += 1 + (self->om_free_provision / 2);
1476 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1477 continue;
1478 }
1479
1480 // 3: allocate a block of new ObjectMonitors
1481 // Both the local and global free lists are empty -- resort to malloc().
1482 // In the current implementation ObjectMonitors are TSM - immortal.
1483 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1484 // each ObjectMonitor to start at the beginning of a cache line,
1485 // so we use align_up().
1486 // A better solution would be to use C++ placement-new.
1512 // Element [0] is reserved for global list linkage
1513 temp[0].set_object(CHAINMARKER);
1514
1515 // Consider carving out this thread's current request from the
1516 // block in hand. This avoids some lock traffic and redundant
1517 // list activity.
1518
1519 prepend_block_to_lists(temp);
1520 }
1521 }
1522
1523 // Place "m" on the caller's private per-thread om_free_list.
1524 // In practice there's no need to clamp or limit the number of
1525 // monitors on a thread's om_free_list as the only non-allocation time
1526 // we'll call om_release() is to return a monitor to the free list after
1527 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1528 // accumulate on a thread's free list.
1529 //
1530 // Key constraint: all ObjectMonitors on a thread's free list and the global
1531 // free list must have their object field set to null. This prevents the
1532 // scavenger -- deflate_monitor_list() or deflate_monitor_list_using_JT()
1533 // -- from reclaiming them while we are trying to release them.
1534
1535 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1536 bool from_per_thread_alloc) {
1537 guarantee(m->header().value() == 0, "invariant");
1538 guarantee(m->object() == NULL, "invariant");
1539 NoSafepointVerifier nsv;
1540
1541 if ((m->is_busy() | m->_recursions) != 0) {
1542 stringStream ss;
1543 fatal("freeing in-use monitor: %s, recursions=" INTX_FORMAT,
1544 m->is_busy_to_string(&ss), m->_recursions);
1545 }
1546 m->set_allocation_state(ObjectMonitor::Free);
1547 // _next_om is used for both per-thread in-use and free lists so
1548 // we have to remove 'm' from the in-use list first (as needed).
1549 if (from_per_thread_alloc) {
1550 // Need to remove 'm' from om_in_use_list.
1551 ObjectMonitor* mid = NULL;
1552 ObjectMonitor* next = NULL;
1553
1622
1623 // At this point mid is disconnected from the in-use list so
1624 // its lock no longer has any effects on the in-use list.
1625 Atomic::dec(&self->om_in_use_count);
1626 // Unlock mid, but leave the next value for any lagging list
1627 // walkers. It will get cleaned up when mid is prepended to
1628 // the thread's free list:
1629 om_unlock(mid);
1630 }
1631
1632 prepend_to_om_free_list(self, m);
1633 guarantee(m->is_free(), "invariant");
1634 }
1635
1636 // Return ObjectMonitors on a moribund thread's free and in-use
1637 // lists to the appropriate global lists. The ObjectMonitors on the
1638 // per-thread in-use list may still be in use by other threads.
1639 //
1640 // We currently call om_flush() from Threads::remove() before the
1641 // thread has been excised from the thread list and is no longer a
1642 // mutator. This means that om_flush() cannot run concurrently with
1643 // a safepoint and interleave with deflate_idle_monitors(). In
1644 // particular, this ensures that the thread's in-use monitors are
1645 // scanned by a GC safepoint, either via Thread::oops_do() (before
1646 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1647 // om_flush() is called).
1648 //
1649 // With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT()
1650 // and deflate_per_thread_idle_monitors_using_JT() (in another thread) can
1651 // run at the same time as om_flush() so we have to follow a careful
1652 // protocol to prevent list corruption.
1653
1654 void ObjectSynchronizer::om_flush(Thread* self) {
1655 // Process the per-thread in-use list first to be consistent.
1656 int in_use_count = 0;
1657 ObjectMonitor* in_use_list = NULL;
1658 ObjectMonitor* in_use_tail = NULL;
1659 NoSafepointVerifier nsv;
1660
1661 // This function can race with a list walker or with an async
1662 // deflater thread so we lock the list head to prevent confusion.
1663 // An async deflater thread checks to see if the target thread
1664 // is exiting, but if it has made it past that check before we
1665 // started exiting, then it is racing to get to the in-use list.
1666 if ((in_use_list = get_list_head_locked(&self->om_in_use_list)) != NULL) {
1667 // At this point, we have locked the in-use list head so a racing
1668 // thread cannot come in after us. However, a racing thread could
1669 // be ahead of us; we'll detect that and delay to let it finish.
1670 //
1684 while (is_locked(cur_om)) {
1685 os::naked_short_sleep(1);
1686 }
1687 // Refetch the possibly changed next field and try again.
1688 cur_om = unmarked_next(in_use_tail);
1689 continue;
1690 }
1691 if (cur_om->object() == NULL) {
1692 // cur_om was deflated and the object ref was cleared while it
1693 // was locked. We happened to see it just after it was unlocked
1694 // (and added to the free list). Refetch the possibly changed
1695 // next field and try again.
1696 cur_om = unmarked_next(in_use_tail);
1697 continue;
1698 }
1699 in_use_tail = cur_om;
1700 in_use_count++;
1701 cur_om = unmarked_next(cur_om);
1702 }
1703 guarantee(in_use_tail != NULL, "invariant");
1704 int l_om_in_use_count = Atomic::load(&self->om_in_use_count);
1705 ADIM_guarantee(l_om_in_use_count == in_use_count, "in-use counts don't match: "
1706 "l_om_in_use_count=%d, in_use_count=%d", l_om_in_use_count, in_use_count);
1707 Atomic::store(&self->om_in_use_count, 0);
1708 // Clear the in-use list head (which also unlocks it):
1709 Atomic::store(&self->om_in_use_list, (ObjectMonitor*)NULL);
1710 om_unlock(in_use_list);
1711 }
1712
1713 int free_count = 0;
1714 ObjectMonitor* free_list = NULL;
1715 ObjectMonitor* free_tail = NULL;
1716 // This function can race with a list walker thread so we lock the
1717 // list head to prevent confusion.
1718 if ((free_list = get_list_head_locked(&self->om_free_list)) != NULL) {
1719 // At this point, we have locked the free list head so a racing
1720 // thread cannot come in after us. However, a racing thread could
1721 // be ahead of us; we'll detect that and delay to let it finish.
1722 //
1723 // The thread is going away. Set 'free_tail' to the last per-thread free
1724 // monitor which will be linked to om_list_globals._free_list below.
1725 //
1726 // Account for the free list head before the loop since it is
1727 // already locked (by this thread):
1728 free_tail = free_list;
1729 free_count++;
1730 for (ObjectMonitor* s = unmarked_next(free_list); s != NULL; s = unmarked_next(s)) {
1731 if (is_locked(s)) {
1732 // s is locked so there must be a racing walker thread ahead
1733 // of us so we'll give it a chance to finish.
1734 while (is_locked(s)) {
1735 os::naked_short_sleep(1);
1736 }
1737 }
1738 free_tail = s;
1739 free_count++;
1740 guarantee(s->object() == NULL, "invariant");
1741 if (s->is_busy()) {
1742 stringStream ss;
1743 fatal("must be !is_busy: %s", s->is_busy_to_string(&ss));
1744 }
1745 }
1746 guarantee(free_tail != NULL, "invariant");
1747 int l_om_free_count = Atomic::load(&self->om_free_count);
1748 ADIM_guarantee(l_om_free_count == free_count, "free counts don't match: "
1749 "l_om_free_count=%d, free_count=%d", l_om_free_count, free_count);
1750 Atomic::store(&self->om_free_count, 0);
1751 Atomic::store(&self->om_free_list, (ObjectMonitor*)NULL);
1752 om_unlock(free_list);
1753 }
1754
1755 if (free_tail != NULL) {
1756 prepend_list_to_global_free_list(free_list, free_tail, free_count);
1757 }
1758
1759 if (in_use_tail != NULL) {
1760 prepend_list_to_global_in_use_list(in_use_list, in_use_tail, in_use_count);
1761 }
1762
1763 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1764 LogStreamHandle(Info, monitorinflation) lsh_info;
1765 LogStream* ls = NULL;
1766 if (log_is_enabled(Debug, monitorinflation)) {
1767 ls = &lsh_debug;
1768 } else if ((free_count != 0 || in_use_count != 0) &&
1769 log_is_enabled(Info, monitorinflation)) {
1808 !SafepointSynchronize::is_at_safepoint(), "invariant");
1809
1810 EventJavaMonitorInflate event;
1811
1812 for (;;) {
1813 const markWord mark = object->mark();
1814 assert(!mark.has_bias_pattern(), "invariant");
1815
1816 // The mark can be in one of the following states:
1817 // * Inflated - just return
1818 // * Stack-locked - coerce it to inflated
1819 // * INFLATING - busy wait for conversion to complete
1820 // * Neutral - aggressively inflate the object.
1821 // * BIASED - Illegal. We should never see this
1822
1823 // CASE: inflated
1824 if (mark.has_monitor()) {
1825 ObjectMonitor* inf = mark.monitor();
1826 markWord dmw = inf->header();
1827 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1828 assert(AsyncDeflateIdleMonitors || inf->object() == object, "invariant");
1829 assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
1830 return inf;
1831 }
1832
1833 // CASE: inflation in progress - inflating over a stack-lock.
1834 // Some other thread is converting from stack-locked to inflated.
1835 // Only that thread can complete inflation -- other threads must wait.
1836 // The INFLATING value is transient.
1837 // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1838 // We could always eliminate polling by parking the thread on some auxiliary list.
1839 if (mark == markWord::INFLATING()) {
1840 read_stable_mark(object);
1841 continue;
1842 }
1843
1844 // CASE: stack-locked
1845 // Could be stack-locked either by this thread or by some other thread.
1846 //
1847 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1848 // to install INFLATING into the mark word. We originally installed INFLATING,
1894 // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1895 // the while preserving the hashCode stability invariants. If the owner
1896 // decides to release the lock while the value is 0, the unlock will fail
1897 // and control will eventually pass from slow_exit() to inflate. The owner
1898 // will then spin, waiting for the 0 value to disappear. Put another way,
1899 // the 0 causes the owner to stall if the owner happens to try to
1900 // drop the lock (restoring the header from the BasicLock to the object)
1901 // while inflation is in-progress. This protocol avoids races that might
1902 // would otherwise permit hashCode values to change or "flicker" for an object.
1903 // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1904 // 0 serves as a "BUSY" inflate-in-progress indicator.
1905
1906
1907 // fetch the displaced mark from the owner's stack.
1908 // The owner can't die or unwind past the lock while our INFLATING
1909 // object is in the mark. Furthermore the owner can't complete
1910 // an unlock on the object, either.
1911 markWord dmw = mark.displaced_mark_helper();
1912 // Catch if the object's header is not neutral (not locked and
1913 // not marked is what we care about here).
1914 ADIM_guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1915
1916 // Setup monitor fields to proper values -- prepare the monitor
1917 m->set_header(dmw);
1918
1919 // Optimization: if the mark.locker stack address is associated
1920 // with this thread we could simply set m->_owner = self.
1921 // Note that a thread can inflate an object
1922 // that it has stack-locked -- as might happen in wait() -- directly
1923 // with CAS. That is, we can avoid the xchg-NULL .... ST idiom.
1924 if (AsyncDeflateIdleMonitors) {
1925 m->set_owner_from(NULL, DEFLATER_MARKER, mark.locker());
1926 } else {
1927 m->set_owner_from(NULL, mark.locker());
1928 }
1929 m->set_object(object);
1930 // TODO-FIXME: assert BasicLock->dhw != 0.
1931
1932 // Must preserve store ordering. The monitor state must
1933 // be stable at the time of publishing the monitor address.
1934 guarantee(object->mark() == markWord::INFLATING(), "invariant");
1935 object->release_set_mark(markWord::encode(m));
1936
1937 // Once ObjectMonitor is configured and the object is associated
1938 // with the ObjectMonitor, it is safe to allow async deflation:
1939 assert(m->is_new(), "freshly allocated monitor must be new");
1940 m->set_allocation_state(ObjectMonitor::Old);
1941
1942 // Hopefully the performance counters are allocated on distinct cache lines
1943 // to avoid false sharing on MP systems ...
1944 OM_PERFDATA_OP(Inflations, inc());
1945 if (log_is_enabled(Trace, monitorinflation)) {
1946 ResourceMark rm(self);
1947 lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
1948 INTPTR_FORMAT ", type='%s'", p2i(object),
1949 object->mark().value(), object->klass()->external_name());
1950 }
1951 if (event.should_commit()) {
1952 post_monitor_inflate_event(&event, object, cause);
1953 }
1954 return m;
1955 }
1956
1957 // CASE: neutral
1958 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
1959 // If we know we're inflating for entry it's better to inflate by swinging a
1960 // pre-locked ObjectMonitor pointer into the object header. A successful
1961 // CAS inflates the object *and* confers ownership to the inflating thread.
1962 // In the current implementation we use a 2-step mechanism where we CAS()
1963 // to inflate and then CAS() again to try to swing _owner from NULL to self.
1964 // An inflateTry() method that we could call from enter() would be useful.
1965
1966 // Catch if the object's header is not neutral (not locked and
1967 // not marked is what we care about here).
1968 ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1969 ObjectMonitor* m = om_alloc(self);
1970 // prepare m for installation - set monitor to initial state
1971 m->Recycle();
1972 m->set_header(mark);
1973 if (AsyncDeflateIdleMonitors) {
1974 // DEFLATER_MARKER is the only non-NULL value we should see here.
1975 m->try_set_owner_from(DEFLATER_MARKER, NULL);
1976 }
1977 m->set_object(object);
1978 m->_Responsible = NULL;
1979 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
1980
1981 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
1982 m->set_header(markWord::zero());
1983 m->set_object(NULL);
1984 m->Recycle();
1985 // om_release() will reset the allocation state from New to Free.
1986 om_release(self, m, true);
1987 m = NULL;
1988 continue;
1989 // interference - the markword changed - just retry.
1990 // The state-transitions are one-way, so there's no chance of
1991 // live-lock -- "Inflated" is an absorbing state.
1992 }
1993
1994 // Once the ObjectMonitor is configured and object is associated
1995 // with the ObjectMonitor, it is safe to allow async deflation:
1996 assert(m->is_new(), "freshly allocated monitor must be new");
1997 m->set_allocation_state(ObjectMonitor::Old);
1998
1999 // Hopefully the performance counters are allocated on distinct
2000 // cache lines to avoid false sharing on MP systems ...
2001 OM_PERFDATA_OP(Inflations, inc());
2002 if (log_is_enabled(Trace, monitorinflation)) {
2003 ResourceMark rm(self);
2004 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
2005 INTPTR_FORMAT ", type='%s'", p2i(object),
2006 object->mark().value(), object->klass()->external_name());
2007 }
2008 if (event.should_commit()) {
2009 post_monitor_inflate_event(&event, object, cause);
2010 }
2011 return m;
2012 }
2013 }
2014
2015
2016 // We maintain a list of in-use monitors for each thread.
2017 //
2018 // For safepoint based deflation:
2019 // deflate_thread_local_monitors() scans a single thread's in-use list, while
2020 // deflate_idle_monitors() scans only a global list of in-use monitors which
2021 // is populated only as a thread dies (see om_flush()).
2022 //
2023 // These operations are called at all safepoints, immediately after mutators
2024 // are stopped, but before any objects have moved. Collectively they traverse
2025 // the population of in-use monitors, deflating where possible. The scavenged
2026 // monitors are returned to the global monitor free list.
2027 //
2028 // Beware that we scavenge at *every* stop-the-world point. Having a large
2029 // number of monitors in-use could negatively impact performance. We also want
2030 // to minimize the total # of monitors in circulation, as they incur a small
2031 // footprint penalty.
2032 //
2033 // Perversely, the heap size -- and thus the STW safepoint rate --
2034 // typically drives the scavenge rate. Large heaps can mean infrequent GC,
2035 // which in turn can mean large(r) numbers of ObjectMonitors in circulation.
2036 // This is an unfortunate aspect of this design.
2037 //
2038 // For async deflation:
2039 // If a special deflation request is made, then the safepoint based
2040 // deflation mechanism is used. Otherwise, an async deflation request
2041 // is registered with the ServiceThread and it is notified.
2042
2043 void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* counters) {
2044 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2045
2046 // The per-thread in-use lists are handled in
2047 // ParallelSPCleanupThreadClosure::do_thread().
2048
2049 if (!AsyncDeflateIdleMonitors) {
2050 // Use the older mechanism for the global in-use list.
2051 ObjectSynchronizer::deflate_idle_monitors(counters);
2052 return;
2053 }
2054
2055 log_debug(monitorinflation)("requesting async deflation of idle monitors.");
2056 // Request deflation of idle monitors by the ServiceThread:
2057 set_is_async_deflation_requested(true);
2058 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
2059 ml.notify_all();
2060
2061 if (log_is_enabled(Debug, monitorinflation)) {
2062 // exit_globals()'s call to audit_and_print_stats() is done
2063 // at the Info level and not at a safepoint.
2064 // For safepoint based deflation, audit_and_print_stats() is called
2065 // in ObjectSynchronizer::finish_deflate_idle_monitors() at the
2066 // Debug level at a safepoint.
2067 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2068 }
2069 }
2070
2071 // Deflate a single monitor if not in-use
2072 // Return true if deflated, false if in-use
2073 bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj,
2074 ObjectMonitor** free_head_p,
2075 ObjectMonitor** free_tail_p) {
2076 bool deflated;
2077 // Normal case ... The monitor is associated with obj.
2078 const markWord mark = obj->mark();
2079 guarantee(mark == markWord::encode(mid), "should match: mark="
2080 INTPTR_FORMAT ", encoded mid=" INTPTR_FORMAT, mark.value(),
2081 markWord::encode(mid).value());
2082 // Make sure that mark.monitor() and markWord::encode() agree:
2083 guarantee(mark.monitor() == mid, "should match: monitor()=" INTPTR_FORMAT
2084 ", mid=" INTPTR_FORMAT, p2i(mark.monitor()), p2i(mid));
2085 const markWord dmw = mid->header();
2086 guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
2087
2088 if (mid->is_busy()) {
2089 // Easy checks are first - the ObjectMonitor is busy so no deflation.
2090 deflated = false;
2091 } else {
2092 // Deflate the monitor if it is no longer being used
2093 // It's idle - scavenge and return to the global free list
2094 // plain old deflation ...
2095 if (log_is_enabled(Trace, monitorinflation)) {
2096 ResourceMark rm;
2097 log_trace(monitorinflation)("deflate_monitor: "
2098 "object=" INTPTR_FORMAT ", mark="
2099 INTPTR_FORMAT ", type='%s'", p2i(obj),
2100 mark.value(), obj->klass()->external_name());
2101 }
2102
2103 // Restore the header back to obj
2104 obj->release_set_mark(dmw);
2105 if (AsyncDeflateIdleMonitors) {
2106 // clear() expects the owner field to be NULL.
2107 // DEFLATER_MARKER is the only non-NULL value we should see here.
2108 mid->try_set_owner_from(DEFLATER_MARKER, NULL);
2109 }
2110 mid->clear();
2111
2112 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
2113 p2i(mid->object()));
2114 assert(mid->is_free(), "invariant");
2115
2116 // Move the deflated ObjectMonitor to the working free list
2117 // defined by free_head_p and free_tail_p.
2118 if (*free_head_p == NULL) *free_head_p = mid;
2119 if (*free_tail_p != NULL) {
2120 // We append to the list so the caller can use mid->_next_om
2121 // to fix the linkages in its context.
2122 ObjectMonitor* prevtail = *free_tail_p;
2123 // Should have been cleaned up by the caller:
2124 // Note: Should not have to lock prevtail here since we're at a
2125 // safepoint and ObjectMonitors on the local free list should
2126 // not be accessed in parallel.
2127 #ifdef ASSERT
2128 ObjectMonitor* l_next_om = prevtail->next_om();
2129 #endif
2130 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2131 prevtail->set_next_om(mid);
2132 }
2133 *free_tail_p = mid;
2134 // At this point, mid->_next_om still refers to its current
2135 // value and another ObjectMonitor's _next_om field still
2136 // refers to this ObjectMonitor. Those linkages have to be
2137 // cleaned up by the caller who has the complete context.
2138 deflated = true;
2139 }
2140 return deflated;
2141 }
2142
2143 // Deflate the specified ObjectMonitor if not in-use using a JavaThread.
2144 // Returns true if it was deflated and false otherwise.
2145 //
2146 // The async deflation protocol sets owner to DEFLATER_MARKER and
2147 // makes contentions negative as signals to contending threads that
2148 // an async deflation is in progress. There are a number of checks
2149 // as part of the protocol to make sure that the calling thread has
2150 // not lost the race to a contending thread.
2151 //
2152 // The ObjectMonitor has been successfully async deflated when:
2153 // (contentions < 0)
2154 // Contending threads that see that condition know to retry their operation.
2155 //
2156 bool ObjectSynchronizer::deflate_monitor_using_JT(ObjectMonitor* mid,
2157 ObjectMonitor** free_head_p,
2158 ObjectMonitor** free_tail_p) {
2159 assert(AsyncDeflateIdleMonitors, "sanity check");
2160 assert(Thread::current()->is_Java_thread(), "precondition");
2161 // A newly allocated ObjectMonitor should not be seen here so we
2162 // avoid an endless inflate/deflate cycle.
2163 assert(mid->is_old(), "must be old: allocation_state=%d",
2164 (int) mid->allocation_state());
2165
2166 if (mid->is_busy()) {
2167 // Easy checks are first - the ObjectMonitor is busy so no deflation.
2168 return false;
2169 }
2170
2171 // Set a NULL owner to DEFLATER_MARKER to force any contending thread
2172 // through the slow path. This is just the first part of the async
2173 // deflation dance.
2174 if (mid->try_set_owner_from(NULL, DEFLATER_MARKER) != NULL) {
2175 // The owner field is no longer NULL so we lost the race since the
2176 // ObjectMonitor is now busy.
2177 return false;
2178 }
2179
2228 mid->clear_common();
2229
2230 assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT,
2231 p2i(mid->object()));
2232 assert(mid->is_free(), "must be free: allocation_state=%d",
2233 (int)mid->allocation_state());
2234
2235 // Move the deflated ObjectMonitor to the working free list
2236 // defined by free_head_p and free_tail_p.
2237 if (*free_head_p == NULL) {
2238 // First one on the list.
2239 *free_head_p = mid;
2240 }
2241 if (*free_tail_p != NULL) {
2242 // We append to the list so the caller can use mid->_next_om
2243 // to fix the linkages in its context.
2244 ObjectMonitor* prevtail = *free_tail_p;
2245 // prevtail should have been cleaned up by the caller:
2246 #ifdef ASSERT
2247 ObjectMonitor* l_next_om = unmarked_next(prevtail);
2248 #endif
2249 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2250 om_lock(prevtail);
2251 prevtail->set_next_om(mid); // prevtail now points to mid (and is unlocked)
2252 }
2253 *free_tail_p = mid;
2254
2255 // At this point, mid->_next_om still refers to its current
2256 // value and another ObjectMonitor's _next_om field still
2257 // refers to this ObjectMonitor. Those linkages have to be
2258 // cleaned up by the caller who has the complete context.
2259
2260 // We leave owner == DEFLATER_MARKER and contentions < 0
2261 // to force any racing threads to retry.
2262 return true; // Success, ObjectMonitor has been deflated.
2263 }
2264
2265 // Walk a given monitor list, and deflate idle monitors.
2266 // The given list could be a per-thread list or a global list.
2267 //
2268 // In the case of parallel processing of thread local monitor lists,
2269 // work is done by Threads::parallel_threads_do() which ensures that
2270 // each Java thread is processed by exactly one worker thread, and
2271 // thus avoid conflicts that would arise when worker threads would
2272 // process the same monitor lists concurrently.
2273 //
2274 // See also ParallelSPCleanupTask and
2275 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
2276 // Threads::parallel_java_threads_do() in thread.cpp.
2277 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor** list_p,
2278 int* count_p,
2279 ObjectMonitor** free_head_p,
2280 ObjectMonitor** free_tail_p) {
2281 ObjectMonitor* cur_mid_in_use = NULL;
2282 ObjectMonitor* mid = NULL;
2283 ObjectMonitor* next = NULL;
2284 int deflated_count = 0;
2285
2286 // This list walk executes at a safepoint and does not race with any
2287 // other list walkers.
2288
2289 for (mid = Atomic::load(list_p); mid != NULL; mid = next) {
2290 next = unmarked_next(mid);
2291 oop obj = (oop) mid->object();
2292 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
2293 // Deflation succeeded and already updated free_head_p and
2294 // free_tail_p as needed. Finish the move to the local free list
2295 // by unlinking mid from the global or per-thread in-use list.
2296 if (cur_mid_in_use == NULL) {
2297 // mid is the list head so switch the list head to next:
2298 Atomic::store(list_p, next);
2299 } else {
2300 // Switch cur_mid_in_use's next field to next:
2301 cur_mid_in_use->set_next_om(next);
2302 }
2303 // At this point mid is disconnected from the in-use list.
2304 deflated_count++;
2305 Atomic::dec(count_p);
2306 // mid is current tail in the free_head_p list so NULL terminate it:
2307 mid->set_next_om(NULL);
2308 } else {
2309 cur_mid_in_use = mid;
2310 }
2311 }
2312 return deflated_count;
2313 }
2314
2315 // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using
2316 // a JavaThread. Returns the number of deflated ObjectMonitors. The given
2317 // list could be a per-thread in-use list or the global in-use list.
2318 // If a safepoint has started, then we save state via saved_mid_in_use_p
2319 // and return to the caller to honor the safepoint.
2320 //
2321 int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor** list_p,
2322 int* count_p,
2323 ObjectMonitor** free_head_p,
2324 ObjectMonitor** free_tail_p,
2325 ObjectMonitor** saved_mid_in_use_p) {
2326 assert(AsyncDeflateIdleMonitors, "sanity check");
2327 JavaThread* self = JavaThread::current();
2328
2329 ObjectMonitor* cur_mid_in_use = NULL;
2330 ObjectMonitor* mid = NULL;
2331 ObjectMonitor* next = NULL;
2332 ObjectMonitor* next_next = NULL;
2333 int deflated_count = 0;
2334 NoSafepointVerifier nsv;
2335
2336 // We use the more complicated lock-cur_mid_in_use-and-mid-as-we-go
2337 // protocol because om_release() can do list deletions in parallel;
2338 // this also prevents races with a list walker thread. We also
2339 // lock-next-next-as-we-go to prevent an om_flush() that is behind
2340 // this thread from passing us.
2341 if (*saved_mid_in_use_p == NULL) {
2342 // No saved state so start at the beginning.
2343 // Lock the list head so we can possibly deflate it:
2344 if ((mid = get_list_head_locked(list_p)) == NULL) {
2345 return 0; // The list is empty so nothing to deflate.
2346 }
2436 }
2437 return deflated_count;
2438 }
2439 }
2440 if (mid == NULL) {
2441 if (cur_mid_in_use != NULL) {
2442 om_unlock(cur_mid_in_use);
2443 }
2444 break; // Reached end of the list so nothing more to deflate.
2445 }
2446
2447 // The current mid's next field is locked at this point. If we have
2448 // a cur_mid_in_use, then it is also locked at this point.
2449 }
2450 // We finished the list without a safepoint starting so there's
2451 // no need to save state.
2452 *saved_mid_in_use_p = NULL;
2453 return deflated_count;
2454 }
2455
2456 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2457 counters->n_in_use = 0; // currently associated with objects
2458 counters->n_in_circulation = 0; // extant
2459 counters->n_scavenged = 0; // reclaimed (global and per-thread)
2460 counters->per_thread_scavenged = 0; // per-thread scavenge total
2461 counters->per_thread_times = 0.0; // per-thread scavenge times
2462 }
2463
2464 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
2465 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2466
2467 if (AsyncDeflateIdleMonitors) {
2468 // Nothing to do when global idle ObjectMonitors are deflated using
2469 // a JavaThread.
2470 return;
2471 }
2472
2473 bool deflated = false;
2474
2475 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2476 ObjectMonitor* free_tail_p = NULL;
2477 elapsedTimer timer;
2478
2479 if (log_is_enabled(Info, monitorinflation)) {
2480 timer.start();
2481 }
2482
2483 // Note: the thread-local monitors lists get deflated in
2484 // a separate pass. See deflate_thread_local_monitors().
2485
2486 // For moribund threads, scan om_list_globals._in_use_list
2487 int deflated_count = 0;
2488 if (Atomic::load(&om_list_globals._in_use_list) != NULL) {
2489 // Update n_in_circulation before om_list_globals._in_use_count is
2490 // updated by deflation.
2491 Atomic::add(&counters->n_in_circulation,
2492 Atomic::load(&om_list_globals._in_use_count));
2493
2494 deflated_count = deflate_monitor_list(&om_list_globals._in_use_list,
2495 &om_list_globals._in_use_count,
2496 &free_head_p, &free_tail_p);
2497 Atomic::add(&counters->n_in_use, Atomic::load(&om_list_globals._in_use_count));
2498 }
2499
2500 if (free_head_p != NULL) {
2501 // Move the deflated ObjectMonitors back to the global free list.
2502 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2503 #ifdef ASSERT
2504 ObjectMonitor* l_next_om = free_tail_p->next_om();
2505 #endif
2506 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2507 prepend_list_to_global_free_list(free_head_p, free_tail_p, deflated_count);
2508 Atomic::add(&counters->n_scavenged, deflated_count);
2509 }
2510 timer.stop();
2511
2512 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2513 LogStreamHandle(Info, monitorinflation) lsh_info;
2514 LogStream* ls = NULL;
2515 if (log_is_enabled(Debug, monitorinflation)) {
2516 ls = &lsh_debug;
2517 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2518 ls = &lsh_info;
2519 }
2520 if (ls != NULL) {
2521 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2522 }
2523 }
2524
2525 class HandshakeForDeflation : public HandshakeClosure {
2526 public:
2527 HandshakeForDeflation() : HandshakeClosure("HandshakeForDeflation") {}
2528
2529 void do_thread(Thread* thread) {
2530 log_trace(monitorinflation)("HandshakeForDeflation::do_thread: thread="
2531 INTPTR_FORMAT, p2i(thread));
2532 }
2533 };
2534
2535 void ObjectSynchronizer::deflate_idle_monitors_using_JT() {
2536 assert(AsyncDeflateIdleMonitors, "sanity check");
2537
2538 // Deflate any global idle monitors.
2539 deflate_global_idle_monitors_using_JT();
2540
2541 int count = 0;
2542 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2543 if (Atomic::load(&jt->om_in_use_count) > 0 && !jt->is_exiting()) {
2544 // This JavaThread is using ObjectMonitors so deflate any that
2545 // are idle unless this JavaThread is exiting; do not race with
2546 // ObjectSynchronizer::om_flush().
2547 deflate_per_thread_idle_monitors_using_JT(jt);
2548 count++;
2549 }
2550 }
2551 if (count > 0) {
2552 log_debug(monitorinflation)("did async deflation of idle monitors for %d thread(s).", count);
2553 }
2554
2555 log_info(monitorinflation)("async global_population=%d, global_in_use_count=%d, "
2556 "global_free_count=%d, global_wait_count=%d",
2557 Atomic::load(&om_list_globals._population),
2558 Atomic::load(&om_list_globals._in_use_count),
2559 Atomic::load(&om_list_globals._free_count),
2560 Atomic::load(&om_list_globals._wait_count));
2561
2562 // The ServiceThread's async deflation request has been processed.
2563 _last_async_deflation_time_ns = os::javaTimeNanos();
2564 set_is_async_deflation_requested(false);
2565
2566 if (Atomic::load(&om_list_globals._wait_count) > 0) {
2567 // There are deflated ObjectMonitors waiting for a handshake
2568 // (or a safepoint) for safety.
2569
2570 ObjectMonitor* list = Atomic::load(&om_list_globals._wait_list);
2571 ADIM_guarantee(list != NULL, "om_list_globals._wait_list must not be NULL");
2572 int count = Atomic::load(&om_list_globals._wait_count);
2573 Atomic::store(&om_list_globals._wait_count, 0);
2574 Atomic::store(&om_list_globals._wait_list, (ObjectMonitor*)NULL);
2575
2576 // Find the tail for prepend_list_to_common(). No need to mark
2577 // ObjectMonitors for this list walk since only the deflater
2578 // thread manages the wait list.
2579 int l_count = 0;
2580 ObjectMonitor* tail = NULL;
2581 for (ObjectMonitor* n = list; n != NULL; n = unmarked_next(n)) {
2582 tail = n;
2583 l_count++;
2584 }
2585 ADIM_guarantee(count == l_count, "count=%d != l_count=%d", count, l_count);
2586
2587 // Will execute a safepoint if !ThreadLocalHandshakes:
2588 HandshakeForDeflation hfd_hc;
2589 Handshake::execute(&hfd_hc);
2590
2591 prepend_list_to_common(list, tail, count, &om_list_globals._free_list,
2592 &om_list_globals._free_count);
2593
2594 log_info(monitorinflation)("moved %d idle monitors from global waiting list to global free list", count);
2595 }
2596 }
2597
2598 // Deflate global idle ObjectMonitors using a JavaThread.
2599 //
2600 void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() {
2601 assert(AsyncDeflateIdleMonitors, "sanity check");
2602 assert(Thread::current()->is_Java_thread(), "precondition");
2603 JavaThread* self = JavaThread::current();
2604
2605 deflate_common_idle_monitors_using_JT(true /* is_global */, self);
2606 }
2607
2608 // Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread.
2609 //
2610 void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) {
2611 assert(AsyncDeflateIdleMonitors, "sanity check");
2612 assert(Thread::current()->is_Java_thread(), "precondition");
2613
2614 deflate_common_idle_monitors_using_JT(false /* !is_global */, target);
2615 }
2616
2617 // Deflate global or per-thread idle ObjectMonitors using a JavaThread.
2618 //
2619 void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) {
2620 JavaThread* self = JavaThread::current();
2621
2622 int deflated_count = 0;
2623 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors
2624 ObjectMonitor* free_tail_p = NULL;
2625 ObjectMonitor* saved_mid_in_use_p = NULL;
2626 elapsedTimer timer;
2627
2628 if (log_is_enabled(Info, monitorinflation)) {
2629 timer.start();
2630 }
2631
2647 local_deflated_count =
2648 deflate_monitor_list_using_JT(&target->om_in_use_list,
2649 &target->om_in_use_count, &free_head_p,
2650 &free_tail_p, &saved_mid_in_use_p);
2651 }
2652 deflated_count += local_deflated_count;
2653
2654 if (free_head_p != NULL) {
2655 // Move the deflated ObjectMonitors to the global free list.
2656 guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count);
2657 // Note: The target thread can be doing an om_alloc() that
2658 // is trying to prepend an ObjectMonitor on its in-use list
2659 // at the same time that we have deflated the current in-use
2660 // list head and put it on the local free list. prepend_to_common()
2661 // will detect the race and retry which avoids list corruption,
2662 // but the next field in free_tail_p can flicker to marked
2663 // and then unmarked while prepend_to_common() is sorting it
2664 // all out.
2665 #ifdef ASSERT
2666 ObjectMonitor* l_next_om = unmarked_next(free_tail_p);
2667 #endif
2668 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2669
2670 prepend_list_to_global_wait_list(free_head_p, free_tail_p, local_deflated_count);
2671
2672 OM_PERFDATA_OP(Deflations, inc(local_deflated_count));
2673 }
2674
2675 if (saved_mid_in_use_p != NULL) {
2676 // deflate_monitor_list_using_JT() detected a safepoint starting.
2677 timer.stop();
2678 {
2679 if (is_global) {
2680 log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint.");
2681 } else {
2682 log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target));
2683 }
2684 assert(SafepointMechanism::should_block(self), "sanity check");
2685 ThreadBlockInVM blocker(self);
2686 }
2687 // Prepare for another loop after the safepoint.
2688 free_head_p = NULL;
2694 } while (saved_mid_in_use_p != NULL);
2695 timer.stop();
2696
2697 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2698 LogStreamHandle(Info, monitorinflation) lsh_info;
2699 LogStream* ls = NULL;
2700 if (log_is_enabled(Debug, monitorinflation)) {
2701 ls = &lsh_debug;
2702 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2703 ls = &lsh_info;
2704 }
2705 if (ls != NULL) {
2706 if (is_global) {
2707 ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2708 } else {
2709 ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count);
2710 }
2711 }
2712 }
2713
2714 void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2715 // Report the cumulative time for deflating each thread's idle
2716 // monitors. Note: if the work is split among more than one
2717 // worker thread, then the reported time will likely be more
2718 // than a beginning to end measurement of the phase.
2719 log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", counters->per_thread_times, counters->per_thread_scavenged);
2720
2721 if (AsyncDeflateIdleMonitors) {
2722 // Nothing to do when idle ObjectMonitors are deflated using
2723 // a JavaThread.
2724 return;
2725 }
2726
2727 if (log_is_enabled(Debug, monitorinflation)) {
2728 // exit_globals()'s call to audit_and_print_stats() is done
2729 // at the Info level and not at a safepoint.
2730 // For async deflation, audit_and_print_stats() is called in
2731 // ObjectSynchronizer::do_safepoint_work() at the Debug level
2732 // at a safepoint.
2733 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2734 } else if (log_is_enabled(Info, monitorinflation)) {
2735 log_info(monitorinflation)("global_population=%d, global_in_use_count=%d, "
2736 "global_free_count=%d, global_wait_count=%d",
2737 Atomic::load(&om_list_globals._population),
2738 Atomic::load(&om_list_globals._in_use_count),
2739 Atomic::load(&om_list_globals._free_count),
2740 Atomic::load(&om_list_globals._wait_count));
2741 }
2742
2743 OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged));
2744 OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation));
2745
2746 GVars.stw_random = os::random();
2747 GVars.stw_cycle++;
2748 }
2749
2750 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
2751 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2752
2753 if (AsyncDeflateIdleMonitors) {
2754 // Nothing to do when per-thread idle ObjectMonitors are deflated
2755 // using a JavaThread.
2756 return;
2757 }
2758
2759 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2760 ObjectMonitor* free_tail_p = NULL;
2761 elapsedTimer timer;
2762
2763 if (log_is_enabled(Info, safepoint, cleanup) ||
2764 log_is_enabled(Info, monitorinflation)) {
2765 timer.start();
2766 }
2767
2768 // Update n_in_circulation before om_in_use_count is updated by deflation.
2769 Atomic::add(&counters->n_in_circulation, Atomic::load(&thread->om_in_use_count));
2770
2771 int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p);
2772 Atomic::add(&counters->n_in_use, Atomic::load(&thread->om_in_use_count));
2773
2774 if (free_head_p != NULL) {
2775 // Move the deflated ObjectMonitors back to the global free list.
2776 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2777 #ifdef ASSERT
2778 ObjectMonitor* l_next_om = free_tail_p->next_om();
2779 #endif
2780 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2781 prepend_list_to_global_free_list(free_head_p, free_tail_p, deflated_count);
2782 Atomic::add(&counters->n_scavenged, deflated_count);
2783 Atomic::add(&counters->per_thread_scavenged, deflated_count);
2784 }
2785
2786 timer.stop();
2787 counters->per_thread_times += timer.seconds();
2788
2789 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2790 LogStreamHandle(Info, monitorinflation) lsh_info;
2791 LogStream* ls = NULL;
2792 if (log_is_enabled(Debug, monitorinflation)) {
2793 ls = &lsh_debug;
2794 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2795 ls = &lsh_info;
2796 }
2797 if (ls != NULL) {
2798 ls->print_cr("jt=" INTPTR_FORMAT ": deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(thread), timer.seconds(), deflated_count);
2799 }
2800 }
2801
2802 // Monitor cleanup on JavaThread::exit
2803
2804 // Iterate through monitor cache and attempt to release thread's monitors
2805 // Gives up on a particular monitor if an exception occurs, but continues
2806 // the overall iteration, swallowing the exception.
2807 class ReleaseJavaMonitorsClosure: public MonitorClosure {
2808 private:
2809 TRAPS;
2810
2811 public:
2812 ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {}
2813 void do_monitor(ObjectMonitor* mid) {
2814 if (mid->owner() == THREAD) {
2815 (void)mid->complete_exit(CHECK);
2816 }
2817 }
2818 };
2819
2820 // Release all inflated monitors owned by THREAD. Lightweight monitors are
2821 // ignored. This is meant to be called during JNI thread detach which assumes
2822 // all remaining monitors are heavyweight. All exceptions are swallowed.
2823 // Scanning the extant monitor list can be time consuming.
2824 // A simple optimization is to add a per-thread flag that indicates a thread
2825 // called jni_monitorenter() during its lifetime.
2826 //
2827 // Instead of No_Savepoint_Verifier it might be cheaper to
2828 // use an idiom of the form:
2829 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
2830 // <code that must not run at safepoint>
2831 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
2832 // Since the tests are extremely cheap we could leave them enabled
2833 // for normal product builds.
2834
2835 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
2836 assert(THREAD == JavaThread::current(), "must be current Java thread");
2837 NoSafepointVerifier nsv;
2838 ReleaseJavaMonitorsClosure rjmc(THREAD);
2839 ObjectSynchronizer::monitors_iterate(&rjmc);
2840 THREAD->clear_pending_exception();
2841 }
2842
2843 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
2844 switch (cause) {
2845 case inflate_cause_vm_internal: return "VM Internal";
2846 case inflate_cause_monitor_enter: return "Monitor Enter";
2847 case inflate_cause_wait: return "Monitor Wait";
2866 return (u_char*)&GVars.hc_sequence;
2867 }
2868
2869 size_t ObjectSynchronizer::get_gvars_size() {
2870 return sizeof(SharedGlobals);
2871 }
2872
2873 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
2874 return (u_char*)&GVars.stw_random;
2875 }
2876
2877 // This function can be called at a safepoint or it can be called when
2878 // we are trying to exit the VM. When we are trying to exit the VM, the
2879 // list walker functions can run in parallel with the other list
2880 // operations so spin-locking is used for safety.
2881 //
2882 // Calls to this function can be added in various places as a debugging
2883 // aid; pass 'true' for the 'on_exit' parameter to have in-use monitor
2884 // details logged at the Info level and 'false' for the 'on_exit'
2885 // parameter to have in-use monitor details logged at the Trace level.
2886 // deflate_monitor_list() no longer uses spin-locking so be careful
2887 // when adding audit_and_print_stats() calls at a safepoint.
2888 //
2889 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
2890 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
2891
2892 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2893 LogStreamHandle(Info, monitorinflation) lsh_info;
2894 LogStreamHandle(Trace, monitorinflation) lsh_trace;
2895 LogStream* ls = NULL;
2896 if (log_is_enabled(Trace, monitorinflation)) {
2897 ls = &lsh_trace;
2898 } else if (log_is_enabled(Debug, monitorinflation)) {
2899 ls = &lsh_debug;
2900 } else if (log_is_enabled(Info, monitorinflation)) {
2901 ls = &lsh_info;
2902 }
2903 assert(ls != NULL, "sanity check");
2904
2905 // Log counts for the global and per-thread monitor lists:
2906 int chk_om_population = log_monitor_list_counts(ls);
2907 int error_cnt = 0;
2966 outputStream * out, int *error_cnt_p) {
2967 stringStream ss;
2968 if (n->is_busy()) {
2969 if (jt != NULL) {
2970 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2971 ": free per-thread monitor must not be busy: %s", p2i(jt),
2972 p2i(n), n->is_busy_to_string(&ss));
2973 } else {
2974 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2975 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
2976 }
2977 *error_cnt_p = *error_cnt_p + 1;
2978 }
2979 if (n->header().value() != 0) {
2980 if (jt != NULL) {
2981 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2982 ": free per-thread monitor must have NULL _header "
2983 "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n),
2984 n->header().value());
2985 *error_cnt_p = *error_cnt_p + 1;
2986 } else if (!AsyncDeflateIdleMonitors) {
2987 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2988 "must have NULL _header field: _header=" INTPTR_FORMAT,
2989 p2i(n), n->header().value());
2990 *error_cnt_p = *error_cnt_p + 1;
2991 }
2992 }
2993 if (n->object() != NULL) {
2994 if (jt != NULL) {
2995 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2996 ": free per-thread monitor must have NULL _object "
2997 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
2998 p2i(n->object()));
2999 } else {
3000 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3001 "must have NULL _object field: _object=" INTPTR_FORMAT,
3002 p2i(n), p2i(n->object()));
3003 }
3004 *error_cnt_p = *error_cnt_p + 1;
3005 }
3006 }
3007
3008 // Lock the next ObjectMonitor for traversal and unlock the current
3009 // ObjectMonitor. Returns the next ObjectMonitor if there is one.
3010 // Otherwise returns NULL (after unlocking the current ObjectMonitor).
|
474
475
476 // The LockNode emitted directly at the synchronization site would have
477 // been too big if it were to have included support for the cases of inflated
478 // recursive enter and exit, so they go here instead.
479 // Note that we can't safely call AsyncPrintJavaStack() from within
480 // quick_enter() as our thread state remains _in_Java.
481
482 bool ObjectSynchronizer::quick_enter(oop obj, Thread* self,
483 BasicLock * lock) {
484 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
485 assert(self->is_Java_thread(), "invariant");
486 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
487 NoSafepointVerifier nsv;
488 if (obj == NULL) return false; // Need to throw NPE
489
490 const markWord mark = obj->mark();
491
492 if (mark.has_monitor()) {
493 ObjectMonitor* const m = mark.monitor();
494 // An async deflation can race us before we manage to make the
495 // ObjectMonitor busy by setting the owner below. If we detect
496 // that race we just bail out to the slow-path here.
497 if (m->object() == NULL) {
498 return false;
499 }
500 Thread* const owner = (Thread *) m->_owner;
501
502 // Lock contention and Transactional Lock Elision (TLE) diagnostics
503 // and observability
504 // Case: light contention possibly amenable to TLE
505 // Case: TLE inimical operations such as nested/recursive synchronization
506
507 if (owner == self) {
508 m->_recursions++;
509 return true;
510 }
511
512 // This Java Monitor is inflated so obj's header will never be
513 // displaced to this thread's BasicLock. Make the displaced header
514 // non-NULL so this BasicLock is not seen as recursive nor as
515 // being locked. We do this unconditionally so that this thread's
516 // BasicLock cannot be mis-interpreted by any stack walkers. For
517 // performance reasons, stack walkers generally first check for
518 // Biased Locking in the object's header, the second check is for
519 // stack-locking in the object's header, the third check is for
965 unsigned t = self->_hashStateX;
966 t ^= (t << 11);
967 self->_hashStateX = self->_hashStateY;
968 self->_hashStateY = self->_hashStateZ;
969 self->_hashStateZ = self->_hashStateW;
970 unsigned v = self->_hashStateW;
971 v = (v ^ (v >> 19)) ^ (t ^ (t >> 8));
972 self->_hashStateW = v;
973 value = v;
974 }
975
976 value &= markWord::hash_mask;
977 if (value == 0) value = 0xBAD;
978 assert(value != markWord::no_hash, "invariant");
979 return value;
980 }
981
982 intptr_t ObjectSynchronizer::FastHashCode(Thread* self, oop obj) {
983 if (UseBiasedLocking) {
984 // NOTE: many places throughout the JVM do not expect a safepoint
985 // to be taken here. However, we only ever bias Java instances and all
986 // of the call sites of identity_hash that might revoke biases have
987 // been checked to make sure they can handle a safepoint. The
988 // added check of the bias pattern is to avoid useless calls to
989 // thread-local storage.
990 if (obj->mark().has_bias_pattern()) {
991 // Handle for oop obj in case of STW safepoint
992 Handle hobj(self, obj);
993 // Relaxing assertion for bug 6320749.
994 assert(Universe::verify_in_progress() ||
995 !SafepointSynchronize::is_at_safepoint(),
996 "biases should not be seen by VM thread here");
997 BiasedLocking::revoke(hobj, JavaThread::current());
998 obj = hobj();
999 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
1000 }
1001 }
1002
1003 // hashCode() is a heap mutator ...
1004 // Relaxing assertion for bug 6320749.
1005 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1006 !SafepointSynchronize::is_at_safepoint(), "invariant");
1171 // Possible mark states: neutral, biased, stack-locked, inflated
1172
1173 if (UseBiasedLocking && h_obj()->mark().has_bias_pattern()) {
1174 // CASE: biased
1175 BiasedLocking::revoke(h_obj, self);
1176 assert(!h_obj->mark().has_bias_pattern(),
1177 "biases should be revoked by now");
1178 }
1179
1180 assert(self == JavaThread::current(), "Can only be called on current thread");
1181 oop obj = h_obj();
1182 markWord mark = read_stable_mark(obj);
1183
1184 // CASE: stack-locked. Mark points to a BasicLock on the owner's stack.
1185 if (mark.has_locker()) {
1186 return self->is_lock_owned((address)mark.locker()) ?
1187 owner_self : owner_other;
1188 }
1189
1190 // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor.
1191 if (mark.has_monitor()) {
1192 // The first stage of async deflation does not affect any field
1193 // used by this comparison so the ObjectMonitor* is usable here.
1194 ObjectMonitor* monitor = mark.monitor();
1195 void* owner = monitor->owner();
1196 if (owner == NULL) return owner_none;
1197 return (owner == self ||
1198 self->is_lock_owned((address)owner)) ? owner_self : owner_other;
1199 }
1200
1201 // CASE: neutral
1202 assert(mark.is_neutral(), "sanity check");
1203 return owner_none; // it's unlocked
1204 }
1205
1206 // FIXME: jvmti should call this
1207 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
1208 if (UseBiasedLocking) {
1209 if (SafepointSynchronize::is_at_safepoint()) {
1210 BiasedLocking::revoke_at_safepoint(h_obj);
1270 // used with block linkage _next_om fields).
1271 block = (PaddedObjectMonitor*)block->next_om();
1272 }
1273 }
1274
1275 static bool monitors_used_above_threshold() {
1276 int population = Atomic::load(&om_list_globals._population);
1277 if (population == 0) {
1278 return false;
1279 }
1280 if (MonitorUsedDeflationThreshold > 0) {
1281 int monitors_used = population - Atomic::load(&om_list_globals._free_count) -
1282 Atomic::load(&om_list_globals._wait_count);
1283 int monitor_usage = (monitors_used * 100LL) / population;
1284 return monitor_usage > MonitorUsedDeflationThreshold;
1285 }
1286 return false;
1287 }
1288
1289 bool ObjectSynchronizer::is_async_deflation_needed() {
1290 if (is_async_deflation_requested()) {
1291 // Async deflation request.
1292 return true;
1293 }
1294 if (AsyncDeflationInterval > 0 &&
1295 time_since_last_async_deflation_ms() > AsyncDeflationInterval &&
1296 monitors_used_above_threshold()) {
1297 // It's been longer than our specified deflate interval and there
1298 // are too many monitors in use. We don't deflate more frequently
1299 // than AsyncDeflationInterval (unless is_async_deflation_requested)
1300 // in order to not swamp the ServiceThread.
1301 return true;
1302 }
1303 return false;
1304 }
1305
1306 bool ObjectSynchronizer::request_deflate_idle_monitors() {
1307 bool is_JavaThread = Thread::current()->is_Java_thread();
1308 bool ret_code = false;
1309
1310 jlong last_time = last_async_deflation_time_ns();
1311 set_is_async_deflation_requested(true);
1312 {
1313 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
1314 ml.notify_all();
1315 }
1316 const int N_CHECKS = 5;
1317 for (int i = 0; i < N_CHECKS; i++) { // sleep for at most 5 seconds
1318 if (last_async_deflation_time_ns() > last_time) {
1319 log_info(monitorinflation)("Async Deflation happened after %d check(s).", i);
1320 ret_code = true;
1321 break;
1322 }
1323 if (is_JavaThread) {
1324 // JavaThread has to honor the blocking protocol.
1325 ThreadBlockInVM tbivm(JavaThread::current());
1326 os::naked_short_sleep(999); // sleep for almost 1 second
1327 } else {
1328 os::naked_short_sleep(999); // sleep for almost 1 second
1329 }
1330 }
1331 if (!ret_code) {
1332 log_info(monitorinflation)("Async Deflation DID NOT happen after %d checks.", N_CHECKS);
1333 }
1334
1335 return ret_code;
1336 }
1337
1338 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1339 return (os::javaTimeNanos() - last_async_deflation_time_ns()) / (NANOUNITS / MILLIUNITS);
1340 }
1341
1342 void ObjectSynchronizer::oops_do(OopClosure* f) {
1343 // We only scan the global used list here (for moribund threads), and
1344 // the thread-local monitors in Thread::oops_do().
1345 global_used_oops_do(f);
1346 }
1347
1348 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
1349 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1350 list_oops_do(Atomic::load(&om_list_globals._in_use_list), f);
1351 }
1352
1353 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
1354 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1355 list_oops_do(thread->om_in_use_list, f);
1356 }
1357
1358 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, OopClosure* f) {
1359 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1360 // The oops_do() phase does not overlap with monitor deflation
1361 // so no need to lock ObjectMonitors for the list traversal.
1362 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
1363 if (mid->object() != NULL) {
1364 f->do_oop((oop*)mid->object_addr());
1365 }
1366 }
1367 }
1368
1369
1370 // -----------------------------------------------------------------------------
1371 // ObjectMonitor Lifecycle
1372 // -----------------------
1373 // Inflation unlinks monitors from om_list_globals._free_list or a per-thread
1374 // free list and associates them with objects. Async deflation disassociates
1375 // idle monitors from objects. Such scavenged monitors are returned to the
1376 // om_list_globals._free_list.
1377 //
1378 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
1379 //
1380 // Lifecycle:
1381 // -- unassigned and on the om_list_globals._free_list
1382 // -- unassigned and on a per-thread free list
1383 // -- assigned to an object. The object is inflated and the mark refers
1384 // to the ObjectMonitor.
1385
1386 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self) {
1387 // A large MAXPRIVATE value reduces both list lock contention
1388 // and list coherency traffic, but also tends to increase the
1389 // number of ObjectMonitors in circulation as well as the
1390 // scavenge costs. As usual, we lean toward time in space-time
1391 // tradeoffs.
1392 const int MAXPRIVATE = 1024;
1393 NoSafepointVerifier nsv;
1394
1395 for (;;) {
1396 ObjectMonitor* m;
1397
1398 // 1: try to allocate from the thread's local om_free_list.
1399 // Threads will attempt to allocate first from their local list, then
1400 // from the global list, and only after those attempts fail will the
1401 // thread attempt to instantiate new monitors. Thread-local free lists
1402 // improve allocation latency, as well as reducing coherency traffic
1403 // on the shared global list.
1404 m = take_from_start_of_om_free_list(self);
1405 if (m != NULL) {
1406 guarantee(m->object() == NULL, "invariant");
1407 m->set_allocation_state(ObjectMonitor::New);
1408 prepend_to_om_in_use_list(self, m);
1409 return m;
1410 }
1411
1412 // 2: try to allocate from the global om_list_globals._free_list
1413 // If we're using thread-local free lists then try
1414 // to reprovision the caller's free list.
1415 if (Atomic::load(&om_list_globals._free_list) != NULL) {
1416 // Reprovision the thread's om_free_list.
1417 // Use bulk transfers to reduce the allocation rate and heat
1418 // on various locks.
1419 for (int i = self->om_free_provision; --i >= 0;) {
1420 ObjectMonitor* take = take_from_start_of_global_free_list();
1421 if (take == NULL) {
1422 break; // No more are available.
1423 }
1424 guarantee(take->object() == NULL, "invariant");
1425 // We allowed 3 field values to linger during async deflation.
1426 // Clear or restore them as appropriate.
1427 take->set_header(markWord::zero());
1428 // DEFLATER_MARKER is the only non-NULL value we should see here.
1429 take->try_set_owner_from(DEFLATER_MARKER, NULL);
1430 if (take->contentions() < 0) {
1431 // Add back max_jint to restore the contentions field to its
1432 // proper value.
1433 take->add_to_contentions(max_jint);
1434
1435 #ifdef ASSERT
1436 jint l_contentions = take->contentions();
1437 assert(l_contentions >= 0, "must not be negative: l_contentions=%d, contentions=%d",
1438 l_contentions, take->contentions());
1439 #endif
1440 }
1441 take->Recycle();
1442 // Since we're taking from the global free-list, take must be Free.
1443 // om_release() also sets the allocation state to Free because it
1444 // is called from other code paths.
1445 assert(take->is_free(), "invariant");
1446 om_release(self, take, false);
1447 }
1448 self->om_free_provision += 1 + (self->om_free_provision / 2);
1449 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1450 continue;
1451 }
1452
1453 // 3: allocate a block of new ObjectMonitors
1454 // Both the local and global free lists are empty -- resort to malloc().
1455 // In the current implementation ObjectMonitors are TSM - immortal.
1456 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1457 // each ObjectMonitor to start at the beginning of a cache line,
1458 // so we use align_up().
1459 // A better solution would be to use C++ placement-new.
1485 // Element [0] is reserved for global list linkage
1486 temp[0].set_object(CHAINMARKER);
1487
1488 // Consider carving out this thread's current request from the
1489 // block in hand. This avoids some lock traffic and redundant
1490 // list activity.
1491
1492 prepend_block_to_lists(temp);
1493 }
1494 }
1495
1496 // Place "m" on the caller's private per-thread om_free_list.
1497 // In practice there's no need to clamp or limit the number of
1498 // monitors on a thread's om_free_list as the only non-allocation time
1499 // we'll call om_release() is to return a monitor to the free list after
1500 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1501 // accumulate on a thread's free list.
1502 //
1503 // Key constraint: all ObjectMonitors on a thread's free list and the global
1504 // free list must have their object field set to null. This prevents the
1505 // scavenger -- deflate_monitor_list_using_JT() -- from reclaiming them
1506 // while we are trying to release them.
1507
1508 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1509 bool from_per_thread_alloc) {
1510 guarantee(m->header().value() == 0, "invariant");
1511 guarantee(m->object() == NULL, "invariant");
1512 NoSafepointVerifier nsv;
1513
1514 if ((m->is_busy() | m->_recursions) != 0) {
1515 stringStream ss;
1516 fatal("freeing in-use monitor: %s, recursions=" INTX_FORMAT,
1517 m->is_busy_to_string(&ss), m->_recursions);
1518 }
1519 m->set_allocation_state(ObjectMonitor::Free);
1520 // _next_om is used for both per-thread in-use and free lists so
1521 // we have to remove 'm' from the in-use list first (as needed).
1522 if (from_per_thread_alloc) {
1523 // Need to remove 'm' from om_in_use_list.
1524 ObjectMonitor* mid = NULL;
1525 ObjectMonitor* next = NULL;
1526
1595
1596 // At this point mid is disconnected from the in-use list so
1597 // its lock no longer has any effects on the in-use list.
1598 Atomic::dec(&self->om_in_use_count);
1599 // Unlock mid, but leave the next value for any lagging list
1600 // walkers. It will get cleaned up when mid is prepended to
1601 // the thread's free list:
1602 om_unlock(mid);
1603 }
1604
1605 prepend_to_om_free_list(self, m);
1606 guarantee(m->is_free(), "invariant");
1607 }
1608
1609 // Return ObjectMonitors on a moribund thread's free and in-use
1610 // lists to the appropriate global lists. The ObjectMonitors on the
1611 // per-thread in-use list may still be in use by other threads.
1612 //
1613 // We currently call om_flush() from Threads::remove() before the
1614 // thread has been excised from the thread list and is no longer a
1615 // mutator. In particular, this ensures that the thread's in-use
1616 // monitors are scanned by a GC safepoint, either via Thread::oops_do()
1617 // (before om_flush() is called) or via ObjectSynchronizer::oops_do()
1618 // (after om_flush() is called).
1619 //
1620 // deflate_global_idle_monitors_using_JT() and
1621 // deflate_per_thread_idle_monitors_using_JT() (in another thread) can
1622 // run at the same time as om_flush() so we have to follow a careful
1623 // protocol to prevent list corruption.
1624
1625 void ObjectSynchronizer::om_flush(Thread* self) {
1626 // Process the per-thread in-use list first to be consistent.
1627 int in_use_count = 0;
1628 ObjectMonitor* in_use_list = NULL;
1629 ObjectMonitor* in_use_tail = NULL;
1630 NoSafepointVerifier nsv;
1631
1632 // This function can race with a list walker or with an async
1633 // deflater thread so we lock the list head to prevent confusion.
1634 // An async deflater thread checks to see if the target thread
1635 // is exiting, but if it has made it past that check before we
1636 // started exiting, then it is racing to get to the in-use list.
1637 if ((in_use_list = get_list_head_locked(&self->om_in_use_list)) != NULL) {
1638 // At this point, we have locked the in-use list head so a racing
1639 // thread cannot come in after us. However, a racing thread could
1640 // be ahead of us; we'll detect that and delay to let it finish.
1641 //
1655 while (is_locked(cur_om)) {
1656 os::naked_short_sleep(1);
1657 }
1658 // Refetch the possibly changed next field and try again.
1659 cur_om = unmarked_next(in_use_tail);
1660 continue;
1661 }
1662 if (cur_om->object() == NULL) {
1663 // cur_om was deflated and the object ref was cleared while it
1664 // was locked. We happened to see it just after it was unlocked
1665 // (and added to the free list). Refetch the possibly changed
1666 // next field and try again.
1667 cur_om = unmarked_next(in_use_tail);
1668 continue;
1669 }
1670 in_use_tail = cur_om;
1671 in_use_count++;
1672 cur_om = unmarked_next(cur_om);
1673 }
1674 guarantee(in_use_tail != NULL, "invariant");
1675 #ifdef ASSERT
1676 int l_om_in_use_count = Atomic::load(&self->om_in_use_count);
1677 assert(l_om_in_use_count == in_use_count, "in-use counts don't match: "
1678 "l_om_in_use_count=%d, in_use_count=%d", l_om_in_use_count, in_use_count);
1679 #endif
1680 Atomic::store(&self->om_in_use_count, 0);
1681 // Clear the in-use list head (which also unlocks it):
1682 Atomic::store(&self->om_in_use_list, (ObjectMonitor*)NULL);
1683 om_unlock(in_use_list);
1684 }
1685
1686 int free_count = 0;
1687 ObjectMonitor* free_list = NULL;
1688 ObjectMonitor* free_tail = NULL;
1689 // This function can race with a list walker thread so we lock the
1690 // list head to prevent confusion.
1691 if ((free_list = get_list_head_locked(&self->om_free_list)) != NULL) {
1692 // At this point, we have locked the free list head so a racing
1693 // thread cannot come in after us. However, a racing thread could
1694 // be ahead of us; we'll detect that and delay to let it finish.
1695 //
1696 // The thread is going away. Set 'free_tail' to the last per-thread free
1697 // monitor which will be linked to om_list_globals._free_list below.
1698 //
1699 // Account for the free list head before the loop since it is
1700 // already locked (by this thread):
1701 free_tail = free_list;
1702 free_count++;
1703 for (ObjectMonitor* s = unmarked_next(free_list); s != NULL; s = unmarked_next(s)) {
1704 if (is_locked(s)) {
1705 // s is locked so there must be a racing walker thread ahead
1706 // of us so we'll give it a chance to finish.
1707 while (is_locked(s)) {
1708 os::naked_short_sleep(1);
1709 }
1710 }
1711 free_tail = s;
1712 free_count++;
1713 guarantee(s->object() == NULL, "invariant");
1714 if (s->is_busy()) {
1715 stringStream ss;
1716 fatal("must be !is_busy: %s", s->is_busy_to_string(&ss));
1717 }
1718 }
1719 guarantee(free_tail != NULL, "invariant");
1720 #ifdef ASSERT
1721 int l_om_free_count = Atomic::load(&self->om_free_count);
1722 assert(l_om_free_count == free_count, "free counts don't match: "
1723 "l_om_free_count=%d, free_count=%d", l_om_free_count, free_count);
1724 #endif
1725 Atomic::store(&self->om_free_count, 0);
1726 Atomic::store(&self->om_free_list, (ObjectMonitor*)NULL);
1727 om_unlock(free_list);
1728 }
1729
1730 if (free_tail != NULL) {
1731 prepend_list_to_global_free_list(free_list, free_tail, free_count);
1732 }
1733
1734 if (in_use_tail != NULL) {
1735 prepend_list_to_global_in_use_list(in_use_list, in_use_tail, in_use_count);
1736 }
1737
1738 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1739 LogStreamHandle(Info, monitorinflation) lsh_info;
1740 LogStream* ls = NULL;
1741 if (log_is_enabled(Debug, monitorinflation)) {
1742 ls = &lsh_debug;
1743 } else if ((free_count != 0 || in_use_count != 0) &&
1744 log_is_enabled(Info, monitorinflation)) {
1783 !SafepointSynchronize::is_at_safepoint(), "invariant");
1784
1785 EventJavaMonitorInflate event;
1786
1787 for (;;) {
1788 const markWord mark = object->mark();
1789 assert(!mark.has_bias_pattern(), "invariant");
1790
1791 // The mark can be in one of the following states:
1792 // * Inflated - just return
1793 // * Stack-locked - coerce it to inflated
1794 // * INFLATING - busy wait for conversion to complete
1795 // * Neutral - aggressively inflate the object.
1796 // * BIASED - Illegal. We should never see this
1797
1798 // CASE: inflated
1799 if (mark.has_monitor()) {
1800 ObjectMonitor* inf = mark.monitor();
1801 markWord dmw = inf->header();
1802 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1803 assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
1804 return inf;
1805 }
1806
1807 // CASE: inflation in progress - inflating over a stack-lock.
1808 // Some other thread is converting from stack-locked to inflated.
1809 // Only that thread can complete inflation -- other threads must wait.
1810 // The INFLATING value is transient.
1811 // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1812 // We could always eliminate polling by parking the thread on some auxiliary list.
1813 if (mark == markWord::INFLATING()) {
1814 read_stable_mark(object);
1815 continue;
1816 }
1817
1818 // CASE: stack-locked
1819 // Could be stack-locked either by this thread or by some other thread.
1820 //
1821 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1822 // to install INFLATING into the mark word. We originally installed INFLATING,
1868 // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1869 // the while preserving the hashCode stability invariants. If the owner
1870 // decides to release the lock while the value is 0, the unlock will fail
1871 // and control will eventually pass from slow_exit() to inflate. The owner
1872 // will then spin, waiting for the 0 value to disappear. Put another way,
1873 // the 0 causes the owner to stall if the owner happens to try to
1874 // drop the lock (restoring the header from the BasicLock to the object)
1875 // while inflation is in-progress. This protocol avoids races that might
1876 // would otherwise permit hashCode values to change or "flicker" for an object.
1877 // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1878 // 0 serves as a "BUSY" inflate-in-progress indicator.
1879
1880
1881 // fetch the displaced mark from the owner's stack.
1882 // The owner can't die or unwind past the lock while our INFLATING
1883 // object is in the mark. Furthermore the owner can't complete
1884 // an unlock on the object, either.
1885 markWord dmw = mark.displaced_mark_helper();
1886 // Catch if the object's header is not neutral (not locked and
1887 // not marked is what we care about here).
1888 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1889
1890 // Setup monitor fields to proper values -- prepare the monitor
1891 m->set_header(dmw);
1892
1893 // Optimization: if the mark.locker stack address is associated
1894 // with this thread we could simply set m->_owner = self.
1895 // Note that a thread can inflate an object
1896 // that it has stack-locked -- as might happen in wait() -- directly
1897 // with CAS. That is, we can avoid the xchg-NULL .... ST idiom.
1898 m->set_owner_from(NULL, DEFLATER_MARKER, mark.locker());
1899 m->set_object(object);
1900 // TODO-FIXME: assert BasicLock->dhw != 0.
1901
1902 // Must preserve store ordering. The monitor state must
1903 // be stable at the time of publishing the monitor address.
1904 guarantee(object->mark() == markWord::INFLATING(), "invariant");
1905 object->release_set_mark(markWord::encode(m));
1906
1907 // Once ObjectMonitor is configured and the object is associated
1908 // with the ObjectMonitor, it is safe to allow async deflation:
1909 assert(m->is_new(), "freshly allocated monitor must be new");
1910 m->set_allocation_state(ObjectMonitor::Old);
1911
1912 // Hopefully the performance counters are allocated on distinct cache lines
1913 // to avoid false sharing on MP systems ...
1914 OM_PERFDATA_OP(Inflations, inc());
1915 if (log_is_enabled(Trace, monitorinflation)) {
1916 ResourceMark rm(self);
1917 lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
1918 INTPTR_FORMAT ", type='%s'", p2i(object),
1919 object->mark().value(), object->klass()->external_name());
1920 }
1921 if (event.should_commit()) {
1922 post_monitor_inflate_event(&event, object, cause);
1923 }
1924 return m;
1925 }
1926
1927 // CASE: neutral
1928 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
1929 // If we know we're inflating for entry it's better to inflate by swinging a
1930 // pre-locked ObjectMonitor pointer into the object header. A successful
1931 // CAS inflates the object *and* confers ownership to the inflating thread.
1932 // In the current implementation we use a 2-step mechanism where we CAS()
1933 // to inflate and then CAS() again to try to swing _owner from NULL to self.
1934 // An inflateTry() method that we could call from enter() would be useful.
1935
1936 // Catch if the object's header is not neutral (not locked and
1937 // not marked is what we care about here).
1938 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1939 ObjectMonitor* m = om_alloc(self);
1940 // prepare m for installation - set monitor to initial state
1941 m->Recycle();
1942 m->set_header(mark);
1943 // DEFLATER_MARKER is the only non-NULL value we should see here.
1944 m->try_set_owner_from(DEFLATER_MARKER, NULL);
1945 m->set_object(object);
1946 m->_Responsible = NULL;
1947 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
1948
1949 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
1950 m->set_header(markWord::zero());
1951 m->set_object(NULL);
1952 m->Recycle();
1953 // om_release() will reset the allocation state from New to Free.
1954 om_release(self, m, true);
1955 m = NULL;
1956 continue;
1957 // interference - the markword changed - just retry.
1958 // The state-transitions are one-way, so there's no chance of
1959 // live-lock -- "Inflated" is an absorbing state.
1960 }
1961
1962 // Once the ObjectMonitor is configured and object is associated
1963 // with the ObjectMonitor, it is safe to allow async deflation:
1964 assert(m->is_new(), "freshly allocated monitor must be new");
1965 m->set_allocation_state(ObjectMonitor::Old);
1966
1967 // Hopefully the performance counters are allocated on distinct
1968 // cache lines to avoid false sharing on MP systems ...
1969 OM_PERFDATA_OP(Inflations, inc());
1970 if (log_is_enabled(Trace, monitorinflation)) {
1971 ResourceMark rm(self);
1972 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
1973 INTPTR_FORMAT ", type='%s'", p2i(object),
1974 object->mark().value(), object->klass()->external_name());
1975 }
1976 if (event.should_commit()) {
1977 post_monitor_inflate_event(&event, object, cause);
1978 }
1979 return m;
1980 }
1981 }
1982
1983
1984 // An async deflation request is registered with the ServiceThread
1985 // and it is notified.
1986 void ObjectSynchronizer::do_safepoint_work() {
1987 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1988
1989 log_debug(monitorinflation)("requesting async deflation of idle monitors.");
1990 // Request deflation of idle monitors by the ServiceThread:
1991 set_is_async_deflation_requested(true);
1992 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
1993 ml.notify_all();
1994
1995 if (log_is_enabled(Debug, monitorinflation)) {
1996 // exit_globals()'s call to audit_and_print_stats() is done
1997 // at the Info level and not at a safepoint.
1998 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
1999 }
2000 }
2001
2002 // Deflate the specified ObjectMonitor if not in-use using a JavaThread.
2003 // Returns true if it was deflated and false otherwise.
2004 //
2005 // The async deflation protocol sets owner to DEFLATER_MARKER and
2006 // makes contentions negative as signals to contending threads that
2007 // an async deflation is in progress. There are a number of checks
2008 // as part of the protocol to make sure that the calling thread has
2009 // not lost the race to a contending thread.
2010 //
2011 // The ObjectMonitor has been successfully async deflated when:
2012 // (contentions < 0)
2013 // Contending threads that see that condition know to retry their operation.
2014 //
2015 bool ObjectSynchronizer::deflate_monitor_using_JT(ObjectMonitor* mid,
2016 ObjectMonitor** free_head_p,
2017 ObjectMonitor** free_tail_p) {
2018 assert(Thread::current()->is_Java_thread(), "precondition");
2019 // A newly allocated ObjectMonitor should not be seen here so we
2020 // avoid an endless inflate/deflate cycle.
2021 assert(mid->is_old(), "must be old: allocation_state=%d",
2022 (int) mid->allocation_state());
2023
2024 if (mid->is_busy()) {
2025 // Easy checks are first - the ObjectMonitor is busy so no deflation.
2026 return false;
2027 }
2028
2029 // Set a NULL owner to DEFLATER_MARKER to force any contending thread
2030 // through the slow path. This is just the first part of the async
2031 // deflation dance.
2032 if (mid->try_set_owner_from(NULL, DEFLATER_MARKER) != NULL) {
2033 // The owner field is no longer NULL so we lost the race since the
2034 // ObjectMonitor is now busy.
2035 return false;
2036 }
2037
2086 mid->clear_common();
2087
2088 assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT,
2089 p2i(mid->object()));
2090 assert(mid->is_free(), "must be free: allocation_state=%d",
2091 (int)mid->allocation_state());
2092
2093 // Move the deflated ObjectMonitor to the working free list
2094 // defined by free_head_p and free_tail_p.
2095 if (*free_head_p == NULL) {
2096 // First one on the list.
2097 *free_head_p = mid;
2098 }
2099 if (*free_tail_p != NULL) {
2100 // We append to the list so the caller can use mid->_next_om
2101 // to fix the linkages in its context.
2102 ObjectMonitor* prevtail = *free_tail_p;
2103 // prevtail should have been cleaned up by the caller:
2104 #ifdef ASSERT
2105 ObjectMonitor* l_next_om = unmarked_next(prevtail);
2106 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2107 #endif
2108 om_lock(prevtail);
2109 prevtail->set_next_om(mid); // prevtail now points to mid (and is unlocked)
2110 }
2111 *free_tail_p = mid;
2112
2113 // At this point, mid->_next_om still refers to its current
2114 // value and another ObjectMonitor's _next_om field still
2115 // refers to this ObjectMonitor. Those linkages have to be
2116 // cleaned up by the caller who has the complete context.
2117
2118 // We leave owner == DEFLATER_MARKER and contentions < 0
2119 // to force any racing threads to retry.
2120 return true; // Success, ObjectMonitor has been deflated.
2121 }
2122
2123 // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using
2124 // a JavaThread. Returns the number of deflated ObjectMonitors. The given
2125 // list could be a per-thread in-use list or the global in-use list.
2126 // If a safepoint has started, then we save state via saved_mid_in_use_p
2127 // and return to the caller to honor the safepoint.
2128 //
2129 int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor** list_p,
2130 int* count_p,
2131 ObjectMonitor** free_head_p,
2132 ObjectMonitor** free_tail_p,
2133 ObjectMonitor** saved_mid_in_use_p) {
2134 JavaThread* self = JavaThread::current();
2135
2136 ObjectMonitor* cur_mid_in_use = NULL;
2137 ObjectMonitor* mid = NULL;
2138 ObjectMonitor* next = NULL;
2139 ObjectMonitor* next_next = NULL;
2140 int deflated_count = 0;
2141 NoSafepointVerifier nsv;
2142
2143 // We use the more complicated lock-cur_mid_in_use-and-mid-as-we-go
2144 // protocol because om_release() can do list deletions in parallel;
2145 // this also prevents races with a list walker thread. We also
2146 // lock-next-next-as-we-go to prevent an om_flush() that is behind
2147 // this thread from passing us.
2148 if (*saved_mid_in_use_p == NULL) {
2149 // No saved state so start at the beginning.
2150 // Lock the list head so we can possibly deflate it:
2151 if ((mid = get_list_head_locked(list_p)) == NULL) {
2152 return 0; // The list is empty so nothing to deflate.
2153 }
2243 }
2244 return deflated_count;
2245 }
2246 }
2247 if (mid == NULL) {
2248 if (cur_mid_in_use != NULL) {
2249 om_unlock(cur_mid_in_use);
2250 }
2251 break; // Reached end of the list so nothing more to deflate.
2252 }
2253
2254 // The current mid's next field is locked at this point. If we have
2255 // a cur_mid_in_use, then it is also locked at this point.
2256 }
2257 // We finished the list without a safepoint starting so there's
2258 // no need to save state.
2259 *saved_mid_in_use_p = NULL;
2260 return deflated_count;
2261 }
2262
2263 class HandshakeForDeflation : public HandshakeClosure {
2264 public:
2265 HandshakeForDeflation() : HandshakeClosure("HandshakeForDeflation") {}
2266
2267 void do_thread(Thread* thread) {
2268 log_trace(monitorinflation)("HandshakeForDeflation::do_thread: thread="
2269 INTPTR_FORMAT, p2i(thread));
2270 }
2271 };
2272
2273 void ObjectSynchronizer::deflate_idle_monitors_using_JT() {
2274 // Deflate any global idle monitors.
2275 deflate_global_idle_monitors_using_JT();
2276
2277 int count = 0;
2278 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2279 if (Atomic::load(&jt->om_in_use_count) > 0 && !jt->is_exiting()) {
2280 // This JavaThread is using ObjectMonitors so deflate any that
2281 // are idle unless this JavaThread is exiting; do not race with
2282 // ObjectSynchronizer::om_flush().
2283 deflate_per_thread_idle_monitors_using_JT(jt);
2284 count++;
2285 }
2286 }
2287 if (count > 0) {
2288 log_debug(monitorinflation)("did async deflation of idle monitors for %d thread(s).", count);
2289 }
2290
2291 log_info(monitorinflation)("async global_population=%d, global_in_use_count=%d, "
2292 "global_free_count=%d, global_wait_count=%d",
2293 Atomic::load(&om_list_globals._population),
2294 Atomic::load(&om_list_globals._in_use_count),
2295 Atomic::load(&om_list_globals._free_count),
2296 Atomic::load(&om_list_globals._wait_count));
2297
2298 // The ServiceThread's async deflation request has been processed.
2299 _last_async_deflation_time_ns = os::javaTimeNanos();
2300 set_is_async_deflation_requested(false);
2301
2302 if (Atomic::load(&om_list_globals._wait_count) > 0) {
2303 // There are deflated ObjectMonitors waiting for a handshake
2304 // (or a safepoint) for safety.
2305
2306 ObjectMonitor* list = Atomic::load(&om_list_globals._wait_list);
2307 assert(list != NULL, "om_list_globals._wait_list must not be NULL");
2308 int count = Atomic::load(&om_list_globals._wait_count);
2309 Atomic::store(&om_list_globals._wait_count, 0);
2310 Atomic::store(&om_list_globals._wait_list, (ObjectMonitor*)NULL);
2311
2312 // Find the tail for prepend_list_to_common(). No need to mark
2313 // ObjectMonitors for this list walk since only the deflater
2314 // thread manages the wait list.
2315 #ifdef ASSERT
2316 int l_count = 0;
2317 #endif
2318 ObjectMonitor* tail = NULL;
2319 for (ObjectMonitor* n = list; n != NULL; n = unmarked_next(n)) {
2320 tail = n;
2321 #ifdef ASSERT
2322 l_count++;
2323 #endif
2324 }
2325 assert(count == l_count, "count=%d != l_count=%d", count, l_count);
2326
2327 // Will execute a safepoint if !ThreadLocalHandshakes:
2328 HandshakeForDeflation hfd_hc;
2329 Handshake::execute(&hfd_hc);
2330
2331 prepend_list_to_common(list, tail, count, &om_list_globals._free_list,
2332 &om_list_globals._free_count);
2333
2334 log_info(monitorinflation)("moved %d idle monitors from global waiting list to global free list", count);
2335 }
2336 }
2337
2338 // Deflate global idle ObjectMonitors using a JavaThread.
2339 //
2340 void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() {
2341 assert(Thread::current()->is_Java_thread(), "precondition");
2342 JavaThread* self = JavaThread::current();
2343
2344 deflate_common_idle_monitors_using_JT(true /* is_global */, self);
2345 }
2346
2347 // Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread.
2348 //
2349 void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) {
2350 assert(Thread::current()->is_Java_thread(), "precondition");
2351
2352 deflate_common_idle_monitors_using_JT(false /* !is_global */, target);
2353 }
2354
2355 // Deflate global or per-thread idle ObjectMonitors using a JavaThread.
2356 //
2357 void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) {
2358 JavaThread* self = JavaThread::current();
2359
2360 int deflated_count = 0;
2361 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors
2362 ObjectMonitor* free_tail_p = NULL;
2363 ObjectMonitor* saved_mid_in_use_p = NULL;
2364 elapsedTimer timer;
2365
2366 if (log_is_enabled(Info, monitorinflation)) {
2367 timer.start();
2368 }
2369
2385 local_deflated_count =
2386 deflate_monitor_list_using_JT(&target->om_in_use_list,
2387 &target->om_in_use_count, &free_head_p,
2388 &free_tail_p, &saved_mid_in_use_p);
2389 }
2390 deflated_count += local_deflated_count;
2391
2392 if (free_head_p != NULL) {
2393 // Move the deflated ObjectMonitors to the global free list.
2394 guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count);
2395 // Note: The target thread can be doing an om_alloc() that
2396 // is trying to prepend an ObjectMonitor on its in-use list
2397 // at the same time that we have deflated the current in-use
2398 // list head and put it on the local free list. prepend_to_common()
2399 // will detect the race and retry which avoids list corruption,
2400 // but the next field in free_tail_p can flicker to marked
2401 // and then unmarked while prepend_to_common() is sorting it
2402 // all out.
2403 #ifdef ASSERT
2404 ObjectMonitor* l_next_om = unmarked_next(free_tail_p);
2405 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2406 #endif
2407
2408 prepend_list_to_global_wait_list(free_head_p, free_tail_p, local_deflated_count);
2409
2410 OM_PERFDATA_OP(Deflations, inc(local_deflated_count));
2411 }
2412
2413 if (saved_mid_in_use_p != NULL) {
2414 // deflate_monitor_list_using_JT() detected a safepoint starting.
2415 timer.stop();
2416 {
2417 if (is_global) {
2418 log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint.");
2419 } else {
2420 log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target));
2421 }
2422 assert(SafepointMechanism::should_block(self), "sanity check");
2423 ThreadBlockInVM blocker(self);
2424 }
2425 // Prepare for another loop after the safepoint.
2426 free_head_p = NULL;
2432 } while (saved_mid_in_use_p != NULL);
2433 timer.stop();
2434
2435 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2436 LogStreamHandle(Info, monitorinflation) lsh_info;
2437 LogStream* ls = NULL;
2438 if (log_is_enabled(Debug, monitorinflation)) {
2439 ls = &lsh_debug;
2440 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2441 ls = &lsh_info;
2442 }
2443 if (ls != NULL) {
2444 if (is_global) {
2445 ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2446 } else {
2447 ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count);
2448 }
2449 }
2450 }
2451
2452 // Monitor cleanup on JavaThread::exit
2453
2454 // Iterate through monitor cache and attempt to release thread's monitors
2455 // Gives up on a particular monitor if an exception occurs, but continues
2456 // the overall iteration, swallowing the exception.
2457 class ReleaseJavaMonitorsClosure: public MonitorClosure {
2458 private:
2459 TRAPS;
2460
2461 public:
2462 ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {}
2463 void do_monitor(ObjectMonitor* mid) {
2464 if (mid->owner() == THREAD) {
2465 (void)mid->complete_exit(CHECK);
2466 }
2467 }
2468 };
2469
2470 // Release all inflated monitors owned by THREAD. Lightweight monitors are
2471 // ignored. This is meant to be called during JNI thread detach which assumes
2472 // all remaining monitors are heavyweight. All exceptions are swallowed.
2473 // Scanning the extant monitor list can be time consuming.
2474 // A simple optimization is to add a per-thread flag that indicates a thread
2475 // called jni_monitorenter() during its lifetime.
2476 //
2477 // Instead of NoSafepointVerifier it might be cheaper to
2478 // use an idiom of the form:
2479 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
2480 // <code that must not run at safepoint>
2481 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
2482 // Since the tests are extremely cheap we could leave them enabled
2483 // for normal product builds.
2484
2485 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
2486 assert(THREAD == JavaThread::current(), "must be current Java thread");
2487 NoSafepointVerifier nsv;
2488 ReleaseJavaMonitorsClosure rjmc(THREAD);
2489 ObjectSynchronizer::monitors_iterate(&rjmc);
2490 THREAD->clear_pending_exception();
2491 }
2492
2493 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
2494 switch (cause) {
2495 case inflate_cause_vm_internal: return "VM Internal";
2496 case inflate_cause_monitor_enter: return "Monitor Enter";
2497 case inflate_cause_wait: return "Monitor Wait";
2516 return (u_char*)&GVars.hc_sequence;
2517 }
2518
2519 size_t ObjectSynchronizer::get_gvars_size() {
2520 return sizeof(SharedGlobals);
2521 }
2522
2523 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
2524 return (u_char*)&GVars.stw_random;
2525 }
2526
2527 // This function can be called at a safepoint or it can be called when
2528 // we are trying to exit the VM. When we are trying to exit the VM, the
2529 // list walker functions can run in parallel with the other list
2530 // operations so spin-locking is used for safety.
2531 //
2532 // Calls to this function can be added in various places as a debugging
2533 // aid; pass 'true' for the 'on_exit' parameter to have in-use monitor
2534 // details logged at the Info level and 'false' for the 'on_exit'
2535 // parameter to have in-use monitor details logged at the Trace level.
2536 //
2537 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
2538 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
2539
2540 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2541 LogStreamHandle(Info, monitorinflation) lsh_info;
2542 LogStreamHandle(Trace, monitorinflation) lsh_trace;
2543 LogStream* ls = NULL;
2544 if (log_is_enabled(Trace, monitorinflation)) {
2545 ls = &lsh_trace;
2546 } else if (log_is_enabled(Debug, monitorinflation)) {
2547 ls = &lsh_debug;
2548 } else if (log_is_enabled(Info, monitorinflation)) {
2549 ls = &lsh_info;
2550 }
2551 assert(ls != NULL, "sanity check");
2552
2553 // Log counts for the global and per-thread monitor lists:
2554 int chk_om_population = log_monitor_list_counts(ls);
2555 int error_cnt = 0;
2614 outputStream * out, int *error_cnt_p) {
2615 stringStream ss;
2616 if (n->is_busy()) {
2617 if (jt != NULL) {
2618 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2619 ": free per-thread monitor must not be busy: %s", p2i(jt),
2620 p2i(n), n->is_busy_to_string(&ss));
2621 } else {
2622 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2623 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
2624 }
2625 *error_cnt_p = *error_cnt_p + 1;
2626 }
2627 if (n->header().value() != 0) {
2628 if (jt != NULL) {
2629 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2630 ": free per-thread monitor must have NULL _header "
2631 "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n),
2632 n->header().value());
2633 *error_cnt_p = *error_cnt_p + 1;
2634 }
2635 }
2636 if (n->object() != NULL) {
2637 if (jt != NULL) {
2638 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2639 ": free per-thread monitor must have NULL _object "
2640 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
2641 p2i(n->object()));
2642 } else {
2643 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2644 "must have NULL _object field: _object=" INTPTR_FORMAT,
2645 p2i(n), p2i(n->object()));
2646 }
2647 *error_cnt_p = *error_cnt_p + 1;
2648 }
2649 }
2650
2651 // Lock the next ObjectMonitor for traversal and unlock the current
2652 // ObjectMonitor. Returns the next ObjectMonitor if there is one.
2653 // Otherwise returns NULL (after unlocking the current ObjectMonitor).
|