101 }
102
103 #else // ndef DTRACE_ENABLED
104
105 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon) {;}
106 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon) {;}
107
108 #endif // ndef DTRACE_ENABLED
109
110 // This exists only as a workaround of dtrace bug 6254741
111 int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
112 DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
113 return 0;
114 }
115
116 #define NINFLATIONLOCKS 256
117 static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];
118
119 // global list of blocks of monitors
120 PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL;
121 // Global ObjectMonitor free list. Newly allocated and deflated
122 // ObjectMonitors are prepended here.
123 ObjectMonitor* volatile ObjectSynchronizer::g_free_list = NULL;
124 // Global ObjectMonitor in-use list. When a JavaThread is exiting,
125 // ObjectMonitors on its per-thread in-use list are prepended here.
126 ObjectMonitor* volatile ObjectSynchronizer::g_om_in_use_list = NULL;
127 int ObjectSynchronizer::g_om_in_use_count = 0; // # on g_om_in_use_list
128 bool volatile ObjectSynchronizer::_is_async_deflation_requested = false;
129 bool volatile ObjectSynchronizer::_is_special_deflation_requested = false;
130 jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0;
131
132 static volatile intptr_t gListLock = 0; // protects global monitor lists
133 static volatile int g_om_free_count = 0; // # on g_free_list
134 static volatile int g_om_population = 0; // # Extant -- in circulation
135
136 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
137
138
139 // =====================> Quick functions
140
141 // The quick_* forms are special fast-path variants used to improve
142 // performance. In the simplest case, a "quick_*" implementation could
143 // simply return false, in which case the caller will perform the necessary
144 // state transitions and call the slow-path form.
145 // The fast-path is designed to handle frequently arising cases in an efficient
146 // manner and is just a degenerate "optimistic" variant of the slow-path.
147 // returns true -- to indicate the call was satisfied.
148 // returns false -- to indicate the call needs the services of the slow-path.
149 // A no-loitering ordinance is in effect for code in the quick_* family
150 // operators: safepoints or indefinite blocking (blocking that might span a
151 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
152 // entry.
153 //
154 // Consider: An interesting optimization is to have the JIT recognize the
155 // following common idiom:
156 // synchronized (someobj) { .... ; notify(); }
157 // That is, we find a notify() or notifyAll() call that immediately precedes
158 // the monitorexit operation. In that case the JIT could fuse the operations
565 //
566 // Performance concern:
567 // OrderAccess::storestore() calls release() which at one time stored 0
568 // into the global volatile OrderAccess::dummy variable. This store was
569 // unnecessary for correctness. Many threads storing into a common location
570 // causes considerable cache migration or "sloshing" on large SMP systems.
571 // As such, I avoided using OrderAccess::storestore(). In some cases
572 // OrderAccess::fence() -- which incurs local latency on the executing
573 // processor -- is a better choice as it scales on SMP systems.
574 //
575 // See http://blogs.oracle.com/dave/entry/biased_locking_in_hotspot for
576 // a discussion of coherency costs. Note that all our current reference
577 // platforms provide strong ST-ST order, so the issue is moot on IA32,
578 // x64, and SPARC.
579 //
580 // As a general policy we use "volatile" to control compiler-based reordering
581 // and explicit fences (barriers) to control for architectural reordering
582 // performed by the CPU(s) or platform.
583
584 struct SharedGlobals {
585 char _pad_prefix[DEFAULT_CACHE_LINE_SIZE];
586 // These are highly shared mostly-read variables.
587 // To avoid false-sharing they need to be the sole occupants of a cache line.
588 volatile int stw_random;
589 volatile int stw_cycle;
590 DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int) * 2);
591 // Hot RW variable -- Sequester to avoid false-sharing
592 volatile int hc_sequence;
593 DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int));
594 };
595
596 static SharedGlobals GVars;
597 static int MonitorScavengeThreshold = 1000000;
598 static volatile int ForceMonitorScavenge = 0; // Scavenge required and pending
599
600 static markWord read_stable_mark(oop obj) {
601 markWord mark = obj->mark();
602 if (!mark.is_being_inflated()) {
603 return mark; // normal fast-path return
604 }
605
606 int its = 0;
607 for (;;) {
608 markWord mark = obj->mark();
609 if (!mark.is_being_inflated()) {
610 return mark; // normal fast-path return
611 }
612
613 // The object is being inflated by some other thread.
981 // Cannot have assertion since this object may have been
982 // locked by another thread when reaching here.
983 // assert(mark.is_neutral(), "sanity check");
984
985 return NULL;
986 }
987 }
988
989 // Visitors ...
990
991 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
992 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
993 while (block != NULL) {
994 assert(block->object() == CHAINMARKER, "must be a block header");
995 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
996 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
997 if (mid->is_active()) {
998 ObjectMonitorHandle omh(mid);
999
1000 if (mid->object() == NULL ||
1001 (AsyncDeflateIdleMonitors && mid->_owner == DEFLATER_MARKER)) {
1002 // Only process with closure if the object is set.
1003 // For async deflation, race here if monitor is not owned!
1004 // The above ref_count bump (in ObjectMonitorHandle ctr)
1005 // will cause subsequent async deflation to skip it.
1006 // However, previous or concurrent async deflation is a race.
1007 continue;
1008 }
1009 closure->do_monitor(mid);
1010 }
1011 }
1012 block = (PaddedObjectMonitor*)block->_next_om;
1013 }
1014 }
1015
1016 static bool monitors_used_above_threshold() {
1017 if (g_om_population == 0) {
1018 return false;
1019 }
1020 if (MonitorUsedDeflationThreshold > 0) {
1021 int monitors_used = g_om_population - g_om_free_count;
1022 int monitor_usage = (monitors_used * 100LL) / g_om_population;
1023 return monitor_usage > MonitorUsedDeflationThreshold;
1024 }
1025 return false;
1026 }
1027
1028 // Returns true if MonitorBound is set (> 0) and if the specified
1029 // cnt is > MonitorBound. Otherwise returns false.
1030 static bool is_MonitorBound_exceeded(const int cnt) {
1031 const int mx = MonitorBound;
1032 return mx > 0 && cnt > mx;
1033 }
1034
1035 bool ObjectSynchronizer::is_async_deflation_needed() {
1036 if (!AsyncDeflateIdleMonitors) {
1037 return false;
1038 }
1039 if (is_async_deflation_requested()) {
1040 // Async deflation request.
1041 return true;
1042 }
1043 if (AsyncDeflationInterval > 0 &&
1044 time_since_last_async_deflation_ms() > AsyncDeflationInterval &&
1045 monitors_used_above_threshold()) {
1046 // It's been longer than our specified deflate interval and there
1047 // are too many monitors in use. We don't deflate more frequently
1048 // than AsyncDeflationInterval (unless is_async_deflation_requested)
1049 // in order to not swamp the ServiceThread.
1050 _last_async_deflation_time_ns = os::javaTimeNanos();
1051 return true;
1052 }
1053 if (is_MonitorBound_exceeded(g_om_population - g_om_free_count)) {
1054 // Not enough ObjectMonitors on the global free list.
1055 return true;
1056 }
1057 return false;
1058 }
1059
1060 bool ObjectSynchronizer::is_safepoint_deflation_needed() {
1061 if (!AsyncDeflateIdleMonitors) {
1062 if (monitors_used_above_threshold()) {
1063 // Too many monitors in use.
1064 return true;
1065 }
1066 return false;
1067 }
1068 if (is_special_deflation_requested()) {
1069 // For AsyncDeflateIdleMonitors only do a safepoint deflation
1070 // if there is a special deflation request.
1071 return true;
1072 }
1073 return false;
1074 }
1075
1076 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1077 return (os::javaTimeNanos() - _last_async_deflation_time_ns) / (NANOUNITS / MILLIUNITS);
1078 }
1079
1080 void ObjectSynchronizer::oops_do(OopClosure* f) {
1081 // We only scan the global used list here (for moribund threads), and
1082 // the thread-local monitors in Thread::oops_do().
1083 global_used_oops_do(f);
1084 }
1085
1086 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
1087 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1088 list_oops_do(g_om_in_use_list, f);
1089 }
1090
1091 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
1092 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1093 list_oops_do(thread->om_in_use_list, f);
1094 }
1095
1096 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, OopClosure* f) {
1097 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1098 ObjectMonitor* mid;
1099 for (mid = list; mid != NULL; mid = mid->_next_om) {
1100 if (mid->object() != NULL) {
1101 f->do_oop((oop*)mid->object_addr());
1102 }
1103 }
1104 }
1105
1106
1107 // -----------------------------------------------------------------------------
1108 // ObjectMonitor Lifecycle
1109 // -----------------------
1110 // Inflation unlinks monitors from the global g_free_list and
1111 // associates them with objects. Deflation -- which occurs at
1112 // STW-time -- disassociates idle monitors from objects. Such
1113 // scavenged monitors are returned to the g_free_list.
1114 //
1115 // The global list is protected by gListLock. All the critical sections
1116 // are short and operate in constant-time.
1117 //
1118 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
1119 //
1120 // Lifecycle:
1121 // -- unassigned and on the global free list
1122 // -- unassigned and on a thread's private om_free_list
1123 // -- assigned to an object. The object is inflated and the mark refers
1124 // to the objectmonitor.
1125
1126
1127 // Constraining monitor pool growth via MonitorBound ...
1128 //
1129 // If MonitorBound is not set (<= 0), MonitorBound checks are disabled.
1130 //
1131 // When safepoint deflation is being used (!AsyncDeflateIdleMonitors):
1132 // The monitor pool is grow-only. We scavenge at STW safepoint-time, but the
1133 // the rate of scavenging is driven primarily by GC. As such, we can find
1134 // an inordinate number of monitors in circulation.
1135 // To avoid that scenario we can artificially induce a STW safepoint
1136 // if the pool appears to be growing past some reasonable bound.
1137 // Generally we favor time in space-time tradeoffs, but as there's no
1140 // we could just loop. In addition, if MonitorBound is set to a low value
1141 // we'll incur more safepoints, which are harmful to performance.
1142 // See also: GuaranteedSafepointInterval
1143 //
1144 // The current implementation uses asynchronous VM operations.
1145 //
1146 // When safepoint deflation is being used and MonitorBound is set, the
1147 // boundry applies to
1148 // (g_om_population - g_om_free_count)
1149 // i.e., if there are not enough ObjectMonitors on the global free list,
1150 // then a safepoint deflation is induced. Picking a good MonitorBound value
1151 // is non-trivial.
1152 //
1153 // When async deflation is being used:
1154 // The monitor pool is still grow-only. Async deflation is requested
1155 // by a safepoint's cleanup phase or by the ServiceThread at periodic
1156 // intervals when is_async_deflation_needed() returns true. In
1157 // addition to other policies that are checked, if there are not
1158 // enough ObjectMonitors on the global free list, then
1159 // is_async_deflation_needed() will return true. The ServiceThread
1160 // calls deflate_global_idle_monitors_using_JT() and also sets the
1161 // per-thread om_request_deflation flag as needed.
1162
1163 static void InduceScavenge(Thread* self, const char * Whence) {
1164 assert(!AsyncDeflateIdleMonitors, "is not used by async deflation");
1165
1166 // Induce STW safepoint to trim monitors
1167 // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
1168 // More precisely, trigger an asynchronous STW safepoint as the number
1169 // of active monitors passes the specified threshold.
1170 // TODO: assert thread state is reasonable
1171
1172 if (ForceMonitorScavenge == 0 && Atomic::xchg (1, &ForceMonitorScavenge) == 0) {
1173 // Induce a 'null' safepoint to scavenge monitors
1174 // Must VM_Operation instance be heap allocated as the op will be enqueue and posted
1175 // to the VMthread and have a lifespan longer than that of this activation record.
1176 // The VMThread will delete the op when completed.
1177 VMThread::execute(new VM_ScavengeMonitors());
1178 }
1179 }
1180
1181 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self,
1182 const InflateCause cause) {
1183 // A large MAXPRIVATE value reduces both list lock contention
1184 // and list coherency traffic, but also tends to increase the
1185 // number of ObjectMonitors in circulation as well as the STW
1186 // scavenge costs. As usual, we lean toward time in space-time
1187 // tradeoffs.
1188 const int MAXPRIVATE = 1024;
1189
1190 if (AsyncDeflateIdleMonitors) {
1191 JavaThread* jt = (JavaThread *)self;
1192 if (jt->om_request_deflation && jt->om_in_use_count > 0 &&
1193 cause != inflate_cause_vm_internal) {
1194 // Deflate any per-thread idle monitors for this JavaThread if
1195 // this is not an internal inflation; internal inflations can
1196 // occur in places where it is not safe to pause for a safepoint.
1197 // Clean up your own mess (Gibbs Rule 45). Otherwise, skip this
1198 // deflation. deflate_global_idle_monitors_using_JT() is called
1199 // by the ServiceThread. Per-thread async deflation is triggered
1200 // by the ServiceThread via om_request_deflation.
1201 debug_only(jt->check_for_valid_safepoint_state(false);)
1202 ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT();
1203 }
1204 }
1205
1206 stringStream ss;
1207 for (;;) {
1208 ObjectMonitor* m;
1209
1210 // 1: try to allocate from the thread's local om_free_list.
1211 // Threads will attempt to allocate first from their local list, then
1212 // from the global list, and only after those attempts fail will the thread
1213 // attempt to instantiate new monitors. Thread-local free lists take
1214 // heat off the gListLock and improve allocation latency, as well as reducing
1215 // coherency traffic on the shared global list.
1216 m = self->om_free_list;
1217 if (m != NULL) {
1218 self->om_free_list = m->_next_om;
1219 self->om_free_count--;
1220 guarantee(m->object() == NULL, "invariant");
1221 m->set_allocation_state(ObjectMonitor::New);
1222 m->_next_om = self->om_in_use_list;
1223 self->om_in_use_list = m;
1224 self->om_in_use_count++;
1225 return m;
1226 }
1227
1228 // 2: try to allocate from the global g_free_list
1229 // CONSIDER: use muxTry() instead of muxAcquire().
1230 // If the muxTry() fails then drop immediately into case 3.
1231 // If we're using thread-local free lists then try
1232 // to reprovision the caller's free list.
1233 if (g_free_list != NULL) {
1234 // Reprovision the thread's om_free_list.
1235 // Use bulk transfers to reduce the allocation rate and heat
1236 // on various locks.
1237 Thread::muxAcquire(&gListLock, "om_alloc(1)");
1238 for (int i = self->om_free_provision; --i >= 0 && g_free_list != NULL;) {
1239 g_om_free_count--;
1240 ObjectMonitor* take = g_free_list;
1241 g_free_list = take->_next_om;
1242 guarantee(take->object() == NULL, "invariant");
1243 if (AsyncDeflateIdleMonitors) {
1244 // We allowed 3 field values to linger during async deflation.
1245 // We clear header and restore ref_count here, but we leave
1246 // owner == DEFLATER_MARKER so the simple C2 ObjectMonitor
1247 // enter optimization can no longer race with async deflation
1248 // and reuse.
1249 take->set_header(markWord::zero());
1250 if (take->ref_count() < 0) {
1251 // Add back max_jint to restore the ref_count field to its
1252 // proper value.
1253 Atomic::add(max_jint, &take->_ref_count);
1254
1255 assert(take->ref_count() >= 0, "must not be negative: ref_count=%d",
1256 take->ref_count());
1257 }
1258 }
1259 take->Recycle();
1260 assert(take->is_free(), "invariant");
1261 om_release(self, take, false);
1262 }
1263 Thread::muxRelease(&gListLock);
1264 self->om_free_provision += 1 + (self->om_free_provision/2);
1265 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1266
1267 if (!AsyncDeflateIdleMonitors &&
1268 is_MonitorBound_exceeded(g_om_population - g_om_free_count)) {
1269 // Not enough ObjectMonitors on the global free list.
1270 // We can't safely induce a STW safepoint from om_alloc() as our thread
1271 // state may not be appropriate for such activities and callers may hold
1272 // naked oops, so instead we defer the action.
1273 InduceScavenge(self, "om_alloc");
1274 }
1275 continue;
1276 }
1277
1278 // 3: allocate a block of new ObjectMonitors
1279 // Both the local and global free lists are empty -- resort to malloc().
1280 // In the current implementation ObjectMonitors are TSM - immortal.
1281 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1282 // each ObjectMonitor to start at the beginning of a cache line,
1283 // so we use align_up().
1284 // A better solution would be to use C++ placement-new.
1285 // BEWARE: As it stands currently, we don't run the ctors!
1286 assert(_BLOCKSIZE > 1, "invariant");
1287 size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE;
1288 PaddedObjectMonitor* temp;
1289 size_t aligned_size = neededsize + (DEFAULT_CACHE_LINE_SIZE - 1);
1290 void* real_malloc_addr = (void*)NEW_C_HEAP_ARRAY(char, aligned_size,
1291 mtInternal);
1292 temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, DEFAULT_CACHE_LINE_SIZE);
1293
1294 // NOTE: (almost) no way to recover if allocation failed.
1295 // We might be able to induce a STW safepoint and scavenge enough
1296 // ObjectMonitors to permit progress.
1297 if (temp == NULL) {
1298 vm_exit_out_of_memory(neededsize, OOM_MALLOC_ERROR,
1299 "Allocate ObjectMonitors");
1300 }
1301 (void)memset((void *) temp, 0, neededsize);
1302
1303 // Format the block.
1304 // initialize the linked list, each monitor points to its next
1305 // forming the single linked free list, the very first monitor
1306 // will points to next block, which forms the block list.
1307 // The trick of using the 1st element in the block as g_block_list
1308 // linkage should be reconsidered. A better implementation would
1309 // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; }
1310
1311 for (int i = 1; i < _BLOCKSIZE; i++) {
1312 temp[i]._next_om = (ObjectMonitor *)&temp[i+1];
1313 assert(temp[i].is_free(), "invariant");
1314 }
1315
1316 // terminate the last monitor as the end of list
1317 temp[_BLOCKSIZE - 1]._next_om = NULL;
1318
1319 // Element [0] is reserved for global list linkage
1320 temp[0].set_object(CHAINMARKER);
1321
1322 // Consider carving out this thread's current request from the
1323 // block in hand. This avoids some lock traffic and redundant
1324 // list activity.
1325
1326 // Acquire the gListLock to manipulate g_block_list and g_free_list.
1327 // An Oyama-Taura-Yonezawa scheme might be more efficient.
1328 Thread::muxAcquire(&gListLock, "om_alloc(2)");
1329 g_om_population += _BLOCKSIZE-1;
1330 g_om_free_count += _BLOCKSIZE-1;
1331
1332 // Add the new block to the list of extant blocks (g_block_list).
1333 // The very first ObjectMonitor in a block is reserved and dedicated.
1334 // It serves as blocklist "next" linkage.
1335 temp[0]._next_om = g_block_list;
1336 // There are lock-free uses of g_block_list so make sure that
1337 // the previous stores happen before we update g_block_list.
1338 OrderAccess::release_store(&g_block_list, temp);
1339
1340 // Add the new string of ObjectMonitors to the global free list
1341 temp[_BLOCKSIZE - 1]._next_om = g_free_list;
1342 g_free_list = temp + 1;
1343 Thread::muxRelease(&gListLock);
1344 }
1345 }
1346
1347 // Place "m" on the caller's private per-thread om_free_list.
1348 // In practice there's no need to clamp or limit the number of
1349 // monitors on a thread's om_free_list as the only non-allocation time
1350 // we'll call om_release() is to return a monitor to the free list after
1351 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1352 // accumulate on a thread's free list.
1353 //
1354 // Key constraint: all ObjectMonitors on a thread's free list and the global
1355 // free list must have their object field set to null. This prevents the
1356 // scavenger -- deflate_monitor_list() or deflate_monitor_list_using_JT()
1357 // -- from reclaiming them while we are trying to release them.
1358
1359 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1360 bool from_per_thread_alloc) {
1361 guarantee(m->header().value() == 0, "invariant");
1362 guarantee(m->object() == NULL, "invariant");
1363 stringStream ss;
1364 guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1365 "%s, recursions=" INTPTR_FORMAT, m->is_busy_to_string(&ss),
1366 m->_recursions);
1367 m->set_allocation_state(ObjectMonitor::Free);
1368 // _next_om is used for both per-thread in-use and free lists so
1369 // we have to remove 'm' from the in-use list first (as needed).
1370 if (from_per_thread_alloc) {
1371 // Need to remove 'm' from om_in_use_list.
1372 ObjectMonitor* cur_mid_in_use = NULL;
1373 bool extracted = false;
1374 for (ObjectMonitor* mid = self->om_in_use_list; mid != NULL; cur_mid_in_use = mid, mid = mid->_next_om) {
1375 if (m == mid) {
1376 // extract from per-thread in-use list
1377 if (mid == self->om_in_use_list) {
1378 self->om_in_use_list = mid->_next_om;
1379 } else if (cur_mid_in_use != NULL) {
1380 cur_mid_in_use->_next_om = mid->_next_om; // maintain the current thread in-use list
1381 }
1382 extracted = true;
1383 self->om_in_use_count--;
1384 break;
1385 }
1386 }
1387 assert(extracted, "Should have extracted from in-use list");
1388 }
1389
1390 m->_next_om = self->om_free_list;
1391 guarantee(m->is_free(), "invariant");
1392 self->om_free_list = m;
1393 self->om_free_count++;
1394 }
1395
1396 // Return ObjectMonitors on a moribund thread's free and in-use
1397 // lists to the appropriate global lists. The ObjectMonitors on the
1398 // per-thread in-use list may still be in use by other threads.
1399 //
1400 // We currently call om_flush() from Threads::remove() before the
1401 // thread has been excised from the thread list and is no longer a
1402 // mutator. This means that om_flush() cannot run concurrently with
1403 // a safepoint and interleave with deflate_idle_monitors(). In
1404 // particular, this ensures that the thread's in-use monitors are
1405 // scanned by a GC safepoint, either via Thread::oops_do() (before
1406 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1407 // om_flush() is called).
1408 //
1409 // With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT()
1410 // and deflate_per_thread_idle_monitors_using_JT() (in another thread) can
1411 // run at the same time as om_flush() so we have to be careful.
1412
1413 void ObjectSynchronizer::om_flush(Thread* self) {
1414 ObjectMonitor* free_list = self->om_free_list;
1415 ObjectMonitor* free_tail = NULL;
1416 int free_count = 0;
1417 if (free_list != NULL) {
1418 ObjectMonitor* s;
1419 // The thread is going away. Set 'free_tail' to the last per-thread free
1420 // monitor which will be linked to g_free_list below under the gListLock.
1421 stringStream ss;
1422 for (s = free_list; s != NULL; s = s->_next_om) {
1423 free_count++;
1424 free_tail = s;
1425 guarantee(s->object() == NULL, "invariant");
1426 guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss));
1427 }
1428 guarantee(free_tail != NULL, "invariant");
1429 ADIM_guarantee(self->om_free_count == free_count, "free-count off");
1430 self->om_free_list = NULL;
1431 self->om_free_count = 0;
1432 }
1433
1434 ObjectMonitor* in_use_list = self->om_in_use_list;
1435 ObjectMonitor* in_use_tail = NULL;
1436 int in_use_count = 0;
1437 if (in_use_list != NULL) {
1438 // The thread is going away, however the ObjectMonitors on the
1439 // om_in_use_list may still be in-use by other threads. Link
1440 // them to in_use_tail, which will be linked into the global
1441 // in-use list g_om_in_use_list below, under the gListLock.
1442 ObjectMonitor *cur_om;
1443 for (cur_om = in_use_list; cur_om != NULL; cur_om = cur_om->_next_om) {
1444 in_use_tail = cur_om;
1445 in_use_count++;
1446 ADIM_guarantee(cur_om->is_active(), "invariant");
1447 }
1448 guarantee(in_use_tail != NULL, "invariant");
1449 ADIM_guarantee(self->om_in_use_count == in_use_count, "in-use count off");
1450 self->om_in_use_list = NULL;
1451 self->om_in_use_count = 0;
1452 }
1453
1454 Thread::muxAcquire(&gListLock, "om_flush");
1455 if (free_tail != NULL) {
1456 free_tail->_next_om = g_free_list;
1457 g_free_list = free_list;
1458 g_om_free_count += free_count;
1459 }
1460
1461 if (in_use_tail != NULL) {
1462 in_use_tail->_next_om = g_om_in_use_list;
1463 g_om_in_use_list = in_use_list;
1464 g_om_in_use_count += in_use_count;
1465 }
1466
1467 Thread::muxRelease(&gListLock);
1468
1469 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1470 LogStreamHandle(Info, monitorinflation) lsh_info;
1471 LogStream* ls = NULL;
1472 if (log_is_enabled(Debug, monitorinflation)) {
1473 ls = &lsh_debug;
1474 } else if ((free_count != 0 || in_use_count != 0) &&
1475 log_is_enabled(Info, monitorinflation)) {
1476 ls = &lsh_info;
1477 }
1478 if (ls != NULL) {
1479 ls->print_cr("om_flush: jt=" INTPTR_FORMAT ", free_count=%d"
1480 ", in_use_count=%d" ", om_free_provision=%d",
1481 p2i(self), free_count, in_use_count, self->om_free_provision);
1482 }
1483 }
1484
1485 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1486 const oop obj,
1487 ObjectSynchronizer::InflateCause cause) {
1488 assert(event != NULL, "invariant");
1565 //
1566 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1567 // to install INFLATING into the mark word. We originally installed INFLATING,
1568 // allocated the objectmonitor, and then finally STed the address of the
1569 // objectmonitor into the mark. This was correct, but artificially lengthened
1570 // the interval in which INFLATED appeared in the mark, thus increasing
1571 // the odds of inflation contention.
1572 //
1573 // We now use per-thread private objectmonitor free lists.
1574 // These list are reprovisioned from the global free list outside the
1575 // critical INFLATING...ST interval. A thread can transfer
1576 // multiple objectmonitors en-mass from the global free list to its local free list.
1577 // This reduces coherency traffic and lock contention on the global free list.
1578 // Using such local free lists, it doesn't matter if the om_alloc() call appears
1579 // before or after the CAS(INFLATING) operation.
1580 // See the comments in om_alloc().
1581
1582 LogStreamHandle(Trace, monitorinflation) lsh;
1583
1584 if (mark.has_locker()) {
1585 ObjectMonitor* m;
1586 if (!AsyncDeflateIdleMonitors || cause == inflate_cause_vm_internal) {
1587 // If !AsyncDeflateIdleMonitors or if an internal inflation, then
1588 // we won't stop for a potential safepoint in om_alloc.
1589 m = om_alloc(self, cause);
1590 } else {
1591 // If AsyncDeflateIdleMonitors and not an internal inflation, then
1592 // we may stop for a safepoint in om_alloc() so protect object.
1593 Handle h_obj(self, object);
1594 m = om_alloc(self, cause);
1595 object = h_obj(); // Refresh object.
1596 }
1597 // Optimistically prepare the objectmonitor - anticipate successful CAS
1598 // We do this before the CAS in order to minimize the length of time
1599 // in which INFLATING appears in the mark.
1600 m->Recycle();
1601 m->_Responsible = NULL;
1602 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // Consider: maintain by type/class
1603
1604 markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1605 if (cmp != mark) {
1606 om_release(self, m, true);
1607 continue; // Interference -- just retry
1608 }
1609
1610 // We've successfully installed INFLATING (0) into the mark-word.
1611 // This is the only case where 0 will appear in a mark-word.
1612 // Only the singular thread that successfully swings the mark-word
1613 // to 0 can perform (or more precisely, complete) inflation.
1614 //
1615 // Why do we CAS a 0 into the mark-word instead of just CASing the
1616 // mark-word from the stack-locked value directly to the new inflated state?
1674 }
1675 if (event.should_commit()) {
1676 post_monitor_inflate_event(&event, object, cause);
1677 }
1678 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
1679 return;
1680 }
1681
1682 // CASE: neutral
1683 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
1684 // If we know we're inflating for entry it's better to inflate by swinging a
1685 // pre-locked ObjectMonitor pointer into the object header. A successful
1686 // CAS inflates the object *and* confers ownership to the inflating thread.
1687 // In the current implementation we use a 2-step mechanism where we CAS()
1688 // to inflate and then CAS() again to try to swing _owner from NULL to self.
1689 // An inflateTry() method that we could call from fast_enter() and slow_enter()
1690 // would be useful.
1691
1692 // Catch if the object's header is not neutral (not locked and
1693 // not marked is what we care about here).
1694 ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1695 ObjectMonitor* m;
1696 if (!AsyncDeflateIdleMonitors || cause == inflate_cause_vm_internal) {
1697 // If !AsyncDeflateIdleMonitors or if an internal inflation, then
1698 // we won't stop for a potential safepoint in om_alloc.
1699 m = om_alloc(self, cause);
1700 } else {
1701 // If AsyncDeflateIdleMonitors and not an internal inflation, then
1702 // we may stop for a safepoint in om_alloc() so protect object.
1703 Handle h_obj(self, object);
1704 m = om_alloc(self, cause);
1705 object = h_obj(); // Refresh object.
1706 }
1707 // prepare m for installation - set monitor to initial state
1708 m->Recycle();
1709 m->set_header(mark);
1710 // If we leave _owner == DEFLATER_MARKER here, then the simple C2
1711 // ObjectMonitor enter optimization can no longer race with async
1712 // deflation and reuse.
1713 m->set_object(object);
1714 m->_Responsible = NULL;
1715 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
1716
1717 omh_p->set_om_ptr(m);
1718 assert(m->is_new(), "freshly allocated monitor must be new");
1719 m->set_allocation_state(ObjectMonitor::Old);
1720
1721 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
1722 m->set_header(markWord::zero());
1723 m->set_object(NULL);
1724 m->Recycle();
1725 omh_p->set_om_ptr(NULL);
1726 // om_release() will reset the allocation state
1727 om_release(self, m, true);
1728 m = NULL;
1729 continue;
1730 // interference - the markword changed - just retry.
1731 // The state-transitions are one-way, so there's no chance of
1732 // live-lock -- "Inflated" is an absorbing state.
1733 }
1734
1735 // Hopefully the performance counters are allocated on distinct
1736 // cache lines to avoid false sharing on MP systems ...
1737 OM_PERFDATA_OP(Inflations, inc());
1738 if (log_is_enabled(Trace, monitorinflation)) {
1739 ResourceMark rm(self);
1740 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
1741 INTPTR_FORMAT ", type='%s'", p2i(object),
1760 // These operations are called at all safepoints, immediately after mutators
1761 // are stopped, but before any objects have moved. Collectively they traverse
1762 // the population of in-use monitors, deflating where possible. The scavenged
1763 // monitors are returned to the global monitor free list.
1764 //
1765 // Beware that we scavenge at *every* stop-the-world point. Having a large
1766 // number of monitors in-use could negatively impact performance. We also want
1767 // to minimize the total # of monitors in circulation, as they incur a small
1768 // footprint penalty.
1769 //
1770 // Perversely, the heap size -- and thus the STW safepoint rate --
1771 // typically drives the scavenge rate. Large heaps can mean infrequent GC,
1772 // which in turn can mean large(r) numbers of ObjectMonitors in circulation.
1773 // This is an unfortunate aspect of this design.
1774 //
1775 // For async deflation:
1776 // If a special deflation request is made, then the safepoint based
1777 // deflation mechanism is used. Otherwise, an async deflation request
1778 // is registered with the ServiceThread and it is notified.
1779
1780 void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* _counters) {
1781 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1782
1783 // The per-thread in-use lists are handled in
1784 // ParallelSPCleanupThreadClosure::do_thread().
1785
1786 if (!AsyncDeflateIdleMonitors || is_special_deflation_requested()) {
1787 // Use the older mechanism for the global in-use list or if a
1788 // special deflation has been requested before the safepoint.
1789 ObjectSynchronizer::deflate_idle_monitors(_counters);
1790 return;
1791 }
1792
1793 log_debug(monitorinflation)("requesting async deflation of idle monitors.");
1794 // Request deflation of idle monitors by the ServiceThread:
1795 set_is_async_deflation_requested(true);
1796 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
1797 ml.notify_all();
1798 }
1799
1800 // Deflate a single monitor if not in-use
1801 // Return true if deflated, false if in-use
1802 bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj,
1803 ObjectMonitor** free_head_p,
1804 ObjectMonitor** free_tail_p) {
1805 bool deflated;
1806 // Normal case ... The monitor is associated with obj.
1807 const markWord mark = obj->mark();
1808 guarantee(mark == markWord::encode(mid), "should match: mark="
1809 INTPTR_FORMAT ", encoded mid=" INTPTR_FORMAT, mark.value(),
1828 "object=" INTPTR_FORMAT ", mark="
1829 INTPTR_FORMAT ", type='%s'", p2i(obj),
1830 mark.value(), obj->klass()->external_name());
1831 }
1832
1833 // Restore the header back to obj
1834 obj->release_set_mark(dmw);
1835 if (AsyncDeflateIdleMonitors) {
1836 // clear() expects the owner field to be NULL and we won't race
1837 // with the simple C2 ObjectMonitor enter optimization since
1838 // we're at a safepoint.
1839 mid->set_owner(NULL);
1840 }
1841 mid->clear();
1842
1843 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
1844 p2i(mid->object()));
1845 assert(mid->is_free(), "invariant");
1846
1847 // Move the deflated ObjectMonitor to the working free list
1848 // defined by free_head_p and free_tail_p.
1849 if (*free_head_p == NULL) *free_head_p = mid;
1850 if (*free_tail_p != NULL) {
1851 // We append to the list so the caller can use mid->_next_om
1852 // to fix the linkages in its context.
1853 ObjectMonitor* prevtail = *free_tail_p;
1854 // Should have been cleaned up by the caller:
1855 assert(prevtail->_next_om == NULL, "cleaned up deflated?");
1856 prevtail->_next_om = mid;
1857 }
1858 *free_tail_p = mid;
1859 // At this point, mid->_next_om still refers to its current
1860 // value and another ObjectMonitor's _next_om field still
1861 // refers to this ObjectMonitor. Those linkages have to be
1862 // cleaned up by the caller who has the complete context.
1863 deflated = true;
1864 }
1865 return deflated;
1866 }
1867
1868 // Deflate the specified ObjectMonitor if not in-use using a JavaThread.
1869 // Returns true if it was deflated and false otherwise.
1870 //
1871 // The async deflation protocol sets owner to DEFLATER_MARKER and
1872 // makes ref_count negative as signals to contending threads that
1873 // an async deflation is in progress. There are a number of checks
1874 // as part of the protocol to make sure that the calling thread has
1875 // not lost the race to a contending thread or to a thread that just
1876 // wants to use the ObjectMonitor*.
1939 const oop obj = (oop) mid->object();
1940 if (log_is_enabled(Trace, monitorinflation)) {
1941 ResourceMark rm;
1942 log_trace(monitorinflation)("deflate_monitor_using_JT: "
1943 "object=" INTPTR_FORMAT ", mark="
1944 INTPTR_FORMAT ", type='%s'",
1945 p2i(obj), obj->mark().value(),
1946 obj->klass()->external_name());
1947 }
1948
1949 // Install the old mark word if nobody else has already done it.
1950 mid->install_displaced_markword_in_object(obj);
1951 mid->clear_using_JT();
1952
1953 assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT,
1954 p2i(mid->object()));
1955 assert(mid->is_free(), "must be free: allocation_state=%d",
1956 (int) mid->allocation_state());
1957
1958 // Move the deflated ObjectMonitor to the working free list
1959 // defined by free_head_p and free_tail_p.
1960 if (*free_head_p == NULL) {
1961 // First one on the list.
1962 *free_head_p = mid;
1963 }
1964 if (*free_tail_p != NULL) {
1965 // We append to the list so the caller can use mid->_next_om
1966 // to fix the linkages in its context.
1967 ObjectMonitor* prevtail = *free_tail_p;
1968 // Should have been cleaned up by the caller:
1969 assert(prevtail->_next_om == NULL, "must be NULL: _next_om="
1970 INTPTR_FORMAT, p2i(prevtail->_next_om));
1971 prevtail->_next_om = mid;
1972 }
1973 *free_tail_p = mid;
1974
1975 // At this point, mid->_next_om still refers to its current
1976 // value and another ObjectMonitor's _next_om field still
1977 // refers to this ObjectMonitor. Those linkages have to be
1978 // cleaned up by the caller who has the complete context.
1979
1980 // We leave owner == DEFLATER_MARKER and ref_count < 0
1981 // to force any racing threads to retry.
1982 return true; // Success, ObjectMonitor has been deflated.
1983 }
1984
1985 // The owner was changed from DEFLATER_MARKER so we lost the
1986 // race since the ObjectMonitor is now busy.
1987
1988 // Add back max_jint to restore the ref_count field to its
1989 // proper value (which may not be what we saw above):
1990 Atomic::add(max_jint, &mid->_ref_count);
1991
1992 assert(mid->ref_count() >= 0, "must not be negative: ref_count=%d",
1993 mid->ref_count());
1994 return false;
1995 }
1996
1997 // The ref_count was no longer 0 so we lost the race since the
1998 // ObjectMonitor is now busy or the ObjectMonitor* is now is use.
1999 // Restore owner to NULL if it is still DEFLATER_MARKER:
2000 Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER);
2001 }
2002
2003 // The owner field is no longer NULL so we lost the race since the
2004 // ObjectMonitor is now busy.
2005 return false;
2006 }
2007
2008 // Walk a given monitor list, and deflate idle monitors
2009 // The given list could be a per-thread list or a global list
2010 // Caller acquires gListLock as needed.
2011 //
2012 // In the case of parallel processing of thread local monitor lists,
2013 // work is done by Threads::parallel_threads_do() which ensures that
2014 // each Java thread is processed by exactly one worker thread, and
2015 // thus avoid conflicts that would arise when worker threads would
2016 // process the same monitor lists concurrently.
2017 //
2018 // See also ParallelSPCleanupTask and
2019 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
2020 // Threads::parallel_java_threads_do() in thread.cpp.
2021 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor** list_p,
2022 ObjectMonitor** free_head_p,
2023 ObjectMonitor** free_tail_p) {
2024 ObjectMonitor* mid;
2025 ObjectMonitor* next;
2026 ObjectMonitor* cur_mid_in_use = NULL;
2027 int deflated_count = 0;
2028
2029 for (mid = *list_p; mid != NULL;) {
2030 oop obj = (oop) mid->object();
2031 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
2032 // Deflation succeeded and already updated free_head_p and
2033 // free_tail_p as needed. Finish the move to the local free list
2034 // by unlinking mid from the global or per-thread in-use list.
2035 if (mid == *list_p) {
2036 *list_p = mid->_next_om;
2037 } else if (cur_mid_in_use != NULL) {
2038 cur_mid_in_use->_next_om = mid->_next_om; // maintain the current thread in-use list
2039 }
2040 next = mid->_next_om;
2041 mid->_next_om = NULL; // This mid is current tail in the free_head_p list
2042 mid = next;
2043 deflated_count++;
2044 } else {
2045 cur_mid_in_use = mid;
2046 mid = mid->_next_om;
2047 }
2048 }
2049 return deflated_count;
2050 }
2051
2052 // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using
2053 // a JavaThread. Returns the number of deflated ObjectMonitors. The given
2054 // list could be a per-thread in-use list or the global in-use list.
2055 // Caller acquires gListLock as appropriate. If a safepoint has started,
2056 // then we save state via saved_mid_in_use_p and return to the caller to
2057 // honor the safepoint.
2058 //
2059 int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor** list_p,
2060 ObjectMonitor** free_head_p,
2061 ObjectMonitor** free_tail_p,
2062 ObjectMonitor** saved_mid_in_use_p) {
2063 assert(AsyncDeflateIdleMonitors, "sanity check");
2064 assert(Thread::current()->is_Java_thread(), "precondition");
2065
2066 ObjectMonitor* mid;
2067 ObjectMonitor* next;
2068 ObjectMonitor* cur_mid_in_use = NULL;
2069 int deflated_count = 0;
2070
2071 if (*saved_mid_in_use_p == NULL) {
2072 // No saved state so start at the beginning.
2073 mid = *list_p;
2074 } else {
2075 // We're restarting after a safepoint so restore the necessary state
2076 // before we resume.
2077 cur_mid_in_use = *saved_mid_in_use_p;
2078 mid = cur_mid_in_use->_next_om;
2079 }
2080 while (mid != NULL) {
2081 // Only try to deflate if there is an associated Java object and if
2082 // mid is old (is not newly allocated and is not newly freed).
2083 if (mid->object() != NULL && mid->is_old() &&
2084 deflate_monitor_using_JT(mid, free_head_p, free_tail_p)) {
2085 // Deflation succeeded so update the in-use list.
2086 if (mid == *list_p) {
2087 *list_p = mid->_next_om;
2088 } else if (cur_mid_in_use != NULL) {
2089 // Maintain the current in-use list.
2090 cur_mid_in_use->_next_om = mid->_next_om;
2091 }
2092 next = mid->_next_om;
2093 mid->_next_om = NULL;
2094 // At this point mid is disconnected from the in-use list
2095 // and is the current tail in the free_head_p list.
2096 mid = next;
2097 deflated_count++;
2098 } else {
2099 // mid is considered in-use if it does not have an associated
2100 // Java object or mid is not old or deflation did not succeed.
2101 // A mid->is_new() node can be seen here when it is freshly
2102 // returned by om_alloc() (and skips the deflation code path).
2103 // A mid->is_old() node can be seen here when deflation failed.
2104 // A mid->is_free() node can be seen here when a fresh node from
2105 // om_alloc() is released by om_release() due to losing the race
2106 // in inflate().
2107
2108 cur_mid_in_use = mid;
2109 mid = mid->_next_om;
2110
2111 if (SafepointSynchronize::is_synchronizing() &&
2112 cur_mid_in_use != *list_p && cur_mid_in_use->is_old()) {
2113 // If a safepoint has started and cur_mid_in_use is not the list
2114 // head and is old, then it is safe to use as saved state. Return
2115 // to the caller so gListLock can be dropped as appropriate
2116 // before blocking.
2117 *saved_mid_in_use_p = cur_mid_in_use;
2118 return deflated_count;
2119 }
2120 }
2121 }
2122 // We finished the list without a safepoint starting so there's
2123 // no need to save state.
2124 *saved_mid_in_use_p = NULL;
2125 return deflated_count;
2126 }
2127
2128 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2129 counters->n_in_use = 0; // currently associated with objects
2130 counters->n_in_circulation = 0; // extant
2131 counters->n_scavenged = 0; // reclaimed (global and per-thread)
2132 counters->per_thread_scavenged = 0; // per-thread scavenge total
2133 counters->per_thread_times = 0.0; // per-thread scavenge times
2134 }
2135
2136 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
2137 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2138
2139 if (AsyncDeflateIdleMonitors) {
2140 // Nothing to do when global idle ObjectMonitors are deflated using
2141 // a JavaThread unless a special deflation has been requested.
2142 if (!is_special_deflation_requested()) {
2143 return;
2144 }
2145 }
2146
2147 bool deflated = false;
2148
2149 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2150 ObjectMonitor* free_tail_p = NULL;
2151 elapsedTimer timer;
2152
2153 if (log_is_enabled(Info, monitorinflation)) {
2154 timer.start();
2155 }
2156
2157 // Prevent om_flush from changing mids in Thread dtor's during deflation
2158 // And in case the vm thread is acquiring a lock during a safepoint
2159 // See e.g. 6320749
2160 Thread::muxAcquire(&gListLock, "deflate_idle_monitors");
2161
2162 // Note: the thread-local monitors lists get deflated in
2163 // a separate pass. See deflate_thread_local_monitors().
2164
2165 // For moribund threads, scan g_om_in_use_list
2166 int deflated_count = 0;
2167 if (g_om_in_use_list) {
2168 counters->n_in_circulation += g_om_in_use_count;
2169 deflated_count = deflate_monitor_list((ObjectMonitor **)&g_om_in_use_list, &free_head_p, &free_tail_p);
2170 g_om_in_use_count -= deflated_count;
2171 counters->n_scavenged += deflated_count;
2172 counters->n_in_use += g_om_in_use_count;
2173 }
2174
2175 if (free_head_p != NULL) {
2176 // Move the deflated ObjectMonitors back to the global free list.
2177 guarantee(free_tail_p != NULL && counters->n_scavenged > 0, "invariant");
2178 assert(free_tail_p->_next_om == NULL, "invariant");
2179 // constant-time list splice - prepend scavenged segment to g_free_list
2180 free_tail_p->_next_om = g_free_list;
2181 g_free_list = free_head_p;
2182 }
2183 Thread::muxRelease(&gListLock);
2184 timer.stop();
2185
2186 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2187 LogStreamHandle(Info, monitorinflation) lsh_info;
2188 LogStream* ls = NULL;
2189 if (log_is_enabled(Debug, monitorinflation)) {
2190 ls = &lsh_debug;
2191 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2192 ls = &lsh_info;
2193 }
2194 if (ls != NULL) {
2195 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2196 }
2197 }
2198
2199 // Deflate global idle ObjectMonitors using a JavaThread.
2200 //
2201 void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() {
2202 assert(AsyncDeflateIdleMonitors, "sanity check");
2203 assert(Thread::current()->is_Java_thread(), "precondition");
2204 JavaThread* self = JavaThread::current();
2205
2206 deflate_common_idle_monitors_using_JT(true /* is_global */, self);
2207 }
2208
2209 // Deflate per-thread idle ObjectMonitors using a JavaThread.
2210 //
2211 void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT() {
2212 assert(AsyncDeflateIdleMonitors, "sanity check");
2213 assert(Thread::current()->is_Java_thread(), "precondition");
2214 JavaThread* self = JavaThread::current();
2215
2216 self->om_request_deflation = false;
2217
2218 deflate_common_idle_monitors_using_JT(false /* !is_global */, self);
2219 }
2220
2221 // Deflate global or per-thread idle ObjectMonitors using a JavaThread.
2222 //
2223 void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* self) {
2224 int deflated_count = 0;
2225 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors
2226 ObjectMonitor* free_tail_p = NULL;
2227 ObjectMonitor* saved_mid_in_use_p = NULL;
2228 elapsedTimer timer;
2229
2230 if (log_is_enabled(Info, monitorinflation)) {
2231 timer.start();
2232 }
2233
2234 if (is_global) {
2235 Thread::muxAcquire(&gListLock, "deflate_global_idle_monitors_using_JT(1)");
2236 OM_PERFDATA_OP(MonExtant, set_value(g_om_in_use_count));
2237 } else {
2238 OM_PERFDATA_OP(MonExtant, inc(self->om_in_use_count));
2239 }
2240
2241 do {
2242 int local_deflated_count;
2243 if (is_global) {
2244 local_deflated_count = deflate_monitor_list_using_JT((ObjectMonitor **)&g_om_in_use_list, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2245 g_om_in_use_count -= local_deflated_count;
2246 } else {
2247 local_deflated_count = deflate_monitor_list_using_JT(self->om_in_use_list_addr(), &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2248 self->om_in_use_count -= local_deflated_count;
2249 }
2250 deflated_count += local_deflated_count;
2251
2252 if (free_head_p != NULL) {
2253 // Move the scavenged ObjectMonitors to the global free list.
2254 guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count);
2255 assert(free_tail_p->_next_om == NULL, "invariant");
2256
2257 if (!is_global) {
2258 Thread::muxAcquire(&gListLock, "deflate_per_thread_idle_monitors_using_JT(2)");
2259 }
2260 // Constant-time list splice - prepend scavenged segment to g_free_list.
2261 free_tail_p->_next_om = g_free_list;
2262 g_free_list = free_head_p;
2263
2264 g_om_free_count += local_deflated_count;
2265 OM_PERFDATA_OP(Deflations, inc(local_deflated_count));
2266 if (!is_global) {
2267 Thread::muxRelease(&gListLock);
2268 }
2269 }
2270
2271 if (saved_mid_in_use_p != NULL) {
2272 // deflate_monitor_list_using_JT() detected a safepoint starting.
2273 if (is_global) {
2274 Thread::muxRelease(&gListLock);
2275 }
2276 timer.stop();
2277 {
2278 if (is_global) {
2279 log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint.");
2280 } else {
2281 log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(self));
2282 }
2283 assert(SafepointSynchronize::is_synchronizing(), "sanity check");
2284 ThreadBlockInVM blocker(self);
2285 }
2286 // Prepare for another loop after the safepoint.
2287 free_head_p = NULL;
2288 free_tail_p = NULL;
2289 if (log_is_enabled(Info, monitorinflation)) {
2290 timer.start();
2291 }
2292 if (is_global) {
2293 Thread::muxAcquire(&gListLock, "deflate_global_idle_monitors_using_JT(3)");
2294 }
2295 }
2296 } while (saved_mid_in_use_p != NULL);
2297 if (is_global) {
2298 Thread::muxRelease(&gListLock);
2299 }
2300 timer.stop();
2301
2302 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2303 LogStreamHandle(Info, monitorinflation) lsh_info;
2304 LogStream* ls = NULL;
2305 if (log_is_enabled(Debug, monitorinflation)) {
2306 ls = &lsh_debug;
2307 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2308 ls = &lsh_info;
2309 }
2310 if (ls != NULL) {
2311 if (is_global) {
2312 ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2313 } else {
2314 ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(self), timer.seconds(), deflated_count);
2315 }
2316 }
2317 }
2318
2319 void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2320 // Report the cumulative time for deflating each thread's idle
2321 // monitors. Note: if the work is split among more than one
2322 // worker thread, then the reported time will likely be more
2323 // than a beginning to end measurement of the phase.
2324 // Note: AsyncDeflateIdleMonitors only deflates per-thread idle
2325 // monitors at a safepoint when a special deflation has been requested.
2326 log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", counters->per_thread_times, counters->per_thread_scavenged);
2327
2328 bool needs_special_deflation = is_special_deflation_requested();
2329 if (!AsyncDeflateIdleMonitors || needs_special_deflation) {
2330 // AsyncDeflateIdleMonitors does not use these counters unless
2331 // there is a special deflation request.
2332
2333 g_om_free_count += counters->n_scavenged;
2334
2335 OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged));
2336 OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation));
2337 }
2338
2339 if (log_is_enabled(Debug, monitorinflation)) {
2340 // exit_globals()'s call to audit_and_print_stats() is done
2341 // at the Info level.
2342 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2343 } else if (log_is_enabled(Info, monitorinflation)) {
2344 Thread::muxAcquire(&gListLock, "finish_deflate_idle_monitors");
2345 log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, "
2346 "g_om_free_count=%d", g_om_population,
2347 g_om_in_use_count, g_om_free_count);
2348 Thread::muxRelease(&gListLock);
2349 }
2350
2351 ForceMonitorScavenge = 0; // Reset
2352 GVars.stw_random = os::random();
2353 GVars.stw_cycle++;
2354 if (needs_special_deflation) {
2355 set_is_special_deflation_requested(false); // special deflation is done
2356 }
2357 }
2358
2359 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
2360 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2361
2362 if (AsyncDeflateIdleMonitors) {
2363 if (!is_special_deflation_requested()) {
2364 // Mark the JavaThread for idle monitor deflation if a special
2365 // deflation has NOT been requested.
2366 if (thread->om_in_use_count > 0) {
2367 // This JavaThread is using monitors so mark it.
2368 thread->om_request_deflation = true;
2369 }
2370 return;
2371 }
2372 }
2373
2374 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2375 ObjectMonitor* free_tail_p = NULL;
2376 elapsedTimer timer;
2377
2378 if (log_is_enabled(Info, safepoint, cleanup) ||
2379 log_is_enabled(Info, monitorinflation)) {
2380 timer.start();
2381 }
2382
2383 int deflated_count = deflate_monitor_list(thread->om_in_use_list_addr(), &free_head_p, &free_tail_p);
2384
2385 Thread::muxAcquire(&gListLock, "deflate_thread_local_monitors");
2386
2387 // Adjust counters
2388 counters->n_in_circulation += thread->om_in_use_count;
2389 thread->om_in_use_count -= deflated_count;
2390 counters->n_scavenged += deflated_count;
2391 counters->n_in_use += thread->om_in_use_count;
2392 counters->per_thread_scavenged += deflated_count;
2393
2394 if (free_head_p != NULL) {
2395 // Move the deflated ObjectMonitors back to the global free list.
2396 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2397 assert(free_tail_p->_next_om == NULL, "invariant");
2398
2399 // constant-time list splice - prepend scavenged segment to g_free_list
2400 free_tail_p->_next_om = g_free_list;
2401 g_free_list = free_head_p;
2402 }
2403
2404 timer.stop();
2405 // Safepoint logging cares about cumulative per_thread_times and
2406 // we'll capture most of the cost, but not the muxRelease() which
2407 // should be cheap.
2408 counters->per_thread_times += timer.seconds();
2409
2410 Thread::muxRelease(&gListLock);
2411
2412 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2413 LogStreamHandle(Info, monitorinflation) lsh_info;
2414 LogStream* ls = NULL;
2415 if (log_is_enabled(Debug, monitorinflation)) {
2416 ls = &lsh_debug;
2417 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2418 ls = &lsh_info;
2419 }
2420 if (ls != NULL) {
2421 ls->print_cr("jt=" INTPTR_FORMAT ": deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(thread), timer.seconds(), deflated_count);
2422 }
2423 }
2424
2425 // Monitor cleanup on JavaThread::exit
2426
2427 // Iterate through monitor cache and attempt to release thread's monitors
2428 // Gives up on a particular monitor if an exception occurs, but continues
2429 // the overall iteration, swallowing the exception.
2430 class ReleaseJavaMonitorsClosure: public MonitorClosure {
2431 private:
2442
2443 // Release all inflated monitors owned by THREAD. Lightweight monitors are
2444 // ignored. This is meant to be called during JNI thread detach which assumes
2445 // all remaining monitors are heavyweight. All exceptions are swallowed.
2446 // Scanning the extant monitor list can be time consuming.
2447 // A simple optimization is to add a per-thread flag that indicates a thread
2448 // called jni_monitorenter() during its lifetime.
2449 //
2450 // Instead of No_Savepoint_Verifier it might be cheaper to
2451 // use an idiom of the form:
2452 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
2453 // <code that must not run at safepoint>
2454 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
2455 // Since the tests are extremely cheap we could leave them enabled
2456 // for normal product builds.
2457
2458 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
2459 assert(THREAD == JavaThread::current(), "must be current Java thread");
2460 NoSafepointVerifier nsv;
2461 ReleaseJavaMonitorsClosure rjmc(THREAD);
2462 Thread::muxAcquire(&gListLock, "release_monitors_owned_by_thread");
2463 ObjectSynchronizer::monitors_iterate(&rjmc);
2464 Thread::muxRelease(&gListLock);
2465 THREAD->clear_pending_exception();
2466 }
2467
2468 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
2469 switch (cause) {
2470 case inflate_cause_vm_internal: return "VM Internal";
2471 case inflate_cause_monitor_enter: return "Monitor Enter";
2472 case inflate_cause_wait: return "Monitor Wait";
2473 case inflate_cause_notify: return "Monitor Notify";
2474 case inflate_cause_hash_code: return "Monitor Hash Code";
2475 case inflate_cause_jni_enter: return "JNI Monitor Enter";
2476 case inflate_cause_jni_exit: return "JNI Monitor Exit";
2477 default:
2478 ShouldNotReachHere();
2479 }
2480 return "Unknown";
2481 }
2482
2483 //------------------------------------------------------------------------------
2484 // Debugging code
2498 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
2499 return (u_char*)&GVars.stw_random;
2500 }
2501
2502 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
2503 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
2504
2505 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2506 LogStreamHandle(Info, monitorinflation) lsh_info;
2507 LogStreamHandle(Trace, monitorinflation) lsh_trace;
2508 LogStream* ls = NULL;
2509 if (log_is_enabled(Trace, monitorinflation)) {
2510 ls = &lsh_trace;
2511 } else if (log_is_enabled(Debug, monitorinflation)) {
2512 ls = &lsh_debug;
2513 } else if (log_is_enabled(Info, monitorinflation)) {
2514 ls = &lsh_info;
2515 }
2516 assert(ls != NULL, "sanity check");
2517
2518 if (!on_exit) {
2519 // Not at VM exit so grab the global list lock.
2520 Thread::muxAcquire(&gListLock, "audit_and_print_stats");
2521 }
2522
2523 // Log counts for the global and per-thread monitor lists:
2524 int chk_om_population = log_monitor_list_counts(ls);
2525 int error_cnt = 0;
2526
2527 ls->print_cr("Checking global lists:");
2528
2529 // Check g_om_population:
2530 if (g_om_population == chk_om_population) {
2531 ls->print_cr("g_om_population=%d equals chk_om_population=%d",
2532 g_om_population, chk_om_population);
2533 } else {
2534 ls->print_cr("ERROR: g_om_population=%d is not equal to "
2535 "chk_om_population=%d", g_om_population,
2536 chk_om_population);
2537 error_cnt++;
2538 }
2539
2540 // Check g_om_in_use_list and g_om_in_use_count:
2541 chk_global_in_use_list_and_count(ls, &error_cnt);
2542
2543 // Check g_free_list and g_om_free_count:
2544 chk_global_free_list_and_count(ls, &error_cnt);
2545
2546 if (!on_exit) {
2547 Thread::muxRelease(&gListLock);
2548 }
2549
2550 ls->print_cr("Checking per-thread lists:");
2551
2552 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2553 // Check om_in_use_list and om_in_use_count:
2554 chk_per_thread_in_use_list_and_count(jt, ls, &error_cnt);
2555
2556 // Check om_free_list and om_free_count:
2557 chk_per_thread_free_list_and_count(jt, ls, &error_cnt);
2558 }
2559
2560 if (error_cnt == 0) {
2561 ls->print_cr("No errors found in monitor list checks.");
2562 } else {
2563 log_error(monitorinflation)("found monitor list errors: error_cnt=%d", error_cnt);
2564 }
2565
2566 if ((on_exit && log_is_enabled(Info, monitorinflation)) ||
2567 (!on_exit && log_is_enabled(Trace, monitorinflation))) {
2568 // When exiting this log output is at the Info level. When called
2569 // at a safepoint, this log output is at the Trace level since
2570 // there can be a lot of it.
2571 log_in_use_monitor_details(ls, on_exit);
2572 }
2573
2574 ls->flush();
2575
2576 guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
2577 }
2578
2579 // Check a free monitor entry; log any errors.
2580 void ObjectSynchronizer::chk_free_entry(JavaThread* jt, ObjectMonitor* n,
2581 outputStream * out, int *error_cnt_p) {
2582 stringStream ss;
2583 if (n->is_busy()) {
2584 if (jt != NULL) {
2585 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2586 ": free per-thread monitor must not be busy: %s", p2i(jt),
2587 p2i(n), n->is_busy_to_string(&ss));
2588 } else {
2589 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2590 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
2591 }
2607 }
2608 if (n->object() != NULL) {
2609 if (jt != NULL) {
2610 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2611 ": free per-thread monitor must have NULL _object "
2612 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
2613 p2i(n->object()));
2614 } else {
2615 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2616 "must have NULL _object field: _object=" INTPTR_FORMAT,
2617 p2i(n), p2i(n->object()));
2618 }
2619 *error_cnt_p = *error_cnt_p + 1;
2620 }
2621 }
2622
2623 // Check the global free list and count; log the results of the checks.
2624 void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out,
2625 int *error_cnt_p) {
2626 int chk_om_free_count = 0;
2627 for (ObjectMonitor* n = g_free_list; n != NULL; n = n->_next_om) {
2628 chk_free_entry(NULL /* jt */, n, out, error_cnt_p);
2629 chk_om_free_count++;
2630 }
2631 if (g_om_free_count == chk_om_free_count) {
2632 out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d",
2633 g_om_free_count, chk_om_free_count);
2634 } else {
2635 out->print_cr("ERROR: g_om_free_count=%d is not equal to "
2636 "chk_om_free_count=%d", g_om_free_count,
2637 chk_om_free_count);
2638 *error_cnt_p = *error_cnt_p + 1;
2639 }
2640 }
2641
2642 // Check the global in-use list and count; log the results of the checks.
2643 void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out,
2644 int *error_cnt_p) {
2645 int chk_om_in_use_count = 0;
2646 for (ObjectMonitor* n = g_om_in_use_list; n != NULL; n = n->_next_om) {
2647 chk_in_use_entry(NULL /* jt */, n, out, error_cnt_p);
2648 chk_om_in_use_count++;
2649 }
2650 if (g_om_in_use_count == chk_om_in_use_count) {
2651 out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d", g_om_in_use_count,
2652 chk_om_in_use_count);
2653 } else {
2654 out->print_cr("ERROR: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d",
2655 g_om_in_use_count, chk_om_in_use_count);
2656 *error_cnt_p = *error_cnt_p + 1;
2657 }
2658 }
2659
2660 // Check an in-use monitor entry; log any errors.
2661 void ObjectSynchronizer::chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,
2662 outputStream * out, int *error_cnt_p) {
2663 if (n->header().value() == 0) {
2664 if (jt != NULL) {
2665 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2666 ": in-use per-thread monitor must have non-NULL _header "
2667 "field.", p2i(jt), p2i(n));
2668 } else {
2669 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
2670 "must have non-NULL _header field.", p2i(n));
2671 }
2672 *error_cnt_p = *error_cnt_p + 1;
2673 }
2674 if (n->object() == NULL) {
2675 if (jt != NULL) {
2704 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2705 ": in-use per-thread monitor's object does not refer "
2706 "to the same monitor: obj=" INTPTR_FORMAT ", mark="
2707 INTPTR_FORMAT ", obj_mon=" INTPTR_FORMAT, p2i(jt),
2708 p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
2709 } else {
2710 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
2711 "monitor's object does not refer to the same monitor: obj="
2712 INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
2713 INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
2714 }
2715 *error_cnt_p = *error_cnt_p + 1;
2716 }
2717 }
2718
2719 // Check the thread's free list and count; log the results of the checks.
2720 void ObjectSynchronizer::chk_per_thread_free_list_and_count(JavaThread *jt,
2721 outputStream * out,
2722 int *error_cnt_p) {
2723 int chk_om_free_count = 0;
2724 for (ObjectMonitor* n = jt->om_free_list; n != NULL; n = n->_next_om) {
2725 chk_free_entry(jt, n, out, error_cnt_p);
2726 chk_om_free_count++;
2727 }
2728 if (jt->om_free_count == chk_om_free_count) {
2729 out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals "
2730 "chk_om_free_count=%d", p2i(jt), jt->om_free_count, chk_om_free_count);
2731 } else {
2732 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not "
2733 "equal to chk_om_free_count=%d", p2i(jt), jt->om_free_count,
2734 chk_om_free_count);
2735 *error_cnt_p = *error_cnt_p + 1;
2736 }
2737 }
2738
2739 // Check the thread's in-use list and count; log the results of the checks.
2740 void ObjectSynchronizer::chk_per_thread_in_use_list_and_count(JavaThread *jt,
2741 outputStream * out,
2742 int *error_cnt_p) {
2743 int chk_om_in_use_count = 0;
2744 for (ObjectMonitor* n = jt->om_in_use_list; n != NULL; n = n->_next_om) {
2745 chk_in_use_entry(jt, n, out, error_cnt_p);
2746 chk_om_in_use_count++;
2747 }
2748 if (jt->om_in_use_count == chk_om_in_use_count) {
2749 out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals "
2750 "chk_om_in_use_count=%d", p2i(jt), jt->om_in_use_count,
2751 chk_om_in_use_count);
2752 } else {
2753 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not "
2754 "equal to chk_om_in_use_count=%d", p2i(jt), jt->om_in_use_count,
2755 chk_om_in_use_count);
2756 *error_cnt_p = *error_cnt_p + 1;
2757 }
2758 }
2759
2760 // Log details about ObjectMonitors on the in-use lists. The 'BHL'
2761 // flags indicate why the entry is in-use, 'object' and 'object type'
2762 // indicate the associated object and its type.
2763 void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out,
2764 bool on_exit) {
2765 if (!on_exit) {
2766 // Not at VM exit so grab the global list lock.
2767 Thread::muxAcquire(&gListLock, "log_in_use_monitor_details");
2768 }
2769
2770 stringStream ss;
2771 if (g_om_in_use_count > 0) {
2772 out->print_cr("In-use global monitor info:");
2773 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
2774 out->print_cr("%18s %s %7s %18s %18s",
2775 "monitor", "BHL", "ref_cnt", "object", "object type");
2776 out->print_cr("================== === ======= ================== ==================");
2777 for (ObjectMonitor* n = g_om_in_use_list; n != NULL; n = n->_next_om) {
2778 const oop obj = (oop) n->object();
2779 const markWord mark = n->header();
2780 ResourceMark rm;
2781 out->print(INTPTR_FORMAT " %d%d%d %7d " INTPTR_FORMAT " %s",
2782 p2i(n), n->is_busy() != 0, mark.hash() != 0,
2783 n->owner() != NULL, (int)n->ref_count(), p2i(obj),
2784 obj->klass()->external_name());
2785 if (n->is_busy() != 0) {
2786 out->print(" (%s)", n->is_busy_to_string(&ss));
2787 ss.reset();
2788 }
2789 out->cr();
2790 }
2791 }
2792
2793 if (!on_exit) {
2794 Thread::muxRelease(&gListLock);
2795 }
2796
2797 out->print_cr("In-use per-thread monitor info:");
2798 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
2799 out->print_cr("%18s %18s %s %7s %18s %18s",
2800 "jt", "monitor", "BHL", "ref_cnt", "object", "object type");
2801 out->print_cr("================== ================== === ======= ================== ==================");
2802 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2803 for (ObjectMonitor* n = jt->om_in_use_list; n != NULL; n = n->_next_om) {
2804 const oop obj = (oop) n->object();
2805 const markWord mark = n->header();
2806 ResourceMark rm;
2807 out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d %7d "
2808 INTPTR_FORMAT " %s", p2i(jt), p2i(n), n->is_busy() != 0,
2809 mark.hash() != 0, n->owner() != NULL, (int)n->ref_count(),
2810 p2i(obj), obj->klass()->external_name());
2811 if (n->is_busy() != 0) {
2812 out->print(" (%s)", n->is_busy_to_string(&ss));
2813 ss.reset();
2814 }
2815 out->cr();
2816 }
2817 }
2818
2819 out->flush();
2820 }
2821
2822 // Log counts for the global and per-thread monitor lists and return
2823 // the population count.
2824 int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) {
2825 int pop_count = 0;
2826 out->print_cr("%18s %10s %10s %10s",
2827 "Global Lists:", "InUse", "Free", "Total");
2828 out->print_cr("================== ========== ========== ==========");
2829 out->print_cr("%18s %10d %10d %10d", "",
2830 g_om_in_use_count, g_om_free_count, g_om_population);
2831 pop_count += g_om_in_use_count + g_om_free_count;
2832
2833 out->print_cr("%18s %10s %10s %10s",
2834 "Per-Thread Lists:", "InUse", "Free", "Provision");
2835 out->print_cr("================== ========== ========== ==========");
2836
2837 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2838 out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt),
2839 jt->om_in_use_count, jt->om_free_count, jt->om_free_provision);
2840 pop_count += jt->om_in_use_count + jt->om_free_count;
2841 }
2842 return pop_count;
2843 }
2844
2845 #ifndef PRODUCT
2846
2847 // Check if monitor belongs to the monitor cache
2848 // The list is grow-only so it's *relatively* safe to traverse
2849 // the list of extant blocks without taking a lock.
2850
2851 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
2852 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
2853 while (block != NULL) {
2854 assert(block->object() == CHAINMARKER, "must be a block header");
2855 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
2856 address mon = (address)monitor;
2857 address blk = (address)block;
2858 size_t diff = mon - blk;
2859 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
2860 return 1;
2861 }
2862 block = (PaddedObjectMonitor*)block->_next_om;
2863 }
2864 return 0;
2865 }
2866
2867 #endif
|
101 }
102
103 #else // ndef DTRACE_ENABLED
104
105 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon) {;}
106 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon) {;}
107
108 #endif // ndef DTRACE_ENABLED
109
110 // This exists only as a workaround of dtrace bug 6254741
111 int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
112 DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
113 return 0;
114 }
115
116 #define NINFLATIONLOCKS 256
117 static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];
118
119 // global list of blocks of monitors
120 PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL;
121 bool volatile ObjectSynchronizer::_is_async_deflation_requested = false;
122 bool volatile ObjectSynchronizer::_is_special_deflation_requested = false;
123 jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0;
124
125 // Global ObjectMonitor free list. Newly allocated and deflated
126 // ObjectMonitors are prepended here.
127 static ObjectMonitor* volatile g_free_list = NULL;
128 // Global ObjectMonitor in-use list. When a JavaThread is exiting,
129 // ObjectMonitors on its per-thread in-use list are prepended here.
130 static ObjectMonitor* volatile g_om_in_use_list = NULL;
131
132 static volatile int g_om_free_count = 0; // # on g_free_list
133 static volatile int g_om_in_use_count = 0; // # on g_om_in_use_list
134 static volatile int g_om_population = 0; // # Extant -- in circulation
135
136 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
137
138
139 // =====================> List Management functions
140
141 // Return true if the ObjectMonitor's next field is marked.
142 // Otherwise returns false.
143 static bool is_next_marked(ObjectMonitor* om) {
144 return ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & 0x1) != 0;
145 }
146
147 // Mark an ObjectMonitor* and return it. Note: the om parameter
148 // may or may not have been marked originally.
149 static ObjectMonitor* mark_om_ptr(ObjectMonitor* om) {
150 return (ObjectMonitor*)((intptr_t)om | 0x1);
151 }
152
153 // Mark the next field in an ObjectMonitor. If marking was successful,
154 // then the unmarked next field is returned via parameter and true is
155 // returned. Otherwise false is returned.
156 static bool mark_next(ObjectMonitor* om, ObjectMonitor** next_p) {
157 // Get current next field without any marking value.
158 ObjectMonitor* next = (ObjectMonitor*)
159 ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1);
160 if (Atomic::cmpxchg(mark_om_ptr(next), &om->_next_om, next) != next) {
161 return false; // Could not mark the next field or it was already marked.
162 }
163 *next_p = next;
164 return true;
165 }
166
167 // Loop until we mark the next field in an ObjectMonitor. The unmarked
168 // next field is returned.
169 static ObjectMonitor* mark_next_loop(ObjectMonitor* om) {
170 ObjectMonitor* next;
171 while (true) {
172 if (mark_next(om, &next)) {
173 // Marked om's next field so return the unmarked value.
174 return next;
175 }
176 }
177 }
178
179 // Set the next field in an ObjectMonitor to the specified value.
180 // The caller of set_next() must be the same thread that marked the
181 // ObjectMonitor.
182 static void set_next(ObjectMonitor* om, ObjectMonitor* value) {
183 OrderAccess::release_store(&om->_next_om, value);
184 }
185
186 // Mark the next field in the list head ObjectMonitor. If marking was
187 // successful, then the mid and the unmarked next field are returned
188 // via parameter and true is returned. Otherwise false is returned.
189 static bool mark_list_head(ObjectMonitor* volatile * list_p,
190 ObjectMonitor** mid_p, ObjectMonitor** next_p) {
191 while (true) {
192 ObjectMonitor* mid = OrderAccess::load_acquire(list_p);
193 if (mid == NULL) {
194 return false; // The list is empty so nothing to mark.
195 }
196 if (mark_next(mid, next_p)) {
197 if (OrderAccess::load_acquire(list_p) != mid) {
198 // The list head changed so we have to retry.
199 set_next(mid, *next_p); // unmark mid
200 continue;
201 }
202 // We marked next field to guard against races.
203 *mid_p = mid;
204 return true;
205 }
206 }
207 }
208
209 // Return the unmarked next field in an ObjectMonitor. Note: the next
210 // field may or may not have been marked originally.
211 static ObjectMonitor* unmarked_next(ObjectMonitor* om) {
212 return (ObjectMonitor*)((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1);
213 }
214
215 #if 0
216 // XXX - this is unused
217 // Unmark the next field in an ObjectMonitor. Requires that the next
218 // field be marked.
219 static void unmark_next(ObjectMonitor* om) {
220 ADIM_guarantee(is_next_marked(om), "next field must be marked: next=" INTPTR_FORMAT, p2i(om->_next_om));
221
222 ObjectMonitor* next = unmarked_next(om);
223 set_next(om, next);
224 }
225 #endif
226
227 volatile int visit_counter = 42;
228 static void chk_for_list_loop(ObjectMonitor* list, int count) {
229 if (!CheckMonitorLists) {
230 return;
231 }
232 int l_visit_counter = Atomic::add(1, &visit_counter);
233 int l_count = 0;
234 ObjectMonitor* prev = NULL;
235 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
236 if (mid->visit_marker == l_visit_counter) {
237 log_error(monitorinflation)("ERROR: prev=" INTPTR_FORMAT ", l_count=%d"
238 " refers to an ObjectMonitor that has"
239 " already been visited: mid=" INTPTR_FORMAT,
240 p2i(prev), l_count, p2i(mid));
241 fatal("list=" INTPTR_FORMAT " of %d items has a loop.", p2i(list), count);
242 }
243 mid->visit_marker = l_visit_counter;
244 prev = mid;
245 if (++l_count > count + 1024 * 1024) {
246 fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d",
247 p2i(list), count, l_count);
248 }
249 }
250 }
251
252 static void chk_om_not_on_list(ObjectMonitor* om, ObjectMonitor* list, int count) {
253 if (!CheckMonitorLists) {
254 return;
255 }
256 guarantee(list != om, "ERROR: om=" INTPTR_FORMAT " must not be head of the "
257 "list=" INTPTR_FORMAT ", count=%d", p2i(om), p2i(list), count);
258 int l_count = 0;
259 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
260 if (unmarked_next(mid) == om) {
261 log_error(monitorinflation)("ERROR: mid=" INTPTR_FORMAT ", l_count=%d"
262 " next_om refers to om=" INTPTR_FORMAT,
263 p2i(mid), l_count, p2i(om));
264 fatal("list=" INTPTR_FORMAT " of %d items has bad next_om value.",
265 p2i(list), count);
266 }
267 if (++l_count > count + 1024 * 1024) {
268 fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d",
269 p2i(list), count, l_count);
270 }
271 }
272 }
273
274 static void chk_om_elems_not_on_list(ObjectMonitor* elems, int elems_count,
275 ObjectMonitor* list, int list_count) {
276 if (!CheckMonitorLists) {
277 return;
278 }
279 chk_for_list_loop(elems, elems_count);
280 for (ObjectMonitor* mid = elems; mid != NULL; mid = unmarked_next(mid)) {
281 chk_om_not_on_list(mid, list, list_count);
282 }
283 }
284
285 // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is
286 // the last ObjectMonitor in the list and there are 'count' on the list.
287 // Also updates the specified *count_p.
288 static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail,
289 int count, ObjectMonitor* volatile* list_p,
290 volatile int* count_p) {
291 chk_for_list_loop(OrderAccess::load_acquire(list_p),
292 OrderAccess::load_acquire(count_p));
293 chk_om_elems_not_on_list(list, count, OrderAccess::load_acquire(list_p),
294 OrderAccess::load_acquire(count_p));
295 while (true) {
296 ObjectMonitor* cur = OrderAccess::load_acquire(list_p);
297 // Prepend list to *list_p.
298 ObjectMonitor* next = NULL;
299 if (!mark_next(tail, &next)) {
300 continue; // failed to mark next field so try it all again
301 }
302 set_next(tail, cur); // tail now points to cur (and unmarks tail)
303 if (cur == NULL) {
304 // No potential race with takers or other prependers since
305 // *list_p is empty.
306 if (Atomic::cmpxchg(list, list_p, cur) == cur) {
307 // Successfully switched *list_p to the list value.
308 Atomic::add(count, count_p);
309 break;
310 }
311 // Implied else: try it all again
312 } else {
313 // Try to mark next field to guard against races:
314 if (!mark_next(cur, &next)) {
315 continue; // failed to mark next field so try it all again
316 }
317 // We marked the next field so try to switch *list_p to the list value.
318 if (Atomic::cmpxchg(list, list_p, cur) != cur) {
319 // The list head has changed so unmark the next field and try again:
320 set_next(cur, next);
321 continue;
322 }
323 Atomic::add(count, count_p);
324 set_next(cur, next); // unmark next field
325 break;
326 }
327 }
328 }
329
330 // Prepend a newly allocated block of ObjectMonitors to g_block_list and
331 // g_free_list. Also updates g_om_population and g_om_free_count.
332 void ObjectSynchronizer::prepend_block_to_lists(PaddedObjectMonitor* new_blk) {
333 // First we handle g_block_list:
334 while (true) {
335 PaddedObjectMonitor* cur = OrderAccess::load_acquire(&g_block_list);
336 // Prepend new_blk to g_block_list. The first ObjectMonitor in
337 // a block is reserved for use as linkage to the next block.
338 OrderAccess::release_store(&new_blk[0]._next_om, cur);
339 if (Atomic::cmpxchg(new_blk, &g_block_list, cur) == cur) {
340 // Successfully switched g_block_list to the new_blk value.
341 Atomic::add(_BLOCKSIZE - 1, &g_om_population);
342 break;
343 }
344 // Implied else: try it all again
345 }
346
347 // Second we handle g_free_list:
348 prepend_list_to_common(new_blk + 1, &new_blk[_BLOCKSIZE - 1], _BLOCKSIZE - 1,
349 &g_free_list, &g_om_free_count);
350 }
351
352 // Prepend a list of ObjectMonitors to g_free_list. 'tail' is the last
353 // ObjectMonitor in the list and there are 'count' on the list. Also
354 // updates g_om_free_count.
355 static void prepend_list_to_g_free_list(ObjectMonitor* list,
356 ObjectMonitor* tail, int count) {
357 prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count);
358 }
359
360 // Prepend a list of ObjectMonitors to g_om_in_use_list. 'tail' is the last
361 // ObjectMonitor in the list and there are 'count' on the list. Also
362 // updates g_om_in_use_list.
363 static void prepend_list_to_g_om_in_use_list(ObjectMonitor* list,
364 ObjectMonitor* tail, int count) {
365 prepend_list_to_common(list, tail, count, &g_om_in_use_list, &g_om_in_use_count);
366 }
367
368 // Prepend an ObjectMonitor to the specified list. Also updates
369 // the specified counter.
370 static void prepend_to_common(ObjectMonitor* m, ObjectMonitor* volatile * list_p,
371 int volatile * count_p) {
372 chk_for_list_loop(OrderAccess::load_acquire(list_p),
373 OrderAccess::load_acquire(count_p));
374 chk_om_not_on_list(m, OrderAccess::load_acquire(list_p),
375 OrderAccess::load_acquire(count_p));
376
377 while (true) {
378 ObjectMonitor* cur = OrderAccess::load_acquire(list_p);
379 // Prepend ObjectMonitor to *list_p.
380 ObjectMonitor* next = NULL;
381 if (!mark_next(m, &next)) {
382 continue; // failed to mark next field so try it all again
383 }
384 set_next(m, cur); // m now points to cur (and unmarks m)
385 if (cur == NULL) {
386 // No potential race with other prependers since *list_p is empty.
387 if (Atomic::cmpxchg(m, list_p, cur) == cur) {
388 // Successfully switched *list_p to 'm'.
389 Atomic::inc(count_p);
390 break;
391 }
392 // Implied else: try it all again
393 } else {
394 // Try to mark next field to guard against races:
395 if (!mark_next(cur, &next)) {
396 continue; // failed to mark next field so try it all again
397 }
398 // We marked the next field so try to switch *list_p to 'm'.
399 if (Atomic::cmpxchg(m, list_p, cur) != cur) {
400 // The list head has changed so unmark the next field and try again:
401 set_next(cur, next);
402 continue;
403 }
404 Atomic::inc(count_p);
405 set_next(cur, next); // unmark next field
406 break;
407 }
408 }
409 }
410
411 // Prepend an ObjectMonitor to a per-thread om_free_list.
412 // Also updates the per-thread om_free_count.
413 static void prepend_to_om_free_list(Thread* self, ObjectMonitor* m) {
414 prepend_to_common(m, &self->om_free_list, &self->om_free_count);
415 }
416
417 // Prepend an ObjectMonitor to a per-thread om_in_use_list.
418 // Also updates the per-thread om_in_use_count.
419 static void prepend_to_om_in_use_list(Thread* self, ObjectMonitor* m) {
420 prepend_to_common(m, &self->om_in_use_list, &self->om_in_use_count);
421 }
422
423 // Take an ObjectMonitor from the start of the specified list. Also
424 // decrements the specified counter. Returns NULL if none are available.
425 static ObjectMonitor* take_from_start_of_common(ObjectMonitor* volatile * list_p,
426 int volatile * count_p) {
427 chk_for_list_loop(OrderAccess::load_acquire(list_p),
428 OrderAccess::load_acquire(count_p));
429
430 ObjectMonitor* next = NULL;
431 ObjectMonitor* take = NULL;
432 // Mark the list head to guard against A-B-A race:
433 if (!mark_list_head(list_p, &take, &next)) {
434 return NULL; // None are available.
435 }
436 // Switch marked list head to next (which unmarks the list head, but
437 // leaves take marked):
438 OrderAccess::release_store(list_p, next);
439 Atomic::dec(count_p);
440 // Unmark take, but leave the next value for any lagging list
441 // walkers. It will get cleaned up when take is prepended to
442 // the in-use list:
443 set_next(take, next);
444 return take;
445 }
446
447 // Take an ObjectMonitor from the start of the global free-list. Also
448 // updates g_om_free_count. Returns NULL if none are available.
449 static ObjectMonitor* take_from_start_of_g_free_list() {
450 return take_from_start_of_common(&g_free_list, &g_om_free_count);
451 }
452
453 // Take an ObjectMonitor from the start of a per-thread free-list.
454 // Also updates om_free_count. Returns NULL if none are available.
455 static ObjectMonitor* take_from_start_of_om_free_list(Thread* self) {
456 return take_from_start_of_common(&self->om_free_list, &self->om_free_count);
457 }
458
459
460 // =====================> Quick functions
461
462 // The quick_* forms are special fast-path variants used to improve
463 // performance. In the simplest case, a "quick_*" implementation could
464 // simply return false, in which case the caller will perform the necessary
465 // state transitions and call the slow-path form.
466 // The fast-path is designed to handle frequently arising cases in an efficient
467 // manner and is just a degenerate "optimistic" variant of the slow-path.
468 // returns true -- to indicate the call was satisfied.
469 // returns false -- to indicate the call needs the services of the slow-path.
470 // A no-loitering ordinance is in effect for code in the quick_* family
471 // operators: safepoints or indefinite blocking (blocking that might span a
472 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
473 // entry.
474 //
475 // Consider: An interesting optimization is to have the JIT recognize the
476 // following common idiom:
477 // synchronized (someobj) { .... ; notify(); }
478 // That is, we find a notify() or notifyAll() call that immediately precedes
479 // the monitorexit operation. In that case the JIT could fuse the operations
886 //
887 // Performance concern:
888 // OrderAccess::storestore() calls release() which at one time stored 0
889 // into the global volatile OrderAccess::dummy variable. This store was
890 // unnecessary for correctness. Many threads storing into a common location
891 // causes considerable cache migration or "sloshing" on large SMP systems.
892 // As such, I avoided using OrderAccess::storestore(). In some cases
893 // OrderAccess::fence() -- which incurs local latency on the executing
894 // processor -- is a better choice as it scales on SMP systems.
895 //
896 // See http://blogs.oracle.com/dave/entry/biased_locking_in_hotspot for
897 // a discussion of coherency costs. Note that all our current reference
898 // platforms provide strong ST-ST order, so the issue is moot on IA32,
899 // x64, and SPARC.
900 //
901 // As a general policy we use "volatile" to control compiler-based reordering
902 // and explicit fences (barriers) to control for architectural reordering
903 // performed by the CPU(s) or platform.
904
905 struct SharedGlobals {
906 char _pad_prefix[OM_CACHE_LINE_SIZE];
907 // These are highly shared mostly-read variables.
908 // To avoid false-sharing they need to be the sole occupants of a cache line.
909 volatile int stw_random;
910 volatile int stw_cycle;
911 DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(volatile int) * 2);
912 // Hot RW variable -- Sequester to avoid false-sharing
913 volatile int hc_sequence;
914 DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile int));
915 };
916
917 static SharedGlobals GVars;
918 static int MonitorScavengeThreshold = 1000000;
919 static volatile int ForceMonitorScavenge = 0; // Scavenge required and pending
920
921 static markWord read_stable_mark(oop obj) {
922 markWord mark = obj->mark();
923 if (!mark.is_being_inflated()) {
924 return mark; // normal fast-path return
925 }
926
927 int its = 0;
928 for (;;) {
929 markWord mark = obj->mark();
930 if (!mark.is_being_inflated()) {
931 return mark; // normal fast-path return
932 }
933
934 // The object is being inflated by some other thread.
1302 // Cannot have assertion since this object may have been
1303 // locked by another thread when reaching here.
1304 // assert(mark.is_neutral(), "sanity check");
1305
1306 return NULL;
1307 }
1308 }
1309
1310 // Visitors ...
1311
1312 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
1313 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
1314 while (block != NULL) {
1315 assert(block->object() == CHAINMARKER, "must be a block header");
1316 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
1317 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
1318 if (mid->is_active()) {
1319 ObjectMonitorHandle omh(mid);
1320
1321 if (mid->object() == NULL ||
1322 (AsyncDeflateIdleMonitors && mid->ref_count() < 0)) {
1323 // Only process with closure if the object is set.
1324 // For async deflation, race here if monitor is not owned!
1325 // The above ref_count bump (in ObjectMonitorHandle ctr)
1326 // will cause subsequent async deflation to skip it.
1327 // However, previous or concurrent async deflation is a race
1328 // so skip this ObjectMonitor if it is being async deflated.
1329 continue;
1330 }
1331 closure->do_monitor(mid);
1332 }
1333 }
1334 // unmarked_next() is not needed with g_block_list (no next field marking).
1335 block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om);
1336 }
1337 }
1338
1339 static bool monitors_used_above_threshold() {
1340 if (OrderAccess::load_acquire(&g_om_population) == 0) {
1341 return false;
1342 }
1343 if (MonitorUsedDeflationThreshold > 0) {
1344 int monitors_used = OrderAccess::load_acquire(&g_om_population) -
1345 OrderAccess::load_acquire(&g_om_free_count);
1346 int monitor_usage = (monitors_used * 100LL) /
1347 OrderAccess::load_acquire(&g_om_population);
1348 return monitor_usage > MonitorUsedDeflationThreshold;
1349 }
1350 return false;
1351 }
1352
1353 // Returns true if MonitorBound is set (> 0) and if the specified
1354 // cnt is > MonitorBound. Otherwise returns false.
1355 static bool is_MonitorBound_exceeded(const int cnt) {
1356 const int mx = MonitorBound;
1357 return mx > 0 && cnt > mx;
1358 }
1359
1360 bool ObjectSynchronizer::is_async_deflation_needed() {
1361 if (!AsyncDeflateIdleMonitors) {
1362 return false;
1363 }
1364 if (is_async_deflation_requested()) {
1365 // Async deflation request.
1366 return true;
1367 }
1368 if (AsyncDeflationInterval > 0 &&
1369 time_since_last_async_deflation_ms() > AsyncDeflationInterval &&
1370 monitors_used_above_threshold()) {
1371 // It's been longer than our specified deflate interval and there
1372 // are too many monitors in use. We don't deflate more frequently
1373 // than AsyncDeflationInterval (unless is_async_deflation_requested)
1374 // in order to not swamp the ServiceThread.
1375 _last_async_deflation_time_ns = os::javaTimeNanos();
1376 return true;
1377 }
1378 if (is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) -
1379 OrderAccess::load_acquire(&g_om_free_count))) {
1380 // Not enough ObjectMonitors on the global free list.
1381 return true;
1382 }
1383 return false;
1384 }
1385
1386 bool ObjectSynchronizer::is_safepoint_deflation_needed() {
1387 if (!AsyncDeflateIdleMonitors) {
1388 if (monitors_used_above_threshold()) {
1389 // Too many monitors in use.
1390 return true;
1391 }
1392 return false;
1393 }
1394 if (is_special_deflation_requested()) {
1395 // For AsyncDeflateIdleMonitors only do a safepoint deflation
1396 // if there is a special deflation request.
1397 return true;
1398 }
1399 return false;
1400 }
1401
1402 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1403 return (os::javaTimeNanos() - _last_async_deflation_time_ns) / (NANOUNITS / MILLIUNITS);
1404 }
1405
1406 void ObjectSynchronizer::oops_do(OopClosure* f) {
1407 // We only scan the global used list here (for moribund threads), and
1408 // the thread-local monitors in Thread::oops_do().
1409 global_used_oops_do(f);
1410 }
1411
1412 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
1413 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1414 list_oops_do(OrderAccess::load_acquire(&g_om_in_use_list), OrderAccess::load_acquire(&g_om_in_use_count), f);
1415 }
1416
1417 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
1418 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1419 list_oops_do(OrderAccess::load_acquire(&thread->om_in_use_list), OrderAccess::load_acquire(&thread->om_in_use_count), f);
1420 }
1421
1422 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, int count, OopClosure* f) {
1423 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1424 chk_for_list_loop(list, count);
1425 // The oops_do() phase does not overlap with monitor deflation
1426 // so no need to update the ObjectMonitor's ref_count for this
1427 // ObjectMonitor* use.
1428 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
1429 if (mid->object() != NULL) {
1430 f->do_oop((oop*)mid->object_addr());
1431 }
1432 }
1433 }
1434
1435
1436 // -----------------------------------------------------------------------------
1437 // ObjectMonitor Lifecycle
1438 // -----------------------
1439 // Inflation unlinks monitors from the global g_free_list and
1440 // associates them with objects. Deflation -- which occurs at
1441 // STW-time -- disassociates idle monitors from objects. Such
1442 // scavenged monitors are returned to the g_free_list.
1443 //
1444 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
1445 //
1446 // Lifecycle:
1447 // -- unassigned and on the global free list
1448 // -- unassigned and on a thread's private om_free_list
1449 // -- assigned to an object. The object is inflated and the mark refers
1450 // to the objectmonitor.
1451
1452
1453 // Constraining monitor pool growth via MonitorBound ...
1454 //
1455 // If MonitorBound is not set (<= 0), MonitorBound checks are disabled.
1456 //
1457 // When safepoint deflation is being used (!AsyncDeflateIdleMonitors):
1458 // The monitor pool is grow-only. We scavenge at STW safepoint-time, but the
1459 // the rate of scavenging is driven primarily by GC. As such, we can find
1460 // an inordinate number of monitors in circulation.
1461 // To avoid that scenario we can artificially induce a STW safepoint
1462 // if the pool appears to be growing past some reasonable bound.
1463 // Generally we favor time in space-time tradeoffs, but as there's no
1466 // we could just loop. In addition, if MonitorBound is set to a low value
1467 // we'll incur more safepoints, which are harmful to performance.
1468 // See also: GuaranteedSafepointInterval
1469 //
1470 // The current implementation uses asynchronous VM operations.
1471 //
1472 // When safepoint deflation is being used and MonitorBound is set, the
1473 // boundry applies to
1474 // (g_om_population - g_om_free_count)
1475 // i.e., if there are not enough ObjectMonitors on the global free list,
1476 // then a safepoint deflation is induced. Picking a good MonitorBound value
1477 // is non-trivial.
1478 //
1479 // When async deflation is being used:
1480 // The monitor pool is still grow-only. Async deflation is requested
1481 // by a safepoint's cleanup phase or by the ServiceThread at periodic
1482 // intervals when is_async_deflation_needed() returns true. In
1483 // addition to other policies that are checked, if there are not
1484 // enough ObjectMonitors on the global free list, then
1485 // is_async_deflation_needed() will return true. The ServiceThread
1486 // calls deflate_global_idle_monitors_using_JT() and also calls
1487 // deflate_per_thread_idle_monitors_using_JT() as needed.
1488
1489 static void InduceScavenge(Thread* self, const char * Whence) {
1490 assert(!AsyncDeflateIdleMonitors, "is not used by async deflation");
1491
1492 // Induce STW safepoint to trim monitors
1493 // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
1494 // More precisely, trigger an asynchronous STW safepoint as the number
1495 // of active monitors passes the specified threshold.
1496 // TODO: assert thread state is reasonable
1497
1498 if (ForceMonitorScavenge == 0 && Atomic::xchg (1, &ForceMonitorScavenge) == 0) {
1499 // Induce a 'null' safepoint to scavenge monitors
1500 // Must VM_Operation instance be heap allocated as the op will be enqueue and posted
1501 // to the VMthread and have a lifespan longer than that of this activation record.
1502 // The VMThread will delete the op when completed.
1503 VMThread::execute(new VM_ScavengeMonitors());
1504 }
1505 }
1506
1507 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self,
1508 const InflateCause cause) {
1509 // A large MAXPRIVATE value reduces both list lock contention
1510 // and list coherency traffic, but also tends to increase the
1511 // number of ObjectMonitors in circulation as well as the STW
1512 // scavenge costs. As usual, we lean toward time in space-time
1513 // tradeoffs.
1514 const int MAXPRIVATE = 1024;
1515
1516 stringStream ss;
1517 for (;;) {
1518 ObjectMonitor* m;
1519
1520 // 1: try to allocate from the thread's local om_free_list.
1521 // Threads will attempt to allocate first from their local list, then
1522 // from the global list, and only after those attempts fail will the
1523 // thread attempt to instantiate new monitors. Thread-local free lists
1524 // improve allocation latency, as well as reducing coherency traffic
1525 // on the shared global list.
1526 m = take_from_start_of_om_free_list(self);
1527 if (m != NULL) {
1528 guarantee(m->object() == NULL, "invariant");
1529 m->set_allocation_state(ObjectMonitor::New);
1530 prepend_to_om_in_use_list(self, m);
1531 return m;
1532 }
1533
1534 // 2: try to allocate from the global g_free_list
1535 // CONSIDER: use muxTry() instead of muxAcquire().
1536 // If the muxTry() fails then drop immediately into case 3.
1537 // If we're using thread-local free lists then try
1538 // to reprovision the caller's free list.
1539 if (OrderAccess::load_acquire(&g_free_list) != NULL) {
1540 // Reprovision the thread's om_free_list.
1541 // Use bulk transfers to reduce the allocation rate and heat
1542 // on various locks.
1543 for (int i = self->om_free_provision; --i >= 0;) {
1544 ObjectMonitor* take = take_from_start_of_g_free_list();
1545 if (take == NULL) {
1546 break; // No more are available.
1547 }
1548 guarantee(take->object() == NULL, "invariant");
1549 if (AsyncDeflateIdleMonitors) {
1550 // We allowed 3 field values to linger during async deflation.
1551 // We clear header and restore ref_count here, but we leave
1552 // owner == DEFLATER_MARKER so the simple C2 ObjectMonitor
1553 // enter optimization can no longer race with async deflation
1554 // and reuse.
1555 take->set_header(markWord::zero());
1556 if (take->ref_count() < 0) {
1557 // Add back max_jint to restore the ref_count field to its
1558 // proper value.
1559 Atomic::add(max_jint, &take->_ref_count);
1560
1561 assert(take->ref_count() >= 0, "must not be negative: ref_count=%d",
1562 take->ref_count());
1563 }
1564 }
1565 take->Recycle();
1566 assert(take->is_free(), "invariant");
1567 om_release(self, take, false);
1568 }
1569 self->om_free_provision += 1 + (self->om_free_provision/2);
1570 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1571
1572 if (!AsyncDeflateIdleMonitors &&
1573 is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) -
1574 OrderAccess::load_acquire(&g_om_free_count))) {
1575 // Not enough ObjectMonitors on the global free list.
1576 // We can't safely induce a STW safepoint from om_alloc() as our thread
1577 // state may not be appropriate for such activities and callers may hold
1578 // naked oops, so instead we defer the action.
1579 InduceScavenge(self, "om_alloc");
1580 }
1581 continue;
1582 }
1583
1584 // 3: allocate a block of new ObjectMonitors
1585 // Both the local and global free lists are empty -- resort to malloc().
1586 // In the current implementation ObjectMonitors are TSM - immortal.
1587 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1588 // each ObjectMonitor to start at the beginning of a cache line,
1589 // so we use align_up().
1590 // A better solution would be to use C++ placement-new.
1591 // BEWARE: As it stands currently, we don't run the ctors!
1592 assert(_BLOCKSIZE > 1, "invariant");
1593 size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE;
1594 PaddedObjectMonitor* temp;
1595 size_t aligned_size = neededsize + (OM_CACHE_LINE_SIZE - 1);
1596 void* real_malloc_addr = (void*)NEW_C_HEAP_ARRAY(char, aligned_size,
1597 mtInternal);
1598 temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, OM_CACHE_LINE_SIZE);
1599
1600 // NOTE: (almost) no way to recover if allocation failed.
1601 // We might be able to induce a STW safepoint and scavenge enough
1602 // ObjectMonitors to permit progress.
1603 if (temp == NULL) {
1604 vm_exit_out_of_memory(neededsize, OOM_MALLOC_ERROR,
1605 "Allocate ObjectMonitors");
1606 }
1607 (void)memset((void *) temp, 0, neededsize);
1608
1609 // Format the block.
1610 // initialize the linked list, each monitor points to its next
1611 // forming the single linked free list, the very first monitor
1612 // will points to next block, which forms the block list.
1613 // The trick of using the 1st element in the block as g_block_list
1614 // linkage should be reconsidered. A better implementation would
1615 // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; }
1616
1617 for (int i = 1; i < _BLOCKSIZE; i++) {
1618 OrderAccess::release_store(&temp[i]._next_om, (ObjectMonitor*)&temp[i+1]);
1619 assert(temp[i].is_free(), "invariant");
1620 }
1621
1622 // terminate the last monitor as the end of list
1623 OrderAccess::release_store(&temp[_BLOCKSIZE - 1]._next_om, (ObjectMonitor*)NULL);
1624
1625 // Element [0] is reserved for global list linkage
1626 temp[0].set_object(CHAINMARKER);
1627
1628 // Consider carving out this thread's current request from the
1629 // block in hand. This avoids some lock traffic and redundant
1630 // list activity.
1631
1632 prepend_block_to_lists(temp);
1633 }
1634 }
1635
1636 // Place "m" on the caller's private per-thread om_free_list.
1637 // In practice there's no need to clamp or limit the number of
1638 // monitors on a thread's om_free_list as the only non-allocation time
1639 // we'll call om_release() is to return a monitor to the free list after
1640 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1641 // accumulate on a thread's free list.
1642 //
1643 // Key constraint: all ObjectMonitors on a thread's free list and the global
1644 // free list must have their object field set to null. This prevents the
1645 // scavenger -- deflate_monitor_list() or deflate_monitor_list_using_JT()
1646 // -- from reclaiming them while we are trying to release them.
1647
1648 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1649 bool from_per_thread_alloc) {
1650 guarantee(m->header().value() == 0, "invariant");
1651 guarantee(m->object() == NULL, "invariant");
1652 stringStream ss;
1653 guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1654 "%s, recursions=" INTPTR_FORMAT, m->is_busy_to_string(&ss),
1655 m->_recursions);
1656 m->set_allocation_state(ObjectMonitor::Free);
1657 // _next_om is used for both per-thread in-use and free lists so
1658 // we have to remove 'm' from the in-use list first (as needed).
1659 if (from_per_thread_alloc) {
1660 // Need to remove 'm' from om_in_use_list.
1661 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
1662 // protocol because async deflation can do list deletions in parallel.
1663 ObjectMonitor* cur_mid_in_use = NULL;
1664 ObjectMonitor* mid = NULL;
1665 ObjectMonitor* next = NULL;
1666 bool extracted = false;
1667
1668 if (!mark_list_head(&self->om_in_use_list, &mid, &next)) {
1669 fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self));
1670 }
1671 while (true) {
1672 if (m == mid) {
1673 // We found 'm' on the per-thread in-use list so try to extract it.
1674 // First try the list head:
1675 if (Atomic::cmpxchg(next, &self->om_in_use_list, mid) != mid) {
1676 // We could not switch the list head to next.
1677 ObjectMonitor* marked_mid = mark_om_ptr(mid);
1678 // Switch cur_mid_in_use's next field to next (which also
1679 // unmarks cur_mid_in_use):
1680 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
1681 if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, marked_mid)
1682 != marked_mid) {
1683 // We could not switch cur_mid_in_use's next field. This
1684 // should not be possible since it was marked so we:
1685 fatal("mid=" INTPTR_FORMAT " must be referred to by the list "
1686 "head: &om_in_use_list=" INTPTR_FORMAT " or by "
1687 "cur_mid_in_use's next field: cur_mid_in_use=" INTPTR_FORMAT
1688 ", next_om=" INTPTR_FORMAT, p2i(mid),
1689 p2i((ObjectMonitor**)&self->om_in_use_list),
1690 p2i(cur_mid_in_use), p2i(cur_mid_in_use->_next_om));
1691 }
1692 }
1693 extracted = true;
1694 Atomic::dec(&self->om_in_use_count);
1695 // Unmark mid, but leave the next value for any lagging list
1696 // walkers. It will get cleaned up when mid is prepended to
1697 // the thread's free list:
1698 set_next(mid, next);
1699 break;
1700 }
1701 if (cur_mid_in_use != NULL) {
1702 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
1703 }
1704 // The next cur_mid_in_use keeps mid's marked next field so
1705 // that it is stable for a possible next field change. It
1706 // cannot be deflated while it is marked.
1707 cur_mid_in_use = mid;
1708 mid = next;
1709 if (mid == NULL) {
1710 // Reached end of the list and didn't find m so:
1711 fatal("must find m=" INTPTR_FORMAT "on om_in_use_list=" INTPTR_FORMAT,
1712 p2i(m), p2i(self->om_in_use_list));
1713 }
1714 // Mark mid's next field so we can possibly extract it:
1715 next = mark_next_loop(mid);
1716 }
1717 }
1718
1719 prepend_to_om_free_list(self, m);
1720 guarantee(m->is_free(), "invariant");
1721 }
1722
1723 // Return ObjectMonitors on a moribund thread's free and in-use
1724 // lists to the appropriate global lists. The ObjectMonitors on the
1725 // per-thread in-use list may still be in use by other threads.
1726 //
1727 // We currently call om_flush() from Threads::remove() before the
1728 // thread has been excised from the thread list and is no longer a
1729 // mutator. This means that om_flush() cannot run concurrently with
1730 // a safepoint and interleave with deflate_idle_monitors(). In
1731 // particular, this ensures that the thread's in-use monitors are
1732 // scanned by a GC safepoint, either via Thread::oops_do() (before
1733 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1734 // om_flush() is called).
1735 //
1736 // With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT()
1737 // and deflate_per_thread_idle_monitors_using_JT() (in another thread) can
1738 // run at the same time as om_flush() so we have to follow a careful
1739 // protocol to prevent list corruption.
1740
1741 void ObjectSynchronizer::om_flush(Thread* self) {
1742 // This function can race with an async deflater thread. Since
1743 // deflation has to process the per-thread in-use list before
1744 // prepending the deflated ObjectMonitors to the global free list,
1745 // we process the per-thread lists in the same order to prevent
1746 // ordering races.
1747 int in_use_count = 0;
1748 ObjectMonitor* in_use_list = NULL;
1749 ObjectMonitor* in_use_tail = NULL;
1750 ObjectMonitor* next = NULL;
1751
1752 // An async deflation thread checks to see if the target thread
1753 // is exiting, but if it has made it past that check before we
1754 // started exiting, then it is racing to get to the in-use list.
1755 if (mark_list_head(&self->om_in_use_list, &in_use_list, &next)) {
1756 chk_for_list_loop(in_use_list, OrderAccess::load_acquire(&self->om_in_use_count));
1757 // At this point, we have marked the in-use list head so an
1758 // async deflation thread cannot come in after us. If an async
1759 // deflation thread is ahead of us, then we'll detect that and
1760 // wait for it to finish its work.
1761 //
1762 // The thread is going away, however the ObjectMonitors on the
1763 // om_in_use_list may still be in-use by other threads. Link
1764 // them to in_use_tail, which will be linked into the global
1765 // in-use list g_om_in_use_list below.
1766 //
1767 // Account for the in-use list head before the loop since it is
1768 // already marked (by this thread):
1769 in_use_tail = in_use_list;
1770 in_use_count++;
1771 for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL;) {
1772 if (is_next_marked(cur_om)) {
1773 // This next field is marked so there must be an async deflater
1774 // thread ahead of us so we'll give it a chance to finish.
1775 while (is_next_marked(cur_om)) {
1776 os::naked_short_sleep(1);
1777 }
1778 // Refetch the possibly changed next field and try again.
1779 cur_om = unmarked_next(in_use_tail);
1780 continue;
1781 }
1782 if (!cur_om->is_active()) {
1783 // cur_om was deflated and the allocation state was changed
1784 // to Free while it was marked. We happened to see it just
1785 // after it was unmarked (and added to the free list).
1786 // Refetch the possibly changed next field and try again.
1787 cur_om = unmarked_next(in_use_tail);
1788 continue;
1789 }
1790 in_use_tail = cur_om;
1791 in_use_count++;
1792 cur_om = unmarked_next(cur_om);
1793 }
1794 guarantee(in_use_tail != NULL, "invariant");
1795 int l_om_in_use_count = OrderAccess::load_acquire(&self->om_in_use_count);
1796 ADIM_guarantee(l_om_in_use_count == in_use_count, "in-use counts don't "
1797 "match: l_om_in_use_count=%d, in_use_count=%d",
1798 l_om_in_use_count, in_use_count);
1799 // Clear the in-use count before unmarking the in-use list head
1800 // to avoid races:
1801 OrderAccess::release_store(&self->om_in_use_count, 0);
1802 // Clear the in-use list head (which also unmarks it):
1803 OrderAccess::release_store(&self->om_in_use_list, (ObjectMonitor*)NULL);
1804 // Unmark the disconnected list head:
1805 set_next(in_use_list, next);
1806 }
1807
1808 int free_count = 0;
1809 ObjectMonitor* free_list = OrderAccess::load_acquire(&self->om_free_list);
1810 ObjectMonitor* free_tail = NULL;
1811 if (free_list != NULL) {
1812 chk_for_list_loop(free_list, OrderAccess::load_acquire(&self->om_free_count));
1813 // The thread is going away. Set 'free_tail' to the last per-thread free
1814 // monitor which will be linked to g_free_list below.
1815 stringStream ss;
1816 for (ObjectMonitor* s = free_list; s != NULL; s = unmarked_next(s)) {
1817 free_count++;
1818 free_tail = s;
1819 guarantee(s->object() == NULL, "invariant");
1820 guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss));
1821 }
1822 guarantee(free_tail != NULL, "invariant");
1823 int l_om_free_count = OrderAccess::load_acquire(&self->om_free_count);
1824 ADIM_guarantee(l_om_free_count == free_count, "free counts don't match: "
1825 "l_om_free_count=%d, free_count=%d", l_om_free_count,
1826 free_count);
1827 OrderAccess::release_store(&self->om_free_list, (ObjectMonitor*)NULL);
1828 OrderAccess::release_store(&self->om_free_count, 0);
1829 }
1830
1831 if (free_tail != NULL) {
1832 prepend_list_to_g_free_list(free_list, free_tail, free_count);
1833 }
1834
1835 if (in_use_tail != NULL) {
1836 prepend_list_to_g_om_in_use_list(in_use_list, in_use_tail, in_use_count);
1837 }
1838
1839 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1840 LogStreamHandle(Info, monitorinflation) lsh_info;
1841 LogStream* ls = NULL;
1842 if (log_is_enabled(Debug, monitorinflation)) {
1843 ls = &lsh_debug;
1844 } else if ((free_count != 0 || in_use_count != 0) &&
1845 log_is_enabled(Info, monitorinflation)) {
1846 ls = &lsh_info;
1847 }
1848 if (ls != NULL) {
1849 ls->print_cr("om_flush: jt=" INTPTR_FORMAT ", free_count=%d"
1850 ", in_use_count=%d" ", om_free_provision=%d",
1851 p2i(self), free_count, in_use_count, self->om_free_provision);
1852 }
1853 }
1854
1855 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1856 const oop obj,
1857 ObjectSynchronizer::InflateCause cause) {
1858 assert(event != NULL, "invariant");
1935 //
1936 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1937 // to install INFLATING into the mark word. We originally installed INFLATING,
1938 // allocated the objectmonitor, and then finally STed the address of the
1939 // objectmonitor into the mark. This was correct, but artificially lengthened
1940 // the interval in which INFLATED appeared in the mark, thus increasing
1941 // the odds of inflation contention.
1942 //
1943 // We now use per-thread private objectmonitor free lists.
1944 // These list are reprovisioned from the global free list outside the
1945 // critical INFLATING...ST interval. A thread can transfer
1946 // multiple objectmonitors en-mass from the global free list to its local free list.
1947 // This reduces coherency traffic and lock contention on the global free list.
1948 // Using such local free lists, it doesn't matter if the om_alloc() call appears
1949 // before or after the CAS(INFLATING) operation.
1950 // See the comments in om_alloc().
1951
1952 LogStreamHandle(Trace, monitorinflation) lsh;
1953
1954 if (mark.has_locker()) {
1955 ObjectMonitor* m = om_alloc(self, cause);
1956 // Optimistically prepare the objectmonitor - anticipate successful CAS
1957 // We do this before the CAS in order to minimize the length of time
1958 // in which INFLATING appears in the mark.
1959 m->Recycle();
1960 m->_Responsible = NULL;
1961 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // Consider: maintain by type/class
1962
1963 markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1964 if (cmp != mark) {
1965 om_release(self, m, true);
1966 continue; // Interference -- just retry
1967 }
1968
1969 // We've successfully installed INFLATING (0) into the mark-word.
1970 // This is the only case where 0 will appear in a mark-word.
1971 // Only the singular thread that successfully swings the mark-word
1972 // to 0 can perform (or more precisely, complete) inflation.
1973 //
1974 // Why do we CAS a 0 into the mark-word instead of just CASing the
1975 // mark-word from the stack-locked value directly to the new inflated state?
2033 }
2034 if (event.should_commit()) {
2035 post_monitor_inflate_event(&event, object, cause);
2036 }
2037 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
2038 return;
2039 }
2040
2041 // CASE: neutral
2042 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
2043 // If we know we're inflating for entry it's better to inflate by swinging a
2044 // pre-locked ObjectMonitor pointer into the object header. A successful
2045 // CAS inflates the object *and* confers ownership to the inflating thread.
2046 // In the current implementation we use a 2-step mechanism where we CAS()
2047 // to inflate and then CAS() again to try to swing _owner from NULL to self.
2048 // An inflateTry() method that we could call from fast_enter() and slow_enter()
2049 // would be useful.
2050
2051 // Catch if the object's header is not neutral (not locked and
2052 // not marked is what we care about here).
2053 ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT,mark.value());
2054 ObjectMonitor* m = om_alloc(self, cause);
2055 // prepare m for installation - set monitor to initial state
2056 m->Recycle();
2057 m->set_header(mark);
2058 // If we leave _owner == DEFLATER_MARKER here, then the simple C2
2059 // ObjectMonitor enter optimization can no longer race with async
2060 // deflation and reuse.
2061 m->set_object(object);
2062 m->_Responsible = NULL;
2063 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
2064
2065 omh_p->set_om_ptr(m);
2066 assert(m->is_new(), "freshly allocated monitor must be new");
2067 m->set_allocation_state(ObjectMonitor::Old);
2068
2069 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
2070 guarantee(!m->owner_is_DEFLATER_MARKER() || m->ref_count() >= 0,
2071 "race between deflation and om_release() with m=" INTPTR_FORMAT
2072 ", _owner=" INTPTR_FORMAT ", ref_count=%d", p2i(m),
2073 p2i(m->_owner), m->ref_count());
2074 m->set_header(markWord::zero());
2075 m->set_object(NULL);
2076 m->Recycle();
2077 omh_p->set_om_ptr(NULL);
2078 // om_release() will reset the allocation state
2079 om_release(self, m, true);
2080 m = NULL;
2081 continue;
2082 // interference - the markword changed - just retry.
2083 // The state-transitions are one-way, so there's no chance of
2084 // live-lock -- "Inflated" is an absorbing state.
2085 }
2086
2087 // Hopefully the performance counters are allocated on distinct
2088 // cache lines to avoid false sharing on MP systems ...
2089 OM_PERFDATA_OP(Inflations, inc());
2090 if (log_is_enabled(Trace, monitorinflation)) {
2091 ResourceMark rm(self);
2092 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
2093 INTPTR_FORMAT ", type='%s'", p2i(object),
2112 // These operations are called at all safepoints, immediately after mutators
2113 // are stopped, but before any objects have moved. Collectively they traverse
2114 // the population of in-use monitors, deflating where possible. The scavenged
2115 // monitors are returned to the global monitor free list.
2116 //
2117 // Beware that we scavenge at *every* stop-the-world point. Having a large
2118 // number of monitors in-use could negatively impact performance. We also want
2119 // to minimize the total # of monitors in circulation, as they incur a small
2120 // footprint penalty.
2121 //
2122 // Perversely, the heap size -- and thus the STW safepoint rate --
2123 // typically drives the scavenge rate. Large heaps can mean infrequent GC,
2124 // which in turn can mean large(r) numbers of ObjectMonitors in circulation.
2125 // This is an unfortunate aspect of this design.
2126 //
2127 // For async deflation:
2128 // If a special deflation request is made, then the safepoint based
2129 // deflation mechanism is used. Otherwise, an async deflation request
2130 // is registered with the ServiceThread and it is notified.
2131
2132 void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* counters) {
2133 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2134
2135 // The per-thread in-use lists are handled in
2136 // ParallelSPCleanupThreadClosure::do_thread().
2137
2138 if (!AsyncDeflateIdleMonitors || is_special_deflation_requested()) {
2139 // Use the older mechanism for the global in-use list or if a
2140 // special deflation has been requested before the safepoint.
2141 ObjectSynchronizer::deflate_idle_monitors(counters);
2142 return;
2143 }
2144
2145 log_debug(monitorinflation)("requesting async deflation of idle monitors.");
2146 // Request deflation of idle monitors by the ServiceThread:
2147 set_is_async_deflation_requested(true);
2148 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
2149 ml.notify_all();
2150 }
2151
2152 // Deflate a single monitor if not in-use
2153 // Return true if deflated, false if in-use
2154 bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj,
2155 ObjectMonitor** free_head_p,
2156 ObjectMonitor** free_tail_p) {
2157 bool deflated;
2158 // Normal case ... The monitor is associated with obj.
2159 const markWord mark = obj->mark();
2160 guarantee(mark == markWord::encode(mid), "should match: mark="
2161 INTPTR_FORMAT ", encoded mid=" INTPTR_FORMAT, mark.value(),
2180 "object=" INTPTR_FORMAT ", mark="
2181 INTPTR_FORMAT ", type='%s'", p2i(obj),
2182 mark.value(), obj->klass()->external_name());
2183 }
2184
2185 // Restore the header back to obj
2186 obj->release_set_mark(dmw);
2187 if (AsyncDeflateIdleMonitors) {
2188 // clear() expects the owner field to be NULL and we won't race
2189 // with the simple C2 ObjectMonitor enter optimization since
2190 // we're at a safepoint.
2191 mid->set_owner(NULL);
2192 }
2193 mid->clear();
2194
2195 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
2196 p2i(mid->object()));
2197 assert(mid->is_free(), "invariant");
2198
2199 // Move the deflated ObjectMonitor to the working free list
2200 // defined by free_head_p and free_tail_p. No races on this list
2201 // so no need for load_acquire() or store_release().
2202 if (*free_head_p == NULL) *free_head_p = mid;
2203 if (*free_tail_p != NULL) {
2204 // We append to the list so the caller can use mid->_next_om
2205 // to fix the linkages in its context.
2206 ObjectMonitor* prevtail = *free_tail_p;
2207 // Should have been cleaned up by the caller:
2208 // Note: Should not have to mark prevtail here since we're at a
2209 // safepoint and ObjectMonitors on the local free list should
2210 // not be accessed in parallel.
2211 assert(prevtail->_next_om == NULL, "must be NULL: _next_om="
2212 INTPTR_FORMAT, p2i(prevtail->_next_om));
2213 set_next(prevtail, mid);
2214 }
2215 *free_tail_p = mid;
2216 // At this point, mid->_next_om still refers to its current
2217 // value and another ObjectMonitor's _next_om field still
2218 // refers to this ObjectMonitor. Those linkages have to be
2219 // cleaned up by the caller who has the complete context.
2220 deflated = true;
2221 }
2222 return deflated;
2223 }
2224
2225 // Deflate the specified ObjectMonitor if not in-use using a JavaThread.
2226 // Returns true if it was deflated and false otherwise.
2227 //
2228 // The async deflation protocol sets owner to DEFLATER_MARKER and
2229 // makes ref_count negative as signals to contending threads that
2230 // an async deflation is in progress. There are a number of checks
2231 // as part of the protocol to make sure that the calling thread has
2232 // not lost the race to a contending thread or to a thread that just
2233 // wants to use the ObjectMonitor*.
2296 const oop obj = (oop) mid->object();
2297 if (log_is_enabled(Trace, monitorinflation)) {
2298 ResourceMark rm;
2299 log_trace(monitorinflation)("deflate_monitor_using_JT: "
2300 "object=" INTPTR_FORMAT ", mark="
2301 INTPTR_FORMAT ", type='%s'",
2302 p2i(obj), obj->mark().value(),
2303 obj->klass()->external_name());
2304 }
2305
2306 // Install the old mark word if nobody else has already done it.
2307 mid->install_displaced_markword_in_object(obj);
2308 mid->clear_using_JT();
2309
2310 assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT,
2311 p2i(mid->object()));
2312 assert(mid->is_free(), "must be free: allocation_state=%d",
2313 (int) mid->allocation_state());
2314
2315 // Move the deflated ObjectMonitor to the working free list
2316 // defined by free_head_p and free_tail_p. No races on this list
2317 // so no need for load_acquire() or store_release().
2318 if (*free_head_p == NULL) {
2319 // First one on the list.
2320 *free_head_p = mid;
2321 }
2322 if (*free_tail_p != NULL) {
2323 // We append to the list so the caller can use mid->_next_om
2324 // to fix the linkages in its context.
2325 ObjectMonitor* prevtail = *free_tail_p;
2326 // Should have been cleaned up by the caller:
2327 ObjectMonitor* next = mark_next_loop(prevtail);
2328 assert(unmarked_next(prevtail) == NULL, "must be NULL: _next_om="
2329 INTPTR_FORMAT, p2i(unmarked_next(prevtail)));
2330 set_next(prevtail, mid); // prevtail now points to mid (and is unmarked)
2331 }
2332 *free_tail_p = mid;
2333
2334 // At this point, mid->_next_om still refers to its current
2335 // value and another ObjectMonitor's _next_om field still
2336 // refers to this ObjectMonitor. Those linkages have to be
2337 // cleaned up by the caller who has the complete context.
2338
2339 // We leave owner == DEFLATER_MARKER and ref_count < 0
2340 // to force any racing threads to retry.
2341 return true; // Success, ObjectMonitor has been deflated.
2342 }
2343
2344 // The owner was changed from DEFLATER_MARKER so we lost the
2345 // race since the ObjectMonitor is now busy.
2346
2347 // Add back max_jint to restore the ref_count field to its
2348 // proper value (which may not be what we saw above):
2349 Atomic::add(max_jint, &mid->_ref_count);
2350
2351 assert(mid->ref_count() >= 0, "must not be negative: ref_count=%d",
2352 mid->ref_count());
2353 return false;
2354 }
2355
2356 // The ref_count was no longer 0 so we lost the race since the
2357 // ObjectMonitor is now busy or the ObjectMonitor* is now is use.
2358 // Restore owner to NULL if it is still DEFLATER_MARKER:
2359 Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER);
2360 }
2361
2362 // The owner field is no longer NULL so we lost the race since the
2363 // ObjectMonitor is now busy.
2364 return false;
2365 }
2366
2367 // Walk a given monitor list, and deflate idle monitors.
2368 // The given list could be a per-thread list or a global list.
2369 //
2370 // In the case of parallel processing of thread local monitor lists,
2371 // work is done by Threads::parallel_threads_do() which ensures that
2372 // each Java thread is processed by exactly one worker thread, and
2373 // thus avoid conflicts that would arise when worker threads would
2374 // process the same monitor lists concurrently.
2375 //
2376 // See also ParallelSPCleanupTask and
2377 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
2378 // Threads::parallel_java_threads_do() in thread.cpp.
2379 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor* volatile * list_p,
2380 int volatile * count_p,
2381 ObjectMonitor** free_head_p,
2382 ObjectMonitor** free_tail_p) {
2383 ObjectMonitor* cur_mid_in_use = NULL;
2384 ObjectMonitor* mid = NULL;
2385 ObjectMonitor* next = NULL;
2386 int deflated_count = 0;
2387
2388 // We use the simpler mark-mid-as-we-go protocol since there are no
2389 // parallel list deletions since we are at a safepoint.
2390 if (!mark_list_head(list_p, &mid, &next)) {
2391 return 0; // The list is empty so nothing to deflate.
2392 }
2393
2394 while (true) {
2395 oop obj = (oop) mid->object();
2396 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
2397 // Deflation succeeded and already updated free_head_p and
2398 // free_tail_p as needed. Finish the move to the local free list
2399 // by unlinking mid from the global or per-thread in-use list.
2400 if (Atomic::cmpxchg(next, list_p, mid) != mid) {
2401 // We could not switch the list head to next.
2402 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
2403 if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, mid) != mid) {
2404 // deflate_monitor_list() is called at a safepoint so the
2405 // global or per-thread in-use list should not be modified
2406 // in parallel so we:
2407 fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: "
2408 "list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: "
2409 "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT,
2410 p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use),
2411 p2i(cur_mid_in_use->_next_om));
2412 }
2413 }
2414 // At this point mid is disconnected from the in-use list so
2415 // its marked next field no longer has any effects.
2416 deflated_count++;
2417 Atomic::dec(count_p);
2418 chk_for_list_loop(OrderAccess::load_acquire(list_p),
2419 OrderAccess::load_acquire(count_p));
2420 chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p),
2421 OrderAccess::load_acquire(count_p));
2422 // mid is current tail in the free_head_p list so NULL terminate it
2423 // (which also unmarks it):
2424 set_next(mid, NULL);
2425
2426 // All the list management is done so move on to the next one:
2427 mid = next;
2428 } else {
2429 set_next(mid, next); // unmark next field
2430
2431 // All the list management is done so move on to the next one:
2432 cur_mid_in_use = mid;
2433 mid = next;
2434 }
2435 if (mid == NULL) {
2436 break; // Reached end of the list so nothing more to deflate.
2437 }
2438 // Mark mid's next field so we can possibly deflate it:
2439 next = mark_next_loop(mid);
2440 }
2441 return deflated_count;
2442 }
2443
2444 // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using
2445 // a JavaThread. Returns the number of deflated ObjectMonitors. The given
2446 // list could be a per-thread in-use list or the global in-use list.
2447 // If a safepoint has started, then we save state via saved_mid_in_use_p
2448 // and return to the caller to honor the safepoint.
2449 //
2450 int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor* volatile * list_p,
2451 int volatile * count_p,
2452 ObjectMonitor** free_head_p,
2453 ObjectMonitor** free_tail_p,
2454 ObjectMonitor** saved_mid_in_use_p) {
2455 assert(AsyncDeflateIdleMonitors, "sanity check");
2456 assert(Thread::current()->is_Java_thread(), "precondition");
2457
2458 ObjectMonitor* cur_mid_in_use = NULL;
2459 ObjectMonitor* mid = NULL;
2460 ObjectMonitor* next = NULL;
2461 ObjectMonitor* next_next = NULL;
2462 int deflated_count = 0;
2463
2464 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
2465 // protocol because om_release() can do list deletions in parallel.
2466 // We also mark-next-next-as-we-go to prevent an om_flush() that is
2467 // behind this thread from passing us.
2468 if (*saved_mid_in_use_p == NULL) {
2469 // No saved state so start at the beginning.
2470 // Mark the list head's next field so we can possibly deflate it:
2471 if (!mark_list_head(list_p, &mid, &next)) {
2472 return 0; // The list is empty so nothing to deflate.
2473 }
2474 } else {
2475 // We're restarting after a safepoint so restore the necessary state
2476 // before we resume.
2477 cur_mid_in_use = *saved_mid_in_use_p;
2478 // Mark cur_mid_in_use's next field so we can possibly update its
2479 // next field to extract a deflated ObjectMonitor.
2480 mid = mark_next_loop(cur_mid_in_use);
2481 if (mid == NULL) {
2482 set_next(cur_mid_in_use, NULL); // unmark next field
2483 *saved_mid_in_use_p = NULL;
2484 return 0; // The remainder is empty so nothing more to deflate.
2485 }
2486 // Mark mid's next field so we can possibly deflate it:
2487 next = mark_next_loop(mid);
2488 }
2489
2490 while (true) {
2491 // The current mid's next field is marked at this point. If we have
2492 // a cur_mid_in_use, then its next field is also marked at this point.
2493
2494 if (next != NULL) {
2495 // We mark the next -> next field so that an om_flush()
2496 // thread that is behind us cannot pass us when we
2497 // unmark the current mid's next field.
2498 next_next = mark_next_loop(next);
2499 }
2500
2501 // Only try to deflate if there is an associated Java object and if
2502 // mid is old (is not newly allocated and is not newly freed).
2503 if (mid->object() != NULL && mid->is_old() &&
2504 deflate_monitor_using_JT(mid, free_head_p, free_tail_p)) {
2505 // Deflation succeeded and already updated free_head_p and
2506 // free_tail_p as needed. Finish the move to the local free list
2507 // by unlinking mid from the global or per-thread in-use list.
2508 if (Atomic::cmpxchg(next, list_p, mid) != mid) {
2509 // We could not switch the list head to next.
2510 ObjectMonitor* marked_mid = mark_om_ptr(mid);
2511 ObjectMonitor* marked_next = mark_om_ptr(next);
2512 // Switch cur_mid_in_use's next field to marked next:
2513 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
2514 if (Atomic::cmpxchg(marked_next, &cur_mid_in_use->_next_om,
2515 marked_mid) != marked_mid) {
2516 // We could not switch cur_mid_in_use's next field. This
2517 // should not be possible since it was marked so we:
2518 fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: "
2519 "&list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: "
2520 "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT,
2521 p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use),
2522 p2i(cur_mid_in_use->_next_om));
2523 }
2524 }
2525 // At this point mid is disconnected from the in-use list so
2526 // its marked next field no longer has any effects.
2527 deflated_count++;
2528 Atomic::dec(count_p);
2529 chk_for_list_loop(OrderAccess::load_acquire(list_p),
2530 OrderAccess::load_acquire(count_p));
2531 chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p),
2532 OrderAccess::load_acquire(count_p));
2533 // mid is current tail in the free_head_p list so NULL terminate it
2534 // (which also unmarks it):
2535 set_next(mid, NULL);
2536
2537 // All the list management is done so move on to the next one:
2538 mid = next; // mid keeps non-NULL next's marked next field
2539 next = next_next;
2540 } else {
2541 // mid is considered in-use if it does not have an associated
2542 // Java object or mid is not old or deflation did not succeed.
2543 // A mid->is_new() node can be seen here when it is freshly
2544 // returned by om_alloc() (and skips the deflation code path).
2545 // A mid->is_old() node can be seen here when deflation failed.
2546 // A mid->is_free() node can be seen here when a fresh node from
2547 // om_alloc() is released by om_release() due to losing the race
2548 // in inflate().
2549
2550 // All the list management is done so move on to the next one:
2551 if (cur_mid_in_use != NULL) {
2552 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2553 }
2554 // The next cur_mid_in_use keeps mid's marked next field so
2555 // that it is stable for a possible next field change. It
2556 // cannot be modified by om_release() while it is marked.
2557 cur_mid_in_use = mid;
2558 mid = next; // mid keeps non-NULL next's marked next field
2559 next = next_next;
2560
2561 if (SafepointSynchronize::is_synchronizing() &&
2562 cur_mid_in_use != OrderAccess::load_acquire(list_p) &&
2563 cur_mid_in_use->is_old()) {
2564 // If a safepoint has started and cur_mid_in_use is not the list
2565 // head and is old, then it is safe to use as saved state. Return
2566 // to the caller before blocking.
2567 *saved_mid_in_use_p = cur_mid_in_use;
2568 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2569 if (mid != NULL) {
2570 set_next(mid, next); // umark mid
2571 }
2572 return deflated_count;
2573 }
2574 }
2575 if (mid == NULL) {
2576 if (cur_mid_in_use != NULL) {
2577 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2578 }
2579 break; // Reached end of the list so nothing more to deflate.
2580 }
2581
2582 // The current mid's next field is marked at this point. If we have
2583 // a cur_mid_in_use, then its next field is also marked at this point.
2584 }
2585 // We finished the list without a safepoint starting so there's
2586 // no need to save state.
2587 *saved_mid_in_use_p = NULL;
2588 return deflated_count;
2589 }
2590
2591 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2592 OrderAccess::release_store(&counters->n_in_use, 0); // currently associated with objects
2593 OrderAccess::release_store(&counters->n_in_circulation, 0); // extant
2594 OrderAccess::release_store(&counters->n_scavenged, 0); // reclaimed (global and per-thread)
2595 OrderAccess::release_store(&counters->per_thread_scavenged, 0); // per-thread scavenge total
2596 counters->per_thread_times = 0.0; // per-thread scavenge times
2597 }
2598
2599 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
2600 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2601
2602 if (AsyncDeflateIdleMonitors) {
2603 // Nothing to do when global idle ObjectMonitors are deflated using
2604 // a JavaThread unless a special deflation has been requested.
2605 if (!is_special_deflation_requested()) {
2606 return;
2607 }
2608 }
2609
2610 bool deflated = false;
2611
2612 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2613 ObjectMonitor* free_tail_p = NULL;
2614 elapsedTimer timer;
2615
2616 if (log_is_enabled(Info, monitorinflation)) {
2617 timer.start();
2618 }
2619
2620 // Note: the thread-local monitors lists get deflated in
2621 // a separate pass. See deflate_thread_local_monitors().
2622
2623 // For moribund threads, scan g_om_in_use_list
2624 int deflated_count = 0;
2625 if (OrderAccess::load_acquire(&g_om_in_use_list) != NULL) {
2626 // Update n_in_circulation before g_om_in_use_count is updated by deflation.
2627 Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_circulation);
2628
2629 deflated_count = deflate_monitor_list(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p);
2630 Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_use);
2631 }
2632
2633 if (free_head_p != NULL) {
2634 // Move the deflated ObjectMonitors back to the global free list.
2635 // No races on the working free list so no need for load_acquire().
2636 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2637 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2638 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2639 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2640 Atomic::add(deflated_count, &counters->n_scavenged);
2641 }
2642 timer.stop();
2643
2644 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2645 LogStreamHandle(Info, monitorinflation) lsh_info;
2646 LogStream* ls = NULL;
2647 if (log_is_enabled(Debug, monitorinflation)) {
2648 ls = &lsh_debug;
2649 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2650 ls = &lsh_info;
2651 }
2652 if (ls != NULL) {
2653 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2654 }
2655 }
2656
2657 // Deflate global idle ObjectMonitors using a JavaThread.
2658 //
2659 void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() {
2660 assert(AsyncDeflateIdleMonitors, "sanity check");
2661 assert(Thread::current()->is_Java_thread(), "precondition");
2662 JavaThread* self = JavaThread::current();
2663
2664 deflate_common_idle_monitors_using_JT(true /* is_global */, self);
2665 }
2666
2667 // Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread.
2668 //
2669 void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) {
2670 assert(AsyncDeflateIdleMonitors, "sanity check");
2671 assert(Thread::current()->is_Java_thread(), "precondition");
2672
2673 deflate_common_idle_monitors_using_JT(false /* !is_global */, target);
2674 }
2675
2676 // Deflate global or per-thread idle ObjectMonitors using a JavaThread.
2677 //
2678 void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) {
2679 JavaThread* self = JavaThread::current();
2680
2681 int deflated_count = 0;
2682 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors
2683 ObjectMonitor* free_tail_p = NULL;
2684 ObjectMonitor* saved_mid_in_use_p = NULL;
2685 elapsedTimer timer;
2686
2687 if (log_is_enabled(Info, monitorinflation)) {
2688 timer.start();
2689 }
2690
2691 if (is_global) {
2692 OM_PERFDATA_OP(MonExtant, set_value(OrderAccess::load_acquire(&g_om_in_use_count)));
2693 } else {
2694 OM_PERFDATA_OP(MonExtant, inc(OrderAccess::load_acquire(&target->om_in_use_count)));
2695 }
2696
2697 do {
2698 int local_deflated_count;
2699 if (is_global) {
2700 local_deflated_count = deflate_monitor_list_using_JT(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2701 } else {
2702 local_deflated_count = deflate_monitor_list_using_JT(&target->om_in_use_list, &target->om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2703 }
2704 deflated_count += local_deflated_count;
2705
2706 if (free_head_p != NULL) {
2707 // Move the deflated ObjectMonitors to the global free list.
2708 // No races on the working list so no need for load_acquire().
2709 guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count);
2710 // Note: The target thread can be doing an om_alloc() that
2711 // is trying to prepend an ObjectMonitor on its in-use list
2712 // at the same time that we have deflated the current in-use
2713 // list head and put it on the local free list. prepend_to_common()
2714 // will detect the race and retry which avoids list corruption,
2715 // but the next field in free_tail_p can flicker to marked
2716 // and then unmarked while prepend_to_common() is sorting it
2717 // all out.
2718 assert(unmarked_next(free_tail_p) == NULL, "must be NULL: _next_om="
2719 INTPTR_FORMAT, p2i(unmarked_next(free_tail_p)));
2720
2721 prepend_list_to_g_free_list(free_head_p, free_tail_p, local_deflated_count);
2722
2723 OM_PERFDATA_OP(Deflations, inc(local_deflated_count));
2724 }
2725
2726 if (saved_mid_in_use_p != NULL) {
2727 // deflate_monitor_list_using_JT() detected a safepoint starting.
2728 timer.stop();
2729 {
2730 if (is_global) {
2731 log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint.");
2732 } else {
2733 log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target));
2734 }
2735 assert(SafepointSynchronize::is_synchronizing(), "sanity check");
2736 ThreadBlockInVM blocker(self);
2737 }
2738 // Prepare for another loop after the safepoint.
2739 free_head_p = NULL;
2740 free_tail_p = NULL;
2741 if (log_is_enabled(Info, monitorinflation)) {
2742 timer.start();
2743 }
2744 }
2745 } while (saved_mid_in_use_p != NULL);
2746 timer.stop();
2747
2748 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2749 LogStreamHandle(Info, monitorinflation) lsh_info;
2750 LogStream* ls = NULL;
2751 if (log_is_enabled(Debug, monitorinflation)) {
2752 ls = &lsh_debug;
2753 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2754 ls = &lsh_info;
2755 }
2756 if (ls != NULL) {
2757 if (is_global) {
2758 ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2759 } else {
2760 ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count);
2761 }
2762 }
2763 }
2764
2765 void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2766 // Report the cumulative time for deflating each thread's idle
2767 // monitors. Note: if the work is split among more than one
2768 // worker thread, then the reported time will likely be more
2769 // than a beginning to end measurement of the phase.
2770 // Note: AsyncDeflateIdleMonitors only deflates per-thread idle
2771 // monitors at a safepoint when a special deflation has been requested.
2772 log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d",
2773 counters->per_thread_times,
2774 OrderAccess::load_acquire(&counters->per_thread_scavenged));
2775
2776 bool needs_special_deflation = is_special_deflation_requested();
2777 if (!AsyncDeflateIdleMonitors || needs_special_deflation) {
2778 // AsyncDeflateIdleMonitors does not use these counters unless
2779 // there is a special deflation request.
2780
2781 OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged));
2782 OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation));
2783 }
2784
2785 if (log_is_enabled(Debug, monitorinflation)) {
2786 // exit_globals()'s call to audit_and_print_stats() is done
2787 // at the Info level.
2788 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2789 } else if (log_is_enabled(Info, monitorinflation)) {
2790 log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, "
2791 "g_om_free_count=%d",
2792 OrderAccess::load_acquire(&g_om_population),
2793 OrderAccess::load_acquire(&g_om_in_use_count),
2794 OrderAccess::load_acquire(&g_om_free_count));
2795 }
2796
2797 ForceMonitorScavenge = 0; // Reset
2798 GVars.stw_random = os::random();
2799 GVars.stw_cycle++;
2800 if (needs_special_deflation) {
2801 set_is_special_deflation_requested(false); // special deflation is done
2802 }
2803 }
2804
2805 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
2806 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2807
2808 if (AsyncDeflateIdleMonitors && !is_special_deflation_requested()) {
2809 // Nothing to do if a special deflation has NOT been requested.
2810 return;
2811 }
2812
2813 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2814 ObjectMonitor* free_tail_p = NULL;
2815 elapsedTimer timer;
2816
2817 if (log_is_enabled(Info, safepoint, cleanup) ||
2818 log_is_enabled(Info, monitorinflation)) {
2819 timer.start();
2820 }
2821
2822 // Update n_in_circulation before om_in_use_count is updated by deflation.
2823 Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_circulation);
2824
2825 int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p);
2826 Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_use);
2827
2828 if (free_head_p != NULL) {
2829 // Move the deflated ObjectMonitors back to the global free list.
2830 // No races on the working list so no need for load_acquire().
2831 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2832 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2833 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2834 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2835 Atomic::add(deflated_count, &counters->n_scavenged);
2836 Atomic::add(deflated_count, &counters->per_thread_scavenged);
2837 }
2838
2839 timer.stop();
2840 // Safepoint logging cares about cumulative per_thread_times and
2841 // we'll capture most of the cost, but not the muxRelease() which
2842 // should be cheap.
2843 counters->per_thread_times += timer.seconds();
2844
2845 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2846 LogStreamHandle(Info, monitorinflation) lsh_info;
2847 LogStream* ls = NULL;
2848 if (log_is_enabled(Debug, monitorinflation)) {
2849 ls = &lsh_debug;
2850 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2851 ls = &lsh_info;
2852 }
2853 if (ls != NULL) {
2854 ls->print_cr("jt=" INTPTR_FORMAT ": deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(thread), timer.seconds(), deflated_count);
2855 }
2856 }
2857
2858 // Monitor cleanup on JavaThread::exit
2859
2860 // Iterate through monitor cache and attempt to release thread's monitors
2861 // Gives up on a particular monitor if an exception occurs, but continues
2862 // the overall iteration, swallowing the exception.
2863 class ReleaseJavaMonitorsClosure: public MonitorClosure {
2864 private:
2875
2876 // Release all inflated monitors owned by THREAD. Lightweight monitors are
2877 // ignored. This is meant to be called during JNI thread detach which assumes
2878 // all remaining monitors are heavyweight. All exceptions are swallowed.
2879 // Scanning the extant monitor list can be time consuming.
2880 // A simple optimization is to add a per-thread flag that indicates a thread
2881 // called jni_monitorenter() during its lifetime.
2882 //
2883 // Instead of No_Savepoint_Verifier it might be cheaper to
2884 // use an idiom of the form:
2885 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
2886 // <code that must not run at safepoint>
2887 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
2888 // Since the tests are extremely cheap we could leave them enabled
2889 // for normal product builds.
2890
2891 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
2892 assert(THREAD == JavaThread::current(), "must be current Java thread");
2893 NoSafepointVerifier nsv;
2894 ReleaseJavaMonitorsClosure rjmc(THREAD);
2895 ObjectSynchronizer::monitors_iterate(&rjmc);
2896 THREAD->clear_pending_exception();
2897 }
2898
2899 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
2900 switch (cause) {
2901 case inflate_cause_vm_internal: return "VM Internal";
2902 case inflate_cause_monitor_enter: return "Monitor Enter";
2903 case inflate_cause_wait: return "Monitor Wait";
2904 case inflate_cause_notify: return "Monitor Notify";
2905 case inflate_cause_hash_code: return "Monitor Hash Code";
2906 case inflate_cause_jni_enter: return "JNI Monitor Enter";
2907 case inflate_cause_jni_exit: return "JNI Monitor Exit";
2908 default:
2909 ShouldNotReachHere();
2910 }
2911 return "Unknown";
2912 }
2913
2914 //------------------------------------------------------------------------------
2915 // Debugging code
2929 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
2930 return (u_char*)&GVars.stw_random;
2931 }
2932
2933 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
2934 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
2935
2936 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2937 LogStreamHandle(Info, monitorinflation) lsh_info;
2938 LogStreamHandle(Trace, monitorinflation) lsh_trace;
2939 LogStream* ls = NULL;
2940 if (log_is_enabled(Trace, monitorinflation)) {
2941 ls = &lsh_trace;
2942 } else if (log_is_enabled(Debug, monitorinflation)) {
2943 ls = &lsh_debug;
2944 } else if (log_is_enabled(Info, monitorinflation)) {
2945 ls = &lsh_info;
2946 }
2947 assert(ls != NULL, "sanity check");
2948
2949 // Log counts for the global and per-thread monitor lists:
2950 int chk_om_population = log_monitor_list_counts(ls);
2951 int error_cnt = 0;
2952
2953 ls->print_cr("Checking global lists:");
2954
2955 // Check g_om_population:
2956 if (OrderAccess::load_acquire(&g_om_population) == chk_om_population) {
2957 ls->print_cr("g_om_population=%d equals chk_om_population=%d",
2958 OrderAccess::load_acquire(&g_om_population),
2959 chk_om_population);
2960 } else {
2961 ls->print_cr("ERROR: g_om_population=%d is not equal to "
2962 "chk_om_population=%d",
2963 OrderAccess::load_acquire(&g_om_population),
2964 chk_om_population);
2965 error_cnt++;
2966 }
2967
2968 // Check g_om_in_use_list and g_om_in_use_count:
2969 chk_global_in_use_list_and_count(ls, &error_cnt);
2970
2971 // Check g_free_list and g_om_free_count:
2972 chk_global_free_list_and_count(ls, &error_cnt);
2973
2974 ls->print_cr("Checking per-thread lists:");
2975
2976 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2977 // Check om_in_use_list and om_in_use_count:
2978 chk_per_thread_in_use_list_and_count(jt, ls, &error_cnt);
2979
2980 // Check om_free_list and om_free_count:
2981 chk_per_thread_free_list_and_count(jt, ls, &error_cnt);
2982 }
2983
2984 if (error_cnt == 0) {
2985 ls->print_cr("No errors found in monitor list checks.");
2986 } else {
2987 log_error(monitorinflation)("found monitor list errors: error_cnt=%d", error_cnt);
2988 }
2989
2990 if ((on_exit && log_is_enabled(Info, monitorinflation)) ||
2991 (!on_exit && log_is_enabled(Trace, monitorinflation))) {
2992 // When exiting this log output is at the Info level. When called
2993 // at a safepoint, this log output is at the Trace level since
2994 // there can be a lot of it.
2995 log_in_use_monitor_details(ls);
2996 }
2997
2998 ls->flush();
2999
3000 guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
3001 }
3002
3003 // Check a free monitor entry; log any errors.
3004 void ObjectSynchronizer::chk_free_entry(JavaThread* jt, ObjectMonitor* n,
3005 outputStream * out, int *error_cnt_p) {
3006 stringStream ss;
3007 if (n->is_busy()) {
3008 if (jt != NULL) {
3009 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3010 ": free per-thread monitor must not be busy: %s", p2i(jt),
3011 p2i(n), n->is_busy_to_string(&ss));
3012 } else {
3013 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3014 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
3015 }
3031 }
3032 if (n->object() != NULL) {
3033 if (jt != NULL) {
3034 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3035 ": free per-thread monitor must have NULL _object "
3036 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
3037 p2i(n->object()));
3038 } else {
3039 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3040 "must have NULL _object field: _object=" INTPTR_FORMAT,
3041 p2i(n), p2i(n->object()));
3042 }
3043 *error_cnt_p = *error_cnt_p + 1;
3044 }
3045 }
3046
3047 // Check the global free list and count; log the results of the checks.
3048 void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out,
3049 int *error_cnt_p) {
3050 int chk_om_free_count = 0;
3051 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_free_list); n != NULL; n = unmarked_next(n)) {
3052 chk_free_entry(NULL /* jt */, n, out, error_cnt_p);
3053 chk_om_free_count++;
3054 }
3055 if (OrderAccess::load_acquire(&g_om_free_count) == chk_om_free_count) {
3056 out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d",
3057 OrderAccess::load_acquire(&g_om_free_count),
3058 chk_om_free_count);
3059 } else {
3060 // With lock free access to g_free_list, it is possible for an
3061 // ObjectMonitor to be prepended to g_free_list after we started
3062 // calculating chk_om_free_count so g_om_free_count may not
3063 // match anymore.
3064 out->print_cr("WARNING: g_om_free_count=%d is not equal to "
3065 "chk_om_free_count=%d",
3066 OrderAccess::load_acquire(&g_om_free_count),
3067 chk_om_free_count);
3068 }
3069 }
3070
3071 // Check the global in-use list and count; log the results of the checks.
3072 void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out,
3073 int *error_cnt_p) {
3074 int chk_om_in_use_count = 0;
3075 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) {
3076 chk_in_use_entry(NULL /* jt */, n, out, error_cnt_p);
3077 chk_om_in_use_count++;
3078 }
3079 if (OrderAccess::load_acquire(&g_om_in_use_count) == chk_om_in_use_count) {
3080 out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d",
3081 OrderAccess::load_acquire(&g_om_in_use_count),
3082 chk_om_in_use_count);
3083 } else {
3084 out->print_cr("ERROR: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d",
3085 OrderAccess::load_acquire(&g_om_in_use_count),
3086 chk_om_in_use_count);
3087 *error_cnt_p = *error_cnt_p + 1;
3088 }
3089 }
3090
3091 // Check an in-use monitor entry; log any errors.
3092 void ObjectSynchronizer::chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,
3093 outputStream * out, int *error_cnt_p) {
3094 if (n->header().value() == 0) {
3095 if (jt != NULL) {
3096 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3097 ": in-use per-thread monitor must have non-NULL _header "
3098 "field.", p2i(jt), p2i(n));
3099 } else {
3100 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
3101 "must have non-NULL _header field.", p2i(n));
3102 }
3103 *error_cnt_p = *error_cnt_p + 1;
3104 }
3105 if (n->object() == NULL) {
3106 if (jt != NULL) {
3135 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3136 ": in-use per-thread monitor's object does not refer "
3137 "to the same monitor: obj=" INTPTR_FORMAT ", mark="
3138 INTPTR_FORMAT ", obj_mon=" INTPTR_FORMAT, p2i(jt),
3139 p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3140 } else {
3141 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
3142 "monitor's object does not refer to the same monitor: obj="
3143 INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
3144 INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3145 }
3146 *error_cnt_p = *error_cnt_p + 1;
3147 }
3148 }
3149
3150 // Check the thread's free list and count; log the results of the checks.
3151 void ObjectSynchronizer::chk_per_thread_free_list_and_count(JavaThread *jt,
3152 outputStream * out,
3153 int *error_cnt_p) {
3154 int chk_om_free_count = 0;
3155 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_free_list); n != NULL; n = unmarked_next(n)) {
3156 chk_free_entry(jt, n, out, error_cnt_p);
3157 chk_om_free_count++;
3158 }
3159 if (OrderAccess::load_acquire(&jt->om_free_count) == chk_om_free_count) {
3160 out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals "
3161 "chk_om_free_count=%d", p2i(jt),
3162 OrderAccess::load_acquire(&jt->om_free_count),
3163 chk_om_free_count);
3164 } else {
3165 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not "
3166 "equal to chk_om_free_count=%d", p2i(jt),
3167 OrderAccess::load_acquire(&jt->om_free_count),
3168 chk_om_free_count);
3169 *error_cnt_p = *error_cnt_p + 1;
3170 }
3171 }
3172
3173 // Check the thread's in-use list and count; log the results of the checks.
3174 void ObjectSynchronizer::chk_per_thread_in_use_list_and_count(JavaThread *jt,
3175 outputStream * out,
3176 int *error_cnt_p) {
3177 int chk_om_in_use_count = 0;
3178 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) {
3179 chk_in_use_entry(jt, n, out, error_cnt_p);
3180 chk_om_in_use_count++;
3181 }
3182 if (OrderAccess::load_acquire(&jt->om_in_use_count) == chk_om_in_use_count) {
3183 out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals "
3184 "chk_om_in_use_count=%d", p2i(jt),
3185 OrderAccess::load_acquire(&jt->om_in_use_count),
3186 chk_om_in_use_count);
3187 } else {
3188 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not "
3189 "equal to chk_om_in_use_count=%d", p2i(jt),
3190 OrderAccess::load_acquire(&jt->om_in_use_count),
3191 chk_om_in_use_count);
3192 *error_cnt_p = *error_cnt_p + 1;
3193 }
3194 }
3195
3196 // Log details about ObjectMonitors on the in-use lists. The 'BHL'
3197 // flags indicate why the entry is in-use, 'object' and 'object type'
3198 // indicate the associated object and its type.
3199 void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out) {
3200 stringStream ss;
3201 if (OrderAccess::load_acquire(&g_om_in_use_count) > 0) {
3202 out->print_cr("In-use global monitor info:");
3203 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3204 out->print_cr("%18s %s %7s %18s %18s",
3205 "monitor", "BHL", "ref_cnt", "object", "object type");
3206 out->print_cr("================== === ======= ================== ==================");
3207 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) {
3208 const oop obj = (oop) n->object();
3209 const markWord mark = n->header();
3210 ResourceMark rm;
3211 out->print(INTPTR_FORMAT " %d%d%d %7d " INTPTR_FORMAT " %s",
3212 p2i(n), n->is_busy() != 0, mark.hash() != 0,
3213 n->owner() != NULL, (int)n->ref_count(), p2i(obj),
3214 obj->klass()->external_name());
3215 if (n->is_busy() != 0) {
3216 out->print(" (%s)", n->is_busy_to_string(&ss));
3217 ss.reset();
3218 }
3219 out->cr();
3220 }
3221 }
3222
3223 out->print_cr("In-use per-thread monitor info:");
3224 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3225 out->print_cr("%18s %18s %s %7s %18s %18s",
3226 "jt", "monitor", "BHL", "ref_cnt", "object", "object type");
3227 out->print_cr("================== ================== === ======= ================== ==================");
3228 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3229 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) {
3230 const oop obj = (oop) n->object();
3231 const markWord mark = n->header();
3232 ResourceMark rm;
3233 out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d %7d "
3234 INTPTR_FORMAT " %s", p2i(jt), p2i(n), n->is_busy() != 0,
3235 mark.hash() != 0, n->owner() != NULL, (int)n->ref_count(),
3236 p2i(obj), obj->klass()->external_name());
3237 if (n->is_busy() != 0) {
3238 out->print(" (%s)", n->is_busy_to_string(&ss));
3239 ss.reset();
3240 }
3241 out->cr();
3242 }
3243 }
3244
3245 out->flush();
3246 }
3247
3248 // Log counts for the global and per-thread monitor lists and return
3249 // the population count.
3250 int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) {
3251 int pop_count = 0;
3252 out->print_cr("%18s %10s %10s %10s",
3253 "Global Lists:", "InUse", "Free", "Total");
3254 out->print_cr("================== ========== ========== ==========");
3255 out->print_cr("%18s %10d %10d %10d", "",
3256 OrderAccess::load_acquire(&g_om_in_use_count),
3257 OrderAccess::load_acquire(&g_om_free_count),
3258 OrderAccess::load_acquire(&g_om_population));
3259 pop_count += OrderAccess::load_acquire(&g_om_in_use_count) +
3260 OrderAccess::load_acquire(&g_om_free_count);
3261
3262 out->print_cr("%18s %10s %10s %10s",
3263 "Per-Thread Lists:", "InUse", "Free", "Provision");
3264 out->print_cr("================== ========== ========== ==========");
3265
3266 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3267 out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt),
3268 OrderAccess::load_acquire(&jt->om_in_use_count),
3269 OrderAccess::load_acquire(&jt->om_free_count),
3270 jt->om_free_provision);
3271 pop_count += OrderAccess::load_acquire(&jt->om_in_use_count) +
3272 OrderAccess::load_acquire(&jt->om_free_count);
3273 }
3274 return pop_count;
3275 }
3276
3277 #ifndef PRODUCT
3278
3279 // Check if monitor belongs to the monitor cache
3280 // The list is grow-only so it's *relatively* safe to traverse
3281 // the list of extant blocks without taking a lock.
3282
3283 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
3284 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
3285 while (block != NULL) {
3286 assert(block->object() == CHAINMARKER, "must be a block header");
3287 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
3288 address mon = (address)monitor;
3289 address blk = (address)block;
3290 size_t diff = mon - blk;
3291 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
3292 return 1;
3293 }
3294 // unmarked_next() is not needed with g_block_list (no next field marking).
3295 block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om);
3296 }
3297 return 0;
3298 }
3299
3300 #endif
|