146 static ListGlobals LVars;
147
148 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
149
150
151 // =====================> Spin-lock functions
152
153 // ObjectMonitors are not lockable outside of this file. We use spin-locks
154 // implemented using a bit in the _next_om field instead of the heavier
155 // weight locking mechanisms for faster list management.
156
157 #define OM_LOCK_BIT 0x1
158
159 // Return true if the ObjectMonitor is locked.
160 // Otherwise returns false.
161 static bool is_locked(ObjectMonitor* om) {
162 return ((intptr_t)Atomic::load(&om->_next_om) & OM_LOCK_BIT) == OM_LOCK_BIT;
163 }
164
165 // Mark an ObjectMonitor* with OM_LOCK_BIT and return it.
166 // Note: the om parameter may or may not have been marked originally.
167 static ObjectMonitor* mark_om_ptr(ObjectMonitor* om) {
168 return (ObjectMonitor*)((intptr_t)om | OM_LOCK_BIT);
169 }
170
171 // Try to lock an ObjectMonitor. Returns true if locking was successful.
172 // Otherwise returns false.
173 static bool try_om_lock(ObjectMonitor* om) {
174 // Get current next field without any OM_LOCK_BIT value.
175 ObjectMonitor* next = (ObjectMonitor*)((intptr_t)Atomic::load(&om->_next_om) & ~OM_LOCK_BIT);
176 if (Atomic::cmpxchg(&om->_next_om, next, mark_om_ptr(next)) != next) {
177 return false; // Cannot lock the ObjectMonitor.
178 }
179 return true;
180 }
181
182 // Lock an ObjectMonitor.
183 static void om_lock(ObjectMonitor* om) {
184 while (true) {
185 if (try_om_lock(om)) {
186 return;
187 }
188 }
189 }
190
191 // Unlock an ObjectMonitor.
192 static void om_unlock(ObjectMonitor* om) {
193 ObjectMonitor* next = Atomic::load(&om->_next_om);
194 guarantee(((intptr_t)next & OM_LOCK_BIT) == OM_LOCK_BIT, "next=" INTPTR_FORMAT
195 " must have OM_LOCK_BIT=%x set.", p2i(next), OM_LOCK_BIT);
196
197 next = (ObjectMonitor*)((intptr_t)next & ~OM_LOCK_BIT); // Clear OM_LOCK_BIT.
198 Atomic::store(&om->_next_om, next);
199 }
200
201 // Get the list head after locking it. Returns the list head or NULL
202 // if the list is empty.
203 static ObjectMonitor* get_list_head_locked(ObjectMonitor** list_p) {
204 while (true) {
205 ObjectMonitor* mid = Atomic::load(list_p);
206 if (mid == NULL) {
207 return NULL; // The list is empty.
208 }
209 if (try_om_lock(mid)) {
210 if (Atomic::load(list_p) != mid) {
211 // The list head changed so we have to retry.
212 om_unlock(mid);
213 continue;
214 }
215 return mid;
216 }
217 }
218 }
219
220 // Return the unmarked next field in an ObjectMonitor. Note: the next
221 // field may or may not have been marked with OM_LOCK_BIT originally.
222 static ObjectMonitor* unmarked_next(ObjectMonitor* om) {
223 return (ObjectMonitor*)((intptr_t)Atomic::load(&om->_next_om) & ~OM_LOCK_BIT);
224 }
225
226 #undef OM_LOCK_BIT
227
228
229 // =====================> List Management functions
230
231 // Set the next field in an ObjectMonitor to the specified value.
232 static void set_next(ObjectMonitor* om, ObjectMonitor* value) {
233 Atomic::store(&om->_next_om, value);
234 }
235
236 // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is
237 // the last ObjectMonitor in the list and there are 'count' on the list.
238 // Also updates the specified *count_p.
239 static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail,
240 int count, ObjectMonitor** list_p,
241 int* count_p) {
242 while (true) {
243 ObjectMonitor* cur = Atomic::load(list_p);
244 // Prepend list to *list_p.
245 if (!try_om_lock(tail)) {
1135 // assert(mark.is_neutral(), "sanity check");
1136
1137 return NULL;
1138 }
1139
1140 // Visitors ...
1141
1142 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
1143 PaddedObjectMonitor* block = Atomic::load(&g_block_list);
1144 while (block != NULL) {
1145 assert(block->object() == CHAINMARKER, "must be a block header");
1146 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
1147 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
1148 oop object = (oop)mid->object();
1149 if (object != NULL) {
1150 // Only process with closure if the object is set.
1151 closure->do_monitor(mid);
1152 }
1153 }
1154 // unmarked_next() is not needed with g_block_list (no locking
1155 // used with with block linkage _next_om fields).
1156 block = (PaddedObjectMonitor*)Atomic::load(&block->_next_om);
1157 }
1158 }
1159
1160 static bool monitors_used_above_threshold() {
1161 if (Atomic::load(&LVars.population) == 0) {
1162 return false;
1163 }
1164 if (MonitorUsedDeflationThreshold > 0) {
1165 int monitors_used = Atomic::load(&LVars.population) - Atomic::load(&LVars.free_count);
1166 int monitor_usage = (monitors_used * 100LL) / Atomic::load(&LVars.population);
1167 return monitor_usage > MonitorUsedDeflationThreshold;
1168 }
1169 return false;
1170 }
1171
1172 // Returns true if MonitorBound is set (> 0) and if the specified
1173 // cnt is > MonitorBound. Otherwise returns false.
1174 static bool is_MonitorBound_exceeded(const int cnt) {
1175 const int mx = MonitorBound;
1176 return mx > 0 && cnt > mx;
1177 }
1178
1179 bool ObjectSynchronizer::is_cleanup_needed() {
1180 if (monitors_used_above_threshold()) {
1181 // Too many monitors in use.
1182 return true;
1183 }
1184 return needs_monitor_scavenge();
1185 }
1186
1261
1262 static void InduceScavenge(Thread* self, const char * Whence) {
1263 // Induce STW safepoint to trim monitors
1264 // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
1265 // More precisely, trigger a cleanup safepoint as the number
1266 // of active monitors passes the specified threshold.
1267 // TODO: assert thread state is reasonable
1268
1269 if (Atomic::xchg(&_forceMonitorScavenge, 1) == 0) {
1270 VMThread::check_for_forced_cleanup();
1271 }
1272 }
1273
1274 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self) {
1275 // A large MAXPRIVATE value reduces both list lock contention
1276 // and list coherency traffic, but also tends to increase the
1277 // number of ObjectMonitors in circulation as well as the STW
1278 // scavenge costs. As usual, we lean toward time in space-time
1279 // tradeoffs.
1280 const int MAXPRIVATE = 1024;
1281
1282 stringStream ss;
1283 for (;;) {
1284 ObjectMonitor* m;
1285
1286 // 1: try to allocate from the thread's local om_free_list.
1287 // Threads will attempt to allocate first from their local list, then
1288 // from the global list, and only after those attempts fail will the
1289 // thread attempt to instantiate new monitors. Thread-local free lists
1290 // improve allocation latency, as well as reducing coherency traffic
1291 // on the shared global list.
1292 m = take_from_start_of_om_free_list(self);
1293 if (m != NULL) {
1294 guarantee(m->object() == NULL, "invariant");
1295 prepend_to_om_in_use_list(self, m);
1296 return m;
1297 }
1298
1299 // 2: try to allocate from the global LVars.free_list
1300 // CONSIDER: use muxTry() instead of muxAcquire().
1368 prepend_block_to_lists(temp);
1369 }
1370 }
1371
1372 // Place "m" on the caller's private per-thread om_free_list.
1373 // In practice there's no need to clamp or limit the number of
1374 // monitors on a thread's om_free_list as the only non-allocation time
1375 // we'll call om_release() is to return a monitor to the free list after
1376 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1377 // accumulate on a thread's free list.
1378 //
1379 // Key constraint: all ObjectMonitors on a thread's free list and the global
1380 // free list must have their object field set to null. This prevents the
1381 // scavenger -- deflate_monitor_list() -- from reclaiming them while we
1382 // are trying to release them.
1383
1384 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1385 bool from_per_thread_alloc) {
1386 guarantee(m->header().value() == 0, "invariant");
1387 guarantee(m->object() == NULL, "invariant");
1388 stringStream ss;
1389 guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1390 "%s, recursions=" INTX_FORMAT, m->is_busy_to_string(&ss),
1391 m->_recursions);
1392 // _next_om is used for both per-thread in-use and free lists so
1393 // we have to remove 'm' from the in-use list first (as needed).
1394 if (from_per_thread_alloc) {
1395 // Need to remove 'm' from om_in_use_list.
1396 ObjectMonitor* cur_mid_in_use = NULL;
1397 ObjectMonitor* mid = NULL;
1398 ObjectMonitor* next = NULL;
1399 bool extracted = false;
1400
1401 // We use the simpler lock-mid-as-we-go protocol to prevent races
1402 // with a list walker thread since there are no parallel list
1403 // deletions (deflations happen at a safepoint).
1404 if ((mid = get_list_head_locked(&self->om_in_use_list)) == NULL) {
1405 fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self));
1406 }
1407 next = unmarked_next(mid);
1408 while (true) {
1409 if (m == mid) {
1410 // We found 'm' on the per-thread in-use list so try to extract it.
1411 if (cur_mid_in_use == NULL) {
1412 // mid is the list head and it is locked. Switch the list head
1413 // to next which unlocks the list head, but leaves mid locked:
1414 Atomic::store(&self->om_in_use_list, next);
1415 } else {
1416 // mid is locked. Switch cur_mid_in_use's next field to next
1417 // which is safe because we have no parallel list deletions,
1418 // but we leave mid locked:
1419 set_next(cur_mid_in_use, next);
1420 }
1421 // At this point mid is disconnected from the in-use list so
1422 // its lock no longer has any effects on the in-use list.
1423 extracted = true;
1424 Atomic::dec(&self->om_in_use_count);
1425 // Unlock mid, but leave the next value for any lagging list
1426 // walkers. It will get cleaned up when mid is prepended to
1427 // the thread's free list:
1428 om_unlock(mid);
1429 break;
1430 } else {
1431 om_unlock(mid);
1432 cur_mid_in_use = mid;
1433 }
1434 // All the list management is done so move on to the next one:
1435 mid = next;
1436 if (mid == NULL) {
1437 // Reached end of the list and didn't find m so:
1438 fatal("must find m=" INTPTR_FORMAT "on om_in_use_list=" INTPTR_FORMAT,
1439 p2i(m), p2i(self->om_in_use_list));
1440 }
1441 // Lock mid so we can possibly extract it:
1442 om_lock(mid);
1443 next = unmarked_next(mid);
1444 }
1445 }
1446
1447 prepend_to_om_free_list(self, m);
1448 }
1449
1450 // Return ObjectMonitors on a moribund thread's free and in-use
1451 // lists to the appropriate global lists. The ObjectMonitors on the
1452 // per-thread in-use list may still be in use by other threads.
1453 //
1454 // We currently call om_flush() from Threads::remove() before the
1455 // thread has been excised from the thread list and is no longer a
1456 // mutator. This means that om_flush() cannot run concurrently with
1457 // a safepoint and interleave with deflate_idle_monitors(). In
1458 // particular, this ensures that the thread's in-use monitors are
1459 // scanned by a GC safepoint, either via Thread::oops_do() (before
1460 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1461 // om_flush() is called).
1462
1463 void ObjectSynchronizer::om_flush(Thread* self) {
1464 // Process the per-thread in-use list first to be consistent.
1465 int in_use_count = 0;
1466 ObjectMonitor* in_use_list = NULL;
1467 ObjectMonitor* in_use_tail = NULL;
1468
1469 // This function can race with a list walker thread so we lock the
1470 // list head to prevent confusion.
1471 if ((in_use_list = get_list_head_locked(&self->om_in_use_list)) != NULL) {
1472 // At this point, we have locked the in-use list head so a racing
1473 // thread cannot come in after us. However, a racing thread could
1474 // be ahead of us; we'll detect that and delay to let it finish.
1475 //
1476 // The thread is going away, however the ObjectMonitors on the
1477 // om_in_use_list may still be in-use by other threads. Link
1478 // them to in_use_tail, which will be linked into the global
1479 // in-use list (LVars.in_use_list) below.
1480 //
1481 // Account for the in-use list head before the loop since it is
1482 // already locked (by this thread):
1483 in_use_tail = in_use_list;
1484 in_use_count++;
1485 for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL; cur_om = unmarked_next(cur_om)) {
1486 if (is_locked(cur_om)) {
1487 // cur_om is locked so there must be a racing walker thread ahead
1826 } else {
1827 // Deflate the monitor if it is no longer being used
1828 // It's idle - scavenge and return to the global free list
1829 // plain old deflation ...
1830 if (log_is_enabled(Trace, monitorinflation)) {
1831 ResourceMark rm;
1832 log_trace(monitorinflation)("deflate_monitor: "
1833 "object=" INTPTR_FORMAT ", mark="
1834 INTPTR_FORMAT ", type='%s'", p2i(obj),
1835 mark.value(), obj->klass()->external_name());
1836 }
1837
1838 // Restore the header back to obj
1839 obj->release_set_mark(dmw);
1840 mid->clear();
1841
1842 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
1843 p2i(mid->object()));
1844
1845 // Move the deflated ObjectMonitor to the working free list
1846 // defined by free_head_p and free_tail_p. The working list is
1847 // local so no need for a memory barrier.
1848 if (*free_head_p == NULL) *free_head_p = mid;
1849 if (*free_tail_p != NULL) {
1850 // We append to the list so the caller can use mid->_next_om
1851 // to fix the linkages in its context.
1852 ObjectMonitor* prevtail = *free_tail_p;
1853 // Should have been cleaned up by the caller:
1854 // Note: Should not have to lock prevtail here since we're at a
1855 // safepoint and ObjectMonitors on the local free list should
1856 // not be accessed in parallel.
1857 #ifdef ASSERT
1858 ObjectMonitor* l_next_om = Atomic::load(&prevtail->_next_om);
1859 #endif
1860 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
1861 set_next(prevtail, mid);
1862 }
1863 *free_tail_p = mid;
1864 // At this point, mid->_next_om still refers to its current
1865 // value and another ObjectMonitor's _next_om field still
1866 // refers to this ObjectMonitor. Those linkages have to be
1867 // cleaned up by the caller who has the complete context.
1874 // The given list could be a per-thread list or a global list.
1875 //
1876 // In the case of parallel processing of thread local monitor lists,
1877 // work is done by Threads::parallel_threads_do() which ensures that
1878 // each Java thread is processed by exactly one worker thread, and
1879 // thus avoid conflicts that would arise when worker threads would
1880 // process the same monitor lists concurrently.
1881 //
1882 // See also ParallelSPCleanupTask and
1883 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
1884 // Threads::parallel_java_threads_do() in thread.cpp.
1885 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor** list_p,
1886 int* count_p,
1887 ObjectMonitor** free_head_p,
1888 ObjectMonitor** free_tail_p) {
1889 ObjectMonitor* cur_mid_in_use = NULL;
1890 ObjectMonitor* mid = NULL;
1891 ObjectMonitor* next = NULL;
1892 int deflated_count = 0;
1893
1894 // We use the simpler lock-mid-as-we-go protocol to prevent races
1895 // with a list walker thread since this caller is the only one doing
1896 // deletions on this list during the safepoint.
1897 if ((mid = get_list_head_locked(list_p)) == NULL) {
1898 return 0; // The list is empty so nothing to deflate.
1899 }
1900 next = unmarked_next(mid);
1901
1902 while (true) {
1903 oop obj = (oop) mid->object();
1904 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
1905 // Deflation succeeded and already updated free_head_p and
1906 // free_tail_p as needed. Finish the move to the local free list
1907 // by unlinking mid from the global or per-thread in-use list.
1908 if (cur_mid_in_use == NULL) {
1909 // mid is the list head and it is locked. Switch the list head
1910 // to next which unlocks the list head, but leaves mid locked:
1911 Atomic::store(list_p, next);
1912 } else {
1913 // mid is locked. Switch cur_mid_in_use's next field to next
1914 // which is safe because we have no parallel list deletions,
1915 // but we leave mid locked:
1916 set_next(cur_mid_in_use, next);
1917 }
1918 // At this point mid is disconnected from the in-use list so
1919 // its lock no longer has any effects on the in-use list.
1920 deflated_count++;
1921 Atomic::dec(count_p);
1922 // mid is current tail in the free_head_p list so NULL terminate it
1923 // (which also unlocks it):
1924 set_next(mid, NULL);
1925 } else {
1926 om_unlock(mid);
1927 cur_mid_in_use = mid;
1928 }
1929 // All the list management is done so move on to the next one:
1930 mid = next;
1931 if (mid == NULL) {
1932 break; // Reached end of the list so nothing more to deflate.
1933 }
1934 // Lock mid so we can possibly deflate it:
1935 om_lock(mid);
1936 next = unmarked_next(mid);
1937 }
1938 return deflated_count;
1939 }
1940
1941 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
1942 counters->n_in_use = 0; // currently associated with objects
1943 counters->n_in_circulation = 0; // extant
1944 counters->n_scavenged = 0; // reclaimed (global and per-thread)
1945 counters->per_thread_scavenged = 0; // per-thread scavenge total
1946 counters->per_thread_times = 0.0; // per-thread scavenge times
1947 OrderAccess::storestore(); // flush inits for worker threads
1948 }
1949
1950 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
1951 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1952 bool deflated = false;
1953
1954 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
1955 ObjectMonitor* free_tail_p = NULL;
1956 elapsedTimer timer;
1957
1958 if (log_is_enabled(Info, monitorinflation)) {
1959 timer.start();
1960 }
1961
1962 // Note: the thread-local monitors lists get deflated in
1963 // a separate pass. See deflate_thread_local_monitors().
1964
1965 // For moribund threads, scan LVars.in_use_list
1966 int deflated_count = 0;
1967 if (Atomic::load(&LVars.in_use_list) != NULL) {
1968 // Update n_in_circulation before LVars.in_use_count is updated by deflation.
1969 Atomic::add(&counters->n_in_circulation, Atomic::load(&LVars.in_use_count));
1970
1971 deflated_count = deflate_monitor_list(&LVars.in_use_list, &LVars.in_use_count, &free_head_p, &free_tail_p);
1972 Atomic::add(&counters->n_in_use, Atomic::load(&LVars.in_use_count));
1973 }
1974
1975 if (free_head_p != NULL) {
1976 // Move the deflated ObjectMonitors back to the global free list.
1977 // The working list is local so no need for a memory barrier.
1978 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
1979 #ifdef ASSERT
1980 ObjectMonitor* l_next_om = Atomic::load(&free_tail_p->_next_om);
1981 #endif
1982 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
1983 prepend_list_to_global_free_list(free_head_p, free_tail_p, deflated_count);
1984 Atomic::add(&counters->n_scavenged, deflated_count);
1985 }
1986 timer.stop();
1987
1988 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1989 LogStreamHandle(Info, monitorinflation) lsh_info;
1990 LogStream* ls = NULL;
1991 if (log_is_enabled(Debug, monitorinflation)) {
1992 ls = &lsh_debug;
1993 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
1994 ls = &lsh_info;
1995 }
1996 if (ls != NULL) {
1997 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2027 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
2028 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2029
2030 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2031 ObjectMonitor* free_tail_p = NULL;
2032 elapsedTimer timer;
2033
2034 if (log_is_enabled(Info, safepoint, cleanup) ||
2035 log_is_enabled(Info, monitorinflation)) {
2036 timer.start();
2037 }
2038
2039 // Update n_in_circulation before om_in_use_count is updated by deflation.
2040 Atomic::add(&counters->n_in_circulation, Atomic::load(&thread->om_in_use_count));
2041
2042 int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p);
2043 Atomic::add(&counters->n_in_use, Atomic::load(&thread->om_in_use_count));
2044
2045 if (free_head_p != NULL) {
2046 // Move the deflated ObjectMonitors back to the global free list.
2047 // The working list is local so no need for a memory barrier.
2048 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2049 #ifdef ASSERT
2050 ObjectMonitor* l_next_om = Atomic::load(&free_tail_p->_next_om);
2051 #endif
2052 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2053 prepend_list_to_global_free_list(free_head_p, free_tail_p, deflated_count);
2054 Atomic::add(&counters->n_scavenged, deflated_count);
2055 Atomic::add(&counters->per_thread_scavenged, deflated_count);
2056 }
2057
2058 timer.stop();
2059 // Safepoint logging cares about cumulative per_thread_times and
2060 // we'll capture most of the cost, but not the muxRelease() which
2061 // should be cheap.
2062 counters->per_thread_times += timer.seconds();
2063
2064 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2065 LogStreamHandle(Info, monitorinflation) lsh_info;
2066 LogStream* ls = NULL;
2067 if (log_is_enabled(Debug, monitorinflation)) {
2568 }
2569
2570 #ifndef PRODUCT
2571
2572 // Check if monitor belongs to the monitor cache
2573 // The list is grow-only so it's *relatively* safe to traverse
2574 // the list of extant blocks without taking a lock.
2575
2576 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
2577 PaddedObjectMonitor* block = Atomic::load(&g_block_list);
2578 while (block != NULL) {
2579 assert(block->object() == CHAINMARKER, "must be a block header");
2580 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
2581 address mon = (address)monitor;
2582 address blk = (address)block;
2583 size_t diff = mon - blk;
2584 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
2585 return 1;
2586 }
2587 // unmarked_next() is not needed with g_block_list (no locking
2588 // used with with block linkage _next_om fields).
2589 block = (PaddedObjectMonitor*)Atomic::load(&block->_next_om);
2590 }
2591 return 0;
2592 }
2593
2594 #endif
|
146 static ListGlobals LVars;
147
148 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
149
150
151 // =====================> Spin-lock functions
152
153 // ObjectMonitors are not lockable outside of this file. We use spin-locks
154 // implemented using a bit in the _next_om field instead of the heavier
155 // weight locking mechanisms for faster list management.
156
157 #define OM_LOCK_BIT 0x1
158
159 // Return true if the ObjectMonitor is locked.
160 // Otherwise returns false.
161 static bool is_locked(ObjectMonitor* om) {
162 return ((intptr_t)Atomic::load(&om->_next_om) & OM_LOCK_BIT) == OM_LOCK_BIT;
163 }
164
165 // Mark an ObjectMonitor* with OM_LOCK_BIT and return it.
166 static ObjectMonitor* mark_om_ptr(ObjectMonitor* om) {
167 return (ObjectMonitor*)((intptr_t)om | OM_LOCK_BIT);
168 }
169
170 // Return the unmarked next field in an ObjectMonitor. Note: the next
171 // field may or may not have been marked with OM_LOCK_BIT originally.
172 static ObjectMonitor* unmarked_next(ObjectMonitor* om) {
173 return (ObjectMonitor*)((intptr_t)Atomic::load(&om->_next_om) & ~OM_LOCK_BIT);
174 }
175
176 // Try to lock an ObjectMonitor. Returns true if locking was successful.
177 // Otherwise returns false.
178 static bool try_om_lock(ObjectMonitor* om) {
179 // Get current next field without any OM_LOCK_BIT value.
180 ObjectMonitor* next = unmarked_next(om);
181 if (Atomic::cmpxchg(&om->_next_om, next, mark_om_ptr(next)) != next) {
182 return false; // Cannot lock the ObjectMonitor.
183 }
184 return true;
185 }
186
187 // Lock an ObjectMonitor.
188 static void om_lock(ObjectMonitor* om) {
189 while (true) {
190 if (try_om_lock(om)) {
191 return;
192 }
193 }
194 }
195
196 // Unlock an ObjectMonitor.
197 static void om_unlock(ObjectMonitor* om) {
198 ObjectMonitor* next = Atomic::load(&om->_next_om);
199 guarantee(((intptr_t)next & OM_LOCK_BIT) == OM_LOCK_BIT, "next=" INTPTR_FORMAT
200 " must have OM_LOCK_BIT=%x set.", p2i(next), OM_LOCK_BIT);
201
202 next = (ObjectMonitor*)((intptr_t)next & ~OM_LOCK_BIT); // Clear OM_LOCK_BIT.
203 Atomic::store(&om->_next_om, next);
204 }
205
206 // Get the list head after locking it. Returns the list head or NULL
207 // if the list is empty.
208 static ObjectMonitor* get_list_head_locked(ObjectMonitor** list_p) {
209 while (true) {
210 ObjectMonitor* mid = Atomic::load(list_p);
211 if (mid == NULL) {
212 return NULL; // The list is empty.
213 }
214 if (try_om_lock(mid)) {
215 if (Atomic::load(list_p) != mid) {
216 // The list head changed before we could lock it so we have to retry.
217 om_unlock(mid);
218 continue;
219 }
220 return mid;
221 }
222 }
223 }
224
225 #undef OM_LOCK_BIT
226
227
228 // =====================> List Management functions
229
230 // Set the next field in an ObjectMonitor to the specified value.
231 static void set_next(ObjectMonitor* om, ObjectMonitor* value) {
232 Atomic::store(&om->_next_om, value);
233 }
234
235 // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is
236 // the last ObjectMonitor in the list and there are 'count' on the list.
237 // Also updates the specified *count_p.
238 static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail,
239 int count, ObjectMonitor** list_p,
240 int* count_p) {
241 while (true) {
242 ObjectMonitor* cur = Atomic::load(list_p);
243 // Prepend list to *list_p.
244 if (!try_om_lock(tail)) {
1134 // assert(mark.is_neutral(), "sanity check");
1135
1136 return NULL;
1137 }
1138
1139 // Visitors ...
1140
1141 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
1142 PaddedObjectMonitor* block = Atomic::load(&g_block_list);
1143 while (block != NULL) {
1144 assert(block->object() == CHAINMARKER, "must be a block header");
1145 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
1146 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
1147 oop object = (oop)mid->object();
1148 if (object != NULL) {
1149 // Only process with closure if the object is set.
1150 closure->do_monitor(mid);
1151 }
1152 }
1153 // unmarked_next() is not needed with g_block_list (no locking
1154 // used with block linkage _next_om fields).
1155 block = (PaddedObjectMonitor*)Atomic::load(&block->_next_om);
1156 }
1157 }
1158
1159 static bool monitors_used_above_threshold() {
1160 int population = Atomic::load(&LVars.population);
1161 if (population == 0) {
1162 return false;
1163 }
1164 if (MonitorUsedDeflationThreshold > 0) {
1165 int monitors_used = population - Atomic::load(&LVars.free_count);
1166 int monitor_usage = (monitors_used * 100LL) / population;
1167 return monitor_usage > MonitorUsedDeflationThreshold;
1168 }
1169 return false;
1170 }
1171
1172 // Returns true if MonitorBound is set (> 0) and if the specified
1173 // cnt is > MonitorBound. Otherwise returns false.
1174 static bool is_MonitorBound_exceeded(const int cnt) {
1175 const int mx = MonitorBound;
1176 return mx > 0 && cnt > mx;
1177 }
1178
1179 bool ObjectSynchronizer::is_cleanup_needed() {
1180 if (monitors_used_above_threshold()) {
1181 // Too many monitors in use.
1182 return true;
1183 }
1184 return needs_monitor_scavenge();
1185 }
1186
1261
1262 static void InduceScavenge(Thread* self, const char * Whence) {
1263 // Induce STW safepoint to trim monitors
1264 // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
1265 // More precisely, trigger a cleanup safepoint as the number
1266 // of active monitors passes the specified threshold.
1267 // TODO: assert thread state is reasonable
1268
1269 if (Atomic::xchg(&_forceMonitorScavenge, 1) == 0) {
1270 VMThread::check_for_forced_cleanup();
1271 }
1272 }
1273
1274 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self) {
1275 // A large MAXPRIVATE value reduces both list lock contention
1276 // and list coherency traffic, but also tends to increase the
1277 // number of ObjectMonitors in circulation as well as the STW
1278 // scavenge costs. As usual, we lean toward time in space-time
1279 // tradeoffs.
1280 const int MAXPRIVATE = 1024;
1281 NoSafepointVerifier nsv;
1282
1283 stringStream ss;
1284 for (;;) {
1285 ObjectMonitor* m;
1286
1287 // 1: try to allocate from the thread's local om_free_list.
1288 // Threads will attempt to allocate first from their local list, then
1289 // from the global list, and only after those attempts fail will the
1290 // thread attempt to instantiate new monitors. Thread-local free lists
1291 // improve allocation latency, as well as reducing coherency traffic
1292 // on the shared global list.
1293 m = take_from_start_of_om_free_list(self);
1294 if (m != NULL) {
1295 guarantee(m->object() == NULL, "invariant");
1296 prepend_to_om_in_use_list(self, m);
1297 return m;
1298 }
1299
1300 // 2: try to allocate from the global LVars.free_list
1301 // CONSIDER: use muxTry() instead of muxAcquire().
1369 prepend_block_to_lists(temp);
1370 }
1371 }
1372
1373 // Place "m" on the caller's private per-thread om_free_list.
1374 // In practice there's no need to clamp or limit the number of
1375 // monitors on a thread's om_free_list as the only non-allocation time
1376 // we'll call om_release() is to return a monitor to the free list after
1377 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1378 // accumulate on a thread's free list.
1379 //
1380 // Key constraint: all ObjectMonitors on a thread's free list and the global
1381 // free list must have their object field set to null. This prevents the
1382 // scavenger -- deflate_monitor_list() -- from reclaiming them while we
1383 // are trying to release them.
1384
1385 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1386 bool from_per_thread_alloc) {
1387 guarantee(m->header().value() == 0, "invariant");
1388 guarantee(m->object() == NULL, "invariant");
1389 NoSafepointVerifier nsv;
1390
1391 stringStream ss;
1392 guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1393 "%s, recursions=" INTX_FORMAT, m->is_busy_to_string(&ss),
1394 m->_recursions);
1395 // _next_om is used for both per-thread in-use and free lists so
1396 // we have to remove 'm' from the in-use list first (as needed).
1397 if (from_per_thread_alloc) {
1398 // Need to remove 'm' from om_in_use_list.
1399 ObjectMonitor* mid = NULL;
1400 ObjectMonitor* next = NULL;
1401
1402 // This list walk can only race with another list walker since
1403 // deflation can only happen at a safepoint so we don't have to
1404 // worry about an ObjectMonitor being removed from this list
1405 // while we are walking it.
1406
1407 // Lock the list head to avoid racing with another list walker.
1408 if ((mid = get_list_head_locked(&self->om_in_use_list)) == NULL) {
1409 fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self));
1410 }
1411 next = unmarked_next(mid);
1412 if (m == mid) {
1413 // First special case:
1414 // 'm' matches mid, is the list head and is locked. Switch the list
1415 // head to next which unlocks the list head, but leaves the extracted
1416 // mid locked:
1417 Atomic::store(&self->om_in_use_list, next);
1418 } else if (m == next) {
1419 // Second special case:
1420 // 'm' matches next after the list head and we already have the list
1421 // head locked so set mid to what we are extracting:
1422 mid = next;
1423 // Lock mid to prevent races with a list walker:
1424 om_lock(mid);
1425 // Update next to what follows mid (if anything):
1426 next = unmarked_next(mid);
1427 // Switch next after the list head to new next which unlocks the
1428 // list head, but leaves the extracted mid locked:
1429 set_next(self->om_in_use_list, next);
1430 } else {
1431 // We have to search the list to find 'm'.
1432 om_unlock(mid); // unlock the list head
1433 guarantee(next != NULL, "thread=" INTPTR_FORMAT ": om_in_use_list=" INTPTR_FORMAT
1434 " is too short.", p2i(self), p2i(self->om_in_use_list));
1435 // Our starting anchor is next after the list head which is the
1436 // last ObjectMonitor we checked:
1437 ObjectMonitor* anchor = next;
1438 while ((mid = unmarked_next(anchor)) != NULL) {
1439 if (m == mid) {
1440 // We found 'm' on the per-thread in-use list so extract it.
1441 om_lock(anchor); // Lock the anchor so we can safely modify it.
1442 // Update next to what follows mid (if anything):
1443 next = unmarked_next(mid);
1444 // Switch next after the anchor to new next which unlocks the
1445 // anchor, but leaves the extracted mid locked:
1446 set_next(anchor, next);
1447 break;
1448 } else {
1449 anchor = mid;
1450 }
1451 }
1452 }
1453
1454 if (mid == NULL) {
1455 // Reached end of the list and didn't find 'm' so:
1456 fatal("thread=" INTPTR_FORMAT " must find m=" INTPTR_FORMAT "on om_in_use_list="
1457 INTPTR_FORMAT, p2i(self), p2i(m), p2i(self->om_in_use_list));
1458 }
1459
1460 // At this point mid is disconnected from the in-use list so
1461 // its lock no longer has any effects on the in-use list.
1462 Atomic::dec(&self->om_in_use_count);
1463 // Unlock mid, but leave the next value for any lagging list
1464 // walkers. It will get cleaned up when mid is prepended to
1465 // the thread's free list:
1466 om_unlock(mid);
1467 }
1468
1469 prepend_to_om_free_list(self, m);
1470 }
1471
1472 // Return ObjectMonitors on a moribund thread's free and in-use
1473 // lists to the appropriate global lists. The ObjectMonitors on the
1474 // per-thread in-use list may still be in use by other threads.
1475 //
1476 // We currently call om_flush() from Threads::remove() before the
1477 // thread has been excised from the thread list and is no longer a
1478 // mutator. This means that om_flush() cannot run concurrently with
1479 // a safepoint and interleave with deflate_idle_monitors(). In
1480 // particular, this ensures that the thread's in-use monitors are
1481 // scanned by a GC safepoint, either via Thread::oops_do() (before
1482 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1483 // om_flush() is called).
1484
1485 void ObjectSynchronizer::om_flush(Thread* self) {
1486 // Process the per-thread in-use list first to be consistent.
1487 int in_use_count = 0;
1488 ObjectMonitor* in_use_list = NULL;
1489 ObjectMonitor* in_use_tail = NULL;
1490 NoSafepointVerifier nsv;
1491
1492 // This function can race with a list walker thread so we lock the
1493 // list head to prevent confusion.
1494 if ((in_use_list = get_list_head_locked(&self->om_in_use_list)) != NULL) {
1495 // At this point, we have locked the in-use list head so a racing
1496 // thread cannot come in after us. However, a racing thread could
1497 // be ahead of us; we'll detect that and delay to let it finish.
1498 //
1499 // The thread is going away, however the ObjectMonitors on the
1500 // om_in_use_list may still be in-use by other threads. Link
1501 // them to in_use_tail, which will be linked into the global
1502 // in-use list (LVars.in_use_list) below.
1503 //
1504 // Account for the in-use list head before the loop since it is
1505 // already locked (by this thread):
1506 in_use_tail = in_use_list;
1507 in_use_count++;
1508 for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL; cur_om = unmarked_next(cur_om)) {
1509 if (is_locked(cur_om)) {
1510 // cur_om is locked so there must be a racing walker thread ahead
1849 } else {
1850 // Deflate the monitor if it is no longer being used
1851 // It's idle - scavenge and return to the global free list
1852 // plain old deflation ...
1853 if (log_is_enabled(Trace, monitorinflation)) {
1854 ResourceMark rm;
1855 log_trace(monitorinflation)("deflate_monitor: "
1856 "object=" INTPTR_FORMAT ", mark="
1857 INTPTR_FORMAT ", type='%s'", p2i(obj),
1858 mark.value(), obj->klass()->external_name());
1859 }
1860
1861 // Restore the header back to obj
1862 obj->release_set_mark(dmw);
1863 mid->clear();
1864
1865 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
1866 p2i(mid->object()));
1867
1868 // Move the deflated ObjectMonitor to the working free list
1869 // defined by free_head_p and free_tail_p.
1870 if (*free_head_p == NULL) *free_head_p = mid;
1871 if (*free_tail_p != NULL) {
1872 // We append to the list so the caller can use mid->_next_om
1873 // to fix the linkages in its context.
1874 ObjectMonitor* prevtail = *free_tail_p;
1875 // Should have been cleaned up by the caller:
1876 // Note: Should not have to lock prevtail here since we're at a
1877 // safepoint and ObjectMonitors on the local free list should
1878 // not be accessed in parallel.
1879 #ifdef ASSERT
1880 ObjectMonitor* l_next_om = Atomic::load(&prevtail->_next_om);
1881 #endif
1882 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
1883 set_next(prevtail, mid);
1884 }
1885 *free_tail_p = mid;
1886 // At this point, mid->_next_om still refers to its current
1887 // value and another ObjectMonitor's _next_om field still
1888 // refers to this ObjectMonitor. Those linkages have to be
1889 // cleaned up by the caller who has the complete context.
1896 // The given list could be a per-thread list or a global list.
1897 //
1898 // In the case of parallel processing of thread local monitor lists,
1899 // work is done by Threads::parallel_threads_do() which ensures that
1900 // each Java thread is processed by exactly one worker thread, and
1901 // thus avoid conflicts that would arise when worker threads would
1902 // process the same monitor lists concurrently.
1903 //
1904 // See also ParallelSPCleanupTask and
1905 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
1906 // Threads::parallel_java_threads_do() in thread.cpp.
1907 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor** list_p,
1908 int* count_p,
1909 ObjectMonitor** free_head_p,
1910 ObjectMonitor** free_tail_p) {
1911 ObjectMonitor* cur_mid_in_use = NULL;
1912 ObjectMonitor* mid = NULL;
1913 ObjectMonitor* next = NULL;
1914 int deflated_count = 0;
1915
1916 // This list walk executes at a safepoint and does not race with any
1917 // other list walkers.
1918
1919 for (mid = Atomic::load(list_p); mid != NULL; mid = next) {
1920 next = unmarked_next(mid);
1921 oop obj = (oop) mid->object();
1922 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
1923 // Deflation succeeded and already updated free_head_p and
1924 // free_tail_p as needed. Finish the move to the local free list
1925 // by unlinking mid from the global or per-thread in-use list.
1926 if (cur_mid_in_use == NULL) {
1927 // mid is the list head so switch the list head to next:
1928 Atomic::store(list_p, next);
1929 } else {
1930 // Switch cur_mid_in_use's next field to next:
1931 set_next(cur_mid_in_use, next);
1932 }
1933 // At this point mid is disconnected from the in-use list.
1934 deflated_count++;
1935 Atomic::dec(count_p);
1936 // mid is current tail in the free_head_p list so NULL terminate it:
1937 set_next(mid, NULL);
1938 } else {
1939 cur_mid_in_use = mid;
1940 }
1941 }
1942 return deflated_count;
1943 }
1944
1945 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
1946 counters->n_in_use = 0; // currently associated with objects
1947 counters->n_in_circulation = 0; // extant
1948 counters->n_scavenged = 0; // reclaimed (global and per-thread)
1949 counters->per_thread_scavenged = 0; // per-thread scavenge total
1950 counters->per_thread_times = 0.0; // per-thread scavenge times
1951 }
1952
1953 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
1954 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1955 bool deflated = false;
1956
1957 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
1958 ObjectMonitor* free_tail_p = NULL;
1959 elapsedTimer timer;
1960
1961 if (log_is_enabled(Info, monitorinflation)) {
1962 timer.start();
1963 }
1964
1965 // Note: the thread-local monitors lists get deflated in
1966 // a separate pass. See deflate_thread_local_monitors().
1967
1968 // For moribund threads, scan LVars.in_use_list
1969 int deflated_count = 0;
1970 if (Atomic::load(&LVars.in_use_list) != NULL) {
1971 // Update n_in_circulation before LVars.in_use_count is updated by deflation.
1972 Atomic::add(&counters->n_in_circulation, Atomic::load(&LVars.in_use_count));
1973
1974 deflated_count = deflate_monitor_list(&LVars.in_use_list, &LVars.in_use_count, &free_head_p, &free_tail_p);
1975 Atomic::add(&counters->n_in_use, Atomic::load(&LVars.in_use_count));
1976 }
1977
1978 if (free_head_p != NULL) {
1979 // Move the deflated ObjectMonitors back to the global free list.
1980 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
1981 #ifdef ASSERT
1982 ObjectMonitor* l_next_om = Atomic::load(&free_tail_p->_next_om);
1983 #endif
1984 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
1985 prepend_list_to_global_free_list(free_head_p, free_tail_p, deflated_count);
1986 Atomic::add(&counters->n_scavenged, deflated_count);
1987 }
1988 timer.stop();
1989
1990 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1991 LogStreamHandle(Info, monitorinflation) lsh_info;
1992 LogStream* ls = NULL;
1993 if (log_is_enabled(Debug, monitorinflation)) {
1994 ls = &lsh_debug;
1995 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
1996 ls = &lsh_info;
1997 }
1998 if (ls != NULL) {
1999 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2029 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
2030 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2031
2032 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2033 ObjectMonitor* free_tail_p = NULL;
2034 elapsedTimer timer;
2035
2036 if (log_is_enabled(Info, safepoint, cleanup) ||
2037 log_is_enabled(Info, monitorinflation)) {
2038 timer.start();
2039 }
2040
2041 // Update n_in_circulation before om_in_use_count is updated by deflation.
2042 Atomic::add(&counters->n_in_circulation, Atomic::load(&thread->om_in_use_count));
2043
2044 int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p);
2045 Atomic::add(&counters->n_in_use, Atomic::load(&thread->om_in_use_count));
2046
2047 if (free_head_p != NULL) {
2048 // Move the deflated ObjectMonitors back to the global free list.
2049 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2050 #ifdef ASSERT
2051 ObjectMonitor* l_next_om = Atomic::load(&free_tail_p->_next_om);
2052 #endif
2053 assert(l_next_om == NULL, "must be NULL: _next_om=" INTPTR_FORMAT, p2i(l_next_om));
2054 prepend_list_to_global_free_list(free_head_p, free_tail_p, deflated_count);
2055 Atomic::add(&counters->n_scavenged, deflated_count);
2056 Atomic::add(&counters->per_thread_scavenged, deflated_count);
2057 }
2058
2059 timer.stop();
2060 // Safepoint logging cares about cumulative per_thread_times and
2061 // we'll capture most of the cost, but not the muxRelease() which
2062 // should be cheap.
2063 counters->per_thread_times += timer.seconds();
2064
2065 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2066 LogStreamHandle(Info, monitorinflation) lsh_info;
2067 LogStream* ls = NULL;
2068 if (log_is_enabled(Debug, monitorinflation)) {
2569 }
2570
2571 #ifndef PRODUCT
2572
2573 // Check if monitor belongs to the monitor cache
2574 // The list is grow-only so it's *relatively* safe to traverse
2575 // the list of extant blocks without taking a lock.
2576
2577 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
2578 PaddedObjectMonitor* block = Atomic::load(&g_block_list);
2579 while (block != NULL) {
2580 assert(block->object() == CHAINMARKER, "must be a block header");
2581 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
2582 address mon = (address)monitor;
2583 address blk = (address)block;
2584 size_t diff = mon - blk;
2585 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
2586 return 1;
2587 }
2588 // unmarked_next() is not needed with g_block_list (no locking
2589 // used with block linkage _next_om fields).
2590 block = (PaddedObjectMonitor*)Atomic::load(&block->_next_om);
2591 }
2592 return 0;
2593 }
2594
2595 #endif
|