1 /*
2 * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "logging/log.hpp"
28 #include "logging/logStream.hpp"
29 #include "jfr/jfrEvents.hpp"
30 #include "memory/allocation.inline.hpp"
31 #include "memory/metaspaceShared.hpp"
32 #include "memory/padded.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/markWord.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "runtime/atomic.hpp"
38 #include "runtime/biasedLocking.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/handshake.hpp"
41 #include "runtime/interfaceSupport.inline.hpp"
42 #include "runtime/mutexLocker.hpp"
43 #include "runtime/objectMonitor.hpp"
44 #include "runtime/objectMonitor.inline.hpp"
45 #include "runtime/osThread.hpp"
46 #include "runtime/safepointVerifiers.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "runtime/synchronizer.hpp"
50 #include "runtime/thread.inline.hpp"
51 #include "runtime/timer.hpp"
52 #include "runtime/vframe.hpp"
53 #include "runtime/vmThread.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/dtrace.hpp"
56 #include "utilities/events.hpp"
57 #include "utilities/preserveException.hpp"
58
59 // The "core" versions of monitor enter and exit reside in this file.
60 // The interpreter and compilers contain specialized transliterated
61 // variants of the enter-exit fast-path operations. See i486.ad fast_lock(),
62 // for instance. If you make changes here, make sure to modify the
63 // interpreter, and both C1 and C2 fast-path inline locking code emission.
64 //
65 // -----------------------------------------------------------------------------
66
67 #ifdef DTRACE_ENABLED
68
69 // Only bother with this argument setup if dtrace is available
70 // TODO-FIXME: probes should not fire when caller is _blocked. assert() accordingly.
71
72 #define DTRACE_MONITOR_PROBE_COMMON(obj, thread) \
73 char* bytes = NULL; \
74 int len = 0; \
75 jlong jtid = SharedRuntime::get_java_tid(thread); \
76 Symbol* klassname = ((oop)(obj))->klass()->name(); \
77 if (klassname != NULL) { \
78 bytes = (char*)klassname->bytes(); \
79 len = klassname->utf8_length(); \
80 }
81
82 #define DTRACE_MONITOR_WAIT_PROBE(monitor, obj, thread, millis) \
83 { \
84 if (DTraceMonitorProbes) { \
85 DTRACE_MONITOR_PROBE_COMMON(obj, thread); \
86 HOTSPOT_MONITOR_WAIT(jtid, \
87 (uintptr_t)(monitor), bytes, len, (millis)); \
88 } \
89 }
90
91 #define HOTSPOT_MONITOR_PROBE_notify HOTSPOT_MONITOR_NOTIFY
92 #define HOTSPOT_MONITOR_PROBE_notifyAll HOTSPOT_MONITOR_NOTIFYALL
93 #define HOTSPOT_MONITOR_PROBE_waited HOTSPOT_MONITOR_WAITED
94
95 #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread) \
96 { \
97 if (DTraceMonitorProbes) { \
98 DTRACE_MONITOR_PROBE_COMMON(obj, thread); \
99 HOTSPOT_MONITOR_PROBE_##probe(jtid, /* probe = waited */ \
100 (uintptr_t)(monitor), bytes, len); \
101 } \
102 }
103
104 #else // ndef DTRACE_ENABLED
105
106 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon) {;}
107 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon) {;}
108
109 #endif // ndef DTRACE_ENABLED
110
111 // This exists only as a workaround of dtrace bug 6254741
112 int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
113 DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
114 return 0;
115 }
116
117 #define NINFLATIONLOCKS 256
118 static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];
119
120 // global list of blocks of monitors
121 PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL;
122 bool volatile ObjectSynchronizer::_is_async_deflation_requested = false;
123 bool volatile ObjectSynchronizer::_is_special_deflation_requested = false;
124 jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0;
125
126 // Global ObjectMonitor free list. Newly allocated and deflated
127 // ObjectMonitors are prepended here.
128 static ObjectMonitor* volatile g_free_list = NULL;
129 // Global ObjectMonitor in-use list. When a JavaThread is exiting,
130 // ObjectMonitors on its per-thread in-use list are prepended here.
131 static ObjectMonitor* volatile g_om_in_use_list = NULL;
132 // Global ObjectMonitor wait list. If HandshakeAfterDeflateIdleMonitors
133 // is true, deflated ObjectMonitors wait on this list until after a
134 // handshake or a safepoint for platforms that don't support handshakes.
135 // After the handshake or safepoint, the deflated ObjectMonitors are
136 // prepended to g_free_list.
137 static ObjectMonitor* volatile g_wait_list = NULL;
138
139 static volatile int g_om_free_count = 0; // # on g_free_list
140 static volatile int g_om_in_use_count = 0; // # on g_om_in_use_list
141 static volatile int g_om_population = 0; // # Extant -- in circulation
142 static volatile int g_om_wait_count = 0; // # on g_wait_list
143
144 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
145
146
147 // =====================> List Management functions
148
149 // Return true if the ObjectMonitor's next field is marked.
150 // Otherwise returns false.
151 static bool is_next_marked(ObjectMonitor* om) {
152 // Use load_acquire() since _next_om fields are updated with a
153 // release_store().
154 return ((intptr_t)Atomic::load_acquire(&om->_next_om) & 0x1) != 0;
155 }
156
157 // Mark an ObjectMonitor* and return it. Note: the om parameter
158 // may or may not have been marked originally.
159 static ObjectMonitor* mark_om_ptr(ObjectMonitor* om) {
160 return (ObjectMonitor*)((intptr_t)om | 0x1);
161 }
162
163 // Mark the next field in an ObjectMonitor. If marking was successful,
164 // then the unmarked next field is returned via parameter and true is
165 // returned. Otherwise false is returned.
166 static bool mark_next(ObjectMonitor* om, ObjectMonitor** next_p) {
167 // Get current next field without any marking value.
168 ObjectMonitor* next = (ObjectMonitor*)((intptr_t)om->_next_om & ~0x1);
169 if (Atomic::cmpxchg(&om->_next_om, next, mark_om_ptr(next)) != next) {
170 return false; // Could not mark the next field or it was already marked.
171 }
172 *next_p = next;
173 return true;
174 }
175
176 // Loop until we mark the next field in an ObjectMonitor. The unmarked
177 // next field is returned.
178 static ObjectMonitor* mark_next_loop(ObjectMonitor* om) {
179 ObjectMonitor* next;
180 while (true) {
181 if (mark_next(om, &next)) {
182 // Marked om's next field so return the unmarked value.
183 return next;
184 }
185 }
186 }
187
188 // Set the next field in an ObjectMonitor to the specified value.
189 // The caller of set_next() must be the same thread that marked the
190 // ObjectMonitor.
191 static void set_next(ObjectMonitor* om, ObjectMonitor* value) {
192 Atomic::release_store(&om->_next_om, value);
193 }
194
195 // Mark the next field in the list head ObjectMonitor. If marking was
196 // successful, then the mid and the unmarked next field are returned
197 // via parameter and true is returned. Otherwise false is returned.
198 static bool mark_list_head(ObjectMonitor* volatile * list_p,
199 ObjectMonitor** mid_p, ObjectMonitor** next_p) {
200 while (true) {
201 ObjectMonitor* mid = *list_p;
202 if (mid == NULL) {
203 return false; // The list is empty so nothing to mark.
204 }
205 if (mark_next(mid, next_p)) {
206 if (*list_p != mid) {
207 // The list head changed so we have to retry.
208 set_next(mid, *next_p); // unmark mid
209 continue;
210 }
211 // We marked next field to guard against races.
212 *mid_p = mid;
213 return true;
214 }
215 }
216 }
217
218 // Return the unmarked next field in an ObjectMonitor. Note: the next
219 // field may or may not have been marked originally.
220 static ObjectMonitor* unmarked_next(ObjectMonitor* om) {
221 // Use load_acquire() since _next_om fields are updated with a
222 // release_store().
223 return (ObjectMonitor*)((intptr_t)Atomic::load_acquire(&om->_next_om) & ~0x1);
224 }
225
226 // Mark the next ObjectMonitor for traversal. The current ObjectMonitor
227 // is unmarked after the next ObjectMonitor is marked. *cur_p and *next_p
228 // are updated to their next values in the list traversal. *cur_p is set
229 // to NULL when the end of the list is reached.
230 static void mark_next_for_traversal(ObjectMonitor** cur_p, ObjectMonitor** next_p) {
231 ObjectMonitor* prev = *cur_p; // Save current for unmarking.
232 if (*next_p == NULL) { // Reached the end of the list.
233 set_next(prev, NULL); // Unmark previous.
234 *cur_p = NULL; // Tell the caller we are done.
235 return;
236 }
237 (void)mark_next_loop(*next_p); // Mark next.
238 set_next(prev, *next_p); // Unmark previous.
239 *cur_p = *next_p; // Update current.
240 *next_p = unmarked_next(*cur_p); // Update next.
241 }
242
243 // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is
244 // the last ObjectMonitor in the list and there are 'count' on the list.
245 // Also updates the specified *count_p.
246 static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail,
247 int count, ObjectMonitor* volatile* list_p,
248 volatile int* count_p) {
249 while (true) {
250 ObjectMonitor* cur = *list_p;
251 // Prepend list to *list_p.
252 ObjectMonitor* next = NULL;
253 if (!mark_next(tail, &next)) {
254 continue; // failed to mark next field so try it all again
255 }
256 set_next(tail, cur); // tail now points to cur (and unmarks tail)
257 if (cur == NULL) {
258 // No potential race with takers or other prependers since
259 // *list_p is empty.
260 if (Atomic::cmpxchg(list_p, cur, list) == cur) {
261 // Successfully switched *list_p to the list value.
262 Atomic::add(count_p, count);
263 break;
264 }
265 // Implied else: try it all again
266 } else {
267 // Try to mark next field to guard against races:
268 if (!mark_next(cur, &next)) {
269 continue; // failed to mark next field so try it all again
270 }
271 // We marked the next field so try to switch *list_p to the list value.
272 if (Atomic::cmpxchg(list_p, cur, list) != cur) {
273 // The list head has changed so unmark the next field and try again:
274 set_next(cur, next);
275 continue;
276 }
277 Atomic::add(count_p, count);
278 set_next(cur, next); // unmark next field
279 break;
280 }
281 }
282 }
283
284 // Prepend a newly allocated block of ObjectMonitors to g_block_list and
285 // g_free_list. Also updates g_om_population and g_om_free_count.
286 void ObjectSynchronizer::prepend_block_to_lists(PaddedObjectMonitor* new_blk) {
287 // First we handle g_block_list:
288 while (true) {
289 PaddedObjectMonitor* cur = g_block_list;
290 // Prepend new_blk to g_block_list. The first ObjectMonitor in
291 // a block is reserved for use as linkage to the next block.
292 new_blk[0]._next_om = cur;
293 if (Atomic::cmpxchg(&g_block_list, cur, new_blk) == cur) {
294 // Successfully switched g_block_list to the new_blk value.
295 Atomic::add(&g_om_population, _BLOCKSIZE - 1);
296 break;
297 }
298 // Implied else: try it all again
299 }
300
301 // Second we handle g_free_list:
302 prepend_list_to_common(new_blk + 1, &new_blk[_BLOCKSIZE - 1], _BLOCKSIZE - 1,
303 &g_free_list, &g_om_free_count);
304 }
305
306 // Prepend a list of ObjectMonitors to g_free_list. 'tail' is the last
307 // ObjectMonitor in the list and there are 'count' on the list. Also
308 // updates g_om_free_count.
309 static void prepend_list_to_g_free_list(ObjectMonitor* list,
310 ObjectMonitor* tail, int count) {
311 prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count);
312 }
313
314 // Prepend a list of ObjectMonitors to g_wait_list. 'tail' is the last
315 // ObjectMonitor in the list and there are 'count' on the list. Also
316 // updates g_om_wait_count.
317 static void prepend_list_to_g_wait_list(ObjectMonitor* list,
318 ObjectMonitor* tail, int count) {
319 assert(HandshakeAfterDeflateIdleMonitors, "sanity check");
320 prepend_list_to_common(list, tail, count, &g_wait_list, &g_om_wait_count);
321 }
322
323 // Prepend a list of ObjectMonitors to g_om_in_use_list. 'tail' is the last
324 // ObjectMonitor in the list and there are 'count' on the list. Also
325 // updates g_om_in_use_list.
326 static void prepend_list_to_g_om_in_use_list(ObjectMonitor* list,
327 ObjectMonitor* tail, int count) {
328 prepend_list_to_common(list, tail, count, &g_om_in_use_list, &g_om_in_use_count);
329 }
330
331 // Prepend an ObjectMonitor to the specified list. Also updates
332 // the specified counter.
333 static void prepend_to_common(ObjectMonitor* m, ObjectMonitor* volatile * list_p,
334 int volatile * count_p) {
335 while (true) {
336 (void)mark_next_loop(m); // mark m so we can safely update its next field
337 ObjectMonitor* cur = NULL;
338 ObjectMonitor* next = NULL;
339 // Mark the list head to guard against A-B-A race:
340 if (mark_list_head(list_p, &cur, &next)) {
341 // List head is now marked so we can safely switch it.
342 set_next(m, cur); // m now points to cur (and unmarks m)
343 *list_p = m; // Switch list head to unmarked m.
344 // mark_list_head() used cmpxchg() above, switching list head can be lazier:
345 OrderAccess::storestore();
346 set_next(cur, next); // Unmark the previous list head.
347 break;
348 }
349 // The list is empty so try to set the list head.
350 assert(cur == NULL, "cur must be NULL: cur=" INTPTR_FORMAT, p2i(cur));
351 set_next(m, cur); // m now points to NULL (and unmarks m)
352 if (Atomic::cmpxchg(list_p, cur, m) == cur) {
353 // List head is now unmarked m.
354 break;
355 }
356 // Implied else: try it all again
357 }
358 Atomic::inc(count_p);
359 }
360
361 // Prepend an ObjectMonitor to a per-thread om_free_list.
362 // Also updates the per-thread om_free_count.
363 static void prepend_to_om_free_list(Thread* self, ObjectMonitor* m) {
364 prepend_to_common(m, &self->om_free_list, &self->om_free_count);
365 }
366
367 // Prepend an ObjectMonitor to a per-thread om_in_use_list.
368 // Also updates the per-thread om_in_use_count.
369 static void prepend_to_om_in_use_list(Thread* self, ObjectMonitor* m) {
370 prepend_to_common(m, &self->om_in_use_list, &self->om_in_use_count);
371 }
372
373 // Take an ObjectMonitor from the start of the specified list. Also
374 // decrements the specified counter. Returns NULL if none are available.
375 static ObjectMonitor* take_from_start_of_common(ObjectMonitor* volatile * list_p,
376 int volatile * count_p) {
377 ObjectMonitor* next = NULL;
378 ObjectMonitor* take = NULL;
379 // Mark the list head to guard against A-B-A race:
380 if (!mark_list_head(list_p, &take, &next)) {
381 return NULL; // None are available.
382 }
383 // Switch marked list head to next (which unmarks the list head, but
384 // leaves take marked):
385 *list_p = next;
386 Atomic::dec(count_p);
387 // mark_list_head() used cmpxchg() above, switching list head can be lazier:
388 OrderAccess::storestore();
389 // Unmark take, but leave the next value for any lagging list
390 // walkers. It will get cleaned up when take is prepended to
391 // the in-use list:
392 set_next(take, next);
393 return take;
394 }
395
396 // Take an ObjectMonitor from the start of the global free-list. Also
397 // updates g_om_free_count. Returns NULL if none are available.
398 static ObjectMonitor* take_from_start_of_g_free_list() {
399 return take_from_start_of_common(&g_free_list, &g_om_free_count);
400 }
401
402 // Take an ObjectMonitor from the start of a per-thread free-list.
403 // Also updates om_free_count. Returns NULL if none are available.
404 static ObjectMonitor* take_from_start_of_om_free_list(Thread* self) {
405 return take_from_start_of_common(&self->om_free_list, &self->om_free_count);
406 }
407
408
409 // =====================> Quick functions
410
411 // The quick_* forms are special fast-path variants used to improve
412 // performance. In the simplest case, a "quick_*" implementation could
413 // simply return false, in which case the caller will perform the necessary
414 // state transitions and call the slow-path form.
415 // The fast-path is designed to handle frequently arising cases in an efficient
416 // manner and is just a degenerate "optimistic" variant of the slow-path.
417 // returns true -- to indicate the call was satisfied.
418 // returns false -- to indicate the call needs the services of the slow-path.
419 // A no-loitering ordinance is in effect for code in the quick_* family
420 // operators: safepoints or indefinite blocking (blocking that might span a
421 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
422 // entry.
423 //
424 // Consider: An interesting optimization is to have the JIT recognize the
425 // following common idiom:
426 // synchronized (someobj) { .... ; notify(); }
427 // That is, we find a notify() or notifyAll() call that immediately precedes
428 // the monitorexit operation. In that case the JIT could fuse the operations
429 // into a single notifyAndExit() runtime primitive.
430
431 bool ObjectSynchronizer::quick_notify(oopDesc* obj, Thread* self, bool all) {
432 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
433 assert(self->is_Java_thread(), "invariant");
434 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
435 NoSafepointVerifier nsv;
436 if (obj == NULL) return false; // slow-path for invalid obj
437 const markWord mark = obj->mark();
438
439 if (mark.has_locker() && self->is_lock_owned((address)mark.locker())) {
440 // Degenerate notify
441 // stack-locked by caller so by definition the implied waitset is empty.
442 return true;
443 }
444
445 if (mark.has_monitor()) {
446 ObjectMonitor* const mon = mark.monitor();
447 assert(mon->object() == obj, "invariant");
448 if (mon->owner() != self) return false; // slow-path for IMS exception
449
450 if (mon->first_waiter() != NULL) {
451 // We have one or more waiters. Since this is an inflated monitor
452 // that we own, we can transfer one or more threads from the waitset
453 // to the entrylist here and now, avoiding the slow-path.
454 if (all) {
455 DTRACE_MONITOR_PROBE(notifyAll, mon, obj, self);
456 } else {
457 DTRACE_MONITOR_PROBE(notify, mon, obj, self);
458 }
459 int free_count = 0;
460 do {
461 mon->INotify(self);
462 ++free_count;
463 } while (mon->first_waiter() != NULL && all);
464 OM_PERFDATA_OP(Notifications, inc(free_count));
465 }
466 return true;
467 }
468
469 // biased locking and any other IMS exception states take the slow-path
470 return false;
471 }
472
473
474 // The LockNode emitted directly at the synchronization site would have
475 // been too big if it were to have included support for the cases of inflated
476 // recursive enter and exit, so they go here instead.
477 // Note that we can't safely call AsyncPrintJavaStack() from within
478 // quick_enter() as our thread state remains _in_Java.
479
480 bool ObjectSynchronizer::quick_enter(oop obj, Thread* self,
481 BasicLock * lock) {
482 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
483 assert(self->is_Java_thread(), "invariant");
484 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
485 NoSafepointVerifier nsv;
486 if (obj == NULL) return false; // Need to throw NPE
487
488 while (true) {
489 const markWord mark = obj->mark();
490
491 if (mark.has_monitor()) {
492 ObjectMonitorHandle omh;
493 if (!omh.save_om_ptr(obj, mark)) {
494 // Lost a race with async deflation so try again.
495 assert(AsyncDeflateIdleMonitors, "sanity check");
496 continue;
497 }
498 ObjectMonitor* const m = omh.om_ptr();
499 assert(m->object() == obj, "invariant");
500 Thread* const owner = (Thread *) m->_owner;
501
502 // Lock contention and Transactional Lock Elision (TLE) diagnostics
503 // and observability
504 // Case: light contention possibly amenable to TLE
505 // Case: TLE inimical operations such as nested/recursive synchronization
506
507 if (owner == self) {
508 m->_recursions++;
509 return true;
510 }
511
512 // This Java Monitor is inflated so obj's header will never be
513 // displaced to this thread's BasicLock. Make the displaced header
514 // non-NULL so this BasicLock is not seen as recursive nor as
515 // being locked. We do this unconditionally so that this thread's
516 // BasicLock cannot be mis-interpreted by any stack walkers. For
517 // performance reasons, stack walkers generally first check for
518 // Biased Locking in the object's header, the second check is for
519 // stack-locking in the object's header, the third check is for
520 // recursive stack-locking in the displaced header in the BasicLock,
521 // and last are the inflated Java Monitor (ObjectMonitor) checks.
522 lock->set_displaced_header(markWord::unused_mark());
523
524 if (owner == NULL && m->try_set_owner_from(self, NULL) == NULL) {
525 assert(m->_recursions == 0, "invariant");
526 return true;
527 }
528
529 if (AsyncDeflateIdleMonitors &&
530 m->try_set_owner_from(self, DEFLATER_MARKER) == DEFLATER_MARKER) {
531 // The deflation protocol finished the first part (setting owner),
532 // but it failed the second part (making ref_count negative) and
533 // bailed. Or the ObjectMonitor was async deflated and reused.
534 // Acquired the monitor.
535 assert(m->_recursions == 0, "invariant");
536 return true;
537 }
538 }
539 break;
540 }
541
542 // Note that we could inflate in quick_enter.
543 // This is likely a useful optimization
544 // Critically, in quick_enter() we must not:
545 // -- perform bias revocation, or
546 // -- block indefinitely, or
547 // -- reach a safepoint
548
549 return false; // revert to slow-path
550 }
551
552 // -----------------------------------------------------------------------------
553 // Monitor Enter/Exit
554 // The interpreter and compiler assembly code tries to lock using the fast path
555 // of this algorithm. Make sure to update that code if the following function is
556 // changed. The implementation is extremely sensitive to race condition. Be careful.
557
558 void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, TRAPS) {
559 if (UseBiasedLocking) {
560 if (!SafepointSynchronize::is_at_safepoint()) {
561 BiasedLocking::revoke(obj, THREAD);
562 } else {
563 BiasedLocking::revoke_at_safepoint(obj);
564 }
565 }
566
567 markWord mark = obj->mark();
568 assert(!mark.has_bias_pattern(), "should not see bias pattern here");
569
570 if (mark.is_neutral()) {
571 // Anticipate successful CAS -- the ST of the displaced mark must
572 // be visible <= the ST performed by the CAS.
573 lock->set_displaced_header(mark);
574 if (mark == obj()->cas_set_mark(markWord::from_pointer(lock), mark)) {
575 return;
576 }
577 // Fall through to inflate() ...
578 } else if (mark.has_locker() &&
579 THREAD->is_lock_owned((address)mark.locker())) {
580 assert(lock != mark.locker(), "must not re-lock the same lock");
581 assert(lock != (BasicLock*)obj->mark().value(), "don't relock with same BasicLock");
582 lock->set_displaced_header(markWord::from_pointer(NULL));
583 return;
584 }
585
586 // The object header will never be displaced to this lock,
587 // so it does not matter what the value is, except that it
588 // must be non-zero to avoid looking like a re-entrant lock,
589 // and must not look locked either.
590 lock->set_displaced_header(markWord::unused_mark());
591 ObjectMonitorHandle omh;
592 inflate(&omh, THREAD, obj(), inflate_cause_monitor_enter);
593 omh.om_ptr()->enter(THREAD);
594 }
595
596 void ObjectSynchronizer::exit(oop object, BasicLock* lock, TRAPS) {
597 markWord mark = object->mark();
598 // We cannot check for Biased Locking if we are racing an inflation.
599 assert(mark == markWord::INFLATING() ||
600 !mark.has_bias_pattern(), "should not see bias pattern here");
601
602 markWord dhw = lock->displaced_header();
603 if (dhw.value() == 0) {
604 // If the displaced header is NULL, then this exit matches up with
605 // a recursive enter. No real work to do here except for diagnostics.
606 #ifndef PRODUCT
607 if (mark != markWord::INFLATING()) {
608 // Only do diagnostics if we are not racing an inflation. Simply
609 // exiting a recursive enter of a Java Monitor that is being
610 // inflated is safe; see the has_monitor() comment below.
611 assert(!mark.is_neutral(), "invariant");
612 assert(!mark.has_locker() ||
613 THREAD->is_lock_owned((address)mark.locker()), "invariant");
614 if (mark.has_monitor()) {
615 // The BasicLock's displaced_header is marked as a recursive
616 // enter and we have an inflated Java Monitor (ObjectMonitor).
617 // This is a special case where the Java Monitor was inflated
618 // after this thread entered the stack-lock recursively. When a
619 // Java Monitor is inflated, we cannot safely walk the Java
620 // Monitor owner's stack and update the BasicLocks because a
621 // Java Monitor can be asynchronously inflated by a thread that
622 // does not own the Java Monitor.
623 ObjectMonitor* m = mark.monitor();
624 assert(((oop)(m->object()))->mark() == mark, "invariant");
625 assert(m->is_entered(THREAD), "invariant");
626 }
627 }
628 #endif
629 return;
630 }
631
632 if (mark == markWord::from_pointer(lock)) {
633 // If the object is stack-locked by the current thread, try to
634 // swing the displaced header from the BasicLock back to the mark.
635 assert(dhw.is_neutral(), "invariant");
636 if (object->cas_set_mark(dhw, mark) == mark) {
637 return;
638 }
639 }
640
641 // We have to take the slow-path of possible inflation and then exit.
642 ObjectMonitorHandle omh;
643 inflate(&omh, THREAD, object, inflate_cause_vm_internal);
644 omh.om_ptr()->exit(true, THREAD);
645 }
646
647 // -----------------------------------------------------------------------------
648 // Class Loader support to workaround deadlocks on the class loader lock objects
649 // Also used by GC
650 // complete_exit()/reenter() are used to wait on a nested lock
651 // i.e. to give up an outer lock completely and then re-enter
652 // Used when holding nested locks - lock acquisition order: lock1 then lock2
653 // 1) complete_exit lock1 - saving recursion count
654 // 2) wait on lock2
655 // 3) when notified on lock2, unlock lock2
656 // 4) reenter lock1 with original recursion count
657 // 5) lock lock2
658 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
659 intx ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
660 if (UseBiasedLocking) {
661 BiasedLocking::revoke(obj, THREAD);
662 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
663 }
664
665 ObjectMonitorHandle omh;
666 inflate(&omh, THREAD, obj(), inflate_cause_vm_internal);
667 intptr_t ret_code = omh.om_ptr()->complete_exit(THREAD);
668 return ret_code;
669 }
670
671 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
672 void ObjectSynchronizer::reenter(Handle obj, intx recursions, TRAPS) {
673 if (UseBiasedLocking) {
674 BiasedLocking::revoke(obj, THREAD);
675 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
676 }
677
678 ObjectMonitorHandle omh;
679 inflate(&omh, THREAD, obj(), inflate_cause_vm_internal);
680 omh.om_ptr()->reenter(recursions, THREAD);
681 }
682 // -----------------------------------------------------------------------------
683 // JNI locks on java objects
684 // NOTE: must use heavy weight monitor to handle jni monitor enter
685 void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) {
686 // the current locking is from JNI instead of Java code
687 if (UseBiasedLocking) {
688 BiasedLocking::revoke(obj, THREAD);
689 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
690 }
691 THREAD->set_current_pending_monitor_is_from_java(false);
692 ObjectMonitorHandle omh;
693 inflate(&omh, THREAD, obj(), inflate_cause_jni_enter);
694 omh.om_ptr()->enter(THREAD);
695 THREAD->set_current_pending_monitor_is_from_java(true);
696 }
697
698 // NOTE: must use heavy weight monitor to handle jni monitor exit
699 void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
700 if (UseBiasedLocking) {
701 Handle h_obj(THREAD, obj);
702 BiasedLocking::revoke(h_obj, THREAD);
703 obj = h_obj();
704 }
705 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
706
707 ObjectMonitorHandle omh;
708 inflate(&omh, THREAD, obj, inflate_cause_jni_exit);
709 ObjectMonitor* monitor = omh.om_ptr();
710 // If this thread has locked the object, exit the monitor. We
711 // intentionally do not use CHECK here because we must exit the
712 // monitor even if an exception is pending.
713 if (monitor->check_owner(THREAD)) {
714 monitor->exit(true, THREAD);
715 }
716 }
717
718 // -----------------------------------------------------------------------------
719 // Internal VM locks on java objects
720 // standard constructor, allows locking failures
721 ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool do_lock) {
722 _dolock = do_lock;
723 _thread = thread;
724 _thread->check_for_valid_safepoint_state();
725 _obj = obj;
726
727 if (_dolock) {
728 ObjectSynchronizer::enter(_obj, &_lock, _thread);
729 }
730 }
731
732 ObjectLocker::~ObjectLocker() {
733 if (_dolock) {
734 ObjectSynchronizer::exit(_obj(), &_lock, _thread);
735 }
736 }
737
738
739 // -----------------------------------------------------------------------------
740 // Wait/Notify/NotifyAll
741 // NOTE: must use heavy weight monitor to handle wait()
742 int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
743 if (UseBiasedLocking) {
744 BiasedLocking::revoke(obj, THREAD);
745 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
746 }
747 if (millis < 0) {
748 THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
749 }
750 ObjectMonitorHandle omh;
751 inflate(&omh, THREAD, obj(), inflate_cause_wait);
752 ObjectMonitor* monitor = omh.om_ptr();
753
754 DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);
755 monitor->wait(millis, true, THREAD);
756
757 // This dummy call is in place to get around dtrace bug 6254741. Once
758 // that's fixed we can uncomment the following line, remove the call
759 // and change this function back into a "void" func.
760 // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
761 int ret_code = dtrace_waited_probe(monitor, obj, THREAD);
762 return ret_code;
763 }
764
765 void ObjectSynchronizer::wait_uninterruptibly(Handle obj, jlong millis, TRAPS) {
766 if (UseBiasedLocking) {
767 BiasedLocking::revoke(obj, THREAD);
768 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
769 }
770 if (millis < 0) {
771 THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
772 }
773 ObjectMonitorHandle omh;
774 inflate(&omh, THREAD, obj(), inflate_cause_wait);
775 omh.om_ptr()->wait(millis, false, THREAD);
776 }
777
778 void ObjectSynchronizer::notify(Handle obj, TRAPS) {
779 if (UseBiasedLocking) {
780 BiasedLocking::revoke(obj, THREAD);
781 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
782 }
783
784 markWord mark = obj->mark();
785 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
786 return;
787 }
788 ObjectMonitorHandle omh;
789 inflate(&omh, THREAD, obj(), inflate_cause_notify);
790 omh.om_ptr()->notify(THREAD);
791 }
792
793 // NOTE: see comment of notify()
794 void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
795 if (UseBiasedLocking) {
796 BiasedLocking::revoke(obj, THREAD);
797 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
798 }
799
800 markWord mark = obj->mark();
801 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
802 return;
803 }
804 ObjectMonitorHandle omh;
805 inflate(&omh, THREAD, obj(), inflate_cause_notify);
806 omh.om_ptr()->notifyAll(THREAD);
807 }
808
809 // -----------------------------------------------------------------------------
810 // Hash Code handling
811 //
812 // Performance concern:
813 // OrderAccess::storestore() calls release() which at one time stored 0
814 // into the global volatile OrderAccess::dummy variable. This store was
815 // unnecessary for correctness. Many threads storing into a common location
816 // causes considerable cache migration or "sloshing" on large SMP systems.
817 // As such, I avoided using OrderAccess::storestore(). In some cases
818 // OrderAccess::fence() -- which incurs local latency on the executing
819 // processor -- is a better choice as it scales on SMP systems.
820 //
821 // See http://blogs.oracle.com/dave/entry/biased_locking_in_hotspot for
822 // a discussion of coherency costs. Note that all our current reference
823 // platforms provide strong ST-ST order, so the issue is moot on IA32,
824 // x64, and SPARC.
825 //
826 // As a general policy we use "volatile" to control compiler-based reordering
827 // and explicit fences (barriers) to control for architectural reordering
828 // performed by the CPU(s) or platform.
829
830 struct SharedGlobals {
831 char _pad_prefix[OM_CACHE_LINE_SIZE];
832 // These are highly shared mostly-read variables.
833 // To avoid false-sharing they need to be the sole occupants of a cache line.
834 volatile int stw_random;
835 volatile int stw_cycle;
836 DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(volatile int) * 2);
837 // Hot RW variable -- Sequester to avoid false-sharing
838 volatile int hc_sequence;
839 DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile int));
840 };
841
842 static SharedGlobals GVars;
843 static int _forceMonitorScavenge = 0; // Scavenge required and pending
844
845 static markWord read_stable_mark(oop obj) {
846 markWord mark = obj->mark();
847 if (!mark.is_being_inflated()) {
848 return mark; // normal fast-path return
849 }
850
851 int its = 0;
852 for (;;) {
853 markWord mark = obj->mark();
854 if (!mark.is_being_inflated()) {
855 return mark; // normal fast-path return
856 }
857
858 // The object is being inflated by some other thread.
859 // The caller of read_stable_mark() must wait for inflation to complete.
860 // Avoid live-lock
861 // TODO: consider calling SafepointSynchronize::do_call_back() while
862 // spinning to see if there's a safepoint pending. If so, immediately
863 // yielding or blocking would be appropriate. Avoid spinning while
864 // there is a safepoint pending.
865 // TODO: add inflation contention performance counters.
866 // TODO: restrict the aggregate number of spinners.
867
868 ++its;
869 if (its > 10000 || !os::is_MP()) {
870 if (its & 1) {
871 os::naked_yield();
872 } else {
873 // Note that the following code attenuates the livelock problem but is not
874 // a complete remedy. A more complete solution would require that the inflating
875 // thread hold the associated inflation lock. The following code simply restricts
876 // the number of spinners to at most one. We'll have N-2 threads blocked
877 // on the inflationlock, 1 thread holding the inflation lock and using
878 // a yield/park strategy, and 1 thread in the midst of inflation.
879 // A more refined approach would be to change the encoding of INFLATING
880 // to allow encapsulation of a native thread pointer. Threads waiting for
881 // inflation to complete would use CAS to push themselves onto a singly linked
882 // list rooted at the markword. Once enqueued, they'd loop, checking a per-thread flag
883 // and calling park(). When inflation was complete the thread that accomplished inflation
884 // would detach the list and set the markword to inflated with a single CAS and
885 // then for each thread on the list, set the flag and unpark() the thread.
886 // This is conceptually similar to muxAcquire-muxRelease, except that muxRelease
887 // wakes at most one thread whereas we need to wake the entire list.
888 int ix = (cast_from_oop<intptr_t>(obj) >> 5) & (NINFLATIONLOCKS-1);
889 int YieldThenBlock = 0;
890 assert(ix >= 0 && ix < NINFLATIONLOCKS, "invariant");
891 assert((NINFLATIONLOCKS & (NINFLATIONLOCKS-1)) == 0, "invariant");
892 Thread::muxAcquire(gInflationLocks + ix, "gInflationLock");
893 while (obj->mark() == markWord::INFLATING()) {
894 // Beware: NakedYield() is advisory and has almost no effect on some platforms
895 // so we periodically call self->_ParkEvent->park(1).
896 // We use a mixed spin/yield/block mechanism.
897 if ((YieldThenBlock++) >= 16) {
898 Thread::current()->_ParkEvent->park(1);
899 } else {
900 os::naked_yield();
901 }
902 }
903 Thread::muxRelease(gInflationLocks + ix);
904 }
905 } else {
906 SpinPause(); // SMP-polite spinning
907 }
908 }
909 }
910
911 // hashCode() generation :
912 //
913 // Possibilities:
914 // * MD5Digest of {obj,stw_random}
915 // * CRC32 of {obj,stw_random} or any linear-feedback shift register function.
916 // * A DES- or AES-style SBox[] mechanism
917 // * One of the Phi-based schemes, such as:
918 // 2654435761 = 2^32 * Phi (golden ratio)
919 // HashCodeValue = ((uintptr_t(obj) >> 3) * 2654435761) ^ GVars.stw_random ;
920 // * A variation of Marsaglia's shift-xor RNG scheme.
921 // * (obj ^ stw_random) is appealing, but can result
922 // in undesirable regularity in the hashCode values of adjacent objects
923 // (objects allocated back-to-back, in particular). This could potentially
924 // result in hashtable collisions and reduced hashtable efficiency.
925 // There are simple ways to "diffuse" the middle address bits over the
926 // generated hashCode values:
927
928 static inline intptr_t get_next_hash(Thread* self, oop obj) {
929 intptr_t value = 0;
930 if (hashCode == 0) {
931 // This form uses global Park-Miller RNG.
932 // On MP system we'll have lots of RW access to a global, so the
933 // mechanism induces lots of coherency traffic.
934 value = os::random();
935 } else if (hashCode == 1) {
936 // This variation has the property of being stable (idempotent)
937 // between STW operations. This can be useful in some of the 1-0
938 // synchronization schemes.
939 intptr_t addr_bits = cast_from_oop<intptr_t>(obj) >> 3;
940 value = addr_bits ^ (addr_bits >> 5) ^ GVars.stw_random;
941 } else if (hashCode == 2) {
942 value = 1; // for sensitivity testing
943 } else if (hashCode == 3) {
944 value = ++GVars.hc_sequence;
945 } else if (hashCode == 4) {
946 value = cast_from_oop<intptr_t>(obj);
947 } else {
948 // Marsaglia's xor-shift scheme with thread-specific state
949 // This is probably the best overall implementation -- we'll
950 // likely make this the default in future releases.
951 unsigned t = self->_hashStateX;
952 t ^= (t << 11);
953 self->_hashStateX = self->_hashStateY;
954 self->_hashStateY = self->_hashStateZ;
955 self->_hashStateZ = self->_hashStateW;
956 unsigned v = self->_hashStateW;
957 v = (v ^ (v >> 19)) ^ (t ^ (t >> 8));
958 self->_hashStateW = v;
959 value = v;
960 }
961
962 value &= markWord::hash_mask;
963 if (value == 0) value = 0xBAD;
964 assert(value != markWord::no_hash, "invariant");
965 return value;
966 }
967
968 intptr_t ObjectSynchronizer::FastHashCode(Thread* self, oop obj) {
969 if (UseBiasedLocking) {
970 // NOTE: many places throughout the JVM do not expect a safepoint
971 // to be taken here, in particular most operations on perm gen
972 // objects. However, we only ever bias Java instances and all of
973 // the call sites of identity_hash that might revoke biases have
974 // been checked to make sure they can handle a safepoint. The
975 // added check of the bias pattern is to avoid useless calls to
976 // thread-local storage.
977 if (obj->mark().has_bias_pattern()) {
978 // Handle for oop obj in case of STW safepoint
979 Handle hobj(self, obj);
980 // Relaxing assertion for bug 6320749.
981 assert(Universe::verify_in_progress() ||
982 !SafepointSynchronize::is_at_safepoint(),
983 "biases should not be seen by VM thread here");
984 BiasedLocking::revoke(hobj, JavaThread::current());
985 obj = hobj();
986 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
987 }
988 }
989
990 // hashCode() is a heap mutator ...
991 // Relaxing assertion for bug 6320749.
992 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
993 !SafepointSynchronize::is_at_safepoint(), "invariant");
994 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
995 self->is_Java_thread() , "invariant");
996 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
997 ((JavaThread *)self)->thread_state() != _thread_blocked, "invariant");
998
999 while (true) {
1000 ObjectMonitor* monitor = NULL;
1001 markWord temp, test;
1002 intptr_t hash;
1003 markWord mark = read_stable_mark(obj);
1004
1005 // object should remain ineligible for biased locking
1006 assert(!mark.has_bias_pattern(), "invariant");
1007
1008 if (mark.is_neutral()) { // if this is a normal header
1009 hash = mark.hash();
1010 if (hash != 0) { // if it has a hash, just return it
1011 return hash;
1012 }
1013 hash = get_next_hash(self, obj); // get a new hash
1014 temp = mark.copy_set_hash(hash); // merge the hash into header
1015 // try to install the hash
1016 test = obj->cas_set_mark(temp, mark);
1017 if (test == mark) { // if the hash was installed, return it
1018 return hash;
1019 }
1020 // Failed to install the hash. It could be that another thread
1021 // installed the hash just before our attempt or inflation has
1022 // occurred or... so we fall thru to inflate the monitor for
1023 // stability and then install the hash.
1024 } else if (mark.has_monitor()) {
1025 ObjectMonitorHandle omh;
1026 if (!omh.save_om_ptr(obj, mark)) {
1027 // Lost a race with async deflation so try again.
1028 assert(AsyncDeflateIdleMonitors, "sanity check");
1029 continue;
1030 }
1031 monitor = omh.om_ptr();
1032 temp = monitor->header();
1033 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1034 hash = temp.hash();
1035 if (hash != 0) { // if it has a hash, just return it
1036 return hash;
1037 }
1038 // Fall thru so we only have one place that installs the hash in
1039 // the ObjectMonitor.
1040 } else if (self->is_lock_owned((address)mark.locker())) {
1041 // This is a stack lock owned by the calling thread so fetch the
1042 // displaced markWord from the BasicLock on the stack.
1043 temp = mark.displaced_mark_helper();
1044 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1045 hash = temp.hash();
1046 if (hash != 0) { // if it has a hash, just return it
1047 return hash;
1048 }
1049 // WARNING:
1050 // The displaced header in the BasicLock on a thread's stack
1051 // is strictly immutable. It CANNOT be changed in ANY cases.
1052 // So we have to inflate the stack lock into an ObjectMonitor
1053 // even if the current thread owns the lock. The BasicLock on
1054 // a thread's stack can be asynchronously read by other threads
1055 // during an inflate() call so any change to that stack memory
1056 // may not propagate to other threads correctly.
1057 }
1058
1059 // Inflate the monitor to set the hash.
1060 ObjectMonitorHandle omh;
1061 inflate(&omh, self, obj, inflate_cause_hash_code);
1062 monitor = omh.om_ptr();
1063 // Load ObjectMonitor's header/dmw field and see if it has a hash.
1064 mark = monitor->header();
1065 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1066 hash = mark.hash();
1067 if (hash == 0) { // if it does not have a hash
1068 hash = get_next_hash(self, obj); // get a new hash
1069 temp = mark.copy_set_hash(hash); // merge the hash into header
1070 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1071 uintptr_t v = Atomic::cmpxchg((volatile uintptr_t*)monitor->header_addr(), mark.value(), temp.value());
1072 test = markWord(v);
1073 if (test != mark) {
1074 // The attempt to update the ObjectMonitor's header/dmw field
1075 // did not work. This can happen if another thread managed to
1076 // merge in the hash just before our cmpxchg().
1077 // ObjectMonitor::install_displaced_markword_in_object()
1078 // does mark the header/dmw field as part of async deflation,
1079 // but that protocol cannot happen now due to the
1080 // ObjectMonitorHandle above.
1081 // If we add any new usages of the header/dmw field, this code
1082 // will need to be updated.
1083 hash = test.hash();
1084 assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value());
1085 assert(hash != 0, "should only have lost the race to a thread that set a non-zero hash");
1086 }
1087 }
1088 // We finally get the hash.
1089 return hash;
1090 }
1091 }
1092
1093 // Deprecated -- use FastHashCode() instead.
1094
1095 intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) {
1096 return FastHashCode(Thread::current(), obj());
1097 }
1098
1099
1100 bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
1101 Handle h_obj) {
1102 if (UseBiasedLocking) {
1103 BiasedLocking::revoke(h_obj, thread);
1104 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1105 }
1106
1107 assert(thread == JavaThread::current(), "Can only be called on current thread");
1108 oop obj = h_obj();
1109
1110 while (true) {
1111 markWord mark = read_stable_mark(obj);
1112
1113 // Uncontended case, header points to stack
1114 if (mark.has_locker()) {
1115 return thread->is_lock_owned((address)mark.locker());
1116 }
1117 // Contended case, header points to ObjectMonitor (tagged pointer)
1118 if (mark.has_monitor()) {
1119 ObjectMonitorHandle omh;
1120 if (!omh.save_om_ptr(obj, mark)) {
1121 // Lost a race with async deflation so try again.
1122 assert(AsyncDeflateIdleMonitors, "sanity check");
1123 continue;
1124 }
1125 bool ret_code = omh.om_ptr()->is_entered(thread) != 0;
1126 return ret_code;
1127 }
1128 // Unlocked case, header in place
1129 assert(mark.is_neutral(), "sanity check");
1130 return false;
1131 }
1132 }
1133
1134 // Be aware of this method could revoke bias of the lock object.
1135 // This method queries the ownership of the lock handle specified by 'h_obj'.
1136 // If the current thread owns the lock, it returns owner_self. If no
1137 // thread owns the lock, it returns owner_none. Otherwise, it will return
1138 // owner_other.
1139 ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership
1140 (JavaThread *self, Handle h_obj) {
1141 // The caller must beware this method can revoke bias, and
1142 // revocation can result in a safepoint.
1143 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
1144 assert(self->thread_state() != _thread_blocked, "invariant");
1145
1146 // Possible mark states: neutral, biased, stack-locked, inflated
1147
1148 if (UseBiasedLocking && h_obj()->mark().has_bias_pattern()) {
1149 // CASE: biased
1150 BiasedLocking::revoke(h_obj, self);
1151 assert(!h_obj->mark().has_bias_pattern(),
1152 "biases should be revoked by now");
1153 }
1154
1155 assert(self == JavaThread::current(), "Can only be called on current thread");
1156 oop obj = h_obj();
1157
1158 while (true) {
1159 markWord mark = read_stable_mark(obj);
1160
1161 // CASE: stack-locked. Mark points to a BasicLock on the owner's stack.
1162 if (mark.has_locker()) {
1163 return self->is_lock_owned((address)mark.locker()) ?
1164 owner_self : owner_other;
1165 }
1166
1167 // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor.
1168 // The Object:ObjectMonitor relationship is stable as long as we're
1169 // not at a safepoint and AsyncDeflateIdleMonitors is false.
1170 if (mark.has_monitor()) {
1171 ObjectMonitorHandle omh;
1172 if (!omh.save_om_ptr(obj, mark)) {
1173 // Lost a race with async deflation so try again.
1174 assert(AsyncDeflateIdleMonitors, "sanity check");
1175 continue;
1176 }
1177 ObjectMonitor* monitor = omh.om_ptr();
1178 void* owner = monitor->_owner;
1179 if (owner == NULL) return owner_none;
1180 return (owner == self ||
1181 self->is_lock_owned((address)owner)) ? owner_self : owner_other;
1182 }
1183
1184 // CASE: neutral
1185 assert(mark.is_neutral(), "sanity check");
1186 return owner_none; // it's unlocked
1187 }
1188 }
1189
1190 // FIXME: jvmti should call this
1191 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
1192 if (UseBiasedLocking) {
1193 if (SafepointSynchronize::is_at_safepoint()) {
1194 BiasedLocking::revoke_at_safepoint(h_obj);
1195 } else {
1196 BiasedLocking::revoke(h_obj, JavaThread::current());
1197 }
1198 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1199 }
1200
1201 oop obj = h_obj();
1202
1203 while (true) {
1204 address owner = NULL;
1205 markWord mark = read_stable_mark(obj);
1206
1207 // Uncontended case, header points to stack
1208 if (mark.has_locker()) {
1209 owner = (address) mark.locker();
1210 }
1211
1212 // Contended case, header points to ObjectMonitor (tagged pointer)
1213 else if (mark.has_monitor()) {
1214 ObjectMonitorHandle omh;
1215 if (!omh.save_om_ptr(obj, mark)) {
1216 // Lost a race with async deflation so try again.
1217 assert(AsyncDeflateIdleMonitors, "sanity check");
1218 continue;
1219 }
1220 ObjectMonitor* monitor = omh.om_ptr();
1221 assert(monitor != NULL, "monitor should be non-null");
1222 owner = (address) monitor->owner();
1223 }
1224
1225 if (owner != NULL) {
1226 // owning_thread_from_monitor_owner() may also return NULL here
1227 return Threads::owning_thread_from_monitor_owner(t_list, owner);
1228 }
1229
1230 // Unlocked case, header in place
1231 // Cannot have assertion since this object may have been
1232 // locked by another thread when reaching here.
1233 // assert(mark.is_neutral(), "sanity check");
1234
1235 return NULL;
1236 }
1237 }
1238
1239 // Visitors ...
1240
1241 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
1242 PaddedObjectMonitor* block = g_block_list;
1243 while (block != NULL) {
1244 assert(block->object() == CHAINMARKER, "must be a block header");
1245 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
1246 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
1247 ObjectMonitorHandle omh;
1248 if (!mid->is_free() && omh.set_om_ptr_if_safe(mid)) {
1249 // The ObjectMonitor* is not free and it has been made safe.
1250 if (mid->object() == NULL) {
1251 // Only process with closure if the object is set.
1252 continue;
1253 }
1254 closure->do_monitor(mid);
1255 }
1256 }
1257 // unmarked_next() is not needed with g_block_list (no next field
1258 // marking) and no load_acquire() needed because _next_om is
1259 // updated before g_block_list is changed with cmpxchg().
1260 block = (PaddedObjectMonitor*)block->_next_om;
1261 }
1262 }
1263
1264 static bool monitors_used_above_threshold() {
1265 if (g_om_population == 0) {
1266 return false;
1267 }
1268 if (MonitorUsedDeflationThreshold > 0) {
1269 int monitors_used = g_om_population - g_om_free_count;
1270 if (HandshakeAfterDeflateIdleMonitors) {
1271 monitors_used -= g_om_wait_count;
1272 }
1273 int monitor_usage = (monitors_used * 100LL) / g_om_population;
1274 return monitor_usage > MonitorUsedDeflationThreshold;
1275 }
1276 return false;
1277 }
1278
1279 // Returns true if MonitorBound is set (> 0) and if the specified
1280 // cnt is > MonitorBound. Otherwise returns false.
1281 static bool is_MonitorBound_exceeded(const int cnt) {
1282 const int mx = MonitorBound;
1283 return mx > 0 && cnt > mx;
1284 }
1285
1286 bool ObjectSynchronizer::is_async_deflation_needed() {
1287 if (!AsyncDeflateIdleMonitors) {
1288 return false;
1289 }
1290 if (is_async_deflation_requested()) {
1291 // Async deflation request.
1292 return true;
1293 }
1294 if (AsyncDeflationInterval > 0 &&
1295 time_since_last_async_deflation_ms() > AsyncDeflationInterval &&
1296 monitors_used_above_threshold()) {
1297 // It's been longer than our specified deflate interval and there
1298 // are too many monitors in use. We don't deflate more frequently
1299 // than AsyncDeflationInterval (unless is_async_deflation_requested)
1300 // in order to not swamp the ServiceThread.
1301 _last_async_deflation_time_ns = os::javaTimeNanos();
1302 return true;
1303 }
1304 int monitors_used = g_om_population - g_om_free_count;
1305 if (HandshakeAfterDeflateIdleMonitors) {
1306 monitors_used -= g_om_wait_count;
1307 }
1308 if (is_MonitorBound_exceeded(monitors_used)) {
1309 // Not enough ObjectMonitors on the global free list.
1310 return true;
1311 }
1312 return false;
1313 }
1314
1315 bool ObjectSynchronizer::needs_monitor_scavenge() {
1316 if (Atomic::load(&_forceMonitorScavenge) == 1) {
1317 log_info(monitorinflation)("Monitor scavenge needed, triggering safepoint cleanup.");
1318 return true;
1319 }
1320 return false;
1321 }
1322
1323 bool ObjectSynchronizer::is_safepoint_deflation_needed() {
1324 if (!AsyncDeflateIdleMonitors) {
1325 if (monitors_used_above_threshold()) {
1326 // Too many monitors in use.
1327 return true;
1328 }
1329 return needs_monitor_scavenge();
1330 }
1331 if (is_special_deflation_requested()) {
1332 // For AsyncDeflateIdleMonitors only do a safepoint deflation
1333 // if there is a special deflation request.
1334 return true;
1335 }
1336 return false;
1337 }
1338
1339 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1340 return (os::javaTimeNanos() - _last_async_deflation_time_ns) / (NANOUNITS / MILLIUNITS);
1341 }
1342
1343 void ObjectSynchronizer::oops_do(OopClosure* f) {
1344 // We only scan the global used list here (for moribund threads), and
1345 // the thread-local monitors in Thread::oops_do().
1346 global_used_oops_do(f);
1347 }
1348
1349 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
1350 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1351 list_oops_do(g_om_in_use_list, g_om_in_use_count, f);
1352 }
1353
1354 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
1355 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1356 list_oops_do(thread->om_in_use_list, thread->om_in_use_count, f);
1357 }
1358
1359 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, int count, OopClosure* f) {
1360 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1361 // The oops_do() phase does not overlap with monitor deflation
1362 // so no need to update the ObjectMonitor's ref_count for this
1363 // ObjectMonitor* use and no need to mark ObjectMonitors for the
1364 // list traversal.
1365 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
1366 if (mid->object() != NULL) {
1367 f->do_oop((oop*)mid->object_addr());
1368 }
1369 }
1370 }
1371
1372
1373 // -----------------------------------------------------------------------------
1374 // ObjectMonitor Lifecycle
1375 // -----------------------
1376 // Inflation unlinks monitors from the global g_free_list and
1377 // associates them with objects. Deflation -- which occurs at
1378 // STW-time -- disassociates idle monitors from objects. Such
1379 // scavenged monitors are returned to the g_free_list.
1380 //
1381 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
1382 //
1383 // Lifecycle:
1384 // -- unassigned and on the global free list
1385 // -- unassigned and on a thread's private om_free_list
1386 // -- assigned to an object. The object is inflated and the mark refers
1387 // to the objectmonitor.
1388
1389
1390 // Constraining monitor pool growth via MonitorBound ...
1391 //
1392 // If MonitorBound is not set (<= 0), MonitorBound checks are disabled.
1393 //
1394 // When safepoint deflation is being used (!AsyncDeflateIdleMonitors):
1395 // The monitor pool is grow-only. We scavenge at STW safepoint-time, but the
1396 // the rate of scavenging is driven primarily by GC. As such, we can find
1397 // an inordinate number of monitors in circulation.
1398 // To avoid that scenario we can artificially induce a STW safepoint
1399 // if the pool appears to be growing past some reasonable bound.
1400 // Generally we favor time in space-time tradeoffs, but as there's no
1401 // natural back-pressure on the # of extant monitors we need to impose some
1402 // type of limit. Beware that if MonitorBound is set to too low a value
1403 // we could just loop. In addition, if MonitorBound is set to a low value
1404 // we'll incur more safepoints, which are harmful to performance.
1405 // See also: GuaranteedSafepointInterval
1406 //
1407 // When safepoint deflation is being used and MonitorBound is set, the
1408 // boundry applies to
1409 // (g_om_population - g_om_free_count)
1410 // i.e., if there are not enough ObjectMonitors on the global free list,
1411 // then a safepoint deflation is induced. Picking a good MonitorBound value
1412 // is non-trivial.
1413 //
1414 // When async deflation is being used:
1415 // The monitor pool is still grow-only. Async deflation is requested
1416 // by a safepoint's cleanup phase or by the ServiceThread at periodic
1417 // intervals when is_async_deflation_needed() returns true. In
1418 // addition to other policies that are checked, if there are not
1419 // enough ObjectMonitors on the global free list, then
1420 // is_async_deflation_needed() will return true. The ServiceThread
1421 // calls deflate_global_idle_monitors_using_JT() and also calls
1422 // deflate_per_thread_idle_monitors_using_JT() as needed.
1423
1424 static void InduceScavenge(Thread* self, const char * Whence) {
1425 assert(!AsyncDeflateIdleMonitors, "is not used by async deflation");
1426
1427 // Induce STW safepoint to trim monitors
1428 // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
1429 // More precisely, trigger a cleanup safepoint as the number
1430 // of active monitors passes the specified threshold.
1431 // TODO: assert thread state is reasonable
1432
1433 if (Atomic::xchg(&_forceMonitorScavenge, 1) == 0) {
1434 VMThread::check_for_forced_cleanup();
1435 }
1436 }
1437
1438 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self,
1439 const InflateCause cause) {
1440 // A large MAXPRIVATE value reduces both list lock contention
1441 // and list coherency traffic, but also tends to increase the
1442 // number of ObjectMonitors in circulation as well as the STW
1443 // scavenge costs. As usual, we lean toward time in space-time
1444 // tradeoffs.
1445 const int MAXPRIVATE = 1024;
1446
1447 stringStream ss;
1448 for (;;) {
1449 ObjectMonitor* m;
1450
1451 // 1: try to allocate from the thread's local om_free_list.
1452 // Threads will attempt to allocate first from their local list, then
1453 // from the global list, and only after those attempts fail will the
1454 // thread attempt to instantiate new monitors. Thread-local free lists
1455 // improve allocation latency, as well as reducing coherency traffic
1456 // on the shared global list.
1457 m = take_from_start_of_om_free_list(self);
1458 if (m != NULL) {
1459 guarantee(m->object() == NULL, "invariant");
1460 m->set_allocation_state(ObjectMonitor::New);
1461 prepend_to_om_in_use_list(self, m);
1462 return m;
1463 }
1464
1465 // 2: try to allocate from the global g_free_list
1466 // CONSIDER: use muxTry() instead of muxAcquire().
1467 // If the muxTry() fails then drop immediately into case 3.
1468 // If we're using thread-local free lists then try
1469 // to reprovision the caller's free list.
1470 if (g_free_list != NULL) {
1471 // Reprovision the thread's om_free_list.
1472 // Use bulk transfers to reduce the allocation rate and heat
1473 // on various locks.
1474 for (int i = self->om_free_provision; --i >= 0;) {
1475 ObjectMonitor* take = take_from_start_of_g_free_list();
1476 if (take == NULL) {
1477 break; // No more are available.
1478 }
1479 guarantee(take->object() == NULL, "invariant");
1480 if (AsyncDeflateIdleMonitors) {
1481 // We allowed 3 field values to linger during async deflation.
1482 // We clear header and restore ref_count here, but we leave
1483 // owner == DEFLATER_MARKER so the simple C2 ObjectMonitor
1484 // enter optimization can no longer race with async deflation
1485 // and reuse.
1486 take->set_header(markWord::zero());
1487 if (take->ref_count() < 0) {
1488 // Add back max_jint to restore the ref_count field to its
1489 // proper value.
1490 Atomic::add(&take->_ref_count, max_jint);
1491
1492 DEBUG_ONLY(jint l_ref_count = take->ref_count();)
1493 assert(l_ref_count >= 0, "must not be negative: l_ref_count=%d, ref_count=%d",
1494 l_ref_count, take->ref_count());
1495 }
1496 }
1497 take->Recycle();
1498 // Since we're taking from the global free-list, take must be Free.
1499 // om_release() also sets the allocation state to Free because it
1500 // is called from other code paths.
1501 assert(take->is_free(), "invariant");
1502 om_release(self, take, false);
1503 }
1504 self->om_free_provision += 1 + (self->om_free_provision / 2);
1505 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1506
1507 if (!AsyncDeflateIdleMonitors &&
1508 is_MonitorBound_exceeded(g_om_population - g_om_free_count)) {
1509 // Not enough ObjectMonitors on the global free list.
1510 // We can't safely induce a STW safepoint from om_alloc() as our thread
1511 // state may not be appropriate for such activities and callers may hold
1512 // naked oops, so instead we defer the action.
1513 InduceScavenge(self, "om_alloc");
1514 }
1515 continue;
1516 }
1517
1518 // 3: allocate a block of new ObjectMonitors
1519 // Both the local and global free lists are empty -- resort to malloc().
1520 // In the current implementation ObjectMonitors are TSM - immortal.
1521 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1522 // each ObjectMonitor to start at the beginning of a cache line,
1523 // so we use align_up().
1524 // A better solution would be to use C++ placement-new.
1525 // BEWARE: As it stands currently, we don't run the ctors!
1526 assert(_BLOCKSIZE > 1, "invariant");
1527 size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE;
1528 PaddedObjectMonitor* temp;
1529 size_t aligned_size = neededsize + (OM_CACHE_LINE_SIZE - 1);
1530 void* real_malloc_addr = NEW_C_HEAP_ARRAY(char, aligned_size, mtInternal);
1531 temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, OM_CACHE_LINE_SIZE);
1532 (void)memset((void *) temp, 0, neededsize);
1533
1534 // Format the block.
1535 // initialize the linked list, each monitor points to its next
1536 // forming the single linked free list, the very first monitor
1537 // will points to next block, which forms the block list.
1538 // The trick of using the 1st element in the block as g_block_list
1539 // linkage should be reconsidered. A better implementation would
1540 // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; }
1541
1542 for (int i = 1; i < _BLOCKSIZE; i++) {
1543 temp[i]._next_om = (ObjectMonitor*)&temp[i + 1];
1544 assert(temp[i].is_free(), "invariant");
1545 }
1546
1547 // terminate the last monitor as the end of list
1548 temp[_BLOCKSIZE - 1]._next_om = (ObjectMonitor*)NULL;
1549
1550 // Element [0] is reserved for global list linkage
1551 temp[0].set_object(CHAINMARKER);
1552
1553 // Consider carving out this thread's current request from the
1554 // block in hand. This avoids some lock traffic and redundant
1555 // list activity.
1556
1557 prepend_block_to_lists(temp);
1558 }
1559 }
1560
1561 // Place "m" on the caller's private per-thread om_free_list.
1562 // In practice there's no need to clamp or limit the number of
1563 // monitors on a thread's om_free_list as the only non-allocation time
1564 // we'll call om_release() is to return a monitor to the free list after
1565 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1566 // accumulate on a thread's free list.
1567 //
1568 // Key constraint: all ObjectMonitors on a thread's free list and the global
1569 // free list must have their object field set to null. This prevents the
1570 // scavenger -- deflate_monitor_list() or deflate_monitor_list_using_JT()
1571 // -- from reclaiming them while we are trying to release them.
1572
1573 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1574 bool from_per_thread_alloc) {
1575 guarantee(m->header().value() == 0, "invariant");
1576 guarantee(m->object() == NULL, "invariant");
1577 stringStream ss;
1578 guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1579 "%s, recursions=" INTX_FORMAT, m->is_busy_to_string(&ss),
1580 m->_recursions);
1581 m->set_allocation_state(ObjectMonitor::Free);
1582 // _next_om is used for both per-thread in-use and free lists so
1583 // we have to remove 'm' from the in-use list first (as needed).
1584 if (from_per_thread_alloc) {
1585 // Need to remove 'm' from om_in_use_list.
1586 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
1587 // protocol because async deflation can do list deletions in parallel.
1588 ObjectMonitor* cur_mid_in_use = NULL;
1589 ObjectMonitor* mid = NULL;
1590 ObjectMonitor* next = NULL;
1591 bool extracted = false;
1592
1593 if (!mark_list_head(&self->om_in_use_list, &mid, &next)) {
1594 fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self));
1595 }
1596 while (true) {
1597 if (m == mid) {
1598 // We found 'm' on the per-thread in-use list so try to extract it.
1599 if (cur_mid_in_use == NULL) {
1600 // mid is the list head and it is marked. Switch the list head
1601 // to next which unmarks the list head, but leaves mid marked:
1602 self->om_in_use_list = next;
1603 // mark_list_head() used cmpxchg() above, switching list head can be lazier:
1604 OrderAccess::storestore();
1605 } else {
1606 // mid and cur_mid_in_use are marked. Switch cur_mid_in_use's
1607 // next field to next which unmarks cur_mid_in_use, but leaves
1608 // mid marked:
1609 Atomic::release_store(&cur_mid_in_use->_next_om, next);
1610 }
1611 extracted = true;
1612 Atomic::dec(&self->om_in_use_count);
1613 // Unmark mid, but leave the next value for any lagging list
1614 // walkers. It will get cleaned up when mid is prepended to
1615 // the thread's free list:
1616 set_next(mid, next);
1617 break;
1618 }
1619 if (cur_mid_in_use != NULL) {
1620 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
1621 }
1622 // The next cur_mid_in_use keeps mid's marked next field so
1623 // that it is stable for a possible next field change. It
1624 // cannot be deflated while it is marked.
1625 cur_mid_in_use = mid;
1626 mid = next;
1627 if (mid == NULL) {
1628 // Reached end of the list and didn't find m so:
1629 fatal("must find m=" INTPTR_FORMAT "on om_in_use_list=" INTPTR_FORMAT,
1630 p2i(m), p2i(self->om_in_use_list));
1631 }
1632 // Mark mid's next field so we can possibly extract it:
1633 next = mark_next_loop(mid);
1634 }
1635 }
1636
1637 prepend_to_om_free_list(self, m);
1638 guarantee(m->is_free(), "invariant");
1639 }
1640
1641 // Return ObjectMonitors on a moribund thread's free and in-use
1642 // lists to the appropriate global lists. The ObjectMonitors on the
1643 // per-thread in-use list may still be in use by other threads.
1644 //
1645 // We currently call om_flush() from Threads::remove() before the
1646 // thread has been excised from the thread list and is no longer a
1647 // mutator. This means that om_flush() cannot run concurrently with
1648 // a safepoint and interleave with deflate_idle_monitors(). In
1649 // particular, this ensures that the thread's in-use monitors are
1650 // scanned by a GC safepoint, either via Thread::oops_do() (before
1651 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1652 // om_flush() is called).
1653 //
1654 // With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT()
1655 // and deflate_per_thread_idle_monitors_using_JT() (in another thread) can
1656 // run at the same time as om_flush() so we have to follow a careful
1657 // protocol to prevent list corruption.
1658
1659 void ObjectSynchronizer::om_flush(Thread* self) {
1660 // This function can race with an async deflater thread. Since
1661 // deflation has to process the per-thread in-use list before
1662 // prepending the deflated ObjectMonitors to the global free list,
1663 // we process the per-thread lists in the same order to prevent
1664 // ordering races.
1665 int in_use_count = 0;
1666 ObjectMonitor* in_use_list = NULL;
1667 ObjectMonitor* in_use_tail = NULL;
1668 ObjectMonitor* next = NULL;
1669
1670 // An async deflation thread checks to see if the target thread
1671 // is exiting, but if it has made it past that check before we
1672 // started exiting, then it is racing to get to the in-use list.
1673 if (mark_list_head(&self->om_in_use_list, &in_use_list, &next)) {
1674 // At this point, we have marked the in-use list head so an
1675 // async deflation thread cannot come in after us. If an async
1676 // deflation thread is ahead of us, then we'll detect that and
1677 // wait for it to finish its work.
1678 //
1679 // The thread is going away, however the ObjectMonitors on the
1680 // om_in_use_list may still be in-use by other threads. Link
1681 // them to in_use_tail, which will be linked into the global
1682 // in-use list g_om_in_use_list below.
1683 //
1684 // Account for the in-use list head before the loop since it is
1685 // already marked (by this thread):
1686 in_use_tail = in_use_list;
1687 in_use_count++;
1688 for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL;) {
1689 if (is_next_marked(cur_om)) {
1690 // This next field is marked so there must be an async deflater
1691 // thread ahead of us so we'll give it a chance to finish.
1692 while (is_next_marked(cur_om)) {
1693 os::naked_short_sleep(1);
1694 }
1695 // Refetch the possibly changed next field and try again.
1696 cur_om = unmarked_next(in_use_tail);
1697 continue;
1698 }
1699 if (cur_om->is_free()) {
1700 // cur_om was deflated and the allocation state was changed
1701 // to Free while it was marked. We happened to see it just
1702 // after it was unmarked (and added to the free list).
1703 // Refetch the possibly changed next field and try again.
1704 cur_om = unmarked_next(in_use_tail);
1705 continue;
1706 }
1707 in_use_tail = cur_om;
1708 in_use_count++;
1709 cur_om = unmarked_next(cur_om);
1710 }
1711 guarantee(in_use_tail != NULL, "invariant");
1712 int l_om_in_use_count = self->om_in_use_count;
1713 ADIM_guarantee(l_om_in_use_count == in_use_count, "in-use counts don't "
1714 "match: l_om_in_use_count=%d, in_use_count=%d",
1715 l_om_in_use_count, in_use_count);
1716 self->om_in_use_count = 0;
1717 // Clear the in-use list head (which also unmarks it):
1718 self->om_in_use_list = (ObjectMonitor*)NULL;
1719 // mark_list_head() used cmpxchg() above, clearing the disconnected list head can be lazier:
1720 OrderAccess::storestore();
1721 set_next(in_use_list, next);
1722 }
1723
1724 int free_count = 0;
1725 ObjectMonitor* free_list = self->om_free_list;
1726 ObjectMonitor* free_tail = NULL;
1727 if (free_list != NULL) {
1728 // The thread is going away. Set 'free_tail' to the last per-thread free
1729 // monitor which will be linked to g_free_list below.
1730 stringStream ss;
1731 for (ObjectMonitor* s = free_list; s != NULL; s = unmarked_next(s)) {
1732 free_count++;
1733 free_tail = s;
1734 guarantee(s->object() == NULL, "invariant");
1735 guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss));
1736 }
1737 guarantee(free_tail != NULL, "invariant");
1738 int l_om_free_count = self->om_free_count;
1739 ADIM_guarantee(l_om_free_count == free_count, "free counts don't match: "
1740 "l_om_free_count=%d, free_count=%d", l_om_free_count,
1741 free_count);
1742 self->om_free_count = 0;
1743 self->om_free_list = NULL;
1744 OrderAccess::storestore(); // Lazier memory is okay for list walkers.
1745 }
1746
1747 if (free_tail != NULL) {
1748 prepend_list_to_g_free_list(free_list, free_tail, free_count);
1749 }
1750
1751 if (in_use_tail != NULL) {
1752 prepend_list_to_g_om_in_use_list(in_use_list, in_use_tail, in_use_count);
1753 }
1754
1755 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1756 LogStreamHandle(Info, monitorinflation) lsh_info;
1757 LogStream* ls = NULL;
1758 if (log_is_enabled(Debug, monitorinflation)) {
1759 ls = &lsh_debug;
1760 } else if ((free_count != 0 || in_use_count != 0) &&
1761 log_is_enabled(Info, monitorinflation)) {
1762 ls = &lsh_info;
1763 }
1764 if (ls != NULL) {
1765 ls->print_cr("om_flush: jt=" INTPTR_FORMAT ", free_count=%d"
1766 ", in_use_count=%d" ", om_free_provision=%d",
1767 p2i(self), free_count, in_use_count, self->om_free_provision);
1768 }
1769 }
1770
1771 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1772 const oop obj,
1773 ObjectSynchronizer::InflateCause cause) {
1774 assert(event != NULL, "invariant");
1775 assert(event->should_commit(), "invariant");
1776 event->set_monitorClass(obj->klass());
1777 event->set_address((uintptr_t)(void*)obj);
1778 event->set_cause((u1)cause);
1779 event->commit();
1780 }
1781
1782 // Fast path code shared by multiple functions
1783 void ObjectSynchronizer::inflate_helper(ObjectMonitorHandle* omh_p, oop obj) {
1784 while (true) {
1785 markWord mark = obj->mark();
1786 if (mark.has_monitor()) {
1787 if (!omh_p->save_om_ptr(obj, mark)) {
1788 // Lost a race with async deflation so try again.
1789 assert(AsyncDeflateIdleMonitors, "sanity check");
1790 continue;
1791 }
1792 ObjectMonitor* monitor = omh_p->om_ptr();
1793 assert(ObjectSynchronizer::verify_objmon_isinpool(monitor), "monitor is invalid");
1794 markWord dmw = monitor->header();
1795 assert(dmw.is_neutral(), "sanity check: header=" INTPTR_FORMAT, dmw.value());
1796 return;
1797 }
1798 inflate(omh_p, Thread::current(), obj, inflate_cause_vm_internal);
1799 return;
1800 }
1801 }
1802
1803 void ObjectSynchronizer::inflate(ObjectMonitorHandle* omh_p, Thread* self,
1804 oop object, const InflateCause cause) {
1805 // Inflate mutates the heap ...
1806 // Relaxing assertion for bug 6320749.
1807 assert(Universe::verify_in_progress() ||
1808 !SafepointSynchronize::is_at_safepoint(), "invariant");
1809
1810 EventJavaMonitorInflate event;
1811
1812 for (;;) {
1813 const markWord mark = object->mark();
1814 assert(!mark.has_bias_pattern(), "invariant");
1815
1816 // The mark can be in one of the following states:
1817 // * Inflated - just return
1818 // * Stack-locked - coerce it to inflated
1819 // * INFLATING - busy wait for conversion to complete
1820 // * Neutral - aggressively inflate the object.
1821 // * BIASED - Illegal. We should never see this
1822
1823 // CASE: inflated
1824 if (mark.has_monitor()) {
1825 if (!omh_p->save_om_ptr(object, mark)) {
1826 // Lost a race with async deflation so try again.
1827 assert(AsyncDeflateIdleMonitors, "sanity check");
1828 continue;
1829 }
1830 ObjectMonitor* inf = omh_p->om_ptr();
1831 markWord dmw = inf->header();
1832 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1833 assert(inf->object() == object, "invariant");
1834 assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
1835 return;
1836 }
1837
1838 // CASE: inflation in progress - inflating over a stack-lock.
1839 // Some other thread is converting from stack-locked to inflated.
1840 // Only that thread can complete inflation -- other threads must wait.
1841 // The INFLATING value is transient.
1842 // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1843 // We could always eliminate polling by parking the thread on some auxiliary list.
1844 if (mark == markWord::INFLATING()) {
1845 read_stable_mark(object);
1846 continue;
1847 }
1848
1849 // CASE: stack-locked
1850 // Could be stack-locked either by this thread or by some other thread.
1851 //
1852 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1853 // to install INFLATING into the mark word. We originally installed INFLATING,
1854 // allocated the objectmonitor, and then finally STed the address of the
1855 // objectmonitor into the mark. This was correct, but artificially lengthened
1856 // the interval in which INFLATED appeared in the mark, thus increasing
1857 // the odds of inflation contention.
1858 //
1859 // We now use per-thread private objectmonitor free lists.
1860 // These list are reprovisioned from the global free list outside the
1861 // critical INFLATING...ST interval. A thread can transfer
1862 // multiple objectmonitors en-mass from the global free list to its local free list.
1863 // This reduces coherency traffic and lock contention on the global free list.
1864 // Using such local free lists, it doesn't matter if the om_alloc() call appears
1865 // before or after the CAS(INFLATING) operation.
1866 // See the comments in om_alloc().
1867
1868 LogStreamHandle(Trace, monitorinflation) lsh;
1869
1870 if (mark.has_locker()) {
1871 ObjectMonitor* m = om_alloc(self, cause);
1872 // Optimistically prepare the objectmonitor - anticipate successful CAS
1873 // We do this before the CAS in order to minimize the length of time
1874 // in which INFLATING appears in the mark.
1875 m->Recycle();
1876 m->_Responsible = NULL;
1877 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // Consider: maintain by type/class
1878
1879 markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1880 if (cmp != mark) {
1881 // om_release() will reset the allocation state from New to Free.
1882 om_release(self, m, true);
1883 continue; // Interference -- just retry
1884 }
1885
1886 // We've successfully installed INFLATING (0) into the mark-word.
1887 // This is the only case where 0 will appear in a mark-word.
1888 // Only the singular thread that successfully swings the mark-word
1889 // to 0 can perform (or more precisely, complete) inflation.
1890 //
1891 // Why do we CAS a 0 into the mark-word instead of just CASing the
1892 // mark-word from the stack-locked value directly to the new inflated state?
1893 // Consider what happens when a thread unlocks a stack-locked object.
1894 // It attempts to use CAS to swing the displaced header value from the
1895 // on-stack BasicLock back into the object header. Recall also that the
1896 // header value (hash code, etc) can reside in (a) the object header, or
1897 // (b) a displaced header associated with the stack-lock, or (c) a displaced
1898 // header in an ObjectMonitor. The inflate() routine must copy the header
1899 // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1900 // the while preserving the hashCode stability invariants. If the owner
1901 // decides to release the lock while the value is 0, the unlock will fail
1902 // and control will eventually pass from slow_exit() to inflate. The owner
1903 // will then spin, waiting for the 0 value to disappear. Put another way,
1904 // the 0 causes the owner to stall if the owner happens to try to
1905 // drop the lock (restoring the header from the BasicLock to the object)
1906 // while inflation is in-progress. This protocol avoids races that might
1907 // would otherwise permit hashCode values to change or "flicker" for an object.
1908 // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1909 // 0 serves as a "BUSY" inflate-in-progress indicator.
1910
1911
1912 // fetch the displaced mark from the owner's stack.
1913 // The owner can't die or unwind past the lock while our INFLATING
1914 // object is in the mark. Furthermore the owner can't complete
1915 // an unlock on the object, either.
1916 markWord dmw = mark.displaced_mark_helper();
1917 // Catch if the object's header is not neutral (not locked and
1918 // not marked is what we care about here).
1919 ADIM_guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1920
1921 // Setup monitor fields to proper values -- prepare the monitor
1922 m->set_header(dmw);
1923
1924 // Optimization: if the mark.locker stack address is associated
1925 // with this thread we could simply set m->_owner = self.
1926 // Note that a thread can inflate an object
1927 // that it has stack-locked -- as might happen in wait() -- directly
1928 // with CAS. That is, we can avoid the xchg-NULL .... ST idiom.
1929 if (AsyncDeflateIdleMonitors) {
1930 m->simply_set_owner_from(mark.locker(), NULL, DEFLATER_MARKER);
1931 } else {
1932 m->simply_set_owner_from(mark.locker(), NULL);
1933 }
1934 m->set_object(object);
1935 // TODO-FIXME: assert BasicLock->dhw != 0.
1936
1937 omh_p->set_om_ptr(m);
1938
1939 // Must preserve store ordering. The monitor state must
1940 // be stable at the time of publishing the monitor address.
1941 guarantee(object->mark() == markWord::INFLATING(), "invariant");
1942 object->release_set_mark(markWord::encode(m));
1943
1944 // Once ObjectMonitor is configured and the object is associated
1945 // with the ObjectMonitor, it is safe to allow async deflation:
1946 assert(m->is_new(), "freshly allocated monitor must be new");
1947 m->set_allocation_state(ObjectMonitor::Old);
1948
1949 // Hopefully the performance counters are allocated on distinct cache lines
1950 // to avoid false sharing on MP systems ...
1951 OM_PERFDATA_OP(Inflations, inc());
1952 if (log_is_enabled(Trace, monitorinflation)) {
1953 ResourceMark rm(self);
1954 lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
1955 INTPTR_FORMAT ", type='%s'", p2i(object),
1956 object->mark().value(), object->klass()->external_name());
1957 }
1958 if (event.should_commit()) {
1959 post_monitor_inflate_event(&event, object, cause);
1960 }
1961 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
1962 return;
1963 }
1964
1965 // CASE: neutral
1966 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
1967 // If we know we're inflating for entry it's better to inflate by swinging a
1968 // pre-locked ObjectMonitor pointer into the object header. A successful
1969 // CAS inflates the object *and* confers ownership to the inflating thread.
1970 // In the current implementation we use a 2-step mechanism where we CAS()
1971 // to inflate and then CAS() again to try to swing _owner from NULL to self.
1972 // An inflateTry() method that we could call from enter() would be useful.
1973
1974 // Catch if the object's header is not neutral (not locked and
1975 // not marked is what we care about here).
1976 ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT,mark.value());
1977 ObjectMonitor* m = om_alloc(self, cause);
1978 // prepare m for installation - set monitor to initial state
1979 m->Recycle();
1980 m->set_header(mark);
1981 // If we leave _owner == DEFLATER_MARKER here, then the simple C2
1982 // ObjectMonitor enter optimization can no longer race with async
1983 // deflation and reuse.
1984 m->set_object(object);
1985 m->_Responsible = NULL;
1986 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
1987
1988 omh_p->set_om_ptr(m);
1989
1990 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
1991 m->set_header(markWord::zero());
1992 m->set_object(NULL);
1993 m->Recycle();
1994 omh_p->set_om_ptr(NULL);
1995 // om_release() will reset the allocation state from New to Free.
1996 om_release(self, m, true);
1997 m = NULL;
1998 continue;
1999 // interference - the markword changed - just retry.
2000 // The state-transitions are one-way, so there's no chance of
2001 // live-lock -- "Inflated" is an absorbing state.
2002 }
2003
2004 // Once the ObjectMonitor is configured and object is associated
2005 // with the ObjectMonitor, it is safe to allow async deflation:
2006 assert(m->is_new(), "freshly allocated monitor must be new");
2007 m->set_allocation_state(ObjectMonitor::Old);
2008
2009 // Hopefully the performance counters are allocated on distinct
2010 // cache lines to avoid false sharing on MP systems ...
2011 OM_PERFDATA_OP(Inflations, inc());
2012 if (log_is_enabled(Trace, monitorinflation)) {
2013 ResourceMark rm(self);
2014 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
2015 INTPTR_FORMAT ", type='%s'", p2i(object),
2016 object->mark().value(), object->klass()->external_name());
2017 }
2018 if (event.should_commit()) {
2019 post_monitor_inflate_event(&event, object, cause);
2020 }
2021 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
2022 return;
2023 }
2024 }
2025
2026
2027 // We maintain a list of in-use monitors for each thread.
2028 //
2029 // For safepoint based deflation:
2030 // deflate_thread_local_monitors() scans a single thread's in-use list, while
2031 // deflate_idle_monitors() scans only a global list of in-use monitors which
2032 // is populated only as a thread dies (see om_flush()).
2033 //
2034 // These operations are called at all safepoints, immediately after mutators
2035 // are stopped, but before any objects have moved. Collectively they traverse
2036 // the population of in-use monitors, deflating where possible. The scavenged
2037 // monitors are returned to the global monitor free list.
2038 //
2039 // Beware that we scavenge at *every* stop-the-world point. Having a large
2040 // number of monitors in-use could negatively impact performance. We also want
2041 // to minimize the total # of monitors in circulation, as they incur a small
2042 // footprint penalty.
2043 //
2044 // Perversely, the heap size -- and thus the STW safepoint rate --
2045 // typically drives the scavenge rate. Large heaps can mean infrequent GC,
2046 // which in turn can mean large(r) numbers of ObjectMonitors in circulation.
2047 // This is an unfortunate aspect of this design.
2048 //
2049 // For async deflation:
2050 // If a special deflation request is made, then the safepoint based
2051 // deflation mechanism is used. Otherwise, an async deflation request
2052 // is registered with the ServiceThread and it is notified.
2053
2054 void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* counters) {
2055 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2056
2057 // The per-thread in-use lists are handled in
2058 // ParallelSPCleanupThreadClosure::do_thread().
2059
2060 if (!AsyncDeflateIdleMonitors || is_special_deflation_requested()) {
2061 // Use the older mechanism for the global in-use list or if a
2062 // special deflation has been requested before the safepoint.
2063 ObjectSynchronizer::deflate_idle_monitors(counters);
2064 return;
2065 }
2066
2067 log_debug(monitorinflation)("requesting async deflation of idle monitors.");
2068 // Request deflation of idle monitors by the ServiceThread:
2069 set_is_async_deflation_requested(true);
2070 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
2071 ml.notify_all();
2072
2073 if (log_is_enabled(Debug, monitorinflation)) {
2074 // exit_globals()'s call to audit_and_print_stats() is done
2075 // at the Info level and not at a safepoint.
2076 // For safepoint based deflation, audit_and_print_stats() is called
2077 // in ObjectSynchronizer::finish_deflate_idle_monitors() at the
2078 // Debug level at a safepoint.
2079 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2080 }
2081 }
2082
2083 // Deflate a single monitor if not in-use
2084 // Return true if deflated, false if in-use
2085 bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj,
2086 ObjectMonitor** free_head_p,
2087 ObjectMonitor** free_tail_p) {
2088 bool deflated;
2089 // Normal case ... The monitor is associated with obj.
2090 const markWord mark = obj->mark();
2091 guarantee(mark == markWord::encode(mid), "should match: mark="
2092 INTPTR_FORMAT ", encoded mid=" INTPTR_FORMAT, mark.value(),
2093 markWord::encode(mid).value());
2094 // Make sure that mark.monitor() and markWord::encode() agree:
2095 guarantee(mark.monitor() == mid, "should match: monitor()=" INTPTR_FORMAT
2096 ", mid=" INTPTR_FORMAT, p2i(mark.monitor()), p2i(mid));
2097 const markWord dmw = mid->header();
2098 guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
2099
2100 if (mid->is_busy() || mid->ref_count() != 0) {
2101 // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor*
2102 // is in use so no deflation.
2103 deflated = false;
2104 } else {
2105 // Deflate the monitor if it is no longer being used
2106 // It's idle - scavenge and return to the global free list
2107 // plain old deflation ...
2108 if (log_is_enabled(Trace, monitorinflation)) {
2109 ResourceMark rm;
2110 log_trace(monitorinflation)("deflate_monitor: "
2111 "object=" INTPTR_FORMAT ", mark="
2112 INTPTR_FORMAT ", type='%s'", p2i(obj),
2113 mark.value(), obj->klass()->external_name());
2114 }
2115
2116 // Restore the header back to obj
2117 obj->release_set_mark(dmw);
2118 if (AsyncDeflateIdleMonitors) {
2119 // clear() expects the owner field to be NULL and we won't race
2120 // with the simple C2 ObjectMonitor enter optimization since
2121 // we're at a safepoint. DEFLATER_MARKER is the only non-NULL
2122 // value we should see here.
2123 mid->try_set_owner_from(NULL, DEFLATER_MARKER);
2124 }
2125 mid->clear();
2126
2127 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
2128 p2i(mid->object()));
2129 assert(mid->is_free(), "invariant");
2130
2131 // Move the deflated ObjectMonitor to the working free list
2132 // defined by free_head_p and free_tail_p. No races on this list
2133 // so no need for load_acquire() or store_release().
2134 if (*free_head_p == NULL) *free_head_p = mid;
2135 if (*free_tail_p != NULL) {
2136 // We append to the list so the caller can use mid->_next_om
2137 // to fix the linkages in its context.
2138 ObjectMonitor* prevtail = *free_tail_p;
2139 // Should have been cleaned up by the caller:
2140 // Note: Should not have to mark prevtail here since we're at a
2141 // safepoint and ObjectMonitors on the local free list should
2142 // not be accessed in parallel.
2143 assert(prevtail->_next_om == NULL, "must be NULL: _next_om="
2144 INTPTR_FORMAT, p2i(prevtail->_next_om));
2145 set_next(prevtail, mid);
2146 }
2147 *free_tail_p = mid;
2148 // At this point, mid->_next_om still refers to its current
2149 // value and another ObjectMonitor's _next_om field still
2150 // refers to this ObjectMonitor. Those linkages have to be
2151 // cleaned up by the caller who has the complete context.
2152 deflated = true;
2153 }
2154 return deflated;
2155 }
2156
2157 // Deflate the specified ObjectMonitor if not in-use using a JavaThread.
2158 // Returns true if it was deflated and false otherwise.
2159 //
2160 // The async deflation protocol sets owner to DEFLATER_MARKER and
2161 // makes ref_count negative as signals to contending threads that
2162 // an async deflation is in progress. There are a number of checks
2163 // as part of the protocol to make sure that the calling thread has
2164 // not lost the race to a contending thread or to a thread that just
2165 // wants to use the ObjectMonitor*.
2166 //
2167 // The ObjectMonitor has been successfully async deflated when:
2168 // (owner == DEFLATER_MARKER && ref_count < 0)
2169 // Contending threads or ObjectMonitor* using threads that see those
2170 // values know to retry their operation.
2171 //
2172 bool ObjectSynchronizer::deflate_monitor_using_JT(ObjectMonitor* mid,
2173 ObjectMonitor** free_head_p,
2174 ObjectMonitor** free_tail_p) {
2175 assert(AsyncDeflateIdleMonitors, "sanity check");
2176 assert(Thread::current()->is_Java_thread(), "precondition");
2177 // A newly allocated ObjectMonitor should not be seen here so we
2178 // avoid an endless inflate/deflate cycle.
2179 assert(mid->is_old(), "must be old: allocation_state=%d",
2180 (int) mid->allocation_state());
2181
2182 if (mid->is_busy() || mid->ref_count() != 0) {
2183 // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor*
2184 // is in use so no deflation.
2185 return false;
2186 }
2187
2188 if (mid->try_set_owner_from(DEFLATER_MARKER, NULL) == NULL) {
2189 // ObjectMonitor is not owned by another thread. Our setting
2190 // owner to DEFLATER_MARKER forces any contending thread through
2191 // the slow path. This is just the first part of the async
2192 // deflation dance.
2193
2194 if (mid->_contentions != 0 || mid->_waiters != 0) {
2195 // Another thread has raced to enter the ObjectMonitor after
2196 // mid->is_busy() above or has already entered and waited on
2197 // it which makes it busy so no deflation. Restore owner to
2198 // NULL if it is still DEFLATER_MARKER.
2199 mid->try_set_owner_from(NULL, DEFLATER_MARKER);
2200 return false;
2201 }
2202
2203 if (Atomic::cmpxchg(&mid->_ref_count, (jint)0, -max_jint) == 0) {
2204 // Make ref_count negative to force any contending threads or
2205 // ObjectMonitor* using threads to retry. This is the second
2206 // part of the async deflation dance.
2207
2208 if (mid->owner_is_DEFLATER_MARKER()) {
2209 // If owner is still DEFLATER_MARKER, then we have successfully
2210 // signaled any contending threads to retry. If it is not, then we
2211 // have lost the race to an entering thread and the ObjectMonitor
2212 // is now busy. This is the third and final part of the async
2213 // deflation dance.
2214 // Note: This owner check solves the ABA problem with ref_count
2215 // where another thread acquired the ObjectMonitor, finished
2216 // using it and restored the ref_count to zero.
2217
2218 // Sanity checks for the races:
2219 guarantee(mid->_contentions == 0, "must be 0: contentions=%d",
2220 mid->_contentions);
2221 guarantee(mid->_waiters == 0, "must be 0: waiters=%d", mid->_waiters);
2222 guarantee(mid->_cxq == NULL, "must be no contending threads: cxq="
2223 INTPTR_FORMAT, p2i(mid->_cxq));
2224 guarantee(mid->_EntryList == NULL,
2225 "must be no entering threads: EntryList=" INTPTR_FORMAT,
2226 p2i(mid->_EntryList));
2227
2228 const oop obj = (oop) mid->object();
2229 if (log_is_enabled(Trace, monitorinflation)) {
2230 ResourceMark rm;
2231 log_trace(monitorinflation)("deflate_monitor_using_JT: "
2232 "object=" INTPTR_FORMAT ", mark="
2233 INTPTR_FORMAT ", type='%s'",
2234 p2i(obj), obj->mark().value(),
2235 obj->klass()->external_name());
2236 }
2237
2238 // Install the old mark word if nobody else has already done it.
2239 mid->install_displaced_markword_in_object(obj);
2240 mid->clear_using_JT();
2241
2242 assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT,
2243 p2i(mid->object()));
2244 assert(mid->is_free(), "must be free: allocation_state=%d",
2245 (int) mid->allocation_state());
2246
2247 // Move the deflated ObjectMonitor to the working free list
2248 // defined by free_head_p and free_tail_p. No races on this list
2249 // so no need for load_acquire() or store_release().
2250 if (*free_head_p == NULL) {
2251 // First one on the list.
2252 *free_head_p = mid;
2253 }
2254 if (*free_tail_p != NULL) {
2255 // We append to the list so the caller can use mid->_next_om
2256 // to fix the linkages in its context.
2257 ObjectMonitor* prevtail = *free_tail_p;
2258 // Should have been cleaned up by the caller:
2259 ObjectMonitor* next = mark_next_loop(prevtail);
2260 assert(unmarked_next(prevtail) == NULL, "must be NULL: _next_om="
2261 INTPTR_FORMAT, p2i(unmarked_next(prevtail)));
2262 set_next(prevtail, mid); // prevtail now points to mid (and is unmarked)
2263 }
2264 *free_tail_p = mid;
2265
2266 // At this point, mid->_next_om still refers to its current
2267 // value and another ObjectMonitor's _next_om field still
2268 // refers to this ObjectMonitor. Those linkages have to be
2269 // cleaned up by the caller who has the complete context.
2270
2271 // We leave owner == DEFLATER_MARKER and ref_count < 0
2272 // to force any racing threads to retry.
2273 return true; // Success, ObjectMonitor has been deflated.
2274 }
2275
2276 // The owner was changed from DEFLATER_MARKER so we lost the
2277 // race since the ObjectMonitor is now busy.
2278
2279 // Add back max_jint to restore the ref_count field to its
2280 // proper value (which may not be what we saw above):
2281 Atomic::add(&mid->_ref_count, max_jint);
2282
2283 DEBUG_ONLY(jint l_ref_count = mid->ref_count();)
2284 assert(l_ref_count >= 0, "must not be negative: l_ref_count=%d, ref_count=%d",
2285 l_ref_count, mid->ref_count());
2286 return false;
2287 }
2288
2289 // The ref_count was no longer 0 so we lost the race since the
2290 // ObjectMonitor is now busy or the ObjectMonitor* is now is use.
2291 // Restore owner to NULL if it is still DEFLATER_MARKER:
2292 mid->try_set_owner_from(NULL, DEFLATER_MARKER);
2293 }
2294
2295 // The owner field is no longer NULL so we lost the race since the
2296 // ObjectMonitor is now busy.
2297 return false;
2298 }
2299
2300 // Walk a given monitor list, and deflate idle monitors.
2301 // The given list could be a per-thread list or a global list.
2302 //
2303 // In the case of parallel processing of thread local monitor lists,
2304 // work is done by Threads::parallel_threads_do() which ensures that
2305 // each Java thread is processed by exactly one worker thread, and
2306 // thus avoid conflicts that would arise when worker threads would
2307 // process the same monitor lists concurrently.
2308 //
2309 // See also ParallelSPCleanupTask and
2310 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
2311 // Threads::parallel_java_threads_do() in thread.cpp.
2312 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor* volatile * list_p,
2313 int volatile * count_p,
2314 ObjectMonitor** free_head_p,
2315 ObjectMonitor** free_tail_p) {
2316 ObjectMonitor* cur_mid_in_use = NULL;
2317 ObjectMonitor* mid = NULL;
2318 ObjectMonitor* next = NULL;
2319 int deflated_count = 0;
2320
2321 // We use the simpler mark-mid-as-we-go protocol since there are no
2322 // parallel list deletions since we are at a safepoint.
2323 if (!mark_list_head(list_p, &mid, &next)) {
2324 return 0; // The list is empty so nothing to deflate.
2325 }
2326
2327 while (true) {
2328 oop obj = (oop) mid->object();
2329 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
2330 // Deflation succeeded and already updated free_head_p and
2331 // free_tail_p as needed. Finish the move to the local free list
2332 // by unlinking mid from the global or per-thread in-use list.
2333 if (cur_mid_in_use == NULL) {
2334 // mid is the list head and it is marked. Switch the list head
2335 // to next which unmarks the list head, but leaves mid marked:
2336 *list_p = next;
2337 // mark_list_head() used cmpxchg() above, switching list head can be lazier:
2338 OrderAccess::storestore();
2339 } else {
2340 // mid is marked. Switch cur_mid_in_use's next field to next
2341 // which is safe because we have no parallel list deletions,
2342 // but we leave mid marked:
2343 Atomic::release_store(&cur_mid_in_use->_next_om, next);
2344 }
2345 // At this point mid is disconnected from the in-use list so
2346 // its marked next field no longer has any effects.
2347 deflated_count++;
2348 Atomic::dec(count_p);
2349 // mid is current tail in the free_head_p list so NULL terminate it
2350 // (which also unmarks it):
2351 set_next(mid, NULL);
2352
2353 // All the list management is done so move on to the next one:
2354 mid = next;
2355 } else {
2356 set_next(mid, next); // unmark next field
2357
2358 // All the list management is done so move on to the next one:
2359 cur_mid_in_use = mid;
2360 mid = next;
2361 }
2362 if (mid == NULL) {
2363 break; // Reached end of the list so nothing more to deflate.
2364 }
2365 // Mark mid's next field so we can possibly deflate it:
2366 next = mark_next_loop(mid);
2367 }
2368 return deflated_count;
2369 }
2370
2371 // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using
2372 // a JavaThread. Returns the number of deflated ObjectMonitors. The given
2373 // list could be a per-thread in-use list or the global in-use list.
2374 // If a safepoint has started, then we save state via saved_mid_in_use_p
2375 // and return to the caller to honor the safepoint.
2376 //
2377 int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor* volatile * list_p,
2378 int volatile * count_p,
2379 ObjectMonitor** free_head_p,
2380 ObjectMonitor** free_tail_p,
2381 ObjectMonitor** saved_mid_in_use_p) {
2382 assert(AsyncDeflateIdleMonitors, "sanity check");
2383 assert(Thread::current()->is_Java_thread(), "precondition");
2384
2385 ObjectMonitor* cur_mid_in_use = NULL;
2386 ObjectMonitor* mid = NULL;
2387 ObjectMonitor* next = NULL;
2388 ObjectMonitor* next_next = NULL;
2389 int deflated_count = 0;
2390
2391 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
2392 // protocol because om_release() can do list deletions in parallel.
2393 // We also mark-next-next-as-we-go to prevent an om_flush() that is
2394 // behind this thread from passing us.
2395 if (*saved_mid_in_use_p == NULL) {
2396 // No saved state so start at the beginning.
2397 // Mark the list head's next field so we can possibly deflate it:
2398 if (!mark_list_head(list_p, &mid, &next)) {
2399 return 0; // The list is empty so nothing to deflate.
2400 }
2401 } else {
2402 // We're restarting after a safepoint so restore the necessary state
2403 // before we resume.
2404 cur_mid_in_use = *saved_mid_in_use_p;
2405 // Mark cur_mid_in_use's next field so we can possibly update its
2406 // next field to extract a deflated ObjectMonitor.
2407 mid = mark_next_loop(cur_mid_in_use);
2408 if (mid == NULL) {
2409 set_next(cur_mid_in_use, NULL); // unmark next field
2410 *saved_mid_in_use_p = NULL;
2411 return 0; // The remainder is empty so nothing more to deflate.
2412 }
2413 // Mark mid's next field so we can possibly deflate it:
2414 next = mark_next_loop(mid);
2415 }
2416
2417 while (true) {
2418 // The current mid's next field is marked at this point. If we have
2419 // a cur_mid_in_use, then its next field is also marked at this point.
2420
2421 if (next != NULL) {
2422 // We mark next's next field so that an om_flush()
2423 // thread that is behind us cannot pass us when we
2424 // unmark the current mid's next field.
2425 next_next = mark_next_loop(next);
2426 }
2427
2428 // Only try to deflate if there is an associated Java object and if
2429 // mid is old (is not newly allocated and is not newly freed).
2430 if (mid->object() != NULL && mid->is_old() &&
2431 deflate_monitor_using_JT(mid, free_head_p, free_tail_p)) {
2432 // Deflation succeeded and already updated free_head_p and
2433 // free_tail_p as needed. Finish the move to the local free list
2434 // by unlinking mid from the global or per-thread in-use list.
2435 if (cur_mid_in_use == NULL) {
2436 // mid is the list head and it is marked. Switch the list head
2437 // to next which is also marked (if not NULL) and also leave
2438 // mid marked:
2439 *list_p = next;
2440 // mark_list_head() used cmpxchg() above, switching list head can be lazier:
2441 OrderAccess::storestore();
2442 } else {
2443 ObjectMonitor* marked_next = mark_om_ptr(next);
2444 // mid and cur_mid_in_use are marked. Switch cur_mid_in_use's
2445 // next field to marked_next and also leave mid marked:
2446 Atomic::release_store(&cur_mid_in_use->_next_om, marked_next);
2447 }
2448 // At this point mid is disconnected from the in-use list so
2449 // its marked next field no longer has any effects.
2450 deflated_count++;
2451 Atomic::dec(count_p);
2452 // mid is current tail in the free_head_p list so NULL terminate it
2453 // (which also unmarks it):
2454 set_next(mid, NULL);
2455
2456 // All the list management is done so move on to the next one:
2457 mid = next; // mid keeps non-NULL next's marked next field
2458 next = next_next;
2459 } else {
2460 // mid is considered in-use if it does not have an associated
2461 // Java object or mid is not old or deflation did not succeed.
2462 // A mid->is_new() node can be seen here when it is freshly
2463 // returned by om_alloc() (and skips the deflation code path).
2464 // A mid->is_old() node can be seen here when deflation failed.
2465 // A mid->is_free() node can be seen here when a fresh node from
2466 // om_alloc() is released by om_release() due to losing the race
2467 // in inflate().
2468
2469 // All the list management is done so move on to the next one:
2470 if (cur_mid_in_use != NULL) {
2471 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2472 }
2473 // The next cur_mid_in_use keeps mid's marked next field so
2474 // that it is stable for a possible next field change. It
2475 // cannot be modified by om_release() while it is marked.
2476 cur_mid_in_use = mid;
2477 mid = next; // mid keeps non-NULL next's marked next field
2478 next = next_next;
2479
2480 if (SafepointSynchronize::is_synchronizing() &&
2481 cur_mid_in_use != *list_p && cur_mid_in_use->is_old()) {
2482 // If a safepoint has started and cur_mid_in_use is not the list
2483 // head and is old, then it is safe to use as saved state. Return
2484 // to the caller before blocking.
2485 *saved_mid_in_use_p = cur_mid_in_use;
2486 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2487 if (mid != NULL) {
2488 set_next(mid, next); // umark mid
2489 }
2490 return deflated_count;
2491 }
2492 }
2493 if (mid == NULL) {
2494 if (cur_mid_in_use != NULL) {
2495 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2496 }
2497 break; // Reached end of the list so nothing more to deflate.
2498 }
2499
2500 // The current mid's next field is marked at this point. If we have
2501 // a cur_mid_in_use, then its next field is also marked at this point.
2502 }
2503 // We finished the list without a safepoint starting so there's
2504 // no need to save state.
2505 *saved_mid_in_use_p = NULL;
2506 return deflated_count;
2507 }
2508
2509 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2510 counters->n_in_use = 0; // currently associated with objects
2511 counters->n_in_circulation = 0; // extant
2512 counters->n_scavenged = 0; // reclaimed (global and per-thread)
2513 counters->per_thread_scavenged = 0; // per-thread scavenge total
2514 counters->per_thread_times = 0.0; // per-thread scavenge times
2515 OrderAccess::storestore(); // flush inits for worker threads
2516 }
2517
2518 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
2519 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2520
2521 if (AsyncDeflateIdleMonitors) {
2522 // Nothing to do when global idle ObjectMonitors are deflated using
2523 // a JavaThread unless a special deflation has been requested.
2524 if (!is_special_deflation_requested()) {
2525 return;
2526 }
2527 }
2528
2529 bool deflated = false;
2530
2531 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2532 ObjectMonitor* free_tail_p = NULL;
2533 elapsedTimer timer;
2534
2535 if (log_is_enabled(Info, monitorinflation)) {
2536 timer.start();
2537 }
2538
2539 // Note: the thread-local monitors lists get deflated in
2540 // a separate pass. See deflate_thread_local_monitors().
2541
2542 // For moribund threads, scan g_om_in_use_list
2543 int deflated_count = 0;
2544 if (g_om_in_use_list != NULL) {
2545 // Update n_in_circulation before g_om_in_use_count is updated by deflation.
2546 Atomic::add(&counters->n_in_circulation, g_om_in_use_count);
2547
2548 deflated_count = deflate_monitor_list(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p);
2549 Atomic::add(&counters->n_in_use, g_om_in_use_count);
2550 }
2551
2552 if (free_head_p != NULL) {
2553 // Move the deflated ObjectMonitors back to the global free list.
2554 // No races on the working free list so no need for load_acquire().
2555 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2556 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2557 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2558 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2559 Atomic::add(&counters->n_scavenged, deflated_count);
2560 }
2561 timer.stop();
2562
2563 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2564 LogStreamHandle(Info, monitorinflation) lsh_info;
2565 LogStream* ls = NULL;
2566 if (log_is_enabled(Debug, monitorinflation)) {
2567 ls = &lsh_debug;
2568 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2569 ls = &lsh_info;
2570 }
2571 if (ls != NULL) {
2572 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2573 }
2574 }
2575
2576 class HandshakeForDeflation : public HandshakeClosure {
2577 public:
2578 HandshakeForDeflation() : HandshakeClosure("HandshakeForDeflation") {}
2579
2580 void do_thread(Thread* thread) {
2581 log_trace(monitorinflation)("HandshakeForDeflation::do_thread: thread="
2582 INTPTR_FORMAT, p2i(thread));
2583 }
2584 };
2585
2586 void ObjectSynchronizer::deflate_idle_monitors_using_JT() {
2587 assert(AsyncDeflateIdleMonitors, "sanity check");
2588
2589 // Deflate any global idle monitors.
2590 deflate_global_idle_monitors_using_JT();
2591
2592 int count = 0;
2593 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2594 if (jt->om_in_use_count > 0 && !jt->is_exiting()) {
2595 // This JavaThread is using ObjectMonitors so deflate any that
2596 // are idle unless this JavaThread is exiting; do not race with
2597 // ObjectSynchronizer::om_flush().
2598 deflate_per_thread_idle_monitors_using_JT(jt);
2599 count++;
2600 }
2601 }
2602 if (count > 0) {
2603 log_debug(monitorinflation)("did async deflation of idle monitors for %d thread(s).", count);
2604 }
2605
2606 log_info(monitorinflation)("async g_om_population=%d, g_om_in_use_count=%d, "
2607 "g_om_free_count=%d, g_om_wait_count=%d",
2608 g_om_population, g_om_in_use_count,
2609 g_om_free_count, g_om_wait_count);
2610
2611 // The ServiceThread's async deflation request has been processed.
2612 set_is_async_deflation_requested(false);
2613
2614 if (HandshakeAfterDeflateIdleMonitors && g_om_wait_count > 0) {
2615 // There are deflated ObjectMonitors waiting for a handshake
2616 // (or a safepoint) for safety.
2617
2618 // g_wait_list and g_om_wait_count are only updated by the calling
2619 // thread so no need for load_acquire() or release_store().
2620 ObjectMonitor* list = g_wait_list;
2621 ADIM_guarantee(list != NULL, "g_wait_list must not be NULL");
2622 int count = g_om_wait_count;
2623 g_om_wait_count = 0;
2624 g_wait_list = NULL;
2625 OrderAccess::storestore(); // Lazier memory sync is okay for list walkers.
2626
2627 // Find the tail for prepend_list_to_common(). No need to mark
2628 // ObjectMonitors for this list walk since only the deflater
2629 // thread manages the wait list.
2630 int l_count = 0;
2631 ObjectMonitor* tail = NULL;
2632 for (ObjectMonitor* n = list; n != NULL; n = unmarked_next(n)) {
2633 tail = n;
2634 l_count++;
2635 }
2636 ADIM_guarantee(count == l_count, "count=%d != l_count=%d", count, l_count);
2637
2638 // Will execute a safepoint if !ThreadLocalHandshakes:
2639 HandshakeForDeflation hfd_hc;
2640 Handshake::execute(&hfd_hc);
2641
2642 prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count);
2643
2644 log_info(monitorinflation)("moved %d idle monitors from global waiting list to global free list", count);
2645 }
2646 }
2647
2648 // Deflate global idle ObjectMonitors using a JavaThread.
2649 //
2650 void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() {
2651 assert(AsyncDeflateIdleMonitors, "sanity check");
2652 assert(Thread::current()->is_Java_thread(), "precondition");
2653 JavaThread* self = JavaThread::current();
2654
2655 deflate_common_idle_monitors_using_JT(true /* is_global */, self);
2656 }
2657
2658 // Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread.
2659 //
2660 void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) {
2661 assert(AsyncDeflateIdleMonitors, "sanity check");
2662 assert(Thread::current()->is_Java_thread(), "precondition");
2663
2664 deflate_common_idle_monitors_using_JT(false /* !is_global */, target);
2665 }
2666
2667 // Deflate global or per-thread idle ObjectMonitors using a JavaThread.
2668 //
2669 void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) {
2670 JavaThread* self = JavaThread::current();
2671
2672 int deflated_count = 0;
2673 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors
2674 ObjectMonitor* free_tail_p = NULL;
2675 ObjectMonitor* saved_mid_in_use_p = NULL;
2676 elapsedTimer timer;
2677
2678 if (log_is_enabled(Info, monitorinflation)) {
2679 timer.start();
2680 }
2681
2682 if (is_global) {
2683 OM_PERFDATA_OP(MonExtant, set_value(g_om_in_use_count));
2684 } else {
2685 OM_PERFDATA_OP(MonExtant, inc(target->om_in_use_count));
2686 }
2687
2688 do {
2689 int local_deflated_count;
2690 if (is_global) {
2691 local_deflated_count = deflate_monitor_list_using_JT(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2692 } else {
2693 local_deflated_count = deflate_monitor_list_using_JT(&target->om_in_use_list, &target->om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2694 }
2695 deflated_count += local_deflated_count;
2696
2697 if (free_head_p != NULL) {
2698 // Move the deflated ObjectMonitors to the global free list.
2699 // No races on the working list so no need for load_acquire().
2700 guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count);
2701 // Note: The target thread can be doing an om_alloc() that
2702 // is trying to prepend an ObjectMonitor on its in-use list
2703 // at the same time that we have deflated the current in-use
2704 // list head and put it on the local free list. prepend_to_common()
2705 // will detect the race and retry which avoids list corruption,
2706 // but the next field in free_tail_p can flicker to marked
2707 // and then unmarked while prepend_to_common() is sorting it
2708 // all out.
2709 assert(unmarked_next(free_tail_p) == NULL, "must be NULL: _next_om="
2710 INTPTR_FORMAT, p2i(unmarked_next(free_tail_p)));
2711
2712 if (HandshakeAfterDeflateIdleMonitors) {
2713 prepend_list_to_g_wait_list(free_head_p, free_tail_p, local_deflated_count);
2714 } else {
2715 prepend_list_to_g_free_list(free_head_p, free_tail_p, local_deflated_count);
2716 }
2717
2718 OM_PERFDATA_OP(Deflations, inc(local_deflated_count));
2719 }
2720
2721 if (saved_mid_in_use_p != NULL) {
2722 // deflate_monitor_list_using_JT() detected a safepoint starting.
2723 timer.stop();
2724 {
2725 if (is_global) {
2726 log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint.");
2727 } else {
2728 log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target));
2729 }
2730 assert(SafepointSynchronize::is_synchronizing(), "sanity check");
2731 ThreadBlockInVM blocker(self);
2732 }
2733 // Prepare for another loop after the safepoint.
2734 free_head_p = NULL;
2735 free_tail_p = NULL;
2736 if (log_is_enabled(Info, monitorinflation)) {
2737 timer.start();
2738 }
2739 }
2740 } while (saved_mid_in_use_p != NULL);
2741 timer.stop();
2742
2743 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2744 LogStreamHandle(Info, monitorinflation) lsh_info;
2745 LogStream* ls = NULL;
2746 if (log_is_enabled(Debug, monitorinflation)) {
2747 ls = &lsh_debug;
2748 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2749 ls = &lsh_info;
2750 }
2751 if (ls != NULL) {
2752 if (is_global) {
2753 ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2754 } else {
2755 ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count);
2756 }
2757 }
2758 }
2759
2760 void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2761 // Report the cumulative time for deflating each thread's idle
2762 // monitors. Note: if the work is split among more than one
2763 // worker thread, then the reported time will likely be more
2764 // than a beginning to end measurement of the phase.
2765 log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", counters->per_thread_times, counters->per_thread_scavenged);
2766
2767 bool needs_special_deflation = is_special_deflation_requested();
2768 if (AsyncDeflateIdleMonitors && !needs_special_deflation) {
2769 // Nothing to do when idle ObjectMonitors are deflated using
2770 // a JavaThread unless a special deflation has been requested.
2771 return;
2772 }
2773
2774 if (log_is_enabled(Debug, monitorinflation)) {
2775 // exit_globals()'s call to audit_and_print_stats() is done
2776 // at the Info level and not at a safepoint.
2777 // For async deflation, audit_and_print_stats() is called in
2778 // ObjectSynchronizer::do_safepoint_work() at the Debug level
2779 // at a safepoint.
2780 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2781 } else if (log_is_enabled(Info, monitorinflation)) {
2782 log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, "
2783 "g_om_free_count=%d, g_om_wait_count=%d",
2784 g_om_population, g_om_in_use_count,
2785 g_om_free_count, g_om_wait_count);
2786 }
2787
2788 Atomic::store(&_forceMonitorScavenge, 0); // Reset
2789
2790 OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged));
2791 OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation));
2792
2793 GVars.stw_random = os::random();
2794 GVars.stw_cycle++;
2795
2796 if (needs_special_deflation) {
2797 set_is_special_deflation_requested(false); // special deflation is done
2798 }
2799 }
2800
2801 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
2802 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2803
2804 if (AsyncDeflateIdleMonitors && !is_special_deflation_requested()) {
2805 // Nothing to do if a special deflation has NOT been requested.
2806 return;
2807 }
2808
2809 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2810 ObjectMonitor* free_tail_p = NULL;
2811 elapsedTimer timer;
2812
2813 if (log_is_enabled(Info, safepoint, cleanup) ||
2814 log_is_enabled(Info, monitorinflation)) {
2815 timer.start();
2816 }
2817
2818 // Update n_in_circulation before om_in_use_count is updated by deflation.
2819 Atomic::add(&counters->n_in_circulation, thread->om_in_use_count);
2820
2821 int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p);
2822 Atomic::add(&counters->n_in_use, thread->om_in_use_count);
2823
2824 if (free_head_p != NULL) {
2825 // Move the deflated ObjectMonitors back to the global free list.
2826 // No races on the working list so no need for load_acquire().
2827 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2828 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2829 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2830 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2831 Atomic::add(&counters->n_scavenged, deflated_count);
2832 Atomic::add(&counters->per_thread_scavenged, deflated_count);
2833 }
2834
2835 timer.stop();
2836 // Safepoint logging cares about cumulative per_thread_times and
2837 // we'll capture most of the cost, but not the muxRelease() which
2838 // should be cheap.
2839 counters->per_thread_times += timer.seconds();
2840
2841 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2842 LogStreamHandle(Info, monitorinflation) lsh_info;
2843 LogStream* ls = NULL;
2844 if (log_is_enabled(Debug, monitorinflation)) {
2845 ls = &lsh_debug;
2846 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2847 ls = &lsh_info;
2848 }
2849 if (ls != NULL) {
2850 ls->print_cr("jt=" INTPTR_FORMAT ": deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(thread), timer.seconds(), deflated_count);
2851 }
2852 }
2853
2854 // Monitor cleanup on JavaThread::exit
2855
2856 // Iterate through monitor cache and attempt to release thread's monitors
2857 // Gives up on a particular monitor if an exception occurs, but continues
2858 // the overall iteration, swallowing the exception.
2859 class ReleaseJavaMonitorsClosure: public MonitorClosure {
2860 private:
2861 TRAPS;
2862
2863 public:
2864 ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {}
2865 void do_monitor(ObjectMonitor* mid) {
2866 if (mid->owner() == THREAD) {
2867 (void)mid->complete_exit(CHECK);
2868 }
2869 }
2870 };
2871
2872 // Release all inflated monitors owned by THREAD. Lightweight monitors are
2873 // ignored. This is meant to be called during JNI thread detach which assumes
2874 // all remaining monitors are heavyweight. All exceptions are swallowed.
2875 // Scanning the extant monitor list can be time consuming.
2876 // A simple optimization is to add a per-thread flag that indicates a thread
2877 // called jni_monitorenter() during its lifetime.
2878 //
2879 // Instead of No_Savepoint_Verifier it might be cheaper to
2880 // use an idiom of the form:
2881 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
2882 // <code that must not run at safepoint>
2883 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
2884 // Since the tests are extremely cheap we could leave them enabled
2885 // for normal product builds.
2886
2887 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
2888 assert(THREAD == JavaThread::current(), "must be current Java thread");
2889 NoSafepointVerifier nsv;
2890 ReleaseJavaMonitorsClosure rjmc(THREAD);
2891 ObjectSynchronizer::monitors_iterate(&rjmc);
2892 THREAD->clear_pending_exception();
2893 }
2894
2895 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
2896 switch (cause) {
2897 case inflate_cause_vm_internal: return "VM Internal";
2898 case inflate_cause_monitor_enter: return "Monitor Enter";
2899 case inflate_cause_wait: return "Monitor Wait";
2900 case inflate_cause_notify: return "Monitor Notify";
2901 case inflate_cause_hash_code: return "Monitor Hash Code";
2902 case inflate_cause_jni_enter: return "JNI Monitor Enter";
2903 case inflate_cause_jni_exit: return "JNI Monitor Exit";
2904 default:
2905 ShouldNotReachHere();
2906 }
2907 return "Unknown";
2908 }
2909
2910 //------------------------------------------------------------------------------
2911 // Debugging code
2912
2913 u_char* ObjectSynchronizer::get_gvars_addr() {
2914 return (u_char*)&GVars;
2915 }
2916
2917 u_char* ObjectSynchronizer::get_gvars_hc_sequence_addr() {
2918 return (u_char*)&GVars.hc_sequence;
2919 }
2920
2921 size_t ObjectSynchronizer::get_gvars_size() {
2922 return sizeof(SharedGlobals);
2923 }
2924
2925 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
2926 return (u_char*)&GVars.stw_random;
2927 }
2928
2929 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
2930 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
2931
2932 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2933 LogStreamHandle(Info, monitorinflation) lsh_info;
2934 LogStreamHandle(Trace, monitorinflation) lsh_trace;
2935 LogStream* ls = NULL;
2936 if (log_is_enabled(Trace, monitorinflation)) {
2937 ls = &lsh_trace;
2938 } else if (log_is_enabled(Debug, monitorinflation)) {
2939 ls = &lsh_debug;
2940 } else if (log_is_enabled(Info, monitorinflation)) {
2941 ls = &lsh_info;
2942 }
2943 assert(ls != NULL, "sanity check");
2944
2945 // Log counts for the global and per-thread monitor lists:
2946 int chk_om_population = log_monitor_list_counts(ls);
2947 int error_cnt = 0;
2948
2949 ls->print_cr("Checking global lists:");
2950
2951 // Check g_om_population:
2952 if (g_om_population == chk_om_population) {
2953 ls->print_cr("g_om_population=%d equals chk_om_population=%d",
2954 g_om_population, chk_om_population);
2955 } else {
2956 // With lock free access to the monitor lists, it is possible for
2957 // log_monitor_list_counts() to return a value that doesn't match
2958 // g_om_population. So far a higher value has been seen in testing
2959 // so something is being double counted by log_monitor_list_counts().
2960 ls->print_cr("WARNING: g_om_population=%d is not equal to "
2961 "chk_om_population=%d", g_om_population, chk_om_population);
2962 }
2963
2964 // Check g_om_in_use_list and g_om_in_use_count:
2965 chk_global_in_use_list_and_count(ls, &error_cnt);
2966
2967 // Check g_free_list and g_om_free_count:
2968 chk_global_free_list_and_count(ls, &error_cnt);
2969
2970 if (HandshakeAfterDeflateIdleMonitors) {
2971 // Check g_wait_list and g_om_wait_count:
2972 chk_global_wait_list_and_count(ls, &error_cnt);
2973 }
2974
2975 ls->print_cr("Checking per-thread lists:");
2976
2977 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2978 // Check om_in_use_list and om_in_use_count:
2979 chk_per_thread_in_use_list_and_count(jt, ls, &error_cnt);
2980
2981 // Check om_free_list and om_free_count:
2982 chk_per_thread_free_list_and_count(jt, ls, &error_cnt);
2983 }
2984
2985 if (error_cnt == 0) {
2986 ls->print_cr("No errors found in monitor list checks.");
2987 } else {
2988 log_error(monitorinflation)("found monitor list errors: error_cnt=%d", error_cnt);
2989 }
2990
2991 if ((on_exit && log_is_enabled(Info, monitorinflation)) ||
2992 (!on_exit && log_is_enabled(Trace, monitorinflation))) {
2993 // When exiting this log output is at the Info level. When called
2994 // at a safepoint, this log output is at the Trace level since
2995 // there can be a lot of it.
2996 log_in_use_monitor_details(ls);
2997 }
2998
2999 ls->flush();
3000
3001 guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
3002 }
3003
3004 // Check a free monitor entry; log any errors.
3005 void ObjectSynchronizer::chk_free_entry(JavaThread* jt, ObjectMonitor* n,
3006 outputStream * out, int *error_cnt_p) {
3007 stringStream ss;
3008 if (n->is_busy()) {
3009 if (jt != NULL) {
3010 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3011 ": free per-thread monitor must not be busy: %s", p2i(jt),
3012 p2i(n), n->is_busy_to_string(&ss));
3013 } else {
3014 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3015 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
3016 }
3017 *error_cnt_p = *error_cnt_p + 1;
3018 }
3019 if (n->header().value() != 0) {
3020 if (jt != NULL) {
3021 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3022 ": free per-thread monitor must have NULL _header "
3023 "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n),
3024 n->header().value());
3025 *error_cnt_p = *error_cnt_p + 1;
3026 } else if (!AsyncDeflateIdleMonitors) {
3027 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3028 "must have NULL _header field: _header=" INTPTR_FORMAT,
3029 p2i(n), n->header().value());
3030 *error_cnt_p = *error_cnt_p + 1;
3031 }
3032 }
3033 if (n->object() != NULL) {
3034 if (jt != NULL) {
3035 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3036 ": free per-thread monitor must have NULL _object "
3037 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
3038 p2i(n->object()));
3039 } else {
3040 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3041 "must have NULL _object field: _object=" INTPTR_FORMAT,
3042 p2i(n), p2i(n->object()));
3043 }
3044 *error_cnt_p = *error_cnt_p + 1;
3045 }
3046 }
3047
3048 // Check the global free list and count; log the results of the checks.
3049 void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out,
3050 int *error_cnt_p) {
3051 int chk_om_free_count = 0;
3052 ObjectMonitor* cur = NULL;
3053 ObjectMonitor* next = NULL;
3054 if (mark_list_head(&g_free_list, &cur, &next)) {
3055 // Marked the global free list head so process the list.
3056 while (true) {
3057 chk_free_entry(NULL /* jt */, cur, out, error_cnt_p);
3058 chk_om_free_count++;
3059
3060 mark_next_for_traversal(&cur, &next);
3061 if (cur == NULL) {
3062 break;
3063 }
3064 }
3065 }
3066 if (g_om_free_count == chk_om_free_count) {
3067 out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d",
3068 g_om_free_count, chk_om_free_count);
3069 } else {
3070 // With lock free access to g_free_list, it is possible for an
3071 // ObjectMonitor to be prepended to g_free_list after we started
3072 // calculating chk_om_free_count so g_om_free_count may not
3073 // match anymore.
3074 out->print_cr("WARNING: g_om_free_count=%d is not equal to "
3075 "chk_om_free_count=%d", g_om_free_count, chk_om_free_count);
3076 }
3077 }
3078
3079 // Check the global wait list and count; log the results of the checks.
3080 void ObjectSynchronizer::chk_global_wait_list_and_count(outputStream * out,
3081 int *error_cnt_p) {
3082 int chk_om_wait_count = 0;
3083 ObjectMonitor* cur = NULL;
3084 ObjectMonitor* next = NULL;
3085 if (mark_list_head(&g_wait_list, &cur, &next)) {
3086 // Marked the global wait list head so process the list.
3087 while (true) {
3088 // Rules for g_wait_list are the same as of g_free_list:
3089 chk_free_entry(NULL /* jt */, cur, out, error_cnt_p);
3090 chk_om_wait_count++;
3091
3092 mark_next_for_traversal(&cur, &next);
3093 if (cur == NULL) {
3094 break;
3095 }
3096 }
3097 }
3098 if (g_om_wait_count == chk_om_wait_count) {
3099 out->print_cr("g_om_wait_count=%d equals chk_om_wait_count=%d",
3100 g_om_wait_count, chk_om_wait_count);
3101 } else {
3102 out->print_cr("ERROR: g_om_wait_count=%d is not equal to "
3103 "chk_om_wait_count=%d", g_om_wait_count, chk_om_wait_count);
3104 *error_cnt_p = *error_cnt_p + 1;
3105 }
3106 }
3107
3108 // Check the global in-use list and count; log the results of the checks.
3109 void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out,
3110 int *error_cnt_p) {
3111 int chk_om_in_use_count = 0;
3112 ObjectMonitor* cur = NULL;
3113 ObjectMonitor* next = NULL;
3114 if (mark_list_head(&g_om_in_use_list, &cur, &next)) {
3115 // Marked the global in-use list head so process the list.
3116 while (true) {
3117 chk_in_use_entry(NULL /* jt */, cur, out, error_cnt_p);
3118 chk_om_in_use_count++;
3119
3120 mark_next_for_traversal(&cur, &next);
3121 if (cur == NULL) {
3122 break;
3123 }
3124 }
3125 }
3126 if (g_om_in_use_count == chk_om_in_use_count) {
3127 out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d",
3128 g_om_in_use_count, chk_om_in_use_count);
3129 } else {
3130 // With lock free access to the monitor lists, it is possible for
3131 // an exiting JavaThread to put its in-use ObjectMonitors on the
3132 // global in-use list after chk_om_in_use_count is calculated above.
3133 out->print_cr("WARNING: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d",
3134 g_om_in_use_count, chk_om_in_use_count);
3135 }
3136 }
3137
3138 // Check an in-use monitor entry; log any errors.
3139 void ObjectSynchronizer::chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,
3140 outputStream * out, int *error_cnt_p) {
3141 if (n->header().value() == 0) {
3142 if (jt != NULL) {
3143 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3144 ": in-use per-thread monitor must have non-NULL _header "
3145 "field.", p2i(jt), p2i(n));
3146 } else {
3147 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
3148 "must have non-NULL _header field.", p2i(n));
3149 }
3150 *error_cnt_p = *error_cnt_p + 1;
3151 }
3152 if (n->object() == NULL) {
3153 if (jt != NULL) {
3154 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3155 ": in-use per-thread monitor must have non-NULL _object "
3156 "field.", p2i(jt), p2i(n));
3157 } else {
3158 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
3159 "must have non-NULL _object field.", p2i(n));
3160 }
3161 *error_cnt_p = *error_cnt_p + 1;
3162 }
3163 const oop obj = (oop)n->object();
3164 const markWord mark = obj->mark();
3165 if (!mark.has_monitor()) {
3166 if (jt != NULL) {
3167 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3168 ": in-use per-thread monitor's object does not think "
3169 "it has a monitor: obj=" INTPTR_FORMAT ", mark="
3170 INTPTR_FORMAT, p2i(jt), p2i(n), p2i(obj), mark.value());
3171 } else {
3172 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
3173 "monitor's object does not think it has a monitor: obj="
3174 INTPTR_FORMAT ", mark=" INTPTR_FORMAT, p2i(n),
3175 p2i(obj), mark.value());
3176 }
3177 *error_cnt_p = *error_cnt_p + 1;
3178 }
3179 ObjectMonitor* const obj_mon = mark.monitor();
3180 if (n != obj_mon) {
3181 if (jt != NULL) {
3182 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3183 ": in-use per-thread monitor's object does not refer "
3184 "to the same monitor: obj=" INTPTR_FORMAT ", mark="
3185 INTPTR_FORMAT ", obj_mon=" INTPTR_FORMAT, p2i(jt),
3186 p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3187 } else {
3188 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
3189 "monitor's object does not refer to the same monitor: obj="
3190 INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
3191 INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3192 }
3193 *error_cnt_p = *error_cnt_p + 1;
3194 }
3195 }
3196
3197 // Check the thread's free list and count; log the results of the checks.
3198 void ObjectSynchronizer::chk_per_thread_free_list_and_count(JavaThread *jt,
3199 outputStream * out,
3200 int *error_cnt_p) {
3201 int chk_om_free_count = 0;
3202 ObjectMonitor* cur = NULL;
3203 ObjectMonitor* next = NULL;
3204 if (mark_list_head(&jt->om_free_list, &cur, &next)) {
3205 // Marked the per-thread free list head so process the list.
3206 while (true) {
3207 chk_free_entry(jt, cur, out, error_cnt_p);
3208 chk_om_free_count++;
3209
3210 mark_next_for_traversal(&cur, &next);
3211 if (cur == NULL) {
3212 break;
3213 }
3214 }
3215 }
3216 if (jt->om_free_count == chk_om_free_count) {
3217 out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals "
3218 "chk_om_free_count=%d", p2i(jt), jt->om_free_count,
3219 chk_om_free_count);
3220 } else {
3221 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not "
3222 "equal to chk_om_free_count=%d", p2i(jt), jt->om_free_count,
3223 chk_om_free_count);
3224 *error_cnt_p = *error_cnt_p + 1;
3225 }
3226 }
3227
3228 // Check the thread's in-use list and count; log the results of the checks.
3229 void ObjectSynchronizer::chk_per_thread_in_use_list_and_count(JavaThread *jt,
3230 outputStream * out,
3231 int *error_cnt_p) {
3232 int chk_om_in_use_count = 0;
3233 ObjectMonitor* cur = NULL;
3234 ObjectMonitor* next = NULL;
3235 if (mark_list_head(&jt->om_in_use_list, &cur, &next)) {
3236 // Marked the per-thread in-use list head so process the list.
3237 while (true) {
3238 chk_in_use_entry(jt, cur, out, error_cnt_p);
3239 chk_om_in_use_count++;
3240
3241 mark_next_for_traversal(&cur, &next);
3242 if (cur == NULL) {
3243 break;
3244 }
3245 }
3246 }
3247 if (jt->om_in_use_count == chk_om_in_use_count) {
3248 out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals "
3249 "chk_om_in_use_count=%d", p2i(jt),
3250 jt->om_in_use_count, chk_om_in_use_count);
3251 } else {
3252 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not "
3253 "equal to chk_om_in_use_count=%d", p2i(jt),
3254 jt->om_in_use_count, chk_om_in_use_count);
3255 *error_cnt_p = *error_cnt_p + 1;
3256 }
3257 }
3258
3259 // Log details about ObjectMonitors on the in-use lists. The 'BHL'
3260 // flags indicate why the entry is in-use, 'object' and 'object type'
3261 // indicate the associated object and its type.
3262 void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out) {
3263 stringStream ss;
3264 if (g_om_in_use_count > 0) {
3265 out->print_cr("In-use global monitor info:");
3266 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3267 out->print_cr("%18s %s %7s %18s %18s",
3268 "monitor", "BHL", "ref_cnt", "object", "object type");
3269 out->print_cr("================== === ======= ================== ==================");
3270 ObjectMonitor* cur = NULL;
3271 ObjectMonitor* next = NULL;
3272 if (mark_list_head(&g_om_in_use_list, &cur, &next)) {
3273 // Marked the global in-use list head so process the list.
3274 while (true) {
3275 const oop obj = (oop) cur->object();
3276 const markWord mark = cur->header();
3277 ResourceMark rm;
3278 out->print(INTPTR_FORMAT " %d%d%d %7d " INTPTR_FORMAT " %s",
3279 p2i(cur), cur->is_busy() != 0, mark.hash() != 0,
3280 cur->owner() != NULL, (int)cur->ref_count(), p2i(obj),
3281 obj->klass()->external_name());
3282 if (cur->is_busy() != 0) {
3283 out->print(" (%s)", cur->is_busy_to_string(&ss));
3284 ss.reset();
3285 }
3286 out->cr();
3287
3288 mark_next_for_traversal(&cur, &next);
3289 if (cur == NULL) {
3290 break;
3291 }
3292 }
3293 }
3294 }
3295
3296 out->print_cr("In-use per-thread monitor info:");
3297 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3298 out->print_cr("%18s %18s %s %7s %18s %18s",
3299 "jt", "monitor", "BHL", "ref_cnt", "object", "object type");
3300 out->print_cr("================== ================== === ======= ================== ==================");
3301 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3302 ObjectMonitor* cur = NULL;
3303 ObjectMonitor* next = NULL;
3304 if (mark_list_head(&jt->om_in_use_list, &cur, &next)) {
3305 // Marked the global in-use list head so process the list.
3306 while (true) {
3307 const oop obj = (oop) cur->object();
3308 const markWord mark = cur->header();
3309 ResourceMark rm;
3310 out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d %7d "
3311 INTPTR_FORMAT " %s", p2i(jt), p2i(cur), cur->is_busy() != 0,
3312 mark.hash() != 0, cur->owner() != NULL, (int)cur->ref_count(),
3313 p2i(obj), obj->klass()->external_name());
3314 if (cur->is_busy() != 0) {
3315 out->print(" (%s)", cur->is_busy_to_string(&ss));
3316 ss.reset();
3317 }
3318 out->cr();
3319
3320 mark_next_for_traversal(&cur, &next);
3321 if (cur == NULL) {
3322 break;
3323 }
3324 }
3325 }
3326 }
3327
3328 out->flush();
3329 }
3330
3331 // Log counts for the global and per-thread monitor lists and return
3332 // the population count.
3333 int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) {
3334 int pop_count = 0;
3335 out->print_cr("%18s %10s %10s %10s %10s",
3336 "Global Lists:", "InUse", "Free", "Wait", "Total");
3337 out->print_cr("================== ========== ========== ========== ==========");
3338 out->print_cr("%18s %10d %10d %10d %10d", "", g_om_in_use_count,
3339 g_om_free_count, g_om_wait_count, g_om_population);
3340 pop_count += g_om_in_use_count + g_om_free_count;
3341 if (HandshakeAfterDeflateIdleMonitors) {
3342 pop_count += g_om_wait_count;
3343 }
3344
3345 out->print_cr("%18s %10s %10s %10s",
3346 "Per-Thread Lists:", "InUse", "Free", "Provision");
3347 out->print_cr("================== ========== ========== ==========");
3348
3349 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3350 out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt),
3351 jt->om_in_use_count, jt->om_free_count, jt->om_free_provision);
3352 pop_count += jt->om_in_use_count + jt->om_free_count;
3353 }
3354 return pop_count;
3355 }
3356
3357 #ifndef PRODUCT
3358
3359 // Check if monitor belongs to the monitor cache
3360 // The list is grow-only so it's *relatively* safe to traverse
3361 // the list of extant blocks without taking a lock.
3362
3363 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
3364 PaddedObjectMonitor* block = g_block_list;
3365 while (block != NULL) {
3366 assert(block->object() == CHAINMARKER, "must be a block header");
3367 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
3368 address mon = (address)monitor;
3369 address blk = (address)block;
3370 size_t diff = mon - blk;
3371 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
3372 return 1;
3373 }
3374 // unmarked_next() is not needed with g_block_list (no next field
3375 // marking) and no load_acquire() needed because _next_om is
3376 // updated before g_block_list is changed with cmpxchg().
3377 block = (PaddedObjectMonitor*)block->_next_om;
3378 }
3379 return 0;
3380 }
3381
3382 #endif