1 /*
2 * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "logging/log.hpp"
28 #include "logging/logStream.hpp"
29 #include "jfr/jfrEvents.hpp"
30 #include "memory/allocation.inline.hpp"
31 #include "memory/metaspaceShared.hpp"
32 #include "memory/padded.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/markWord.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "runtime/atomic.hpp"
38 #include "runtime/biasedLocking.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/handshake.hpp"
41 #include "runtime/interfaceSupport.inline.hpp"
42 #include "runtime/mutexLocker.hpp"
43 #include "runtime/objectMonitor.hpp"
44 #include "runtime/objectMonitor.inline.hpp"
45 #include "runtime/osThread.hpp"
46 #include "runtime/safepointVerifiers.hpp"
47 #include "runtime/sharedRuntime.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "runtime/synchronizer.hpp"
50 #include "runtime/thread.inline.hpp"
51 #include "runtime/timer.hpp"
52 #include "runtime/vframe.hpp"
53 #include "runtime/vmThread.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/dtrace.hpp"
56 #include "utilities/events.hpp"
57 #include "utilities/preserveException.hpp"
58
59 // The "core" versions of monitor enter and exit reside in this file.
60 // The interpreter and compilers contain specialized transliterated
61 // variants of the enter-exit fast-path operations. See i486.ad fast_lock(),
62 // for instance. If you make changes here, make sure to modify the
63 // interpreter, and both C1 and C2 fast-path inline locking code emission.
64 //
65 // -----------------------------------------------------------------------------
66
67 #ifdef DTRACE_ENABLED
68
69 // Only bother with this argument setup if dtrace is available
70 // TODO-FIXME: probes should not fire when caller is _blocked. assert() accordingly.
71
72 #define DTRACE_MONITOR_PROBE_COMMON(obj, thread) \
73 char* bytes = NULL; \
74 int len = 0; \
75 jlong jtid = SharedRuntime::get_java_tid(thread); \
76 Symbol* klassname = ((oop)(obj))->klass()->name(); \
77 if (klassname != NULL) { \
78 bytes = (char*)klassname->bytes(); \
79 len = klassname->utf8_length(); \
80 }
81
82 #define DTRACE_MONITOR_WAIT_PROBE(monitor, obj, thread, millis) \
83 { \
84 if (DTraceMonitorProbes) { \
85 DTRACE_MONITOR_PROBE_COMMON(obj, thread); \
86 HOTSPOT_MONITOR_WAIT(jtid, \
87 (uintptr_t)(monitor), bytes, len, (millis)); \
88 } \
89 }
90
91 #define HOTSPOT_MONITOR_PROBE_notify HOTSPOT_MONITOR_NOTIFY
92 #define HOTSPOT_MONITOR_PROBE_notifyAll HOTSPOT_MONITOR_NOTIFYALL
93 #define HOTSPOT_MONITOR_PROBE_waited HOTSPOT_MONITOR_WAITED
94
95 #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread) \
96 { \
97 if (DTraceMonitorProbes) { \
98 DTRACE_MONITOR_PROBE_COMMON(obj, thread); \
99 HOTSPOT_MONITOR_PROBE_##probe(jtid, /* probe = waited */ \
100 (uintptr_t)(monitor), bytes, len); \
101 } \
102 }
103
104 #else // ndef DTRACE_ENABLED
105
106 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon) {;}
107 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon) {;}
108
109 #endif // ndef DTRACE_ENABLED
110
111 // This exists only as a workaround of dtrace bug 6254741
112 int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
113 DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
114 return 0;
115 }
116
117 #define NINFLATIONLOCKS 256
118 static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];
119
120 // global list of blocks of monitors
121 PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL;
122 bool volatile ObjectSynchronizer::_is_async_deflation_requested = false;
123 bool volatile ObjectSynchronizer::_is_special_deflation_requested = false;
124 jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0;
125
126 // Global ObjectMonitor free list. Newly allocated and deflated
127 // ObjectMonitors are prepended here.
128 static ObjectMonitor* volatile g_free_list = NULL;
129 // Global ObjectMonitor in-use list. When a JavaThread is exiting,
130 // ObjectMonitors on its per-thread in-use list are prepended here.
131 static ObjectMonitor* volatile g_om_in_use_list = NULL;
132 // Global ObjectMonitor wait list. If HandshakeAfterDeflateIdleMonitors
133 // is true, deflated ObjectMonitors wait on this list until after a
134 // handshake or a safepoint for platforms that don't support handshakes.
135 // After the handshake or safepoint, the deflated ObjectMonitors are
136 // prepended to g_free_list.
137 static ObjectMonitor* volatile g_wait_list = NULL;
138
139 static volatile int g_om_free_count = 0; // # on g_free_list
140 static volatile int g_om_in_use_count = 0; // # on g_om_in_use_list
141 static volatile int g_om_population = 0; // # Extant -- in circulation
142 static volatile int g_om_wait_count = 0; // # on g_wait_list
143
144 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
145
146
147 // =====================> List Management functions
148
149 // Return true if the ObjectMonitor's next field is marked.
150 // Otherwise returns false.
151 static bool is_next_marked(ObjectMonitor* om) {
152 return ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & 0x1) != 0;
153 }
154
155 // Mark an ObjectMonitor* and return it. Note: the om parameter
156 // may or may not have been marked originally.
157 static ObjectMonitor* mark_om_ptr(ObjectMonitor* om) {
158 return (ObjectMonitor*)((intptr_t)om | 0x1);
159 }
160
161 // Mark the next field in an ObjectMonitor. If marking was successful,
162 // then the unmarked next field is returned via parameter and true is
163 // returned. Otherwise false is returned.
164 static bool mark_next(ObjectMonitor* om, ObjectMonitor** next_p) {
165 // Get current next field without any marking value.
166 ObjectMonitor* next = (ObjectMonitor*)
167 ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1);
168 if (Atomic::cmpxchg(mark_om_ptr(next), &om->_next_om, next) != next) {
169 return false; // Could not mark the next field or it was already marked.
170 }
171 *next_p = next;
172 return true;
173 }
174
175 // Loop until we mark the next field in an ObjectMonitor. The unmarked
176 // next field is returned.
177 static ObjectMonitor* mark_next_loop(ObjectMonitor* om) {
178 ObjectMonitor* next;
179 while (true) {
180 if (mark_next(om, &next)) {
181 // Marked om's next field so return the unmarked value.
182 return next;
183 }
184 }
185 }
186
187 // Set the next field in an ObjectMonitor to the specified value.
188 // The caller of set_next() must be the same thread that marked the
189 // ObjectMonitor.
190 static void set_next(ObjectMonitor* om, ObjectMonitor* value) {
191 OrderAccess::release_store(&om->_next_om, value);
192 }
193
194 // Mark the next field in the list head ObjectMonitor. If marking was
195 // successful, then the mid and the unmarked next field are returned
196 // via parameter and true is returned. Otherwise false is returned.
197 static bool mark_list_head(ObjectMonitor* volatile * list_p,
198 ObjectMonitor** mid_p, ObjectMonitor** next_p) {
199 while (true) {
200 ObjectMonitor* mid = OrderAccess::load_acquire(list_p);
201 if (mid == NULL) {
202 return false; // The list is empty so nothing to mark.
203 }
204 if (mark_next(mid, next_p)) {
205 if (OrderAccess::load_acquire(list_p) != mid) {
206 // The list head changed so we have to retry.
207 set_next(mid, *next_p); // unmark mid
208 continue;
209 }
210 // We marked next field to guard against races.
211 *mid_p = mid;
212 return true;
213 }
214 }
215 }
216
217 // Return the unmarked next field in an ObjectMonitor. Note: the next
218 // field may or may not have been marked originally.
219 static ObjectMonitor* unmarked_next(ObjectMonitor* om) {
220 return (ObjectMonitor*)((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1);
221 }
222
223 // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is
224 // the last ObjectMonitor in the list and there are 'count' on the list.
225 // Also updates the specified *count_p.
226 static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail,
227 int count, ObjectMonitor* volatile* list_p,
228 volatile int* count_p) {
229 while (true) {
230 ObjectMonitor* cur = OrderAccess::load_acquire(list_p);
231 // Prepend list to *list_p.
232 ObjectMonitor* next = NULL;
233 if (!mark_next(tail, &next)) {
234 continue; // failed to mark next field so try it all again
235 }
236 set_next(tail, cur); // tail now points to cur (and unmarks tail)
237 if (cur == NULL) {
238 // No potential race with takers or other prependers since
239 // *list_p is empty.
240 if (Atomic::cmpxchg(list, list_p, cur) == cur) {
241 // Successfully switched *list_p to the list value.
242 Atomic::add(count, count_p);
243 break;
244 }
245 // Implied else: try it all again
246 } else {
247 // Try to mark next field to guard against races:
248 if (!mark_next(cur, &next)) {
249 continue; // failed to mark next field so try it all again
250 }
251 // We marked the next field so try to switch *list_p to the list value.
252 if (Atomic::cmpxchg(list, list_p, cur) != cur) {
253 // The list head has changed so unmark the next field and try again:
254 set_next(cur, next);
255 continue;
256 }
257 Atomic::add(count, count_p);
258 set_next(cur, next); // unmark next field
259 break;
260 }
261 }
262 }
263
264 // Prepend a newly allocated block of ObjectMonitors to g_block_list and
265 // g_free_list. Also updates g_om_population and g_om_free_count.
266 void ObjectSynchronizer::prepend_block_to_lists(PaddedObjectMonitor* new_blk) {
267 // First we handle g_block_list:
268 while (true) {
269 PaddedObjectMonitor* cur = g_block_list;
270 // Prepend new_blk to g_block_list. The first ObjectMonitor in
271 // a block is reserved for use as linkage to the next block.
272 new_blk[0]._next_om = cur;
273 if (Atomic::cmpxchg(new_blk, &g_block_list, cur) == cur) {
274 // Successfully switched g_block_list to the new_blk value.
275 Atomic::add(_BLOCKSIZE - 1, &g_om_population);
276 break;
277 }
278 // Implied else: try it all again
279 }
280
281 // Second we handle g_free_list:
282 prepend_list_to_common(new_blk + 1, &new_blk[_BLOCKSIZE - 1], _BLOCKSIZE - 1,
283 &g_free_list, &g_om_free_count);
284 }
285
286 // Prepend a list of ObjectMonitors to g_free_list. 'tail' is the last
287 // ObjectMonitor in the list and there are 'count' on the list. Also
288 // updates g_om_free_count.
289 static void prepend_list_to_g_free_list(ObjectMonitor* list,
290 ObjectMonitor* tail, int count) {
291 prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count);
292 }
293
294 // Prepend a list of ObjectMonitors to g_wait_list. 'tail' is the last
295 // ObjectMonitor in the list and there are 'count' on the list. Also
296 // updates g_om_wait_count.
297 static void prepend_list_to_g_wait_list(ObjectMonitor* list,
298 ObjectMonitor* tail, int count) {
299 assert(HandshakeAfterDeflateIdleMonitors, "sanity check");
300 prepend_list_to_common(list, tail, count, &g_wait_list, &g_om_wait_count);
301 }
302
303 // Prepend a list of ObjectMonitors to g_om_in_use_list. 'tail' is the last
304 // ObjectMonitor in the list and there are 'count' on the list. Also
305 // updates g_om_in_use_list.
306 static void prepend_list_to_g_om_in_use_list(ObjectMonitor* list,
307 ObjectMonitor* tail, int count) {
308 prepend_list_to_common(list, tail, count, &g_om_in_use_list, &g_om_in_use_count);
309 }
310
311 // Prepend an ObjectMonitor to the specified list. Also updates
312 // the specified counter.
313 static void prepend_to_common(ObjectMonitor* m, ObjectMonitor* volatile * list_p,
314 int volatile * count_p) {
315 while (true) {
316 (void)mark_next_loop(m); // mark m so we can safely update its next field
317 ObjectMonitor* cur = NULL;
318 ObjectMonitor* next = NULL;
319 // Mark the list head to guard against A-B-A race:
320 if (mark_list_head(list_p, &cur, &next)) {
321 // List head is now marked so we can safely switch it.
322 set_next(m, cur); // m now points to cur (and unmarks m)
323 OrderAccess::release_store(list_p, m); // Switch list head to unmarked m.
324 set_next(cur, next); // Unmark the previous list head.
325 break;
326 }
327 // The list is empty so try to set the list head.
328 assert(cur == NULL, "cur must be NULL: cur=" INTPTR_FORMAT, p2i(cur));
329 set_next(m, cur); // m now points to NULL (and unmarks m)
330 if (Atomic::cmpxchg(m, list_p, cur) == cur) {
331 // List head is now unmarked m.
332 break;
333 }
334 // Implied else: try it all again
335 }
336 Atomic::inc(count_p);
337 }
338
339 // Prepend an ObjectMonitor to a per-thread om_free_list.
340 // Also updates the per-thread om_free_count.
341 static void prepend_to_om_free_list(Thread* self, ObjectMonitor* m) {
342 prepend_to_common(m, &self->om_free_list, &self->om_free_count);
343 }
344
345 // Prepend an ObjectMonitor to a per-thread om_in_use_list.
346 // Also updates the per-thread om_in_use_count.
347 static void prepend_to_om_in_use_list(Thread* self, ObjectMonitor* m) {
348 prepend_to_common(m, &self->om_in_use_list, &self->om_in_use_count);
349 }
350
351 // Take an ObjectMonitor from the start of the specified list. Also
352 // decrements the specified counter. Returns NULL if none are available.
353 static ObjectMonitor* take_from_start_of_common(ObjectMonitor* volatile * list_p,
354 int volatile * count_p) {
355 ObjectMonitor* next = NULL;
356 ObjectMonitor* take = NULL;
357 // Mark the list head to guard against A-B-A race:
358 if (!mark_list_head(list_p, &take, &next)) {
359 return NULL; // None are available.
360 }
361 // Switch marked list head to next (which unmarks the list head, but
362 // leaves take marked):
363 OrderAccess::release_store(list_p, next);
364 Atomic::dec(count_p);
365 // Unmark take, but leave the next value for any lagging list
366 // walkers. It will get cleaned up when take is prepended to
367 // the in-use list:
368 set_next(take, next);
369 return take;
370 }
371
372 // Take an ObjectMonitor from the start of the global free-list. Also
373 // updates g_om_free_count. Returns NULL if none are available.
374 static ObjectMonitor* take_from_start_of_g_free_list() {
375 return take_from_start_of_common(&g_free_list, &g_om_free_count);
376 }
377
378 // Take an ObjectMonitor from the start of a per-thread free-list.
379 // Also updates om_free_count. Returns NULL if none are available.
380 static ObjectMonitor* take_from_start_of_om_free_list(Thread* self) {
381 return take_from_start_of_common(&self->om_free_list, &self->om_free_count);
382 }
383
384
385 // =====================> Quick functions
386
387 // The quick_* forms are special fast-path variants used to improve
388 // performance. In the simplest case, a "quick_*" implementation could
389 // simply return false, in which case the caller will perform the necessary
390 // state transitions and call the slow-path form.
391 // The fast-path is designed to handle frequently arising cases in an efficient
392 // manner and is just a degenerate "optimistic" variant of the slow-path.
393 // returns true -- to indicate the call was satisfied.
394 // returns false -- to indicate the call needs the services of the slow-path.
395 // A no-loitering ordinance is in effect for code in the quick_* family
396 // operators: safepoints or indefinite blocking (blocking that might span a
397 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
398 // entry.
399 //
400 // Consider: An interesting optimization is to have the JIT recognize the
401 // following common idiom:
402 // synchronized (someobj) { .... ; notify(); }
403 // That is, we find a notify() or notifyAll() call that immediately precedes
404 // the monitorexit operation. In that case the JIT could fuse the operations
405 // into a single notifyAndExit() runtime primitive.
406
407 bool ObjectSynchronizer::quick_notify(oopDesc* obj, Thread* self, bool all) {
408 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
409 assert(self->is_Java_thread(), "invariant");
410 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
411 NoSafepointVerifier nsv;
412 if (obj == NULL) return false; // slow-path for invalid obj
413 const markWord mark = obj->mark();
414
415 if (mark.has_locker() && self->is_lock_owned((address)mark.locker())) {
416 // Degenerate notify
417 // stack-locked by caller so by definition the implied waitset is empty.
418 return true;
419 }
420
421 if (mark.has_monitor()) {
422 ObjectMonitor* const mon = mark.monitor();
423 assert(mon->object() == obj, "invariant");
424 if (mon->owner() != self) return false; // slow-path for IMS exception
425
426 if (mon->first_waiter() != NULL) {
427 // We have one or more waiters. Since this is an inflated monitor
428 // that we own, we can transfer one or more threads from the waitset
429 // to the entrylist here and now, avoiding the slow-path.
430 if (all) {
431 DTRACE_MONITOR_PROBE(notifyAll, mon, obj, self);
432 } else {
433 DTRACE_MONITOR_PROBE(notify, mon, obj, self);
434 }
435 int free_count = 0;
436 do {
437 mon->INotify(self);
438 ++free_count;
439 } while (mon->first_waiter() != NULL && all);
440 OM_PERFDATA_OP(Notifications, inc(free_count));
441 }
442 return true;
443 }
444
445 // biased locking and any other IMS exception states take the slow-path
446 return false;
447 }
448
449
450 // The LockNode emitted directly at the synchronization site would have
451 // been too big if it were to have included support for the cases of inflated
452 // recursive enter and exit, so they go here instead.
453 // Note that we can't safely call AsyncPrintJavaStack() from within
454 // quick_enter() as our thread state remains _in_Java.
455
456 bool ObjectSynchronizer::quick_enter(oop obj, Thread* self,
457 BasicLock * lock) {
458 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
459 assert(self->is_Java_thread(), "invariant");
460 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
461 NoSafepointVerifier nsv;
462 if (obj == NULL) return false; // Need to throw NPE
463
464 while (true) {
465 const markWord mark = obj->mark();
466
467 if (mark.has_monitor()) {
468 ObjectMonitorHandle omh;
469 if (!omh.save_om_ptr(obj, mark)) {
470 // Lost a race with async deflation so try again.
471 assert(AsyncDeflateIdleMonitors, "sanity check");
472 continue;
473 }
474 ObjectMonitor* const m = omh.om_ptr();
475 assert(m->object() == obj, "invariant");
476 Thread* const owner = (Thread *) m->_owner;
477
478 // Lock contention and Transactional Lock Elision (TLE) diagnostics
479 // and observability
480 // Case: light contention possibly amenable to TLE
481 // Case: TLE inimical operations such as nested/recursive synchronization
482
483 if (owner == self) {
484 m->_recursions++;
485 return true;
486 }
487
488 // This Java Monitor is inflated so obj's header will never be
489 // displaced to this thread's BasicLock. Make the displaced header
490 // non-NULL so this BasicLock is not seen as recursive nor as
491 // being locked. We do this unconditionally so that this thread's
492 // BasicLock cannot be mis-interpreted by any stack walkers. For
493 // performance reasons, stack walkers generally first check for
494 // Biased Locking in the object's header, the second check is for
495 // stack-locking in the object's header, the third check is for
496 // recursive stack-locking in the displaced header in the BasicLock,
497 // and last are the inflated Java Monitor (ObjectMonitor) checks.
498 lock->set_displaced_header(markWord::unused_mark());
499
500 if (owner == NULL && m->try_set_owner_from(self, NULL) == NULL) {
501 assert(m->_recursions == 0, "invariant");
502 return true;
503 }
504
505 if (AsyncDeflateIdleMonitors &&
506 m->try_set_owner_from(self, DEFLATER_MARKER) == DEFLATER_MARKER) {
507 // The deflation protocol finished the first part (setting owner),
508 // but it failed the second part (making ref_count negative) and
509 // bailed. Or the ObjectMonitor was async deflated and reused.
510 // Acquired the monitor.
511 assert(m->_recursions == 0, "invariant");
512 return true;
513 }
514 }
515 break;
516 }
517
518 // Note that we could inflate in quick_enter.
519 // This is likely a useful optimization
520 // Critically, in quick_enter() we must not:
521 // -- perform bias revocation, or
522 // -- block indefinitely, or
523 // -- reach a safepoint
524
525 return false; // revert to slow-path
526 }
527
528 // -----------------------------------------------------------------------------
529 // Monitor Enter/Exit
530 // The interpreter and compiler assembly code tries to lock using the fast path
531 // of this algorithm. Make sure to update that code if the following function is
532 // changed. The implementation is extremely sensitive to race condition. Be careful.
533
534 void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, TRAPS) {
535 if (UseBiasedLocking) {
536 if (!SafepointSynchronize::is_at_safepoint()) {
537 BiasedLocking::revoke(obj, THREAD);
538 } else {
539 BiasedLocking::revoke_at_safepoint(obj);
540 }
541 }
542
543 markWord mark = obj->mark();
544 assert(!mark.has_bias_pattern(), "should not see bias pattern here");
545
546 if (mark.is_neutral()) {
547 // Anticipate successful CAS -- the ST of the displaced mark must
548 // be visible <= the ST performed by the CAS.
549 lock->set_displaced_header(mark);
550 if (mark == obj()->cas_set_mark(markWord::from_pointer(lock), mark)) {
551 return;
552 }
553 // Fall through to inflate() ...
554 } else if (mark.has_locker() &&
555 THREAD->is_lock_owned((address)mark.locker())) {
556 assert(lock != mark.locker(), "must not re-lock the same lock");
557 assert(lock != (BasicLock*)obj->mark().value(), "don't relock with same BasicLock");
558 lock->set_displaced_header(markWord::from_pointer(NULL));
559 return;
560 }
561
562 // The object header will never be displaced to this lock,
563 // so it does not matter what the value is, except that it
564 // must be non-zero to avoid looking like a re-entrant lock,
565 // and must not look locked either.
566 lock->set_displaced_header(markWord::unused_mark());
567 ObjectMonitorHandle omh;
568 inflate(&omh, THREAD, obj(), inflate_cause_monitor_enter);
569 omh.om_ptr()->enter(THREAD);
570 }
571
572 void ObjectSynchronizer::exit(oop object, BasicLock* lock, TRAPS) {
573 markWord mark = object->mark();
574 // We cannot check for Biased Locking if we are racing an inflation.
575 assert(mark == markWord::INFLATING() ||
576 !mark.has_bias_pattern(), "should not see bias pattern here");
577
578 markWord dhw = lock->displaced_header();
579 if (dhw.value() == 0) {
580 // If the displaced header is NULL, then this exit matches up with
581 // a recursive enter. No real work to do here except for diagnostics.
582 #ifndef PRODUCT
583 if (mark != markWord::INFLATING()) {
584 // Only do diagnostics if we are not racing an inflation. Simply
585 // exiting a recursive enter of a Java Monitor that is being
586 // inflated is safe; see the has_monitor() comment below.
587 assert(!mark.is_neutral(), "invariant");
588 assert(!mark.has_locker() ||
589 THREAD->is_lock_owned((address)mark.locker()), "invariant");
590 if (mark.has_monitor()) {
591 // The BasicLock's displaced_header is marked as a recursive
592 // enter and we have an inflated Java Monitor (ObjectMonitor).
593 // This is a special case where the Java Monitor was inflated
594 // after this thread entered the stack-lock recursively. When a
595 // Java Monitor is inflated, we cannot safely walk the Java
596 // Monitor owner's stack and update the BasicLocks because a
597 // Java Monitor can be asynchronously inflated by a thread that
598 // does not own the Java Monitor.
599 ObjectMonitor* m = mark.monitor();
600 assert(((oop)(m->object()))->mark() == mark, "invariant");
601 assert(m->is_entered(THREAD), "invariant");
602 }
603 }
604 #endif
605 return;
606 }
607
608 if (mark == markWord::from_pointer(lock)) {
609 // If the object is stack-locked by the current thread, try to
610 // swing the displaced header from the BasicLock back to the mark.
611 assert(dhw.is_neutral(), "invariant");
612 if (object->cas_set_mark(dhw, mark) == mark) {
613 return;
614 }
615 }
616
617 // We have to take the slow-path of possible inflation and then exit.
618 ObjectMonitorHandle omh;
619 inflate(&omh, THREAD, object, inflate_cause_vm_internal);
620 omh.om_ptr()->exit(true, THREAD);
621 }
622
623 // -----------------------------------------------------------------------------
624 // Class Loader support to workaround deadlocks on the class loader lock objects
625 // Also used by GC
626 // complete_exit()/reenter() are used to wait on a nested lock
627 // i.e. to give up an outer lock completely and then re-enter
628 // Used when holding nested locks - lock acquisition order: lock1 then lock2
629 // 1) complete_exit lock1 - saving recursion count
630 // 2) wait on lock2
631 // 3) when notified on lock2, unlock lock2
632 // 4) reenter lock1 with original recursion count
633 // 5) lock lock2
634 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
635 intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
636 if (UseBiasedLocking) {
637 BiasedLocking::revoke(obj, THREAD);
638 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
639 }
640
641 ObjectMonitorHandle omh;
642 inflate(&omh, THREAD, obj(), inflate_cause_vm_internal);
643 intptr_t ret_code = omh.om_ptr()->complete_exit(THREAD);
644 return ret_code;
645 }
646
647 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
648 void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) {
649 if (UseBiasedLocking) {
650 BiasedLocking::revoke(obj, THREAD);
651 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
652 }
653
654 ObjectMonitorHandle omh;
655 inflate(&omh, THREAD, obj(), inflate_cause_vm_internal);
656 omh.om_ptr()->reenter(recursion, THREAD);
657 }
658 // -----------------------------------------------------------------------------
659 // JNI locks on java objects
660 // NOTE: must use heavy weight monitor to handle jni monitor enter
661 void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) {
662 // the current locking is from JNI instead of Java code
663 if (UseBiasedLocking) {
664 BiasedLocking::revoke(obj, THREAD);
665 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
666 }
667 THREAD->set_current_pending_monitor_is_from_java(false);
668 ObjectMonitorHandle omh;
669 inflate(&omh, THREAD, obj(), inflate_cause_jni_enter);
670 omh.om_ptr()->enter(THREAD);
671 THREAD->set_current_pending_monitor_is_from_java(true);
672 }
673
674 // NOTE: must use heavy weight monitor to handle jni monitor exit
675 void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
676 if (UseBiasedLocking) {
677 Handle h_obj(THREAD, obj);
678 BiasedLocking::revoke(h_obj, THREAD);
679 obj = h_obj();
680 }
681 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
682
683 ObjectMonitorHandle omh;
684 inflate(&omh, THREAD, obj, inflate_cause_jni_exit);
685 ObjectMonitor* monitor = omh.om_ptr();
686 // If this thread has locked the object, exit the monitor. We
687 // intentionally do not use CHECK here because we must exit the
688 // monitor even if an exception is pending.
689 if (monitor->check_owner(THREAD)) {
690 monitor->exit(true, THREAD);
691 }
692 }
693
694 // -----------------------------------------------------------------------------
695 // Internal VM locks on java objects
696 // standard constructor, allows locking failures
697 ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool do_lock) {
698 _dolock = do_lock;
699 _thread = thread;
700 _thread->check_for_valid_safepoint_state();
701 _obj = obj;
702
703 if (_dolock) {
704 ObjectSynchronizer::enter(_obj, &_lock, _thread);
705 }
706 }
707
708 ObjectLocker::~ObjectLocker() {
709 if (_dolock) {
710 ObjectSynchronizer::exit(_obj(), &_lock, _thread);
711 }
712 }
713
714
715 // -----------------------------------------------------------------------------
716 // Wait/Notify/NotifyAll
717 // NOTE: must use heavy weight monitor to handle wait()
718 int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
719 if (UseBiasedLocking) {
720 BiasedLocking::revoke(obj, THREAD);
721 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
722 }
723 if (millis < 0) {
724 THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
725 }
726 ObjectMonitorHandle omh;
727 inflate(&omh, THREAD, obj(), inflate_cause_wait);
728 ObjectMonitor* monitor = omh.om_ptr();
729
730 DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);
731 monitor->wait(millis, true, THREAD);
732
733 // This dummy call is in place to get around dtrace bug 6254741. Once
734 // that's fixed we can uncomment the following line, remove the call
735 // and change this function back into a "void" func.
736 // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
737 int ret_code = dtrace_waited_probe(monitor, obj, THREAD);
738 return ret_code;
739 }
740
741 void ObjectSynchronizer::wait_uninterruptibly(Handle obj, jlong millis, TRAPS) {
742 if (UseBiasedLocking) {
743 BiasedLocking::revoke(obj, THREAD);
744 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
745 }
746 if (millis < 0) {
747 THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
748 }
749 ObjectMonitorHandle omh;
750 inflate(&omh, THREAD, obj(), inflate_cause_wait);
751 omh.om_ptr()->wait(millis, false, THREAD);
752 }
753
754 void ObjectSynchronizer::notify(Handle obj, TRAPS) {
755 if (UseBiasedLocking) {
756 BiasedLocking::revoke(obj, THREAD);
757 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
758 }
759
760 markWord mark = obj->mark();
761 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
762 return;
763 }
764 ObjectMonitorHandle omh;
765 inflate(&omh, THREAD, obj(), inflate_cause_notify);
766 omh.om_ptr()->notify(THREAD);
767 }
768
769 // NOTE: see comment of notify()
770 void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
771 if (UseBiasedLocking) {
772 BiasedLocking::revoke(obj, THREAD);
773 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
774 }
775
776 markWord mark = obj->mark();
777 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
778 return;
779 }
780 ObjectMonitorHandle omh;
781 inflate(&omh, THREAD, obj(), inflate_cause_notify);
782 omh.om_ptr()->notifyAll(THREAD);
783 }
784
785 // -----------------------------------------------------------------------------
786 // Hash Code handling
787 //
788 // Performance concern:
789 // OrderAccess::storestore() calls release() which at one time stored 0
790 // into the global volatile OrderAccess::dummy variable. This store was
791 // unnecessary for correctness. Many threads storing into a common location
792 // causes considerable cache migration or "sloshing" on large SMP systems.
793 // As such, I avoided using OrderAccess::storestore(). In some cases
794 // OrderAccess::fence() -- which incurs local latency on the executing
795 // processor -- is a better choice as it scales on SMP systems.
796 //
797 // See http://blogs.oracle.com/dave/entry/biased_locking_in_hotspot for
798 // a discussion of coherency costs. Note that all our current reference
799 // platforms provide strong ST-ST order, so the issue is moot on IA32,
800 // x64, and SPARC.
801 //
802 // As a general policy we use "volatile" to control compiler-based reordering
803 // and explicit fences (barriers) to control for architectural reordering
804 // performed by the CPU(s) or platform.
805
806 struct SharedGlobals {
807 char _pad_prefix[OM_CACHE_LINE_SIZE];
808 // These are highly shared mostly-read variables.
809 // To avoid false-sharing they need to be the sole occupants of a cache line.
810 volatile int stw_random;
811 volatile int stw_cycle;
812 DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(volatile int) * 2);
813 // Hot RW variable -- Sequester to avoid false-sharing
814 volatile int hc_sequence;
815 DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile int));
816 };
817
818 static SharedGlobals GVars;
819 static int MonitorScavengeThreshold = 1000000;
820 static volatile int ForceMonitorScavenge = 0; // Scavenge required and pending
821
822 static markWord read_stable_mark(oop obj) {
823 markWord mark = obj->mark();
824 if (!mark.is_being_inflated()) {
825 return mark; // normal fast-path return
826 }
827
828 int its = 0;
829 for (;;) {
830 markWord mark = obj->mark();
831 if (!mark.is_being_inflated()) {
832 return mark; // normal fast-path return
833 }
834
835 // The object is being inflated by some other thread.
836 // The caller of read_stable_mark() must wait for inflation to complete.
837 // Avoid live-lock
838 // TODO: consider calling SafepointSynchronize::do_call_back() while
839 // spinning to see if there's a safepoint pending. If so, immediately
840 // yielding or blocking would be appropriate. Avoid spinning while
841 // there is a safepoint pending.
842 // TODO: add inflation contention performance counters.
843 // TODO: restrict the aggregate number of spinners.
844
845 ++its;
846 if (its > 10000 || !os::is_MP()) {
847 if (its & 1) {
848 os::naked_yield();
849 } else {
850 // Note that the following code attenuates the livelock problem but is not
851 // a complete remedy. A more complete solution would require that the inflating
852 // thread hold the associated inflation lock. The following code simply restricts
853 // the number of spinners to at most one. We'll have N-2 threads blocked
854 // on the inflationlock, 1 thread holding the inflation lock and using
855 // a yield/park strategy, and 1 thread in the midst of inflation.
856 // A more refined approach would be to change the encoding of INFLATING
857 // to allow encapsulation of a native thread pointer. Threads waiting for
858 // inflation to complete would use CAS to push themselves onto a singly linked
859 // list rooted at the markword. Once enqueued, they'd loop, checking a per-thread flag
860 // and calling park(). When inflation was complete the thread that accomplished inflation
861 // would detach the list and set the markword to inflated with a single CAS and
862 // then for each thread on the list, set the flag and unpark() the thread.
863 // This is conceptually similar to muxAcquire-muxRelease, except that muxRelease
864 // wakes at most one thread whereas we need to wake the entire list.
865 int ix = (cast_from_oop<intptr_t>(obj) >> 5) & (NINFLATIONLOCKS-1);
866 int YieldThenBlock = 0;
867 assert(ix >= 0 && ix < NINFLATIONLOCKS, "invariant");
868 assert((NINFLATIONLOCKS & (NINFLATIONLOCKS-1)) == 0, "invariant");
869 Thread::muxAcquire(gInflationLocks + ix, "gInflationLock");
870 while (obj->mark() == markWord::INFLATING()) {
871 // Beware: NakedYield() is advisory and has almost no effect on some platforms
872 // so we periodically call self->_ParkEvent->park(1).
873 // We use a mixed spin/yield/block mechanism.
874 if ((YieldThenBlock++) >= 16) {
875 Thread::current()->_ParkEvent->park(1);
876 } else {
877 os::naked_yield();
878 }
879 }
880 Thread::muxRelease(gInflationLocks + ix);
881 }
882 } else {
883 SpinPause(); // SMP-polite spinning
884 }
885 }
886 }
887
888 // hashCode() generation :
889 //
890 // Possibilities:
891 // * MD5Digest of {obj,stw_random}
892 // * CRC32 of {obj,stw_random} or any linear-feedback shift register function.
893 // * A DES- or AES-style SBox[] mechanism
894 // * One of the Phi-based schemes, such as:
895 // 2654435761 = 2^32 * Phi (golden ratio)
896 // HashCodeValue = ((uintptr_t(obj) >> 3) * 2654435761) ^ GVars.stw_random ;
897 // * A variation of Marsaglia's shift-xor RNG scheme.
898 // * (obj ^ stw_random) is appealing, but can result
899 // in undesirable regularity in the hashCode values of adjacent objects
900 // (objects allocated back-to-back, in particular). This could potentially
901 // result in hashtable collisions and reduced hashtable efficiency.
902 // There are simple ways to "diffuse" the middle address bits over the
903 // generated hashCode values:
904
905 static inline intptr_t get_next_hash(Thread* self, oop obj) {
906 intptr_t value = 0;
907 if (hashCode == 0) {
908 // This form uses global Park-Miller RNG.
909 // On MP system we'll have lots of RW access to a global, so the
910 // mechanism induces lots of coherency traffic.
911 value = os::random();
912 } else if (hashCode == 1) {
913 // This variation has the property of being stable (idempotent)
914 // between STW operations. This can be useful in some of the 1-0
915 // synchronization schemes.
916 intptr_t addr_bits = cast_from_oop<intptr_t>(obj) >> 3;
917 value = addr_bits ^ (addr_bits >> 5) ^ GVars.stw_random;
918 } else if (hashCode == 2) {
919 value = 1; // for sensitivity testing
920 } else if (hashCode == 3) {
921 value = ++GVars.hc_sequence;
922 } else if (hashCode == 4) {
923 value = cast_from_oop<intptr_t>(obj);
924 } else {
925 // Marsaglia's xor-shift scheme with thread-specific state
926 // This is probably the best overall implementation -- we'll
927 // likely make this the default in future releases.
928 unsigned t = self->_hashStateX;
929 t ^= (t << 11);
930 self->_hashStateX = self->_hashStateY;
931 self->_hashStateY = self->_hashStateZ;
932 self->_hashStateZ = self->_hashStateW;
933 unsigned v = self->_hashStateW;
934 v = (v ^ (v >> 19)) ^ (t ^ (t >> 8));
935 self->_hashStateW = v;
936 value = v;
937 }
938
939 value &= markWord::hash_mask;
940 if (value == 0) value = 0xBAD;
941 assert(value != markWord::no_hash, "invariant");
942 return value;
943 }
944
945 intptr_t ObjectSynchronizer::FastHashCode(Thread* self, oop obj) {
946 if (UseBiasedLocking) {
947 // NOTE: many places throughout the JVM do not expect a safepoint
948 // to be taken here, in particular most operations on perm gen
949 // objects. However, we only ever bias Java instances and all of
950 // the call sites of identity_hash that might revoke biases have
951 // been checked to make sure they can handle a safepoint. The
952 // added check of the bias pattern is to avoid useless calls to
953 // thread-local storage.
954 if (obj->mark().has_bias_pattern()) {
955 // Handle for oop obj in case of STW safepoint
956 Handle hobj(self, obj);
957 // Relaxing assertion for bug 6320749.
958 assert(Universe::verify_in_progress() ||
959 !SafepointSynchronize::is_at_safepoint(),
960 "biases should not be seen by VM thread here");
961 BiasedLocking::revoke(hobj, JavaThread::current());
962 obj = hobj();
963 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
964 }
965 }
966
967 // hashCode() is a heap mutator ...
968 // Relaxing assertion for bug 6320749.
969 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
970 !SafepointSynchronize::is_at_safepoint(), "invariant");
971 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
972 self->is_Java_thread() , "invariant");
973 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
974 ((JavaThread *)self)->thread_state() != _thread_blocked, "invariant");
975
976 while (true) {
977 ObjectMonitor* monitor = NULL;
978 markWord temp, test;
979 intptr_t hash;
980 markWord mark = read_stable_mark(obj);
981
982 // object should remain ineligible for biased locking
983 assert(!mark.has_bias_pattern(), "invariant");
984
985 if (mark.is_neutral()) {
986 hash = mark.hash(); // this is a normal header
987 if (hash != 0) { // if it has hash, just return it
988 return hash;
989 }
990 hash = get_next_hash(self, obj); // allocate a new hash code
991 temp = mark.copy_set_hash(hash); // merge the hash code into header
992 // use (machine word version) atomic operation to install the hash
993 test = obj->cas_set_mark(temp, mark);
994 if (test == mark) {
995 return hash;
996 }
997 // If atomic operation failed, we must inflate the header
998 // into heavy weight monitor. We could add more code here
999 // for fast path, but it does not worth the complexity.
1000 } else if (mark.has_monitor()) {
1001 ObjectMonitorHandle omh;
1002 if (!omh.save_om_ptr(obj, mark)) {
1003 // Lost a race with async deflation so try again.
1004 assert(AsyncDeflateIdleMonitors, "sanity check");
1005 continue;
1006 }
1007 monitor = omh.om_ptr();
1008 temp = monitor->header();
1009 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1010 hash = temp.hash();
1011 if (hash != 0) {
1012 return hash;
1013 }
1014 // Skip to the following code to reduce code size
1015 } else if (self->is_lock_owned((address)mark.locker())) {
1016 temp = mark.displaced_mark_helper(); // this is a lightweight monitor owned
1017 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1018 hash = temp.hash(); // by current thread, check if the displaced
1019 if (hash != 0) { // header contains hash code
1020 return hash;
1021 }
1022 // WARNING:
1023 // The displaced header in the BasicLock on a thread's stack
1024 // is strictly immutable. It CANNOT be changed in ANY cases.
1025 // So we have to inflate the stack lock into an ObjectMonitor
1026 // even if the current thread owns the lock. The BasicLock on
1027 // a thread's stack can be asynchronously read by other threads
1028 // during an inflate() call so any change to that stack memory
1029 // may not propagate to other threads correctly.
1030 }
1031
1032 // Inflate the monitor to set hash code
1033 ObjectMonitorHandle omh;
1034 inflate(&omh, self, obj, inflate_cause_hash_code);
1035 monitor = omh.om_ptr();
1036 // Load displaced header and check it has hash code
1037 mark = monitor->header();
1038 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1039 hash = mark.hash();
1040 if (hash == 0) {
1041 hash = get_next_hash(self, obj);
1042 temp = mark.copy_set_hash(hash); // merge hash code into header
1043 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1044 uintptr_t v = Atomic::cmpxchg(temp.value(), (volatile uintptr_t*)monitor->header_addr(), mark.value());
1045 test = markWord(v);
1046 if (test != mark) {
1047 // The only non-deflation update to the ObjectMonitor's
1048 // header/dmw field is to merge in the hash code. If someone
1049 // adds a new usage of the header/dmw field, please update
1050 // this code.
1051 // ObjectMonitor::install_displaced_markword_in_object()
1052 // does mark the header/dmw field as part of async deflation,
1053 // but that protocol cannot happen now due to the
1054 // ObjectMonitorHandle above.
1055 hash = test.hash();
1056 assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value());
1057 assert(hash != 0, "Trivial unexpected object/monitor header usage.");
1058 }
1059 }
1060 // We finally get the hash
1061 return hash;
1062 }
1063 }
1064
1065 // Deprecated -- use FastHashCode() instead.
1066
1067 intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) {
1068 return FastHashCode(Thread::current(), obj());
1069 }
1070
1071
1072 bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
1073 Handle h_obj) {
1074 if (UseBiasedLocking) {
1075 BiasedLocking::revoke(h_obj, thread);
1076 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1077 }
1078
1079 assert(thread == JavaThread::current(), "Can only be called on current thread");
1080 oop obj = h_obj();
1081
1082 while (true) {
1083 markWord mark = read_stable_mark(obj);
1084
1085 // Uncontended case, header points to stack
1086 if (mark.has_locker()) {
1087 return thread->is_lock_owned((address)mark.locker());
1088 }
1089 // Contended case, header points to ObjectMonitor (tagged pointer)
1090 if (mark.has_monitor()) {
1091 ObjectMonitorHandle omh;
1092 if (!omh.save_om_ptr(obj, mark)) {
1093 // Lost a race with async deflation so try again.
1094 assert(AsyncDeflateIdleMonitors, "sanity check");
1095 continue;
1096 }
1097 bool ret_code = omh.om_ptr()->is_entered(thread) != 0;
1098 return ret_code;
1099 }
1100 // Unlocked case, header in place
1101 assert(mark.is_neutral(), "sanity check");
1102 return false;
1103 }
1104 }
1105
1106 // Be aware of this method could revoke bias of the lock object.
1107 // This method queries the ownership of the lock handle specified by 'h_obj'.
1108 // If the current thread owns the lock, it returns owner_self. If no
1109 // thread owns the lock, it returns owner_none. Otherwise, it will return
1110 // owner_other.
1111 ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership
1112 (JavaThread *self, Handle h_obj) {
1113 // The caller must beware this method can revoke bias, and
1114 // revocation can result in a safepoint.
1115 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
1116 assert(self->thread_state() != _thread_blocked, "invariant");
1117
1118 // Possible mark states: neutral, biased, stack-locked, inflated
1119
1120 if (UseBiasedLocking && h_obj()->mark().has_bias_pattern()) {
1121 // CASE: biased
1122 BiasedLocking::revoke(h_obj, self);
1123 assert(!h_obj->mark().has_bias_pattern(),
1124 "biases should be revoked by now");
1125 }
1126
1127 assert(self == JavaThread::current(), "Can only be called on current thread");
1128 oop obj = h_obj();
1129
1130 while (true) {
1131 markWord mark = read_stable_mark(obj);
1132
1133 // CASE: stack-locked. Mark points to a BasicLock on the owner's stack.
1134 if (mark.has_locker()) {
1135 return self->is_lock_owned((address)mark.locker()) ?
1136 owner_self : owner_other;
1137 }
1138
1139 // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor.
1140 // The Object:ObjectMonitor relationship is stable as long as we're
1141 // not at a safepoint and AsyncDeflateIdleMonitors is false.
1142 if (mark.has_monitor()) {
1143 ObjectMonitorHandle omh;
1144 if (!omh.save_om_ptr(obj, mark)) {
1145 // Lost a race with async deflation so try again.
1146 assert(AsyncDeflateIdleMonitors, "sanity check");
1147 continue;
1148 }
1149 ObjectMonitor* monitor = omh.om_ptr();
1150 void* owner = monitor->_owner;
1151 if (owner == NULL) return owner_none;
1152 return (owner == self ||
1153 self->is_lock_owned((address)owner)) ? owner_self : owner_other;
1154 }
1155
1156 // CASE: neutral
1157 assert(mark.is_neutral(), "sanity check");
1158 return owner_none; // it's unlocked
1159 }
1160 }
1161
1162 // FIXME: jvmti should call this
1163 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
1164 if (UseBiasedLocking) {
1165 if (SafepointSynchronize::is_at_safepoint()) {
1166 BiasedLocking::revoke_at_safepoint(h_obj);
1167 } else {
1168 BiasedLocking::revoke(h_obj, JavaThread::current());
1169 }
1170 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1171 }
1172
1173 oop obj = h_obj();
1174
1175 while (true) {
1176 address owner = NULL;
1177 markWord mark = read_stable_mark(obj);
1178
1179 // Uncontended case, header points to stack
1180 if (mark.has_locker()) {
1181 owner = (address) mark.locker();
1182 }
1183
1184 // Contended case, header points to ObjectMonitor (tagged pointer)
1185 else if (mark.has_monitor()) {
1186 ObjectMonitorHandle omh;
1187 if (!omh.save_om_ptr(obj, mark)) {
1188 // Lost a race with async deflation so try again.
1189 assert(AsyncDeflateIdleMonitors, "sanity check");
1190 continue;
1191 }
1192 ObjectMonitor* monitor = omh.om_ptr();
1193 assert(monitor != NULL, "monitor should be non-null");
1194 owner = (address) monitor->owner();
1195 }
1196
1197 if (owner != NULL) {
1198 // owning_thread_from_monitor_owner() may also return NULL here
1199 return Threads::owning_thread_from_monitor_owner(t_list, owner);
1200 }
1201
1202 // Unlocked case, header in place
1203 // Cannot have assertion since this object may have been
1204 // locked by another thread when reaching here.
1205 // assert(mark.is_neutral(), "sanity check");
1206
1207 return NULL;
1208 }
1209 }
1210
1211 // Visitors ...
1212
1213 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
1214 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
1215 while (block != NULL) {
1216 assert(block->object() == CHAINMARKER, "must be a block header");
1217 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
1218 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
1219 if (mid->is_active()) {
1220 ObjectMonitorHandle omh(mid);
1221
1222 if (mid->object() == NULL ||
1223 (AsyncDeflateIdleMonitors && mid->ref_count() < 0)) {
1224 // Only process with closure if the object is set.
1225 // For async deflation, race here if monitor is not owned!
1226 // The above ref_count bump (in ObjectMonitorHandle ctr)
1227 // will cause subsequent async deflation to skip it.
1228 // However, previous or concurrent async deflation is a race
1229 // so skip this ObjectMonitor if it is being async deflated.
1230 continue;
1231 }
1232 closure->do_monitor(mid);
1233 }
1234 }
1235 // unmarked_next() is not needed with g_block_list (no next field marking).
1236 block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om);
1237 }
1238 }
1239
1240 static bool monitors_used_above_threshold() {
1241 if (OrderAccess::load_acquire(&g_om_population) == 0) {
1242 return false;
1243 }
1244 if (MonitorUsedDeflationThreshold > 0) {
1245 int monitors_used = OrderAccess::load_acquire(&g_om_population) -
1246 OrderAccess::load_acquire(&g_om_free_count);
1247 if (HandshakeAfterDeflateIdleMonitors) {
1248 monitors_used -= OrderAccess::load_acquire(&g_om_wait_count);
1249 }
1250 int monitor_usage = (monitors_used * 100LL) /
1251 OrderAccess::load_acquire(&g_om_population);
1252 return monitor_usage > MonitorUsedDeflationThreshold;
1253 }
1254 return false;
1255 }
1256
1257 // Returns true if MonitorBound is set (> 0) and if the specified
1258 // cnt is > MonitorBound. Otherwise returns false.
1259 static bool is_MonitorBound_exceeded(const int cnt) {
1260 const int mx = MonitorBound;
1261 return mx > 0 && cnt > mx;
1262 }
1263
1264 bool ObjectSynchronizer::is_async_deflation_needed() {
1265 if (!AsyncDeflateIdleMonitors) {
1266 return false;
1267 }
1268 if (is_async_deflation_requested()) {
1269 // Async deflation request.
1270 return true;
1271 }
1272 if (AsyncDeflationInterval > 0 &&
1273 time_since_last_async_deflation_ms() > AsyncDeflationInterval &&
1274 monitors_used_above_threshold()) {
1275 // It's been longer than our specified deflate interval and there
1276 // are too many monitors in use. We don't deflate more frequently
1277 // than AsyncDeflationInterval (unless is_async_deflation_requested)
1278 // in order to not swamp the ServiceThread.
1279 _last_async_deflation_time_ns = os::javaTimeNanos();
1280 return true;
1281 }
1282 int monitors_used = OrderAccess::load_acquire(&g_om_population) -
1283 OrderAccess::load_acquire(&g_om_free_count);
1284 if (HandshakeAfterDeflateIdleMonitors) {
1285 monitors_used -= OrderAccess::load_acquire(&g_om_wait_count);
1286 }
1287 if (is_MonitorBound_exceeded(monitors_used)) {
1288 // Not enough ObjectMonitors on the global free list.
1289 return true;
1290 }
1291 return false;
1292 }
1293
1294 bool ObjectSynchronizer::is_safepoint_deflation_needed() {
1295 if (!AsyncDeflateIdleMonitors) {
1296 if (monitors_used_above_threshold()) {
1297 // Too many monitors in use.
1298 return true;
1299 }
1300 return false;
1301 }
1302 if (is_special_deflation_requested()) {
1303 // For AsyncDeflateIdleMonitors only do a safepoint deflation
1304 // if there is a special deflation request.
1305 return true;
1306 }
1307 return false;
1308 }
1309
1310 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1311 return (os::javaTimeNanos() - _last_async_deflation_time_ns) / (NANOUNITS / MILLIUNITS);
1312 }
1313
1314 void ObjectSynchronizer::oops_do(OopClosure* f) {
1315 // We only scan the global used list here (for moribund threads), and
1316 // the thread-local monitors in Thread::oops_do().
1317 global_used_oops_do(f);
1318 }
1319
1320 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
1321 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1322 list_oops_do(OrderAccess::load_acquire(&g_om_in_use_list), OrderAccess::load_acquire(&g_om_in_use_count), f);
1323 }
1324
1325 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
1326 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1327 list_oops_do(OrderAccess::load_acquire(&thread->om_in_use_list), OrderAccess::load_acquire(&thread->om_in_use_count), f);
1328 }
1329
1330 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, int count, OopClosure* f) {
1331 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1332 // The oops_do() phase does not overlap with monitor deflation
1333 // so no need to update the ObjectMonitor's ref_count for this
1334 // ObjectMonitor* use.
1335 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
1336 if (mid->object() != NULL) {
1337 f->do_oop((oop*)mid->object_addr());
1338 }
1339 }
1340 }
1341
1342
1343 // -----------------------------------------------------------------------------
1344 // ObjectMonitor Lifecycle
1345 // -----------------------
1346 // Inflation unlinks monitors from the global g_free_list and
1347 // associates them with objects. Deflation -- which occurs at
1348 // STW-time -- disassociates idle monitors from objects. Such
1349 // scavenged monitors are returned to the g_free_list.
1350 //
1351 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
1352 //
1353 // Lifecycle:
1354 // -- unassigned and on the global free list
1355 // -- unassigned and on a thread's private om_free_list
1356 // -- assigned to an object. The object is inflated and the mark refers
1357 // to the objectmonitor.
1358
1359
1360 // Constraining monitor pool growth via MonitorBound ...
1361 //
1362 // If MonitorBound is not set (<= 0), MonitorBound checks are disabled.
1363 //
1364 // When safepoint deflation is being used (!AsyncDeflateIdleMonitors):
1365 // The monitor pool is grow-only. We scavenge at STW safepoint-time, but the
1366 // the rate of scavenging is driven primarily by GC. As such, we can find
1367 // an inordinate number of monitors in circulation.
1368 // To avoid that scenario we can artificially induce a STW safepoint
1369 // if the pool appears to be growing past some reasonable bound.
1370 // Generally we favor time in space-time tradeoffs, but as there's no
1371 // natural back-pressure on the # of extant monitors we need to impose some
1372 // type of limit. Beware that if MonitorBound is set to too low a value
1373 // we could just loop. In addition, if MonitorBound is set to a low value
1374 // we'll incur more safepoints, which are harmful to performance.
1375 // See also: GuaranteedSafepointInterval
1376 //
1377 // The current implementation uses asynchronous VM operations.
1378 //
1379 // When safepoint deflation is being used and MonitorBound is set, the
1380 // boundry applies to
1381 // (g_om_population - g_om_free_count)
1382 // i.e., if there are not enough ObjectMonitors on the global free list,
1383 // then a safepoint deflation is induced. Picking a good MonitorBound value
1384 // is non-trivial.
1385 //
1386 // When async deflation is being used:
1387 // The monitor pool is still grow-only. Async deflation is requested
1388 // by a safepoint's cleanup phase or by the ServiceThread at periodic
1389 // intervals when is_async_deflation_needed() returns true. In
1390 // addition to other policies that are checked, if there are not
1391 // enough ObjectMonitors on the global free list, then
1392 // is_async_deflation_needed() will return true. The ServiceThread
1393 // calls deflate_global_idle_monitors_using_JT() and also calls
1394 // deflate_per_thread_idle_monitors_using_JT() as needed.
1395
1396 static void InduceScavenge(Thread* self, const char * Whence) {
1397 assert(!AsyncDeflateIdleMonitors, "is not used by async deflation");
1398
1399 // Induce STW safepoint to trim monitors
1400 // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
1401 // More precisely, trigger an asynchronous STW safepoint as the number
1402 // of active monitors passes the specified threshold.
1403 // TODO: assert thread state is reasonable
1404
1405 if (ForceMonitorScavenge == 0 && Atomic::xchg (1, &ForceMonitorScavenge) == 0) {
1406 // Induce a 'null' safepoint to scavenge monitors
1407 // Must VM_Operation instance be heap allocated as the op will be enqueue and posted
1408 // to the VMthread and have a lifespan longer than that of this activation record.
1409 // The VMThread will delete the op when completed.
1410 VMThread::execute(new VM_ScavengeMonitors());
1411 }
1412 }
1413
1414 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self,
1415 const InflateCause cause) {
1416 // A large MAXPRIVATE value reduces both list lock contention
1417 // and list coherency traffic, but also tends to increase the
1418 // number of ObjectMonitors in circulation as well as the STW
1419 // scavenge costs. As usual, we lean toward time in space-time
1420 // tradeoffs.
1421 const int MAXPRIVATE = 1024;
1422
1423 stringStream ss;
1424 for (;;) {
1425 ObjectMonitor* m;
1426
1427 // 1: try to allocate from the thread's local om_free_list.
1428 // Threads will attempt to allocate first from their local list, then
1429 // from the global list, and only after those attempts fail will the
1430 // thread attempt to instantiate new monitors. Thread-local free lists
1431 // improve allocation latency, as well as reducing coherency traffic
1432 // on the shared global list.
1433 m = take_from_start_of_om_free_list(self);
1434 if (m != NULL) {
1435 guarantee(m->object() == NULL, "invariant");
1436 m->set_allocation_state(ObjectMonitor::New);
1437 prepend_to_om_in_use_list(self, m);
1438 return m;
1439 }
1440
1441 // 2: try to allocate from the global g_free_list
1442 // CONSIDER: use muxTry() instead of muxAcquire().
1443 // If the muxTry() fails then drop immediately into case 3.
1444 // If we're using thread-local free lists then try
1445 // to reprovision the caller's free list.
1446 if (OrderAccess::load_acquire(&g_free_list) != NULL) {
1447 // Reprovision the thread's om_free_list.
1448 // Use bulk transfers to reduce the allocation rate and heat
1449 // on various locks.
1450 for (int i = self->om_free_provision; --i >= 0;) {
1451 ObjectMonitor* take = take_from_start_of_g_free_list();
1452 if (take == NULL) {
1453 break; // No more are available.
1454 }
1455 guarantee(take->object() == NULL, "invariant");
1456 if (AsyncDeflateIdleMonitors) {
1457 // We allowed 3 field values to linger during async deflation.
1458 // We clear header and restore ref_count here, but we leave
1459 // owner == DEFLATER_MARKER so the simple C2 ObjectMonitor
1460 // enter optimization can no longer race with async deflation
1461 // and reuse.
1462 take->set_header(markWord::zero());
1463 if (take->ref_count() < 0) {
1464 // Add back max_jint to restore the ref_count field to its
1465 // proper value.
1466 Atomic::add(max_jint, &take->_ref_count);
1467
1468 assert(take->ref_count() >= 0, "must not be negative: ref_count=%d",
1469 take->ref_count());
1470 }
1471 }
1472 take->Recycle();
1473 // Since we're taking from the global free-list, take must be Free.
1474 // om_release() also sets the allocation state to Free because it
1475 // is called from other code paths.
1476 assert(take->is_free(), "invariant");
1477 om_release(self, take, false);
1478 }
1479 self->om_free_provision += 1 + (self->om_free_provision/2);
1480 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1481
1482 if (!AsyncDeflateIdleMonitors &&
1483 is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) -
1484 OrderAccess::load_acquire(&g_om_free_count))) {
1485 // Not enough ObjectMonitors on the global free list.
1486 // We can't safely induce a STW safepoint from om_alloc() as our thread
1487 // state may not be appropriate for such activities and callers may hold
1488 // naked oops, so instead we defer the action.
1489 InduceScavenge(self, "om_alloc");
1490 }
1491 continue;
1492 }
1493
1494 // 3: allocate a block of new ObjectMonitors
1495 // Both the local and global free lists are empty -- resort to malloc().
1496 // In the current implementation ObjectMonitors are TSM - immortal.
1497 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1498 // each ObjectMonitor to start at the beginning of a cache line,
1499 // so we use align_up().
1500 // A better solution would be to use C++ placement-new.
1501 // BEWARE: As it stands currently, we don't run the ctors!
1502 assert(_BLOCKSIZE > 1, "invariant");
1503 size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE;
1504 PaddedObjectMonitor* temp;
1505 size_t aligned_size = neededsize + (OM_CACHE_LINE_SIZE - 1);
1506 void* real_malloc_addr = NEW_C_HEAP_ARRAY(char, aligned_size, mtInternal);
1507 temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, OM_CACHE_LINE_SIZE);
1508 (void)memset((void *) temp, 0, neededsize);
1509
1510 // Format the block.
1511 // initialize the linked list, each monitor points to its next
1512 // forming the single linked free list, the very first monitor
1513 // will points to next block, which forms the block list.
1514 // The trick of using the 1st element in the block as g_block_list
1515 // linkage should be reconsidered. A better implementation would
1516 // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; }
1517
1518 for (int i = 1; i < _BLOCKSIZE; i++) {
1519 OrderAccess::release_store(&temp[i]._next_om, (ObjectMonitor*)&temp[i+1]);
1520 assert(temp[i].is_free(), "invariant");
1521 }
1522
1523 // terminate the last monitor as the end of list
1524 OrderAccess::release_store(&temp[_BLOCKSIZE - 1]._next_om, (ObjectMonitor*)NULL);
1525
1526 // Element [0] is reserved for global list linkage
1527 temp[0].set_object(CHAINMARKER);
1528
1529 // Consider carving out this thread's current request from the
1530 // block in hand. This avoids some lock traffic and redundant
1531 // list activity.
1532
1533 prepend_block_to_lists(temp);
1534 }
1535 }
1536
1537 // Place "m" on the caller's private per-thread om_free_list.
1538 // In practice there's no need to clamp or limit the number of
1539 // monitors on a thread's om_free_list as the only non-allocation time
1540 // we'll call om_release() is to return a monitor to the free list after
1541 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1542 // accumulate on a thread's free list.
1543 //
1544 // Key constraint: all ObjectMonitors on a thread's free list and the global
1545 // free list must have their object field set to null. This prevents the
1546 // scavenger -- deflate_monitor_list() or deflate_monitor_list_using_JT()
1547 // -- from reclaiming them while we are trying to release them.
1548
1549 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1550 bool from_per_thread_alloc) {
1551 guarantee(m->header().value() == 0, "invariant");
1552 guarantee(m->object() == NULL, "invariant");
1553 stringStream ss;
1554 guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1555 "%s, recursions=" INTX_FORMAT, m->is_busy_to_string(&ss),
1556 m->_recursions);
1557 m->set_allocation_state(ObjectMonitor::Free);
1558 // _next_om is used for both per-thread in-use and free lists so
1559 // we have to remove 'm' from the in-use list first (as needed).
1560 if (from_per_thread_alloc) {
1561 // Need to remove 'm' from om_in_use_list.
1562 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
1563 // protocol because async deflation can do list deletions in parallel.
1564 ObjectMonitor* cur_mid_in_use = NULL;
1565 ObjectMonitor* mid = NULL;
1566 ObjectMonitor* next = NULL;
1567 bool extracted = false;
1568
1569 if (!mark_list_head(&self->om_in_use_list, &mid, &next)) {
1570 fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self));
1571 }
1572 while (true) {
1573 if (m == mid) {
1574 // We found 'm' on the per-thread in-use list so try to extract it.
1575 if (cur_mid_in_use == NULL) {
1576 // mid is the list head and it is marked. Switch the list head
1577 // to next which unmarks the list head, but leaves mid marked:
1578 OrderAccess::release_store(&self->om_in_use_list, next);
1579 } else {
1580 // mid and cur_mid_in_use are marked. Switch cur_mid_in_use's
1581 // next field to next which unmarks cur_mid_in_use, but leaves
1582 // mid marked:
1583 OrderAccess::release_store(&cur_mid_in_use->_next_om, next);
1584 }
1585 extracted = true;
1586 Atomic::dec(&self->om_in_use_count);
1587 // Unmark mid, but leave the next value for any lagging list
1588 // walkers. It will get cleaned up when mid is prepended to
1589 // the thread's free list:
1590 set_next(mid, next);
1591 break;
1592 }
1593 if (cur_mid_in_use != NULL) {
1594 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
1595 }
1596 // The next cur_mid_in_use keeps mid's marked next field so
1597 // that it is stable for a possible next field change. It
1598 // cannot be deflated while it is marked.
1599 cur_mid_in_use = mid;
1600 mid = next;
1601 if (mid == NULL) {
1602 // Reached end of the list and didn't find m so:
1603 fatal("must find m=" INTPTR_FORMAT "on om_in_use_list=" INTPTR_FORMAT,
1604 p2i(m), p2i(self->om_in_use_list));
1605 }
1606 // Mark mid's next field so we can possibly extract it:
1607 next = mark_next_loop(mid);
1608 }
1609 }
1610
1611 prepend_to_om_free_list(self, m);
1612 guarantee(m->is_free(), "invariant");
1613 }
1614
1615 // Return ObjectMonitors on a moribund thread's free and in-use
1616 // lists to the appropriate global lists. The ObjectMonitors on the
1617 // per-thread in-use list may still be in use by other threads.
1618 //
1619 // We currently call om_flush() from Threads::remove() before the
1620 // thread has been excised from the thread list and is no longer a
1621 // mutator. This means that om_flush() cannot run concurrently with
1622 // a safepoint and interleave with deflate_idle_monitors(). In
1623 // particular, this ensures that the thread's in-use monitors are
1624 // scanned by a GC safepoint, either via Thread::oops_do() (before
1625 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1626 // om_flush() is called).
1627 //
1628 // With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT()
1629 // and deflate_per_thread_idle_monitors_using_JT() (in another thread) can
1630 // run at the same time as om_flush() so we have to follow a careful
1631 // protocol to prevent list corruption.
1632
1633 void ObjectSynchronizer::om_flush(Thread* self) {
1634 // This function can race with an async deflater thread. Since
1635 // deflation has to process the per-thread in-use list before
1636 // prepending the deflated ObjectMonitors to the global free list,
1637 // we process the per-thread lists in the same order to prevent
1638 // ordering races.
1639 int in_use_count = 0;
1640 ObjectMonitor* in_use_list = NULL;
1641 ObjectMonitor* in_use_tail = NULL;
1642 ObjectMonitor* next = NULL;
1643
1644 // An async deflation thread checks to see if the target thread
1645 // is exiting, but if it has made it past that check before we
1646 // started exiting, then it is racing to get to the in-use list.
1647 if (mark_list_head(&self->om_in_use_list, &in_use_list, &next)) {
1648 // At this point, we have marked the in-use list head so an
1649 // async deflation thread cannot come in after us. If an async
1650 // deflation thread is ahead of us, then we'll detect that and
1651 // wait for it to finish its work.
1652 //
1653 // The thread is going away, however the ObjectMonitors on the
1654 // om_in_use_list may still be in-use by other threads. Link
1655 // them to in_use_tail, which will be linked into the global
1656 // in-use list g_om_in_use_list below.
1657 //
1658 // Account for the in-use list head before the loop since it is
1659 // already marked (by this thread):
1660 in_use_tail = in_use_list;
1661 in_use_count++;
1662 for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL;) {
1663 if (is_next_marked(cur_om)) {
1664 // This next field is marked so there must be an async deflater
1665 // thread ahead of us so we'll give it a chance to finish.
1666 while (is_next_marked(cur_om)) {
1667 os::naked_short_sleep(1);
1668 }
1669 // Refetch the possibly changed next field and try again.
1670 cur_om = unmarked_next(in_use_tail);
1671 continue;
1672 }
1673 if (!cur_om->is_active()) {
1674 // cur_om was deflated and the allocation state was changed
1675 // to Free while it was marked. We happened to see it just
1676 // after it was unmarked (and added to the free list).
1677 // Refetch the possibly changed next field and try again.
1678 cur_om = unmarked_next(in_use_tail);
1679 continue;
1680 }
1681 in_use_tail = cur_om;
1682 in_use_count++;
1683 cur_om = unmarked_next(cur_om);
1684 }
1685 guarantee(in_use_tail != NULL, "invariant");
1686 int l_om_in_use_count = OrderAccess::load_acquire(&self->om_in_use_count);
1687 ADIM_guarantee(l_om_in_use_count == in_use_count, "in-use counts don't "
1688 "match: l_om_in_use_count=%d, in_use_count=%d",
1689 l_om_in_use_count, in_use_count);
1690 // Clear the in-use count before unmarking the in-use list head
1691 // to avoid races:
1692 OrderAccess::release_store(&self->om_in_use_count, 0);
1693 // Clear the in-use list head (which also unmarks it):
1694 OrderAccess::release_store(&self->om_in_use_list, (ObjectMonitor*)NULL);
1695 // Unmark the disconnected list head:
1696 set_next(in_use_list, next);
1697 }
1698
1699 int free_count = 0;
1700 ObjectMonitor* free_list = OrderAccess::load_acquire(&self->om_free_list);
1701 ObjectMonitor* free_tail = NULL;
1702 if (free_list != NULL) {
1703 // The thread is going away. Set 'free_tail' to the last per-thread free
1704 // monitor which will be linked to g_free_list below.
1705 stringStream ss;
1706 for (ObjectMonitor* s = free_list; s != NULL; s = unmarked_next(s)) {
1707 free_count++;
1708 free_tail = s;
1709 guarantee(s->object() == NULL, "invariant");
1710 guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss));
1711 }
1712 guarantee(free_tail != NULL, "invariant");
1713 int l_om_free_count = OrderAccess::load_acquire(&self->om_free_count);
1714 ADIM_guarantee(l_om_free_count == free_count, "free counts don't match: "
1715 "l_om_free_count=%d, free_count=%d", l_om_free_count,
1716 free_count);
1717 OrderAccess::release_store(&self->om_free_list, (ObjectMonitor*)NULL);
1718 OrderAccess::release_store(&self->om_free_count, 0);
1719 }
1720
1721 if (free_tail != NULL) {
1722 prepend_list_to_g_free_list(free_list, free_tail, free_count);
1723 }
1724
1725 if (in_use_tail != NULL) {
1726 prepend_list_to_g_om_in_use_list(in_use_list, in_use_tail, in_use_count);
1727 }
1728
1729 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1730 LogStreamHandle(Info, monitorinflation) lsh_info;
1731 LogStream* ls = NULL;
1732 if (log_is_enabled(Debug, monitorinflation)) {
1733 ls = &lsh_debug;
1734 } else if ((free_count != 0 || in_use_count != 0) &&
1735 log_is_enabled(Info, monitorinflation)) {
1736 ls = &lsh_info;
1737 }
1738 if (ls != NULL) {
1739 ls->print_cr("om_flush: jt=" INTPTR_FORMAT ", free_count=%d"
1740 ", in_use_count=%d" ", om_free_provision=%d",
1741 p2i(self), free_count, in_use_count, self->om_free_provision);
1742 }
1743 }
1744
1745 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1746 const oop obj,
1747 ObjectSynchronizer::InflateCause cause) {
1748 assert(event != NULL, "invariant");
1749 assert(event->should_commit(), "invariant");
1750 event->set_monitorClass(obj->klass());
1751 event->set_address((uintptr_t)(void*)obj);
1752 event->set_cause((u1)cause);
1753 event->commit();
1754 }
1755
1756 // Fast path code shared by multiple functions
1757 void ObjectSynchronizer::inflate_helper(ObjectMonitorHandle* omh_p, oop obj) {
1758 while (true) {
1759 markWord mark = obj->mark();
1760 if (mark.has_monitor()) {
1761 if (!omh_p->save_om_ptr(obj, mark)) {
1762 // Lost a race with async deflation so try again.
1763 assert(AsyncDeflateIdleMonitors, "sanity check");
1764 continue;
1765 }
1766 ObjectMonitor* monitor = omh_p->om_ptr();
1767 assert(ObjectSynchronizer::verify_objmon_isinpool(monitor), "monitor is invalid");
1768 markWord dmw = monitor->header();
1769 assert(dmw.is_neutral(), "sanity check: header=" INTPTR_FORMAT, dmw.value());
1770 return;
1771 }
1772 inflate(omh_p, Thread::current(), obj, inflate_cause_vm_internal);
1773 return;
1774 }
1775 }
1776
1777 void ObjectSynchronizer::inflate(ObjectMonitorHandle* omh_p, Thread* self,
1778 oop object, const InflateCause cause) {
1779 // Inflate mutates the heap ...
1780 // Relaxing assertion for bug 6320749.
1781 assert(Universe::verify_in_progress() ||
1782 !SafepointSynchronize::is_at_safepoint(), "invariant");
1783
1784 EventJavaMonitorInflate event;
1785
1786 for (;;) {
1787 const markWord mark = object->mark();
1788 assert(!mark.has_bias_pattern(), "invariant");
1789
1790 // The mark can be in one of the following states:
1791 // * Inflated - just return
1792 // * Stack-locked - coerce it to inflated
1793 // * INFLATING - busy wait for conversion to complete
1794 // * Neutral - aggressively inflate the object.
1795 // * BIASED - Illegal. We should never see this
1796
1797 // CASE: inflated
1798 if (mark.has_monitor()) {
1799 if (!omh_p->save_om_ptr(object, mark)) {
1800 // Lost a race with async deflation so try again.
1801 assert(AsyncDeflateIdleMonitors, "sanity check");
1802 continue;
1803 }
1804 ObjectMonitor* inf = omh_p->om_ptr();
1805 markWord dmw = inf->header();
1806 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1807 assert(inf->object() == object, "invariant");
1808 assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
1809 return;
1810 }
1811
1812 // CASE: inflation in progress - inflating over a stack-lock.
1813 // Some other thread is converting from stack-locked to inflated.
1814 // Only that thread can complete inflation -- other threads must wait.
1815 // The INFLATING value is transient.
1816 // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1817 // We could always eliminate polling by parking the thread on some auxiliary list.
1818 if (mark == markWord::INFLATING()) {
1819 read_stable_mark(object);
1820 continue;
1821 }
1822
1823 // CASE: stack-locked
1824 // Could be stack-locked either by this thread or by some other thread.
1825 //
1826 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1827 // to install INFLATING into the mark word. We originally installed INFLATING,
1828 // allocated the objectmonitor, and then finally STed the address of the
1829 // objectmonitor into the mark. This was correct, but artificially lengthened
1830 // the interval in which INFLATED appeared in the mark, thus increasing
1831 // the odds of inflation contention.
1832 //
1833 // We now use per-thread private objectmonitor free lists.
1834 // These list are reprovisioned from the global free list outside the
1835 // critical INFLATING...ST interval. A thread can transfer
1836 // multiple objectmonitors en-mass from the global free list to its local free list.
1837 // This reduces coherency traffic and lock contention on the global free list.
1838 // Using such local free lists, it doesn't matter if the om_alloc() call appears
1839 // before or after the CAS(INFLATING) operation.
1840 // See the comments in om_alloc().
1841
1842 LogStreamHandle(Trace, monitorinflation) lsh;
1843
1844 if (mark.has_locker()) {
1845 ObjectMonitor* m = om_alloc(self, cause);
1846 // Optimistically prepare the objectmonitor - anticipate successful CAS
1847 // We do this before the CAS in order to minimize the length of time
1848 // in which INFLATING appears in the mark.
1849 m->Recycle();
1850 m->_Responsible = NULL;
1851 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // Consider: maintain by type/class
1852
1853 markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1854 if (cmp != mark) {
1855 // om_release() will reset the allocation state from New to Free.
1856 om_release(self, m, true);
1857 continue; // Interference -- just retry
1858 }
1859
1860 // We've successfully installed INFLATING (0) into the mark-word.
1861 // This is the only case where 0 will appear in a mark-word.
1862 // Only the singular thread that successfully swings the mark-word
1863 // to 0 can perform (or more precisely, complete) inflation.
1864 //
1865 // Why do we CAS a 0 into the mark-word instead of just CASing the
1866 // mark-word from the stack-locked value directly to the new inflated state?
1867 // Consider what happens when a thread unlocks a stack-locked object.
1868 // It attempts to use CAS to swing the displaced header value from the
1869 // on-stack BasicLock back into the object header. Recall also that the
1870 // header value (hash code, etc) can reside in (a) the object header, or
1871 // (b) a displaced header associated with the stack-lock, or (c) a displaced
1872 // header in an ObjectMonitor. The inflate() routine must copy the header
1873 // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1874 // the while preserving the hashCode stability invariants. If the owner
1875 // decides to release the lock while the value is 0, the unlock will fail
1876 // and control will eventually pass from slow_exit() to inflate. The owner
1877 // will then spin, waiting for the 0 value to disappear. Put another way,
1878 // the 0 causes the owner to stall if the owner happens to try to
1879 // drop the lock (restoring the header from the BasicLock to the object)
1880 // while inflation is in-progress. This protocol avoids races that might
1881 // would otherwise permit hashCode values to change or "flicker" for an object.
1882 // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1883 // 0 serves as a "BUSY" inflate-in-progress indicator.
1884
1885
1886 // fetch the displaced mark from the owner's stack.
1887 // The owner can't die or unwind past the lock while our INFLATING
1888 // object is in the mark. Furthermore the owner can't complete
1889 // an unlock on the object, either.
1890 markWord dmw = mark.displaced_mark_helper();
1891 // Catch if the object's header is not neutral (not locked and
1892 // not marked is what we care about here).
1893 ADIM_guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1894
1895 // Setup monitor fields to proper values -- prepare the monitor
1896 m->set_header(dmw);
1897
1898 // Optimization: if the mark.locker stack address is associated
1899 // with this thread we could simply set m->_owner = self.
1900 // Note that a thread can inflate an object
1901 // that it has stack-locked -- as might happen in wait() -- directly
1902 // with CAS. That is, we can avoid the xchg-NULL .... ST idiom.
1903 if (AsyncDeflateIdleMonitors) {
1904 m->set_owner_from(mark.locker(), NULL, DEFLATER_MARKER);
1905 } else {
1906 m->set_owner_from(mark.locker(), NULL);
1907 }
1908 m->set_object(object);
1909 // TODO-FIXME: assert BasicLock->dhw != 0.
1910
1911 omh_p->set_om_ptr(m);
1912
1913 // Must preserve store ordering. The monitor state must
1914 // be stable at the time of publishing the monitor address.
1915 guarantee(object->mark() == markWord::INFLATING(), "invariant");
1916 object->release_set_mark(markWord::encode(m));
1917
1918 // Once ObjectMonitor is configured and the object is associated
1919 // with the ObjectMonitor, it is safe to allow async deflation:
1920 assert(m->is_new(), "freshly allocated monitor must be new");
1921 m->set_allocation_state(ObjectMonitor::Old);
1922
1923 // Hopefully the performance counters are allocated on distinct cache lines
1924 // to avoid false sharing on MP systems ...
1925 OM_PERFDATA_OP(Inflations, inc());
1926 if (log_is_enabled(Trace, monitorinflation)) {
1927 ResourceMark rm(self);
1928 lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
1929 INTPTR_FORMAT ", type='%s'", p2i(object),
1930 object->mark().value(), object->klass()->external_name());
1931 }
1932 if (event.should_commit()) {
1933 post_monitor_inflate_event(&event, object, cause);
1934 }
1935 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
1936 return;
1937 }
1938
1939 // CASE: neutral
1940 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
1941 // If we know we're inflating for entry it's better to inflate by swinging a
1942 // pre-locked ObjectMonitor pointer into the object header. A successful
1943 // CAS inflates the object *and* confers ownership to the inflating thread.
1944 // In the current implementation we use a 2-step mechanism where we CAS()
1945 // to inflate and then CAS() again to try to swing _owner from NULL to self.
1946 // An inflateTry() method that we could call from enter() would be useful.
1947
1948 // Catch if the object's header is not neutral (not locked and
1949 // not marked is what we care about here).
1950 ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT,mark.value());
1951 ObjectMonitor* m = om_alloc(self, cause);
1952 // prepare m for installation - set monitor to initial state
1953 m->Recycle();
1954 m->set_header(mark);
1955 // If we leave _owner == DEFLATER_MARKER here, then the simple C2
1956 // ObjectMonitor enter optimization can no longer race with async
1957 // deflation and reuse.
1958 m->set_object(object);
1959 m->_Responsible = NULL;
1960 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
1961
1962 omh_p->set_om_ptr(m);
1963
1964 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
1965 m->set_header(markWord::zero());
1966 m->set_object(NULL);
1967 m->Recycle();
1968 omh_p->set_om_ptr(NULL);
1969 // om_release() will reset the allocation state from New to Free.
1970 om_release(self, m, true);
1971 m = NULL;
1972 continue;
1973 // interference - the markword changed - just retry.
1974 // The state-transitions are one-way, so there's no chance of
1975 // live-lock -- "Inflated" is an absorbing state.
1976 }
1977
1978 // Once the ObjectMonitor is configured and object is associated
1979 // with the ObjectMonitor, it is safe to allow async deflation:
1980 assert(m->is_new(), "freshly allocated monitor must be new");
1981 m->set_allocation_state(ObjectMonitor::Old);
1982
1983 // Hopefully the performance counters are allocated on distinct
1984 // cache lines to avoid false sharing on MP systems ...
1985 OM_PERFDATA_OP(Inflations, inc());
1986 if (log_is_enabled(Trace, monitorinflation)) {
1987 ResourceMark rm(self);
1988 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
1989 INTPTR_FORMAT ", type='%s'", p2i(object),
1990 object->mark().value(), object->klass()->external_name());
1991 }
1992 if (event.should_commit()) {
1993 post_monitor_inflate_event(&event, object, cause);
1994 }
1995 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
1996 return;
1997 }
1998 }
1999
2000
2001 // We maintain a list of in-use monitors for each thread.
2002 //
2003 // For safepoint based deflation:
2004 // deflate_thread_local_monitors() scans a single thread's in-use list, while
2005 // deflate_idle_monitors() scans only a global list of in-use monitors which
2006 // is populated only as a thread dies (see om_flush()).
2007 //
2008 // These operations are called at all safepoints, immediately after mutators
2009 // are stopped, but before any objects have moved. Collectively they traverse
2010 // the population of in-use monitors, deflating where possible. The scavenged
2011 // monitors are returned to the global monitor free list.
2012 //
2013 // Beware that we scavenge at *every* stop-the-world point. Having a large
2014 // number of monitors in-use could negatively impact performance. We also want
2015 // to minimize the total # of monitors in circulation, as they incur a small
2016 // footprint penalty.
2017 //
2018 // Perversely, the heap size -- and thus the STW safepoint rate --
2019 // typically drives the scavenge rate. Large heaps can mean infrequent GC,
2020 // which in turn can mean large(r) numbers of ObjectMonitors in circulation.
2021 // This is an unfortunate aspect of this design.
2022 //
2023 // For async deflation:
2024 // If a special deflation request is made, then the safepoint based
2025 // deflation mechanism is used. Otherwise, an async deflation request
2026 // is registered with the ServiceThread and it is notified.
2027
2028 void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* counters) {
2029 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2030
2031 // The per-thread in-use lists are handled in
2032 // ParallelSPCleanupThreadClosure::do_thread().
2033
2034 if (!AsyncDeflateIdleMonitors || is_special_deflation_requested()) {
2035 // Use the older mechanism for the global in-use list or if a
2036 // special deflation has been requested before the safepoint.
2037 ObjectSynchronizer::deflate_idle_monitors(counters);
2038 return;
2039 }
2040
2041 log_debug(monitorinflation)("requesting async deflation of idle monitors.");
2042 // Request deflation of idle monitors by the ServiceThread:
2043 set_is_async_deflation_requested(true);
2044 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
2045 ml.notify_all();
2046 }
2047
2048 // Deflate a single monitor if not in-use
2049 // Return true if deflated, false if in-use
2050 bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj,
2051 ObjectMonitor** free_head_p,
2052 ObjectMonitor** free_tail_p) {
2053 bool deflated;
2054 // Normal case ... The monitor is associated with obj.
2055 const markWord mark = obj->mark();
2056 guarantee(mark == markWord::encode(mid), "should match: mark="
2057 INTPTR_FORMAT ", encoded mid=" INTPTR_FORMAT, mark.value(),
2058 markWord::encode(mid).value());
2059 // Make sure that mark.monitor() and markWord::encode() agree:
2060 guarantee(mark.monitor() == mid, "should match: monitor()=" INTPTR_FORMAT
2061 ", mid=" INTPTR_FORMAT, p2i(mark.monitor()), p2i(mid));
2062 const markWord dmw = mid->header();
2063 guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
2064
2065 if (mid->is_busy() || mid->ref_count() != 0) {
2066 // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor*
2067 // is in use so no deflation.
2068 deflated = false;
2069 } else {
2070 // Deflate the monitor if it is no longer being used
2071 // It's idle - scavenge and return to the global free list
2072 // plain old deflation ...
2073 if (log_is_enabled(Trace, monitorinflation)) {
2074 ResourceMark rm;
2075 log_trace(monitorinflation)("deflate_monitor: "
2076 "object=" INTPTR_FORMAT ", mark="
2077 INTPTR_FORMAT ", type='%s'", p2i(obj),
2078 mark.value(), obj->klass()->external_name());
2079 }
2080
2081 // Restore the header back to obj
2082 obj->release_set_mark(dmw);
2083 if (AsyncDeflateIdleMonitors) {
2084 // clear() expects the owner field to be NULL and we won't race
2085 // with the simple C2 ObjectMonitor enter optimization since
2086 // we're at a safepoint. DEFLATER_MARKER is the only non-NULL
2087 // value we should see here.
2088 mid->try_set_owner_from(NULL, DEFLATER_MARKER);
2089 }
2090 mid->clear();
2091
2092 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
2093 p2i(mid->object()));
2094 assert(mid->is_free(), "invariant");
2095
2096 // Move the deflated ObjectMonitor to the working free list
2097 // defined by free_head_p and free_tail_p. No races on this list
2098 // so no need for load_acquire() or store_release().
2099 if (*free_head_p == NULL) *free_head_p = mid;
2100 if (*free_tail_p != NULL) {
2101 // We append to the list so the caller can use mid->_next_om
2102 // to fix the linkages in its context.
2103 ObjectMonitor* prevtail = *free_tail_p;
2104 // Should have been cleaned up by the caller:
2105 // Note: Should not have to mark prevtail here since we're at a
2106 // safepoint and ObjectMonitors on the local free list should
2107 // not be accessed in parallel.
2108 assert(prevtail->_next_om == NULL, "must be NULL: _next_om="
2109 INTPTR_FORMAT, p2i(prevtail->_next_om));
2110 set_next(prevtail, mid);
2111 }
2112 *free_tail_p = mid;
2113 // At this point, mid->_next_om still refers to its current
2114 // value and another ObjectMonitor's _next_om field still
2115 // refers to this ObjectMonitor. Those linkages have to be
2116 // cleaned up by the caller who has the complete context.
2117 deflated = true;
2118 }
2119 return deflated;
2120 }
2121
2122 // Deflate the specified ObjectMonitor if not in-use using a JavaThread.
2123 // Returns true if it was deflated and false otherwise.
2124 //
2125 // The async deflation protocol sets owner to DEFLATER_MARKER and
2126 // makes ref_count negative as signals to contending threads that
2127 // an async deflation is in progress. There are a number of checks
2128 // as part of the protocol to make sure that the calling thread has
2129 // not lost the race to a contending thread or to a thread that just
2130 // wants to use the ObjectMonitor*.
2131 //
2132 // The ObjectMonitor has been successfully async deflated when:
2133 // (owner == DEFLATER_MARKER && ref_count < 0)
2134 // Contending threads or ObjectMonitor* using threads that see those
2135 // values know to retry their operation.
2136 //
2137 bool ObjectSynchronizer::deflate_monitor_using_JT(ObjectMonitor* mid,
2138 ObjectMonitor** free_head_p,
2139 ObjectMonitor** free_tail_p) {
2140 assert(AsyncDeflateIdleMonitors, "sanity check");
2141 assert(Thread::current()->is_Java_thread(), "precondition");
2142 // A newly allocated ObjectMonitor should not be seen here so we
2143 // avoid an endless inflate/deflate cycle.
2144 assert(mid->is_old(), "must be old: allocation_state=%d",
2145 (int) mid->allocation_state());
2146
2147 if (mid->is_busy() || mid->ref_count() != 0) {
2148 // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor*
2149 // is in use so no deflation.
2150 return false;
2151 }
2152
2153 if (mid->try_set_owner_from(DEFLATER_MARKER, NULL) == NULL) {
2154 // ObjectMonitor is not owned by another thread. Our setting
2155 // owner to DEFLATER_MARKER forces any contending thread through
2156 // the slow path. This is just the first part of the async
2157 // deflation dance.
2158
2159 if (mid->_contentions != 0 || mid->_waiters != 0) {
2160 // Another thread has raced to enter the ObjectMonitor after
2161 // mid->is_busy() above or has already entered and waited on
2162 // it which makes it busy so no deflation. Restore owner to
2163 // NULL if it is still DEFLATER_MARKER.
2164 mid->try_set_owner_from(NULL, DEFLATER_MARKER);
2165 return false;
2166 }
2167
2168 if (Atomic::cmpxchg(-max_jint, &mid->_ref_count, (jint)0) == 0) {
2169 // Make ref_count negative to force any contending threads or
2170 // ObjectMonitor* using threads to retry. This is the second
2171 // part of the async deflation dance.
2172
2173 if (mid->owner_is_DEFLATER_MARKER()) {
2174 // If owner is still DEFLATER_MARKER, then we have successfully
2175 // signaled any contending threads to retry. If it is not, then we
2176 // have lost the race to an entering thread and the ObjectMonitor
2177 // is now busy. This is the third and final part of the async
2178 // deflation dance.
2179 // Note: This owner check solves the ABA problem with ref_count
2180 // where another thread acquired the ObjectMonitor, finished
2181 // using it and restored the ref_count to zero.
2182
2183 // Sanity checks for the races:
2184 guarantee(mid->_contentions == 0, "must be 0: contentions=%d",
2185 mid->_contentions);
2186 guarantee(mid->_waiters == 0, "must be 0: waiters=%d", mid->_waiters);
2187 guarantee(mid->_cxq == NULL, "must be no contending threads: cxq="
2188 INTPTR_FORMAT, p2i(mid->_cxq));
2189 guarantee(mid->_EntryList == NULL,
2190 "must be no entering threads: EntryList=" INTPTR_FORMAT,
2191 p2i(mid->_EntryList));
2192
2193 const oop obj = (oop) mid->object();
2194 if (log_is_enabled(Trace, monitorinflation)) {
2195 ResourceMark rm;
2196 log_trace(monitorinflation)("deflate_monitor_using_JT: "
2197 "object=" INTPTR_FORMAT ", mark="
2198 INTPTR_FORMAT ", type='%s'",
2199 p2i(obj), obj->mark().value(),
2200 obj->klass()->external_name());
2201 }
2202
2203 // Install the old mark word if nobody else has already done it.
2204 mid->install_displaced_markword_in_object(obj);
2205 mid->clear_using_JT();
2206
2207 assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT,
2208 p2i(mid->object()));
2209 assert(mid->is_free(), "must be free: allocation_state=%d",
2210 (int) mid->allocation_state());
2211
2212 // Move the deflated ObjectMonitor to the working free list
2213 // defined by free_head_p and free_tail_p. No races on this list
2214 // so no need for load_acquire() or store_release().
2215 if (*free_head_p == NULL) {
2216 // First one on the list.
2217 *free_head_p = mid;
2218 }
2219 if (*free_tail_p != NULL) {
2220 // We append to the list so the caller can use mid->_next_om
2221 // to fix the linkages in its context.
2222 ObjectMonitor* prevtail = *free_tail_p;
2223 // Should have been cleaned up by the caller:
2224 ObjectMonitor* next = mark_next_loop(prevtail);
2225 assert(unmarked_next(prevtail) == NULL, "must be NULL: _next_om="
2226 INTPTR_FORMAT, p2i(unmarked_next(prevtail)));
2227 set_next(prevtail, mid); // prevtail now points to mid (and is unmarked)
2228 }
2229 *free_tail_p = mid;
2230
2231 // At this point, mid->_next_om still refers to its current
2232 // value and another ObjectMonitor's _next_om field still
2233 // refers to this ObjectMonitor. Those linkages have to be
2234 // cleaned up by the caller who has the complete context.
2235
2236 // We leave owner == DEFLATER_MARKER and ref_count < 0
2237 // to force any racing threads to retry.
2238 return true; // Success, ObjectMonitor has been deflated.
2239 }
2240
2241 // The owner was changed from DEFLATER_MARKER so we lost the
2242 // race since the ObjectMonitor is now busy.
2243
2244 // Add back max_jint to restore the ref_count field to its
2245 // proper value (which may not be what we saw above):
2246 Atomic::add(max_jint, &mid->_ref_count);
2247
2248 assert(mid->ref_count() >= 0, "must not be negative: ref_count=%d",
2249 mid->ref_count());
2250 return false;
2251 }
2252
2253 // The ref_count was no longer 0 so we lost the race since the
2254 // ObjectMonitor is now busy or the ObjectMonitor* is now is use.
2255 // Restore owner to NULL if it is still DEFLATER_MARKER:
2256 mid->try_set_owner_from(NULL, DEFLATER_MARKER);
2257 }
2258
2259 // The owner field is no longer NULL so we lost the race since the
2260 // ObjectMonitor is now busy.
2261 return false;
2262 }
2263
2264 // Walk a given monitor list, and deflate idle monitors.
2265 // The given list could be a per-thread list or a global list.
2266 //
2267 // In the case of parallel processing of thread local monitor lists,
2268 // work is done by Threads::parallel_threads_do() which ensures that
2269 // each Java thread is processed by exactly one worker thread, and
2270 // thus avoid conflicts that would arise when worker threads would
2271 // process the same monitor lists concurrently.
2272 //
2273 // See also ParallelSPCleanupTask and
2274 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
2275 // Threads::parallel_java_threads_do() in thread.cpp.
2276 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor* volatile * list_p,
2277 int volatile * count_p,
2278 ObjectMonitor** free_head_p,
2279 ObjectMonitor** free_tail_p) {
2280 ObjectMonitor* cur_mid_in_use = NULL;
2281 ObjectMonitor* mid = NULL;
2282 ObjectMonitor* next = NULL;
2283 int deflated_count = 0;
2284
2285 // We use the simpler mark-mid-as-we-go protocol since there are no
2286 // parallel list deletions since we are at a safepoint.
2287 if (!mark_list_head(list_p, &mid, &next)) {
2288 return 0; // The list is empty so nothing to deflate.
2289 }
2290
2291 while (true) {
2292 oop obj = (oop) mid->object();
2293 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
2294 // Deflation succeeded and already updated free_head_p and
2295 // free_tail_p as needed. Finish the move to the local free list
2296 // by unlinking mid from the global or per-thread in-use list.
2297 if (cur_mid_in_use == NULL) {
2298 // mid is the list head and it is marked. Switch the list head
2299 // to next which unmarks the list head, but leaves mid marked:
2300 OrderAccess::release_store(list_p, next);
2301 } else {
2302 // mid is marked. Switch cur_mid_in_use's next field to next
2303 // which is safe because we have no parallel list deletions,
2304 // but we leave mid marked:
2305 OrderAccess::release_store(&cur_mid_in_use->_next_om, next);
2306 }
2307 // At this point mid is disconnected from the in-use list so
2308 // its marked next field no longer has any effects.
2309 deflated_count++;
2310 Atomic::dec(count_p);
2311 // mid is current tail in the free_head_p list so NULL terminate it
2312 // (which also unmarks it):
2313 set_next(mid, NULL);
2314
2315 // All the list management is done so move on to the next one:
2316 mid = next;
2317 } else {
2318 set_next(mid, next); // unmark next field
2319
2320 // All the list management is done so move on to the next one:
2321 cur_mid_in_use = mid;
2322 mid = next;
2323 }
2324 if (mid == NULL) {
2325 break; // Reached end of the list so nothing more to deflate.
2326 }
2327 // Mark mid's next field so we can possibly deflate it:
2328 next = mark_next_loop(mid);
2329 }
2330 return deflated_count;
2331 }
2332
2333 // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using
2334 // a JavaThread. Returns the number of deflated ObjectMonitors. The given
2335 // list could be a per-thread in-use list or the global in-use list.
2336 // If a safepoint has started, then we save state via saved_mid_in_use_p
2337 // and return to the caller to honor the safepoint.
2338 //
2339 int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor* volatile * list_p,
2340 int volatile * count_p,
2341 ObjectMonitor** free_head_p,
2342 ObjectMonitor** free_tail_p,
2343 ObjectMonitor** saved_mid_in_use_p) {
2344 assert(AsyncDeflateIdleMonitors, "sanity check");
2345 assert(Thread::current()->is_Java_thread(), "precondition");
2346
2347 ObjectMonitor* cur_mid_in_use = NULL;
2348 ObjectMonitor* mid = NULL;
2349 ObjectMonitor* next = NULL;
2350 ObjectMonitor* next_next = NULL;
2351 int deflated_count = 0;
2352
2353 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
2354 // protocol because om_release() can do list deletions in parallel.
2355 // We also mark-next-next-as-we-go to prevent an om_flush() that is
2356 // behind this thread from passing us.
2357 if (*saved_mid_in_use_p == NULL) {
2358 // No saved state so start at the beginning.
2359 // Mark the list head's next field so we can possibly deflate it:
2360 if (!mark_list_head(list_p, &mid, &next)) {
2361 return 0; // The list is empty so nothing to deflate.
2362 }
2363 } else {
2364 // We're restarting after a safepoint so restore the necessary state
2365 // before we resume.
2366 cur_mid_in_use = *saved_mid_in_use_p;
2367 // Mark cur_mid_in_use's next field so we can possibly update its
2368 // next field to extract a deflated ObjectMonitor.
2369 mid = mark_next_loop(cur_mid_in_use);
2370 if (mid == NULL) {
2371 set_next(cur_mid_in_use, NULL); // unmark next field
2372 *saved_mid_in_use_p = NULL;
2373 return 0; // The remainder is empty so nothing more to deflate.
2374 }
2375 // Mark mid's next field so we can possibly deflate it:
2376 next = mark_next_loop(mid);
2377 }
2378
2379 while (true) {
2380 // The current mid's next field is marked at this point. If we have
2381 // a cur_mid_in_use, then its next field is also marked at this point.
2382
2383 if (next != NULL) {
2384 // We mark next's next field so that an om_flush()
2385 // thread that is behind us cannot pass us when we
2386 // unmark the current mid's next field.
2387 next_next = mark_next_loop(next);
2388 }
2389
2390 // Only try to deflate if there is an associated Java object and if
2391 // mid is old (is not newly allocated and is not newly freed).
2392 if (mid->object() != NULL && mid->is_old() &&
2393 deflate_monitor_using_JT(mid, free_head_p, free_tail_p)) {
2394 // Deflation succeeded and already updated free_head_p and
2395 // free_tail_p as needed. Finish the move to the local free list
2396 // by unlinking mid from the global or per-thread in-use list.
2397 if (cur_mid_in_use == NULL) {
2398 // mid is the list head and it is marked. Switch the list head
2399 // to next which is also marked (if not NULL) and also leave
2400 // mid marked:
2401 OrderAccess::release_store(list_p, next);
2402 } else {
2403 ObjectMonitor* marked_next = mark_om_ptr(next);
2404 // mid and cur_mid_in_use are marked. Switch cur_mid_in_use's
2405 // next field to marked_next and also leave mid marked:
2406 OrderAccess::release_store(&cur_mid_in_use->_next_om, marked_next);
2407 }
2408 // At this point mid is disconnected from the in-use list so
2409 // its marked next field no longer has any effects.
2410 deflated_count++;
2411 Atomic::dec(count_p);
2412 // mid is current tail in the free_head_p list so NULL terminate it
2413 // (which also unmarks it):
2414 set_next(mid, NULL);
2415
2416 // All the list management is done so move on to the next one:
2417 mid = next; // mid keeps non-NULL next's marked next field
2418 next = next_next;
2419 } else {
2420 // mid is considered in-use if it does not have an associated
2421 // Java object or mid is not old or deflation did not succeed.
2422 // A mid->is_new() node can be seen here when it is freshly
2423 // returned by om_alloc() (and skips the deflation code path).
2424 // A mid->is_old() node can be seen here when deflation failed.
2425 // A mid->is_free() node can be seen here when a fresh node from
2426 // om_alloc() is released by om_release() due to losing the race
2427 // in inflate().
2428
2429 // All the list management is done so move on to the next one:
2430 if (cur_mid_in_use != NULL) {
2431 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2432 }
2433 // The next cur_mid_in_use keeps mid's marked next field so
2434 // that it is stable for a possible next field change. It
2435 // cannot be modified by om_release() while it is marked.
2436 cur_mid_in_use = mid;
2437 mid = next; // mid keeps non-NULL next's marked next field
2438 next = next_next;
2439
2440 if (SafepointSynchronize::is_synchronizing() &&
2441 cur_mid_in_use != OrderAccess::load_acquire(list_p) &&
2442 cur_mid_in_use->is_old()) {
2443 // If a safepoint has started and cur_mid_in_use is not the list
2444 // head and is old, then it is safe to use as saved state. Return
2445 // to the caller before blocking.
2446 *saved_mid_in_use_p = cur_mid_in_use;
2447 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2448 if (mid != NULL) {
2449 set_next(mid, next); // umark mid
2450 }
2451 return deflated_count;
2452 }
2453 }
2454 if (mid == NULL) {
2455 if (cur_mid_in_use != NULL) {
2456 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2457 }
2458 break; // Reached end of the list so nothing more to deflate.
2459 }
2460
2461 // The current mid's next field is marked at this point. If we have
2462 // a cur_mid_in_use, then its next field is also marked at this point.
2463 }
2464 // We finished the list without a safepoint starting so there's
2465 // no need to save state.
2466 *saved_mid_in_use_p = NULL;
2467 return deflated_count;
2468 }
2469
2470 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2471 OrderAccess::release_store(&counters->n_in_use, 0); // currently associated with objects
2472 OrderAccess::release_store(&counters->n_in_circulation, 0); // extant
2473 OrderAccess::release_store(&counters->n_scavenged, 0); // reclaimed (global and per-thread)
2474 OrderAccess::release_store(&counters->per_thread_scavenged, 0); // per-thread scavenge total
2475 counters->per_thread_times = 0.0; // per-thread scavenge times
2476 }
2477
2478 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
2479 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2480
2481 if (AsyncDeflateIdleMonitors) {
2482 // Nothing to do when global idle ObjectMonitors are deflated using
2483 // a JavaThread unless a special deflation has been requested.
2484 if (!is_special_deflation_requested()) {
2485 return;
2486 }
2487 }
2488
2489 bool deflated = false;
2490
2491 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2492 ObjectMonitor* free_tail_p = NULL;
2493 elapsedTimer timer;
2494
2495 if (log_is_enabled(Info, monitorinflation)) {
2496 timer.start();
2497 }
2498
2499 // Note: the thread-local monitors lists get deflated in
2500 // a separate pass. See deflate_thread_local_monitors().
2501
2502 // For moribund threads, scan g_om_in_use_list
2503 int deflated_count = 0;
2504 if (OrderAccess::load_acquire(&g_om_in_use_list) != NULL) {
2505 // Update n_in_circulation before g_om_in_use_count is updated by deflation.
2506 Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_circulation);
2507
2508 deflated_count = deflate_monitor_list(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p);
2509 Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_use);
2510 }
2511
2512 if (free_head_p != NULL) {
2513 // Move the deflated ObjectMonitors back to the global free list.
2514 // No races on the working free list so no need for load_acquire().
2515 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2516 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2517 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2518 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2519 Atomic::add(deflated_count, &counters->n_scavenged);
2520 }
2521 timer.stop();
2522
2523 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2524 LogStreamHandle(Info, monitorinflation) lsh_info;
2525 LogStream* ls = NULL;
2526 if (log_is_enabled(Debug, monitorinflation)) {
2527 ls = &lsh_debug;
2528 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2529 ls = &lsh_info;
2530 }
2531 if (ls != NULL) {
2532 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2533 }
2534 }
2535
2536 class HandshakeForDeflation : public ThreadClosure {
2537 public:
2538 void do_thread(Thread* thread) {
2539 log_trace(monitorinflation)("HandshakeForDeflation::do_thread: thread="
2540 INTPTR_FORMAT, p2i(thread));
2541 }
2542 };
2543
2544 void ObjectSynchronizer::deflate_idle_monitors_using_JT() {
2545 assert(AsyncDeflateIdleMonitors, "sanity check");
2546
2547 // Deflate any global idle monitors.
2548 deflate_global_idle_monitors_using_JT();
2549
2550 int count = 0;
2551 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2552 if (jt->om_in_use_count > 0 && !jt->is_exiting()) {
2553 // This JavaThread is using ObjectMonitors so deflate any that
2554 // are idle unless this JavaThread is exiting; do not race with
2555 // ObjectSynchronizer::om_flush().
2556 deflate_per_thread_idle_monitors_using_JT(jt);
2557 count++;
2558 }
2559 }
2560 if (count > 0) {
2561 log_debug(monitorinflation)("did async deflation of idle monitors for %d thread(s).", count);
2562 }
2563 // The ServiceThread's async deflation request has been processed.
2564 set_is_async_deflation_requested(false);
2565
2566 if (HandshakeAfterDeflateIdleMonitors && g_om_wait_count > 0) {
2567 // There are deflated ObjectMonitors waiting for a handshake
2568 // (or a safepoint) for safety.
2569
2570 // g_wait_list and g_om_wait_count are only updated by the calling
2571 // thread so no need for load_acquire() or release_store().
2572 ObjectMonitor* list = g_wait_list;
2573 ADIM_guarantee(list != NULL, "g_wait_list must not be NULL");
2574 int count = g_om_wait_count;
2575 g_wait_list = NULL;
2576 g_om_wait_count = 0;
2577
2578 // Find the tail for prepend_list_to_common().
2579 int l_count = 0;
2580 ObjectMonitor* tail = NULL;
2581 for (ObjectMonitor* n = list; n != NULL; n = unmarked_next(n)) {
2582 tail = n;
2583 l_count++;
2584 }
2585 ADIM_guarantee(count == l_count, "count=%d != l_count=%d", count, l_count);
2586
2587 // Will execute a safepoint if !ThreadLocalHandshakes:
2588 HandshakeForDeflation hfd_tc;
2589 Handshake::execute(&hfd_tc);
2590
2591 prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count);
2592
2593 log_info(monitorinflation)("moved %d idle monitors from global waiting list to global free list", count);
2594 }
2595 }
2596
2597 // Deflate global idle ObjectMonitors using a JavaThread.
2598 //
2599 void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() {
2600 assert(AsyncDeflateIdleMonitors, "sanity check");
2601 assert(Thread::current()->is_Java_thread(), "precondition");
2602 JavaThread* self = JavaThread::current();
2603
2604 deflate_common_idle_monitors_using_JT(true /* is_global */, self);
2605 }
2606
2607 // Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread.
2608 //
2609 void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) {
2610 assert(AsyncDeflateIdleMonitors, "sanity check");
2611 assert(Thread::current()->is_Java_thread(), "precondition");
2612
2613 deflate_common_idle_monitors_using_JT(false /* !is_global */, target);
2614 }
2615
2616 // Deflate global or per-thread idle ObjectMonitors using a JavaThread.
2617 //
2618 void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) {
2619 JavaThread* self = JavaThread::current();
2620
2621 int deflated_count = 0;
2622 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors
2623 ObjectMonitor* free_tail_p = NULL;
2624 ObjectMonitor* saved_mid_in_use_p = NULL;
2625 elapsedTimer timer;
2626
2627 if (log_is_enabled(Info, monitorinflation)) {
2628 timer.start();
2629 }
2630
2631 if (is_global) {
2632 OM_PERFDATA_OP(MonExtant, set_value(OrderAccess::load_acquire(&g_om_in_use_count)));
2633 } else {
2634 OM_PERFDATA_OP(MonExtant, inc(OrderAccess::load_acquire(&target->om_in_use_count)));
2635 }
2636
2637 do {
2638 int local_deflated_count;
2639 if (is_global) {
2640 local_deflated_count = deflate_monitor_list_using_JT(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2641 } else {
2642 local_deflated_count = deflate_monitor_list_using_JT(&target->om_in_use_list, &target->om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2643 }
2644 deflated_count += local_deflated_count;
2645
2646 if (free_head_p != NULL) {
2647 // Move the deflated ObjectMonitors to the global free list.
2648 // No races on the working list so no need for load_acquire().
2649 guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count);
2650 // Note: The target thread can be doing an om_alloc() that
2651 // is trying to prepend an ObjectMonitor on its in-use list
2652 // at the same time that we have deflated the current in-use
2653 // list head and put it on the local free list. prepend_to_common()
2654 // will detect the race and retry which avoids list corruption,
2655 // but the next field in free_tail_p can flicker to marked
2656 // and then unmarked while prepend_to_common() is sorting it
2657 // all out.
2658 assert(unmarked_next(free_tail_p) == NULL, "must be NULL: _next_om="
2659 INTPTR_FORMAT, p2i(unmarked_next(free_tail_p)));
2660
2661 if (HandshakeAfterDeflateIdleMonitors) {
2662 prepend_list_to_g_wait_list(free_head_p, free_tail_p, local_deflated_count);
2663 } else {
2664 prepend_list_to_g_free_list(free_head_p, free_tail_p, local_deflated_count);
2665 }
2666
2667 OM_PERFDATA_OP(Deflations, inc(local_deflated_count));
2668 }
2669
2670 if (saved_mid_in_use_p != NULL) {
2671 // deflate_monitor_list_using_JT() detected a safepoint starting.
2672 timer.stop();
2673 {
2674 if (is_global) {
2675 log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint.");
2676 } else {
2677 log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target));
2678 }
2679 assert(SafepointSynchronize::is_synchronizing(), "sanity check");
2680 ThreadBlockInVM blocker(self);
2681 }
2682 // Prepare for another loop after the safepoint.
2683 free_head_p = NULL;
2684 free_tail_p = NULL;
2685 if (log_is_enabled(Info, monitorinflation)) {
2686 timer.start();
2687 }
2688 }
2689 } while (saved_mid_in_use_p != NULL);
2690 timer.stop();
2691
2692 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2693 LogStreamHandle(Info, monitorinflation) lsh_info;
2694 LogStream* ls = NULL;
2695 if (log_is_enabled(Debug, monitorinflation)) {
2696 ls = &lsh_debug;
2697 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2698 ls = &lsh_info;
2699 }
2700 if (ls != NULL) {
2701 if (is_global) {
2702 ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2703 } else {
2704 ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count);
2705 }
2706 }
2707 }
2708
2709 void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2710 // Report the cumulative time for deflating each thread's idle
2711 // monitors. Note: if the work is split among more than one
2712 // worker thread, then the reported time will likely be more
2713 // than a beginning to end measurement of the phase.
2714 // Note: AsyncDeflateIdleMonitors only deflates per-thread idle
2715 // monitors at a safepoint when a special deflation has been requested.
2716 log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d",
2717 counters->per_thread_times,
2718 OrderAccess::load_acquire(&counters->per_thread_scavenged));
2719
2720 bool needs_special_deflation = is_special_deflation_requested();
2721 if (!AsyncDeflateIdleMonitors || needs_special_deflation) {
2722 // AsyncDeflateIdleMonitors does not use these counters unless
2723 // there is a special deflation request.
2724
2725 OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged));
2726 OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation));
2727 }
2728
2729 if (log_is_enabled(Debug, monitorinflation)) {
2730 // exit_globals()'s call to audit_and_print_stats() is done
2731 // at the Info level.
2732 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2733 } else if (log_is_enabled(Info, monitorinflation)) {
2734 log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, "
2735 "g_om_free_count=%d, g_om_wait_count=%d",
2736 OrderAccess::load_acquire(&g_om_population),
2737 OrderAccess::load_acquire(&g_om_in_use_count),
2738 OrderAccess::load_acquire(&g_om_free_count),
2739 OrderAccess::load_acquire(&g_om_wait_count));
2740 }
2741
2742 ForceMonitorScavenge = 0; // Reset
2743 GVars.stw_random = os::random();
2744 GVars.stw_cycle++;
2745 if (needs_special_deflation) {
2746 set_is_special_deflation_requested(false); // special deflation is done
2747 }
2748 }
2749
2750 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
2751 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2752
2753 if (AsyncDeflateIdleMonitors && !is_special_deflation_requested()) {
2754 // Nothing to do if a special deflation has NOT been requested.
2755 return;
2756 }
2757
2758 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2759 ObjectMonitor* free_tail_p = NULL;
2760 elapsedTimer timer;
2761
2762 if (log_is_enabled(Info, safepoint, cleanup) ||
2763 log_is_enabled(Info, monitorinflation)) {
2764 timer.start();
2765 }
2766
2767 // Update n_in_circulation before om_in_use_count is updated by deflation.
2768 Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_circulation);
2769
2770 int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p);
2771 Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_use);
2772
2773 if (free_head_p != NULL) {
2774 // Move the deflated ObjectMonitors back to the global free list.
2775 // No races on the working list so no need for load_acquire().
2776 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2777 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2778 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2779 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2780 Atomic::add(deflated_count, &counters->n_scavenged);
2781 Atomic::add(deflated_count, &counters->per_thread_scavenged);
2782 }
2783
2784 timer.stop();
2785 // Safepoint logging cares about cumulative per_thread_times and
2786 // we'll capture most of the cost, but not the muxRelease() which
2787 // should be cheap.
2788 counters->per_thread_times += timer.seconds();
2789
2790 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2791 LogStreamHandle(Info, monitorinflation) lsh_info;
2792 LogStream* ls = NULL;
2793 if (log_is_enabled(Debug, monitorinflation)) {
2794 ls = &lsh_debug;
2795 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2796 ls = &lsh_info;
2797 }
2798 if (ls != NULL) {
2799 ls->print_cr("jt=" INTPTR_FORMAT ": deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(thread), timer.seconds(), deflated_count);
2800 }
2801 }
2802
2803 // Monitor cleanup on JavaThread::exit
2804
2805 // Iterate through monitor cache and attempt to release thread's monitors
2806 // Gives up on a particular monitor if an exception occurs, but continues
2807 // the overall iteration, swallowing the exception.
2808 class ReleaseJavaMonitorsClosure: public MonitorClosure {
2809 private:
2810 TRAPS;
2811
2812 public:
2813 ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {}
2814 void do_monitor(ObjectMonitor* mid) {
2815 if (mid->owner() == THREAD) {
2816 (void)mid->complete_exit(CHECK);
2817 }
2818 }
2819 };
2820
2821 // Release all inflated monitors owned by THREAD. Lightweight monitors are
2822 // ignored. This is meant to be called during JNI thread detach which assumes
2823 // all remaining monitors are heavyweight. All exceptions are swallowed.
2824 // Scanning the extant monitor list can be time consuming.
2825 // A simple optimization is to add a per-thread flag that indicates a thread
2826 // called jni_monitorenter() during its lifetime.
2827 //
2828 // Instead of No_Savepoint_Verifier it might be cheaper to
2829 // use an idiom of the form:
2830 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
2831 // <code that must not run at safepoint>
2832 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
2833 // Since the tests are extremely cheap we could leave them enabled
2834 // for normal product builds.
2835
2836 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
2837 assert(THREAD == JavaThread::current(), "must be current Java thread");
2838 NoSafepointVerifier nsv;
2839 ReleaseJavaMonitorsClosure rjmc(THREAD);
2840 ObjectSynchronizer::monitors_iterate(&rjmc);
2841 THREAD->clear_pending_exception();
2842 }
2843
2844 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
2845 switch (cause) {
2846 case inflate_cause_vm_internal: return "VM Internal";
2847 case inflate_cause_monitor_enter: return "Monitor Enter";
2848 case inflate_cause_wait: return "Monitor Wait";
2849 case inflate_cause_notify: return "Monitor Notify";
2850 case inflate_cause_hash_code: return "Monitor Hash Code";
2851 case inflate_cause_jni_enter: return "JNI Monitor Enter";
2852 case inflate_cause_jni_exit: return "JNI Monitor Exit";
2853 default:
2854 ShouldNotReachHere();
2855 }
2856 return "Unknown";
2857 }
2858
2859 //------------------------------------------------------------------------------
2860 // Debugging code
2861
2862 u_char* ObjectSynchronizer::get_gvars_addr() {
2863 return (u_char*)&GVars;
2864 }
2865
2866 u_char* ObjectSynchronizer::get_gvars_hc_sequence_addr() {
2867 return (u_char*)&GVars.hc_sequence;
2868 }
2869
2870 size_t ObjectSynchronizer::get_gvars_size() {
2871 return sizeof(SharedGlobals);
2872 }
2873
2874 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
2875 return (u_char*)&GVars.stw_random;
2876 }
2877
2878 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
2879 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
2880
2881 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2882 LogStreamHandle(Info, monitorinflation) lsh_info;
2883 LogStreamHandle(Trace, monitorinflation) lsh_trace;
2884 LogStream* ls = NULL;
2885 if (log_is_enabled(Trace, monitorinflation)) {
2886 ls = &lsh_trace;
2887 } else if (log_is_enabled(Debug, monitorinflation)) {
2888 ls = &lsh_debug;
2889 } else if (log_is_enabled(Info, monitorinflation)) {
2890 ls = &lsh_info;
2891 }
2892 assert(ls != NULL, "sanity check");
2893
2894 // Log counts for the global and per-thread monitor lists:
2895 int chk_om_population = log_monitor_list_counts(ls);
2896 int error_cnt = 0;
2897
2898 ls->print_cr("Checking global lists:");
2899
2900 // Check g_om_population:
2901 if (OrderAccess::load_acquire(&g_om_population) == chk_om_population) {
2902 ls->print_cr("g_om_population=%d equals chk_om_population=%d",
2903 OrderAccess::load_acquire(&g_om_population),
2904 chk_om_population);
2905 } else {
2906 // With lock free access to the monitor lists, it is possible for
2907 // log_monitor_list_counts() to return a value that doesn't match
2908 // g_om_population. So far a higher value has been seen in testing
2909 // so something is being double counted by log_monitor_list_counts().
2910 ls->print_cr("WARNING: g_om_population=%d is not equal to "
2911 "chk_om_population=%d",
2912 OrderAccess::load_acquire(&g_om_population),
2913 chk_om_population);
2914 }
2915
2916 // Check g_om_in_use_list and g_om_in_use_count:
2917 chk_global_in_use_list_and_count(ls, &error_cnt);
2918
2919 // Check g_free_list and g_om_free_count:
2920 chk_global_free_list_and_count(ls, &error_cnt);
2921
2922 if (HandshakeAfterDeflateIdleMonitors) {
2923 // Check g_wait_list and g_om_wait_count:
2924 chk_global_wait_list_and_count(ls, &error_cnt);
2925 }
2926
2927 ls->print_cr("Checking per-thread lists:");
2928
2929 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2930 // Check om_in_use_list and om_in_use_count:
2931 chk_per_thread_in_use_list_and_count(jt, ls, &error_cnt);
2932
2933 // Check om_free_list and om_free_count:
2934 chk_per_thread_free_list_and_count(jt, ls, &error_cnt);
2935 }
2936
2937 if (error_cnt == 0) {
2938 ls->print_cr("No errors found in monitor list checks.");
2939 } else {
2940 log_error(monitorinflation)("found monitor list errors: error_cnt=%d", error_cnt);
2941 }
2942
2943 if ((on_exit && log_is_enabled(Info, monitorinflation)) ||
2944 (!on_exit && log_is_enabled(Trace, monitorinflation))) {
2945 // When exiting this log output is at the Info level. When called
2946 // at a safepoint, this log output is at the Trace level since
2947 // there can be a lot of it.
2948 log_in_use_monitor_details(ls);
2949 }
2950
2951 ls->flush();
2952
2953 guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
2954 }
2955
2956 // Check a free monitor entry; log any errors.
2957 void ObjectSynchronizer::chk_free_entry(JavaThread* jt, ObjectMonitor* n,
2958 outputStream * out, int *error_cnt_p) {
2959 stringStream ss;
2960 if (n->is_busy()) {
2961 if (jt != NULL) {
2962 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2963 ": free per-thread monitor must not be busy: %s", p2i(jt),
2964 p2i(n), n->is_busy_to_string(&ss));
2965 } else {
2966 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2967 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
2968 }
2969 *error_cnt_p = *error_cnt_p + 1;
2970 }
2971 if (n->header().value() != 0) {
2972 if (jt != NULL) {
2973 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2974 ": free per-thread monitor must have NULL _header "
2975 "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n),
2976 n->header().value());
2977 *error_cnt_p = *error_cnt_p + 1;
2978 } else if (!AsyncDeflateIdleMonitors) {
2979 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2980 "must have NULL _header field: _header=" INTPTR_FORMAT,
2981 p2i(n), n->header().value());
2982 *error_cnt_p = *error_cnt_p + 1;
2983 }
2984 }
2985 if (n->object() != NULL) {
2986 if (jt != NULL) {
2987 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2988 ": free per-thread monitor must have NULL _object "
2989 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
2990 p2i(n->object()));
2991 } else {
2992 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2993 "must have NULL _object field: _object=" INTPTR_FORMAT,
2994 p2i(n), p2i(n->object()));
2995 }
2996 *error_cnt_p = *error_cnt_p + 1;
2997 }
2998 }
2999
3000 // Check the global free list and count; log the results of the checks.
3001 void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out,
3002 int *error_cnt_p) {
3003 int chk_om_free_count = 0;
3004 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_free_list); n != NULL; n = unmarked_next(n)) {
3005 chk_free_entry(NULL /* jt */, n, out, error_cnt_p);
3006 chk_om_free_count++;
3007 }
3008 if (OrderAccess::load_acquire(&g_om_free_count) == chk_om_free_count) {
3009 out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d",
3010 OrderAccess::load_acquire(&g_om_free_count),
3011 chk_om_free_count);
3012 } else {
3013 // With lock free access to g_free_list, it is possible for an
3014 // ObjectMonitor to be prepended to g_free_list after we started
3015 // calculating chk_om_free_count so g_om_free_count may not
3016 // match anymore.
3017 out->print_cr("WARNING: g_om_free_count=%d is not equal to "
3018 "chk_om_free_count=%d",
3019 OrderAccess::load_acquire(&g_om_free_count),
3020 chk_om_free_count);
3021 }
3022 }
3023
3024 // Check the global wait list and count; log the results of the checks.
3025 void ObjectSynchronizer::chk_global_wait_list_and_count(outputStream * out,
3026 int *error_cnt_p) {
3027 int chk_om_wait_count = 0;
3028 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_wait_list); n != NULL; n = unmarked_next(n)) {
3029 // Rules for g_wait_list are the same as of g_free_list:
3030 chk_free_entry(NULL /* jt */, n, out, error_cnt_p);
3031 chk_om_wait_count++;
3032 }
3033 if (OrderAccess::load_acquire(&g_om_wait_count) == chk_om_wait_count) {
3034 out->print_cr("g_om_wait_count=%d equals chk_om_wait_count=%d",
3035 OrderAccess::load_acquire(&g_om_wait_count),
3036 chk_om_wait_count);
3037 } else {
3038 out->print_cr("ERROR: g_om_wait_count=%d is not equal to "
3039 "chk_om_wait_count=%d",
3040 OrderAccess::load_acquire(&g_om_wait_count),
3041 chk_om_wait_count);
3042 *error_cnt_p = *error_cnt_p + 1;
3043 }
3044 }
3045
3046 // Check the global in-use list and count; log the results of the checks.
3047 void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out,
3048 int *error_cnt_p) {
3049 int chk_om_in_use_count = 0;
3050 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) {
3051 chk_in_use_entry(NULL /* jt */, n, out, error_cnt_p);
3052 chk_om_in_use_count++;
3053 }
3054 if (OrderAccess::load_acquire(&g_om_in_use_count) == chk_om_in_use_count) {
3055 out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d",
3056 OrderAccess::load_acquire(&g_om_in_use_count),
3057 chk_om_in_use_count);
3058 } else {
3059 // With lock free access to the monitor lists, it is possible for
3060 // an exiting JavaThread to put its in-use ObjectMonitors on the
3061 // global in-use list after chk_om_in_use_count is calculated above.
3062 out->print_cr("WARNING: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d",
3063 OrderAccess::load_acquire(&g_om_in_use_count),
3064 chk_om_in_use_count);
3065 }
3066 }
3067
3068 // Check an in-use monitor entry; log any errors.
3069 void ObjectSynchronizer::chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,
3070 outputStream * out, int *error_cnt_p) {
3071 if (n->header().value() == 0) {
3072 if (jt != NULL) {
3073 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3074 ": in-use per-thread monitor must have non-NULL _header "
3075 "field.", p2i(jt), p2i(n));
3076 } else {
3077 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
3078 "must have non-NULL _header field.", p2i(n));
3079 }
3080 *error_cnt_p = *error_cnt_p + 1;
3081 }
3082 if (n->object() == NULL) {
3083 if (jt != NULL) {
3084 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3085 ": in-use per-thread monitor must have non-NULL _object "
3086 "field.", p2i(jt), p2i(n));
3087 } else {
3088 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
3089 "must have non-NULL _object field.", p2i(n));
3090 }
3091 *error_cnt_p = *error_cnt_p + 1;
3092 }
3093 const oop obj = (oop)n->object();
3094 const markWord mark = obj->mark();
3095 if (!mark.has_monitor()) {
3096 if (jt != NULL) {
3097 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3098 ": in-use per-thread monitor's object does not think "
3099 "it has a monitor: obj=" INTPTR_FORMAT ", mark="
3100 INTPTR_FORMAT, p2i(jt), p2i(n), p2i(obj), mark.value());
3101 } else {
3102 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
3103 "monitor's object does not think it has a monitor: obj="
3104 INTPTR_FORMAT ", mark=" INTPTR_FORMAT, p2i(n),
3105 p2i(obj), mark.value());
3106 }
3107 *error_cnt_p = *error_cnt_p + 1;
3108 }
3109 ObjectMonitor* const obj_mon = mark.monitor();
3110 if (n != obj_mon) {
3111 if (jt != NULL) {
3112 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3113 ": in-use per-thread monitor's object does not refer "
3114 "to the same monitor: obj=" INTPTR_FORMAT ", mark="
3115 INTPTR_FORMAT ", obj_mon=" INTPTR_FORMAT, p2i(jt),
3116 p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3117 } else {
3118 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
3119 "monitor's object does not refer to the same monitor: obj="
3120 INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
3121 INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3122 }
3123 *error_cnt_p = *error_cnt_p + 1;
3124 }
3125 }
3126
3127 // Check the thread's free list and count; log the results of the checks.
3128 void ObjectSynchronizer::chk_per_thread_free_list_and_count(JavaThread *jt,
3129 outputStream * out,
3130 int *error_cnt_p) {
3131 int chk_om_free_count = 0;
3132 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_free_list); n != NULL; n = unmarked_next(n)) {
3133 chk_free_entry(jt, n, out, error_cnt_p);
3134 chk_om_free_count++;
3135 }
3136 if (OrderAccess::load_acquire(&jt->om_free_count) == chk_om_free_count) {
3137 out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals "
3138 "chk_om_free_count=%d", p2i(jt),
3139 OrderAccess::load_acquire(&jt->om_free_count),
3140 chk_om_free_count);
3141 } else {
3142 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not "
3143 "equal to chk_om_free_count=%d", p2i(jt),
3144 OrderAccess::load_acquire(&jt->om_free_count),
3145 chk_om_free_count);
3146 *error_cnt_p = *error_cnt_p + 1;
3147 }
3148 }
3149
3150 // Check the thread's in-use list and count; log the results of the checks.
3151 void ObjectSynchronizer::chk_per_thread_in_use_list_and_count(JavaThread *jt,
3152 outputStream * out,
3153 int *error_cnt_p) {
3154 int chk_om_in_use_count = 0;
3155 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) {
3156 chk_in_use_entry(jt, n, out, error_cnt_p);
3157 chk_om_in_use_count++;
3158 }
3159 if (OrderAccess::load_acquire(&jt->om_in_use_count) == chk_om_in_use_count) {
3160 out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals "
3161 "chk_om_in_use_count=%d", p2i(jt),
3162 OrderAccess::load_acquire(&jt->om_in_use_count),
3163 chk_om_in_use_count);
3164 } else {
3165 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not "
3166 "equal to chk_om_in_use_count=%d", p2i(jt),
3167 OrderAccess::load_acquire(&jt->om_in_use_count),
3168 chk_om_in_use_count);
3169 *error_cnt_p = *error_cnt_p + 1;
3170 }
3171 }
3172
3173 // Log details about ObjectMonitors on the in-use lists. The 'BHL'
3174 // flags indicate why the entry is in-use, 'object' and 'object type'
3175 // indicate the associated object and its type.
3176 void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out) {
3177 stringStream ss;
3178 if (OrderAccess::load_acquire(&g_om_in_use_count) > 0) {
3179 out->print_cr("In-use global monitor info:");
3180 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3181 out->print_cr("%18s %s %7s %18s %18s",
3182 "monitor", "BHL", "ref_cnt", "object", "object type");
3183 out->print_cr("================== === ======= ================== ==================");
3184 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) {
3185 const oop obj = (oop) n->object();
3186 const markWord mark = n->header();
3187 ResourceMark rm;
3188 out->print(INTPTR_FORMAT " %d%d%d %7d " INTPTR_FORMAT " %s",
3189 p2i(n), n->is_busy() != 0, mark.hash() != 0,
3190 n->owner() != NULL, (int)n->ref_count(), p2i(obj),
3191 obj->klass()->external_name());
3192 if (n->is_busy() != 0) {
3193 out->print(" (%s)", n->is_busy_to_string(&ss));
3194 ss.reset();
3195 }
3196 out->cr();
3197 }
3198 }
3199
3200 out->print_cr("In-use per-thread monitor info:");
3201 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3202 out->print_cr("%18s %18s %s %7s %18s %18s",
3203 "jt", "monitor", "BHL", "ref_cnt", "object", "object type");
3204 out->print_cr("================== ================== === ======= ================== ==================");
3205 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3206 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) {
3207 const oop obj = (oop) n->object();
3208 const markWord mark = n->header();
3209 ResourceMark rm;
3210 out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d %7d "
3211 INTPTR_FORMAT " %s", p2i(jt), p2i(n), n->is_busy() != 0,
3212 mark.hash() != 0, n->owner() != NULL, (int)n->ref_count(),
3213 p2i(obj), obj->klass()->external_name());
3214 if (n->is_busy() != 0) {
3215 out->print(" (%s)", n->is_busy_to_string(&ss));
3216 ss.reset();
3217 }
3218 out->cr();
3219 }
3220 }
3221
3222 out->flush();
3223 }
3224
3225 // Log counts for the global and per-thread monitor lists and return
3226 // the population count.
3227 int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) {
3228 int pop_count = 0;
3229 out->print_cr("%18s %10s %10s %10s %10s",
3230 "Global Lists:", "InUse", "Free", "Wait", "Total");
3231 out->print_cr("================== ========== ========== ========== ==========");
3232 out->print_cr("%18s %10d %10d %10d %10d", "",
3233 OrderAccess::load_acquire(&g_om_in_use_count),
3234 OrderAccess::load_acquire(&g_om_free_count),
3235 OrderAccess::load_acquire(&g_om_wait_count),
3236 OrderAccess::load_acquire(&g_om_population));
3237 pop_count += OrderAccess::load_acquire(&g_om_in_use_count) +
3238 OrderAccess::load_acquire(&g_om_free_count);
3239 if (HandshakeAfterDeflateIdleMonitors) {
3240 pop_count += OrderAccess::load_acquire(&g_om_wait_count);
3241 }
3242
3243 out->print_cr("%18s %10s %10s %10s",
3244 "Per-Thread Lists:", "InUse", "Free", "Provision");
3245 out->print_cr("================== ========== ========== ==========");
3246
3247 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3248 out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt),
3249 OrderAccess::load_acquire(&jt->om_in_use_count),
3250 OrderAccess::load_acquire(&jt->om_free_count),
3251 jt->om_free_provision);
3252 pop_count += OrderAccess::load_acquire(&jt->om_in_use_count) +
3253 OrderAccess::load_acquire(&jt->om_free_count);
3254 }
3255 return pop_count;
3256 }
3257
3258 #ifndef PRODUCT
3259
3260 // Check if monitor belongs to the monitor cache
3261 // The list is grow-only so it's *relatively* safe to traverse
3262 // the list of extant blocks without taking a lock.
3263
3264 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
3265 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
3266 while (block != NULL) {
3267 assert(block->object() == CHAINMARKER, "must be a block header");
3268 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
3269 address mon = (address)monitor;
3270 address blk = (address)block;
3271 size_t diff = mon - blk;
3272 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
3273 return 1;
3274 }
3275 // unmarked_next() is not needed with g_block_list (no next field marking).
3276 block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om);
3277 }
3278 return 0;
3279 }
3280
3281 #endif