1 /*
2 * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "logging/log.hpp"
28 #include "logging/logStream.hpp"
29 #include "jfr/jfrEvents.hpp"
30 #include "memory/allocation.inline.hpp"
31 #include "memory/metaspaceShared.hpp"
32 #include "memory/padded.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/markWord.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "runtime/atomic.hpp"
38 #include "runtime/biasedLocking.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/interfaceSupport.inline.hpp"
41 #include "runtime/mutexLocker.hpp"
42 #include "runtime/objectMonitor.hpp"
43 #include "runtime/objectMonitor.inline.hpp"
44 #include "runtime/osThread.hpp"
45 #include "runtime/safepointVerifiers.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "runtime/synchronizer.hpp"
49 #include "runtime/thread.inline.hpp"
50 #include "runtime/timer.hpp"
51 #include "runtime/vframe.hpp"
52 #include "runtime/vmThread.hpp"
53 #include "utilities/align.hpp"
54 #include "utilities/dtrace.hpp"
55 #include "utilities/events.hpp"
56 #include "utilities/preserveException.hpp"
57
58 // The "core" versions of monitor enter and exit reside in this file.
59 // The interpreter and compilers contain specialized transliterated
60 // variants of the enter-exit fast-path operations. See i486.ad fast_lock(),
61 // for instance. If you make changes here, make sure to modify the
62 // interpreter, and both C1 and C2 fast-path inline locking code emission.
63 //
64 // -----------------------------------------------------------------------------
65
66 #ifdef DTRACE_ENABLED
67
68 // Only bother with this argument setup if dtrace is available
69 // TODO-FIXME: probes should not fire when caller is _blocked. assert() accordingly.
70
71 #define DTRACE_MONITOR_PROBE_COMMON(obj, thread) \
72 char* bytes = NULL; \
73 int len = 0; \
74 jlong jtid = SharedRuntime::get_java_tid(thread); \
75 Symbol* klassname = ((oop)(obj))->klass()->name(); \
76 if (klassname != NULL) { \
77 bytes = (char*)klassname->bytes(); \
78 len = klassname->utf8_length(); \
79 }
80
81 #define DTRACE_MONITOR_WAIT_PROBE(monitor, obj, thread, millis) \
82 { \
83 if (DTraceMonitorProbes) { \
84 DTRACE_MONITOR_PROBE_COMMON(obj, thread); \
85 HOTSPOT_MONITOR_WAIT(jtid, \
86 (uintptr_t)(monitor), bytes, len, (millis)); \
87 } \
88 }
89
90 #define HOTSPOT_MONITOR_PROBE_notify HOTSPOT_MONITOR_NOTIFY
91 #define HOTSPOT_MONITOR_PROBE_notifyAll HOTSPOT_MONITOR_NOTIFYALL
92 #define HOTSPOT_MONITOR_PROBE_waited HOTSPOT_MONITOR_WAITED
93
94 #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread) \
95 { \
96 if (DTraceMonitorProbes) { \
97 DTRACE_MONITOR_PROBE_COMMON(obj, thread); \
98 HOTSPOT_MONITOR_PROBE_##probe(jtid, /* probe = waited */ \
99 (uintptr_t)(monitor), bytes, len); \
100 } \
101 }
102
103 #else // ndef DTRACE_ENABLED
104
105 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon) {;}
106 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon) {;}
107
108 #endif // ndef DTRACE_ENABLED
109
110 // This exists only as a workaround of dtrace bug 6254741
111 int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
112 DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
113 return 0;
114 }
115
116 #define NINFLATIONLOCKS 256
117 static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];
118
119 // global list of blocks of monitors
120 PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL;
121 bool volatile ObjectSynchronizer::_is_async_deflation_requested = false;
122 bool volatile ObjectSynchronizer::_is_special_deflation_requested = false;
123 jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0;
124
125 // Global ObjectMonitor free list. Newly allocated and deflated
126 // ObjectMonitors are prepended here.
127 static ObjectMonitor* volatile g_free_list = NULL;
128 // Global ObjectMonitor in-use list. When a JavaThread is exiting,
129 // ObjectMonitors on its per-thread in-use list are prepended here.
130 static ObjectMonitor* volatile g_om_in_use_list = NULL;
131
132 static volatile int g_om_free_count = 0; // # on g_free_list
133 static volatile int g_om_in_use_count = 0; // # on g_om_in_use_list
134 static volatile int g_om_population = 0; // # Extant -- in circulation
135
136 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
137
138
139 // =====================> List Management functions
140
141 // Return true if the ObjectMonitor's next field is marked.
142 // Otherwise returns false.
143 static bool is_next_marked(ObjectMonitor* om) {
144 return ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & 0x1) != 0;
145 }
146
147 // Mark an ObjectMonitor* and return it. Note: the om parameter
148 // may or may not have been marked originally.
149 static ObjectMonitor* mark_om_ptr(ObjectMonitor* om) {
150 return (ObjectMonitor*)((intptr_t)om | 0x1);
151 }
152
153 // Mark the next field in an ObjectMonitor. If marking was successful,
154 // then the unmarked next field is returned via parameter and true is
155 // returned. Otherwise false is returned.
156 static bool mark_next(ObjectMonitor* om, ObjectMonitor** next_p) {
157 // Get current next field without any marking value.
158 ObjectMonitor* next = (ObjectMonitor*)
159 ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1);
160 if (Atomic::cmpxchg(mark_om_ptr(next), &om->_next_om, next) != next) {
161 return false; // Could not mark the next field or it was already marked.
162 }
163 *next_p = next;
164 return true;
165 }
166
167 // Loop until we mark the next field in an ObjectMonitor. The unmarked
168 // next field is returned.
169 static ObjectMonitor* mark_next_loop(ObjectMonitor* om) {
170 ObjectMonitor* next;
171 while (true) {
172 if (mark_next(om, &next)) {
173 // Marked om's next field so return the unmarked value.
174 return next;
175 }
176 }
177 }
178
179 // Set the next field in an ObjectMonitor to the specified value.
180 // The caller of set_next() must be the same thread that marked the
181 // ObjectMonitor.
182 static void set_next(ObjectMonitor* om, ObjectMonitor* value) {
183 OrderAccess::release_store(&om->_next_om, value);
184 }
185
186 // Mark the next field in the list head ObjectMonitor. If marking was
187 // successful, then the mid and the unmarked next field are returned
188 // via parameter and true is returned. Otherwise false is returned.
189 static bool mark_list_head(ObjectMonitor* volatile * list_p,
190 ObjectMonitor** mid_p, ObjectMonitor** next_p) {
191 while (true) {
192 ObjectMonitor* mid = OrderAccess::load_acquire(list_p);
193 if (mid == NULL) {
194 return false; // The list is empty so nothing to mark.
195 }
196 if (mark_next(mid, next_p)) {
197 if (OrderAccess::load_acquire(list_p) != mid) {
198 // The list head changed so we have to retry.
199 set_next(mid, *next_p); // unmark mid
200 continue;
201 }
202 // We marked next field to guard against races.
203 *mid_p = mid;
204 return true;
205 }
206 }
207 }
208
209 // Return the unmarked next field in an ObjectMonitor. Note: the next
210 // field may or may not have been marked originally.
211 static ObjectMonitor* unmarked_next(ObjectMonitor* om) {
212 return (ObjectMonitor*)((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1);
213 }
214
215 #if 0
216 // XXX - this is unused
217 // Unmark the next field in an ObjectMonitor. Requires that the next
218 // field be marked.
219 static void unmark_next(ObjectMonitor* om) {
220 ADIM_guarantee(is_next_marked(om), "next field must be marked: next=" INTPTR_FORMAT, p2i(om->_next_om));
221
222 ObjectMonitor* next = unmarked_next(om);
223 set_next(om, next);
224 }
225 #endif
226
227 volatile int visit_counter = 42;
228 static void chk_for_list_loop(ObjectMonitor* list, int count) {
229 if (!CheckMonitorLists) {
230 return;
231 }
232 int l_visit_counter = Atomic::add(1, &visit_counter);
233 int l_count = 0;
234 ObjectMonitor* prev = NULL;
235 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
236 if (mid->visit_marker == l_visit_counter) {
237 log_error(monitorinflation)("ERROR: prev=" INTPTR_FORMAT ", l_count=%d"
238 " refers to an ObjectMonitor that has"
239 " already been visited: mid=" INTPTR_FORMAT,
240 p2i(prev), l_count, p2i(mid));
241 fatal("list=" INTPTR_FORMAT " of %d items has a loop.", p2i(list), count);
242 }
243 mid->visit_marker = l_visit_counter;
244 prev = mid;
245 if (++l_count > count + 1024 * 1024) {
246 fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d",
247 p2i(list), count, l_count);
248 }
249 }
250 }
251
252 static void chk_om_not_on_list(ObjectMonitor* om, ObjectMonitor* list, int count) {
253 if (!CheckMonitorLists) {
254 return;
255 }
256 guarantee(list != om, "ERROR: om=" INTPTR_FORMAT " must not be head of the "
257 "list=" INTPTR_FORMAT ", count=%d", p2i(om), p2i(list), count);
258 int l_count = 0;
259 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
260 if (unmarked_next(mid) == om) {
261 log_error(monitorinflation)("ERROR: mid=" INTPTR_FORMAT ", l_count=%d"
262 " next_om refers to om=" INTPTR_FORMAT,
263 p2i(mid), l_count, p2i(om));
264 fatal("list=" INTPTR_FORMAT " of %d items has bad next_om value.",
265 p2i(list), count);
266 }
267 if (++l_count > count + 1024 * 1024) {
268 fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d",
269 p2i(list), count, l_count);
270 }
271 }
272 }
273
274 static void chk_om_elems_not_on_list(ObjectMonitor* elems, int elems_count,
275 ObjectMonitor* list, int list_count) {
276 if (!CheckMonitorLists) {
277 return;
278 }
279 chk_for_list_loop(elems, elems_count);
280 for (ObjectMonitor* mid = elems; mid != NULL; mid = unmarked_next(mid)) {
281 chk_om_not_on_list(mid, list, list_count);
282 }
283 }
284
285 // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is
286 // the last ObjectMonitor in the list and there are 'count' on the list.
287 // Also updates the specified *count_p.
288 static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail,
289 int count, ObjectMonitor* volatile* list_p,
290 volatile int* count_p) {
291 chk_for_list_loop(OrderAccess::load_acquire(list_p),
292 OrderAccess::load_acquire(count_p));
293 chk_om_elems_not_on_list(list, count, OrderAccess::load_acquire(list_p),
294 OrderAccess::load_acquire(count_p));
295 while (true) {
296 ObjectMonitor* cur = OrderAccess::load_acquire(list_p);
297 // Prepend list to *list_p.
298 ObjectMonitor* next = NULL;
299 if (!mark_next(tail, &next)) {
300 continue; // failed to mark next field so try it all again
301 }
302 set_next(tail, cur); // tail now points to cur (and unmarks tail)
303 if (cur == NULL) {
304 // No potential race with takers or other prependers since
305 // *list_p is empty.
306 if (Atomic::cmpxchg(list, list_p, cur) == cur) {
307 // Successfully switched *list_p to the list value.
308 Atomic::add(count, count_p);
309 break;
310 }
311 // Implied else: try it all again
312 } else {
313 // Try to mark next field to guard against races:
314 if (!mark_next(cur, &next)) {
315 continue; // failed to mark next field so try it all again
316 }
317 // We marked the next field so try to switch *list_p to the list value.
318 if (Atomic::cmpxchg(list, list_p, cur) != cur) {
319 // The list head has changed so unmark the next field and try again:
320 set_next(cur, next);
321 continue;
322 }
323 Atomic::add(count, count_p);
324 set_next(cur, next); // unmark next field
325 break;
326 }
327 }
328 }
329
330 // Prepend a newly allocated block of ObjectMonitors to g_block_list and
331 // g_free_list. Also updates g_om_population and g_om_free_count.
332 void ObjectSynchronizer::prepend_block_to_lists(PaddedObjectMonitor* new_blk) {
333 // First we handle g_block_list:
334 while (true) {
335 PaddedObjectMonitor* cur = OrderAccess::load_acquire(&g_block_list);
336 // Prepend new_blk to g_block_list. The first ObjectMonitor in
337 // a block is reserved for use as linkage to the next block.
338 OrderAccess::release_store(&new_blk[0]._next_om, cur);
339 if (Atomic::cmpxchg(new_blk, &g_block_list, cur) == cur) {
340 // Successfully switched g_block_list to the new_blk value.
341 Atomic::add(_BLOCKSIZE - 1, &g_om_population);
342 break;
343 }
344 // Implied else: try it all again
345 }
346
347 // Second we handle g_free_list:
348 prepend_list_to_common(new_blk + 1, &new_blk[_BLOCKSIZE - 1], _BLOCKSIZE - 1,
349 &g_free_list, &g_om_free_count);
350 }
351
352 // Prepend a list of ObjectMonitors to g_free_list. 'tail' is the last
353 // ObjectMonitor in the list and there are 'count' on the list. Also
354 // updates g_om_free_count.
355 static void prepend_list_to_g_free_list(ObjectMonitor* list,
356 ObjectMonitor* tail, int count) {
357 prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count);
358 }
359
360 // Prepend a list of ObjectMonitors to g_om_in_use_list. 'tail' is the last
361 // ObjectMonitor in the list and there are 'count' on the list. Also
362 // updates g_om_in_use_list.
363 static void prepend_list_to_g_om_in_use_list(ObjectMonitor* list,
364 ObjectMonitor* tail, int count) {
365 prepend_list_to_common(list, tail, count, &g_om_in_use_list, &g_om_in_use_count);
366 }
367
368 // Prepend an ObjectMonitor to the specified list. Also updates
369 // the specified counter.
370 static void prepend_to_common(ObjectMonitor* m, ObjectMonitor* volatile * list_p,
371 int volatile * count_p) {
372 chk_for_list_loop(OrderAccess::load_acquire(list_p),
373 OrderAccess::load_acquire(count_p));
374 chk_om_not_on_list(m, OrderAccess::load_acquire(list_p),
375 OrderAccess::load_acquire(count_p));
376
377 while (true) {
378 ObjectMonitor* cur = OrderAccess::load_acquire(list_p);
379 // Prepend ObjectMonitor to *list_p.
380 ObjectMonitor* next = NULL;
381 if (!mark_next(m, &next)) {
382 continue; // failed to mark next field so try it all again
383 }
384 set_next(m, cur); // m now points to cur (and unmarks m)
385 if (cur == NULL) {
386 // No potential race with other prependers since *list_p is empty.
387 if (Atomic::cmpxchg(m, list_p, cur) == cur) {
388 // Successfully switched *list_p to 'm'.
389 Atomic::inc(count_p);
390 break;
391 }
392 // Implied else: try it all again
393 } else {
394 // Try to mark next field to guard against races:
395 if (!mark_next(cur, &next)) {
396 continue; // failed to mark next field so try it all again
397 }
398 // We marked the next field so try to switch *list_p to 'm'.
399 if (Atomic::cmpxchg(m, list_p, cur) != cur) {
400 // The list head has changed so unmark the next field and try again:
401 set_next(cur, next);
402 continue;
403 }
404 Atomic::inc(count_p);
405 set_next(cur, next); // unmark next field
406 break;
407 }
408 }
409 }
410
411 // Prepend an ObjectMonitor to a per-thread om_free_list.
412 // Also updates the per-thread om_free_count.
413 static void prepend_to_om_free_list(Thread* self, ObjectMonitor* m) {
414 prepend_to_common(m, &self->om_free_list, &self->om_free_count);
415 }
416
417 // Prepend an ObjectMonitor to a per-thread om_in_use_list.
418 // Also updates the per-thread om_in_use_count.
419 static void prepend_to_om_in_use_list(Thread* self, ObjectMonitor* m) {
420 prepend_to_common(m, &self->om_in_use_list, &self->om_in_use_count);
421 }
422
423 // Take an ObjectMonitor from the start of the specified list. Also
424 // decrements the specified counter. Returns NULL if none are available.
425 static ObjectMonitor* take_from_start_of_common(ObjectMonitor* volatile * list_p,
426 int volatile * count_p) {
427 chk_for_list_loop(OrderAccess::load_acquire(list_p),
428 OrderAccess::load_acquire(count_p));
429
430 ObjectMonitor* next = NULL;
431 ObjectMonitor* take = NULL;
432 // Mark the list head to guard against A-B-A race:
433 if (!mark_list_head(list_p, &take, &next)) {
434 return NULL; // None are available.
435 }
436 // Switch marked list head to next (which unmarks the list head, but
437 // leaves take marked):
438 OrderAccess::release_store(list_p, next);
439 Atomic::dec(count_p);
440 // Unmark take, but leave the next value for any lagging list
441 // walkers. It will get cleaned up when take is prepended to
442 // the in-use list:
443 set_next(take, next);
444 return take;
445 }
446
447 // Take an ObjectMonitor from the start of the global free-list. Also
448 // updates g_om_free_count. Returns NULL if none are available.
449 static ObjectMonitor* take_from_start_of_g_free_list() {
450 return take_from_start_of_common(&g_free_list, &g_om_free_count);
451 }
452
453 // Take an ObjectMonitor from the start of a per-thread free-list.
454 // Also updates om_free_count. Returns NULL if none are available.
455 static ObjectMonitor* take_from_start_of_om_free_list(Thread* self) {
456 return take_from_start_of_common(&self->om_free_list, &self->om_free_count);
457 }
458
459
460 // =====================> Quick functions
461
462 // The quick_* forms are special fast-path variants used to improve
463 // performance. In the simplest case, a "quick_*" implementation could
464 // simply return false, in which case the caller will perform the necessary
465 // state transitions and call the slow-path form.
466 // The fast-path is designed to handle frequently arising cases in an efficient
467 // manner and is just a degenerate "optimistic" variant of the slow-path.
468 // returns true -- to indicate the call was satisfied.
469 // returns false -- to indicate the call needs the services of the slow-path.
470 // A no-loitering ordinance is in effect for code in the quick_* family
471 // operators: safepoints or indefinite blocking (blocking that might span a
472 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
473 // entry.
474 //
475 // Consider: An interesting optimization is to have the JIT recognize the
476 // following common idiom:
477 // synchronized (someobj) { .... ; notify(); }
478 // That is, we find a notify() or notifyAll() call that immediately precedes
479 // the monitorexit operation. In that case the JIT could fuse the operations
480 // into a single notifyAndExit() runtime primitive.
481
482 bool ObjectSynchronizer::quick_notify(oopDesc* obj, Thread* self, bool all) {
483 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
484 assert(self->is_Java_thread(), "invariant");
485 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
486 NoSafepointVerifier nsv;
487 if (obj == NULL) return false; // slow-path for invalid obj
488 const markWord mark = obj->mark();
489
490 if (mark.has_locker() && self->is_lock_owned((address)mark.locker())) {
491 // Degenerate notify
492 // stack-locked by caller so by definition the implied waitset is empty.
493 return true;
494 }
495
496 if (mark.has_monitor()) {
497 ObjectMonitor* const mon = mark.monitor();
498 assert(mon->object() == obj, "invariant");
499 if (mon->owner() != self) return false; // slow-path for IMS exception
500
501 if (mon->first_waiter() != NULL) {
502 // We have one or more waiters. Since this is an inflated monitor
503 // that we own, we can transfer one or more threads from the waitset
504 // to the entrylist here and now, avoiding the slow-path.
505 if (all) {
506 DTRACE_MONITOR_PROBE(notifyAll, mon, obj, self);
507 } else {
508 DTRACE_MONITOR_PROBE(notify, mon, obj, self);
509 }
510 int free_count = 0;
511 do {
512 mon->INotify(self);
513 ++free_count;
514 } while (mon->first_waiter() != NULL && all);
515 OM_PERFDATA_OP(Notifications, inc(free_count));
516 }
517 return true;
518 }
519
520 // biased locking and any other IMS exception states take the slow-path
521 return false;
522 }
523
524
525 // The LockNode emitted directly at the synchronization site would have
526 // been too big if it were to have included support for the cases of inflated
527 // recursive enter and exit, so they go here instead.
528 // Note that we can't safely call AsyncPrintJavaStack() from within
529 // quick_enter() as our thread state remains _in_Java.
530
531 bool ObjectSynchronizer::quick_enter(oop obj, Thread* self,
532 BasicLock * lock) {
533 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
534 assert(self->is_Java_thread(), "invariant");
535 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
536 NoSafepointVerifier nsv;
537 if (obj == NULL) return false; // Need to throw NPE
538
539 while (true) {
540 const markWord mark = obj->mark();
541
542 if (mark.has_monitor()) {
543 ObjectMonitorHandle omh;
544 if (!omh.save_om_ptr(obj, mark)) {
545 // Lost a race with async deflation so try again.
546 assert(AsyncDeflateIdleMonitors, "sanity check");
547 continue;
548 }
549 ObjectMonitor* const m = omh.om_ptr();
550 assert(m->object() == obj, "invariant");
551 Thread* const owner = (Thread *) m->_owner;
552
553 // Lock contention and Transactional Lock Elision (TLE) diagnostics
554 // and observability
555 // Case: light contention possibly amenable to TLE
556 // Case: TLE inimical operations such as nested/recursive synchronization
557
558 if (owner == self) {
559 m->_recursions++;
560 return true;
561 }
562
563 // This Java Monitor is inflated so obj's header will never be
564 // displaced to this thread's BasicLock. Make the displaced header
565 // non-NULL so this BasicLock is not seen as recursive nor as
566 // being locked. We do this unconditionally so that this thread's
567 // BasicLock cannot be mis-interpreted by any stack walkers. For
568 // performance reasons, stack walkers generally first check for
569 // Biased Locking in the object's header, the second check is for
570 // stack-locking in the object's header, the third check is for
571 // recursive stack-locking in the displaced header in the BasicLock,
572 // and last are the inflated Java Monitor (ObjectMonitor) checks.
573 lock->set_displaced_header(markWord::unused_mark());
574
575 if (owner == NULL && Atomic::replace_if_null(self, &(m->_owner))) {
576 assert(m->_recursions == 0, "invariant");
577 return true;
578 }
579
580 if (AsyncDeflateIdleMonitors &&
581 Atomic::cmpxchg(self, &m->_owner, DEFLATER_MARKER) == DEFLATER_MARKER) {
582 // The deflation protocol finished the first part (setting owner),
583 // but it failed the second part (making ref_count negative) and
584 // bailed. Or the ObjectMonitor was async deflated and reused.
585 // Acquired the monitor.
586 assert(m->_recursions == 0, "invariant");
587 return true;
588 }
589 }
590 break;
591 }
592
593 // Note that we could inflate in quick_enter.
594 // This is likely a useful optimization
595 // Critically, in quick_enter() we must not:
596 // -- perform bias revocation, or
597 // -- block indefinitely, or
598 // -- reach a safepoint
599
600 return false; // revert to slow-path
601 }
602
603 // -----------------------------------------------------------------------------
604 // Monitor Enter/Exit
605 // The interpreter and compiler assembly code tries to lock using the fast path
606 // of this algorithm. Make sure to update that code if the following function is
607 // changed. The implementation is extremely sensitive to race condition. Be careful.
608
609 void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, TRAPS) {
610 if (UseBiasedLocking) {
611 if (!SafepointSynchronize::is_at_safepoint()) {
612 BiasedLocking::revoke(obj, THREAD);
613 } else {
614 BiasedLocking::revoke_at_safepoint(obj);
615 }
616 }
617
618 markWord mark = obj->mark();
619 assert(!mark.has_bias_pattern(), "should not see bias pattern here");
620
621 if (mark.is_neutral()) {
622 // Anticipate successful CAS -- the ST of the displaced mark must
623 // be visible <= the ST performed by the CAS.
624 lock->set_displaced_header(mark);
625 if (mark == obj()->cas_set_mark(markWord::from_pointer(lock), mark)) {
626 return;
627 }
628 // Fall through to inflate() ...
629 } else if (mark.has_locker() &&
630 THREAD->is_lock_owned((address)mark.locker())) {
631 assert(lock != mark.locker(), "must not re-lock the same lock");
632 assert(lock != (BasicLock*)obj->mark().value(), "don't relock with same BasicLock");
633 lock->set_displaced_header(markWord::from_pointer(NULL));
634 return;
635 }
636
637 // The object header will never be displaced to this lock,
638 // so it does not matter what the value is, except that it
639 // must be non-zero to avoid looking like a re-entrant lock,
640 // and must not look locked either.
641 lock->set_displaced_header(markWord::unused_mark());
642 ObjectMonitorHandle omh;
643 inflate(&omh, THREAD, obj(), inflate_cause_monitor_enter);
644 omh.om_ptr()->enter(THREAD);
645 }
646
647 void ObjectSynchronizer::exit(oop object, BasicLock* lock, TRAPS) {
648 markWord mark = object->mark();
649 // We cannot check for Biased Locking if we are racing an inflation.
650 assert(mark == markWord::INFLATING() ||
651 !mark.has_bias_pattern(), "should not see bias pattern here");
652
653 markWord dhw = lock->displaced_header();
654 if (dhw.value() == 0) {
655 // If the displaced header is NULL, then this exit matches up with
656 // a recursive enter. No real work to do here except for diagnostics.
657 #ifndef PRODUCT
658 if (mark != markWord::INFLATING()) {
659 // Only do diagnostics if we are not racing an inflation. Simply
660 // exiting a recursive enter of a Java Monitor that is being
661 // inflated is safe; see the has_monitor() comment below.
662 assert(!mark.is_neutral(), "invariant");
663 assert(!mark.has_locker() ||
664 THREAD->is_lock_owned((address)mark.locker()), "invariant");
665 if (mark.has_monitor()) {
666 // The BasicLock's displaced_header is marked as a recursive
667 // enter and we have an inflated Java Monitor (ObjectMonitor).
668 // This is a special case where the Java Monitor was inflated
669 // after this thread entered the stack-lock recursively. When a
670 // Java Monitor is inflated, we cannot safely walk the Java
671 // Monitor owner's stack and update the BasicLocks because a
672 // Java Monitor can be asynchronously inflated by a thread that
673 // does not own the Java Monitor.
674 ObjectMonitor* m = mark.monitor();
675 assert(((oop)(m->object()))->mark() == mark, "invariant");
676 assert(m->is_entered(THREAD), "invariant");
677 }
678 }
679 #endif
680 return;
681 }
682
683 if (mark == markWord::from_pointer(lock)) {
684 // If the object is stack-locked by the current thread, try to
685 // swing the displaced header from the BasicLock back to the mark.
686 assert(dhw.is_neutral(), "invariant");
687 if (object->cas_set_mark(dhw, mark) == mark) {
688 return;
689 }
690 }
691
692 // We have to take the slow-path of possible inflation and then exit.
693 ObjectMonitorHandle omh;
694 inflate(&omh, THREAD, object, inflate_cause_vm_internal);
695 omh.om_ptr()->exit(true, THREAD);
696 }
697
698 // -----------------------------------------------------------------------------
699 // Class Loader support to workaround deadlocks on the class loader lock objects
700 // Also used by GC
701 // complete_exit()/reenter() are used to wait on a nested lock
702 // i.e. to give up an outer lock completely and then re-enter
703 // Used when holding nested locks - lock acquisition order: lock1 then lock2
704 // 1) complete_exit lock1 - saving recursion count
705 // 2) wait on lock2
706 // 3) when notified on lock2, unlock lock2
707 // 4) reenter lock1 with original recursion count
708 // 5) lock lock2
709 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
710 intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
711 if (UseBiasedLocking) {
712 BiasedLocking::revoke(obj, THREAD);
713 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
714 }
715
716 ObjectMonitorHandle omh;
717 inflate(&omh, THREAD, obj(), inflate_cause_vm_internal);
718 intptr_t ret_code = omh.om_ptr()->complete_exit(THREAD);
719 return ret_code;
720 }
721
722 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
723 void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) {
724 if (UseBiasedLocking) {
725 BiasedLocking::revoke(obj, THREAD);
726 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
727 }
728
729 ObjectMonitorHandle omh;
730 inflate(&omh, THREAD, obj(), inflate_cause_vm_internal);
731 omh.om_ptr()->reenter(recursion, THREAD);
732 }
733 // -----------------------------------------------------------------------------
734 // JNI locks on java objects
735 // NOTE: must use heavy weight monitor to handle jni monitor enter
736 void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) {
737 // the current locking is from JNI instead of Java code
738 if (UseBiasedLocking) {
739 BiasedLocking::revoke(obj, THREAD);
740 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
741 }
742 THREAD->set_current_pending_monitor_is_from_java(false);
743 ObjectMonitorHandle omh;
744 inflate(&omh, THREAD, obj(), inflate_cause_jni_enter);
745 omh.om_ptr()->enter(THREAD);
746 THREAD->set_current_pending_monitor_is_from_java(true);
747 }
748
749 // NOTE: must use heavy weight monitor to handle jni monitor exit
750 void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
751 if (UseBiasedLocking) {
752 Handle h_obj(THREAD, obj);
753 BiasedLocking::revoke(h_obj, THREAD);
754 obj = h_obj();
755 }
756 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
757
758 ObjectMonitorHandle omh;
759 inflate(&omh, THREAD, obj, inflate_cause_jni_exit);
760 ObjectMonitor* monitor = omh.om_ptr();
761 // If this thread has locked the object, exit the monitor. We
762 // intentionally do not use CHECK here because we must exit the
763 // monitor even if an exception is pending.
764 if (monitor->check_owner(THREAD)) {
765 monitor->exit(true, THREAD);
766 }
767 }
768
769 // -----------------------------------------------------------------------------
770 // Internal VM locks on java objects
771 // standard constructor, allows locking failures
772 ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool do_lock) {
773 _dolock = do_lock;
774 _thread = thread;
775 _thread->check_for_valid_safepoint_state();
776 _obj = obj;
777
778 if (_dolock) {
779 ObjectSynchronizer::enter(_obj, &_lock, _thread);
780 }
781 }
782
783 ObjectLocker::~ObjectLocker() {
784 if (_dolock) {
785 ObjectSynchronizer::exit(_obj(), &_lock, _thread);
786 }
787 }
788
789
790 // -----------------------------------------------------------------------------
791 // Wait/Notify/NotifyAll
792 // NOTE: must use heavy weight monitor to handle wait()
793 int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
794 if (UseBiasedLocking) {
795 BiasedLocking::revoke(obj, THREAD);
796 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
797 }
798 if (millis < 0) {
799 THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
800 }
801 ObjectMonitorHandle omh;
802 inflate(&omh, THREAD, obj(), inflate_cause_wait);
803 ObjectMonitor* monitor = omh.om_ptr();
804
805 DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);
806 monitor->wait(millis, true, THREAD);
807
808 // This dummy call is in place to get around dtrace bug 6254741. Once
809 // that's fixed we can uncomment the following line, remove the call
810 // and change this function back into a "void" func.
811 // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
812 int ret_code = dtrace_waited_probe(monitor, obj, THREAD);
813 return ret_code;
814 }
815
816 void ObjectSynchronizer::wait_uninterruptibly(Handle obj, jlong millis, TRAPS) {
817 if (UseBiasedLocking) {
818 BiasedLocking::revoke(obj, THREAD);
819 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
820 }
821 if (millis < 0) {
822 THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
823 }
824 ObjectMonitorHandle omh;
825 inflate(&omh, THREAD, obj(), inflate_cause_wait);
826 omh.om_ptr()->wait(millis, false, THREAD);
827 }
828
829 void ObjectSynchronizer::notify(Handle obj, TRAPS) {
830 if (UseBiasedLocking) {
831 BiasedLocking::revoke(obj, THREAD);
832 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
833 }
834
835 markWord mark = obj->mark();
836 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
837 return;
838 }
839 ObjectMonitorHandle omh;
840 inflate(&omh, THREAD, obj(), inflate_cause_notify);
841 omh.om_ptr()->notify(THREAD);
842 }
843
844 // NOTE: see comment of notify()
845 void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
846 if (UseBiasedLocking) {
847 BiasedLocking::revoke(obj, THREAD);
848 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
849 }
850
851 markWord mark = obj->mark();
852 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
853 return;
854 }
855 ObjectMonitorHandle omh;
856 inflate(&omh, THREAD, obj(), inflate_cause_notify);
857 omh.om_ptr()->notifyAll(THREAD);
858 }
859
860 // -----------------------------------------------------------------------------
861 // Hash Code handling
862 //
863 // Performance concern:
864 // OrderAccess::storestore() calls release() which at one time stored 0
865 // into the global volatile OrderAccess::dummy variable. This store was
866 // unnecessary for correctness. Many threads storing into a common location
867 // causes considerable cache migration or "sloshing" on large SMP systems.
868 // As such, I avoided using OrderAccess::storestore(). In some cases
869 // OrderAccess::fence() -- which incurs local latency on the executing
870 // processor -- is a better choice as it scales on SMP systems.
871 //
872 // See http://blogs.oracle.com/dave/entry/biased_locking_in_hotspot for
873 // a discussion of coherency costs. Note that all our current reference
874 // platforms provide strong ST-ST order, so the issue is moot on IA32,
875 // x64, and SPARC.
876 //
877 // As a general policy we use "volatile" to control compiler-based reordering
878 // and explicit fences (barriers) to control for architectural reordering
879 // performed by the CPU(s) or platform.
880
881 struct SharedGlobals {
882 char _pad_prefix[OM_CACHE_LINE_SIZE];
883 // These are highly shared mostly-read variables.
884 // To avoid false-sharing they need to be the sole occupants of a cache line.
885 volatile int stw_random;
886 volatile int stw_cycle;
887 DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(volatile int) * 2);
888 // Hot RW variable -- Sequester to avoid false-sharing
889 volatile int hc_sequence;
890 DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile int));
891 };
892
893 static SharedGlobals GVars;
894 static int MonitorScavengeThreshold = 1000000;
895 static volatile int ForceMonitorScavenge = 0; // Scavenge required and pending
896
897 static markWord read_stable_mark(oop obj) {
898 markWord mark = obj->mark();
899 if (!mark.is_being_inflated()) {
900 return mark; // normal fast-path return
901 }
902
903 int its = 0;
904 for (;;) {
905 markWord mark = obj->mark();
906 if (!mark.is_being_inflated()) {
907 return mark; // normal fast-path return
908 }
909
910 // The object is being inflated by some other thread.
911 // The caller of read_stable_mark() must wait for inflation to complete.
912 // Avoid live-lock
913 // TODO: consider calling SafepointSynchronize::do_call_back() while
914 // spinning to see if there's a safepoint pending. If so, immediately
915 // yielding or blocking would be appropriate. Avoid spinning while
916 // there is a safepoint pending.
917 // TODO: add inflation contention performance counters.
918 // TODO: restrict the aggregate number of spinners.
919
920 ++its;
921 if (its > 10000 || !os::is_MP()) {
922 if (its & 1) {
923 os::naked_yield();
924 } else {
925 // Note that the following code attenuates the livelock problem but is not
926 // a complete remedy. A more complete solution would require that the inflating
927 // thread hold the associated inflation lock. The following code simply restricts
928 // the number of spinners to at most one. We'll have N-2 threads blocked
929 // on the inflationlock, 1 thread holding the inflation lock and using
930 // a yield/park strategy, and 1 thread in the midst of inflation.
931 // A more refined approach would be to change the encoding of INFLATING
932 // to allow encapsulation of a native thread pointer. Threads waiting for
933 // inflation to complete would use CAS to push themselves onto a singly linked
934 // list rooted at the markword. Once enqueued, they'd loop, checking a per-thread flag
935 // and calling park(). When inflation was complete the thread that accomplished inflation
936 // would detach the list and set the markword to inflated with a single CAS and
937 // then for each thread on the list, set the flag and unpark() the thread.
938 // This is conceptually similar to muxAcquire-muxRelease, except that muxRelease
939 // wakes at most one thread whereas we need to wake the entire list.
940 int ix = (cast_from_oop<intptr_t>(obj) >> 5) & (NINFLATIONLOCKS-1);
941 int YieldThenBlock = 0;
942 assert(ix >= 0 && ix < NINFLATIONLOCKS, "invariant");
943 assert((NINFLATIONLOCKS & (NINFLATIONLOCKS-1)) == 0, "invariant");
944 Thread::muxAcquire(gInflationLocks + ix, "gInflationLock");
945 while (obj->mark() == markWord::INFLATING()) {
946 // Beware: NakedYield() is advisory and has almost no effect on some platforms
947 // so we periodically call self->_ParkEvent->park(1).
948 // We use a mixed spin/yield/block mechanism.
949 if ((YieldThenBlock++) >= 16) {
950 Thread::current()->_ParkEvent->park(1);
951 } else {
952 os::naked_yield();
953 }
954 }
955 Thread::muxRelease(gInflationLocks + ix);
956 }
957 } else {
958 SpinPause(); // SMP-polite spinning
959 }
960 }
961 }
962
963 // hashCode() generation :
964 //
965 // Possibilities:
966 // * MD5Digest of {obj,stw_random}
967 // * CRC32 of {obj,stw_random} or any linear-feedback shift register function.
968 // * A DES- or AES-style SBox[] mechanism
969 // * One of the Phi-based schemes, such as:
970 // 2654435761 = 2^32 * Phi (golden ratio)
971 // HashCodeValue = ((uintptr_t(obj) >> 3) * 2654435761) ^ GVars.stw_random ;
972 // * A variation of Marsaglia's shift-xor RNG scheme.
973 // * (obj ^ stw_random) is appealing, but can result
974 // in undesirable regularity in the hashCode values of adjacent objects
975 // (objects allocated back-to-back, in particular). This could potentially
976 // result in hashtable collisions and reduced hashtable efficiency.
977 // There are simple ways to "diffuse" the middle address bits over the
978 // generated hashCode values:
979
980 static inline intptr_t get_next_hash(Thread* self, oop obj) {
981 intptr_t value = 0;
982 if (hashCode == 0) {
983 // This form uses global Park-Miller RNG.
984 // On MP system we'll have lots of RW access to a global, so the
985 // mechanism induces lots of coherency traffic.
986 value = os::random();
987 } else if (hashCode == 1) {
988 // This variation has the property of being stable (idempotent)
989 // between STW operations. This can be useful in some of the 1-0
990 // synchronization schemes.
991 intptr_t addr_bits = cast_from_oop<intptr_t>(obj) >> 3;
992 value = addr_bits ^ (addr_bits >> 5) ^ GVars.stw_random;
993 } else if (hashCode == 2) {
994 value = 1; // for sensitivity testing
995 } else if (hashCode == 3) {
996 value = ++GVars.hc_sequence;
997 } else if (hashCode == 4) {
998 value = cast_from_oop<intptr_t>(obj);
999 } else {
1000 // Marsaglia's xor-shift scheme with thread-specific state
1001 // This is probably the best overall implementation -- we'll
1002 // likely make this the default in future releases.
1003 unsigned t = self->_hashStateX;
1004 t ^= (t << 11);
1005 self->_hashStateX = self->_hashStateY;
1006 self->_hashStateY = self->_hashStateZ;
1007 self->_hashStateZ = self->_hashStateW;
1008 unsigned v = self->_hashStateW;
1009 v = (v ^ (v >> 19)) ^ (t ^ (t >> 8));
1010 self->_hashStateW = v;
1011 value = v;
1012 }
1013
1014 value &= markWord::hash_mask;
1015 if (value == 0) value = 0xBAD;
1016 assert(value != markWord::no_hash, "invariant");
1017 return value;
1018 }
1019
1020 intptr_t ObjectSynchronizer::FastHashCode(Thread* self, oop obj) {
1021 if (UseBiasedLocking) {
1022 // NOTE: many places throughout the JVM do not expect a safepoint
1023 // to be taken here, in particular most operations on perm gen
1024 // objects. However, we only ever bias Java instances and all of
1025 // the call sites of identity_hash that might revoke biases have
1026 // been checked to make sure they can handle a safepoint. The
1027 // added check of the bias pattern is to avoid useless calls to
1028 // thread-local storage.
1029 if (obj->mark().has_bias_pattern()) {
1030 // Handle for oop obj in case of STW safepoint
1031 Handle hobj(self, obj);
1032 // Relaxing assertion for bug 6320749.
1033 assert(Universe::verify_in_progress() ||
1034 !SafepointSynchronize::is_at_safepoint(),
1035 "biases should not be seen by VM thread here");
1036 BiasedLocking::revoke(hobj, JavaThread::current());
1037 obj = hobj();
1038 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
1039 }
1040 }
1041
1042 // hashCode() is a heap mutator ...
1043 // Relaxing assertion for bug 6320749.
1044 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1045 !SafepointSynchronize::is_at_safepoint(), "invariant");
1046 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1047 self->is_Java_thread() , "invariant");
1048 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1049 ((JavaThread *)self)->thread_state() != _thread_blocked, "invariant");
1050
1051 while (true) {
1052 ObjectMonitor* monitor = NULL;
1053 markWord temp, test;
1054 intptr_t hash;
1055 markWord mark = read_stable_mark(obj);
1056
1057 // object should remain ineligible for biased locking
1058 assert(!mark.has_bias_pattern(), "invariant");
1059
1060 if (mark.is_neutral()) {
1061 hash = mark.hash(); // this is a normal header
1062 if (hash != 0) { // if it has hash, just return it
1063 return hash;
1064 }
1065 hash = get_next_hash(self, obj); // allocate a new hash code
1066 temp = mark.copy_set_hash(hash); // merge the hash code into header
1067 // use (machine word version) atomic operation to install the hash
1068 test = obj->cas_set_mark(temp, mark);
1069 if (test == mark) {
1070 return hash;
1071 }
1072 // If atomic operation failed, we must inflate the header
1073 // into heavy weight monitor. We could add more code here
1074 // for fast path, but it does not worth the complexity.
1075 } else if (mark.has_monitor()) {
1076 ObjectMonitorHandle omh;
1077 if (!omh.save_om_ptr(obj, mark)) {
1078 // Lost a race with async deflation so try again.
1079 assert(AsyncDeflateIdleMonitors, "sanity check");
1080 continue;
1081 }
1082 monitor = omh.om_ptr();
1083 temp = monitor->header();
1084 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1085 hash = temp.hash();
1086 if (hash != 0) {
1087 return hash;
1088 }
1089 // Skip to the following code to reduce code size
1090 } else if (self->is_lock_owned((address)mark.locker())) {
1091 temp = mark.displaced_mark_helper(); // this is a lightweight monitor owned
1092 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1093 hash = temp.hash(); // by current thread, check if the displaced
1094 if (hash != 0) { // header contains hash code
1095 return hash;
1096 }
1097 // WARNING:
1098 // The displaced header in the BasicLock on a thread's stack
1099 // is strictly immutable. It CANNOT be changed in ANY cases.
1100 // So we have to inflate the stack lock into an ObjectMonitor
1101 // even if the current thread owns the lock. The BasicLock on
1102 // a thread's stack can be asynchronously read by other threads
1103 // during an inflate() call so any change to that stack memory
1104 // may not propagate to other threads correctly.
1105 }
1106
1107 // Inflate the monitor to set hash code
1108 ObjectMonitorHandle omh;
1109 inflate(&omh, self, obj, inflate_cause_hash_code);
1110 monitor = omh.om_ptr();
1111 // Load displaced header and check it has hash code
1112 mark = monitor->header();
1113 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1114 hash = mark.hash();
1115 if (hash == 0) {
1116 hash = get_next_hash(self, obj);
1117 temp = mark.copy_set_hash(hash); // merge hash code into header
1118 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1119 uintptr_t v = Atomic::cmpxchg(temp.value(), (volatile uintptr_t*)monitor->header_addr(), mark.value());
1120 test = markWord(v);
1121 if (test != mark) {
1122 // The only non-deflation update to the ObjectMonitor's
1123 // header/dmw field is to merge in the hash code. If someone
1124 // adds a new usage of the header/dmw field, please update
1125 // this code.
1126 // ObjectMonitor::install_displaced_markword_in_object()
1127 // does mark the header/dmw field as part of async deflation,
1128 // but that protocol cannot happen now due to the
1129 // ObjectMonitorHandle above.
1130 hash = test.hash();
1131 assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value());
1132 assert(hash != 0, "Trivial unexpected object/monitor header usage.");
1133 }
1134 }
1135 // We finally get the hash
1136 return hash;
1137 }
1138 }
1139
1140 // Deprecated -- use FastHashCode() instead.
1141
1142 intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) {
1143 return FastHashCode(Thread::current(), obj());
1144 }
1145
1146
1147 bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
1148 Handle h_obj) {
1149 if (UseBiasedLocking) {
1150 BiasedLocking::revoke(h_obj, thread);
1151 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1152 }
1153
1154 assert(thread == JavaThread::current(), "Can only be called on current thread");
1155 oop obj = h_obj();
1156
1157 while (true) {
1158 markWord mark = read_stable_mark(obj);
1159
1160 // Uncontended case, header points to stack
1161 if (mark.has_locker()) {
1162 return thread->is_lock_owned((address)mark.locker());
1163 }
1164 // Contended case, header points to ObjectMonitor (tagged pointer)
1165 if (mark.has_monitor()) {
1166 ObjectMonitorHandle omh;
1167 if (!omh.save_om_ptr(obj, mark)) {
1168 // Lost a race with async deflation so try again.
1169 assert(AsyncDeflateIdleMonitors, "sanity check");
1170 continue;
1171 }
1172 bool ret_code = omh.om_ptr()->is_entered(thread) != 0;
1173 return ret_code;
1174 }
1175 // Unlocked case, header in place
1176 assert(mark.is_neutral(), "sanity check");
1177 return false;
1178 }
1179 }
1180
1181 // Be aware of this method could revoke bias of the lock object.
1182 // This method queries the ownership of the lock handle specified by 'h_obj'.
1183 // If the current thread owns the lock, it returns owner_self. If no
1184 // thread owns the lock, it returns owner_none. Otherwise, it will return
1185 // owner_other.
1186 ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership
1187 (JavaThread *self, Handle h_obj) {
1188 // The caller must beware this method can revoke bias, and
1189 // revocation can result in a safepoint.
1190 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
1191 assert(self->thread_state() != _thread_blocked, "invariant");
1192
1193 // Possible mark states: neutral, biased, stack-locked, inflated
1194
1195 if (UseBiasedLocking && h_obj()->mark().has_bias_pattern()) {
1196 // CASE: biased
1197 BiasedLocking::revoke(h_obj, self);
1198 assert(!h_obj->mark().has_bias_pattern(),
1199 "biases should be revoked by now");
1200 }
1201
1202 assert(self == JavaThread::current(), "Can only be called on current thread");
1203 oop obj = h_obj();
1204
1205 while (true) {
1206 markWord mark = read_stable_mark(obj);
1207
1208 // CASE: stack-locked. Mark points to a BasicLock on the owner's stack.
1209 if (mark.has_locker()) {
1210 return self->is_lock_owned((address)mark.locker()) ?
1211 owner_self : owner_other;
1212 }
1213
1214 // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor.
1215 // The Object:ObjectMonitor relationship is stable as long as we're
1216 // not at a safepoint and AsyncDeflateIdleMonitors is false.
1217 if (mark.has_monitor()) {
1218 ObjectMonitorHandle omh;
1219 if (!omh.save_om_ptr(obj, mark)) {
1220 // Lost a race with async deflation so try again.
1221 assert(AsyncDeflateIdleMonitors, "sanity check");
1222 continue;
1223 }
1224 ObjectMonitor* monitor = omh.om_ptr();
1225 void* owner = monitor->_owner;
1226 if (owner == NULL) return owner_none;
1227 return (owner == self ||
1228 self->is_lock_owned((address)owner)) ? owner_self : owner_other;
1229 }
1230
1231 // CASE: neutral
1232 assert(mark.is_neutral(), "sanity check");
1233 return owner_none; // it's unlocked
1234 }
1235 }
1236
1237 // FIXME: jvmti should call this
1238 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
1239 if (UseBiasedLocking) {
1240 if (SafepointSynchronize::is_at_safepoint()) {
1241 BiasedLocking::revoke_at_safepoint(h_obj);
1242 } else {
1243 BiasedLocking::revoke(h_obj, JavaThread::current());
1244 }
1245 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1246 }
1247
1248 oop obj = h_obj();
1249
1250 while (true) {
1251 address owner = NULL;
1252 markWord mark = read_stable_mark(obj);
1253
1254 // Uncontended case, header points to stack
1255 if (mark.has_locker()) {
1256 owner = (address) mark.locker();
1257 }
1258
1259 // Contended case, header points to ObjectMonitor (tagged pointer)
1260 else if (mark.has_monitor()) {
1261 ObjectMonitorHandle omh;
1262 if (!omh.save_om_ptr(obj, mark)) {
1263 // Lost a race with async deflation so try again.
1264 assert(AsyncDeflateIdleMonitors, "sanity check");
1265 continue;
1266 }
1267 ObjectMonitor* monitor = omh.om_ptr();
1268 assert(monitor != NULL, "monitor should be non-null");
1269 owner = (address) monitor->owner();
1270 }
1271
1272 if (owner != NULL) {
1273 // owning_thread_from_monitor_owner() may also return NULL here
1274 return Threads::owning_thread_from_monitor_owner(t_list, owner);
1275 }
1276
1277 // Unlocked case, header in place
1278 // Cannot have assertion since this object may have been
1279 // locked by another thread when reaching here.
1280 // assert(mark.is_neutral(), "sanity check");
1281
1282 return NULL;
1283 }
1284 }
1285
1286 // Visitors ...
1287
1288 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
1289 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
1290 while (block != NULL) {
1291 assert(block->object() == CHAINMARKER, "must be a block header");
1292 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
1293 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
1294 if (mid->is_active()) {
1295 ObjectMonitorHandle omh(mid);
1296
1297 if (mid->object() == NULL ||
1298 (AsyncDeflateIdleMonitors && mid->ref_count() < 0)) {
1299 // Only process with closure if the object is set.
1300 // For async deflation, race here if monitor is not owned!
1301 // The above ref_count bump (in ObjectMonitorHandle ctr)
1302 // will cause subsequent async deflation to skip it.
1303 // However, previous or concurrent async deflation is a race
1304 // so skip this ObjectMonitor if it is being async deflated.
1305 continue;
1306 }
1307 closure->do_monitor(mid);
1308 }
1309 }
1310 // unmarked_next() is not needed with g_block_list (no next field marking).
1311 block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om);
1312 }
1313 }
1314
1315 static bool monitors_used_above_threshold() {
1316 if (OrderAccess::load_acquire(&g_om_population) == 0) {
1317 return false;
1318 }
1319 if (MonitorUsedDeflationThreshold > 0) {
1320 int monitors_used = OrderAccess::load_acquire(&g_om_population) -
1321 OrderAccess::load_acquire(&g_om_free_count);
1322 int monitor_usage = (monitors_used * 100LL) /
1323 OrderAccess::load_acquire(&g_om_population);
1324 return monitor_usage > MonitorUsedDeflationThreshold;
1325 }
1326 return false;
1327 }
1328
1329 // Returns true if MonitorBound is set (> 0) and if the specified
1330 // cnt is > MonitorBound. Otherwise returns false.
1331 static bool is_MonitorBound_exceeded(const int cnt) {
1332 const int mx = MonitorBound;
1333 return mx > 0 && cnt > mx;
1334 }
1335
1336 bool ObjectSynchronizer::is_async_deflation_needed() {
1337 if (!AsyncDeflateIdleMonitors) {
1338 return false;
1339 }
1340 if (is_async_deflation_requested()) {
1341 // Async deflation request.
1342 return true;
1343 }
1344 if (AsyncDeflationInterval > 0 &&
1345 time_since_last_async_deflation_ms() > AsyncDeflationInterval &&
1346 monitors_used_above_threshold()) {
1347 // It's been longer than our specified deflate interval and there
1348 // are too many monitors in use. We don't deflate more frequently
1349 // than AsyncDeflationInterval (unless is_async_deflation_requested)
1350 // in order to not swamp the ServiceThread.
1351 _last_async_deflation_time_ns = os::javaTimeNanos();
1352 return true;
1353 }
1354 if (is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) -
1355 OrderAccess::load_acquire(&g_om_free_count))) {
1356 // Not enough ObjectMonitors on the global free list.
1357 return true;
1358 }
1359 return false;
1360 }
1361
1362 bool ObjectSynchronizer::is_safepoint_deflation_needed() {
1363 if (!AsyncDeflateIdleMonitors) {
1364 if (monitors_used_above_threshold()) {
1365 // Too many monitors in use.
1366 return true;
1367 }
1368 return false;
1369 }
1370 if (is_special_deflation_requested()) {
1371 // For AsyncDeflateIdleMonitors only do a safepoint deflation
1372 // if there is a special deflation request.
1373 return true;
1374 }
1375 return false;
1376 }
1377
1378 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1379 return (os::javaTimeNanos() - _last_async_deflation_time_ns) / (NANOUNITS / MILLIUNITS);
1380 }
1381
1382 void ObjectSynchronizer::oops_do(OopClosure* f) {
1383 // We only scan the global used list here (for moribund threads), and
1384 // the thread-local monitors in Thread::oops_do().
1385 global_used_oops_do(f);
1386 }
1387
1388 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
1389 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1390 list_oops_do(OrderAccess::load_acquire(&g_om_in_use_list), OrderAccess::load_acquire(&g_om_in_use_count), f);
1391 }
1392
1393 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
1394 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1395 list_oops_do(OrderAccess::load_acquire(&thread->om_in_use_list), OrderAccess::load_acquire(&thread->om_in_use_count), f);
1396 }
1397
1398 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, int count, OopClosure* f) {
1399 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1400 chk_for_list_loop(list, count);
1401 // The oops_do() phase does not overlap with monitor deflation
1402 // so no need to update the ObjectMonitor's ref_count for this
1403 // ObjectMonitor* use.
1404 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
1405 if (mid->object() != NULL) {
1406 f->do_oop((oop*)mid->object_addr());
1407 }
1408 }
1409 }
1410
1411
1412 // -----------------------------------------------------------------------------
1413 // ObjectMonitor Lifecycle
1414 // -----------------------
1415 // Inflation unlinks monitors from the global g_free_list and
1416 // associates them with objects. Deflation -- which occurs at
1417 // STW-time -- disassociates idle monitors from objects. Such
1418 // scavenged monitors are returned to the g_free_list.
1419 //
1420 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
1421 //
1422 // Lifecycle:
1423 // -- unassigned and on the global free list
1424 // -- unassigned and on a thread's private om_free_list
1425 // -- assigned to an object. The object is inflated and the mark refers
1426 // to the objectmonitor.
1427
1428
1429 // Constraining monitor pool growth via MonitorBound ...
1430 //
1431 // If MonitorBound is not set (<= 0), MonitorBound checks are disabled.
1432 //
1433 // When safepoint deflation is being used (!AsyncDeflateIdleMonitors):
1434 // The monitor pool is grow-only. We scavenge at STW safepoint-time, but the
1435 // the rate of scavenging is driven primarily by GC. As such, we can find
1436 // an inordinate number of monitors in circulation.
1437 // To avoid that scenario we can artificially induce a STW safepoint
1438 // if the pool appears to be growing past some reasonable bound.
1439 // Generally we favor time in space-time tradeoffs, but as there's no
1440 // natural back-pressure on the # of extant monitors we need to impose some
1441 // type of limit. Beware that if MonitorBound is set to too low a value
1442 // we could just loop. In addition, if MonitorBound is set to a low value
1443 // we'll incur more safepoints, which are harmful to performance.
1444 // See also: GuaranteedSafepointInterval
1445 //
1446 // The current implementation uses asynchronous VM operations.
1447 //
1448 // When safepoint deflation is being used and MonitorBound is set, the
1449 // boundry applies to
1450 // (g_om_population - g_om_free_count)
1451 // i.e., if there are not enough ObjectMonitors on the global free list,
1452 // then a safepoint deflation is induced. Picking a good MonitorBound value
1453 // is non-trivial.
1454 //
1455 // When async deflation is being used:
1456 // The monitor pool is still grow-only. Async deflation is requested
1457 // by a safepoint's cleanup phase or by the ServiceThread at periodic
1458 // intervals when is_async_deflation_needed() returns true. In
1459 // addition to other policies that are checked, if there are not
1460 // enough ObjectMonitors on the global free list, then
1461 // is_async_deflation_needed() will return true. The ServiceThread
1462 // calls deflate_global_idle_monitors_using_JT() and also calls
1463 // deflate_per_thread_idle_monitors_using_JT() as needed.
1464
1465 static void InduceScavenge(Thread* self, const char * Whence) {
1466 assert(!AsyncDeflateIdleMonitors, "is not used by async deflation");
1467
1468 // Induce STW safepoint to trim monitors
1469 // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
1470 // More precisely, trigger an asynchronous STW safepoint as the number
1471 // of active monitors passes the specified threshold.
1472 // TODO: assert thread state is reasonable
1473
1474 if (ForceMonitorScavenge == 0 && Atomic::xchg (1, &ForceMonitorScavenge) == 0) {
1475 // Induce a 'null' safepoint to scavenge monitors
1476 // Must VM_Operation instance be heap allocated as the op will be enqueue and posted
1477 // to the VMthread and have a lifespan longer than that of this activation record.
1478 // The VMThread will delete the op when completed.
1479 VMThread::execute(new VM_ScavengeMonitors());
1480 }
1481 }
1482
1483 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self,
1484 const InflateCause cause) {
1485 // A large MAXPRIVATE value reduces both list lock contention
1486 // and list coherency traffic, but also tends to increase the
1487 // number of ObjectMonitors in circulation as well as the STW
1488 // scavenge costs. As usual, we lean toward time in space-time
1489 // tradeoffs.
1490 const int MAXPRIVATE = 1024;
1491
1492 stringStream ss;
1493 for (;;) {
1494 ObjectMonitor* m;
1495
1496 // 1: try to allocate from the thread's local om_free_list.
1497 // Threads will attempt to allocate first from their local list, then
1498 // from the global list, and only after those attempts fail will the
1499 // thread attempt to instantiate new monitors. Thread-local free lists
1500 // improve allocation latency, as well as reducing coherency traffic
1501 // on the shared global list.
1502 m = take_from_start_of_om_free_list(self);
1503 if (m != NULL) {
1504 guarantee(m->object() == NULL, "invariant");
1505 m->set_allocation_state(ObjectMonitor::New);
1506 prepend_to_om_in_use_list(self, m);
1507 return m;
1508 }
1509
1510 // 2: try to allocate from the global g_free_list
1511 // CONSIDER: use muxTry() instead of muxAcquire().
1512 // If the muxTry() fails then drop immediately into case 3.
1513 // If we're using thread-local free lists then try
1514 // to reprovision the caller's free list.
1515 if (OrderAccess::load_acquire(&g_free_list) != NULL) {
1516 // Reprovision the thread's om_free_list.
1517 // Use bulk transfers to reduce the allocation rate and heat
1518 // on various locks.
1519 for (int i = self->om_free_provision; --i >= 0;) {
1520 ObjectMonitor* take = take_from_start_of_g_free_list();
1521 if (take == NULL) {
1522 break; // No more are available.
1523 }
1524 guarantee(take->object() == NULL, "invariant");
1525 if (AsyncDeflateIdleMonitors) {
1526 // We allowed 3 field values to linger during async deflation.
1527 // We clear header and restore ref_count here, but we leave
1528 // owner == DEFLATER_MARKER so the simple C2 ObjectMonitor
1529 // enter optimization can no longer race with async deflation
1530 // and reuse.
1531 take->set_header(markWord::zero());
1532 if (take->ref_count() < 0) {
1533 // Add back max_jint to restore the ref_count field to its
1534 // proper value.
1535 Atomic::add(max_jint, &take->_ref_count);
1536
1537 assert(take->ref_count() >= 0, "must not be negative: ref_count=%d",
1538 take->ref_count());
1539 }
1540 }
1541 take->Recycle();
1542 assert(take->is_free(), "invariant");
1543 om_release(self, take, false);
1544 }
1545 self->om_free_provision += 1 + (self->om_free_provision/2);
1546 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1547
1548 if (!AsyncDeflateIdleMonitors &&
1549 is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) -
1550 OrderAccess::load_acquire(&g_om_free_count))) {
1551 // Not enough ObjectMonitors on the global free list.
1552 // We can't safely induce a STW safepoint from om_alloc() as our thread
1553 // state may not be appropriate for such activities and callers may hold
1554 // naked oops, so instead we defer the action.
1555 InduceScavenge(self, "om_alloc");
1556 }
1557 continue;
1558 }
1559
1560 // 3: allocate a block of new ObjectMonitors
1561 // Both the local and global free lists are empty -- resort to malloc().
1562 // In the current implementation ObjectMonitors are TSM - immortal.
1563 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1564 // each ObjectMonitor to start at the beginning of a cache line,
1565 // so we use align_up().
1566 // A better solution would be to use C++ placement-new.
1567 // BEWARE: As it stands currently, we don't run the ctors!
1568 assert(_BLOCKSIZE > 1, "invariant");
1569 size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE;
1570 PaddedObjectMonitor* temp;
1571 size_t aligned_size = neededsize + (OM_CACHE_LINE_SIZE - 1);
1572 void* real_malloc_addr = NEW_C_HEAP_ARRAY(char, aligned_size, mtInternal);
1573 temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, OM_CACHE_LINE_SIZE);
1574 (void)memset((void *) temp, 0, neededsize);
1575
1576 // Format the block.
1577 // initialize the linked list, each monitor points to its next
1578 // forming the single linked free list, the very first monitor
1579 // will points to next block, which forms the block list.
1580 // The trick of using the 1st element in the block as g_block_list
1581 // linkage should be reconsidered. A better implementation would
1582 // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; }
1583
1584 for (int i = 1; i < _BLOCKSIZE; i++) {
1585 OrderAccess::release_store(&temp[i]._next_om, (ObjectMonitor*)&temp[i+1]);
1586 assert(temp[i].is_free(), "invariant");
1587 }
1588
1589 // terminate the last monitor as the end of list
1590 OrderAccess::release_store(&temp[_BLOCKSIZE - 1]._next_om, (ObjectMonitor*)NULL);
1591
1592 // Element [0] is reserved for global list linkage
1593 temp[0].set_object(CHAINMARKER);
1594
1595 // Consider carving out this thread's current request from the
1596 // block in hand. This avoids some lock traffic and redundant
1597 // list activity.
1598
1599 prepend_block_to_lists(temp);
1600 }
1601 }
1602
1603 // Place "m" on the caller's private per-thread om_free_list.
1604 // In practice there's no need to clamp or limit the number of
1605 // monitors on a thread's om_free_list as the only non-allocation time
1606 // we'll call om_release() is to return a monitor to the free list after
1607 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1608 // accumulate on a thread's free list.
1609 //
1610 // Key constraint: all ObjectMonitors on a thread's free list and the global
1611 // free list must have their object field set to null. This prevents the
1612 // scavenger -- deflate_monitor_list() or deflate_monitor_list_using_JT()
1613 // -- from reclaiming them while we are trying to release them.
1614
1615 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1616 bool from_per_thread_alloc) {
1617 guarantee(m->header().value() == 0, "invariant");
1618 guarantee(m->object() == NULL, "invariant");
1619 stringStream ss;
1620 guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1621 "%s, recursions=" INTX_FORMAT, m->is_busy_to_string(&ss),
1622 m->_recursions);
1623 m->set_allocation_state(ObjectMonitor::Free);
1624 // _next_om is used for both per-thread in-use and free lists so
1625 // we have to remove 'm' from the in-use list first (as needed).
1626 if (from_per_thread_alloc) {
1627 // Need to remove 'm' from om_in_use_list.
1628 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
1629 // protocol because async deflation can do list deletions in parallel.
1630 ObjectMonitor* cur_mid_in_use = NULL;
1631 ObjectMonitor* mid = NULL;
1632 ObjectMonitor* next = NULL;
1633 bool extracted = false;
1634
1635 if (!mark_list_head(&self->om_in_use_list, &mid, &next)) {
1636 fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self));
1637 }
1638 while (true) {
1639 if (m == mid) {
1640 // We found 'm' on the per-thread in-use list so try to extract it.
1641 // First try the list head:
1642 if (Atomic::cmpxchg(next, &self->om_in_use_list, mid) != mid) {
1643 // We could not switch the list head to next.
1644 ObjectMonitor* marked_mid = mark_om_ptr(mid);
1645 // Switch cur_mid_in_use's next field to next (which also
1646 // unmarks cur_mid_in_use):
1647 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
1648 if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, marked_mid)
1649 != marked_mid) {
1650 // We could not switch cur_mid_in_use's next field. This
1651 // should not be possible since it was marked so we:
1652 fatal("mid=" INTPTR_FORMAT " must be referred to by the list "
1653 "head: &om_in_use_list=" INTPTR_FORMAT " or by "
1654 "cur_mid_in_use's next field: cur_mid_in_use=" INTPTR_FORMAT
1655 ", next_om=" INTPTR_FORMAT, p2i(mid),
1656 p2i((ObjectMonitor**)&self->om_in_use_list),
1657 p2i(cur_mid_in_use), p2i(cur_mid_in_use->_next_om));
1658 }
1659 }
1660 extracted = true;
1661 Atomic::dec(&self->om_in_use_count);
1662 // Unmark mid, but leave the next value for any lagging list
1663 // walkers. It will get cleaned up when mid is prepended to
1664 // the thread's free list:
1665 set_next(mid, next);
1666 break;
1667 }
1668 if (cur_mid_in_use != NULL) {
1669 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
1670 }
1671 // The next cur_mid_in_use keeps mid's marked next field so
1672 // that it is stable for a possible next field change. It
1673 // cannot be deflated while it is marked.
1674 cur_mid_in_use = mid;
1675 mid = next;
1676 if (mid == NULL) {
1677 // Reached end of the list and didn't find m so:
1678 fatal("must find m=" INTPTR_FORMAT "on om_in_use_list=" INTPTR_FORMAT,
1679 p2i(m), p2i(self->om_in_use_list));
1680 }
1681 // Mark mid's next field so we can possibly extract it:
1682 next = mark_next_loop(mid);
1683 }
1684 }
1685
1686 prepend_to_om_free_list(self, m);
1687 guarantee(m->is_free(), "invariant");
1688 }
1689
1690 // Return ObjectMonitors on a moribund thread's free and in-use
1691 // lists to the appropriate global lists. The ObjectMonitors on the
1692 // per-thread in-use list may still be in use by other threads.
1693 //
1694 // We currently call om_flush() from Threads::remove() before the
1695 // thread has been excised from the thread list and is no longer a
1696 // mutator. This means that om_flush() cannot run concurrently with
1697 // a safepoint and interleave with deflate_idle_monitors(). In
1698 // particular, this ensures that the thread's in-use monitors are
1699 // scanned by a GC safepoint, either via Thread::oops_do() (before
1700 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1701 // om_flush() is called).
1702 //
1703 // With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT()
1704 // and deflate_per_thread_idle_monitors_using_JT() (in another thread) can
1705 // run at the same time as om_flush() so we have to follow a careful
1706 // protocol to prevent list corruption.
1707
1708 void ObjectSynchronizer::om_flush(Thread* self) {
1709 // This function can race with an async deflater thread. Since
1710 // deflation has to process the per-thread in-use list before
1711 // prepending the deflated ObjectMonitors to the global free list,
1712 // we process the per-thread lists in the same order to prevent
1713 // ordering races.
1714 int in_use_count = 0;
1715 ObjectMonitor* in_use_list = NULL;
1716 ObjectMonitor* in_use_tail = NULL;
1717 ObjectMonitor* next = NULL;
1718
1719 // An async deflation thread checks to see if the target thread
1720 // is exiting, but if it has made it past that check before we
1721 // started exiting, then it is racing to get to the in-use list.
1722 if (mark_list_head(&self->om_in_use_list, &in_use_list, &next)) {
1723 chk_for_list_loop(in_use_list, OrderAccess::load_acquire(&self->om_in_use_count));
1724 // At this point, we have marked the in-use list head so an
1725 // async deflation thread cannot come in after us. If an async
1726 // deflation thread is ahead of us, then we'll detect that and
1727 // wait for it to finish its work.
1728 //
1729 // The thread is going away, however the ObjectMonitors on the
1730 // om_in_use_list may still be in-use by other threads. Link
1731 // them to in_use_tail, which will be linked into the global
1732 // in-use list g_om_in_use_list below.
1733 //
1734 // Account for the in-use list head before the loop since it is
1735 // already marked (by this thread):
1736 in_use_tail = in_use_list;
1737 in_use_count++;
1738 for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL;) {
1739 if (is_next_marked(cur_om)) {
1740 // This next field is marked so there must be an async deflater
1741 // thread ahead of us so we'll give it a chance to finish.
1742 while (is_next_marked(cur_om)) {
1743 os::naked_short_sleep(1);
1744 }
1745 // Refetch the possibly changed next field and try again.
1746 cur_om = unmarked_next(in_use_tail);
1747 continue;
1748 }
1749 if (!cur_om->is_active()) {
1750 // cur_om was deflated and the allocation state was changed
1751 // to Free while it was marked. We happened to see it just
1752 // after it was unmarked (and added to the free list).
1753 // Refetch the possibly changed next field and try again.
1754 cur_om = unmarked_next(in_use_tail);
1755 continue;
1756 }
1757 in_use_tail = cur_om;
1758 in_use_count++;
1759 cur_om = unmarked_next(cur_om);
1760 }
1761 guarantee(in_use_tail != NULL, "invariant");
1762 int l_om_in_use_count = OrderAccess::load_acquire(&self->om_in_use_count);
1763 ADIM_guarantee(l_om_in_use_count == in_use_count, "in-use counts don't "
1764 "match: l_om_in_use_count=%d, in_use_count=%d",
1765 l_om_in_use_count, in_use_count);
1766 // Clear the in-use count before unmarking the in-use list head
1767 // to avoid races:
1768 OrderAccess::release_store(&self->om_in_use_count, 0);
1769 // Clear the in-use list head (which also unmarks it):
1770 OrderAccess::release_store(&self->om_in_use_list, (ObjectMonitor*)NULL);
1771 // Unmark the disconnected list head:
1772 set_next(in_use_list, next);
1773 }
1774
1775 int free_count = 0;
1776 ObjectMonitor* free_list = OrderAccess::load_acquire(&self->om_free_list);
1777 ObjectMonitor* free_tail = NULL;
1778 if (free_list != NULL) {
1779 chk_for_list_loop(free_list, OrderAccess::load_acquire(&self->om_free_count));
1780 // The thread is going away. Set 'free_tail' to the last per-thread free
1781 // monitor which will be linked to g_free_list below.
1782 stringStream ss;
1783 for (ObjectMonitor* s = free_list; s != NULL; s = unmarked_next(s)) {
1784 free_count++;
1785 free_tail = s;
1786 guarantee(s->object() == NULL, "invariant");
1787 guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss));
1788 }
1789 guarantee(free_tail != NULL, "invariant");
1790 int l_om_free_count = OrderAccess::load_acquire(&self->om_free_count);
1791 ADIM_guarantee(l_om_free_count == free_count, "free counts don't match: "
1792 "l_om_free_count=%d, free_count=%d", l_om_free_count,
1793 free_count);
1794 OrderAccess::release_store(&self->om_free_list, (ObjectMonitor*)NULL);
1795 OrderAccess::release_store(&self->om_free_count, 0);
1796 }
1797
1798 if (free_tail != NULL) {
1799 prepend_list_to_g_free_list(free_list, free_tail, free_count);
1800 }
1801
1802 if (in_use_tail != NULL) {
1803 prepend_list_to_g_om_in_use_list(in_use_list, in_use_tail, in_use_count);
1804 }
1805
1806 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1807 LogStreamHandle(Info, monitorinflation) lsh_info;
1808 LogStream* ls = NULL;
1809 if (log_is_enabled(Debug, monitorinflation)) {
1810 ls = &lsh_debug;
1811 } else if ((free_count != 0 || in_use_count != 0) &&
1812 log_is_enabled(Info, monitorinflation)) {
1813 ls = &lsh_info;
1814 }
1815 if (ls != NULL) {
1816 ls->print_cr("om_flush: jt=" INTPTR_FORMAT ", free_count=%d"
1817 ", in_use_count=%d" ", om_free_provision=%d",
1818 p2i(self), free_count, in_use_count, self->om_free_provision);
1819 }
1820 }
1821
1822 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1823 const oop obj,
1824 ObjectSynchronizer::InflateCause cause) {
1825 assert(event != NULL, "invariant");
1826 assert(event->should_commit(), "invariant");
1827 event->set_monitorClass(obj->klass());
1828 event->set_address((uintptr_t)(void*)obj);
1829 event->set_cause((u1)cause);
1830 event->commit();
1831 }
1832
1833 // Fast path code shared by multiple functions
1834 void ObjectSynchronizer::inflate_helper(ObjectMonitorHandle* omh_p, oop obj) {
1835 while (true) {
1836 markWord mark = obj->mark();
1837 if (mark.has_monitor()) {
1838 if (!omh_p->save_om_ptr(obj, mark)) {
1839 // Lost a race with async deflation so try again.
1840 assert(AsyncDeflateIdleMonitors, "sanity check");
1841 continue;
1842 }
1843 ObjectMonitor* monitor = omh_p->om_ptr();
1844 assert(ObjectSynchronizer::verify_objmon_isinpool(monitor), "monitor is invalid");
1845 markWord dmw = monitor->header();
1846 assert(dmw.is_neutral(), "sanity check: header=" INTPTR_FORMAT, dmw.value());
1847 return;
1848 }
1849 inflate(omh_p, Thread::current(), obj, inflate_cause_vm_internal);
1850 return;
1851 }
1852 }
1853
1854 void ObjectSynchronizer::inflate(ObjectMonitorHandle* omh_p, Thread* self,
1855 oop object, const InflateCause cause) {
1856 // Inflate mutates the heap ...
1857 // Relaxing assertion for bug 6320749.
1858 assert(Universe::verify_in_progress() ||
1859 !SafepointSynchronize::is_at_safepoint(), "invariant");
1860
1861 EventJavaMonitorInflate event;
1862
1863 for (;;) {
1864 const markWord mark = object->mark();
1865 assert(!mark.has_bias_pattern(), "invariant");
1866
1867 // The mark can be in one of the following states:
1868 // * Inflated - just return
1869 // * Stack-locked - coerce it to inflated
1870 // * INFLATING - busy wait for conversion to complete
1871 // * Neutral - aggressively inflate the object.
1872 // * BIASED - Illegal. We should never see this
1873
1874 // CASE: inflated
1875 if (mark.has_monitor()) {
1876 if (!omh_p->save_om_ptr(object, mark)) {
1877 // Lost a race with async deflation so try again.
1878 assert(AsyncDeflateIdleMonitors, "sanity check");
1879 continue;
1880 }
1881 ObjectMonitor* inf = omh_p->om_ptr();
1882 markWord dmw = inf->header();
1883 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1884 assert(inf->object() == object, "invariant");
1885 assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
1886 return;
1887 }
1888
1889 // CASE: inflation in progress - inflating over a stack-lock.
1890 // Some other thread is converting from stack-locked to inflated.
1891 // Only that thread can complete inflation -- other threads must wait.
1892 // The INFLATING value is transient.
1893 // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1894 // We could always eliminate polling by parking the thread on some auxiliary list.
1895 if (mark == markWord::INFLATING()) {
1896 read_stable_mark(object);
1897 continue;
1898 }
1899
1900 // CASE: stack-locked
1901 // Could be stack-locked either by this thread or by some other thread.
1902 //
1903 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1904 // to install INFLATING into the mark word. We originally installed INFLATING,
1905 // allocated the objectmonitor, and then finally STed the address of the
1906 // objectmonitor into the mark. This was correct, but artificially lengthened
1907 // the interval in which INFLATED appeared in the mark, thus increasing
1908 // the odds of inflation contention.
1909 //
1910 // We now use per-thread private objectmonitor free lists.
1911 // These list are reprovisioned from the global free list outside the
1912 // critical INFLATING...ST interval. A thread can transfer
1913 // multiple objectmonitors en-mass from the global free list to its local free list.
1914 // This reduces coherency traffic and lock contention on the global free list.
1915 // Using such local free lists, it doesn't matter if the om_alloc() call appears
1916 // before or after the CAS(INFLATING) operation.
1917 // See the comments in om_alloc().
1918
1919 LogStreamHandle(Trace, monitorinflation) lsh;
1920
1921 if (mark.has_locker()) {
1922 ObjectMonitor* m = om_alloc(self, cause);
1923 // Optimistically prepare the objectmonitor - anticipate successful CAS
1924 // We do this before the CAS in order to minimize the length of time
1925 // in which INFLATING appears in the mark.
1926 m->Recycle();
1927 m->_Responsible = NULL;
1928 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // Consider: maintain by type/class
1929
1930 markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1931 if (cmp != mark) {
1932 om_release(self, m, true);
1933 continue; // Interference -- just retry
1934 }
1935
1936 // We've successfully installed INFLATING (0) into the mark-word.
1937 // This is the only case where 0 will appear in a mark-word.
1938 // Only the singular thread that successfully swings the mark-word
1939 // to 0 can perform (or more precisely, complete) inflation.
1940 //
1941 // Why do we CAS a 0 into the mark-word instead of just CASing the
1942 // mark-word from the stack-locked value directly to the new inflated state?
1943 // Consider what happens when a thread unlocks a stack-locked object.
1944 // It attempts to use CAS to swing the displaced header value from the
1945 // on-stack BasicLock back into the object header. Recall also that the
1946 // header value (hash code, etc) can reside in (a) the object header, or
1947 // (b) a displaced header associated with the stack-lock, or (c) a displaced
1948 // header in an ObjectMonitor. The inflate() routine must copy the header
1949 // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1950 // the while preserving the hashCode stability invariants. If the owner
1951 // decides to release the lock while the value is 0, the unlock will fail
1952 // and control will eventually pass from slow_exit() to inflate. The owner
1953 // will then spin, waiting for the 0 value to disappear. Put another way,
1954 // the 0 causes the owner to stall if the owner happens to try to
1955 // drop the lock (restoring the header from the BasicLock to the object)
1956 // while inflation is in-progress. This protocol avoids races that might
1957 // would otherwise permit hashCode values to change or "flicker" for an object.
1958 // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1959 // 0 serves as a "BUSY" inflate-in-progress indicator.
1960
1961
1962 // fetch the displaced mark from the owner's stack.
1963 // The owner can't die or unwind past the lock while our INFLATING
1964 // object is in the mark. Furthermore the owner can't complete
1965 // an unlock on the object, either.
1966 markWord dmw = mark.displaced_mark_helper();
1967 // Catch if the object's header is not neutral (not locked and
1968 // not marked is what we care about here).
1969 ADIM_guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1970
1971 // Setup monitor fields to proper values -- prepare the monitor
1972 m->set_header(dmw);
1973
1974 // Optimization: if the mark.locker stack address is associated
1975 // with this thread we could simply set m->_owner = self.
1976 // Note that a thread can inflate an object
1977 // that it has stack-locked -- as might happen in wait() -- directly
1978 // with CAS. That is, we can avoid the xchg-NULL .... ST idiom.
1979 m->set_owner(mark.locker());
1980 m->set_object(object);
1981 // TODO-FIXME: assert BasicLock->dhw != 0.
1982
1983 omh_p->set_om_ptr(m);
1984 assert(m->is_new(), "freshly allocated monitor must be new");
1985 m->set_allocation_state(ObjectMonitor::Old);
1986
1987 // Must preserve store ordering. The monitor state must
1988 // be stable at the time of publishing the monitor address.
1989 guarantee(object->mark() == markWord::INFLATING(), "invariant");
1990 object->release_set_mark(markWord::encode(m));
1991
1992 // Hopefully the performance counters are allocated on distinct cache lines
1993 // to avoid false sharing on MP systems ...
1994 OM_PERFDATA_OP(Inflations, inc());
1995 if (log_is_enabled(Trace, monitorinflation)) {
1996 ResourceMark rm(self);
1997 lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
1998 INTPTR_FORMAT ", type='%s'", p2i(object),
1999 object->mark().value(), object->klass()->external_name());
2000 }
2001 if (event.should_commit()) {
2002 post_monitor_inflate_event(&event, object, cause);
2003 }
2004 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
2005 return;
2006 }
2007
2008 // CASE: neutral
2009 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
2010 // If we know we're inflating for entry it's better to inflate by swinging a
2011 // pre-locked ObjectMonitor pointer into the object header. A successful
2012 // CAS inflates the object *and* confers ownership to the inflating thread.
2013 // In the current implementation we use a 2-step mechanism where we CAS()
2014 // to inflate and then CAS() again to try to swing _owner from NULL to self.
2015 // An inflateTry() method that we could call from enter() would be useful.
2016
2017 // Catch if the object's header is not neutral (not locked and
2018 // not marked is what we care about here).
2019 ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT,mark.value());
2020 ObjectMonitor* m = om_alloc(self, cause);
2021 // prepare m for installation - set monitor to initial state
2022 m->Recycle();
2023 m->set_header(mark);
2024 // If we leave _owner == DEFLATER_MARKER here, then the simple C2
2025 // ObjectMonitor enter optimization can no longer race with async
2026 // deflation and reuse.
2027 m->set_object(object);
2028 m->_Responsible = NULL;
2029 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
2030
2031 omh_p->set_om_ptr(m);
2032 assert(m->is_new(), "freshly allocated monitor must be new");
2033 m->set_allocation_state(ObjectMonitor::Old);
2034
2035 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
2036 guarantee(!m->owner_is_DEFLATER_MARKER() || m->ref_count() >= 0,
2037 "race between deflation and om_release() with m=" INTPTR_FORMAT
2038 ", _owner=" INTPTR_FORMAT ", ref_count=%d", p2i(m),
2039 p2i(m->_owner), m->ref_count());
2040 m->set_header(markWord::zero());
2041 m->set_object(NULL);
2042 m->Recycle();
2043 omh_p->set_om_ptr(NULL);
2044 // om_release() will reset the allocation state
2045 om_release(self, m, true);
2046 m = NULL;
2047 continue;
2048 // interference - the markword changed - just retry.
2049 // The state-transitions are one-way, so there's no chance of
2050 // live-lock -- "Inflated" is an absorbing state.
2051 }
2052
2053 // Hopefully the performance counters are allocated on distinct
2054 // cache lines to avoid false sharing on MP systems ...
2055 OM_PERFDATA_OP(Inflations, inc());
2056 if (log_is_enabled(Trace, monitorinflation)) {
2057 ResourceMark rm(self);
2058 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
2059 INTPTR_FORMAT ", type='%s'", p2i(object),
2060 object->mark().value(), object->klass()->external_name());
2061 }
2062 if (event.should_commit()) {
2063 post_monitor_inflate_event(&event, object, cause);
2064 }
2065 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
2066 return;
2067 }
2068 }
2069
2070
2071 // We maintain a list of in-use monitors for each thread.
2072 //
2073 // For safepoint based deflation:
2074 // deflate_thread_local_monitors() scans a single thread's in-use list, while
2075 // deflate_idle_monitors() scans only a global list of in-use monitors which
2076 // is populated only as a thread dies (see om_flush()).
2077 //
2078 // These operations are called at all safepoints, immediately after mutators
2079 // are stopped, but before any objects have moved. Collectively they traverse
2080 // the population of in-use monitors, deflating where possible. The scavenged
2081 // monitors are returned to the global monitor free list.
2082 //
2083 // Beware that we scavenge at *every* stop-the-world point. Having a large
2084 // number of monitors in-use could negatively impact performance. We also want
2085 // to minimize the total # of monitors in circulation, as they incur a small
2086 // footprint penalty.
2087 //
2088 // Perversely, the heap size -- and thus the STW safepoint rate --
2089 // typically drives the scavenge rate. Large heaps can mean infrequent GC,
2090 // which in turn can mean large(r) numbers of ObjectMonitors in circulation.
2091 // This is an unfortunate aspect of this design.
2092 //
2093 // For async deflation:
2094 // If a special deflation request is made, then the safepoint based
2095 // deflation mechanism is used. Otherwise, an async deflation request
2096 // is registered with the ServiceThread and it is notified.
2097
2098 void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* counters) {
2099 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2100
2101 // The per-thread in-use lists are handled in
2102 // ParallelSPCleanupThreadClosure::do_thread().
2103
2104 if (!AsyncDeflateIdleMonitors || is_special_deflation_requested()) {
2105 // Use the older mechanism for the global in-use list or if a
2106 // special deflation has been requested before the safepoint.
2107 ObjectSynchronizer::deflate_idle_monitors(counters);
2108 return;
2109 }
2110
2111 log_debug(monitorinflation)("requesting async deflation of idle monitors.");
2112 // Request deflation of idle monitors by the ServiceThread:
2113 set_is_async_deflation_requested(true);
2114 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
2115 ml.notify_all();
2116 }
2117
2118 // Deflate a single monitor if not in-use
2119 // Return true if deflated, false if in-use
2120 bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj,
2121 ObjectMonitor** free_head_p,
2122 ObjectMonitor** free_tail_p) {
2123 bool deflated;
2124 // Normal case ... The monitor is associated with obj.
2125 const markWord mark = obj->mark();
2126 guarantee(mark == markWord::encode(mid), "should match: mark="
2127 INTPTR_FORMAT ", encoded mid=" INTPTR_FORMAT, mark.value(),
2128 markWord::encode(mid).value());
2129 // Make sure that mark.monitor() and markWord::encode() agree:
2130 guarantee(mark.monitor() == mid, "should match: monitor()=" INTPTR_FORMAT
2131 ", mid=" INTPTR_FORMAT, p2i(mark.monitor()), p2i(mid));
2132 const markWord dmw = mid->header();
2133 guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
2134
2135 if (mid->is_busy() || mid->ref_count() != 0) {
2136 // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor*
2137 // is in use so no deflation.
2138 deflated = false;
2139 } else {
2140 // Deflate the monitor if it is no longer being used
2141 // It's idle - scavenge and return to the global free list
2142 // plain old deflation ...
2143 if (log_is_enabled(Trace, monitorinflation)) {
2144 ResourceMark rm;
2145 log_trace(monitorinflation)("deflate_monitor: "
2146 "object=" INTPTR_FORMAT ", mark="
2147 INTPTR_FORMAT ", type='%s'", p2i(obj),
2148 mark.value(), obj->klass()->external_name());
2149 }
2150
2151 // Restore the header back to obj
2152 obj->release_set_mark(dmw);
2153 if (AsyncDeflateIdleMonitors) {
2154 // clear() expects the owner field to be NULL and we won't race
2155 // with the simple C2 ObjectMonitor enter optimization since
2156 // we're at a safepoint.
2157 mid->set_owner(NULL);
2158 }
2159 mid->clear();
2160
2161 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
2162 p2i(mid->object()));
2163 assert(mid->is_free(), "invariant");
2164
2165 // Move the deflated ObjectMonitor to the working free list
2166 // defined by free_head_p and free_tail_p. No races on this list
2167 // so no need for load_acquire() or store_release().
2168 if (*free_head_p == NULL) *free_head_p = mid;
2169 if (*free_tail_p != NULL) {
2170 // We append to the list so the caller can use mid->_next_om
2171 // to fix the linkages in its context.
2172 ObjectMonitor* prevtail = *free_tail_p;
2173 // Should have been cleaned up by the caller:
2174 // Note: Should not have to mark prevtail here since we're at a
2175 // safepoint and ObjectMonitors on the local free list should
2176 // not be accessed in parallel.
2177 assert(prevtail->_next_om == NULL, "must be NULL: _next_om="
2178 INTPTR_FORMAT, p2i(prevtail->_next_om));
2179 set_next(prevtail, mid);
2180 }
2181 *free_tail_p = mid;
2182 // At this point, mid->_next_om still refers to its current
2183 // value and another ObjectMonitor's _next_om field still
2184 // refers to this ObjectMonitor. Those linkages have to be
2185 // cleaned up by the caller who has the complete context.
2186 deflated = true;
2187 }
2188 return deflated;
2189 }
2190
2191 // Deflate the specified ObjectMonitor if not in-use using a JavaThread.
2192 // Returns true if it was deflated and false otherwise.
2193 //
2194 // The async deflation protocol sets owner to DEFLATER_MARKER and
2195 // makes ref_count negative as signals to contending threads that
2196 // an async deflation is in progress. There are a number of checks
2197 // as part of the protocol to make sure that the calling thread has
2198 // not lost the race to a contending thread or to a thread that just
2199 // wants to use the ObjectMonitor*.
2200 //
2201 // The ObjectMonitor has been successfully async deflated when:
2202 // (owner == DEFLATER_MARKER && ref_count < 0)
2203 // Contending threads or ObjectMonitor* using threads that see those
2204 // values know to retry their operation.
2205 //
2206 bool ObjectSynchronizer::deflate_monitor_using_JT(ObjectMonitor* mid,
2207 ObjectMonitor** free_head_p,
2208 ObjectMonitor** free_tail_p) {
2209 assert(AsyncDeflateIdleMonitors, "sanity check");
2210 assert(Thread::current()->is_Java_thread(), "precondition");
2211 // A newly allocated ObjectMonitor should not be seen here so we
2212 // avoid an endless inflate/deflate cycle.
2213 assert(mid->is_old(), "must be old: allocation_state=%d",
2214 (int) mid->allocation_state());
2215
2216 if (mid->is_busy() || mid->ref_count() != 0) {
2217 // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor*
2218 // is in use so no deflation.
2219 return false;
2220 }
2221
2222 if (Atomic::replace_if_null(DEFLATER_MARKER, &(mid->_owner))) {
2223 // ObjectMonitor is not owned by another thread. Our setting
2224 // owner to DEFLATER_MARKER forces any contending thread through
2225 // the slow path. This is just the first part of the async
2226 // deflation dance.
2227
2228 if (mid->_contentions != 0 || mid->_waiters != 0) {
2229 // Another thread has raced to enter the ObjectMonitor after
2230 // mid->is_busy() above or has already entered and waited on
2231 // it which makes it busy so no deflation. Restore owner to
2232 // NULL if it is still DEFLATER_MARKER.
2233 Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER);
2234 return false;
2235 }
2236
2237 if (Atomic::cmpxchg(-max_jint, &mid->_ref_count, (jint)0) == 0) {
2238 // Make ref_count negative to force any contending threads or
2239 // ObjectMonitor* using threads to retry. This is the second
2240 // part of the async deflation dance.
2241
2242 if (mid->owner_is_DEFLATER_MARKER()) {
2243 // If owner is still DEFLATER_MARKER, then we have successfully
2244 // signaled any contending threads to retry. If it is not, then we
2245 // have lost the race to an entering thread and the ObjectMonitor
2246 // is now busy. This is the third and final part of the async
2247 // deflation dance.
2248 // Note: This owner check solves the ABA problem with ref_count
2249 // where another thread acquired the ObjectMonitor, finished
2250 // using it and restored the ref_count to zero.
2251
2252 // Sanity checks for the races:
2253 guarantee(mid->_contentions == 0, "must be 0: contentions=%d",
2254 mid->_contentions);
2255 guarantee(mid->_waiters == 0, "must be 0: waiters=%d", mid->_waiters);
2256 guarantee(mid->_cxq == NULL, "must be no contending threads: cxq="
2257 INTPTR_FORMAT, p2i(mid->_cxq));
2258 guarantee(mid->_EntryList == NULL,
2259 "must be no entering threads: EntryList=" INTPTR_FORMAT,
2260 p2i(mid->_EntryList));
2261
2262 const oop obj = (oop) mid->object();
2263 if (log_is_enabled(Trace, monitorinflation)) {
2264 ResourceMark rm;
2265 log_trace(monitorinflation)("deflate_monitor_using_JT: "
2266 "object=" INTPTR_FORMAT ", mark="
2267 INTPTR_FORMAT ", type='%s'",
2268 p2i(obj), obj->mark().value(),
2269 obj->klass()->external_name());
2270 }
2271
2272 // Install the old mark word if nobody else has already done it.
2273 mid->install_displaced_markword_in_object(obj);
2274 mid->clear_using_JT();
2275
2276 assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT,
2277 p2i(mid->object()));
2278 assert(mid->is_free(), "must be free: allocation_state=%d",
2279 (int) mid->allocation_state());
2280
2281 // Move the deflated ObjectMonitor to the working free list
2282 // defined by free_head_p and free_tail_p. No races on this list
2283 // so no need for load_acquire() or store_release().
2284 if (*free_head_p == NULL) {
2285 // First one on the list.
2286 *free_head_p = mid;
2287 }
2288 if (*free_tail_p != NULL) {
2289 // We append to the list so the caller can use mid->_next_om
2290 // to fix the linkages in its context.
2291 ObjectMonitor* prevtail = *free_tail_p;
2292 // Should have been cleaned up by the caller:
2293 ObjectMonitor* next = mark_next_loop(prevtail);
2294 assert(unmarked_next(prevtail) == NULL, "must be NULL: _next_om="
2295 INTPTR_FORMAT, p2i(unmarked_next(prevtail)));
2296 set_next(prevtail, mid); // prevtail now points to mid (and is unmarked)
2297 }
2298 *free_tail_p = mid;
2299
2300 // At this point, mid->_next_om still refers to its current
2301 // value and another ObjectMonitor's _next_om field still
2302 // refers to this ObjectMonitor. Those linkages have to be
2303 // cleaned up by the caller who has the complete context.
2304
2305 // We leave owner == DEFLATER_MARKER and ref_count < 0
2306 // to force any racing threads to retry.
2307 return true; // Success, ObjectMonitor has been deflated.
2308 }
2309
2310 // The owner was changed from DEFLATER_MARKER so we lost the
2311 // race since the ObjectMonitor is now busy.
2312
2313 // Add back max_jint to restore the ref_count field to its
2314 // proper value (which may not be what we saw above):
2315 Atomic::add(max_jint, &mid->_ref_count);
2316
2317 assert(mid->ref_count() >= 0, "must not be negative: ref_count=%d",
2318 mid->ref_count());
2319 return false;
2320 }
2321
2322 // The ref_count was no longer 0 so we lost the race since the
2323 // ObjectMonitor is now busy or the ObjectMonitor* is now is use.
2324 // Restore owner to NULL if it is still DEFLATER_MARKER:
2325 Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER);
2326 }
2327
2328 // The owner field is no longer NULL so we lost the race since the
2329 // ObjectMonitor is now busy.
2330 return false;
2331 }
2332
2333 // Walk a given monitor list, and deflate idle monitors.
2334 // The given list could be a per-thread list or a global list.
2335 //
2336 // In the case of parallel processing of thread local monitor lists,
2337 // work is done by Threads::parallel_threads_do() which ensures that
2338 // each Java thread is processed by exactly one worker thread, and
2339 // thus avoid conflicts that would arise when worker threads would
2340 // process the same monitor lists concurrently.
2341 //
2342 // See also ParallelSPCleanupTask and
2343 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
2344 // Threads::parallel_java_threads_do() in thread.cpp.
2345 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor* volatile * list_p,
2346 int volatile * count_p,
2347 ObjectMonitor** free_head_p,
2348 ObjectMonitor** free_tail_p) {
2349 ObjectMonitor* cur_mid_in_use = NULL;
2350 ObjectMonitor* mid = NULL;
2351 ObjectMonitor* next = NULL;
2352 int deflated_count = 0;
2353
2354 // We use the simpler mark-mid-as-we-go protocol since there are no
2355 // parallel list deletions since we are at a safepoint.
2356 if (!mark_list_head(list_p, &mid, &next)) {
2357 return 0; // The list is empty so nothing to deflate.
2358 }
2359
2360 while (true) {
2361 oop obj = (oop) mid->object();
2362 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
2363 // Deflation succeeded and already updated free_head_p and
2364 // free_tail_p as needed. Finish the move to the local free list
2365 // by unlinking mid from the global or per-thread in-use list.
2366 if (Atomic::cmpxchg(next, list_p, mid) != mid) {
2367 // We could not switch the list head to next.
2368 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
2369 if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, mid) != mid) {
2370 // deflate_monitor_list() is called at a safepoint so the
2371 // global or per-thread in-use list should not be modified
2372 // in parallel so we:
2373 fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: "
2374 "list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: "
2375 "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT,
2376 p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use),
2377 p2i(cur_mid_in_use->_next_om));
2378 }
2379 }
2380 // At this point mid is disconnected from the in-use list so
2381 // its marked next field no longer has any effects.
2382 deflated_count++;
2383 Atomic::dec(count_p);
2384 chk_for_list_loop(OrderAccess::load_acquire(list_p),
2385 OrderAccess::load_acquire(count_p));
2386 chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p),
2387 OrderAccess::load_acquire(count_p));
2388 // mid is current tail in the free_head_p list so NULL terminate it
2389 // (which also unmarks it):
2390 set_next(mid, NULL);
2391
2392 // All the list management is done so move on to the next one:
2393 mid = next;
2394 } else {
2395 set_next(mid, next); // unmark next field
2396
2397 // All the list management is done so move on to the next one:
2398 cur_mid_in_use = mid;
2399 mid = next;
2400 }
2401 if (mid == NULL) {
2402 break; // Reached end of the list so nothing more to deflate.
2403 }
2404 // Mark mid's next field so we can possibly deflate it:
2405 next = mark_next_loop(mid);
2406 }
2407 return deflated_count;
2408 }
2409
2410 // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using
2411 // a JavaThread. Returns the number of deflated ObjectMonitors. The given
2412 // list could be a per-thread in-use list or the global in-use list.
2413 // If a safepoint has started, then we save state via saved_mid_in_use_p
2414 // and return to the caller to honor the safepoint.
2415 //
2416 int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor* volatile * list_p,
2417 int volatile * count_p,
2418 ObjectMonitor** free_head_p,
2419 ObjectMonitor** free_tail_p,
2420 ObjectMonitor** saved_mid_in_use_p) {
2421 assert(AsyncDeflateIdleMonitors, "sanity check");
2422 assert(Thread::current()->is_Java_thread(), "precondition");
2423
2424 ObjectMonitor* cur_mid_in_use = NULL;
2425 ObjectMonitor* mid = NULL;
2426 ObjectMonitor* next = NULL;
2427 ObjectMonitor* next_next = NULL;
2428 int deflated_count = 0;
2429
2430 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
2431 // protocol because om_release() can do list deletions in parallel.
2432 // We also mark-next-next-as-we-go to prevent an om_flush() that is
2433 // behind this thread from passing us.
2434 if (*saved_mid_in_use_p == NULL) {
2435 // No saved state so start at the beginning.
2436 // Mark the list head's next field so we can possibly deflate it:
2437 if (!mark_list_head(list_p, &mid, &next)) {
2438 return 0; // The list is empty so nothing to deflate.
2439 }
2440 } else {
2441 // We're restarting after a safepoint so restore the necessary state
2442 // before we resume.
2443 cur_mid_in_use = *saved_mid_in_use_p;
2444 // Mark cur_mid_in_use's next field so we can possibly update its
2445 // next field to extract a deflated ObjectMonitor.
2446 mid = mark_next_loop(cur_mid_in_use);
2447 if (mid == NULL) {
2448 set_next(cur_mid_in_use, NULL); // unmark next field
2449 *saved_mid_in_use_p = NULL;
2450 return 0; // The remainder is empty so nothing more to deflate.
2451 }
2452 // Mark mid's next field so we can possibly deflate it:
2453 next = mark_next_loop(mid);
2454 }
2455
2456 while (true) {
2457 // The current mid's next field is marked at this point. If we have
2458 // a cur_mid_in_use, then its next field is also marked at this point.
2459
2460 if (next != NULL) {
2461 // We mark the next -> next field so that an om_flush()
2462 // thread that is behind us cannot pass us when we
2463 // unmark the current mid's next field.
2464 next_next = mark_next_loop(next);
2465 }
2466
2467 // Only try to deflate if there is an associated Java object and if
2468 // mid is old (is not newly allocated and is not newly freed).
2469 if (mid->object() != NULL && mid->is_old() &&
2470 deflate_monitor_using_JT(mid, free_head_p, free_tail_p)) {
2471 // Deflation succeeded and already updated free_head_p and
2472 // free_tail_p as needed. Finish the move to the local free list
2473 // by unlinking mid from the global or per-thread in-use list.
2474 if (Atomic::cmpxchg(next, list_p, mid) != mid) {
2475 // We could not switch the list head to next.
2476 ObjectMonitor* marked_mid = mark_om_ptr(mid);
2477 ObjectMonitor* marked_next = mark_om_ptr(next);
2478 // Switch cur_mid_in_use's next field to marked next:
2479 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
2480 if (Atomic::cmpxchg(marked_next, &cur_mid_in_use->_next_om,
2481 marked_mid) != marked_mid) {
2482 // We could not switch cur_mid_in_use's next field. This
2483 // should not be possible since it was marked so we:
2484 fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: "
2485 "&list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: "
2486 "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT,
2487 p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use),
2488 p2i(cur_mid_in_use->_next_om));
2489 }
2490 }
2491 // At this point mid is disconnected from the in-use list so
2492 // its marked next field no longer has any effects.
2493 deflated_count++;
2494 Atomic::dec(count_p);
2495 chk_for_list_loop(OrderAccess::load_acquire(list_p),
2496 OrderAccess::load_acquire(count_p));
2497 chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p),
2498 OrderAccess::load_acquire(count_p));
2499 // mid is current tail in the free_head_p list so NULL terminate it
2500 // (which also unmarks it):
2501 set_next(mid, NULL);
2502
2503 // All the list management is done so move on to the next one:
2504 mid = next; // mid keeps non-NULL next's marked next field
2505 next = next_next;
2506 } else {
2507 // mid is considered in-use if it does not have an associated
2508 // Java object or mid is not old or deflation did not succeed.
2509 // A mid->is_new() node can be seen here when it is freshly
2510 // returned by om_alloc() (and skips the deflation code path).
2511 // A mid->is_old() node can be seen here when deflation failed.
2512 // A mid->is_free() node can be seen here when a fresh node from
2513 // om_alloc() is released by om_release() due to losing the race
2514 // in inflate().
2515
2516 // All the list management is done so move on to the next one:
2517 if (cur_mid_in_use != NULL) {
2518 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2519 }
2520 // The next cur_mid_in_use keeps mid's marked next field so
2521 // that it is stable for a possible next field change. It
2522 // cannot be modified by om_release() while it is marked.
2523 cur_mid_in_use = mid;
2524 mid = next; // mid keeps non-NULL next's marked next field
2525 next = next_next;
2526
2527 if (SafepointSynchronize::is_synchronizing() &&
2528 cur_mid_in_use != OrderAccess::load_acquire(list_p) &&
2529 cur_mid_in_use->is_old()) {
2530 // If a safepoint has started and cur_mid_in_use is not the list
2531 // head and is old, then it is safe to use as saved state. Return
2532 // to the caller before blocking.
2533 *saved_mid_in_use_p = cur_mid_in_use;
2534 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2535 if (mid != NULL) {
2536 set_next(mid, next); // umark mid
2537 }
2538 return deflated_count;
2539 }
2540 }
2541 if (mid == NULL) {
2542 if (cur_mid_in_use != NULL) {
2543 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2544 }
2545 break; // Reached end of the list so nothing more to deflate.
2546 }
2547
2548 // The current mid's next field is marked at this point. If we have
2549 // a cur_mid_in_use, then its next field is also marked at this point.
2550 }
2551 // We finished the list without a safepoint starting so there's
2552 // no need to save state.
2553 *saved_mid_in_use_p = NULL;
2554 return deflated_count;
2555 }
2556
2557 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2558 OrderAccess::release_store(&counters->n_in_use, 0); // currently associated with objects
2559 OrderAccess::release_store(&counters->n_in_circulation, 0); // extant
2560 OrderAccess::release_store(&counters->n_scavenged, 0); // reclaimed (global and per-thread)
2561 OrderAccess::release_store(&counters->per_thread_scavenged, 0); // per-thread scavenge total
2562 counters->per_thread_times = 0.0; // per-thread scavenge times
2563 }
2564
2565 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
2566 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2567
2568 if (AsyncDeflateIdleMonitors) {
2569 // Nothing to do when global idle ObjectMonitors are deflated using
2570 // a JavaThread unless a special deflation has been requested.
2571 if (!is_special_deflation_requested()) {
2572 return;
2573 }
2574 }
2575
2576 bool deflated = false;
2577
2578 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2579 ObjectMonitor* free_tail_p = NULL;
2580 elapsedTimer timer;
2581
2582 if (log_is_enabled(Info, monitorinflation)) {
2583 timer.start();
2584 }
2585
2586 // Note: the thread-local monitors lists get deflated in
2587 // a separate pass. See deflate_thread_local_monitors().
2588
2589 // For moribund threads, scan g_om_in_use_list
2590 int deflated_count = 0;
2591 if (OrderAccess::load_acquire(&g_om_in_use_list) != NULL) {
2592 // Update n_in_circulation before g_om_in_use_count is updated by deflation.
2593 Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_circulation);
2594
2595 deflated_count = deflate_monitor_list(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p);
2596 Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_use);
2597 }
2598
2599 if (free_head_p != NULL) {
2600 // Move the deflated ObjectMonitors back to the global free list.
2601 // No races on the working free list so no need for load_acquire().
2602 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2603 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2604 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2605 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2606 Atomic::add(deflated_count, &counters->n_scavenged);
2607 }
2608 timer.stop();
2609
2610 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2611 LogStreamHandle(Info, monitorinflation) lsh_info;
2612 LogStream* ls = NULL;
2613 if (log_is_enabled(Debug, monitorinflation)) {
2614 ls = &lsh_debug;
2615 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2616 ls = &lsh_info;
2617 }
2618 if (ls != NULL) {
2619 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2620 }
2621 }
2622
2623 // Deflate global idle ObjectMonitors using a JavaThread.
2624 //
2625 void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() {
2626 assert(AsyncDeflateIdleMonitors, "sanity check");
2627 assert(Thread::current()->is_Java_thread(), "precondition");
2628 JavaThread* self = JavaThread::current();
2629
2630 deflate_common_idle_monitors_using_JT(true /* is_global */, self);
2631 }
2632
2633 // Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread.
2634 //
2635 void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) {
2636 assert(AsyncDeflateIdleMonitors, "sanity check");
2637 assert(Thread::current()->is_Java_thread(), "precondition");
2638
2639 deflate_common_idle_monitors_using_JT(false /* !is_global */, target);
2640 }
2641
2642 // Deflate global or per-thread idle ObjectMonitors using a JavaThread.
2643 //
2644 void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) {
2645 JavaThread* self = JavaThread::current();
2646
2647 int deflated_count = 0;
2648 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors
2649 ObjectMonitor* free_tail_p = NULL;
2650 ObjectMonitor* saved_mid_in_use_p = NULL;
2651 elapsedTimer timer;
2652
2653 if (log_is_enabled(Info, monitorinflation)) {
2654 timer.start();
2655 }
2656
2657 if (is_global) {
2658 OM_PERFDATA_OP(MonExtant, set_value(OrderAccess::load_acquire(&g_om_in_use_count)));
2659 } else {
2660 OM_PERFDATA_OP(MonExtant, inc(OrderAccess::load_acquire(&target->om_in_use_count)));
2661 }
2662
2663 do {
2664 int local_deflated_count;
2665 if (is_global) {
2666 local_deflated_count = deflate_monitor_list_using_JT(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2667 } else {
2668 local_deflated_count = deflate_monitor_list_using_JT(&target->om_in_use_list, &target->om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2669 }
2670 deflated_count += local_deflated_count;
2671
2672 if (free_head_p != NULL) {
2673 // Move the deflated ObjectMonitors to the global free list.
2674 // No races on the working list so no need for load_acquire().
2675 guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count);
2676 // Note: The target thread can be doing an om_alloc() that
2677 // is trying to prepend an ObjectMonitor on its in-use list
2678 // at the same time that we have deflated the current in-use
2679 // list head and put it on the local free list. prepend_to_common()
2680 // will detect the race and retry which avoids list corruption,
2681 // but the next field in free_tail_p can flicker to marked
2682 // and then unmarked while prepend_to_common() is sorting it
2683 // all out.
2684 assert(unmarked_next(free_tail_p) == NULL, "must be NULL: _next_om="
2685 INTPTR_FORMAT, p2i(unmarked_next(free_tail_p)));
2686
2687 prepend_list_to_g_free_list(free_head_p, free_tail_p, local_deflated_count);
2688
2689 OM_PERFDATA_OP(Deflations, inc(local_deflated_count));
2690 }
2691
2692 if (saved_mid_in_use_p != NULL) {
2693 // deflate_monitor_list_using_JT() detected a safepoint starting.
2694 timer.stop();
2695 {
2696 if (is_global) {
2697 log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint.");
2698 } else {
2699 log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target));
2700 }
2701 assert(SafepointSynchronize::is_synchronizing(), "sanity check");
2702 ThreadBlockInVM blocker(self);
2703 }
2704 // Prepare for another loop after the safepoint.
2705 free_head_p = NULL;
2706 free_tail_p = NULL;
2707 if (log_is_enabled(Info, monitorinflation)) {
2708 timer.start();
2709 }
2710 }
2711 } while (saved_mid_in_use_p != NULL);
2712 timer.stop();
2713
2714 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2715 LogStreamHandle(Info, monitorinflation) lsh_info;
2716 LogStream* ls = NULL;
2717 if (log_is_enabled(Debug, monitorinflation)) {
2718 ls = &lsh_debug;
2719 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2720 ls = &lsh_info;
2721 }
2722 if (ls != NULL) {
2723 if (is_global) {
2724 ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2725 } else {
2726 ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count);
2727 }
2728 }
2729 }
2730
2731 void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2732 // Report the cumulative time for deflating each thread's idle
2733 // monitors. Note: if the work is split among more than one
2734 // worker thread, then the reported time will likely be more
2735 // than a beginning to end measurement of the phase.
2736 // Note: AsyncDeflateIdleMonitors only deflates per-thread idle
2737 // monitors at a safepoint when a special deflation has been requested.
2738 log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d",
2739 counters->per_thread_times,
2740 OrderAccess::load_acquire(&counters->per_thread_scavenged));
2741
2742 bool needs_special_deflation = is_special_deflation_requested();
2743 if (!AsyncDeflateIdleMonitors || needs_special_deflation) {
2744 // AsyncDeflateIdleMonitors does not use these counters unless
2745 // there is a special deflation request.
2746
2747 OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged));
2748 OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation));
2749 }
2750
2751 if (log_is_enabled(Debug, monitorinflation)) {
2752 // exit_globals()'s call to audit_and_print_stats() is done
2753 // at the Info level.
2754 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2755 } else if (log_is_enabled(Info, monitorinflation)) {
2756 log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, "
2757 "g_om_free_count=%d",
2758 OrderAccess::load_acquire(&g_om_population),
2759 OrderAccess::load_acquire(&g_om_in_use_count),
2760 OrderAccess::load_acquire(&g_om_free_count));
2761 }
2762
2763 ForceMonitorScavenge = 0; // Reset
2764 GVars.stw_random = os::random();
2765 GVars.stw_cycle++;
2766 if (needs_special_deflation) {
2767 set_is_special_deflation_requested(false); // special deflation is done
2768 }
2769 }
2770
2771 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
2772 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2773
2774 if (AsyncDeflateIdleMonitors && !is_special_deflation_requested()) {
2775 // Nothing to do if a special deflation has NOT been requested.
2776 return;
2777 }
2778
2779 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2780 ObjectMonitor* free_tail_p = NULL;
2781 elapsedTimer timer;
2782
2783 if (log_is_enabled(Info, safepoint, cleanup) ||
2784 log_is_enabled(Info, monitorinflation)) {
2785 timer.start();
2786 }
2787
2788 // Update n_in_circulation before om_in_use_count is updated by deflation.
2789 Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_circulation);
2790
2791 int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p);
2792 Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_use);
2793
2794 if (free_head_p != NULL) {
2795 // Move the deflated ObjectMonitors back to the global free list.
2796 // No races on the working list so no need for load_acquire().
2797 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2798 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2799 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2800 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2801 Atomic::add(deflated_count, &counters->n_scavenged);
2802 Atomic::add(deflated_count, &counters->per_thread_scavenged);
2803 }
2804
2805 timer.stop();
2806 // Safepoint logging cares about cumulative per_thread_times and
2807 // we'll capture most of the cost, but not the muxRelease() which
2808 // should be cheap.
2809 counters->per_thread_times += timer.seconds();
2810
2811 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2812 LogStreamHandle(Info, monitorinflation) lsh_info;
2813 LogStream* ls = NULL;
2814 if (log_is_enabled(Debug, monitorinflation)) {
2815 ls = &lsh_debug;
2816 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2817 ls = &lsh_info;
2818 }
2819 if (ls != NULL) {
2820 ls->print_cr("jt=" INTPTR_FORMAT ": deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(thread), timer.seconds(), deflated_count);
2821 }
2822 }
2823
2824 // Monitor cleanup on JavaThread::exit
2825
2826 // Iterate through monitor cache and attempt to release thread's monitors
2827 // Gives up on a particular monitor if an exception occurs, but continues
2828 // the overall iteration, swallowing the exception.
2829 class ReleaseJavaMonitorsClosure: public MonitorClosure {
2830 private:
2831 TRAPS;
2832
2833 public:
2834 ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {}
2835 void do_monitor(ObjectMonitor* mid) {
2836 if (mid->owner() == THREAD) {
2837 (void)mid->complete_exit(CHECK);
2838 }
2839 }
2840 };
2841
2842 // Release all inflated monitors owned by THREAD. Lightweight monitors are
2843 // ignored. This is meant to be called during JNI thread detach which assumes
2844 // all remaining monitors are heavyweight. All exceptions are swallowed.
2845 // Scanning the extant monitor list can be time consuming.
2846 // A simple optimization is to add a per-thread flag that indicates a thread
2847 // called jni_monitorenter() during its lifetime.
2848 //
2849 // Instead of No_Savepoint_Verifier it might be cheaper to
2850 // use an idiom of the form:
2851 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
2852 // <code that must not run at safepoint>
2853 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
2854 // Since the tests are extremely cheap we could leave them enabled
2855 // for normal product builds.
2856
2857 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
2858 assert(THREAD == JavaThread::current(), "must be current Java thread");
2859 NoSafepointVerifier nsv;
2860 ReleaseJavaMonitorsClosure rjmc(THREAD);
2861 ObjectSynchronizer::monitors_iterate(&rjmc);
2862 THREAD->clear_pending_exception();
2863 }
2864
2865 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
2866 switch (cause) {
2867 case inflate_cause_vm_internal: return "VM Internal";
2868 case inflate_cause_monitor_enter: return "Monitor Enter";
2869 case inflate_cause_wait: return "Monitor Wait";
2870 case inflate_cause_notify: return "Monitor Notify";
2871 case inflate_cause_hash_code: return "Monitor Hash Code";
2872 case inflate_cause_jni_enter: return "JNI Monitor Enter";
2873 case inflate_cause_jni_exit: return "JNI Monitor Exit";
2874 default:
2875 ShouldNotReachHere();
2876 }
2877 return "Unknown";
2878 }
2879
2880 //------------------------------------------------------------------------------
2881 // Debugging code
2882
2883 u_char* ObjectSynchronizer::get_gvars_addr() {
2884 return (u_char*)&GVars;
2885 }
2886
2887 u_char* ObjectSynchronizer::get_gvars_hc_sequence_addr() {
2888 return (u_char*)&GVars.hc_sequence;
2889 }
2890
2891 size_t ObjectSynchronizer::get_gvars_size() {
2892 return sizeof(SharedGlobals);
2893 }
2894
2895 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
2896 return (u_char*)&GVars.stw_random;
2897 }
2898
2899 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
2900 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
2901
2902 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2903 LogStreamHandle(Info, monitorinflation) lsh_info;
2904 LogStreamHandle(Trace, monitorinflation) lsh_trace;
2905 LogStream* ls = NULL;
2906 if (log_is_enabled(Trace, monitorinflation)) {
2907 ls = &lsh_trace;
2908 } else if (log_is_enabled(Debug, monitorinflation)) {
2909 ls = &lsh_debug;
2910 } else if (log_is_enabled(Info, monitorinflation)) {
2911 ls = &lsh_info;
2912 }
2913 assert(ls != NULL, "sanity check");
2914
2915 // Log counts for the global and per-thread monitor lists:
2916 int chk_om_population = log_monitor_list_counts(ls);
2917 int error_cnt = 0;
2918
2919 ls->print_cr("Checking global lists:");
2920
2921 // Check g_om_population:
2922 if (OrderAccess::load_acquire(&g_om_population) == chk_om_population) {
2923 ls->print_cr("g_om_population=%d equals chk_om_population=%d",
2924 OrderAccess::load_acquire(&g_om_population),
2925 chk_om_population);
2926 } else {
2927 ls->print_cr("ERROR: g_om_population=%d is not equal to "
2928 "chk_om_population=%d",
2929 OrderAccess::load_acquire(&g_om_population),
2930 chk_om_population);
2931 error_cnt++;
2932 }
2933
2934 // Check g_om_in_use_list and g_om_in_use_count:
2935 chk_global_in_use_list_and_count(ls, &error_cnt);
2936
2937 // Check g_free_list and g_om_free_count:
2938 chk_global_free_list_and_count(ls, &error_cnt);
2939
2940 ls->print_cr("Checking per-thread lists:");
2941
2942 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2943 // Check om_in_use_list and om_in_use_count:
2944 chk_per_thread_in_use_list_and_count(jt, ls, &error_cnt);
2945
2946 // Check om_free_list and om_free_count:
2947 chk_per_thread_free_list_and_count(jt, ls, &error_cnt);
2948 }
2949
2950 if (error_cnt == 0) {
2951 ls->print_cr("No errors found in monitor list checks.");
2952 } else {
2953 log_error(monitorinflation)("found monitor list errors: error_cnt=%d", error_cnt);
2954 }
2955
2956 if ((on_exit && log_is_enabled(Info, monitorinflation)) ||
2957 (!on_exit && log_is_enabled(Trace, monitorinflation))) {
2958 // When exiting this log output is at the Info level. When called
2959 // at a safepoint, this log output is at the Trace level since
2960 // there can be a lot of it.
2961 log_in_use_monitor_details(ls);
2962 }
2963
2964 ls->flush();
2965
2966 guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
2967 }
2968
2969 // Check a free monitor entry; log any errors.
2970 void ObjectSynchronizer::chk_free_entry(JavaThread* jt, ObjectMonitor* n,
2971 outputStream * out, int *error_cnt_p) {
2972 stringStream ss;
2973 if (n->is_busy()) {
2974 if (jt != NULL) {
2975 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2976 ": free per-thread monitor must not be busy: %s", p2i(jt),
2977 p2i(n), n->is_busy_to_string(&ss));
2978 } else {
2979 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2980 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
2981 }
2982 *error_cnt_p = *error_cnt_p + 1;
2983 }
2984 if (n->header().value() != 0) {
2985 if (jt != NULL) {
2986 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2987 ": free per-thread monitor must have NULL _header "
2988 "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n),
2989 n->header().value());
2990 *error_cnt_p = *error_cnt_p + 1;
2991 } else if (!AsyncDeflateIdleMonitors) {
2992 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
2993 "must have NULL _header field: _header=" INTPTR_FORMAT,
2994 p2i(n), n->header().value());
2995 *error_cnt_p = *error_cnt_p + 1;
2996 }
2997 }
2998 if (n->object() != NULL) {
2999 if (jt != NULL) {
3000 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3001 ": free per-thread monitor must have NULL _object "
3002 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
3003 p2i(n->object()));
3004 } else {
3005 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3006 "must have NULL _object field: _object=" INTPTR_FORMAT,
3007 p2i(n), p2i(n->object()));
3008 }
3009 *error_cnt_p = *error_cnt_p + 1;
3010 }
3011 }
3012
3013 // Check the global free list and count; log the results of the checks.
3014 void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out,
3015 int *error_cnt_p) {
3016 int chk_om_free_count = 0;
3017 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_free_list); n != NULL; n = unmarked_next(n)) {
3018 chk_free_entry(NULL /* jt */, n, out, error_cnt_p);
3019 chk_om_free_count++;
3020 }
3021 if (OrderAccess::load_acquire(&g_om_free_count) == chk_om_free_count) {
3022 out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d",
3023 OrderAccess::load_acquire(&g_om_free_count),
3024 chk_om_free_count);
3025 } else {
3026 // With lock free access to g_free_list, it is possible for an
3027 // ObjectMonitor to be prepended to g_free_list after we started
3028 // calculating chk_om_free_count so g_om_free_count may not
3029 // match anymore.
3030 out->print_cr("WARNING: g_om_free_count=%d is not equal to "
3031 "chk_om_free_count=%d",
3032 OrderAccess::load_acquire(&g_om_free_count),
3033 chk_om_free_count);
3034 }
3035 }
3036
3037 // Check the global in-use list and count; log the results of the checks.
3038 void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out,
3039 int *error_cnt_p) {
3040 int chk_om_in_use_count = 0;
3041 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) {
3042 chk_in_use_entry(NULL /* jt */, n, out, error_cnt_p);
3043 chk_om_in_use_count++;
3044 }
3045 if (OrderAccess::load_acquire(&g_om_in_use_count) == chk_om_in_use_count) {
3046 out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d",
3047 OrderAccess::load_acquire(&g_om_in_use_count),
3048 chk_om_in_use_count);
3049 } else {
3050 out->print_cr("ERROR: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d",
3051 OrderAccess::load_acquire(&g_om_in_use_count),
3052 chk_om_in_use_count);
3053 *error_cnt_p = *error_cnt_p + 1;
3054 }
3055 }
3056
3057 // Check an in-use monitor entry; log any errors.
3058 void ObjectSynchronizer::chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,
3059 outputStream * out, int *error_cnt_p) {
3060 if (n->header().value() == 0) {
3061 if (jt != NULL) {
3062 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3063 ": in-use per-thread monitor must have non-NULL _header "
3064 "field.", p2i(jt), p2i(n));
3065 } else {
3066 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
3067 "must have non-NULL _header field.", p2i(n));
3068 }
3069 *error_cnt_p = *error_cnt_p + 1;
3070 }
3071 if (n->object() == NULL) {
3072 if (jt != NULL) {
3073 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3074 ": in-use per-thread monitor must have non-NULL _object "
3075 "field.", p2i(jt), p2i(n));
3076 } else {
3077 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
3078 "must have non-NULL _object field.", p2i(n));
3079 }
3080 *error_cnt_p = *error_cnt_p + 1;
3081 }
3082 const oop obj = (oop)n->object();
3083 const markWord mark = obj->mark();
3084 if (!mark.has_monitor()) {
3085 if (jt != NULL) {
3086 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3087 ": in-use per-thread monitor's object does not think "
3088 "it has a monitor: obj=" INTPTR_FORMAT ", mark="
3089 INTPTR_FORMAT, p2i(jt), p2i(n), p2i(obj), mark.value());
3090 } else {
3091 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
3092 "monitor's object does not think it has a monitor: obj="
3093 INTPTR_FORMAT ", mark=" INTPTR_FORMAT, p2i(n),
3094 p2i(obj), mark.value());
3095 }
3096 *error_cnt_p = *error_cnt_p + 1;
3097 }
3098 ObjectMonitor* const obj_mon = mark.monitor();
3099 if (n != obj_mon) {
3100 if (jt != NULL) {
3101 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3102 ": in-use per-thread monitor's object does not refer "
3103 "to the same monitor: obj=" INTPTR_FORMAT ", mark="
3104 INTPTR_FORMAT ", obj_mon=" INTPTR_FORMAT, p2i(jt),
3105 p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3106 } else {
3107 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
3108 "monitor's object does not refer to the same monitor: obj="
3109 INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
3110 INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3111 }
3112 *error_cnt_p = *error_cnt_p + 1;
3113 }
3114 }
3115
3116 // Check the thread's free list and count; log the results of the checks.
3117 void ObjectSynchronizer::chk_per_thread_free_list_and_count(JavaThread *jt,
3118 outputStream * out,
3119 int *error_cnt_p) {
3120 int chk_om_free_count = 0;
3121 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_free_list); n != NULL; n = unmarked_next(n)) {
3122 chk_free_entry(jt, n, out, error_cnt_p);
3123 chk_om_free_count++;
3124 }
3125 if (OrderAccess::load_acquire(&jt->om_free_count) == chk_om_free_count) {
3126 out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals "
3127 "chk_om_free_count=%d", p2i(jt),
3128 OrderAccess::load_acquire(&jt->om_free_count),
3129 chk_om_free_count);
3130 } else {
3131 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not "
3132 "equal to chk_om_free_count=%d", p2i(jt),
3133 OrderAccess::load_acquire(&jt->om_free_count),
3134 chk_om_free_count);
3135 *error_cnt_p = *error_cnt_p + 1;
3136 }
3137 }
3138
3139 // Check the thread's in-use list and count; log the results of the checks.
3140 void ObjectSynchronizer::chk_per_thread_in_use_list_and_count(JavaThread *jt,
3141 outputStream * out,
3142 int *error_cnt_p) {
3143 int chk_om_in_use_count = 0;
3144 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) {
3145 chk_in_use_entry(jt, n, out, error_cnt_p);
3146 chk_om_in_use_count++;
3147 }
3148 if (OrderAccess::load_acquire(&jt->om_in_use_count) == chk_om_in_use_count) {
3149 out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals "
3150 "chk_om_in_use_count=%d", p2i(jt),
3151 OrderAccess::load_acquire(&jt->om_in_use_count),
3152 chk_om_in_use_count);
3153 } else {
3154 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not "
3155 "equal to chk_om_in_use_count=%d", p2i(jt),
3156 OrderAccess::load_acquire(&jt->om_in_use_count),
3157 chk_om_in_use_count);
3158 *error_cnt_p = *error_cnt_p + 1;
3159 }
3160 }
3161
3162 // Log details about ObjectMonitors on the in-use lists. The 'BHL'
3163 // flags indicate why the entry is in-use, 'object' and 'object type'
3164 // indicate the associated object and its type.
3165 void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out) {
3166 stringStream ss;
3167 if (OrderAccess::load_acquire(&g_om_in_use_count) > 0) {
3168 out->print_cr("In-use global monitor info:");
3169 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3170 out->print_cr("%18s %s %7s %18s %18s",
3171 "monitor", "BHL", "ref_cnt", "object", "object type");
3172 out->print_cr("================== === ======= ================== ==================");
3173 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) {
3174 const oop obj = (oop) n->object();
3175 const markWord mark = n->header();
3176 ResourceMark rm;
3177 out->print(INTPTR_FORMAT " %d%d%d %7d " INTPTR_FORMAT " %s",
3178 p2i(n), n->is_busy() != 0, mark.hash() != 0,
3179 n->owner() != NULL, (int)n->ref_count(), p2i(obj),
3180 obj->klass()->external_name());
3181 if (n->is_busy() != 0) {
3182 out->print(" (%s)", n->is_busy_to_string(&ss));
3183 ss.reset();
3184 }
3185 out->cr();
3186 }
3187 }
3188
3189 out->print_cr("In-use per-thread monitor info:");
3190 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3191 out->print_cr("%18s %18s %s %7s %18s %18s",
3192 "jt", "monitor", "BHL", "ref_cnt", "object", "object type");
3193 out->print_cr("================== ================== === ======= ================== ==================");
3194 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3195 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) {
3196 const oop obj = (oop) n->object();
3197 const markWord mark = n->header();
3198 ResourceMark rm;
3199 out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d %7d "
3200 INTPTR_FORMAT " %s", p2i(jt), p2i(n), n->is_busy() != 0,
3201 mark.hash() != 0, n->owner() != NULL, (int)n->ref_count(),
3202 p2i(obj), obj->klass()->external_name());
3203 if (n->is_busy() != 0) {
3204 out->print(" (%s)", n->is_busy_to_string(&ss));
3205 ss.reset();
3206 }
3207 out->cr();
3208 }
3209 }
3210
3211 out->flush();
3212 }
3213
3214 // Log counts for the global and per-thread monitor lists and return
3215 // the population count.
3216 int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) {
3217 int pop_count = 0;
3218 out->print_cr("%18s %10s %10s %10s",
3219 "Global Lists:", "InUse", "Free", "Total");
3220 out->print_cr("================== ========== ========== ==========");
3221 out->print_cr("%18s %10d %10d %10d", "",
3222 OrderAccess::load_acquire(&g_om_in_use_count),
3223 OrderAccess::load_acquire(&g_om_free_count),
3224 OrderAccess::load_acquire(&g_om_population));
3225 pop_count += OrderAccess::load_acquire(&g_om_in_use_count) +
3226 OrderAccess::load_acquire(&g_om_free_count);
3227
3228 out->print_cr("%18s %10s %10s %10s",
3229 "Per-Thread Lists:", "InUse", "Free", "Provision");
3230 out->print_cr("================== ========== ========== ==========");
3231
3232 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3233 out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt),
3234 OrderAccess::load_acquire(&jt->om_in_use_count),
3235 OrderAccess::load_acquire(&jt->om_free_count),
3236 jt->om_free_provision);
3237 pop_count += OrderAccess::load_acquire(&jt->om_in_use_count) +
3238 OrderAccess::load_acquire(&jt->om_free_count);
3239 }
3240 return pop_count;
3241 }
3242
3243 #ifndef PRODUCT
3244
3245 // Check if monitor belongs to the monitor cache
3246 // The list is grow-only so it's *relatively* safe to traverse
3247 // the list of extant blocks without taking a lock.
3248
3249 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
3250 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
3251 while (block != NULL) {
3252 assert(block->object() == CHAINMARKER, "must be a block header");
3253 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
3254 address mon = (address)monitor;
3255 address blk = (address)block;
3256 size_t diff = mon - blk;
3257 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
3258 return 1;
3259 }
3260 // unmarked_next() is not needed with g_block_list (no next field marking).
3261 block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om);
3262 }
3263 return 0;
3264 }
3265
3266 #endif