1 /*
2 * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "logging/log.hpp"
28 #include "logging/logStream.hpp"
29 #include "jfr/jfrEvents.hpp"
30 #include "memory/allocation.inline.hpp"
31 #include "memory/metaspaceShared.hpp"
32 #include "memory/padded.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/markWord.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "runtime/atomic.hpp"
38 #include "runtime/biasedLocking.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/interfaceSupport.inline.hpp"
41 #include "runtime/mutexLocker.hpp"
42 #include "runtime/objectMonitor.hpp"
43 #include "runtime/objectMonitor.inline.hpp"
44 #include "runtime/osThread.hpp"
45 #include "runtime/safepointVerifiers.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "runtime/synchronizer.hpp"
49 #include "runtime/thread.inline.hpp"
50 #include "runtime/timer.hpp"
51 #include "runtime/vframe.hpp"
52 #include "runtime/vmThread.hpp"
53 #include "utilities/align.hpp"
54 #include "utilities/dtrace.hpp"
55 #include "utilities/events.hpp"
56 #include "utilities/preserveException.hpp"
57
58 // The "core" versions of monitor enter and exit reside in this file.
59 // The interpreter and compilers contain specialized transliterated
60 // variants of the enter-exit fast-path operations. See i486.ad fast_lock(),
61 // for instance. If you make changes here, make sure to modify the
62 // interpreter, and both C1 and C2 fast-path inline locking code emission.
63 //
64 // -----------------------------------------------------------------------------
65
66 #ifdef DTRACE_ENABLED
67
68 // Only bother with this argument setup if dtrace is available
69 // TODO-FIXME: probes should not fire when caller is _blocked. assert() accordingly.
70
71 #define DTRACE_MONITOR_PROBE_COMMON(obj, thread) \
72 char* bytes = NULL; \
73 int len = 0; \
74 jlong jtid = SharedRuntime::get_java_tid(thread); \
75 Symbol* klassname = ((oop)(obj))->klass()->name(); \
76 if (klassname != NULL) { \
77 bytes = (char*)klassname->bytes(); \
78 len = klassname->utf8_length(); \
79 }
80
81 #define DTRACE_MONITOR_WAIT_PROBE(monitor, obj, thread, millis) \
82 { \
83 if (DTraceMonitorProbes) { \
84 DTRACE_MONITOR_PROBE_COMMON(obj, thread); \
85 HOTSPOT_MONITOR_WAIT(jtid, \
86 (uintptr_t)(monitor), bytes, len, (millis)); \
87 } \
88 }
89
90 #define HOTSPOT_MONITOR_PROBE_notify HOTSPOT_MONITOR_NOTIFY
91 #define HOTSPOT_MONITOR_PROBE_notifyAll HOTSPOT_MONITOR_NOTIFYALL
92 #define HOTSPOT_MONITOR_PROBE_waited HOTSPOT_MONITOR_WAITED
93
94 #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread) \
95 { \
96 if (DTraceMonitorProbes) { \
97 DTRACE_MONITOR_PROBE_COMMON(obj, thread); \
98 HOTSPOT_MONITOR_PROBE_##probe(jtid, /* probe = waited */ \
99 (uintptr_t)(monitor), bytes, len); \
100 } \
101 }
102
103 #else // ndef DTRACE_ENABLED
104
105 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon) {;}
106 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon) {;}
107
108 #endif // ndef DTRACE_ENABLED
109
110 // This exists only as a workaround of dtrace bug 6254741
111 int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
112 DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
113 return 0;
114 }
115
116 #define NINFLATIONLOCKS 256
117 static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];
118
119 // global list of blocks of monitors
120 PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL;
121 bool volatile ObjectSynchronizer::_is_async_deflation_requested = false;
122 bool volatile ObjectSynchronizer::_is_special_deflation_requested = false;
123 jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0;
124
125 // Global ObjectMonitor free list. Newly allocated and deflated
126 // ObjectMonitors are prepended here.
127 static ObjectMonitor* volatile g_free_list = NULL;
128 // Global ObjectMonitor in-use list. When a JavaThread is exiting,
129 // ObjectMonitors on its per-thread in-use list are prepended here.
130 static ObjectMonitor* volatile g_om_in_use_list = NULL;
131
132 static volatile int g_om_free_count = 0; // # on g_free_list
133 static volatile int g_om_in_use_count = 0; // # on g_om_in_use_list
134 static volatile int g_om_population = 0; // # Extant -- in circulation
135
136 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
137
138
139 // =====================> List Management functions
140
141 // Return true if the ObjectMonitor's next field is marked.
142 // Otherwise returns false.
143 static bool is_next_marked(ObjectMonitor* om) {
144 return ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & 0x1) != 0;
145 }
146
147 // Mark an ObjectMonitor* and return it. Note: the om parameter
148 // may or may not have been marked originally.
149 static ObjectMonitor* mark_om_ptr(ObjectMonitor* om) {
150 return (ObjectMonitor*)((intptr_t)om | 0x1);
151 }
152
153 // Mark the next field in an ObjectMonitor. If marking was successful,
154 // then the unmarked next field is returned via parameter and true is
155 // returned. Otherwise false is returned.
156 static bool mark_next(ObjectMonitor* om, ObjectMonitor** next_p) {
157 // Get current next field without any marking value.
158 ObjectMonitor* next = (ObjectMonitor*)
159 ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1);
160 if (Atomic::cmpxchg(mark_om_ptr(next), &om->_next_om, next) != next) {
161 return false; // Could not mark the next field or it was already marked.
162 }
163 *next_p = next;
164 return true;
165 }
166
167 // Loop until we mark the next field in an ObjectMonitor. The unmarked
168 // next field is returned.
169 static ObjectMonitor* mark_next_loop(ObjectMonitor* om) {
170 ObjectMonitor* next;
171 while (true) {
172 if (mark_next(om, &next)) {
173 // Marked om's next field so return the unmarked value.
174 return next;
175 }
176 }
177 }
178
179 // Set the next field in an ObjectMonitor to the specified value.
180 // The caller of set_next() must be the same thread that marked the
181 // ObjectMonitor.
182 static void set_next(ObjectMonitor* om, ObjectMonitor* value) {
183 OrderAccess::release_store(&om->_next_om, value);
184 }
185
186 // Mark the next field in the list head ObjectMonitor. If marking was
187 // successful, then the mid and the unmarked next field are returned
188 // via parameter and true is returned. Otherwise false is returned.
189 static bool mark_list_head(ObjectMonitor* volatile * list_p,
190 ObjectMonitor** mid_p, ObjectMonitor** next_p) {
191 while (true) {
192 ObjectMonitor* mid = OrderAccess::load_acquire(list_p);
193 if (mid == NULL) {
194 return false; // The list is empty so nothing to mark.
195 }
196 if (mark_next(mid, next_p)) {
197 if (OrderAccess::load_acquire(list_p) != mid) {
198 // The list head changed so we have to retry.
199 set_next(mid, *next_p); // unmark mid
200 continue;
201 }
202 // We marked next field to guard against races.
203 *mid_p = mid;
204 return true;
205 }
206 }
207 }
208
209 // Return the unmarked next field in an ObjectMonitor. Note: the next
210 // field may or may not have been marked originally.
211 static ObjectMonitor* unmarked_next(ObjectMonitor* om) {
212 return (ObjectMonitor*)((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1);
213 }
214
215 #if 0
216 // XXX - this is unused
217 // Unmark the next field in an ObjectMonitor. Requires that the next
218 // field be marked.
219 static void unmark_next(ObjectMonitor* om) {
220 ADIM_guarantee(is_next_marked(om), "next field must be marked: next=" INTPTR_FORMAT, p2i(om->_next_om));
221
222 ObjectMonitor* next = unmarked_next(om);
223 set_next(om, next);
224 }
225 #endif
226
227 volatile int visit_counter = 42;
228 static void chk_for_list_loop(ObjectMonitor* list, int count) {
229 if (!CheckMonitorLists) {
230 return;
231 }
232 int l_visit_counter = Atomic::add(1, &visit_counter);
233 int l_count = 0;
234 ObjectMonitor* prev = NULL;
235 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
236 if (mid->visit_marker == l_visit_counter) {
237 log_error(monitorinflation)("ERROR: prev=" INTPTR_FORMAT ", l_count=%d"
238 " refers to an ObjectMonitor that has"
239 " already been visited: mid=" INTPTR_FORMAT,
240 p2i(prev), l_count, p2i(mid));
241 fatal("list=" INTPTR_FORMAT " of %d items has a loop.", p2i(list), count);
242 }
243 mid->visit_marker = l_visit_counter;
244 prev = mid;
245 if (++l_count > count + 1024 * 1024) {
246 fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d",
247 p2i(list), count, l_count);
248 }
249 }
250 }
251
252 static void chk_om_not_on_list(ObjectMonitor* om, ObjectMonitor* list, int count) {
253 if (!CheckMonitorLists) {
254 return;
255 }
256 guarantee(list != om, "ERROR: om=" INTPTR_FORMAT " must not be head of the "
257 "list=" INTPTR_FORMAT ", count=%d", p2i(om), p2i(list), count);
258 int l_count = 0;
259 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
260 if (unmarked_next(mid) == om) {
261 log_error(monitorinflation)("ERROR: mid=" INTPTR_FORMAT ", l_count=%d"
262 " next_om refers to om=" INTPTR_FORMAT,
263 p2i(mid), l_count, p2i(om));
264 fatal("list=" INTPTR_FORMAT " of %d items has bad next_om value.",
265 p2i(list), count);
266 }
267 if (++l_count > count + 1024 * 1024) {
268 fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d",
269 p2i(list), count, l_count);
270 }
271 }
272 }
273
274 static void chk_om_elems_not_on_list(ObjectMonitor* elems, int elems_count,
275 ObjectMonitor* list, int list_count) {
276 if (!CheckMonitorLists) {
277 return;
278 }
279 chk_for_list_loop(elems, elems_count);
280 for (ObjectMonitor* mid = elems; mid != NULL; mid = unmarked_next(mid)) {
281 chk_om_not_on_list(mid, list, list_count);
282 }
283 }
284
285 // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is
286 // the last ObjectMonitor in the list and there are 'count' on the list.
287 // Also updates the specified *count_p.
288 static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail,
289 int count, ObjectMonitor* volatile* list_p,
290 volatile int* count_p) {
291 chk_for_list_loop(OrderAccess::load_acquire(list_p),
292 OrderAccess::load_acquire(count_p));
293 chk_om_elems_not_on_list(list, count, OrderAccess::load_acquire(list_p),
294 OrderAccess::load_acquire(count_p));
295 while (true) {
296 ObjectMonitor* cur = OrderAccess::load_acquire(list_p);
297 // Prepend list to *list_p.
298 ObjectMonitor* next = NULL;
299 if (!mark_next(tail, &next)) {
300 continue; // failed to mark next field so try it all again
301 }
302 set_next(tail, cur); // tail now points to cur (and unmarks tail)
303 if (cur == NULL) {
304 // No potential race with takers or other prependers since
305 // *list_p is empty.
306 if (Atomic::cmpxchg(list, list_p, cur) == cur) {
307 // Successfully switched *list_p to the list value.
308 Atomic::add(count, count_p);
309 break;
310 }
311 // Implied else: try it all again
312 } else {
313 // Try to mark next field to guard against races:
314 if (!mark_next(cur, &next)) {
315 continue; // failed to mark next field so try it all again
316 }
317 // We marked the next field so try to switch *list_p to the list value.
318 if (Atomic::cmpxchg(list, list_p, cur) != cur) {
319 // The list head has changed so unmark the next field and try again:
320 set_next(cur, next);
321 continue;
322 }
323 Atomic::add(count, count_p);
324 set_next(cur, next); // unmark next field
325 break;
326 }
327 }
328 }
329
330 // Prepend a newly allocated block of ObjectMonitors to g_block_list and
331 // g_free_list. Also updates g_om_population and g_om_free_count.
332 void ObjectSynchronizer::prepend_block_to_lists(PaddedObjectMonitor* new_blk) {
333 // First we handle g_block_list:
334 while (true) {
335 PaddedObjectMonitor* cur = OrderAccess::load_acquire(&g_block_list);
336 // Prepend new_blk to g_block_list. The first ObjectMonitor in
337 // a block is reserved for use as linkage to the next block.
338 OrderAccess::release_store(&new_blk[0]._next_om, cur);
339 if (Atomic::cmpxchg(new_blk, &g_block_list, cur) == cur) {
340 // Successfully switched g_block_list to the new_blk value.
341 Atomic::add(_BLOCKSIZE - 1, &g_om_population);
342 break;
343 }
344 // Implied else: try it all again
345 }
346
347 // Second we handle g_free_list:
348 prepend_list_to_common(new_blk + 1, &new_blk[_BLOCKSIZE - 1], _BLOCKSIZE - 1,
349 &g_free_list, &g_om_free_count);
350 }
351
352 // Prepend a list of ObjectMonitors to g_free_list. 'tail' is the last
353 // ObjectMonitor in the list and there are 'count' on the list. Also
354 // updates g_om_free_count.
355 static void prepend_list_to_g_free_list(ObjectMonitor* list,
356 ObjectMonitor* tail, int count) {
357 prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count);
358 }
359
360 // Prepend a list of ObjectMonitors to g_om_in_use_list. 'tail' is the last
361 // ObjectMonitor in the list and there are 'count' on the list. Also
362 // updates g_om_in_use_list.
363 static void prepend_list_to_g_om_in_use_list(ObjectMonitor* list,
364 ObjectMonitor* tail, int count) {
365 prepend_list_to_common(list, tail, count, &g_om_in_use_list, &g_om_in_use_count);
366 }
367
368 // Prepend an ObjectMonitor to the specified list. Also updates
369 // the specified counter.
370 static void prepend_to_common(ObjectMonitor* m, ObjectMonitor* volatile * list_p,
371 int volatile * count_p) {
372 chk_for_list_loop(OrderAccess::load_acquire(list_p),
373 OrderAccess::load_acquire(count_p));
374 chk_om_not_on_list(m, OrderAccess::load_acquire(list_p),
375 OrderAccess::load_acquire(count_p));
376
377 while (true) {
378 ObjectMonitor* cur = OrderAccess::load_acquire(list_p);
379 // Prepend ObjectMonitor to *list_p.
380 ObjectMonitor* next = NULL;
381 if (!mark_next(m, &next)) {
382 continue; // failed to mark next field so try it all again
383 }
384 set_next(m, cur); // m now points to cur (and unmarks m)
385 if (cur == NULL) {
386 // No potential race with other prependers since *list_p is empty.
387 if (Atomic::cmpxchg(m, list_p, cur) == cur) {
388 // Successfully switched *list_p to 'm'.
389 Atomic::inc(count_p);
390 break;
391 }
392 // Implied else: try it all again
393 } else {
394 // Try to mark next field to guard against races:
395 if (!mark_next(cur, &next)) {
396 continue; // failed to mark next field so try it all again
397 }
398 // We marked the next field so try to switch *list_p to 'm'.
399 if (Atomic::cmpxchg(m, list_p, cur) != cur) {
400 // The list head has changed so unmark the next field and try again:
401 set_next(cur, next);
402 continue;
403 }
404 Atomic::inc(count_p);
405 set_next(cur, next); // unmark next field
406 break;
407 }
408 }
409 }
410
411 // Prepend an ObjectMonitor to a per-thread om_free_list.
412 // Also updates the per-thread om_free_count.
413 static void prepend_to_om_free_list(Thread* self, ObjectMonitor* m) {
414 prepend_to_common(m, &self->om_free_list, &self->om_free_count);
415 }
416
417 // Prepend an ObjectMonitor to a per-thread om_in_use_list.
418 // Also updates the per-thread om_in_use_count.
419 static void prepend_to_om_in_use_list(Thread* self, ObjectMonitor* m) {
420 prepend_to_common(m, &self->om_in_use_list, &self->om_in_use_count);
421 }
422
423 // Take an ObjectMonitor from the start of the specified list. Also
424 // decrements the specified counter. Returns NULL if none are available.
425 static ObjectMonitor* take_from_start_of_common(ObjectMonitor* volatile * list_p,
426 int volatile * count_p) {
427 chk_for_list_loop(OrderAccess::load_acquire(list_p),
428 OrderAccess::load_acquire(count_p));
429
430 ObjectMonitor* next = NULL;
431 ObjectMonitor* take = NULL;
432 // Mark the list head to guard against A-B-A race:
433 if (!mark_list_head(list_p, &take, &next)) {
434 return NULL; // None are available.
435 }
436 // Switch marked list head to next (which unmarks the list head, but
437 // leaves take marked):
438 OrderAccess::release_store(list_p, next);
439 Atomic::dec(count_p);
440 // Unmark take, but leave the next value for any lagging list
441 // walkers. It will get cleaned up when take is prepended to
442 // the in-use list:
443 set_next(take, next);
444 return take;
445 }
446
447 // Take an ObjectMonitor from the start of the global free-list. Also
448 // updates g_om_free_count. Returns NULL if none are available.
449 static ObjectMonitor* take_from_start_of_g_free_list() {
450 return take_from_start_of_common(&g_free_list, &g_om_free_count);
451 }
452
453 // Take an ObjectMonitor from the start of a per-thread free-list.
454 // Also updates om_free_count. Returns NULL if none are available.
455 static ObjectMonitor* take_from_start_of_om_free_list(Thread* self) {
456 return take_from_start_of_common(&self->om_free_list, &self->om_free_count);
457 }
458
459
460 // =====================> Quick functions
461
462 // The quick_* forms are special fast-path variants used to improve
463 // performance. In the simplest case, a "quick_*" implementation could
464 // simply return false, in which case the caller will perform the necessary
465 // state transitions and call the slow-path form.
466 // The fast-path is designed to handle frequently arising cases in an efficient
467 // manner and is just a degenerate "optimistic" variant of the slow-path.
468 // returns true -- to indicate the call was satisfied.
469 // returns false -- to indicate the call needs the services of the slow-path.
470 // A no-loitering ordinance is in effect for code in the quick_* family
471 // operators: safepoints or indefinite blocking (blocking that might span a
472 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
473 // entry.
474 //
475 // Consider: An interesting optimization is to have the JIT recognize the
476 // following common idiom:
477 // synchronized (someobj) { .... ; notify(); }
478 // That is, we find a notify() or notifyAll() call that immediately precedes
479 // the monitorexit operation. In that case the JIT could fuse the operations
480 // into a single notifyAndExit() runtime primitive.
481
482 bool ObjectSynchronizer::quick_notify(oopDesc* obj, Thread* self, bool all) {
483 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
484 assert(self->is_Java_thread(), "invariant");
485 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
486 NoSafepointVerifier nsv;
487 if (obj == NULL) return false; // slow-path for invalid obj
488 const markWord mark = obj->mark();
489
490 if (mark.has_locker() && self->is_lock_owned((address)mark.locker())) {
491 // Degenerate notify
492 // stack-locked by caller so by definition the implied waitset is empty.
493 return true;
494 }
495
496 if (mark.has_monitor()) {
497 ObjectMonitor* const mon = mark.monitor();
498 assert(oopDesc::equals((oop) mon->object(), obj), "invariant");
499 if (mon->owner() != self) return false; // slow-path for IMS exception
500
501 if (mon->first_waiter() != NULL) {
502 // We have one or more waiters. Since this is an inflated monitor
503 // that we own, we can transfer one or more threads from the waitset
504 // to the entrylist here and now, avoiding the slow-path.
505 if (all) {
506 DTRACE_MONITOR_PROBE(notifyAll, mon, obj, self);
507 } else {
508 DTRACE_MONITOR_PROBE(notify, mon, obj, self);
509 }
510 int free_count = 0;
511 do {
512 mon->INotify(self);
513 ++free_count;
514 } while (mon->first_waiter() != NULL && all);
515 OM_PERFDATA_OP(Notifications, inc(free_count));
516 }
517 return true;
518 }
519
520 // biased locking and any other IMS exception states take the slow-path
521 return false;
522 }
523
524
525 // The LockNode emitted directly at the synchronization site would have
526 // been too big if it were to have included support for the cases of inflated
527 // recursive enter and exit, so they go here instead.
528 // Note that we can't safely call AsyncPrintJavaStack() from within
529 // quick_enter() as our thread state remains _in_Java.
530
531 bool ObjectSynchronizer::quick_enter(oop obj, Thread* self,
532 BasicLock * lock) {
533 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
534 assert(self->is_Java_thread(), "invariant");
535 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
536 NoSafepointVerifier nsv;
537 if (obj == NULL) return false; // Need to throw NPE
538
539 while (true) {
540 const markWord mark = obj->mark();
541
542 if (mark.has_monitor()) {
543 ObjectMonitorHandle omh;
544 if (!omh.save_om_ptr(obj, mark)) {
545 // Lost a race with async deflation so try again.
546 assert(AsyncDeflateIdleMonitors, "sanity check");
547 continue;
548 }
549 ObjectMonitor* const m = omh.om_ptr();
550 assert(oopDesc::equals((oop) m->object(), obj), "invariant");
551 Thread* const owner = (Thread *) m->_owner;
552
553 // Lock contention and Transactional Lock Elision (TLE) diagnostics
554 // and observability
555 // Case: light contention possibly amenable to TLE
556 // Case: TLE inimical operations such as nested/recursive synchronization
557
558 if (owner == self) {
559 m->_recursions++;
560 return true;
561 }
562
563 // This Java Monitor is inflated so obj's header will never be
564 // displaced to this thread's BasicLock. Make the displaced header
565 // non-NULL so this BasicLock is not seen as recursive nor as
566 // being locked. We do this unconditionally so that this thread's
567 // BasicLock cannot be mis-interpreted by any stack walkers. For
568 // performance reasons, stack walkers generally first check for
569 // Biased Locking in the object's header, the second check is for
570 // stack-locking in the object's header, the third check is for
571 // recursive stack-locking in the displaced header in the BasicLock,
572 // and last are the inflated Java Monitor (ObjectMonitor) checks.
573 lock->set_displaced_header(markWord::unused_mark());
574
575 if (owner == NULL && Atomic::replace_if_null(self, &(m->_owner))) {
576 assert(m->_recursions == 0, "invariant");
577 return true;
578 }
579
580 if (AsyncDeflateIdleMonitors &&
581 Atomic::cmpxchg(self, &m->_owner, DEFLATER_MARKER) == DEFLATER_MARKER) {
582 // The deflation protocol finished the first part (setting owner),
583 // but it failed the second part (making ref_count negative) and
584 // bailed. Or the ObjectMonitor was async deflated and reused.
585 // Acquired the monitor.
586 assert(m->_recursions == 0, "invariant");
587 return true;
588 }
589 }
590 break;
591 }
592
593 // Note that we could inflate in quick_enter.
594 // This is likely a useful optimization
595 // Critically, in quick_enter() we must not:
596 // -- perform bias revocation, or
597 // -- block indefinitely, or
598 // -- reach a safepoint
599
600 return false; // revert to slow-path
601 }
602
603 // -----------------------------------------------------------------------------
604 // Fast Monitor Enter/Exit
605 // This the fast monitor enter. The interpreter and compiler use
606 // some assembly copies of this code. Make sure update those code
607 // if the following function is changed. The implementation is
608 // extremely sensitive to race condition. Be careful.
609
610 void ObjectSynchronizer::fast_enter(Handle obj, BasicLock* lock,
611 bool attempt_rebias, TRAPS) {
612 if (UseBiasedLocking) {
613 if (!SafepointSynchronize::is_at_safepoint()) {
614 BiasedLocking::Condition cond = BiasedLocking::revoke_and_rebias(obj, attempt_rebias, THREAD);
615 if (cond == BiasedLocking::BIAS_REVOKED_AND_REBIASED) {
616 return;
617 }
618 } else {
619 assert(!attempt_rebias, "can not rebias toward VM thread");
620 BiasedLocking::revoke_at_safepoint(obj);
621 }
622 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
623 }
624
625 slow_enter(obj, lock, THREAD);
626 }
627
628 void ObjectSynchronizer::fast_exit(oop object, BasicLock* lock, TRAPS) {
629 markWord mark = object->mark();
630 // We cannot check for Biased Locking if we are racing an inflation.
631 assert(mark == markWord::INFLATING() ||
632 !mark.has_bias_pattern(), "should not see bias pattern here");
633
634 markWord dhw = lock->displaced_header();
635 if (dhw.value() == 0) {
636 // If the displaced header is NULL, then this exit matches up with
637 // a recursive enter. No real work to do here except for diagnostics.
638 #ifndef PRODUCT
639 if (mark != markWord::INFLATING()) {
640 // Only do diagnostics if we are not racing an inflation. Simply
641 // exiting a recursive enter of a Java Monitor that is being
642 // inflated is safe; see the has_monitor() comment below.
643 assert(!mark.is_neutral(), "invariant");
644 assert(!mark.has_locker() ||
645 THREAD->is_lock_owned((address)mark.locker()), "invariant");
646 if (mark.has_monitor()) {
647 // The BasicLock's displaced_header is marked as a recursive
648 // enter and we have an inflated Java Monitor (ObjectMonitor).
649 // This is a special case where the Java Monitor was inflated
650 // after this thread entered the stack-lock recursively. When a
651 // Java Monitor is inflated, we cannot safely walk the Java
652 // Monitor owner's stack and update the BasicLocks because a
653 // Java Monitor can be asynchronously inflated by a thread that
654 // does not own the Java Monitor.
655 ObjectMonitor* m = mark.monitor();
656 assert(((oop)(m->object()))->mark() == mark, "invariant");
657 assert(m->is_entered(THREAD), "invariant");
658 }
659 }
660 #endif
661 return;
662 }
663
664 if (mark == markWord::from_pointer(lock)) {
665 // If the object is stack-locked by the current thread, try to
666 // swing the displaced header from the BasicLock back to the mark.
667 assert(dhw.is_neutral(), "invariant");
668 if (object->cas_set_mark(dhw, mark) == mark) {
669 return;
670 }
671 }
672
673 // We have to take the slow-path of possible inflation and then exit.
674 ObjectMonitorHandle omh;
675 inflate(&omh, THREAD, object, inflate_cause_vm_internal);
676 omh.om_ptr()->exit(true, THREAD);
677 }
678
679 // -----------------------------------------------------------------------------
680 // Interpreter/Compiler Slow Case
681 // This routine is used to handle interpreter/compiler slow case
682 // We don't need to use fast path here, because it must have been
683 // failed in the interpreter/compiler code.
684 void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) {
685 markWord mark = obj->mark();
686 assert(!mark.has_bias_pattern(), "should not see bias pattern here");
687
688 if (mark.is_neutral()) {
689 // Anticipate successful CAS -- the ST of the displaced mark must
690 // be visible <= the ST performed by the CAS.
691 lock->set_displaced_header(mark);
692 if (mark == obj()->cas_set_mark(markWord::from_pointer(lock), mark)) {
693 return;
694 }
695 // Fall through to inflate() ...
696 } else if (mark.has_locker() &&
697 THREAD->is_lock_owned((address)mark.locker())) {
698 assert(lock != mark.locker(), "must not re-lock the same lock");
699 assert(lock != (BasicLock*)obj->mark().value(), "don't relock with same BasicLock");
700 lock->set_displaced_header(markWord::from_pointer(NULL));
701 return;
702 }
703
704 // The object header will never be displaced to this lock,
705 // so it does not matter what the value is, except that it
706 // must be non-zero to avoid looking like a re-entrant lock,
707 // and must not look locked either.
708 lock->set_displaced_header(markWord::unused_mark());
709 ObjectMonitorHandle omh;
710 inflate(&omh, THREAD, obj(), inflate_cause_monitor_enter);
711 omh.om_ptr()->enter(THREAD);
712 }
713
714 // This routine is used to handle interpreter/compiler slow case
715 // We don't need to use fast path here, because it must have
716 // failed in the interpreter/compiler code. Simply use the heavy
717 // weight monitor should be ok, unless someone find otherwise.
718 void ObjectSynchronizer::slow_exit(oop object, BasicLock* lock, TRAPS) {
719 fast_exit(object, lock, THREAD);
720 }
721
722 // -----------------------------------------------------------------------------
723 // Class Loader support to workaround deadlocks on the class loader lock objects
724 // Also used by GC
725 // complete_exit()/reenter() are used to wait on a nested lock
726 // i.e. to give up an outer lock completely and then re-enter
727 // Used when holding nested locks - lock acquisition order: lock1 then lock2
728 // 1) complete_exit lock1 - saving recursion count
729 // 2) wait on lock2
730 // 3) when notified on lock2, unlock lock2
731 // 4) reenter lock1 with original recursion count
732 // 5) lock lock2
733 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
734 intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
735 if (UseBiasedLocking) {
736 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
737 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
738 }
739
740 ObjectMonitorHandle omh;
741 inflate(&omh, THREAD, obj(), inflate_cause_vm_internal);
742 intptr_t ret_code = omh.om_ptr()->complete_exit(THREAD);
743 return ret_code;
744 }
745
746 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
747 void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) {
748 if (UseBiasedLocking) {
749 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
750 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
751 }
752
753 ObjectMonitorHandle omh;
754 inflate(&omh, THREAD, obj(), inflate_cause_vm_internal);
755 omh.om_ptr()->reenter(recursion, THREAD);
756 }
757 // -----------------------------------------------------------------------------
758 // JNI locks on java objects
759 // NOTE: must use heavy weight monitor to handle jni monitor enter
760 void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) {
761 // the current locking is from JNI instead of Java code
762 if (UseBiasedLocking) {
763 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
764 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
765 }
766 THREAD->set_current_pending_monitor_is_from_java(false);
767 ObjectMonitorHandle omh;
768 inflate(&omh, THREAD, obj(), inflate_cause_jni_enter);
769 omh.om_ptr()->enter(THREAD);
770 THREAD->set_current_pending_monitor_is_from_java(true);
771 }
772
773 // NOTE: must use heavy weight monitor to handle jni monitor exit
774 void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
775 if (UseBiasedLocking) {
776 Handle h_obj(THREAD, obj);
777 BiasedLocking::revoke_and_rebias(h_obj, false, THREAD);
778 obj = h_obj();
779 }
780 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
781
782 ObjectMonitorHandle omh;
783 inflate(&omh, THREAD, obj, inflate_cause_jni_exit);
784 ObjectMonitor* monitor = omh.om_ptr();
785 // If this thread has locked the object, exit the monitor. We
786 // intentionally do not use CHECK here because we must exit the
787 // monitor even if an exception is pending.
788 if (monitor->check_owner(THREAD)) {
789 monitor->exit(true, THREAD);
790 }
791 }
792
793 // -----------------------------------------------------------------------------
794 // Internal VM locks on java objects
795 // standard constructor, allows locking failures
796 ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool do_lock) {
797 _dolock = do_lock;
798 _thread = thread;
799 _thread->check_for_valid_safepoint_state(false);
800 _obj = obj;
801
802 if (_dolock) {
803 ObjectSynchronizer::fast_enter(_obj, &_lock, false, _thread);
804 }
805 }
806
807 ObjectLocker::~ObjectLocker() {
808 if (_dolock) {
809 ObjectSynchronizer::fast_exit(_obj(), &_lock, _thread);
810 }
811 }
812
813
814 // -----------------------------------------------------------------------------
815 // Wait/Notify/NotifyAll
816 // NOTE: must use heavy weight monitor to handle wait()
817 int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
818 if (UseBiasedLocking) {
819 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
820 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
821 }
822 if (millis < 0) {
823 THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
824 }
825 ObjectMonitorHandle omh;
826 inflate(&omh, THREAD, obj(), inflate_cause_wait);
827 ObjectMonitor* monitor = omh.om_ptr();
828
829 DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);
830 monitor->wait(millis, true, THREAD);
831
832 // This dummy call is in place to get around dtrace bug 6254741. Once
833 // that's fixed we can uncomment the following line, remove the call
834 // and change this function back into a "void" func.
835 // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
836 int ret_code = dtrace_waited_probe(monitor, obj, THREAD);
837 return ret_code;
838 }
839
840 void ObjectSynchronizer::wait_uninterruptibly(Handle obj, jlong millis, TRAPS) {
841 if (UseBiasedLocking) {
842 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
843 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
844 }
845 if (millis < 0) {
846 THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
847 }
848 ObjectMonitorHandle omh;
849 inflate(&omh, THREAD, obj(), inflate_cause_wait);
850 omh.om_ptr()->wait(millis, false, THREAD);
851 }
852
853 void ObjectSynchronizer::notify(Handle obj, TRAPS) {
854 if (UseBiasedLocking) {
855 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
856 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
857 }
858
859 markWord mark = obj->mark();
860 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
861 return;
862 }
863 ObjectMonitorHandle omh;
864 inflate(&omh, THREAD, obj(), inflate_cause_notify);
865 omh.om_ptr()->notify(THREAD);
866 }
867
868 // NOTE: see comment of notify()
869 void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
870 if (UseBiasedLocking) {
871 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
872 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
873 }
874
875 markWord mark = obj->mark();
876 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
877 return;
878 }
879 ObjectMonitorHandle omh;
880 inflate(&omh, THREAD, obj(), inflate_cause_notify);
881 omh.om_ptr()->notifyAll(THREAD);
882 }
883
884 // -----------------------------------------------------------------------------
885 // Hash Code handling
886 //
887 // Performance concern:
888 // OrderAccess::storestore() calls release() which at one time stored 0
889 // into the global volatile OrderAccess::dummy variable. This store was
890 // unnecessary for correctness. Many threads storing into a common location
891 // causes considerable cache migration or "sloshing" on large SMP systems.
892 // As such, I avoided using OrderAccess::storestore(). In some cases
893 // OrderAccess::fence() -- which incurs local latency on the executing
894 // processor -- is a better choice as it scales on SMP systems.
895 //
896 // See http://blogs.oracle.com/dave/entry/biased_locking_in_hotspot for
897 // a discussion of coherency costs. Note that all our current reference
898 // platforms provide strong ST-ST order, so the issue is moot on IA32,
899 // x64, and SPARC.
900 //
901 // As a general policy we use "volatile" to control compiler-based reordering
902 // and explicit fences (barriers) to control for architectural reordering
903 // performed by the CPU(s) or platform.
904
905 struct SharedGlobals {
906 char _pad_prefix[OM_CACHE_LINE_SIZE];
907 // These are highly shared mostly-read variables.
908 // To avoid false-sharing they need to be the sole occupants of a cache line.
909 volatile int stw_random;
910 volatile int stw_cycle;
911 DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(volatile int) * 2);
912 // Hot RW variable -- Sequester to avoid false-sharing
913 volatile int hc_sequence;
914 DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile int));
915 };
916
917 static SharedGlobals GVars;
918 static int MonitorScavengeThreshold = 1000000;
919 static volatile int ForceMonitorScavenge = 0; // Scavenge required and pending
920
921 static markWord read_stable_mark(oop obj) {
922 markWord mark = obj->mark();
923 if (!mark.is_being_inflated()) {
924 return mark; // normal fast-path return
925 }
926
927 int its = 0;
928 for (;;) {
929 markWord mark = obj->mark();
930 if (!mark.is_being_inflated()) {
931 return mark; // normal fast-path return
932 }
933
934 // The object is being inflated by some other thread.
935 // The caller of read_stable_mark() must wait for inflation to complete.
936 // Avoid live-lock
937 // TODO: consider calling SafepointSynchronize::do_call_back() while
938 // spinning to see if there's a safepoint pending. If so, immediately
939 // yielding or blocking would be appropriate. Avoid spinning while
940 // there is a safepoint pending.
941 // TODO: add inflation contention performance counters.
942 // TODO: restrict the aggregate number of spinners.
943
944 ++its;
945 if (its > 10000 || !os::is_MP()) {
946 if (its & 1) {
947 os::naked_yield();
948 } else {
949 // Note that the following code attenuates the livelock problem but is not
950 // a complete remedy. A more complete solution would require that the inflating
951 // thread hold the associated inflation lock. The following code simply restricts
952 // the number of spinners to at most one. We'll have N-2 threads blocked
953 // on the inflationlock, 1 thread holding the inflation lock and using
954 // a yield/park strategy, and 1 thread in the midst of inflation.
955 // A more refined approach would be to change the encoding of INFLATING
956 // to allow encapsulation of a native thread pointer. Threads waiting for
957 // inflation to complete would use CAS to push themselves onto a singly linked
958 // list rooted at the markword. Once enqueued, they'd loop, checking a per-thread flag
959 // and calling park(). When inflation was complete the thread that accomplished inflation
960 // would detach the list and set the markword to inflated with a single CAS and
961 // then for each thread on the list, set the flag and unpark() the thread.
962 // This is conceptually similar to muxAcquire-muxRelease, except that muxRelease
963 // wakes at most one thread whereas we need to wake the entire list.
964 int ix = (cast_from_oop<intptr_t>(obj) >> 5) & (NINFLATIONLOCKS-1);
965 int YieldThenBlock = 0;
966 assert(ix >= 0 && ix < NINFLATIONLOCKS, "invariant");
967 assert((NINFLATIONLOCKS & (NINFLATIONLOCKS-1)) == 0, "invariant");
968 Thread::muxAcquire(gInflationLocks + ix, "gInflationLock");
969 while (obj->mark() == markWord::INFLATING()) {
970 // Beware: NakedYield() is advisory and has almost no effect on some platforms
971 // so we periodically call self->_ParkEvent->park(1).
972 // We use a mixed spin/yield/block mechanism.
973 if ((YieldThenBlock++) >= 16) {
974 Thread::current()->_ParkEvent->park(1);
975 } else {
976 os::naked_yield();
977 }
978 }
979 Thread::muxRelease(gInflationLocks + ix);
980 }
981 } else {
982 SpinPause(); // SMP-polite spinning
983 }
984 }
985 }
986
987 // hashCode() generation :
988 //
989 // Possibilities:
990 // * MD5Digest of {obj,stw_random}
991 // * CRC32 of {obj,stw_random} or any linear-feedback shift register function.
992 // * A DES- or AES-style SBox[] mechanism
993 // * One of the Phi-based schemes, such as:
994 // 2654435761 = 2^32 * Phi (golden ratio)
995 // HashCodeValue = ((uintptr_t(obj) >> 3) * 2654435761) ^ GVars.stw_random ;
996 // * A variation of Marsaglia's shift-xor RNG scheme.
997 // * (obj ^ stw_random) is appealing, but can result
998 // in undesirable regularity in the hashCode values of adjacent objects
999 // (objects allocated back-to-back, in particular). This could potentially
1000 // result in hashtable collisions and reduced hashtable efficiency.
1001 // There are simple ways to "diffuse" the middle address bits over the
1002 // generated hashCode values:
1003
1004 static inline intptr_t get_next_hash(Thread* self, oop obj) {
1005 intptr_t value = 0;
1006 if (hashCode == 0) {
1007 // This form uses global Park-Miller RNG.
1008 // On MP system we'll have lots of RW access to a global, so the
1009 // mechanism induces lots of coherency traffic.
1010 value = os::random();
1011 } else if (hashCode == 1) {
1012 // This variation has the property of being stable (idempotent)
1013 // between STW operations. This can be useful in some of the 1-0
1014 // synchronization schemes.
1015 intptr_t addr_bits = cast_from_oop<intptr_t>(obj) >> 3;
1016 value = addr_bits ^ (addr_bits >> 5) ^ GVars.stw_random;
1017 } else if (hashCode == 2) {
1018 value = 1; // for sensitivity testing
1019 } else if (hashCode == 3) {
1020 value = ++GVars.hc_sequence;
1021 } else if (hashCode == 4) {
1022 value = cast_from_oop<intptr_t>(obj);
1023 } else {
1024 // Marsaglia's xor-shift scheme with thread-specific state
1025 // This is probably the best overall implementation -- we'll
1026 // likely make this the default in future releases.
1027 unsigned t = self->_hashStateX;
1028 t ^= (t << 11);
1029 self->_hashStateX = self->_hashStateY;
1030 self->_hashStateY = self->_hashStateZ;
1031 self->_hashStateZ = self->_hashStateW;
1032 unsigned v = self->_hashStateW;
1033 v = (v ^ (v >> 19)) ^ (t ^ (t >> 8));
1034 self->_hashStateW = v;
1035 value = v;
1036 }
1037
1038 value &= markWord::hash_mask;
1039 if (value == 0) value = 0xBAD;
1040 assert(value != markWord::no_hash, "invariant");
1041 return value;
1042 }
1043
1044 intptr_t ObjectSynchronizer::FastHashCode(Thread* self, oop obj) {
1045 if (UseBiasedLocking) {
1046 // NOTE: many places throughout the JVM do not expect a safepoint
1047 // to be taken here, in particular most operations on perm gen
1048 // objects. However, we only ever bias Java instances and all of
1049 // the call sites of identity_hash that might revoke biases have
1050 // been checked to make sure they can handle a safepoint. The
1051 // added check of the bias pattern is to avoid useless calls to
1052 // thread-local storage.
1053 if (obj->mark().has_bias_pattern()) {
1054 // Handle for oop obj in case of STW safepoint
1055 Handle hobj(self, obj);
1056 // Relaxing assertion for bug 6320749.
1057 assert(Universe::verify_in_progress() ||
1058 !SafepointSynchronize::is_at_safepoint(),
1059 "biases should not be seen by VM thread here");
1060 BiasedLocking::revoke_and_rebias(hobj, false, JavaThread::current());
1061 obj = hobj();
1062 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
1063 }
1064 }
1065
1066 // hashCode() is a heap mutator ...
1067 // Relaxing assertion for bug 6320749.
1068 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1069 !SafepointSynchronize::is_at_safepoint(), "invariant");
1070 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1071 self->is_Java_thread() , "invariant");
1072 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1073 ((JavaThread *)self)->thread_state() != _thread_blocked, "invariant");
1074
1075 while (true) {
1076 ObjectMonitor* monitor = NULL;
1077 markWord temp, test;
1078 intptr_t hash;
1079 markWord mark = read_stable_mark(obj);
1080
1081 // object should remain ineligible for biased locking
1082 assert(!mark.has_bias_pattern(), "invariant");
1083
1084 if (mark.is_neutral()) {
1085 hash = mark.hash(); // this is a normal header
1086 if (hash != 0) { // if it has hash, just return it
1087 return hash;
1088 }
1089 hash = get_next_hash(self, obj); // allocate a new hash code
1090 temp = mark.copy_set_hash(hash); // merge the hash code into header
1091 // use (machine word version) atomic operation to install the hash
1092 test = obj->cas_set_mark(temp, mark);
1093 if (test == mark) {
1094 return hash;
1095 }
1096 // If atomic operation failed, we must inflate the header
1097 // into heavy weight monitor. We could add more code here
1098 // for fast path, but it does not worth the complexity.
1099 } else if (mark.has_monitor()) {
1100 ObjectMonitorHandle omh;
1101 if (!omh.save_om_ptr(obj, mark)) {
1102 // Lost a race with async deflation so try again.
1103 assert(AsyncDeflateIdleMonitors, "sanity check");
1104 continue;
1105 }
1106 monitor = omh.om_ptr();
1107 temp = monitor->header();
1108 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1109 hash = temp.hash();
1110 if (hash != 0) {
1111 return hash;
1112 }
1113 // Skip to the following code to reduce code size
1114 } else if (self->is_lock_owned((address)mark.locker())) {
1115 temp = mark.displaced_mark_helper(); // this is a lightweight monitor owned
1116 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1117 hash = temp.hash(); // by current thread, check if the displaced
1118 if (hash != 0) { // header contains hash code
1119 return hash;
1120 }
1121 // WARNING:
1122 // The displaced header in the BasicLock on a thread's stack
1123 // is strictly immutable. It CANNOT be changed in ANY cases.
1124 // So we have to inflate the stack lock into an ObjectMonitor
1125 // even if the current thread owns the lock. The BasicLock on
1126 // a thread's stack can be asynchronously read by other threads
1127 // during an inflate() call so any change to that stack memory
1128 // may not propagate to other threads correctly.
1129 }
1130
1131 // Inflate the monitor to set hash code
1132 ObjectMonitorHandle omh;
1133 inflate(&omh, self, obj, inflate_cause_hash_code);
1134 monitor = omh.om_ptr();
1135 // Load displaced header and check it has hash code
1136 mark = monitor->header();
1137 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1138 hash = mark.hash();
1139 if (hash == 0) {
1140 hash = get_next_hash(self, obj);
1141 temp = mark.copy_set_hash(hash); // merge hash code into header
1142 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1143 uintptr_t v = Atomic::cmpxchg(temp.value(), (volatile uintptr_t*)monitor->header_addr(), mark.value());
1144 test = markWord(v);
1145 if (test != mark) {
1146 // The only non-deflation update to the ObjectMonitor's
1147 // header/dmw field is to merge in the hash code. If someone
1148 // adds a new usage of the header/dmw field, please update
1149 // this code.
1150 // ObjectMonitor::install_displaced_markword_in_object()
1151 // does mark the header/dmw field as part of async deflation,
1152 // but that protocol cannot happen now due to the
1153 // ObjectMonitorHandle above.
1154 hash = test.hash();
1155 assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value());
1156 assert(hash != 0, "Trivial unexpected object/monitor header usage.");
1157 }
1158 }
1159 // We finally get the hash
1160 return hash;
1161 }
1162 }
1163
1164 // Deprecated -- use FastHashCode() instead.
1165
1166 intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) {
1167 return FastHashCode(Thread::current(), obj());
1168 }
1169
1170
1171 bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
1172 Handle h_obj) {
1173 if (UseBiasedLocking) {
1174 BiasedLocking::revoke_and_rebias(h_obj, false, thread);
1175 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1176 }
1177
1178 assert(thread == JavaThread::current(), "Can only be called on current thread");
1179 oop obj = h_obj();
1180
1181 while (true) {
1182 markWord mark = read_stable_mark(obj);
1183
1184 // Uncontended case, header points to stack
1185 if (mark.has_locker()) {
1186 return thread->is_lock_owned((address)mark.locker());
1187 }
1188 // Contended case, header points to ObjectMonitor (tagged pointer)
1189 if (mark.has_monitor()) {
1190 ObjectMonitorHandle omh;
1191 if (!omh.save_om_ptr(obj, mark)) {
1192 // Lost a race with async deflation so try again.
1193 assert(AsyncDeflateIdleMonitors, "sanity check");
1194 continue;
1195 }
1196 bool ret_code = omh.om_ptr()->is_entered(thread) != 0;
1197 return ret_code;
1198 }
1199 // Unlocked case, header in place
1200 assert(mark.is_neutral(), "sanity check");
1201 return false;
1202 }
1203 }
1204
1205 // Be aware of this method could revoke bias of the lock object.
1206 // This method queries the ownership of the lock handle specified by 'h_obj'.
1207 // If the current thread owns the lock, it returns owner_self. If no
1208 // thread owns the lock, it returns owner_none. Otherwise, it will return
1209 // owner_other.
1210 ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership
1211 (JavaThread *self, Handle h_obj) {
1212 // The caller must beware this method can revoke bias, and
1213 // revocation can result in a safepoint.
1214 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
1215 assert(self->thread_state() != _thread_blocked, "invariant");
1216
1217 // Possible mark states: neutral, biased, stack-locked, inflated
1218
1219 if (UseBiasedLocking && h_obj()->mark().has_bias_pattern()) {
1220 // CASE: biased
1221 BiasedLocking::revoke_and_rebias(h_obj, false, self);
1222 assert(!h_obj->mark().has_bias_pattern(),
1223 "biases should be revoked by now");
1224 }
1225
1226 assert(self == JavaThread::current(), "Can only be called on current thread");
1227 oop obj = h_obj();
1228
1229 while (true) {
1230 markWord mark = read_stable_mark(obj);
1231
1232 // CASE: stack-locked. Mark points to a BasicLock on the owner's stack.
1233 if (mark.has_locker()) {
1234 return self->is_lock_owned((address)mark.locker()) ?
1235 owner_self : owner_other;
1236 }
1237
1238 // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor.
1239 // The Object:ObjectMonitor relationship is stable as long as we're
1240 // not at a safepoint and AsyncDeflateIdleMonitors is false.
1241 if (mark.has_monitor()) {
1242 ObjectMonitorHandle omh;
1243 if (!omh.save_om_ptr(obj, mark)) {
1244 // Lost a race with async deflation so try again.
1245 assert(AsyncDeflateIdleMonitors, "sanity check");
1246 continue;
1247 }
1248 ObjectMonitor* monitor = omh.om_ptr();
1249 void* owner = monitor->_owner;
1250 if (owner == NULL) return owner_none;
1251 return (owner == self ||
1252 self->is_lock_owned((address)owner)) ? owner_self : owner_other;
1253 }
1254
1255 // CASE: neutral
1256 assert(mark.is_neutral(), "sanity check");
1257 return owner_none; // it's unlocked
1258 }
1259 }
1260
1261 // FIXME: jvmti should call this
1262 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
1263 if (UseBiasedLocking) {
1264 if (SafepointSynchronize::is_at_safepoint()) {
1265 BiasedLocking::revoke_at_safepoint(h_obj);
1266 } else {
1267 BiasedLocking::revoke_and_rebias(h_obj, false, JavaThread::current());
1268 }
1269 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1270 }
1271
1272 oop obj = h_obj();
1273
1274 while (true) {
1275 address owner = NULL;
1276 markWord mark = read_stable_mark(obj);
1277
1278 // Uncontended case, header points to stack
1279 if (mark.has_locker()) {
1280 owner = (address) mark.locker();
1281 }
1282
1283 // Contended case, header points to ObjectMonitor (tagged pointer)
1284 else if (mark.has_monitor()) {
1285 ObjectMonitorHandle omh;
1286 if (!omh.save_om_ptr(obj, mark)) {
1287 // Lost a race with async deflation so try again.
1288 assert(AsyncDeflateIdleMonitors, "sanity check");
1289 continue;
1290 }
1291 ObjectMonitor* monitor = omh.om_ptr();
1292 assert(monitor != NULL, "monitor should be non-null");
1293 owner = (address) monitor->owner();
1294 }
1295
1296 if (owner != NULL) {
1297 // owning_thread_from_monitor_owner() may also return NULL here
1298 return Threads::owning_thread_from_monitor_owner(t_list, owner);
1299 }
1300
1301 // Unlocked case, header in place
1302 // Cannot have assertion since this object may have been
1303 // locked by another thread when reaching here.
1304 // assert(mark.is_neutral(), "sanity check");
1305
1306 return NULL;
1307 }
1308 }
1309
1310 // Visitors ...
1311
1312 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
1313 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
1314 while (block != NULL) {
1315 assert(block->object() == CHAINMARKER, "must be a block header");
1316 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
1317 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
1318 if (mid->is_active()) {
1319 ObjectMonitorHandle omh(mid);
1320
1321 if (mid->object() == NULL ||
1322 (AsyncDeflateIdleMonitors && mid->ref_count() < 0)) {
1323 // Only process with closure if the object is set.
1324 // For async deflation, race here if monitor is not owned!
1325 // The above ref_count bump (in ObjectMonitorHandle ctr)
1326 // will cause subsequent async deflation to skip it.
1327 // However, previous or concurrent async deflation is a race
1328 // so skip this ObjectMonitor if it is being async deflated.
1329 continue;
1330 }
1331 closure->do_monitor(mid);
1332 }
1333 }
1334 // unmarked_next() is not needed with g_block_list (no next field marking).
1335 block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om);
1336 }
1337 }
1338
1339 static bool monitors_used_above_threshold() {
1340 if (OrderAccess::load_acquire(&g_om_population) == 0) {
1341 return false;
1342 }
1343 if (MonitorUsedDeflationThreshold > 0) {
1344 int monitors_used = OrderAccess::load_acquire(&g_om_population) -
1345 OrderAccess::load_acquire(&g_om_free_count);
1346 int monitor_usage = (monitors_used * 100LL) /
1347 OrderAccess::load_acquire(&g_om_population);
1348 return monitor_usage > MonitorUsedDeflationThreshold;
1349 }
1350 return false;
1351 }
1352
1353 // Returns true if MonitorBound is set (> 0) and if the specified
1354 // cnt is > MonitorBound. Otherwise returns false.
1355 static bool is_MonitorBound_exceeded(const int cnt) {
1356 const int mx = MonitorBound;
1357 return mx > 0 && cnt > mx;
1358 }
1359
1360 bool ObjectSynchronizer::is_async_deflation_needed() {
1361 if (!AsyncDeflateIdleMonitors) {
1362 return false;
1363 }
1364 if (is_async_deflation_requested()) {
1365 // Async deflation request.
1366 return true;
1367 }
1368 if (AsyncDeflationInterval > 0 &&
1369 time_since_last_async_deflation_ms() > AsyncDeflationInterval &&
1370 monitors_used_above_threshold()) {
1371 // It's been longer than our specified deflate interval and there
1372 // are too many monitors in use. We don't deflate more frequently
1373 // than AsyncDeflationInterval (unless is_async_deflation_requested)
1374 // in order to not swamp the ServiceThread.
1375 _last_async_deflation_time_ns = os::javaTimeNanos();
1376 return true;
1377 }
1378 if (is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) -
1379 OrderAccess::load_acquire(&g_om_free_count))) {
1380 // Not enough ObjectMonitors on the global free list.
1381 return true;
1382 }
1383 return false;
1384 }
1385
1386 bool ObjectSynchronizer::is_safepoint_deflation_needed() {
1387 if (!AsyncDeflateIdleMonitors) {
1388 if (monitors_used_above_threshold()) {
1389 // Too many monitors in use.
1390 return true;
1391 }
1392 return false;
1393 }
1394 if (is_special_deflation_requested()) {
1395 // For AsyncDeflateIdleMonitors only do a safepoint deflation
1396 // if there is a special deflation request.
1397 return true;
1398 }
1399 return false;
1400 }
1401
1402 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1403 return (os::javaTimeNanos() - _last_async_deflation_time_ns) / (NANOUNITS / MILLIUNITS);
1404 }
1405
1406 void ObjectSynchronizer::oops_do(OopClosure* f) {
1407 // We only scan the global used list here (for moribund threads), and
1408 // the thread-local monitors in Thread::oops_do().
1409 global_used_oops_do(f);
1410 }
1411
1412 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
1413 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1414 list_oops_do(OrderAccess::load_acquire(&g_om_in_use_list), OrderAccess::load_acquire(&g_om_in_use_count), f);
1415 }
1416
1417 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
1418 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1419 list_oops_do(OrderAccess::load_acquire(&thread->om_in_use_list), OrderAccess::load_acquire(&thread->om_in_use_count), f);
1420 }
1421
1422 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, int count, OopClosure* f) {
1423 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1424 chk_for_list_loop(list, count);
1425 // The oops_do() phase does not overlap with monitor deflation
1426 // so no need to update the ObjectMonitor's ref_count for this
1427 // ObjectMonitor* use.
1428 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
1429 if (mid->object() != NULL) {
1430 f->do_oop((oop*)mid->object_addr());
1431 }
1432 }
1433 }
1434
1435
1436 // -----------------------------------------------------------------------------
1437 // ObjectMonitor Lifecycle
1438 // -----------------------
1439 // Inflation unlinks monitors from the global g_free_list and
1440 // associates them with objects. Deflation -- which occurs at
1441 // STW-time -- disassociates idle monitors from objects. Such
1442 // scavenged monitors are returned to the g_free_list.
1443 //
1444 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
1445 //
1446 // Lifecycle:
1447 // -- unassigned and on the global free list
1448 // -- unassigned and on a thread's private om_free_list
1449 // -- assigned to an object. The object is inflated and the mark refers
1450 // to the objectmonitor.
1451
1452
1453 // Constraining monitor pool growth via MonitorBound ...
1454 //
1455 // If MonitorBound is not set (<= 0), MonitorBound checks are disabled.
1456 //
1457 // When safepoint deflation is being used (!AsyncDeflateIdleMonitors):
1458 // The monitor pool is grow-only. We scavenge at STW safepoint-time, but the
1459 // the rate of scavenging is driven primarily by GC. As such, we can find
1460 // an inordinate number of monitors in circulation.
1461 // To avoid that scenario we can artificially induce a STW safepoint
1462 // if the pool appears to be growing past some reasonable bound.
1463 // Generally we favor time in space-time tradeoffs, but as there's no
1464 // natural back-pressure on the # of extant monitors we need to impose some
1465 // type of limit. Beware that if MonitorBound is set to too low a value
1466 // we could just loop. In addition, if MonitorBound is set to a low value
1467 // we'll incur more safepoints, which are harmful to performance.
1468 // See also: GuaranteedSafepointInterval
1469 //
1470 // The current implementation uses asynchronous VM operations.
1471 //
1472 // When safepoint deflation is being used and MonitorBound is set, the
1473 // boundry applies to
1474 // (g_om_population - g_om_free_count)
1475 // i.e., if there are not enough ObjectMonitors on the global free list,
1476 // then a safepoint deflation is induced. Picking a good MonitorBound value
1477 // is non-trivial.
1478 //
1479 // When async deflation is being used:
1480 // The monitor pool is still grow-only. Async deflation is requested
1481 // by a safepoint's cleanup phase or by the ServiceThread at periodic
1482 // intervals when is_async_deflation_needed() returns true. In
1483 // addition to other policies that are checked, if there are not
1484 // enough ObjectMonitors on the global free list, then
1485 // is_async_deflation_needed() will return true. The ServiceThread
1486 // calls deflate_global_idle_monitors_using_JT() and also calls
1487 // deflate_per_thread_idle_monitors_using_JT() as needed.
1488
1489 static void InduceScavenge(Thread* self, const char * Whence) {
1490 assert(!AsyncDeflateIdleMonitors, "is not used by async deflation");
1491
1492 // Induce STW safepoint to trim monitors
1493 // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
1494 // More precisely, trigger an asynchronous STW safepoint as the number
1495 // of active monitors passes the specified threshold.
1496 // TODO: assert thread state is reasonable
1497
1498 if (ForceMonitorScavenge == 0 && Atomic::xchg (1, &ForceMonitorScavenge) == 0) {
1499 // Induce a 'null' safepoint to scavenge monitors
1500 // Must VM_Operation instance be heap allocated as the op will be enqueue and posted
1501 // to the VMthread and have a lifespan longer than that of this activation record.
1502 // The VMThread will delete the op when completed.
1503 VMThread::execute(new VM_ScavengeMonitors());
1504 }
1505 }
1506
1507 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self,
1508 const InflateCause cause) {
1509 // A large MAXPRIVATE value reduces both list lock contention
1510 // and list coherency traffic, but also tends to increase the
1511 // number of ObjectMonitors in circulation as well as the STW
1512 // scavenge costs. As usual, we lean toward time in space-time
1513 // tradeoffs.
1514 const int MAXPRIVATE = 1024;
1515
1516 stringStream ss;
1517 for (;;) {
1518 ObjectMonitor* m;
1519
1520 // 1: try to allocate from the thread's local om_free_list.
1521 // Threads will attempt to allocate first from their local list, then
1522 // from the global list, and only after those attempts fail will the
1523 // thread attempt to instantiate new monitors. Thread-local free lists
1524 // improve allocation latency, as well as reducing coherency traffic
1525 // on the shared global list.
1526 m = take_from_start_of_om_free_list(self);
1527 if (m != NULL) {
1528 guarantee(m->object() == NULL, "invariant");
1529 m->set_allocation_state(ObjectMonitor::New);
1530 prepend_to_om_in_use_list(self, m);
1531 return m;
1532 }
1533
1534 // 2: try to allocate from the global g_free_list
1535 // CONSIDER: use muxTry() instead of muxAcquire().
1536 // If the muxTry() fails then drop immediately into case 3.
1537 // If we're using thread-local free lists then try
1538 // to reprovision the caller's free list.
1539 if (OrderAccess::load_acquire(&g_free_list) != NULL) {
1540 // Reprovision the thread's om_free_list.
1541 // Use bulk transfers to reduce the allocation rate and heat
1542 // on various locks.
1543 for (int i = self->om_free_provision; --i >= 0;) {
1544 ObjectMonitor* take = take_from_start_of_g_free_list();
1545 if (take == NULL) {
1546 break; // No more are available.
1547 }
1548 guarantee(take->object() == NULL, "invariant");
1549 if (AsyncDeflateIdleMonitors) {
1550 // We allowed 3 field values to linger during async deflation.
1551 // We clear header and restore ref_count here, but we leave
1552 // owner == DEFLATER_MARKER so the simple C2 ObjectMonitor
1553 // enter optimization can no longer race with async deflation
1554 // and reuse.
1555 take->set_header(markWord::zero());
1556 if (take->ref_count() < 0) {
1557 // Add back max_jint to restore the ref_count field to its
1558 // proper value.
1559 Atomic::add(max_jint, &take->_ref_count);
1560
1561 assert(take->ref_count() >= 0, "must not be negative: ref_count=%d",
1562 take->ref_count());
1563 }
1564 }
1565 take->Recycle();
1566 assert(take->is_free(), "invariant");
1567 om_release(self, take, false);
1568 }
1569 self->om_free_provision += 1 + (self->om_free_provision/2);
1570 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1571
1572 if (!AsyncDeflateIdleMonitors &&
1573 is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) -
1574 OrderAccess::load_acquire(&g_om_free_count))) {
1575 // Not enough ObjectMonitors on the global free list.
1576 // We can't safely induce a STW safepoint from om_alloc() as our thread
1577 // state may not be appropriate for such activities and callers may hold
1578 // naked oops, so instead we defer the action.
1579 InduceScavenge(self, "om_alloc");
1580 }
1581 continue;
1582 }
1583
1584 // 3: allocate a block of new ObjectMonitors
1585 // Both the local and global free lists are empty -- resort to malloc().
1586 // In the current implementation ObjectMonitors are TSM - immortal.
1587 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1588 // each ObjectMonitor to start at the beginning of a cache line,
1589 // so we use align_up().
1590 // A better solution would be to use C++ placement-new.
1591 // BEWARE: As it stands currently, we don't run the ctors!
1592 assert(_BLOCKSIZE > 1, "invariant");
1593 size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE;
1594 PaddedObjectMonitor* temp;
1595 size_t aligned_size = neededsize + (OM_CACHE_LINE_SIZE - 1);
1596 void* real_malloc_addr = (void*)NEW_C_HEAP_ARRAY(char, aligned_size,
1597 mtInternal);
1598 temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, OM_CACHE_LINE_SIZE);
1599
1600 // NOTE: (almost) no way to recover if allocation failed.
1601 // We might be able to induce a STW safepoint and scavenge enough
1602 // ObjectMonitors to permit progress.
1603 if (temp == NULL) {
1604 vm_exit_out_of_memory(neededsize, OOM_MALLOC_ERROR,
1605 "Allocate ObjectMonitors");
1606 }
1607 (void)memset((void *) temp, 0, neededsize);
1608
1609 // Format the block.
1610 // initialize the linked list, each monitor points to its next
1611 // forming the single linked free list, the very first monitor
1612 // will points to next block, which forms the block list.
1613 // The trick of using the 1st element in the block as g_block_list
1614 // linkage should be reconsidered. A better implementation would
1615 // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; }
1616
1617 for (int i = 1; i < _BLOCKSIZE; i++) {
1618 OrderAccess::release_store(&temp[i]._next_om, (ObjectMonitor*)&temp[i+1]);
1619 assert(temp[i].is_free(), "invariant");
1620 }
1621
1622 // terminate the last monitor as the end of list
1623 OrderAccess::release_store(&temp[_BLOCKSIZE - 1]._next_om, (ObjectMonitor*)NULL);
1624
1625 // Element [0] is reserved for global list linkage
1626 temp[0].set_object(CHAINMARKER);
1627
1628 // Consider carving out this thread's current request from the
1629 // block in hand. This avoids some lock traffic and redundant
1630 // list activity.
1631
1632 prepend_block_to_lists(temp);
1633 }
1634 }
1635
1636 // Place "m" on the caller's private per-thread om_free_list.
1637 // In practice there's no need to clamp or limit the number of
1638 // monitors on a thread's om_free_list as the only non-allocation time
1639 // we'll call om_release() is to return a monitor to the free list after
1640 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1641 // accumulate on a thread's free list.
1642 //
1643 // Key constraint: all ObjectMonitors on a thread's free list and the global
1644 // free list must have their object field set to null. This prevents the
1645 // scavenger -- deflate_monitor_list() or deflate_monitor_list_using_JT()
1646 // -- from reclaiming them while we are trying to release them.
1647
1648 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1649 bool from_per_thread_alloc) {
1650 guarantee(m->header().value() == 0, "invariant");
1651 guarantee(m->object() == NULL, "invariant");
1652 stringStream ss;
1653 guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1654 "%s, recursions=" INTPTR_FORMAT, m->is_busy_to_string(&ss),
1655 m->_recursions);
1656 m->set_allocation_state(ObjectMonitor::Free);
1657 // _next_om is used for both per-thread in-use and free lists so
1658 // we have to remove 'm' from the in-use list first (as needed).
1659 if (from_per_thread_alloc) {
1660 // Need to remove 'm' from om_in_use_list.
1661 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
1662 // protocol because async deflation can do list deletions in parallel.
1663 ObjectMonitor* cur_mid_in_use = NULL;
1664 ObjectMonitor* mid = NULL;
1665 ObjectMonitor* next = NULL;
1666 bool extracted = false;
1667
1668 if (!mark_list_head(&self->om_in_use_list, &mid, &next)) {
1669 fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self));
1670 }
1671 while (true) {
1672 if (m == mid) {
1673 // We found 'm' on the per-thread in-use list so try to extract it.
1674 // First try the list head:
1675 if (Atomic::cmpxchg(next, &self->om_in_use_list, mid) != mid) {
1676 // We could not switch the list head to next.
1677 ObjectMonitor* marked_mid = mark_om_ptr(mid);
1678 // Switch cur_mid_in_use's next field to next (which also
1679 // unmarks cur_mid_in_use):
1680 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
1681 if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, marked_mid)
1682 != marked_mid) {
1683 // We could not switch cur_mid_in_use's next field. This
1684 // should not be possible since it was marked so we:
1685 fatal("mid=" INTPTR_FORMAT " must be referred to by the list "
1686 "head: &om_in_use_list=" INTPTR_FORMAT " or by "
1687 "cur_mid_in_use's next field: cur_mid_in_use=" INTPTR_FORMAT
1688 ", next_om=" INTPTR_FORMAT, p2i(mid),
1689 p2i((ObjectMonitor**)&self->om_in_use_list),
1690 p2i(cur_mid_in_use), p2i(cur_mid_in_use->_next_om));
1691 }
1692 }
1693 extracted = true;
1694 Atomic::dec(&self->om_in_use_count);
1695 // Unmark mid, but leave the next value for any lagging list
1696 // walkers. It will get cleaned up when mid is prepended to
1697 // the thread's free list:
1698 set_next(mid, next);
1699 break;
1700 }
1701 if (cur_mid_in_use != NULL) {
1702 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
1703 }
1704 // The next cur_mid_in_use keeps mid's marked next field so
1705 // that it is stable for a possible next field change. It
1706 // cannot be deflated while it is marked.
1707 cur_mid_in_use = mid;
1708 mid = next;
1709 if (mid == NULL) {
1710 // Reached end of the list and didn't find m so:
1711 fatal("must find m=" INTPTR_FORMAT "on om_in_use_list=" INTPTR_FORMAT,
1712 p2i(m), p2i(self->om_in_use_list));
1713 }
1714 // Mark mid's next field so we can possibly extract it:
1715 next = mark_next_loop(mid);
1716 }
1717 }
1718
1719 prepend_to_om_free_list(self, m);
1720 guarantee(m->is_free(), "invariant");
1721 }
1722
1723 // Return ObjectMonitors on a moribund thread's free and in-use
1724 // lists to the appropriate global lists. The ObjectMonitors on the
1725 // per-thread in-use list may still be in use by other threads.
1726 //
1727 // We currently call om_flush() from Threads::remove() before the
1728 // thread has been excised from the thread list and is no longer a
1729 // mutator. This means that om_flush() cannot run concurrently with
1730 // a safepoint and interleave with deflate_idle_monitors(). In
1731 // particular, this ensures that the thread's in-use monitors are
1732 // scanned by a GC safepoint, either via Thread::oops_do() (before
1733 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1734 // om_flush() is called).
1735 //
1736 // With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT()
1737 // and deflate_per_thread_idle_monitors_using_JT() (in another thread) can
1738 // run at the same time as om_flush() so we have to follow a careful
1739 // protocol to prevent list corruption.
1740
1741 void ObjectSynchronizer::om_flush(Thread* self) {
1742 // This function can race with an async deflater thread. Since
1743 // deflation has to process the per-thread in-use list before
1744 // prepending the deflated ObjectMonitors to the global free list,
1745 // we process the per-thread lists in the same order to prevent
1746 // ordering races.
1747 int in_use_count = 0;
1748 ObjectMonitor* in_use_list = NULL;
1749 ObjectMonitor* in_use_tail = NULL;
1750 ObjectMonitor* next = NULL;
1751
1752 // An async deflation thread checks to see if the target thread
1753 // is exiting, but if it has made it past that check before we
1754 // started exiting, then it is racing to get to the in-use list.
1755 if (mark_list_head(&self->om_in_use_list, &in_use_list, &next)) {
1756 chk_for_list_loop(in_use_list, OrderAccess::load_acquire(&self->om_in_use_count));
1757 // At this point, we have marked the in-use list head so an
1758 // async deflation thread cannot come in after us. If an async
1759 // deflation thread is ahead of us, then we'll detect that and
1760 // wait for it to finish its work.
1761 //
1762 // The thread is going away, however the ObjectMonitors on the
1763 // om_in_use_list may still be in-use by other threads. Link
1764 // them to in_use_tail, which will be linked into the global
1765 // in-use list g_om_in_use_list below.
1766 //
1767 // Account for the in-use list head before the loop since it is
1768 // already marked (by this thread):
1769 in_use_tail = in_use_list;
1770 in_use_count++;
1771 for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL;) {
1772 if (is_next_marked(cur_om)) {
1773 // This next field is marked so there must be an async deflater
1774 // thread ahead of us so we'll give it a chance to finish.
1775 while (is_next_marked(cur_om)) {
1776 os::naked_short_sleep(1);
1777 }
1778 // Refetch the possibly changed next field and try again.
1779 cur_om = unmarked_next(in_use_tail);
1780 continue;
1781 }
1782 if (!cur_om->is_active()) {
1783 // cur_om was deflated and the allocation state was changed
1784 // to Free while it was marked. We happened to see it just
1785 // after it was unmarked (and added to the free list).
1786 // Refetch the possibly changed next field and try again.
1787 cur_om = unmarked_next(in_use_tail);
1788 continue;
1789 }
1790 in_use_tail = cur_om;
1791 in_use_count++;
1792 cur_om = unmarked_next(cur_om);
1793 }
1794 guarantee(in_use_tail != NULL, "invariant");
1795 int l_om_in_use_count = OrderAccess::load_acquire(&self->om_in_use_count);
1796 ADIM_guarantee(l_om_in_use_count == in_use_count, "in-use counts don't "
1797 "match: l_om_in_use_count=%d, in_use_count=%d",
1798 l_om_in_use_count, in_use_count);
1799 // Clear the in-use count before unmarking the in-use list head
1800 // to avoid races:
1801 OrderAccess::release_store(&self->om_in_use_count, 0);
1802 // Clear the in-use list head (which also unmarks it):
1803 OrderAccess::release_store(&self->om_in_use_list, (ObjectMonitor*)NULL);
1804 // Unmark the disconnected list head:
1805 set_next(in_use_list, next);
1806 }
1807
1808 int free_count = 0;
1809 ObjectMonitor* free_list = OrderAccess::load_acquire(&self->om_free_list);
1810 ObjectMonitor* free_tail = NULL;
1811 if (free_list != NULL) {
1812 chk_for_list_loop(free_list, OrderAccess::load_acquire(&self->om_free_count));
1813 // The thread is going away. Set 'free_tail' to the last per-thread free
1814 // monitor which will be linked to g_free_list below.
1815 stringStream ss;
1816 for (ObjectMonitor* s = free_list; s != NULL; s = unmarked_next(s)) {
1817 free_count++;
1818 free_tail = s;
1819 guarantee(s->object() == NULL, "invariant");
1820 guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss));
1821 }
1822 guarantee(free_tail != NULL, "invariant");
1823 int l_om_free_count = OrderAccess::load_acquire(&self->om_free_count);
1824 ADIM_guarantee(l_om_free_count == free_count, "free counts don't match: "
1825 "l_om_free_count=%d, free_count=%d", l_om_free_count,
1826 free_count);
1827 OrderAccess::release_store(&self->om_free_list, (ObjectMonitor*)NULL);
1828 OrderAccess::release_store(&self->om_free_count, 0);
1829 }
1830
1831 if (free_tail != NULL) {
1832 prepend_list_to_g_free_list(free_list, free_tail, free_count);
1833 }
1834
1835 if (in_use_tail != NULL) {
1836 prepend_list_to_g_om_in_use_list(in_use_list, in_use_tail, in_use_count);
1837 }
1838
1839 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1840 LogStreamHandle(Info, monitorinflation) lsh_info;
1841 LogStream* ls = NULL;
1842 if (log_is_enabled(Debug, monitorinflation)) {
1843 ls = &lsh_debug;
1844 } else if ((free_count != 0 || in_use_count != 0) &&
1845 log_is_enabled(Info, monitorinflation)) {
1846 ls = &lsh_info;
1847 }
1848 if (ls != NULL) {
1849 ls->print_cr("om_flush: jt=" INTPTR_FORMAT ", free_count=%d"
1850 ", in_use_count=%d" ", om_free_provision=%d",
1851 p2i(self), free_count, in_use_count, self->om_free_provision);
1852 }
1853 }
1854
1855 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1856 const oop obj,
1857 ObjectSynchronizer::InflateCause cause) {
1858 assert(event != NULL, "invariant");
1859 assert(event->should_commit(), "invariant");
1860 event->set_monitorClass(obj->klass());
1861 event->set_address((uintptr_t)(void*)obj);
1862 event->set_cause((u1)cause);
1863 event->commit();
1864 }
1865
1866 // Fast path code shared by multiple functions
1867 void ObjectSynchronizer::inflate_helper(ObjectMonitorHandle* omh_p, oop obj) {
1868 while (true) {
1869 markWord mark = obj->mark();
1870 if (mark.has_monitor()) {
1871 if (!omh_p->save_om_ptr(obj, mark)) {
1872 // Lost a race with async deflation so try again.
1873 assert(AsyncDeflateIdleMonitors, "sanity check");
1874 continue;
1875 }
1876 ObjectMonitor* monitor = omh_p->om_ptr();
1877 assert(ObjectSynchronizer::verify_objmon_isinpool(monitor), "monitor is invalid");
1878 markWord dmw = monitor->header();
1879 assert(dmw.is_neutral(), "sanity check: header=" INTPTR_FORMAT, dmw.value());
1880 return;
1881 }
1882 inflate(omh_p, Thread::current(), obj, inflate_cause_vm_internal);
1883 return;
1884 }
1885 }
1886
1887 void ObjectSynchronizer::inflate(ObjectMonitorHandle* omh_p, Thread* self,
1888 oop object, const InflateCause cause) {
1889 // Inflate mutates the heap ...
1890 // Relaxing assertion for bug 6320749.
1891 assert(Universe::verify_in_progress() ||
1892 !SafepointSynchronize::is_at_safepoint(), "invariant");
1893
1894 EventJavaMonitorInflate event;
1895
1896 for (;;) {
1897 const markWord mark = object->mark();
1898 assert(!mark.has_bias_pattern(), "invariant");
1899
1900 // The mark can be in one of the following states:
1901 // * Inflated - just return
1902 // * Stack-locked - coerce it to inflated
1903 // * INFLATING - busy wait for conversion to complete
1904 // * Neutral - aggressively inflate the object.
1905 // * BIASED - Illegal. We should never see this
1906
1907 // CASE: inflated
1908 if (mark.has_monitor()) {
1909 if (!omh_p->save_om_ptr(object, mark)) {
1910 // Lost a race with async deflation so try again.
1911 assert(AsyncDeflateIdleMonitors, "sanity check");
1912 continue;
1913 }
1914 ObjectMonitor* inf = omh_p->om_ptr();
1915 markWord dmw = inf->header();
1916 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1917 assert(oopDesc::equals((oop) inf->object(), object), "invariant");
1918 assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
1919 return;
1920 }
1921
1922 // CASE: inflation in progress - inflating over a stack-lock.
1923 // Some other thread is converting from stack-locked to inflated.
1924 // Only that thread can complete inflation -- other threads must wait.
1925 // The INFLATING value is transient.
1926 // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1927 // We could always eliminate polling by parking the thread on some auxiliary list.
1928 if (mark == markWord::INFLATING()) {
1929 read_stable_mark(object);
1930 continue;
1931 }
1932
1933 // CASE: stack-locked
1934 // Could be stack-locked either by this thread or by some other thread.
1935 //
1936 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1937 // to install INFLATING into the mark word. We originally installed INFLATING,
1938 // allocated the objectmonitor, and then finally STed the address of the
1939 // objectmonitor into the mark. This was correct, but artificially lengthened
1940 // the interval in which INFLATED appeared in the mark, thus increasing
1941 // the odds of inflation contention.
1942 //
1943 // We now use per-thread private objectmonitor free lists.
1944 // These list are reprovisioned from the global free list outside the
1945 // critical INFLATING...ST interval. A thread can transfer
1946 // multiple objectmonitors en-mass from the global free list to its local free list.
1947 // This reduces coherency traffic and lock contention on the global free list.
1948 // Using such local free lists, it doesn't matter if the om_alloc() call appears
1949 // before or after the CAS(INFLATING) operation.
1950 // See the comments in om_alloc().
1951
1952 LogStreamHandle(Trace, monitorinflation) lsh;
1953
1954 if (mark.has_locker()) {
1955 ObjectMonitor* m = om_alloc(self, cause);
1956 // Optimistically prepare the objectmonitor - anticipate successful CAS
1957 // We do this before the CAS in order to minimize the length of time
1958 // in which INFLATING appears in the mark.
1959 m->Recycle();
1960 m->_Responsible = NULL;
1961 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // Consider: maintain by type/class
1962
1963 markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1964 if (cmp != mark) {
1965 om_release(self, m, true);
1966 continue; // Interference -- just retry
1967 }
1968
1969 // We've successfully installed INFLATING (0) into the mark-word.
1970 // This is the only case where 0 will appear in a mark-word.
1971 // Only the singular thread that successfully swings the mark-word
1972 // to 0 can perform (or more precisely, complete) inflation.
1973 //
1974 // Why do we CAS a 0 into the mark-word instead of just CASing the
1975 // mark-word from the stack-locked value directly to the new inflated state?
1976 // Consider what happens when a thread unlocks a stack-locked object.
1977 // It attempts to use CAS to swing the displaced header value from the
1978 // on-stack BasicLock back into the object header. Recall also that the
1979 // header value (hash code, etc) can reside in (a) the object header, or
1980 // (b) a displaced header associated with the stack-lock, or (c) a displaced
1981 // header in an ObjectMonitor. The inflate() routine must copy the header
1982 // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1983 // the while preserving the hashCode stability invariants. If the owner
1984 // decides to release the lock while the value is 0, the unlock will fail
1985 // and control will eventually pass from slow_exit() to inflate. The owner
1986 // will then spin, waiting for the 0 value to disappear. Put another way,
1987 // the 0 causes the owner to stall if the owner happens to try to
1988 // drop the lock (restoring the header from the BasicLock to the object)
1989 // while inflation is in-progress. This protocol avoids races that might
1990 // would otherwise permit hashCode values to change or "flicker" for an object.
1991 // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1992 // 0 serves as a "BUSY" inflate-in-progress indicator.
1993
1994
1995 // fetch the displaced mark from the owner's stack.
1996 // The owner can't die or unwind past the lock while our INFLATING
1997 // object is in the mark. Furthermore the owner can't complete
1998 // an unlock on the object, either.
1999 markWord dmw = mark.displaced_mark_helper();
2000 // Catch if the object's header is not neutral (not locked and
2001 // not marked is what we care about here).
2002 ADIM_guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
2003
2004 // Setup monitor fields to proper values -- prepare the monitor
2005 m->set_header(dmw);
2006
2007 // Optimization: if the mark.locker stack address is associated
2008 // with this thread we could simply set m->_owner = self.
2009 // Note that a thread can inflate an object
2010 // that it has stack-locked -- as might happen in wait() -- directly
2011 // with CAS. That is, we can avoid the xchg-NULL .... ST idiom.
2012 m->set_owner(mark.locker());
2013 m->set_object(object);
2014 // TODO-FIXME: assert BasicLock->dhw != 0.
2015
2016 omh_p->set_om_ptr(m);
2017 assert(m->is_new(), "freshly allocated monitor must be new");
2018 m->set_allocation_state(ObjectMonitor::Old);
2019
2020 // Must preserve store ordering. The monitor state must
2021 // be stable at the time of publishing the monitor address.
2022 guarantee(object->mark() == markWord::INFLATING(), "invariant");
2023 object->release_set_mark(markWord::encode(m));
2024
2025 // Hopefully the performance counters are allocated on distinct cache lines
2026 // to avoid false sharing on MP systems ...
2027 OM_PERFDATA_OP(Inflations, inc());
2028 if (log_is_enabled(Trace, monitorinflation)) {
2029 ResourceMark rm(self);
2030 lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
2031 INTPTR_FORMAT ", type='%s'", p2i(object),
2032 object->mark().value(), object->klass()->external_name());
2033 }
2034 if (event.should_commit()) {
2035 post_monitor_inflate_event(&event, object, cause);
2036 }
2037 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
2038 return;
2039 }
2040
2041 // CASE: neutral
2042 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
2043 // If we know we're inflating for entry it's better to inflate by swinging a
2044 // pre-locked ObjectMonitor pointer into the object header. A successful
2045 // CAS inflates the object *and* confers ownership to the inflating thread.
2046 // In the current implementation we use a 2-step mechanism where we CAS()
2047 // to inflate and then CAS() again to try to swing _owner from NULL to self.
2048 // An inflateTry() method that we could call from fast_enter() and slow_enter()
2049 // would be useful.
2050
2051 // Catch if the object's header is not neutral (not locked and
2052 // not marked is what we care about here).
2053 ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT,mark.value());
2054 ObjectMonitor* m = om_alloc(self, cause);
2055 // prepare m for installation - set monitor to initial state
2056 m->Recycle();
2057 m->set_header(mark);
2058 // If we leave _owner == DEFLATER_MARKER here, then the simple C2
2059 // ObjectMonitor enter optimization can no longer race with async
2060 // deflation and reuse.
2061 m->set_object(object);
2062 m->_Responsible = NULL;
2063 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
2064
2065 omh_p->set_om_ptr(m);
2066 assert(m->is_new(), "freshly allocated monitor must be new");
2067 m->set_allocation_state(ObjectMonitor::Old);
2068
2069 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
2070 guarantee(!m->owner_is_DEFLATER_MARKER() || m->ref_count() >= 0,
2071 "race between deflation and om_release() with m=" INTPTR_FORMAT
2072 ", _owner=" INTPTR_FORMAT ", ref_count=%d", p2i(m),
2073 p2i(m->_owner), m->ref_count());
2074 m->set_header(markWord::zero());
2075 m->set_object(NULL);
2076 m->Recycle();
2077 omh_p->set_om_ptr(NULL);
2078 // om_release() will reset the allocation state
2079 om_release(self, m, true);
2080 m = NULL;
2081 continue;
2082 // interference - the markword changed - just retry.
2083 // The state-transitions are one-way, so there's no chance of
2084 // live-lock -- "Inflated" is an absorbing state.
2085 }
2086
2087 // Hopefully the performance counters are allocated on distinct
2088 // cache lines to avoid false sharing on MP systems ...
2089 OM_PERFDATA_OP(Inflations, inc());
2090 if (log_is_enabled(Trace, monitorinflation)) {
2091 ResourceMark rm(self);
2092 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
2093 INTPTR_FORMAT ", type='%s'", p2i(object),
2094 object->mark().value(), object->klass()->external_name());
2095 }
2096 if (event.should_commit()) {
2097 post_monitor_inflate_event(&event, object, cause);
2098 }
2099 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
2100 return;
2101 }
2102 }
2103
2104
2105 // We maintain a list of in-use monitors for each thread.
2106 //
2107 // For safepoint based deflation:
2108 // deflate_thread_local_monitors() scans a single thread's in-use list, while
2109 // deflate_idle_monitors() scans only a global list of in-use monitors which
2110 // is populated only as a thread dies (see om_flush()).
2111 //
2112 // These operations are called at all safepoints, immediately after mutators
2113 // are stopped, but before any objects have moved. Collectively they traverse
2114 // the population of in-use monitors, deflating where possible. The scavenged
2115 // monitors are returned to the global monitor free list.
2116 //
2117 // Beware that we scavenge at *every* stop-the-world point. Having a large
2118 // number of monitors in-use could negatively impact performance. We also want
2119 // to minimize the total # of monitors in circulation, as they incur a small
2120 // footprint penalty.
2121 //
2122 // Perversely, the heap size -- and thus the STW safepoint rate --
2123 // typically drives the scavenge rate. Large heaps can mean infrequent GC,
2124 // which in turn can mean large(r) numbers of ObjectMonitors in circulation.
2125 // This is an unfortunate aspect of this design.
2126 //
2127 // For async deflation:
2128 // If a special deflation request is made, then the safepoint based
2129 // deflation mechanism is used. Otherwise, an async deflation request
2130 // is registered with the ServiceThread and it is notified.
2131
2132 void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* counters) {
2133 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2134
2135 // The per-thread in-use lists are handled in
2136 // ParallelSPCleanupThreadClosure::do_thread().
2137
2138 if (!AsyncDeflateIdleMonitors || is_special_deflation_requested()) {
2139 // Use the older mechanism for the global in-use list or if a
2140 // special deflation has been requested before the safepoint.
2141 ObjectSynchronizer::deflate_idle_monitors(counters);
2142 return;
2143 }
2144
2145 log_debug(monitorinflation)("requesting async deflation of idle monitors.");
2146 // Request deflation of idle monitors by the ServiceThread:
2147 set_is_async_deflation_requested(true);
2148 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
2149 ml.notify_all();
2150 }
2151
2152 // Deflate a single monitor if not in-use
2153 // Return true if deflated, false if in-use
2154 bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj,
2155 ObjectMonitor** free_head_p,
2156 ObjectMonitor** free_tail_p) {
2157 bool deflated;
2158 // Normal case ... The monitor is associated with obj.
2159 const markWord mark = obj->mark();
2160 guarantee(mark == markWord::encode(mid), "should match: mark="
2161 INTPTR_FORMAT ", encoded mid=" INTPTR_FORMAT, mark.value(),
2162 markWord::encode(mid).value());
2163 // Make sure that mark.monitor() and markWord::encode() agree:
2164 guarantee(mark.monitor() == mid, "should match: monitor()=" INTPTR_FORMAT
2165 ", mid=" INTPTR_FORMAT, p2i(mark.monitor()), p2i(mid));
2166 const markWord dmw = mid->header();
2167 guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
2168
2169 if (mid->is_busy() || mid->ref_count() != 0) {
2170 // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor*
2171 // is in use so no deflation.
2172 deflated = false;
2173 } else {
2174 // Deflate the monitor if it is no longer being used
2175 // It's idle - scavenge and return to the global free list
2176 // plain old deflation ...
2177 if (log_is_enabled(Trace, monitorinflation)) {
2178 ResourceMark rm;
2179 log_trace(monitorinflation)("deflate_monitor: "
2180 "object=" INTPTR_FORMAT ", mark="
2181 INTPTR_FORMAT ", type='%s'", p2i(obj),
2182 mark.value(), obj->klass()->external_name());
2183 }
2184
2185 // Restore the header back to obj
2186 obj->release_set_mark(dmw);
2187 if (AsyncDeflateIdleMonitors) {
2188 // clear() expects the owner field to be NULL and we won't race
2189 // with the simple C2 ObjectMonitor enter optimization since
2190 // we're at a safepoint.
2191 mid->set_owner(NULL);
2192 }
2193 mid->clear();
2194
2195 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
2196 p2i(mid->object()));
2197 assert(mid->is_free(), "invariant");
2198
2199 // Move the deflated ObjectMonitor to the working free list
2200 // defined by free_head_p and free_tail_p. No races on this list
2201 // so no need for load_acquire() or store_release().
2202 if (*free_head_p == NULL) *free_head_p = mid;
2203 if (*free_tail_p != NULL) {
2204 // We append to the list so the caller can use mid->_next_om
2205 // to fix the linkages in its context.
2206 ObjectMonitor* prevtail = *free_tail_p;
2207 // Should have been cleaned up by the caller:
2208 // Note: Should not have to mark prevtail here since we're at a
2209 // safepoint and ObjectMonitors on the local free list should
2210 // not be accessed in parallel.
2211 assert(prevtail->_next_om == NULL, "must be NULL: _next_om="
2212 INTPTR_FORMAT, p2i(prevtail->_next_om));
2213 set_next(prevtail, mid);
2214 }
2215 *free_tail_p = mid;
2216 // At this point, mid->_next_om still refers to its current
2217 // value and another ObjectMonitor's _next_om field still
2218 // refers to this ObjectMonitor. Those linkages have to be
2219 // cleaned up by the caller who has the complete context.
2220 deflated = true;
2221 }
2222 return deflated;
2223 }
2224
2225 // Deflate the specified ObjectMonitor if not in-use using a JavaThread.
2226 // Returns true if it was deflated and false otherwise.
2227 //
2228 // The async deflation protocol sets owner to DEFLATER_MARKER and
2229 // makes ref_count negative as signals to contending threads that
2230 // an async deflation is in progress. There are a number of checks
2231 // as part of the protocol to make sure that the calling thread has
2232 // not lost the race to a contending thread or to a thread that just
2233 // wants to use the ObjectMonitor*.
2234 //
2235 // The ObjectMonitor has been successfully async deflated when:
2236 // (owner == DEFLATER_MARKER && ref_count < 0)
2237 // Contending threads or ObjectMonitor* using threads that see those
2238 // values know to retry their operation.
2239 //
2240 bool ObjectSynchronizer::deflate_monitor_using_JT(ObjectMonitor* mid,
2241 ObjectMonitor** free_head_p,
2242 ObjectMonitor** free_tail_p) {
2243 assert(AsyncDeflateIdleMonitors, "sanity check");
2244 assert(Thread::current()->is_Java_thread(), "precondition");
2245 // A newly allocated ObjectMonitor should not be seen here so we
2246 // avoid an endless inflate/deflate cycle.
2247 assert(mid->is_old(), "must be old: allocation_state=%d",
2248 (int) mid->allocation_state());
2249
2250 if (mid->is_busy() || mid->ref_count() != 0) {
2251 // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor*
2252 // is in use so no deflation.
2253 return false;
2254 }
2255
2256 if (Atomic::replace_if_null(DEFLATER_MARKER, &(mid->_owner))) {
2257 // ObjectMonitor is not owned by another thread. Our setting
2258 // owner to DEFLATER_MARKER forces any contending thread through
2259 // the slow path. This is just the first part of the async
2260 // deflation dance.
2261
2262 if (mid->_contentions != 0 || mid->_waiters != 0) {
2263 // Another thread has raced to enter the ObjectMonitor after
2264 // mid->is_busy() above or has already entered and waited on
2265 // it which makes it busy so no deflation. Restore owner to
2266 // NULL if it is still DEFLATER_MARKER.
2267 Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER);
2268 return false;
2269 }
2270
2271 if (Atomic::cmpxchg(-max_jint, &mid->_ref_count, (jint)0) == 0) {
2272 // Make ref_count negative to force any contending threads or
2273 // ObjectMonitor* using threads to retry. This is the second
2274 // part of the async deflation dance.
2275
2276 if (mid->owner_is_DEFLATER_MARKER()) {
2277 // If owner is still DEFLATER_MARKER, then we have successfully
2278 // signaled any contending threads to retry. If it is not, then we
2279 // have lost the race to an entering thread and the ObjectMonitor
2280 // is now busy. This is the third and final part of the async
2281 // deflation dance.
2282 // Note: This owner check solves the ABA problem with ref_count
2283 // where another thread acquired the ObjectMonitor, finished
2284 // using it and restored the ref_count to zero.
2285
2286 // Sanity checks for the races:
2287 guarantee(mid->_contentions == 0, "must be 0: contentions=%d",
2288 mid->_contentions);
2289 guarantee(mid->_waiters == 0, "must be 0: waiters=%d", mid->_waiters);
2290 guarantee(mid->_cxq == NULL, "must be no contending threads: cxq="
2291 INTPTR_FORMAT, p2i(mid->_cxq));
2292 guarantee(mid->_EntryList == NULL,
2293 "must be no entering threads: EntryList=" INTPTR_FORMAT,
2294 p2i(mid->_EntryList));
2295
2296 const oop obj = (oop) mid->object();
2297 if (log_is_enabled(Trace, monitorinflation)) {
2298 ResourceMark rm;
2299 log_trace(monitorinflation)("deflate_monitor_using_JT: "
2300 "object=" INTPTR_FORMAT ", mark="
2301 INTPTR_FORMAT ", type='%s'",
2302 p2i(obj), obj->mark().value(),
2303 obj->klass()->external_name());
2304 }
2305
2306 // Install the old mark word if nobody else has already done it.
2307 mid->install_displaced_markword_in_object(obj);
2308 mid->clear_using_JT();
2309
2310 assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT,
2311 p2i(mid->object()));
2312 assert(mid->is_free(), "must be free: allocation_state=%d",
2313 (int) mid->allocation_state());
2314
2315 // Move the deflated ObjectMonitor to the working free list
2316 // defined by free_head_p and free_tail_p. No races on this list
2317 // so no need for load_acquire() or store_release().
2318 if (*free_head_p == NULL) {
2319 // First one on the list.
2320 *free_head_p = mid;
2321 }
2322 if (*free_tail_p != NULL) {
2323 // We append to the list so the caller can use mid->_next_om
2324 // to fix the linkages in its context.
2325 ObjectMonitor* prevtail = *free_tail_p;
2326 // Should have been cleaned up by the caller:
2327 ObjectMonitor* next = mark_next_loop(prevtail);
2328 assert(unmarked_next(prevtail) == NULL, "must be NULL: _next_om="
2329 INTPTR_FORMAT, p2i(unmarked_next(prevtail)));
2330 set_next(prevtail, mid); // prevtail now points to mid (and is unmarked)
2331 }
2332 *free_tail_p = mid;
2333
2334 // At this point, mid->_next_om still refers to its current
2335 // value and another ObjectMonitor's _next_om field still
2336 // refers to this ObjectMonitor. Those linkages have to be
2337 // cleaned up by the caller who has the complete context.
2338
2339 // We leave owner == DEFLATER_MARKER and ref_count < 0
2340 // to force any racing threads to retry.
2341 return true; // Success, ObjectMonitor has been deflated.
2342 }
2343
2344 // The owner was changed from DEFLATER_MARKER so we lost the
2345 // race since the ObjectMonitor is now busy.
2346
2347 // Add back max_jint to restore the ref_count field to its
2348 // proper value (which may not be what we saw above):
2349 Atomic::add(max_jint, &mid->_ref_count);
2350
2351 assert(mid->ref_count() >= 0, "must not be negative: ref_count=%d",
2352 mid->ref_count());
2353 return false;
2354 }
2355
2356 // The ref_count was no longer 0 so we lost the race since the
2357 // ObjectMonitor is now busy or the ObjectMonitor* is now is use.
2358 // Restore owner to NULL if it is still DEFLATER_MARKER:
2359 Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER);
2360 }
2361
2362 // The owner field is no longer NULL so we lost the race since the
2363 // ObjectMonitor is now busy.
2364 return false;
2365 }
2366
2367 // Walk a given monitor list, and deflate idle monitors.
2368 // The given list could be a per-thread list or a global list.
2369 //
2370 // In the case of parallel processing of thread local monitor lists,
2371 // work is done by Threads::parallel_threads_do() which ensures that
2372 // each Java thread is processed by exactly one worker thread, and
2373 // thus avoid conflicts that would arise when worker threads would
2374 // process the same monitor lists concurrently.
2375 //
2376 // See also ParallelSPCleanupTask and
2377 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
2378 // Threads::parallel_java_threads_do() in thread.cpp.
2379 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor* volatile * list_p,
2380 int volatile * count_p,
2381 ObjectMonitor** free_head_p,
2382 ObjectMonitor** free_tail_p) {
2383 ObjectMonitor* cur_mid_in_use = NULL;
2384 ObjectMonitor* mid = NULL;
2385 ObjectMonitor* next = NULL;
2386 int deflated_count = 0;
2387
2388 // We use the simpler mark-mid-as-we-go protocol since there are no
2389 // parallel list deletions since we are at a safepoint.
2390 if (!mark_list_head(list_p, &mid, &next)) {
2391 return 0; // The list is empty so nothing to deflate.
2392 }
2393
2394 while (true) {
2395 oop obj = (oop) mid->object();
2396 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
2397 // Deflation succeeded and already updated free_head_p and
2398 // free_tail_p as needed. Finish the move to the local free list
2399 // by unlinking mid from the global or per-thread in-use list.
2400 if (Atomic::cmpxchg(next, list_p, mid) != mid) {
2401 // We could not switch the list head to next.
2402 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
2403 if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, mid) != mid) {
2404 // deflate_monitor_list() is called at a safepoint so the
2405 // global or per-thread in-use list should not be modified
2406 // in parallel so we:
2407 fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: "
2408 "list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: "
2409 "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT,
2410 p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use),
2411 p2i(cur_mid_in_use->_next_om));
2412 }
2413 }
2414 // At this point mid is disconnected from the in-use list so
2415 // its marked next field no longer has any effects.
2416 deflated_count++;
2417 Atomic::dec(count_p);
2418 chk_for_list_loop(OrderAccess::load_acquire(list_p),
2419 OrderAccess::load_acquire(count_p));
2420 chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p),
2421 OrderAccess::load_acquire(count_p));
2422 // mid is current tail in the free_head_p list so NULL terminate it
2423 // (which also unmarks it):
2424 set_next(mid, NULL);
2425
2426 // All the list management is done so move on to the next one:
2427 mid = next;
2428 } else {
2429 set_next(mid, next); // unmark next field
2430
2431 // All the list management is done so move on to the next one:
2432 cur_mid_in_use = mid;
2433 mid = next;
2434 }
2435 if (mid == NULL) {
2436 break; // Reached end of the list so nothing more to deflate.
2437 }
2438 // Mark mid's next field so we can possibly deflate it:
2439 next = mark_next_loop(mid);
2440 }
2441 return deflated_count;
2442 }
2443
2444 // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using
2445 // a JavaThread. Returns the number of deflated ObjectMonitors. The given
2446 // list could be a per-thread in-use list or the global in-use list.
2447 // If a safepoint has started, then we save state via saved_mid_in_use_p
2448 // and return to the caller to honor the safepoint.
2449 //
2450 int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor* volatile * list_p,
2451 int volatile * count_p,
2452 ObjectMonitor** free_head_p,
2453 ObjectMonitor** free_tail_p,
2454 ObjectMonitor** saved_mid_in_use_p) {
2455 assert(AsyncDeflateIdleMonitors, "sanity check");
2456 assert(Thread::current()->is_Java_thread(), "precondition");
2457
2458 ObjectMonitor* cur_mid_in_use = NULL;
2459 ObjectMonitor* mid = NULL;
2460 ObjectMonitor* next = NULL;
2461 ObjectMonitor* next_next = NULL;
2462 int deflated_count = 0;
2463
2464 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
2465 // protocol because om_release() can do list deletions in parallel.
2466 // We also mark-next-next-as-we-go to prevent an om_flush() that is
2467 // behind this thread from passing us.
2468 if (*saved_mid_in_use_p == NULL) {
2469 // No saved state so start at the beginning.
2470 // Mark the list head's next field so we can possibly deflate it:
2471 if (!mark_list_head(list_p, &mid, &next)) {
2472 return 0; // The list is empty so nothing to deflate.
2473 }
2474 } else {
2475 // We're restarting after a safepoint so restore the necessary state
2476 // before we resume.
2477 cur_mid_in_use = *saved_mid_in_use_p;
2478 // Mark cur_mid_in_use's next field so we can possibly update its
2479 // next field to extract a deflated ObjectMonitor.
2480 mid = mark_next_loop(cur_mid_in_use);
2481 if (mid == NULL) {
2482 set_next(cur_mid_in_use, NULL); // unmark next field
2483 *saved_mid_in_use_p = NULL;
2484 return 0; // The remainder is empty so nothing more to deflate.
2485 }
2486 // Mark mid's next field so we can possibly deflate it:
2487 next = mark_next_loop(mid);
2488 }
2489
2490 while (true) {
2491 // The current mid's next field is marked at this point. If we have
2492 // a cur_mid_in_use, then its next field is also marked at this point.
2493
2494 if (next != NULL) {
2495 // We mark the next -> next field so that an om_flush()
2496 // thread that is behind us cannot pass us when we
2497 // unmark the current mid's next field.
2498 next_next = mark_next_loop(next);
2499 }
2500
2501 // Only try to deflate if there is an associated Java object and if
2502 // mid is old (is not newly allocated and is not newly freed).
2503 if (mid->object() != NULL && mid->is_old() &&
2504 deflate_monitor_using_JT(mid, free_head_p, free_tail_p)) {
2505 // Deflation succeeded and already updated free_head_p and
2506 // free_tail_p as needed. Finish the move to the local free list
2507 // by unlinking mid from the global or per-thread in-use list.
2508 if (Atomic::cmpxchg(next, list_p, mid) != mid) {
2509 // We could not switch the list head to next.
2510 ObjectMonitor* marked_mid = mark_om_ptr(mid);
2511 ObjectMonitor* marked_next = mark_om_ptr(next);
2512 // Switch cur_mid_in_use's next field to marked next:
2513 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
2514 if (Atomic::cmpxchg(marked_next, &cur_mid_in_use->_next_om,
2515 marked_mid) != marked_mid) {
2516 // We could not switch cur_mid_in_use's next field. This
2517 // should not be possible since it was marked so we:
2518 fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: "
2519 "&list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: "
2520 "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT,
2521 p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use),
2522 p2i(cur_mid_in_use->_next_om));
2523 }
2524 }
2525 // At this point mid is disconnected from the in-use list so
2526 // its marked next field no longer has any effects.
2527 deflated_count++;
2528 Atomic::dec(count_p);
2529 chk_for_list_loop(OrderAccess::load_acquire(list_p),
2530 OrderAccess::load_acquire(count_p));
2531 chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p),
2532 OrderAccess::load_acquire(count_p));
2533 // mid is current tail in the free_head_p list so NULL terminate it
2534 // (which also unmarks it):
2535 set_next(mid, NULL);
2536
2537 // All the list management is done so move on to the next one:
2538 mid = next; // mid keeps non-NULL next's marked next field
2539 next = next_next;
2540 } else {
2541 // mid is considered in-use if it does not have an associated
2542 // Java object or mid is not old or deflation did not succeed.
2543 // A mid->is_new() node can be seen here when it is freshly
2544 // returned by om_alloc() (and skips the deflation code path).
2545 // A mid->is_old() node can be seen here when deflation failed.
2546 // A mid->is_free() node can be seen here when a fresh node from
2547 // om_alloc() is released by om_release() due to losing the race
2548 // in inflate().
2549
2550 // All the list management is done so move on to the next one:
2551 if (cur_mid_in_use != NULL) {
2552 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2553 }
2554 // The next cur_mid_in_use keeps mid's marked next field so
2555 // that it is stable for a possible next field change. It
2556 // cannot be modified by om_release() while it is marked.
2557 cur_mid_in_use = mid;
2558 mid = next; // mid keeps non-NULL next's marked next field
2559 next = next_next;
2560
2561 if (SafepointSynchronize::is_synchronizing() &&
2562 cur_mid_in_use != OrderAccess::load_acquire(list_p) &&
2563 cur_mid_in_use->is_old()) {
2564 // If a safepoint has started and cur_mid_in_use is not the list
2565 // head and is old, then it is safe to use as saved state. Return
2566 // to the caller before blocking.
2567 *saved_mid_in_use_p = cur_mid_in_use;
2568 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2569 if (mid != NULL) {
2570 set_next(mid, next); // umark mid
2571 }
2572 return deflated_count;
2573 }
2574 }
2575 if (mid == NULL) {
2576 if (cur_mid_in_use != NULL) {
2577 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2578 }
2579 break; // Reached end of the list so nothing more to deflate.
2580 }
2581
2582 // The current mid's next field is marked at this point. If we have
2583 // a cur_mid_in_use, then its next field is also marked at this point.
2584 }
2585 // We finished the list without a safepoint starting so there's
2586 // no need to save state.
2587 *saved_mid_in_use_p = NULL;
2588 return deflated_count;
2589 }
2590
2591 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2592 OrderAccess::release_store(&counters->n_in_use, 0); // currently associated with objects
2593 OrderAccess::release_store(&counters->n_in_circulation, 0); // extant
2594 OrderAccess::release_store(&counters->n_scavenged, 0); // reclaimed (global and per-thread)
2595 OrderAccess::release_store(&counters->per_thread_scavenged, 0); // per-thread scavenge total
2596 counters->per_thread_times = 0.0; // per-thread scavenge times
2597 }
2598
2599 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
2600 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2601
2602 if (AsyncDeflateIdleMonitors) {
2603 // Nothing to do when global idle ObjectMonitors are deflated using
2604 // a JavaThread unless a special deflation has been requested.
2605 if (!is_special_deflation_requested()) {
2606 return;
2607 }
2608 }
2609
2610 bool deflated = false;
2611
2612 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2613 ObjectMonitor* free_tail_p = NULL;
2614 elapsedTimer timer;
2615
2616 if (log_is_enabled(Info, monitorinflation)) {
2617 timer.start();
2618 }
2619
2620 // Note: the thread-local monitors lists get deflated in
2621 // a separate pass. See deflate_thread_local_monitors().
2622
2623 // For moribund threads, scan g_om_in_use_list
2624 int deflated_count = 0;
2625 if (OrderAccess::load_acquire(&g_om_in_use_list) != NULL) {
2626 // Update n_in_circulation before g_om_in_use_count is updated by deflation.
2627 Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_circulation);
2628
2629 deflated_count = deflate_monitor_list(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p);
2630 Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_use);
2631 }
2632
2633 if (free_head_p != NULL) {
2634 // Move the deflated ObjectMonitors back to the global free list.
2635 // No races on the working free list so no need for load_acquire().
2636 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2637 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2638 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2639 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2640 Atomic::add(deflated_count, &counters->n_scavenged);
2641 }
2642 timer.stop();
2643
2644 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2645 LogStreamHandle(Info, monitorinflation) lsh_info;
2646 LogStream* ls = NULL;
2647 if (log_is_enabled(Debug, monitorinflation)) {
2648 ls = &lsh_debug;
2649 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2650 ls = &lsh_info;
2651 }
2652 if (ls != NULL) {
2653 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2654 }
2655 }
2656
2657 // Deflate global idle ObjectMonitors using a JavaThread.
2658 //
2659 void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() {
2660 assert(AsyncDeflateIdleMonitors, "sanity check");
2661 assert(Thread::current()->is_Java_thread(), "precondition");
2662 JavaThread* self = JavaThread::current();
2663
2664 deflate_common_idle_monitors_using_JT(true /* is_global */, self);
2665 }
2666
2667 // Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread.
2668 //
2669 void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) {
2670 assert(AsyncDeflateIdleMonitors, "sanity check");
2671 assert(Thread::current()->is_Java_thread(), "precondition");
2672
2673 deflate_common_idle_monitors_using_JT(false /* !is_global */, target);
2674 }
2675
2676 // Deflate global or per-thread idle ObjectMonitors using a JavaThread.
2677 //
2678 void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) {
2679 JavaThread* self = JavaThread::current();
2680
2681 int deflated_count = 0;
2682 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors
2683 ObjectMonitor* free_tail_p = NULL;
2684 ObjectMonitor* saved_mid_in_use_p = NULL;
2685 elapsedTimer timer;
2686
2687 if (log_is_enabled(Info, monitorinflation)) {
2688 timer.start();
2689 }
2690
2691 if (is_global) {
2692 OM_PERFDATA_OP(MonExtant, set_value(OrderAccess::load_acquire(&g_om_in_use_count)));
2693 } else {
2694 OM_PERFDATA_OP(MonExtant, inc(OrderAccess::load_acquire(&target->om_in_use_count)));
2695 }
2696
2697 do {
2698 int local_deflated_count;
2699 if (is_global) {
2700 local_deflated_count = deflate_monitor_list_using_JT(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2701 } else {
2702 local_deflated_count = deflate_monitor_list_using_JT(&target->om_in_use_list, &target->om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2703 }
2704 deflated_count += local_deflated_count;
2705
2706 if (free_head_p != NULL) {
2707 // Move the deflated ObjectMonitors to the global free list.
2708 // No races on the working list so no need for load_acquire().
2709 guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count);
2710 // Note: The target thread can be doing an om_alloc() that
2711 // is trying to prepend an ObjectMonitor on its in-use list
2712 // at the same time that we have deflated the current in-use
2713 // list head and put it on the local free list. prepend_to_common()
2714 // will detect the race and retry which avoids list corruption,
2715 // but the next field in free_tail_p can flicker to marked
2716 // and then unmarked while prepend_to_common() is sorting it
2717 // all out.
2718 assert(unmarked_next(free_tail_p) == NULL, "must be NULL: _next_om="
2719 INTPTR_FORMAT, p2i(unmarked_next(free_tail_p)));
2720
2721 prepend_list_to_g_free_list(free_head_p, free_tail_p, local_deflated_count);
2722
2723 OM_PERFDATA_OP(Deflations, inc(local_deflated_count));
2724 }
2725
2726 if (saved_mid_in_use_p != NULL) {
2727 // deflate_monitor_list_using_JT() detected a safepoint starting.
2728 timer.stop();
2729 {
2730 if (is_global) {
2731 log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint.");
2732 } else {
2733 log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target));
2734 }
2735 assert(SafepointSynchronize::is_synchronizing(), "sanity check");
2736 ThreadBlockInVM blocker(self);
2737 }
2738 // Prepare for another loop after the safepoint.
2739 free_head_p = NULL;
2740 free_tail_p = NULL;
2741 if (log_is_enabled(Info, monitorinflation)) {
2742 timer.start();
2743 }
2744 }
2745 } while (saved_mid_in_use_p != NULL);
2746 timer.stop();
2747
2748 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2749 LogStreamHandle(Info, monitorinflation) lsh_info;
2750 LogStream* ls = NULL;
2751 if (log_is_enabled(Debug, monitorinflation)) {
2752 ls = &lsh_debug;
2753 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2754 ls = &lsh_info;
2755 }
2756 if (ls != NULL) {
2757 if (is_global) {
2758 ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2759 } else {
2760 ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count);
2761 }
2762 }
2763 }
2764
2765 void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2766 // Report the cumulative time for deflating each thread's idle
2767 // monitors. Note: if the work is split among more than one
2768 // worker thread, then the reported time will likely be more
2769 // than a beginning to end measurement of the phase.
2770 // Note: AsyncDeflateIdleMonitors only deflates per-thread idle
2771 // monitors at a safepoint when a special deflation has been requested.
2772 log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d",
2773 counters->per_thread_times,
2774 OrderAccess::load_acquire(&counters->per_thread_scavenged));
2775
2776 bool needs_special_deflation = is_special_deflation_requested();
2777 if (!AsyncDeflateIdleMonitors || needs_special_deflation) {
2778 // AsyncDeflateIdleMonitors does not use these counters unless
2779 // there is a special deflation request.
2780
2781 OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged));
2782 OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation));
2783 }
2784
2785 if (log_is_enabled(Debug, monitorinflation)) {
2786 // exit_globals()'s call to audit_and_print_stats() is done
2787 // at the Info level.
2788 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2789 } else if (log_is_enabled(Info, monitorinflation)) {
2790 log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, "
2791 "g_om_free_count=%d",
2792 OrderAccess::load_acquire(&g_om_population),
2793 OrderAccess::load_acquire(&g_om_in_use_count),
2794 OrderAccess::load_acquire(&g_om_free_count));
2795 }
2796
2797 ForceMonitorScavenge = 0; // Reset
2798 GVars.stw_random = os::random();
2799 GVars.stw_cycle++;
2800 if (needs_special_deflation) {
2801 set_is_special_deflation_requested(false); // special deflation is done
2802 }
2803 }
2804
2805 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
2806 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2807
2808 if (AsyncDeflateIdleMonitors && !is_special_deflation_requested()) {
2809 // Nothing to do if a special deflation has NOT been requested.
2810 return;
2811 }
2812
2813 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2814 ObjectMonitor* free_tail_p = NULL;
2815 elapsedTimer timer;
2816
2817 if (log_is_enabled(Info, safepoint, cleanup) ||
2818 log_is_enabled(Info, monitorinflation)) {
2819 timer.start();
2820 }
2821
2822 // Update n_in_circulation before om_in_use_count is updated by deflation.
2823 Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_circulation);
2824
2825 int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p);
2826 Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_use);
2827
2828 if (free_head_p != NULL) {
2829 // Move the deflated ObjectMonitors back to the global free list.
2830 // No races on the working list so no need for load_acquire().
2831 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2832 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2833 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2834 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2835 Atomic::add(deflated_count, &counters->n_scavenged);
2836 Atomic::add(deflated_count, &counters->per_thread_scavenged);
2837 }
2838
2839 timer.stop();
2840 // Safepoint logging cares about cumulative per_thread_times and
2841 // we'll capture most of the cost, but not the muxRelease() which
2842 // should be cheap.
2843 counters->per_thread_times += timer.seconds();
2844
2845 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2846 LogStreamHandle(Info, monitorinflation) lsh_info;
2847 LogStream* ls = NULL;
2848 if (log_is_enabled(Debug, monitorinflation)) {
2849 ls = &lsh_debug;
2850 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2851 ls = &lsh_info;
2852 }
2853 if (ls != NULL) {
2854 ls->print_cr("jt=" INTPTR_FORMAT ": deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(thread), timer.seconds(), deflated_count);
2855 }
2856 }
2857
2858 // Monitor cleanup on JavaThread::exit
2859
2860 // Iterate through monitor cache and attempt to release thread's monitors
2861 // Gives up on a particular monitor if an exception occurs, but continues
2862 // the overall iteration, swallowing the exception.
2863 class ReleaseJavaMonitorsClosure: public MonitorClosure {
2864 private:
2865 TRAPS;
2866
2867 public:
2868 ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {}
2869 void do_monitor(ObjectMonitor* mid) {
2870 if (mid->owner() == THREAD) {
2871 (void)mid->complete_exit(CHECK);
2872 }
2873 }
2874 };
2875
2876 // Release all inflated monitors owned by THREAD. Lightweight monitors are
2877 // ignored. This is meant to be called during JNI thread detach which assumes
2878 // all remaining monitors are heavyweight. All exceptions are swallowed.
2879 // Scanning the extant monitor list can be time consuming.
2880 // A simple optimization is to add a per-thread flag that indicates a thread
2881 // called jni_monitorenter() during its lifetime.
2882 //
2883 // Instead of No_Savepoint_Verifier it might be cheaper to
2884 // use an idiom of the form:
2885 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
2886 // <code that must not run at safepoint>
2887 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
2888 // Since the tests are extremely cheap we could leave them enabled
2889 // for normal product builds.
2890
2891 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
2892 assert(THREAD == JavaThread::current(), "must be current Java thread");
2893 NoSafepointVerifier nsv;
2894 ReleaseJavaMonitorsClosure rjmc(THREAD);
2895 ObjectSynchronizer::monitors_iterate(&rjmc);
2896 THREAD->clear_pending_exception();
2897 }
2898
2899 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
2900 switch (cause) {
2901 case inflate_cause_vm_internal: return "VM Internal";
2902 case inflate_cause_monitor_enter: return "Monitor Enter";
2903 case inflate_cause_wait: return "Monitor Wait";
2904 case inflate_cause_notify: return "Monitor Notify";
2905 case inflate_cause_hash_code: return "Monitor Hash Code";
2906 case inflate_cause_jni_enter: return "JNI Monitor Enter";
2907 case inflate_cause_jni_exit: return "JNI Monitor Exit";
2908 default:
2909 ShouldNotReachHere();
2910 }
2911 return "Unknown";
2912 }
2913
2914 //------------------------------------------------------------------------------
2915 // Debugging code
2916
2917 u_char* ObjectSynchronizer::get_gvars_addr() {
2918 return (u_char*)&GVars;
2919 }
2920
2921 u_char* ObjectSynchronizer::get_gvars_hc_sequence_addr() {
2922 return (u_char*)&GVars.hc_sequence;
2923 }
2924
2925 size_t ObjectSynchronizer::get_gvars_size() {
2926 return sizeof(SharedGlobals);
2927 }
2928
2929 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
2930 return (u_char*)&GVars.stw_random;
2931 }
2932
2933 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
2934 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
2935
2936 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2937 LogStreamHandle(Info, monitorinflation) lsh_info;
2938 LogStreamHandle(Trace, monitorinflation) lsh_trace;
2939 LogStream* ls = NULL;
2940 if (log_is_enabled(Trace, monitorinflation)) {
2941 ls = &lsh_trace;
2942 } else if (log_is_enabled(Debug, monitorinflation)) {
2943 ls = &lsh_debug;
2944 } else if (log_is_enabled(Info, monitorinflation)) {
2945 ls = &lsh_info;
2946 }
2947 assert(ls != NULL, "sanity check");
2948
2949 // Log counts for the global and per-thread monitor lists:
2950 int chk_om_population = log_monitor_list_counts(ls);
2951 int error_cnt = 0;
2952
2953 ls->print_cr("Checking global lists:");
2954
2955 // Check g_om_population:
2956 if (OrderAccess::load_acquire(&g_om_population) == chk_om_population) {
2957 ls->print_cr("g_om_population=%d equals chk_om_population=%d",
2958 OrderAccess::load_acquire(&g_om_population),
2959 chk_om_population);
2960 } else {
2961 ls->print_cr("ERROR: g_om_population=%d is not equal to "
2962 "chk_om_population=%d",
2963 OrderAccess::load_acquire(&g_om_population),
2964 chk_om_population);
2965 error_cnt++;
2966 }
2967
2968 // Check g_om_in_use_list and g_om_in_use_count:
2969 chk_global_in_use_list_and_count(ls, &error_cnt);
2970
2971 // Check g_free_list and g_om_free_count:
2972 chk_global_free_list_and_count(ls, &error_cnt);
2973
2974 ls->print_cr("Checking per-thread lists:");
2975
2976 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2977 // Check om_in_use_list and om_in_use_count:
2978 chk_per_thread_in_use_list_and_count(jt, ls, &error_cnt);
2979
2980 // Check om_free_list and om_free_count:
2981 chk_per_thread_free_list_and_count(jt, ls, &error_cnt);
2982 }
2983
2984 if (error_cnt == 0) {
2985 ls->print_cr("No errors found in monitor list checks.");
2986 } else {
2987 log_error(monitorinflation)("found monitor list errors: error_cnt=%d", error_cnt);
2988 }
2989
2990 if ((on_exit && log_is_enabled(Info, monitorinflation)) ||
2991 (!on_exit && log_is_enabled(Trace, monitorinflation))) {
2992 // When exiting this log output is at the Info level. When called
2993 // at a safepoint, this log output is at the Trace level since
2994 // there can be a lot of it.
2995 log_in_use_monitor_details(ls);
2996 }
2997
2998 ls->flush();
2999
3000 guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
3001 }
3002
3003 // Check a free monitor entry; log any errors.
3004 void ObjectSynchronizer::chk_free_entry(JavaThread* jt, ObjectMonitor* n,
3005 outputStream * out, int *error_cnt_p) {
3006 stringStream ss;
3007 if (n->is_busy()) {
3008 if (jt != NULL) {
3009 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3010 ": free per-thread monitor must not be busy: %s", p2i(jt),
3011 p2i(n), n->is_busy_to_string(&ss));
3012 } else {
3013 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3014 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
3015 }
3016 *error_cnt_p = *error_cnt_p + 1;
3017 }
3018 if (n->header().value() != 0) {
3019 if (jt != NULL) {
3020 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3021 ": free per-thread monitor must have NULL _header "
3022 "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n),
3023 n->header().value());
3024 *error_cnt_p = *error_cnt_p + 1;
3025 } else if (!AsyncDeflateIdleMonitors) {
3026 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3027 "must have NULL _header field: _header=" INTPTR_FORMAT,
3028 p2i(n), n->header().value());
3029 *error_cnt_p = *error_cnt_p + 1;
3030 }
3031 }
3032 if (n->object() != NULL) {
3033 if (jt != NULL) {
3034 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3035 ": free per-thread monitor must have NULL _object "
3036 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
3037 p2i(n->object()));
3038 } else {
3039 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3040 "must have NULL _object field: _object=" INTPTR_FORMAT,
3041 p2i(n), p2i(n->object()));
3042 }
3043 *error_cnt_p = *error_cnt_p + 1;
3044 }
3045 }
3046
3047 // Check the global free list and count; log the results of the checks.
3048 void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out,
3049 int *error_cnt_p) {
3050 int chk_om_free_count = 0;
3051 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_free_list); n != NULL; n = unmarked_next(n)) {
3052 chk_free_entry(NULL /* jt */, n, out, error_cnt_p);
3053 chk_om_free_count++;
3054 }
3055 if (OrderAccess::load_acquire(&g_om_free_count) == chk_om_free_count) {
3056 out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d",
3057 OrderAccess::load_acquire(&g_om_free_count),
3058 chk_om_free_count);
3059 } else {
3060 // With lock free access to g_free_list, it is possible for an
3061 // ObjectMonitor to be prepended to g_free_list after we started
3062 // calculating chk_om_free_count so g_om_free_count may not
3063 // match anymore.
3064 out->print_cr("WARNING: g_om_free_count=%d is not equal to "
3065 "chk_om_free_count=%d",
3066 OrderAccess::load_acquire(&g_om_free_count),
3067 chk_om_free_count);
3068 }
3069 }
3070
3071 // Check the global in-use list and count; log the results of the checks.
3072 void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out,
3073 int *error_cnt_p) {
3074 int chk_om_in_use_count = 0;
3075 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) {
3076 chk_in_use_entry(NULL /* jt */, n, out, error_cnt_p);
3077 chk_om_in_use_count++;
3078 }
3079 if (OrderAccess::load_acquire(&g_om_in_use_count) == chk_om_in_use_count) {
3080 out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d",
3081 OrderAccess::load_acquire(&g_om_in_use_count),
3082 chk_om_in_use_count);
3083 } else {
3084 out->print_cr("ERROR: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d",
3085 OrderAccess::load_acquire(&g_om_in_use_count),
3086 chk_om_in_use_count);
3087 *error_cnt_p = *error_cnt_p + 1;
3088 }
3089 }
3090
3091 // Check an in-use monitor entry; log any errors.
3092 void ObjectSynchronizer::chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,
3093 outputStream * out, int *error_cnt_p) {
3094 if (n->header().value() == 0) {
3095 if (jt != NULL) {
3096 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3097 ": in-use per-thread monitor must have non-NULL _header "
3098 "field.", p2i(jt), p2i(n));
3099 } else {
3100 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
3101 "must have non-NULL _header field.", p2i(n));
3102 }
3103 *error_cnt_p = *error_cnt_p + 1;
3104 }
3105 if (n->object() == NULL) {
3106 if (jt != NULL) {
3107 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3108 ": in-use per-thread monitor must have non-NULL _object "
3109 "field.", p2i(jt), p2i(n));
3110 } else {
3111 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
3112 "must have non-NULL _object field.", p2i(n));
3113 }
3114 *error_cnt_p = *error_cnt_p + 1;
3115 }
3116 const oop obj = (oop)n->object();
3117 const markWord mark = obj->mark();
3118 if (!mark.has_monitor()) {
3119 if (jt != NULL) {
3120 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3121 ": in-use per-thread monitor's object does not think "
3122 "it has a monitor: obj=" INTPTR_FORMAT ", mark="
3123 INTPTR_FORMAT, p2i(jt), p2i(n), p2i(obj), mark.value());
3124 } else {
3125 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
3126 "monitor's object does not think it has a monitor: obj="
3127 INTPTR_FORMAT ", mark=" INTPTR_FORMAT, p2i(n),
3128 p2i(obj), mark.value());
3129 }
3130 *error_cnt_p = *error_cnt_p + 1;
3131 }
3132 ObjectMonitor* const obj_mon = mark.monitor();
3133 if (n != obj_mon) {
3134 if (jt != NULL) {
3135 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3136 ": in-use per-thread monitor's object does not refer "
3137 "to the same monitor: obj=" INTPTR_FORMAT ", mark="
3138 INTPTR_FORMAT ", obj_mon=" INTPTR_FORMAT, p2i(jt),
3139 p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3140 } else {
3141 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
3142 "monitor's object does not refer to the same monitor: obj="
3143 INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
3144 INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3145 }
3146 *error_cnt_p = *error_cnt_p + 1;
3147 }
3148 }
3149
3150 // Check the thread's free list and count; log the results of the checks.
3151 void ObjectSynchronizer::chk_per_thread_free_list_and_count(JavaThread *jt,
3152 outputStream * out,
3153 int *error_cnt_p) {
3154 int chk_om_free_count = 0;
3155 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_free_list); n != NULL; n = unmarked_next(n)) {
3156 chk_free_entry(jt, n, out, error_cnt_p);
3157 chk_om_free_count++;
3158 }
3159 if (OrderAccess::load_acquire(&jt->om_free_count) == chk_om_free_count) {
3160 out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals "
3161 "chk_om_free_count=%d", p2i(jt),
3162 OrderAccess::load_acquire(&jt->om_free_count),
3163 chk_om_free_count);
3164 } else {
3165 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not "
3166 "equal to chk_om_free_count=%d", p2i(jt),
3167 OrderAccess::load_acquire(&jt->om_free_count),
3168 chk_om_free_count);
3169 *error_cnt_p = *error_cnt_p + 1;
3170 }
3171 }
3172
3173 // Check the thread's in-use list and count; log the results of the checks.
3174 void ObjectSynchronizer::chk_per_thread_in_use_list_and_count(JavaThread *jt,
3175 outputStream * out,
3176 int *error_cnt_p) {
3177 int chk_om_in_use_count = 0;
3178 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) {
3179 chk_in_use_entry(jt, n, out, error_cnt_p);
3180 chk_om_in_use_count++;
3181 }
3182 if (OrderAccess::load_acquire(&jt->om_in_use_count) == chk_om_in_use_count) {
3183 out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals "
3184 "chk_om_in_use_count=%d", p2i(jt),
3185 OrderAccess::load_acquire(&jt->om_in_use_count),
3186 chk_om_in_use_count);
3187 } else {
3188 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not "
3189 "equal to chk_om_in_use_count=%d", p2i(jt),
3190 OrderAccess::load_acquire(&jt->om_in_use_count),
3191 chk_om_in_use_count);
3192 *error_cnt_p = *error_cnt_p + 1;
3193 }
3194 }
3195
3196 // Log details about ObjectMonitors on the in-use lists. The 'BHL'
3197 // flags indicate why the entry is in-use, 'object' and 'object type'
3198 // indicate the associated object and its type.
3199 void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out) {
3200 stringStream ss;
3201 if (OrderAccess::load_acquire(&g_om_in_use_count) > 0) {
3202 out->print_cr("In-use global monitor info:");
3203 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3204 out->print_cr("%18s %s %7s %18s %18s",
3205 "monitor", "BHL", "ref_cnt", "object", "object type");
3206 out->print_cr("================== === ======= ================== ==================");
3207 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) {
3208 const oop obj = (oop) n->object();
3209 const markWord mark = n->header();
3210 ResourceMark rm;
3211 out->print(INTPTR_FORMAT " %d%d%d %7d " INTPTR_FORMAT " %s",
3212 p2i(n), n->is_busy() != 0, mark.hash() != 0,
3213 n->owner() != NULL, (int)n->ref_count(), p2i(obj),
3214 obj->klass()->external_name());
3215 if (n->is_busy() != 0) {
3216 out->print(" (%s)", n->is_busy_to_string(&ss));
3217 ss.reset();
3218 }
3219 out->cr();
3220 }
3221 }
3222
3223 out->print_cr("In-use per-thread monitor info:");
3224 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3225 out->print_cr("%18s %18s %s %7s %18s %18s",
3226 "jt", "monitor", "BHL", "ref_cnt", "object", "object type");
3227 out->print_cr("================== ================== === ======= ================== ==================");
3228 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3229 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) {
3230 const oop obj = (oop) n->object();
3231 const markWord mark = n->header();
3232 ResourceMark rm;
3233 out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d %7d "
3234 INTPTR_FORMAT " %s", p2i(jt), p2i(n), n->is_busy() != 0,
3235 mark.hash() != 0, n->owner() != NULL, (int)n->ref_count(),
3236 p2i(obj), obj->klass()->external_name());
3237 if (n->is_busy() != 0) {
3238 out->print(" (%s)", n->is_busy_to_string(&ss));
3239 ss.reset();
3240 }
3241 out->cr();
3242 }
3243 }
3244
3245 out->flush();
3246 }
3247
3248 // Log counts for the global and per-thread monitor lists and return
3249 // the population count.
3250 int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) {
3251 int pop_count = 0;
3252 out->print_cr("%18s %10s %10s %10s",
3253 "Global Lists:", "InUse", "Free", "Total");
3254 out->print_cr("================== ========== ========== ==========");
3255 out->print_cr("%18s %10d %10d %10d", "",
3256 OrderAccess::load_acquire(&g_om_in_use_count),
3257 OrderAccess::load_acquire(&g_om_free_count),
3258 OrderAccess::load_acquire(&g_om_population));
3259 pop_count += OrderAccess::load_acquire(&g_om_in_use_count) +
3260 OrderAccess::load_acquire(&g_om_free_count);
3261
3262 out->print_cr("%18s %10s %10s %10s",
3263 "Per-Thread Lists:", "InUse", "Free", "Provision");
3264 out->print_cr("================== ========== ========== ==========");
3265
3266 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3267 out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt),
3268 OrderAccess::load_acquire(&jt->om_in_use_count),
3269 OrderAccess::load_acquire(&jt->om_free_count),
3270 jt->om_free_provision);
3271 pop_count += OrderAccess::load_acquire(&jt->om_in_use_count) +
3272 OrderAccess::load_acquire(&jt->om_free_count);
3273 }
3274 return pop_count;
3275 }
3276
3277 #ifndef PRODUCT
3278
3279 // Check if monitor belongs to the monitor cache
3280 // The list is grow-only so it's *relatively* safe to traverse
3281 // the list of extant blocks without taking a lock.
3282
3283 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
3284 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
3285 while (block != NULL) {
3286 assert(block->object() == CHAINMARKER, "must be a block header");
3287 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
3288 address mon = (address)monitor;
3289 address blk = (address)block;
3290 size_t diff = mon - blk;
3291 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
3292 return 1;
3293 }
3294 // unmarked_next() is not needed with g_block_list (no next field marking).
3295 block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om);
3296 }
3297 return 0;
3298 }
3299
3300 #endif