101 }
102
103 #else // ndef DTRACE_ENABLED
104
105 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon) {;}
106 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon) {;}
107
108 #endif // ndef DTRACE_ENABLED
109
110 // This exists only as a workaround of dtrace bug 6254741
111 int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
112 DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
113 return 0;
114 }
115
116 #define NINFLATIONLOCKS 256
117 static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];
118
119 // global list of blocks of monitors
120 PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL;
121 // Global ObjectMonitor free list. Newly allocated and deflated
122 // ObjectMonitors are prepended here.
123 ObjectMonitor* volatile ObjectSynchronizer::g_free_list = NULL;
124 // Global ObjectMonitor in-use list. When a JavaThread is exiting,
125 // ObjectMonitors on its per-thread in-use list are prepended here.
126 ObjectMonitor* volatile ObjectSynchronizer::g_om_in_use_list = NULL;
127 int ObjectSynchronizer::g_om_in_use_count = 0; // # on g_om_in_use_list
128
129 static volatile intptr_t gListLock = 0; // protects global monitor lists
130 static volatile int g_om_free_count = 0; // # on g_free_list
131 static volatile int g_om_population = 0; // # Extant -- in circulation
132
133 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
134
135
136 // =====================> Quick functions
137
138 // The quick_* forms are special fast-path variants used to improve
139 // performance. In the simplest case, a "quick_*" implementation could
140 // simply return false, in which case the caller will perform the necessary
141 // state transitions and call the slow-path form.
142 // The fast-path is designed to handle frequently arising cases in an efficient
143 // manner and is just a degenerate "optimistic" variant of the slow-path.
144 // returns true -- to indicate the call was satisfied.
145 // returns false -- to indicate the call needs the services of the slow-path.
146 // A no-loitering ordinance is in effect for code in the quick_* family
147 // operators: safepoints or indefinite blocking (blocking that might span a
148 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
149 // entry.
150 //
151 // Consider: An interesting optimization is to have the JIT recognize the
152 // following common idiom:
153 // synchronized (someobj) { .... ; notify(); }
154 // That is, we find a notify() or notifyAll() call that immediately precedes
155 // the monitorexit operation. In that case the JIT could fuse the operations
194 }
195
196 // biased locking and any other IMS exception states take the slow-path
197 return false;
198 }
199
200
201 // The LockNode emitted directly at the synchronization site would have
202 // been too big if it were to have included support for the cases of inflated
203 // recursive enter and exit, so they go here instead.
204 // Note that we can't safely call AsyncPrintJavaStack() from within
205 // quick_enter() as our thread state remains _in_Java.
206
207 bool ObjectSynchronizer::quick_enter(oop obj, Thread* self,
208 BasicLock * lock) {
209 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
210 assert(self->is_Java_thread(), "invariant");
211 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
212 NoSafepointVerifier nsv;
213 if (obj == NULL) return false; // Need to throw NPE
214 const markWord mark = obj->mark();
215
216 if (mark.has_monitor()) {
217 ObjectMonitor* const m = mark.monitor();
218 assert(oopDesc::equals((oop) m->object(), obj), "invariant");
219 Thread* const owner = (Thread *) m->_owner;
220
221 // Lock contention and Transactional Lock Elision (TLE) diagnostics
222 // and observability
223 // Case: light contention possibly amenable to TLE
224 // Case: TLE inimical operations such as nested/recursive synchronization
225
226 if (owner == self) {
227 m->_recursions++;
228 return true;
229 }
230
231 // This Java Monitor is inflated so obj's header will never be
232 // displaced to this thread's BasicLock. Make the displaced header
233 // non-NULL so this BasicLock is not seen as recursive nor as
234 // being locked. We do this unconditionally so that this thread's
235 // BasicLock cannot be mis-interpreted by any stack walkers. For
236 // performance reasons, stack walkers generally first check for
237 // Biased Locking in the object's header, the second check is for
238 // stack-locking in the object's header, the third check is for
239 // recursive stack-locking in the displaced header in the BasicLock,
240 // and last are the inflated Java Monitor (ObjectMonitor) checks.
241 lock->set_displaced_header(markWord::unused_mark());
242
243 if (owner == NULL && Atomic::replace_if_null(self, &(m->_owner))) {
244 assert(m->_recursions == 0, "invariant");
245 return true;
246 }
247 }
248
249 // Note that we could inflate in quick_enter.
250 // This is likely a useful optimization
251 // Critically, in quick_enter() we must not:
252 // -- perform bias revocation, or
253 // -- block indefinitely, or
254 // -- reach a safepoint
255
256 return false; // revert to slow-path
257 }
258
259 // -----------------------------------------------------------------------------
260 // Fast Monitor Enter/Exit
261 // This the fast monitor enter. The interpreter and compiler use
262 // some assembly copies of this code. Make sure update those code
263 // if the following function is changed. The implementation is
264 // extremely sensitive to race condition. Be careful.
265
266 void ObjectSynchronizer::fast_enter(Handle obj, BasicLock* lock,
310 // does not own the Java Monitor.
311 ObjectMonitor* m = mark.monitor();
312 assert(((oop)(m->object()))->mark() == mark, "invariant");
313 assert(m->is_entered(THREAD), "invariant");
314 }
315 }
316 #endif
317 return;
318 }
319
320 if (mark == markWord::from_pointer(lock)) {
321 // If the object is stack-locked by the current thread, try to
322 // swing the displaced header from the BasicLock back to the mark.
323 assert(dhw.is_neutral(), "invariant");
324 if (object->cas_set_mark(dhw, mark) == mark) {
325 return;
326 }
327 }
328
329 // We have to take the slow-path of possible inflation and then exit.
330 inflate(THREAD, object, inflate_cause_vm_internal)->exit(true, THREAD);
331 }
332
333 // -----------------------------------------------------------------------------
334 // Interpreter/Compiler Slow Case
335 // This routine is used to handle interpreter/compiler slow case
336 // We don't need to use fast path here, because it must have been
337 // failed in the interpreter/compiler code.
338 void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) {
339 markWord mark = obj->mark();
340 assert(!mark.has_bias_pattern(), "should not see bias pattern here");
341
342 if (mark.is_neutral()) {
343 // Anticipate successful CAS -- the ST of the displaced mark must
344 // be visible <= the ST performed by the CAS.
345 lock->set_displaced_header(mark);
346 if (mark == obj()->cas_set_mark(markWord::from_pointer(lock), mark)) {
347 return;
348 }
349 // Fall through to inflate() ...
350 } else if (mark.has_locker() &&
351 THREAD->is_lock_owned((address)mark.locker())) {
352 assert(lock != mark.locker(), "must not re-lock the same lock");
353 assert(lock != (BasicLock*)obj->mark().value(), "don't relock with same BasicLock");
354 lock->set_displaced_header(markWord::from_pointer(NULL));
355 return;
356 }
357
358 // The object header will never be displaced to this lock,
359 // so it does not matter what the value is, except that it
360 // must be non-zero to avoid looking like a re-entrant lock,
361 // and must not look locked either.
362 lock->set_displaced_header(markWord::unused_mark());
363 inflate(THREAD, obj(), inflate_cause_monitor_enter)->enter(THREAD);
364 }
365
366 // This routine is used to handle interpreter/compiler slow case
367 // We don't need to use fast path here, because it must have
368 // failed in the interpreter/compiler code. Simply use the heavy
369 // weight monitor should be ok, unless someone find otherwise.
370 void ObjectSynchronizer::slow_exit(oop object, BasicLock* lock, TRAPS) {
371 fast_exit(object, lock, THREAD);
372 }
373
374 // -----------------------------------------------------------------------------
375 // Class Loader support to workaround deadlocks on the class loader lock objects
376 // Also used by GC
377 // complete_exit()/reenter() are used to wait on a nested lock
378 // i.e. to give up an outer lock completely and then re-enter
379 // Used when holding nested locks - lock acquisition order: lock1 then lock2
380 // 1) complete_exit lock1 - saving recursion count
381 // 2) wait on lock2
382 // 3) when notified on lock2, unlock lock2
383 // 4) reenter lock1 with original recursion count
384 // 5) lock lock2
385 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
386 intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
387 if (UseBiasedLocking) {
388 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
389 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
390 }
391
392 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal);
393
394 return monitor->complete_exit(THREAD);
395 }
396
397 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
398 void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) {
399 if (UseBiasedLocking) {
400 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
401 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
402 }
403
404 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal);
405
406 monitor->reenter(recursion, THREAD);
407 }
408 // -----------------------------------------------------------------------------
409 // JNI locks on java objects
410 // NOTE: must use heavy weight monitor to handle jni monitor enter
411 void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) {
412 // the current locking is from JNI instead of Java code
413 if (UseBiasedLocking) {
414 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
415 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
416 }
417 THREAD->set_current_pending_monitor_is_from_java(false);
418 inflate(THREAD, obj(), inflate_cause_jni_enter)->enter(THREAD);
419 THREAD->set_current_pending_monitor_is_from_java(true);
420 }
421
422 // NOTE: must use heavy weight monitor to handle jni monitor exit
423 void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
424 if (UseBiasedLocking) {
425 Handle h_obj(THREAD, obj);
426 BiasedLocking::revoke_and_rebias(h_obj, false, THREAD);
427 obj = h_obj();
428 }
429 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
430
431 ObjectMonitor* monitor = inflate(THREAD, obj, inflate_cause_jni_exit);
432 // If this thread has locked the object, exit the monitor. We
433 // intentionally do not use CHECK here because we must exit the
434 // monitor even if an exception is pending.
435 if (monitor->check_owner(THREAD)) {
436 monitor->exit(true, THREAD);
437 }
438 }
439
440 // -----------------------------------------------------------------------------
441 // Internal VM locks on java objects
442 // standard constructor, allows locking failures
443 ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool do_lock) {
444 _dolock = do_lock;
445 _thread = thread;
446 _thread->check_for_valid_safepoint_state(false);
447 _obj = obj;
448
449 if (_dolock) {
450 ObjectSynchronizer::fast_enter(_obj, &_lock, false, _thread);
451 }
452 }
453
454 ObjectLocker::~ObjectLocker() {
455 if (_dolock) {
456 ObjectSynchronizer::fast_exit(_obj(), &_lock, _thread);
457 }
458 }
459
460
461 // -----------------------------------------------------------------------------
462 // Wait/Notify/NotifyAll
463 // NOTE: must use heavy weight monitor to handle wait()
464 int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
465 if (UseBiasedLocking) {
466 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
467 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
468 }
469 if (millis < 0) {
470 THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
471 }
472 ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_wait);
473
474 DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);
475 monitor->wait(millis, true, THREAD);
476
477 // This dummy call is in place to get around dtrace bug 6254741. Once
478 // that's fixed we can uncomment the following line, remove the call
479 // and change this function back into a "void" func.
480 // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
481 return dtrace_waited_probe(monitor, obj, THREAD);
482 }
483
484 void ObjectSynchronizer::wait_uninterruptibly(Handle obj, jlong millis, TRAPS) {
485 if (UseBiasedLocking) {
486 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
487 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
488 }
489 if (millis < 0) {
490 THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
491 }
492 inflate(THREAD, obj(), inflate_cause_wait)->wait(millis, false, THREAD);
493 }
494
495 void ObjectSynchronizer::notify(Handle obj, TRAPS) {
496 if (UseBiasedLocking) {
497 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
498 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
499 }
500
501 markWord mark = obj->mark();
502 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
503 return;
504 }
505 inflate(THREAD, obj(), inflate_cause_notify)->notify(THREAD);
506 }
507
508 // NOTE: see comment of notify()
509 void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
510 if (UseBiasedLocking) {
511 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
512 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
513 }
514
515 markWord mark = obj->mark();
516 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
517 return;
518 }
519 inflate(THREAD, obj(), inflate_cause_notify)->notifyAll(THREAD);
520 }
521
522 // -----------------------------------------------------------------------------
523 // Hash Code handling
524 //
525 // Performance concern:
526 // OrderAccess::storestore() calls release() which at one time stored 0
527 // into the global volatile OrderAccess::dummy variable. This store was
528 // unnecessary for correctness. Many threads storing into a common location
529 // causes considerable cache migration or "sloshing" on large SMP systems.
530 // As such, I avoided using OrderAccess::storestore(). In some cases
531 // OrderAccess::fence() -- which incurs local latency on the executing
532 // processor -- is a better choice as it scales on SMP systems.
533 //
534 // See http://blogs.oracle.com/dave/entry/biased_locking_in_hotspot for
535 // a discussion of coherency costs. Note that all our current reference
536 // platforms provide strong ST-ST order, so the issue is moot on IA32,
537 // x64, and SPARC.
538 //
539 // As a general policy we use "volatile" to control compiler-based reordering
540 // and explicit fences (barriers) to control for architectural reordering
541 // performed by the CPU(s) or platform.
542
543 struct SharedGlobals {
544 char _pad_prefix[DEFAULT_CACHE_LINE_SIZE];
545 // These are highly shared mostly-read variables.
546 // To avoid false-sharing they need to be the sole occupants of a cache line.
547 volatile int stw_random;
548 volatile int stw_cycle;
549 DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int) * 2);
550 // Hot RW variable -- Sequester to avoid false-sharing
551 volatile int hc_sequence;
552 DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile int));
553 };
554
555 static SharedGlobals GVars;
556 static int MonitorScavengeThreshold = 1000000;
557 static volatile int ForceMonitorScavenge = 0; // Scavenge required and pending
558
559 static markWord read_stable_mark(oop obj) {
560 markWord mark = obj->mark();
561 if (!mark.is_being_inflated()) {
562 return mark; // normal fast-path return
563 }
564
565 int its = 0;
566 for (;;) {
567 markWord mark = obj->mark();
568 if (!mark.is_being_inflated()) {
569 return mark; // normal fast-path return
570 }
571
572 // The object is being inflated by some other thread.
693 Handle hobj(self, obj);
694 // Relaxing assertion for bug 6320749.
695 assert(Universe::verify_in_progress() ||
696 !SafepointSynchronize::is_at_safepoint(),
697 "biases should not be seen by VM thread here");
698 BiasedLocking::revoke_and_rebias(hobj, false, JavaThread::current());
699 obj = hobj();
700 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
701 }
702 }
703
704 // hashCode() is a heap mutator ...
705 // Relaxing assertion for bug 6320749.
706 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
707 !SafepointSynchronize::is_at_safepoint(), "invariant");
708 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
709 self->is_Java_thread() , "invariant");
710 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
711 ((JavaThread *)self)->thread_state() != _thread_blocked, "invariant");
712
713 ObjectMonitor* monitor = NULL;
714 markWord temp, test;
715 intptr_t hash;
716 markWord mark = read_stable_mark(obj);
717
718 // object should remain ineligible for biased locking
719 assert(!mark.has_bias_pattern(), "invariant");
720
721 if (mark.is_neutral()) {
722 hash = mark.hash(); // this is a normal header
723 if (hash != 0) { // if it has hash, just return it
724 return hash;
725 }
726 hash = get_next_hash(self, obj); // allocate a new hash code
727 temp = mark.copy_set_hash(hash); // merge the hash code into header
728 // use (machine word version) atomic operation to install the hash
729 test = obj->cas_set_mark(temp, mark);
730 if (test == mark) {
731 return hash;
732 }
733 // If atomic operation failed, we must inflate the header
734 // into heavy weight monitor. We could add more code here
735 // for fast path, but it does not worth the complexity.
736 } else if (mark.has_monitor()) {
737 monitor = mark.monitor();
738 temp = monitor->header();
739 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
740 hash = temp.hash();
741 if (hash != 0) {
742 return hash;
743 }
744 // Skip to the following code to reduce code size
745 } else if (self->is_lock_owned((address)mark.locker())) {
746 temp = mark.displaced_mark_helper(); // this is a lightweight monitor owned
747 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
748 hash = temp.hash(); // by current thread, check if the displaced
749 if (hash != 0) { // header contains hash code
750 return hash;
751 }
752 // WARNING:
753 // The displaced header in the BasicLock on a thread's stack
754 // is strictly immutable. It CANNOT be changed in ANY cases.
755 // So we have to inflate the stack lock into an ObjectMonitor
756 // even if the current thread owns the lock. The BasicLock on
757 // a thread's stack can be asynchronously read by other threads
758 // during an inflate() call so any change to that stack memory
759 // may not propagate to other threads correctly.
760 }
761
762 // Inflate the monitor to set hash code
763 monitor = inflate(self, obj, inflate_cause_hash_code);
764 // Load displaced header and check it has hash code
765 mark = monitor->header();
766 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
767 hash = mark.hash();
768 if (hash == 0) {
769 hash = get_next_hash(self, obj);
770 temp = mark.copy_set_hash(hash); // merge hash code into header
771 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
772 uintptr_t v = Atomic::cmpxchg(temp.value(), (volatile uintptr_t*)monitor->header_addr(), mark.value());
773 test = markWord(v);
774 if (test != mark) {
775 // The only non-deflation update to the ObjectMonitor's
776 // header/dmw field is to merge in the hash code. If someone
777 // adds a new usage of the header/dmw field, please update
778 // this code.
779 hash = test.hash();
780 assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value());
781 assert(hash != 0, "Trivial unexpected object/monitor header usage.");
782 }
783 }
784 // We finally get the hash
785 return hash;
786 }
787
788 // Deprecated -- use FastHashCode() instead.
789
790 intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) {
791 return FastHashCode(Thread::current(), obj());
792 }
793
794
795 bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
796 Handle h_obj) {
797 if (UseBiasedLocking) {
798 BiasedLocking::revoke_and_rebias(h_obj, false, thread);
799 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
800 }
801
802 assert(thread == JavaThread::current(), "Can only be called on current thread");
803 oop obj = h_obj();
804
805 markWord mark = read_stable_mark(obj);
806
807 // Uncontended case, header points to stack
808 if (mark.has_locker()) {
809 return thread->is_lock_owned((address)mark.locker());
810 }
811 // Contended case, header points to ObjectMonitor (tagged pointer)
812 if (mark.has_monitor()) {
813 ObjectMonitor* monitor = mark.monitor();
814 return monitor->is_entered(thread) != 0;
815 }
816 // Unlocked case, header in place
817 assert(mark.is_neutral(), "sanity check");
818 return false;
819 }
820
821 // Be aware of this method could revoke bias of the lock object.
822 // This method queries the ownership of the lock handle specified by 'h_obj'.
823 // If the current thread owns the lock, it returns owner_self. If no
824 // thread owns the lock, it returns owner_none. Otherwise, it will return
825 // owner_other.
826 ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership
827 (JavaThread *self, Handle h_obj) {
828 // The caller must beware this method can revoke bias, and
829 // revocation can result in a safepoint.
830 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
831 assert(self->thread_state() != _thread_blocked, "invariant");
832
833 // Possible mark states: neutral, biased, stack-locked, inflated
834
835 if (UseBiasedLocking && h_obj()->mark().has_bias_pattern()) {
836 // CASE: biased
837 BiasedLocking::revoke_and_rebias(h_obj, false, self);
838 assert(!h_obj->mark().has_bias_pattern(),
839 "biases should be revoked by now");
840 }
841
842 assert(self == JavaThread::current(), "Can only be called on current thread");
843 oop obj = h_obj();
844 markWord mark = read_stable_mark(obj);
845
846 // CASE: stack-locked. Mark points to a BasicLock on the owner's stack.
847 if (mark.has_locker()) {
848 return self->is_lock_owned((address)mark.locker()) ?
849 owner_self : owner_other;
850 }
851
852 // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor.
853 // The Object:ObjectMonitor relationship is stable as long as we're
854 // not at a safepoint.
855 if (mark.has_monitor()) {
856 void* owner = mark.monitor()->_owner;
857 if (owner == NULL) return owner_none;
858 return (owner == self ||
859 self->is_lock_owned((address)owner)) ? owner_self : owner_other;
860 }
861
862 // CASE: neutral
863 assert(mark.is_neutral(), "sanity check");
864 return owner_none; // it's unlocked
865 }
866
867 // FIXME: jvmti should call this
868 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
869 if (UseBiasedLocking) {
870 if (SafepointSynchronize::is_at_safepoint()) {
871 BiasedLocking::revoke_at_safepoint(h_obj);
872 } else {
873 BiasedLocking::revoke_and_rebias(h_obj, false, JavaThread::current());
874 }
875 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
876 }
877
878 oop obj = h_obj();
879 address owner = NULL;
880
881 markWord mark = read_stable_mark(obj);
882
883 // Uncontended case, header points to stack
884 if (mark.has_locker()) {
885 owner = (address) mark.locker();
886 }
887
888 // Contended case, header points to ObjectMonitor (tagged pointer)
889 else if (mark.has_monitor()) {
890 ObjectMonitor* monitor = mark.monitor();
891 assert(monitor != NULL, "monitor should be non-null");
892 owner = (address) monitor->owner();
893 }
894
895 if (owner != NULL) {
896 // owning_thread_from_monitor_owner() may also return NULL here
897 return Threads::owning_thread_from_monitor_owner(t_list, owner);
898 }
899
900 // Unlocked case, header in place
901 // Cannot have assertion since this object may have been
902 // locked by another thread when reaching here.
903 // assert(mark.is_neutral(), "sanity check");
904
905 return NULL;
906 }
907
908 // Visitors ...
909
910 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
911 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
912 while (block != NULL) {
913 assert(block->object() == CHAINMARKER, "must be a block header");
914 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
915 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
916 oop object = (oop)mid->object();
917 if (object != NULL) {
918 // Only process with closure if the object is set.
919 closure->do_monitor(mid);
920 }
921 }
922 block = (PaddedObjectMonitor*)block->_next_om;
923 }
924 }
925
926 static bool monitors_used_above_threshold() {
927 if (g_om_population == 0) {
928 return false;
929 }
930 int monitors_used = g_om_population - g_om_free_count;
931 int monitor_usage = (monitors_used * 100LL) / g_om_population;
932 return monitor_usage > MonitorUsedDeflationThreshold;
933 }
934
935 bool ObjectSynchronizer::is_cleanup_needed() {
936 if (MonitorUsedDeflationThreshold > 0) {
937 return monitors_used_above_threshold();
938 }
939 return false;
940 }
941
942 void ObjectSynchronizer::oops_do(OopClosure* f) {
943 // We only scan the global used list here (for moribund threads), and
944 // the thread-local monitors in Thread::oops_do().
945 global_used_oops_do(f);
946 }
947
948 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
949 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
950 list_oops_do(g_om_in_use_list, f);
951 }
952
953 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
954 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
955 list_oops_do(thread->om_in_use_list, f);
956 }
957
958 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, OopClosure* f) {
959 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
960 ObjectMonitor* mid;
961 for (mid = list; mid != NULL; mid = mid->_next_om) {
962 if (mid->object() != NULL) {
963 f->do_oop((oop*)mid->object_addr());
964 }
965 }
966 }
967
968
969 // -----------------------------------------------------------------------------
970 // ObjectMonitor Lifecycle
971 // -----------------------
972 // Inflation unlinks monitors from the global g_free_list and
973 // associates them with objects. Deflation -- which occurs at
974 // STW-time -- disassociates idle monitors from objects. Such
975 // scavenged monitors are returned to the g_free_list.
976 //
977 // The global list is protected by gListLock. All the critical sections
978 // are short and operate in constant-time.
979 //
980 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
981 //
982 // Lifecycle:
983 // -- unassigned and on the global free list
984 // -- unassigned and on a thread's private om_free_list
985 // -- assigned to an object. The object is inflated and the mark refers
986 // to the objectmonitor.
987
988
989 // Constraining monitor pool growth via MonitorBound ...
990 //
991 // If MonitorBound is not set (<= 0), MonitorBound checks are disabled.
992 //
993 // The monitor pool is grow-only. We scavenge at STW safepoint-time, but the
994 // the rate of scavenging is driven primarily by GC. As such, we can find
995 // an inordinate number of monitors in circulation.
996 // To avoid that scenario we can artificially induce a STW safepoint
997 // if the pool appears to be growing past some reasonable bound.
998 // Generally we favor time in space-time tradeoffs, but as there's no
999 // natural back-pressure on the # of extant monitors we need to impose some
1000 // type of limit. Beware that if MonitorBound is set to too low a value
1001 // we could just loop. In addition, if MonitorBound is set to a low value
1002 // we'll incur more safepoints, which are harmful to performance.
1003 // See also: GuaranteedSafepointInterval
1004 //
1005 // The current implementation uses asynchronous VM operations.
1006 //
1007 // If MonitorBound is set, the boundry applies to
1008 // (g_om_population - g_om_free_count)
1009 // i.e., if there are not enough ObjectMonitors on the global free list,
1010 // then a safepoint deflation is induced. Picking a good MonitorBound value
1011 // is non-trivial.
1012
1013 static void InduceScavenge(Thread* self, const char * Whence) {
1014 // Induce STW safepoint to trim monitors
1015 // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
1016 // More precisely, trigger an asynchronous STW safepoint as the number
1017 // of active monitors passes the specified threshold.
1018 // TODO: assert thread state is reasonable
1019
1020 if (ForceMonitorScavenge == 0 && Atomic::xchg (1, &ForceMonitorScavenge) == 0) {
1021 // Induce a 'null' safepoint to scavenge monitors
1022 // Must VM_Operation instance be heap allocated as the op will be enqueue and posted
1023 // to the VMthread and have a lifespan longer than that of this activation record.
1024 // The VMThread will delete the op when completed.
1025 VMThread::execute(new VM_ScavengeMonitors());
1026 }
1027 }
1028
1029 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self) {
1030 // A large MAXPRIVATE value reduces both list lock contention
1031 // and list coherency traffic, but also tends to increase the
1032 // number of ObjectMonitors in circulation as well as the STW
1033 // scavenge costs. As usual, we lean toward time in space-time
1034 // tradeoffs.
1035 const int MAXPRIVATE = 1024;
1036 stringStream ss;
1037 for (;;) {
1038 ObjectMonitor* m;
1039
1040 // 1: try to allocate from the thread's local om_free_list.
1041 // Threads will attempt to allocate first from their local list, then
1042 // from the global list, and only after those attempts fail will the thread
1043 // attempt to instantiate new monitors. Thread-local free lists take
1044 // heat off the gListLock and improve allocation latency, as well as reducing
1045 // coherency traffic on the shared global list.
1046 m = self->om_free_list;
1047 if (m != NULL) {
1048 self->om_free_list = m->_next_om;
1049 self->om_free_count--;
1050 guarantee(m->object() == NULL, "invariant");
1051 m->_next_om = self->om_in_use_list;
1052 self->om_in_use_list = m;
1053 self->om_in_use_count++;
1054 return m;
1055 }
1056
1057 // 2: try to allocate from the global g_free_list
1058 // CONSIDER: use muxTry() instead of muxAcquire().
1059 // If the muxTry() fails then drop immediately into case 3.
1060 // If we're using thread-local free lists then try
1061 // to reprovision the caller's free list.
1062 if (g_free_list != NULL) {
1063 // Reprovision the thread's om_free_list.
1064 // Use bulk transfers to reduce the allocation rate and heat
1065 // on various locks.
1066 Thread::muxAcquire(&gListLock, "om_alloc(1)");
1067 for (int i = self->om_free_provision; --i >= 0 && g_free_list != NULL;) {
1068 g_om_free_count--;
1069 ObjectMonitor* take = g_free_list;
1070 g_free_list = take->_next_om;
1071 guarantee(take->object() == NULL, "invariant");
1072 take->Recycle();
1073 om_release(self, take, false);
1074 }
1075 Thread::muxRelease(&gListLock);
1076 self->om_free_provision += 1 + (self->om_free_provision/2);
1077 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1078
1079 const int mx = MonitorBound;
1080 if (mx > 0 && (g_om_population-g_om_free_count) > mx) {
1081 // Not enough ObjectMonitors on the global free list.
1082 // We can't safely induce a STW safepoint from om_alloc() as our thread
1083 // state may not be appropriate for such activities and callers may hold
1084 // naked oops, so instead we defer the action.
1085 InduceScavenge(self, "om_alloc");
1086 }
1087 continue;
1088 }
1089
1090 // 3: allocate a block of new ObjectMonitors
1091 // Both the local and global free lists are empty -- resort to malloc().
1092 // In the current implementation ObjectMonitors are TSM - immortal.
1093 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1094 // each ObjectMonitor to start at the beginning of a cache line,
1095 // so we use align_up().
1096 // A better solution would be to use C++ placement-new.
1097 // BEWARE: As it stands currently, we don't run the ctors!
1098 assert(_BLOCKSIZE > 1, "invariant");
1099 size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE;
1100 PaddedObjectMonitor* temp;
1101 size_t aligned_size = neededsize + (DEFAULT_CACHE_LINE_SIZE - 1);
1102 void* real_malloc_addr = (void*)NEW_C_HEAP_ARRAY(char, aligned_size,
1103 mtInternal);
1104 temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, DEFAULT_CACHE_LINE_SIZE);
1105
1106 // NOTE: (almost) no way to recover if allocation failed.
1107 // We might be able to induce a STW safepoint and scavenge enough
1108 // ObjectMonitors to permit progress.
1109 if (temp == NULL) {
1110 vm_exit_out_of_memory(neededsize, OOM_MALLOC_ERROR,
1111 "Allocate ObjectMonitors");
1112 }
1113 (void)memset((void *) temp, 0, neededsize);
1114
1115 // Format the block.
1116 // initialize the linked list, each monitor points to its next
1117 // forming the single linked free list, the very first monitor
1118 // will points to next block, which forms the block list.
1119 // The trick of using the 1st element in the block as g_block_list
1120 // linkage should be reconsidered. A better implementation would
1121 // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; }
1122
1123 for (int i = 1; i < _BLOCKSIZE; i++) {
1124 temp[i]._next_om = (ObjectMonitor *)&temp[i+1];
1125 }
1126
1127 // terminate the last monitor as the end of list
1128 temp[_BLOCKSIZE - 1]._next_om = NULL;
1129
1130 // Element [0] is reserved for global list linkage
1131 temp[0].set_object(CHAINMARKER);
1132
1133 // Consider carving out this thread's current request from the
1134 // block in hand. This avoids some lock traffic and redundant
1135 // list activity.
1136
1137 // Acquire the gListLock to manipulate g_block_list and g_free_list.
1138 // An Oyama-Taura-Yonezawa scheme might be more efficient.
1139 Thread::muxAcquire(&gListLock, "om_alloc(2)");
1140 g_om_population += _BLOCKSIZE-1;
1141 g_om_free_count += _BLOCKSIZE-1;
1142
1143 // Add the new block to the list of extant blocks (g_block_list).
1144 // The very first ObjectMonitor in a block is reserved and dedicated.
1145 // It serves as blocklist "next" linkage.
1146 temp[0]._next_om = g_block_list;
1147 // There are lock-free uses of g_block_list so make sure that
1148 // the previous stores happen before we update g_block_list.
1149 OrderAccess::release_store(&g_block_list, temp);
1150
1151 // Add the new string of ObjectMonitors to the global free list
1152 temp[_BLOCKSIZE - 1]._next_om = g_free_list;
1153 g_free_list = temp + 1;
1154 Thread::muxRelease(&gListLock);
1155 }
1156 }
1157
1158 // Place "m" on the caller's private per-thread om_free_list.
1159 // In practice there's no need to clamp or limit the number of
1160 // monitors on a thread's om_free_list as the only non-allocation time
1161 // we'll call om_release() is to return a monitor to the free list after
1162 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1163 // accumulate on a thread's free list.
1164 //
1165 // Key constraint: all ObjectMonitors on a thread's free list and the global
1166 // free list must have their object field set to null. This prevents the
1167 // scavenger -- deflate_monitor_list() -- from reclaiming them while we
1168 // are trying to release them.
1169
1170 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1171 bool from_per_thread_alloc) {
1172 guarantee(m->header().value() == 0, "invariant");
1173 guarantee(m->object() == NULL, "invariant");
1174 stringStream ss;
1175 guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1176 "%s, recursions=" INTPTR_FORMAT, m->is_busy_to_string(&ss),
1177 m->_recursions);
1178 // _next_om is used for both per-thread in-use and free lists so
1179 // we have to remove 'm' from the in-use list first (as needed).
1180 if (from_per_thread_alloc) {
1181 // Need to remove 'm' from om_in_use_list.
1182 ObjectMonitor* cur_mid_in_use = NULL;
1183 bool extracted = false;
1184 for (ObjectMonitor* mid = self->om_in_use_list; mid != NULL; cur_mid_in_use = mid, mid = mid->_next_om) {
1185 if (m == mid) {
1186 // extract from per-thread in-use list
1187 if (mid == self->om_in_use_list) {
1188 self->om_in_use_list = mid->_next_om;
1189 } else if (cur_mid_in_use != NULL) {
1190 cur_mid_in_use->_next_om = mid->_next_om; // maintain the current thread in-use list
1191 }
1192 extracted = true;
1193 self->om_in_use_count--;
1194 break;
1195 }
1196 }
1197 assert(extracted, "Should have extracted from in-use list");
1198 }
1199
1200 m->_next_om = self->om_free_list;
1201 self->om_free_list = m;
1202 self->om_free_count++;
1203 }
1204
1205 // Return ObjectMonitors on a moribund thread's free and in-use
1206 // lists to the appropriate global lists. The ObjectMonitors on the
1207 // per-thread in-use list may still be in use by other threads.
1208 //
1209 // We currently call om_flush() from Threads::remove() before the
1210 // thread has been excised from the thread list and is no longer a
1211 // mutator. This means that om_flush() cannot run concurrently with
1212 // a safepoint and interleave with deflate_idle_monitors(). In
1213 // particular, this ensures that the thread's in-use monitors are
1214 // scanned by a GC safepoint, either via Thread::oops_do() (before
1215 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1216 // om_flush() is called).
1217
1218 void ObjectSynchronizer::om_flush(Thread* self) {
1219 ObjectMonitor* free_list = self->om_free_list;
1220 ObjectMonitor* free_tail = NULL;
1221 int free_count = 0;
1222 if (free_list != NULL) {
1223 ObjectMonitor* s;
1224 // The thread is going away. Set 'free_tail' to the last per-thread free
1225 // monitor which will be linked to g_free_list below under the gListLock.
1226 stringStream ss;
1227 for (s = free_list; s != NULL; s = s->_next_om) {
1228 free_count++;
1229 free_tail = s;
1230 guarantee(s->object() == NULL, "invariant");
1231 guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss));
1232 }
1233 guarantee(free_tail != NULL, "invariant");
1234 assert(self->om_free_count == free_count, "free-count off");
1235 self->om_free_list = NULL;
1236 self->om_free_count = 0;
1237 }
1238
1239 ObjectMonitor* in_use_list = self->om_in_use_list;
1240 ObjectMonitor* in_use_tail = NULL;
1241 int in_use_count = 0;
1242 if (in_use_list != NULL) {
1243 // The thread is going away, however the ObjectMonitors on the
1244 // om_in_use_list may still be in-use by other threads. Link
1245 // them to in_use_tail, which will be linked into the global
1246 // in-use list g_om_in_use_list below, under the gListLock.
1247 ObjectMonitor *cur_om;
1248 for (cur_om = in_use_list; cur_om != NULL; cur_om = cur_om->_next_om) {
1249 in_use_tail = cur_om;
1250 in_use_count++;
1251 }
1252 guarantee(in_use_tail != NULL, "invariant");
1253 assert(self->om_in_use_count == in_use_count, "in-use count off");
1254 self->om_in_use_list = NULL;
1255 self->om_in_use_count = 0;
1256 }
1257
1258 Thread::muxAcquire(&gListLock, "om_flush");
1259 if (free_tail != NULL) {
1260 free_tail->_next_om = g_free_list;
1261 g_free_list = free_list;
1262 g_om_free_count += free_count;
1263 }
1264
1265 if (in_use_tail != NULL) {
1266 in_use_tail->_next_om = g_om_in_use_list;
1267 g_om_in_use_list = in_use_list;
1268 g_om_in_use_count += in_use_count;
1269 }
1270
1271 Thread::muxRelease(&gListLock);
1272
1273 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1274 LogStreamHandle(Info, monitorinflation) lsh_info;
1275 LogStream* ls = NULL;
1276 if (log_is_enabled(Debug, monitorinflation)) {
1277 ls = &lsh_debug;
1278 } else if ((free_count != 0 || in_use_count != 0) &&
1279 log_is_enabled(Info, monitorinflation)) {
1280 ls = &lsh_info;
1281 }
1282 if (ls != NULL) {
1283 ls->print_cr("om_flush: jt=" INTPTR_FORMAT ", free_count=%d"
1284 ", in_use_count=%d" ", om_free_provision=%d",
1285 p2i(self), free_count, in_use_count, self->om_free_provision);
1286 }
1287 }
1288
1289 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1290 const oop obj,
1291 ObjectSynchronizer::InflateCause cause) {
1292 assert(event != NULL, "invariant");
1293 assert(event->should_commit(), "invariant");
1294 event->set_monitorClass(obj->klass());
1295 event->set_address((uintptr_t)(void*)obj);
1296 event->set_cause((u1)cause);
1297 event->commit();
1298 }
1299
1300 // Fast path code shared by multiple functions
1301 void ObjectSynchronizer::inflate_helper(oop obj) {
1302 markWord mark = obj->mark();
1303 if (mark.has_monitor()) {
1304 assert(ObjectSynchronizer::verify_objmon_isinpool(mark.monitor()), "monitor is invalid");
1305 assert(mark.monitor()->header().is_neutral(), "monitor must record a good object header");
1306 return;
1307 }
1308 inflate(Thread::current(), obj, inflate_cause_vm_internal);
1309 }
1310
1311 ObjectMonitor* ObjectSynchronizer::inflate(Thread* self,
1312 oop object,
1313 const InflateCause cause) {
1314 // Inflate mutates the heap ...
1315 // Relaxing assertion for bug 6320749.
1316 assert(Universe::verify_in_progress() ||
1317 !SafepointSynchronize::is_at_safepoint(), "invariant");
1318
1319 EventJavaMonitorInflate event;
1320
1321 for (;;) {
1322 const markWord mark = object->mark();
1323 assert(!mark.has_bias_pattern(), "invariant");
1324
1325 // The mark can be in one of the following states:
1326 // * Inflated - just return
1327 // * Stack-locked - coerce it to inflated
1328 // * INFLATING - busy wait for conversion to complete
1329 // * Neutral - aggressively inflate the object.
1330 // * BIASED - Illegal. We should never see this
1331
1332 // CASE: inflated
1333 if (mark.has_monitor()) {
1334 ObjectMonitor* inf = mark.monitor();
1335 markWord dmw = inf->header();
1336 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1337 assert(oopDesc::equals((oop) inf->object(), object), "invariant");
1338 assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
1339 return inf;
1340 }
1341
1342 // CASE: inflation in progress - inflating over a stack-lock.
1343 // Some other thread is converting from stack-locked to inflated.
1344 // Only that thread can complete inflation -- other threads must wait.
1345 // The INFLATING value is transient.
1346 // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1347 // We could always eliminate polling by parking the thread on some auxiliary list.
1348 if (mark == markWord::INFLATING()) {
1349 read_stable_mark(object);
1350 continue;
1351 }
1352
1353 // CASE: stack-locked
1354 // Could be stack-locked either by this thread or by some other thread.
1355 //
1356 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1357 // to install INFLATING into the mark word. We originally installed INFLATING,
1358 // allocated the objectmonitor, and then finally STed the address of the
1359 // objectmonitor into the mark. This was correct, but artificially lengthened
1360 // the interval in which INFLATED appeared in the mark, thus increasing
1361 // the odds of inflation contention.
1362 //
1363 // We now use per-thread private objectmonitor free lists.
1364 // These list are reprovisioned from the global free list outside the
1365 // critical INFLATING...ST interval. A thread can transfer
1366 // multiple objectmonitors en-mass from the global free list to its local free list.
1367 // This reduces coherency traffic and lock contention on the global free list.
1368 // Using such local free lists, it doesn't matter if the om_alloc() call appears
1369 // before or after the CAS(INFLATING) operation.
1370 // See the comments in om_alloc().
1371
1372 LogStreamHandle(Trace, monitorinflation) lsh;
1373
1374 if (mark.has_locker()) {
1375 ObjectMonitor* m = om_alloc(self);
1376 // Optimistically prepare the objectmonitor - anticipate successful CAS
1377 // We do this before the CAS in order to minimize the length of time
1378 // in which INFLATING appears in the mark.
1379 m->Recycle();
1380 m->_Responsible = NULL;
1381 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // Consider: maintain by type/class
1382
1383 markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1384 if (cmp != mark) {
1385 om_release(self, m, true);
1386 continue; // Interference -- just retry
1387 }
1388
1389 // We've successfully installed INFLATING (0) into the mark-word.
1390 // This is the only case where 0 will appear in a mark-word.
1391 // Only the singular thread that successfully swings the mark-word
1392 // to 0 can perform (or more precisely, complete) inflation.
1393 //
1394 // Why do we CAS a 0 into the mark-word instead of just CASing the
1395 // mark-word from the stack-locked value directly to the new inflated state?
1402 // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1403 // the while preserving the hashCode stability invariants. If the owner
1404 // decides to release the lock while the value is 0, the unlock will fail
1405 // and control will eventually pass from slow_exit() to inflate. The owner
1406 // will then spin, waiting for the 0 value to disappear. Put another way,
1407 // the 0 causes the owner to stall if the owner happens to try to
1408 // drop the lock (restoring the header from the BasicLock to the object)
1409 // while inflation is in-progress. This protocol avoids races that might
1410 // would otherwise permit hashCode values to change or "flicker" for an object.
1411 // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1412 // 0 serves as a "BUSY" inflate-in-progress indicator.
1413
1414
1415 // fetch the displaced mark from the owner's stack.
1416 // The owner can't die or unwind past the lock while our INFLATING
1417 // object is in the mark. Furthermore the owner can't complete
1418 // an unlock on the object, either.
1419 markWord dmw = mark.displaced_mark_helper();
1420 // Catch if the object's header is not neutral (not locked and
1421 // not marked is what we care about here).
1422 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1423
1424 // Setup monitor fields to proper values -- prepare the monitor
1425 m->set_header(dmw);
1426
1427 // Optimization: if the mark.locker stack address is associated
1428 // with this thread we could simply set m->_owner = self.
1429 // Note that a thread can inflate an object
1430 // that it has stack-locked -- as might happen in wait() -- directly
1431 // with CAS. That is, we can avoid the xchg-NULL .... ST idiom.
1432 m->set_owner(mark.locker());
1433 m->set_object(object);
1434 // TODO-FIXME: assert BasicLock->dhw != 0.
1435
1436 // Must preserve store ordering. The monitor state must
1437 // be stable at the time of publishing the monitor address.
1438 guarantee(object->mark() == markWord::INFLATING(), "invariant");
1439 object->release_set_mark(markWord::encode(m));
1440
1441 // Hopefully the performance counters are allocated on distinct cache lines
1442 // to avoid false sharing on MP systems ...
1443 OM_PERFDATA_OP(Inflations, inc());
1444 if (log_is_enabled(Trace, monitorinflation)) {
1445 ResourceMark rm(self);
1446 lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
1447 INTPTR_FORMAT ", type='%s'", p2i(object),
1448 object->mark().value(), object->klass()->external_name());
1449 }
1450 if (event.should_commit()) {
1451 post_monitor_inflate_event(&event, object, cause);
1452 }
1453 return m;
1454 }
1455
1456 // CASE: neutral
1457 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
1458 // If we know we're inflating for entry it's better to inflate by swinging a
1459 // pre-locked ObjectMonitor pointer into the object header. A successful
1460 // CAS inflates the object *and* confers ownership to the inflating thread.
1461 // In the current implementation we use a 2-step mechanism where we CAS()
1462 // to inflate and then CAS() again to try to swing _owner from NULL to self.
1463 // An inflateTry() method that we could call from fast_enter() and slow_enter()
1464 // would be useful.
1465
1466 // Catch if the object's header is not neutral (not locked and
1467 // not marked is what we care about here).
1468 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1469 ObjectMonitor* m = om_alloc(self);
1470 // prepare m for installation - set monitor to initial state
1471 m->Recycle();
1472 m->set_header(mark);
1473 m->set_object(object);
1474 m->_Responsible = NULL;
1475 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
1476
1477 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
1478 m->set_header(markWord::zero());
1479 m->set_object(NULL);
1480 m->Recycle();
1481 om_release(self, m, true);
1482 m = NULL;
1483 continue;
1484 // interference - the markword changed - just retry.
1485 // The state-transitions are one-way, so there's no chance of
1486 // live-lock -- "Inflated" is an absorbing state.
1487 }
1488
1489 // Hopefully the performance counters are allocated on distinct
1490 // cache lines to avoid false sharing on MP systems ...
1491 OM_PERFDATA_OP(Inflations, inc());
1492 if (log_is_enabled(Trace, monitorinflation)) {
1493 ResourceMark rm(self);
1494 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
1495 INTPTR_FORMAT ", type='%s'", p2i(object),
1496 object->mark().value(), object->klass()->external_name());
1497 }
1498 if (event.should_commit()) {
1499 post_monitor_inflate_event(&event, object, cause);
1500 }
1501 return m;
1502 }
1503 }
1504
1505
1506 // We maintain a list of in-use monitors for each thread.
1507 //
1508 // deflate_thread_local_monitors() scans a single thread's in-use list, while
1509 // deflate_idle_monitors() scans only a global list of in-use monitors which
1510 // is populated only as a thread dies (see om_flush()).
1511 //
1512 // These operations are called at all safepoints, immediately after mutators
1513 // are stopped, but before any objects have moved. Collectively they traverse
1514 // the population of in-use monitors, deflating where possible. The scavenged
1515 // monitors are returned to the global monitor free list.
1516 //
1517 // Beware that we scavenge at *every* stop-the-world point. Having a large
1518 // number of monitors in-use could negatively impact performance. We also want
1519 // to minimize the total # of monitors in circulation, as they incur a small
1520 // footprint penalty.
1521 //
1522 // Perversely, the heap size -- and thus the STW safepoint rate --
1523 // typically drives the scavenge rate. Large heaps can mean infrequent GC,
1524 // which in turn can mean large(r) numbers of ObjectMonitors in circulation.
1525 // This is an unfortunate aspect of this design.
1526
1527 // Deflate a single monitor if not in-use
1528 // Return true if deflated, false if in-use
1529 bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj,
1530 ObjectMonitor** free_head_p,
1531 ObjectMonitor** free_tail_p) {
1532 bool deflated;
1533 // Normal case ... The monitor is associated with obj.
1534 const markWord mark = obj->mark();
1535 guarantee(mark == markWord::encode(mid), "should match: mark="
1536 INTPTR_FORMAT ", encoded mid=" INTPTR_FORMAT, mark.value(),
1537 markWord::encode(mid).value());
1538 // Make sure that mark.monitor() and markWord::encode() agree:
1539 guarantee(mark.monitor() == mid, "should match: monitor()=" INTPTR_FORMAT
1540 ", mid=" INTPTR_FORMAT, p2i(mark.monitor()), p2i(mid));
1541 const markWord dmw = mid->header();
1542 guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1543
1544 if (mid->is_busy()) {
1545 deflated = false;
1546 } else {
1547 // Deflate the monitor if it is no longer being used
1548 // It's idle - scavenge and return to the global free list
1549 // plain old deflation ...
1550 if (log_is_enabled(Trace, monitorinflation)) {
1551 ResourceMark rm;
1552 log_trace(monitorinflation)("deflate_monitor: "
1553 "object=" INTPTR_FORMAT ", mark="
1554 INTPTR_FORMAT ", type='%s'", p2i(obj),
1555 mark.value(), obj->klass()->external_name());
1556 }
1557
1558 // Restore the header back to obj
1559 obj->release_set_mark(dmw);
1560 mid->clear();
1561
1562 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
1563 p2i(mid->object()));
1564
1565 // Move the deflated ObjectMonitor to the working free list
1566 // defined by free_head_p and free_tail_p.
1567 if (*free_head_p == NULL) *free_head_p = mid;
1568 if (*free_tail_p != NULL) {
1569 // We append to the list so the caller can use mid->_next_om
1570 // to fix the linkages in its context.
1571 ObjectMonitor* prevtail = *free_tail_p;
1572 // Should have been cleaned up by the caller:
1573 assert(prevtail->_next_om == NULL, "cleaned up deflated?");
1574 prevtail->_next_om = mid;
1575 }
1576 *free_tail_p = mid;
1577 // At this point, mid->_next_om still refers to its current
1578 // value and another ObjectMonitor's _next_om field still
1579 // refers to this ObjectMonitor. Those linkages have to be
1580 // cleaned up by the caller who has the complete context.
1581 deflated = true;
1582 }
1583 return deflated;
1584 }
1585
1586 // Walk a given monitor list, and deflate idle monitors
1587 // The given list could be a per-thread list or a global list
1588 // Caller acquires gListLock as needed.
1589 //
1590 // In the case of parallel processing of thread local monitor lists,
1591 // work is done by Threads::parallel_threads_do() which ensures that
1592 // each Java thread is processed by exactly one worker thread, and
1593 // thus avoid conflicts that would arise when worker threads would
1594 // process the same monitor lists concurrently.
1595 //
1596 // See also ParallelSPCleanupTask and
1597 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
1598 // Threads::parallel_java_threads_do() in thread.cpp.
1599 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor** list_p,
1600 ObjectMonitor** free_head_p,
1601 ObjectMonitor** free_tail_p) {
1602 ObjectMonitor* mid;
1603 ObjectMonitor* next;
1604 ObjectMonitor* cur_mid_in_use = NULL;
1605 int deflated_count = 0;
1606
1607 for (mid = *list_p; mid != NULL;) {
1608 oop obj = (oop) mid->object();
1609 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
1610 // Deflation succeeded and already updated free_head_p and
1611 // free_tail_p as needed. Finish the move to the local free list
1612 // by unlinking mid from the global or per-thread in-use list.
1613 if (mid == *list_p) {
1614 *list_p = mid->_next_om;
1615 } else if (cur_mid_in_use != NULL) {
1616 cur_mid_in_use->_next_om = mid->_next_om; // maintain the current thread in-use list
1617 }
1618 next = mid->_next_om;
1619 mid->_next_om = NULL; // This mid is current tail in the free_head_p list
1620 mid = next;
1621 deflated_count++;
1622 } else {
1623 cur_mid_in_use = mid;
1624 mid = mid->_next_om;
1625 }
1626 }
1627 return deflated_count;
1628 }
1629
1630 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
1631 counters->n_in_use = 0; // currently associated with objects
1632 counters->n_in_circulation = 0; // extant
1633 counters->n_scavenged = 0; // reclaimed (global and per-thread)
1634 counters->per_thread_scavenged = 0; // per-thread scavenge total
1635 counters->per_thread_times = 0.0; // per-thread scavenge times
1636 }
1637
1638 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
1639 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1640 bool deflated = false;
1641
1642 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
1643 ObjectMonitor* free_tail_p = NULL;
1644 elapsedTimer timer;
1645
1646 if (log_is_enabled(Info, monitorinflation)) {
1647 timer.start();
1648 }
1649
1650 // Prevent om_flush from changing mids in Thread dtor's during deflation
1651 // And in case the vm thread is acquiring a lock during a safepoint
1652 // See e.g. 6320749
1653 Thread::muxAcquire(&gListLock, "deflate_idle_monitors");
1654
1655 // Note: the thread-local monitors lists get deflated in
1656 // a separate pass. See deflate_thread_local_monitors().
1657
1658 // For moribund threads, scan g_om_in_use_list
1659 int deflated_count = 0;
1660 if (g_om_in_use_list) {
1661 counters->n_in_circulation += g_om_in_use_count;
1662 deflated_count = deflate_monitor_list((ObjectMonitor **)&g_om_in_use_list, &free_head_p, &free_tail_p);
1663 g_om_in_use_count -= deflated_count;
1664 counters->n_scavenged += deflated_count;
1665 counters->n_in_use += g_om_in_use_count;
1666 }
1667
1668 if (free_head_p != NULL) {
1669 // Move the deflated ObjectMonitors back to the global free list.
1670 guarantee(free_tail_p != NULL && counters->n_scavenged > 0, "invariant");
1671 assert(free_tail_p->_next_om == NULL, "invariant");
1672 // constant-time list splice - prepend scavenged segment to g_free_list
1673 free_tail_p->_next_om = g_free_list;
1674 g_free_list = free_head_p;
1675 }
1676 Thread::muxRelease(&gListLock);
1677 timer.stop();
1678
1679 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1680 LogStreamHandle(Info, monitorinflation) lsh_info;
1681 LogStream* ls = NULL;
1682 if (log_is_enabled(Debug, monitorinflation)) {
1683 ls = &lsh_debug;
1684 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
1685 ls = &lsh_info;
1686 }
1687 if (ls != NULL) {
1688 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
1689 }
1690 }
1691
1692 void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) {
1693 // Report the cumulative time for deflating each thread's idle
1694 // monitors. Note: if the work is split among more than one
1695 // worker thread, then the reported time will likely be more
1696 // than a beginning to end measurement of the phase.
1697 log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d", counters->per_thread_times, counters->per_thread_scavenged);
1698
1699 g_om_free_count += counters->n_scavenged;
1700
1701 if (log_is_enabled(Debug, monitorinflation)) {
1702 // exit_globals()'s call to audit_and_print_stats() is done
1703 // at the Info level.
1704 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
1705 } else if (log_is_enabled(Info, monitorinflation)) {
1706 Thread::muxAcquire(&gListLock, "finish_deflate_idle_monitors");
1707 log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, "
1708 "g_om_free_count=%d", g_om_population,
1709 g_om_in_use_count, g_om_free_count);
1710 Thread::muxRelease(&gListLock);
1711 }
1712
1713 ForceMonitorScavenge = 0; // Reset
1714
1715 OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged));
1716 OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation));
1717
1718 GVars.stw_random = os::random();
1719 GVars.stw_cycle++;
1720 }
1721
1722 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
1723 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1724
1725 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
1726 ObjectMonitor* free_tail_p = NULL;
1727 elapsedTimer timer;
1728
1729 if (log_is_enabled(Info, safepoint, cleanup) ||
1730 log_is_enabled(Info, monitorinflation)) {
1731 timer.start();
1732 }
1733
1734 int deflated_count = deflate_monitor_list(thread->om_in_use_list_addr(), &free_head_p, &free_tail_p);
1735
1736 Thread::muxAcquire(&gListLock, "deflate_thread_local_monitors");
1737
1738 // Adjust counters
1739 counters->n_in_circulation += thread->om_in_use_count;
1740 thread->om_in_use_count -= deflated_count;
1741 counters->n_scavenged += deflated_count;
1742 counters->n_in_use += thread->om_in_use_count;
1743 counters->per_thread_scavenged += deflated_count;
1744
1745 if (free_head_p != NULL) {
1746 // Move the deflated ObjectMonitors back to the global free list.
1747 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
1748 assert(free_tail_p->_next_om == NULL, "invariant");
1749
1750 // constant-time list splice - prepend scavenged segment to g_free_list
1751 free_tail_p->_next_om = g_free_list;
1752 g_free_list = free_head_p;
1753 }
1754
1755 timer.stop();
1756 // Safepoint logging cares about cumulative per_thread_times and
1757 // we'll capture most of the cost, but not the muxRelease() which
1758 // should be cheap.
1759 counters->per_thread_times += timer.seconds();
1760
1761 Thread::muxRelease(&gListLock);
1762
1763 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1764 LogStreamHandle(Info, monitorinflation) lsh_info;
1765 LogStream* ls = NULL;
1766 if (log_is_enabled(Debug, monitorinflation)) {
1767 ls = &lsh_debug;
1768 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
1769 ls = &lsh_info;
1770 }
1771 if (ls != NULL) {
1772 ls->print_cr("jt=" INTPTR_FORMAT ": deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(thread), timer.seconds(), deflated_count);
1773 }
1774 }
1775
1776 // Monitor cleanup on JavaThread::exit
1777
1778 // Iterate through monitor cache and attempt to release thread's monitors
1779 // Gives up on a particular monitor if an exception occurs, but continues
1780 // the overall iteration, swallowing the exception.
1781 class ReleaseJavaMonitorsClosure: public MonitorClosure {
1782 private:
1793
1794 // Release all inflated monitors owned by THREAD. Lightweight monitors are
1795 // ignored. This is meant to be called during JNI thread detach which assumes
1796 // all remaining monitors are heavyweight. All exceptions are swallowed.
1797 // Scanning the extant monitor list can be time consuming.
1798 // A simple optimization is to add a per-thread flag that indicates a thread
1799 // called jni_monitorenter() during its lifetime.
1800 //
1801 // Instead of No_Savepoint_Verifier it might be cheaper to
1802 // use an idiom of the form:
1803 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
1804 // <code that must not run at safepoint>
1805 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
1806 // Since the tests are extremely cheap we could leave them enabled
1807 // for normal product builds.
1808
1809 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
1810 assert(THREAD == JavaThread::current(), "must be current Java thread");
1811 NoSafepointVerifier nsv;
1812 ReleaseJavaMonitorsClosure rjmc(THREAD);
1813 Thread::muxAcquire(&gListLock, "release_monitors_owned_by_thread");
1814 ObjectSynchronizer::monitors_iterate(&rjmc);
1815 Thread::muxRelease(&gListLock);
1816 THREAD->clear_pending_exception();
1817 }
1818
1819 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
1820 switch (cause) {
1821 case inflate_cause_vm_internal: return "VM Internal";
1822 case inflate_cause_monitor_enter: return "Monitor Enter";
1823 case inflate_cause_wait: return "Monitor Wait";
1824 case inflate_cause_notify: return "Monitor Notify";
1825 case inflate_cause_hash_code: return "Monitor Hash Code";
1826 case inflate_cause_jni_enter: return "JNI Monitor Enter";
1827 case inflate_cause_jni_exit: return "JNI Monitor Exit";
1828 default:
1829 ShouldNotReachHere();
1830 }
1831 return "Unknown";
1832 }
1833
1834 //------------------------------------------------------------------------------
1835 // Debugging code
1849 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
1850 return (u_char*)&GVars.stw_random;
1851 }
1852
1853 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
1854 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
1855
1856 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1857 LogStreamHandle(Info, monitorinflation) lsh_info;
1858 LogStreamHandle(Trace, monitorinflation) lsh_trace;
1859 LogStream* ls = NULL;
1860 if (log_is_enabled(Trace, monitorinflation)) {
1861 ls = &lsh_trace;
1862 } else if (log_is_enabled(Debug, monitorinflation)) {
1863 ls = &lsh_debug;
1864 } else if (log_is_enabled(Info, monitorinflation)) {
1865 ls = &lsh_info;
1866 }
1867 assert(ls != NULL, "sanity check");
1868
1869 if (!on_exit) {
1870 // Not at VM exit so grab the global list lock.
1871 Thread::muxAcquire(&gListLock, "audit_and_print_stats");
1872 }
1873
1874 // Log counts for the global and per-thread monitor lists:
1875 int chk_om_population = log_monitor_list_counts(ls);
1876 int error_cnt = 0;
1877
1878 ls->print_cr("Checking global lists:");
1879
1880 // Check g_om_population:
1881 if (g_om_population == chk_om_population) {
1882 ls->print_cr("g_om_population=%d equals chk_om_population=%d",
1883 g_om_population, chk_om_population);
1884 } else {
1885 ls->print_cr("ERROR: g_om_population=%d is not equal to "
1886 "chk_om_population=%d", g_om_population,
1887 chk_om_population);
1888 error_cnt++;
1889 }
1890
1891 // Check g_om_in_use_list and g_om_in_use_count:
1892 chk_global_in_use_list_and_count(ls, &error_cnt);
1893
1894 // Check g_free_list and g_om_free_count:
1895 chk_global_free_list_and_count(ls, &error_cnt);
1896
1897 if (!on_exit) {
1898 Thread::muxRelease(&gListLock);
1899 }
1900
1901 ls->print_cr("Checking per-thread lists:");
1902
1903 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
1904 // Check om_in_use_list and om_in_use_count:
1905 chk_per_thread_in_use_list_and_count(jt, ls, &error_cnt);
1906
1907 // Check om_free_list and om_free_count:
1908 chk_per_thread_free_list_and_count(jt, ls, &error_cnt);
1909 }
1910
1911 if (error_cnt == 0) {
1912 ls->print_cr("No errors found in monitor list checks.");
1913 } else {
1914 log_error(monitorinflation)("found monitor list errors: error_cnt=%d", error_cnt);
1915 }
1916
1917 if ((on_exit && log_is_enabled(Info, monitorinflation)) ||
1918 (!on_exit && log_is_enabled(Trace, monitorinflation))) {
1919 // When exiting this log output is at the Info level. When called
1920 // at a safepoint, this log output is at the Trace level since
1921 // there can be a lot of it.
1922 log_in_use_monitor_details(ls, on_exit);
1923 }
1924
1925 ls->flush();
1926
1927 guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
1928 }
1929
1930 // Check a free monitor entry; log any errors.
1931 void ObjectSynchronizer::chk_free_entry(JavaThread* jt, ObjectMonitor* n,
1932 outputStream * out, int *error_cnt_p) {
1933 stringStream ss;
1934 if (n->is_busy()) {
1935 if (jt != NULL) {
1936 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
1937 ": free per-thread monitor must not be busy: %s", p2i(jt),
1938 p2i(n), n->is_busy_to_string(&ss));
1939 } else {
1940 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
1941 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
1942 }
1943 *error_cnt_p = *error_cnt_p + 1;
1944 }
1945 if (n->header().value() != 0) {
1946 if (jt != NULL) {
1947 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
1948 ": free per-thread monitor must have NULL _header "
1949 "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n),
1950 n->header().value());
1951 } else {
1952 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
1953 "must have NULL _header field: _header=" INTPTR_FORMAT,
1954 p2i(n), n->header().value());
1955 }
1956 *error_cnt_p = *error_cnt_p + 1;
1957 }
1958 if (n->object() != NULL) {
1959 if (jt != NULL) {
1960 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
1961 ": free per-thread monitor must have NULL _object "
1962 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
1963 p2i(n->object()));
1964 } else {
1965 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
1966 "must have NULL _object field: _object=" INTPTR_FORMAT,
1967 p2i(n), p2i(n->object()));
1968 }
1969 *error_cnt_p = *error_cnt_p + 1;
1970 }
1971 }
1972
1973 // Check the global free list and count; log the results of the checks.
1974 void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out,
1975 int *error_cnt_p) {
1976 int chk_om_free_count = 0;
1977 for (ObjectMonitor* n = g_free_list; n != NULL; n = n->_next_om) {
1978 chk_free_entry(NULL /* jt */, n, out, error_cnt_p);
1979 chk_om_free_count++;
1980 }
1981 if (g_om_free_count == chk_om_free_count) {
1982 out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d",
1983 g_om_free_count, chk_om_free_count);
1984 } else {
1985 out->print_cr("ERROR: g_om_free_count=%d is not equal to "
1986 "chk_om_free_count=%d", g_om_free_count,
1987 chk_om_free_count);
1988 *error_cnt_p = *error_cnt_p + 1;
1989 }
1990 }
1991
1992 // Check the global in-use list and count; log the results of the checks.
1993 void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out,
1994 int *error_cnt_p) {
1995 int chk_om_in_use_count = 0;
1996 for (ObjectMonitor* n = g_om_in_use_list; n != NULL; n = n->_next_om) {
1997 chk_in_use_entry(NULL /* jt */, n, out, error_cnt_p);
1998 chk_om_in_use_count++;
1999 }
2000 if (g_om_in_use_count == chk_om_in_use_count) {
2001 out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d", g_om_in_use_count,
2002 chk_om_in_use_count);
2003 } else {
2004 out->print_cr("ERROR: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d",
2005 g_om_in_use_count, chk_om_in_use_count);
2006 *error_cnt_p = *error_cnt_p + 1;
2007 }
2008 }
2009
2010 // Check an in-use monitor entry; log any errors.
2011 void ObjectSynchronizer::chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,
2012 outputStream * out, int *error_cnt_p) {
2013 if (n->header().value() == 0) {
2014 if (jt != NULL) {
2015 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2016 ": in-use per-thread monitor must have non-NULL _header "
2017 "field.", p2i(jt), p2i(n));
2018 } else {
2019 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
2020 "must have non-NULL _header field.", p2i(n));
2021 }
2022 *error_cnt_p = *error_cnt_p + 1;
2023 }
2024 if (n->object() == NULL) {
2025 if (jt != NULL) {
2054 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
2055 ": in-use per-thread monitor's object does not refer "
2056 "to the same monitor: obj=" INTPTR_FORMAT ", mark="
2057 INTPTR_FORMAT ", obj_mon=" INTPTR_FORMAT, p2i(jt),
2058 p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
2059 } else {
2060 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
2061 "monitor's object does not refer to the same monitor: obj="
2062 INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
2063 INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
2064 }
2065 *error_cnt_p = *error_cnt_p + 1;
2066 }
2067 }
2068
2069 // Check the thread's free list and count; log the results of the checks.
2070 void ObjectSynchronizer::chk_per_thread_free_list_and_count(JavaThread *jt,
2071 outputStream * out,
2072 int *error_cnt_p) {
2073 int chk_om_free_count = 0;
2074 for (ObjectMonitor* n = jt->om_free_list; n != NULL; n = n->_next_om) {
2075 chk_free_entry(jt, n, out, error_cnt_p);
2076 chk_om_free_count++;
2077 }
2078 if (jt->om_free_count == chk_om_free_count) {
2079 out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals "
2080 "chk_om_free_count=%d", p2i(jt), jt->om_free_count, chk_om_free_count);
2081 } else {
2082 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not "
2083 "equal to chk_om_free_count=%d", p2i(jt), jt->om_free_count,
2084 chk_om_free_count);
2085 *error_cnt_p = *error_cnt_p + 1;
2086 }
2087 }
2088
2089 // Check the thread's in-use list and count; log the results of the checks.
2090 void ObjectSynchronizer::chk_per_thread_in_use_list_and_count(JavaThread *jt,
2091 outputStream * out,
2092 int *error_cnt_p) {
2093 int chk_om_in_use_count = 0;
2094 for (ObjectMonitor* n = jt->om_in_use_list; n != NULL; n = n->_next_om) {
2095 chk_in_use_entry(jt, n, out, error_cnt_p);
2096 chk_om_in_use_count++;
2097 }
2098 if (jt->om_in_use_count == chk_om_in_use_count) {
2099 out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals "
2100 "chk_om_in_use_count=%d", p2i(jt), jt->om_in_use_count,
2101 chk_om_in_use_count);
2102 } else {
2103 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not "
2104 "equal to chk_om_in_use_count=%d", p2i(jt), jt->om_in_use_count,
2105 chk_om_in_use_count);
2106 *error_cnt_p = *error_cnt_p + 1;
2107 }
2108 }
2109
2110 // Log details about ObjectMonitors on the in-use lists. The 'BHL'
2111 // flags indicate why the entry is in-use, 'object' and 'object type'
2112 // indicate the associated object and its type.
2113 void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out,
2114 bool on_exit) {
2115 if (!on_exit) {
2116 // Not at VM exit so grab the global list lock.
2117 Thread::muxAcquire(&gListLock, "log_in_use_monitor_details");
2118 }
2119
2120 stringStream ss;
2121 if (g_om_in_use_count > 0) {
2122 out->print_cr("In-use global monitor info:");
2123 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
2124 out->print_cr("%18s %s %18s %18s",
2125 "monitor", "BHL", "object", "object type");
2126 out->print_cr("================== === ================== ==================");
2127 for (ObjectMonitor* n = g_om_in_use_list; n != NULL; n = n->_next_om) {
2128 const oop obj = (oop) n->object();
2129 const markWord mark = n->header();
2130 ResourceMark rm;
2131 out->print(INTPTR_FORMAT " %d%d%d " INTPTR_FORMAT " %s", p2i(n),
2132 n->is_busy() != 0, mark.hash() != 0, n->owner() != NULL,
2133 p2i(obj), obj->klass()->external_name());
2134 if (n->is_busy() != 0) {
2135 out->print(" (%s)", n->is_busy_to_string(&ss));
2136 ss.reset();
2137 }
2138 out->cr();
2139 }
2140 }
2141
2142 if (!on_exit) {
2143 Thread::muxRelease(&gListLock);
2144 }
2145
2146 out->print_cr("In-use per-thread monitor info:");
2147 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
2148 out->print_cr("%18s %18s %s %18s %18s",
2149 "jt", "monitor", "BHL", "object", "object type");
2150 out->print_cr("================== ================== === ================== ==================");
2151 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2152 for (ObjectMonitor* n = jt->om_in_use_list; n != NULL; n = n->_next_om) {
2153 const oop obj = (oop) n->object();
2154 const markWord mark = n->header();
2155 ResourceMark rm;
2156 out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d " INTPTR_FORMAT
2157 " %s", p2i(jt), p2i(n), n->is_busy() != 0,
2158 mark.hash() != 0, n->owner() != NULL, p2i(obj),
2159 obj->klass()->external_name());
2160 if (n->is_busy() != 0) {
2161 out->print(" (%s)", n->is_busy_to_string(&ss));
2162 ss.reset();
2163 }
2164 out->cr();
2165 }
2166 }
2167
2168 out->flush();
2169 }
2170
2171 // Log counts for the global and per-thread monitor lists and return
2172 // the population count.
2173 int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) {
2174 int pop_count = 0;
2175 out->print_cr("%18s %10s %10s %10s",
2176 "Global Lists:", "InUse", "Free", "Total");
2177 out->print_cr("================== ========== ========== ==========");
2178 out->print_cr("%18s %10d %10d %10d", "",
2179 g_om_in_use_count, g_om_free_count, g_om_population);
2180 pop_count += g_om_in_use_count + g_om_free_count;
2181
2182 out->print_cr("%18s %10s %10s %10s",
2183 "Per-Thread Lists:", "InUse", "Free", "Provision");
2184 out->print_cr("================== ========== ========== ==========");
2185
2186 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2187 out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt),
2188 jt->om_in_use_count, jt->om_free_count, jt->om_free_provision);
2189 pop_count += jt->om_in_use_count + jt->om_free_count;
2190 }
2191 return pop_count;
2192 }
2193
2194 #ifndef PRODUCT
2195
2196 // Check if monitor belongs to the monitor cache
2197 // The list is grow-only so it's *relatively* safe to traverse
2198 // the list of extant blocks without taking a lock.
2199
2200 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
2201 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
2202 while (block != NULL) {
2203 assert(block->object() == CHAINMARKER, "must be a block header");
2204 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
2205 address mon = (address)monitor;
2206 address blk = (address)block;
2207 size_t diff = mon - blk;
2208 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
2209 return 1;
2210 }
2211 block = (PaddedObjectMonitor*)block->_next_om;
2212 }
2213 return 0;
2214 }
2215
2216 #endif
|
101 }
102
103 #else // ndef DTRACE_ENABLED
104
105 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon) {;}
106 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon) {;}
107
108 #endif // ndef DTRACE_ENABLED
109
110 // This exists only as a workaround of dtrace bug 6254741
111 int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
112 DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
113 return 0;
114 }
115
116 #define NINFLATIONLOCKS 256
117 static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];
118
119 // global list of blocks of monitors
120 PaddedObjectMonitor* volatile ObjectSynchronizer::g_block_list = NULL;
121 bool volatile ObjectSynchronizer::_is_async_deflation_requested = false;
122 bool volatile ObjectSynchronizer::_is_special_deflation_requested = false;
123 jlong ObjectSynchronizer::_last_async_deflation_time_ns = 0;
124
125 // Global ObjectMonitor free list. Newly allocated and deflated
126 // ObjectMonitors are prepended here.
127 static ObjectMonitor* volatile g_free_list = NULL;
128 // Global ObjectMonitor in-use list. When a JavaThread is exiting,
129 // ObjectMonitors on its per-thread in-use list are prepended here.
130 static ObjectMonitor* volatile g_om_in_use_list = NULL;
131
132 static volatile int g_om_free_count = 0; // # on g_free_list
133 static volatile int g_om_in_use_count = 0; // # on g_om_in_use_list
134 static volatile int g_om_population = 0; // # Extant -- in circulation
135
136 #define CHAINMARKER (cast_to_oop<intptr_t>(-1))
137
138
139 // =====================> List Management functions
140
141 // Return true if the ObjectMonitor's next field is marked.
142 // Otherwise returns false.
143 static bool is_next_marked(ObjectMonitor* om) {
144 return ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & 0x1) != 0;
145 }
146
147 // Mark an ObjectMonitor* and return it. Note: the om parameter
148 // may or may not have been marked originally.
149 static ObjectMonitor* mark_om_ptr(ObjectMonitor* om) {
150 return (ObjectMonitor*)((intptr_t)om | 0x1);
151 }
152
153 // Mark the next field in an ObjectMonitor. If marking was successful,
154 // then the unmarked next field is returned via parameter and true is
155 // returned. Otherwise false is returned.
156 static bool mark_next(ObjectMonitor* om, ObjectMonitor** next_p) {
157 // Get current next field without any marking value.
158 ObjectMonitor* next = (ObjectMonitor*)
159 ((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1);
160 if (Atomic::cmpxchg(mark_om_ptr(next), &om->_next_om, next) != next) {
161 return false; // Could not mark the next field or it was already marked.
162 }
163 *next_p = next;
164 return true;
165 }
166
167 // Loop until we mark the next field in an ObjectMonitor. The unmarked
168 // next field is returned.
169 static ObjectMonitor* mark_next_loop(ObjectMonitor* om) {
170 ObjectMonitor* next;
171 while (true) {
172 if (mark_next(om, &next)) {
173 // Marked om's next field so return the unmarked value.
174 return next;
175 }
176 }
177 }
178
179 // Set the next field in an ObjectMonitor to the specified value.
180 // The caller of set_next() must be the same thread that marked the
181 // ObjectMonitor.
182 static void set_next(ObjectMonitor* om, ObjectMonitor* value) {
183 OrderAccess::release_store(&om->_next_om, value);
184 }
185
186 // Mark the next field in the list head ObjectMonitor. If marking was
187 // successful, then the mid and the unmarked next field are returned
188 // via parameter and true is returned. Otherwise false is returned.
189 static bool mark_list_head(ObjectMonitor* volatile * list_p,
190 ObjectMonitor** mid_p, ObjectMonitor** next_p) {
191 while (true) {
192 ObjectMonitor* mid = OrderAccess::load_acquire(list_p);
193 if (mid == NULL) {
194 return false; // The list is empty so nothing to mark.
195 }
196 if (mark_next(mid, next_p)) {
197 if (OrderAccess::load_acquire(list_p) != mid) {
198 // The list head changed so we have to retry.
199 set_next(mid, *next_p); // unmark mid
200 continue;
201 }
202 // We marked next field to guard against races.
203 *mid_p = mid;
204 return true;
205 }
206 }
207 }
208
209 // Return the unmarked next field in an ObjectMonitor. Note: the next
210 // field may or may not have been marked originally.
211 static ObjectMonitor* unmarked_next(ObjectMonitor* om) {
212 return (ObjectMonitor*)((intptr_t)OrderAccess::load_acquire(&om->_next_om) & ~0x1);
213 }
214
215 #if 0
216 // XXX - this is unused
217 // Unmark the next field in an ObjectMonitor. Requires that the next
218 // field be marked.
219 static void unmark_next(ObjectMonitor* om) {
220 ADIM_guarantee(is_next_marked(om), "next field must be marked: next=" INTPTR_FORMAT, p2i(om->_next_om));
221
222 ObjectMonitor* next = unmarked_next(om);
223 set_next(om, next);
224 }
225 #endif
226
227 volatile int visit_counter = 42;
228 static void chk_for_list_loop(ObjectMonitor* list, int count) {
229 if (!CheckMonitorLists) {
230 return;
231 }
232 int l_visit_counter = Atomic::add(1, &visit_counter);
233 int l_count = 0;
234 ObjectMonitor* prev = NULL;
235 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
236 if (mid->visit_marker == l_visit_counter) {
237 log_error(monitorinflation)("ERROR: prev=" INTPTR_FORMAT ", l_count=%d"
238 " refers to an ObjectMonitor that has"
239 " already been visited: mid=" INTPTR_FORMAT,
240 p2i(prev), l_count, p2i(mid));
241 fatal("list=" INTPTR_FORMAT " of %d items has a loop.", p2i(list), count);
242 }
243 mid->visit_marker = l_visit_counter;
244 prev = mid;
245 if (++l_count > count + 1024 * 1024) {
246 fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d",
247 p2i(list), count, l_count);
248 }
249 }
250 }
251
252 static void chk_om_not_on_list(ObjectMonitor* om, ObjectMonitor* list, int count) {
253 if (!CheckMonitorLists) {
254 return;
255 }
256 guarantee(list != om, "ERROR: om=" INTPTR_FORMAT " must not be head of the "
257 "list=" INTPTR_FORMAT ", count=%d", p2i(om), p2i(list), count);
258 int l_count = 0;
259 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
260 if (unmarked_next(mid) == om) {
261 log_error(monitorinflation)("ERROR: mid=" INTPTR_FORMAT ", l_count=%d"
262 " next_om refers to om=" INTPTR_FORMAT,
263 p2i(mid), l_count, p2i(om));
264 fatal("list=" INTPTR_FORMAT " of %d items has bad next_om value.",
265 p2i(list), count);
266 }
267 if (++l_count > count + 1024 * 1024) {
268 fatal("list=" INTPTR_FORMAT " of %d items may have a loop; l_count=%d",
269 p2i(list), count, l_count);
270 }
271 }
272 }
273
274 static void chk_om_elems_not_on_list(ObjectMonitor* elems, int elems_count,
275 ObjectMonitor* list, int list_count) {
276 if (!CheckMonitorLists) {
277 return;
278 }
279 chk_for_list_loop(elems, elems_count);
280 for (ObjectMonitor* mid = elems; mid != NULL; mid = unmarked_next(mid)) {
281 chk_om_not_on_list(mid, list, list_count);
282 }
283 }
284
285 // Prepend a list of ObjectMonitors to the specified *list_p. 'tail' is
286 // the last ObjectMonitor in the list and there are 'count' on the list.
287 // Also updates the specified *count_p.
288 static void prepend_list_to_common(ObjectMonitor* list, ObjectMonitor* tail,
289 int count, ObjectMonitor* volatile* list_p,
290 volatile int* count_p) {
291 chk_for_list_loop(OrderAccess::load_acquire(list_p),
292 OrderAccess::load_acquire(count_p));
293 chk_om_elems_not_on_list(list, count, OrderAccess::load_acquire(list_p),
294 OrderAccess::load_acquire(count_p));
295 while (true) {
296 ObjectMonitor* cur = OrderAccess::load_acquire(list_p);
297 // Prepend list to *list_p.
298 ObjectMonitor* next = NULL;
299 if (!mark_next(tail, &next)) {
300 continue; // failed to mark next field so try it all again
301 }
302 set_next(tail, cur); // tail now points to cur (and unmarks tail)
303 if (cur == NULL) {
304 // No potential race with takers or other prependers since
305 // *list_p is empty.
306 if (Atomic::cmpxchg(list, list_p, cur) == cur) {
307 // Successfully switched *list_p to the list value.
308 Atomic::add(count, count_p);
309 break;
310 }
311 // Implied else: try it all again
312 } else {
313 // Try to mark next field to guard against races:
314 if (!mark_next(cur, &next)) {
315 continue; // failed to mark next field so try it all again
316 }
317 // We marked the next field so try to switch *list_p to the list value.
318 if (Atomic::cmpxchg(list, list_p, cur) != cur) {
319 // The list head has changed so unmark the next field and try again:
320 set_next(cur, next);
321 continue;
322 }
323 Atomic::add(count, count_p);
324 set_next(cur, next); // unmark next field
325 break;
326 }
327 }
328 }
329
330 // Prepend a newly allocated block of ObjectMonitors to g_block_list and
331 // g_free_list. Also updates g_om_population and g_om_free_count.
332 void ObjectSynchronizer::prepend_block_to_lists(PaddedObjectMonitor* new_blk) {
333 // First we handle g_block_list:
334 while (true) {
335 PaddedObjectMonitor* cur = OrderAccess::load_acquire(&g_block_list);
336 // Prepend new_blk to g_block_list. The first ObjectMonitor in
337 // a block is reserved for use as linkage to the next block.
338 OrderAccess::release_store(&new_blk[0]._next_om, cur);
339 if (Atomic::cmpxchg(new_blk, &g_block_list, cur) == cur) {
340 // Successfully switched g_block_list to the new_blk value.
341 Atomic::add(_BLOCKSIZE - 1, &g_om_population);
342 break;
343 }
344 // Implied else: try it all again
345 }
346
347 // Second we handle g_free_list:
348 prepend_list_to_common(new_blk + 1, &new_blk[_BLOCKSIZE - 1], _BLOCKSIZE - 1,
349 &g_free_list, &g_om_free_count);
350 }
351
352 // Prepend a list of ObjectMonitors to g_free_list. 'tail' is the last
353 // ObjectMonitor in the list and there are 'count' on the list. Also
354 // updates g_om_free_count.
355 static void prepend_list_to_g_free_list(ObjectMonitor* list,
356 ObjectMonitor* tail, int count) {
357 prepend_list_to_common(list, tail, count, &g_free_list, &g_om_free_count);
358 }
359
360 // Prepend a list of ObjectMonitors to g_om_in_use_list. 'tail' is the last
361 // ObjectMonitor in the list and there are 'count' on the list. Also
362 // updates g_om_in_use_list.
363 static void prepend_list_to_g_om_in_use_list(ObjectMonitor* list,
364 ObjectMonitor* tail, int count) {
365 prepend_list_to_common(list, tail, count, &g_om_in_use_list, &g_om_in_use_count);
366 }
367
368 // Prepend an ObjectMonitor to the specified list. Also updates
369 // the specified counter.
370 static void prepend_to_common(ObjectMonitor* m, ObjectMonitor* volatile * list_p,
371 int volatile * count_p) {
372 chk_for_list_loop(OrderAccess::load_acquire(list_p),
373 OrderAccess::load_acquire(count_p));
374 chk_om_not_on_list(m, OrderAccess::load_acquire(list_p),
375 OrderAccess::load_acquire(count_p));
376
377 while (true) {
378 ObjectMonitor* cur = OrderAccess::load_acquire(list_p);
379 // Prepend ObjectMonitor to *list_p.
380 ObjectMonitor* next = NULL;
381 if (!mark_next(m, &next)) {
382 continue; // failed to mark next field so try it all again
383 }
384 set_next(m, cur); // m now points to cur (and unmarks m)
385 if (cur == NULL) {
386 // No potential race with other prependers since *list_p is empty.
387 if (Atomic::cmpxchg(m, list_p, cur) == cur) {
388 // Successfully switched *list_p to 'm'.
389 Atomic::inc(count_p);
390 break;
391 }
392 // Implied else: try it all again
393 } else {
394 // Try to mark next field to guard against races:
395 if (!mark_next(cur, &next)) {
396 continue; // failed to mark next field so try it all again
397 }
398 // We marked the next field so try to switch *list_p to 'm'.
399 if (Atomic::cmpxchg(m, list_p, cur) != cur) {
400 // The list head has changed so unmark the next field and try again:
401 set_next(cur, next);
402 continue;
403 }
404 Atomic::inc(count_p);
405 set_next(cur, next); // unmark next field
406 break;
407 }
408 }
409 }
410
411 // Prepend an ObjectMonitor to a per-thread om_free_list.
412 // Also updates the per-thread om_free_count.
413 static void prepend_to_om_free_list(Thread* self, ObjectMonitor* m) {
414 prepend_to_common(m, &self->om_free_list, &self->om_free_count);
415 }
416
417 // Prepend an ObjectMonitor to a per-thread om_in_use_list.
418 // Also updates the per-thread om_in_use_count.
419 static void prepend_to_om_in_use_list(Thread* self, ObjectMonitor* m) {
420 prepend_to_common(m, &self->om_in_use_list, &self->om_in_use_count);
421 }
422
423 // Take an ObjectMonitor from the start of the specified list. Also
424 // decrements the specified counter. Returns NULL if none are available.
425 static ObjectMonitor* take_from_start_of_common(ObjectMonitor* volatile * list_p,
426 int volatile * count_p) {
427 chk_for_list_loop(OrderAccess::load_acquire(list_p),
428 OrderAccess::load_acquire(count_p));
429
430 ObjectMonitor* next = NULL;
431 ObjectMonitor* take = NULL;
432 // Mark the list head to guard against A-B-A race:
433 if (!mark_list_head(list_p, &take, &next)) {
434 return NULL; // None are available.
435 }
436 // Switch marked list head to next (which unmarks the list head, but
437 // leaves take marked):
438 OrderAccess::release_store(list_p, next);
439 Atomic::dec(count_p);
440 // Unmark take, but leave the next value for any lagging list
441 // walkers. It will get cleaned up when take is prepended to
442 // the in-use list:
443 set_next(take, next);
444 return take;
445 }
446
447 // Take an ObjectMonitor from the start of the global free-list. Also
448 // updates g_om_free_count. Returns NULL if none are available.
449 static ObjectMonitor* take_from_start_of_g_free_list() {
450 return take_from_start_of_common(&g_free_list, &g_om_free_count);
451 }
452
453 // Take an ObjectMonitor from the start of a per-thread free-list.
454 // Also updates om_free_count. Returns NULL if none are available.
455 static ObjectMonitor* take_from_start_of_om_free_list(Thread* self) {
456 return take_from_start_of_common(&self->om_free_list, &self->om_free_count);
457 }
458
459
460 // =====================> Quick functions
461
462 // The quick_* forms are special fast-path variants used to improve
463 // performance. In the simplest case, a "quick_*" implementation could
464 // simply return false, in which case the caller will perform the necessary
465 // state transitions and call the slow-path form.
466 // The fast-path is designed to handle frequently arising cases in an efficient
467 // manner and is just a degenerate "optimistic" variant of the slow-path.
468 // returns true -- to indicate the call was satisfied.
469 // returns false -- to indicate the call needs the services of the slow-path.
470 // A no-loitering ordinance is in effect for code in the quick_* family
471 // operators: safepoints or indefinite blocking (blocking that might span a
472 // safepoint) are forbidden. Generally the thread_state() is _in_Java upon
473 // entry.
474 //
475 // Consider: An interesting optimization is to have the JIT recognize the
476 // following common idiom:
477 // synchronized (someobj) { .... ; notify(); }
478 // That is, we find a notify() or notifyAll() call that immediately precedes
479 // the monitorexit operation. In that case the JIT could fuse the operations
518 }
519
520 // biased locking and any other IMS exception states take the slow-path
521 return false;
522 }
523
524
525 // The LockNode emitted directly at the synchronization site would have
526 // been too big if it were to have included support for the cases of inflated
527 // recursive enter and exit, so they go here instead.
528 // Note that we can't safely call AsyncPrintJavaStack() from within
529 // quick_enter() as our thread state remains _in_Java.
530
531 bool ObjectSynchronizer::quick_enter(oop obj, Thread* self,
532 BasicLock * lock) {
533 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
534 assert(self->is_Java_thread(), "invariant");
535 assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");
536 NoSafepointVerifier nsv;
537 if (obj == NULL) return false; // Need to throw NPE
538
539 while (true) {
540 const markWord mark = obj->mark();
541
542 if (mark.has_monitor()) {
543 ObjectMonitorHandle omh;
544 if (!omh.save_om_ptr(obj, mark)) {
545 // Lost a race with async deflation so try again.
546 assert(AsyncDeflateIdleMonitors, "sanity check");
547 continue;
548 }
549 ObjectMonitor* const m = omh.om_ptr();
550 assert(oopDesc::equals((oop) m->object(), obj), "invariant");
551 Thread* const owner = (Thread *) m->_owner;
552
553 // Lock contention and Transactional Lock Elision (TLE) diagnostics
554 // and observability
555 // Case: light contention possibly amenable to TLE
556 // Case: TLE inimical operations such as nested/recursive synchronization
557
558 if (owner == self) {
559 m->_recursions++;
560 return true;
561 }
562
563 // This Java Monitor is inflated so obj's header will never be
564 // displaced to this thread's BasicLock. Make the displaced header
565 // non-NULL so this BasicLock is not seen as recursive nor as
566 // being locked. We do this unconditionally so that this thread's
567 // BasicLock cannot be mis-interpreted by any stack walkers. For
568 // performance reasons, stack walkers generally first check for
569 // Biased Locking in the object's header, the second check is for
570 // stack-locking in the object's header, the third check is for
571 // recursive stack-locking in the displaced header in the BasicLock,
572 // and last are the inflated Java Monitor (ObjectMonitor) checks.
573 lock->set_displaced_header(markWord::unused_mark());
574
575 if (owner == NULL && Atomic::replace_if_null(self, &(m->_owner))) {
576 assert(m->_recursions == 0, "invariant");
577 return true;
578 }
579
580 if (AsyncDeflateIdleMonitors &&
581 Atomic::cmpxchg(self, &m->_owner, DEFLATER_MARKER) == DEFLATER_MARKER) {
582 // The deflation protocol finished the first part (setting owner),
583 // but it failed the second part (making ref_count negative) and
584 // bailed. Or the ObjectMonitor was async deflated and reused.
585 // Acquired the monitor.
586 assert(m->_recursions == 0, "invariant");
587 return true;
588 }
589 }
590 break;
591 }
592
593 // Note that we could inflate in quick_enter.
594 // This is likely a useful optimization
595 // Critically, in quick_enter() we must not:
596 // -- perform bias revocation, or
597 // -- block indefinitely, or
598 // -- reach a safepoint
599
600 return false; // revert to slow-path
601 }
602
603 // -----------------------------------------------------------------------------
604 // Fast Monitor Enter/Exit
605 // This the fast monitor enter. The interpreter and compiler use
606 // some assembly copies of this code. Make sure update those code
607 // if the following function is changed. The implementation is
608 // extremely sensitive to race condition. Be careful.
609
610 void ObjectSynchronizer::fast_enter(Handle obj, BasicLock* lock,
654 // does not own the Java Monitor.
655 ObjectMonitor* m = mark.monitor();
656 assert(((oop)(m->object()))->mark() == mark, "invariant");
657 assert(m->is_entered(THREAD), "invariant");
658 }
659 }
660 #endif
661 return;
662 }
663
664 if (mark == markWord::from_pointer(lock)) {
665 // If the object is stack-locked by the current thread, try to
666 // swing the displaced header from the BasicLock back to the mark.
667 assert(dhw.is_neutral(), "invariant");
668 if (object->cas_set_mark(dhw, mark) == mark) {
669 return;
670 }
671 }
672
673 // We have to take the slow-path of possible inflation and then exit.
674 ObjectMonitorHandle omh;
675 inflate(&omh, THREAD, object, inflate_cause_vm_internal);
676 omh.om_ptr()->exit(true, THREAD);
677 }
678
679 // -----------------------------------------------------------------------------
680 // Interpreter/Compiler Slow Case
681 // This routine is used to handle interpreter/compiler slow case
682 // We don't need to use fast path here, because it must have been
683 // failed in the interpreter/compiler code.
684 void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) {
685 markWord mark = obj->mark();
686 assert(!mark.has_bias_pattern(), "should not see bias pattern here");
687
688 if (mark.is_neutral()) {
689 // Anticipate successful CAS -- the ST of the displaced mark must
690 // be visible <= the ST performed by the CAS.
691 lock->set_displaced_header(mark);
692 if (mark == obj()->cas_set_mark(markWord::from_pointer(lock), mark)) {
693 return;
694 }
695 // Fall through to inflate() ...
696 } else if (mark.has_locker() &&
697 THREAD->is_lock_owned((address)mark.locker())) {
698 assert(lock != mark.locker(), "must not re-lock the same lock");
699 assert(lock != (BasicLock*)obj->mark().value(), "don't relock with same BasicLock");
700 lock->set_displaced_header(markWord::from_pointer(NULL));
701 return;
702 }
703
704 // The object header will never be displaced to this lock,
705 // so it does not matter what the value is, except that it
706 // must be non-zero to avoid looking like a re-entrant lock,
707 // and must not look locked either.
708 lock->set_displaced_header(markWord::unused_mark());
709 ObjectMonitorHandle omh;
710 inflate(&omh, THREAD, obj(), inflate_cause_monitor_enter);
711 omh.om_ptr()->enter(THREAD);
712 }
713
714 // This routine is used to handle interpreter/compiler slow case
715 // We don't need to use fast path here, because it must have
716 // failed in the interpreter/compiler code. Simply use the heavy
717 // weight monitor should be ok, unless someone find otherwise.
718 void ObjectSynchronizer::slow_exit(oop object, BasicLock* lock, TRAPS) {
719 fast_exit(object, lock, THREAD);
720 }
721
722 // -----------------------------------------------------------------------------
723 // Class Loader support to workaround deadlocks on the class loader lock objects
724 // Also used by GC
725 // complete_exit()/reenter() are used to wait on a nested lock
726 // i.e. to give up an outer lock completely and then re-enter
727 // Used when holding nested locks - lock acquisition order: lock1 then lock2
728 // 1) complete_exit lock1 - saving recursion count
729 // 2) wait on lock2
730 // 3) when notified on lock2, unlock lock2
731 // 4) reenter lock1 with original recursion count
732 // 5) lock lock2
733 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
734 intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
735 if (UseBiasedLocking) {
736 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
737 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
738 }
739
740 ObjectMonitorHandle omh;
741 inflate(&omh, THREAD, obj(), inflate_cause_vm_internal);
742 intptr_t ret_code = omh.om_ptr()->complete_exit(THREAD);
743 return ret_code;
744 }
745
746 // NOTE: must use heavy weight monitor to handle complete_exit/reenter()
747 void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) {
748 if (UseBiasedLocking) {
749 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
750 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
751 }
752
753 ObjectMonitorHandle omh;
754 inflate(&omh, THREAD, obj(), inflate_cause_vm_internal);
755 omh.om_ptr()->reenter(recursion, THREAD);
756 }
757 // -----------------------------------------------------------------------------
758 // JNI locks on java objects
759 // NOTE: must use heavy weight monitor to handle jni monitor enter
760 void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) {
761 // the current locking is from JNI instead of Java code
762 if (UseBiasedLocking) {
763 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
764 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
765 }
766 THREAD->set_current_pending_monitor_is_from_java(false);
767 ObjectMonitorHandle omh;
768 inflate(&omh, THREAD, obj(), inflate_cause_jni_enter);
769 omh.om_ptr()->enter(THREAD);
770 THREAD->set_current_pending_monitor_is_from_java(true);
771 }
772
773 // NOTE: must use heavy weight monitor to handle jni monitor exit
774 void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
775 if (UseBiasedLocking) {
776 Handle h_obj(THREAD, obj);
777 BiasedLocking::revoke_and_rebias(h_obj, false, THREAD);
778 obj = h_obj();
779 }
780 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
781
782 ObjectMonitorHandle omh;
783 inflate(&omh, THREAD, obj, inflate_cause_jni_exit);
784 ObjectMonitor* monitor = omh.om_ptr();
785 // If this thread has locked the object, exit the monitor. We
786 // intentionally do not use CHECK here because we must exit the
787 // monitor even if an exception is pending.
788 if (monitor->check_owner(THREAD)) {
789 monitor->exit(true, THREAD);
790 }
791 }
792
793 // -----------------------------------------------------------------------------
794 // Internal VM locks on java objects
795 // standard constructor, allows locking failures
796 ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool do_lock) {
797 _dolock = do_lock;
798 _thread = thread;
799 _thread->check_for_valid_safepoint_state(false);
800 _obj = obj;
801
802 if (_dolock) {
803 ObjectSynchronizer::fast_enter(_obj, &_lock, false, _thread);
804 }
805 }
806
807 ObjectLocker::~ObjectLocker() {
808 if (_dolock) {
809 ObjectSynchronizer::fast_exit(_obj(), &_lock, _thread);
810 }
811 }
812
813
814 // -----------------------------------------------------------------------------
815 // Wait/Notify/NotifyAll
816 // NOTE: must use heavy weight monitor to handle wait()
817 int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
818 if (UseBiasedLocking) {
819 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
820 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
821 }
822 if (millis < 0) {
823 THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
824 }
825 ObjectMonitorHandle omh;
826 inflate(&omh, THREAD, obj(), inflate_cause_wait);
827 ObjectMonitor* monitor = omh.om_ptr();
828
829 DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);
830 monitor->wait(millis, true, THREAD);
831
832 // This dummy call is in place to get around dtrace bug 6254741. Once
833 // that's fixed we can uncomment the following line, remove the call
834 // and change this function back into a "void" func.
835 // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
836 int ret_code = dtrace_waited_probe(monitor, obj, THREAD);
837 return ret_code;
838 }
839
840 void ObjectSynchronizer::wait_uninterruptibly(Handle obj, jlong millis, TRAPS) {
841 if (UseBiasedLocking) {
842 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
843 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
844 }
845 if (millis < 0) {
846 THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
847 }
848 ObjectMonitorHandle omh;
849 inflate(&omh, THREAD, obj(), inflate_cause_wait);
850 omh.om_ptr()->wait(millis, false, THREAD);
851 }
852
853 void ObjectSynchronizer::notify(Handle obj, TRAPS) {
854 if (UseBiasedLocking) {
855 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
856 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
857 }
858
859 markWord mark = obj->mark();
860 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
861 return;
862 }
863 ObjectMonitorHandle omh;
864 inflate(&omh, THREAD, obj(), inflate_cause_notify);
865 omh.om_ptr()->notify(THREAD);
866 }
867
868 // NOTE: see comment of notify()
869 void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
870 if (UseBiasedLocking) {
871 BiasedLocking::revoke_and_rebias(obj, false, THREAD);
872 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
873 }
874
875 markWord mark = obj->mark();
876 if (mark.has_locker() && THREAD->is_lock_owned((address)mark.locker())) {
877 return;
878 }
879 ObjectMonitorHandle omh;
880 inflate(&omh, THREAD, obj(), inflate_cause_notify);
881 omh.om_ptr()->notifyAll(THREAD);
882 }
883
884 // -----------------------------------------------------------------------------
885 // Hash Code handling
886 //
887 // Performance concern:
888 // OrderAccess::storestore() calls release() which at one time stored 0
889 // into the global volatile OrderAccess::dummy variable. This store was
890 // unnecessary for correctness. Many threads storing into a common location
891 // causes considerable cache migration or "sloshing" on large SMP systems.
892 // As such, I avoided using OrderAccess::storestore(). In some cases
893 // OrderAccess::fence() -- which incurs local latency on the executing
894 // processor -- is a better choice as it scales on SMP systems.
895 //
896 // See http://blogs.oracle.com/dave/entry/biased_locking_in_hotspot for
897 // a discussion of coherency costs. Note that all our current reference
898 // platforms provide strong ST-ST order, so the issue is moot on IA32,
899 // x64, and SPARC.
900 //
901 // As a general policy we use "volatile" to control compiler-based reordering
902 // and explicit fences (barriers) to control for architectural reordering
903 // performed by the CPU(s) or platform.
904
905 struct SharedGlobals {
906 char _pad_prefix[OM_CACHE_LINE_SIZE];
907 // These are highly shared mostly-read variables.
908 // To avoid false-sharing they need to be the sole occupants of a cache line.
909 volatile int stw_random;
910 volatile int stw_cycle;
911 DEFINE_PAD_MINUS_SIZE(1, OM_CACHE_LINE_SIZE, sizeof(volatile int) * 2);
912 // Hot RW variable -- Sequester to avoid false-sharing
913 volatile int hc_sequence;
914 DEFINE_PAD_MINUS_SIZE(2, OM_CACHE_LINE_SIZE, sizeof(volatile int));
915 };
916
917 static SharedGlobals GVars;
918 static int MonitorScavengeThreshold = 1000000;
919 static volatile int ForceMonitorScavenge = 0; // Scavenge required and pending
920
921 static markWord read_stable_mark(oop obj) {
922 markWord mark = obj->mark();
923 if (!mark.is_being_inflated()) {
924 return mark; // normal fast-path return
925 }
926
927 int its = 0;
928 for (;;) {
929 markWord mark = obj->mark();
930 if (!mark.is_being_inflated()) {
931 return mark; // normal fast-path return
932 }
933
934 // The object is being inflated by some other thread.
1055 Handle hobj(self, obj);
1056 // Relaxing assertion for bug 6320749.
1057 assert(Universe::verify_in_progress() ||
1058 !SafepointSynchronize::is_at_safepoint(),
1059 "biases should not be seen by VM thread here");
1060 BiasedLocking::revoke_and_rebias(hobj, false, JavaThread::current());
1061 obj = hobj();
1062 assert(!obj->mark().has_bias_pattern(), "biases should be revoked by now");
1063 }
1064 }
1065
1066 // hashCode() is a heap mutator ...
1067 // Relaxing assertion for bug 6320749.
1068 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1069 !SafepointSynchronize::is_at_safepoint(), "invariant");
1070 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1071 self->is_Java_thread() , "invariant");
1072 assert(Universe::verify_in_progress() || DumpSharedSpaces ||
1073 ((JavaThread *)self)->thread_state() != _thread_blocked, "invariant");
1074
1075 while (true) {
1076 ObjectMonitor* monitor = NULL;
1077 markWord temp, test;
1078 intptr_t hash;
1079 markWord mark = read_stable_mark(obj);
1080
1081 // object should remain ineligible for biased locking
1082 assert(!mark.has_bias_pattern(), "invariant");
1083
1084 if (mark.is_neutral()) {
1085 hash = mark.hash(); // this is a normal header
1086 if (hash != 0) { // if it has hash, just return it
1087 return hash;
1088 }
1089 hash = get_next_hash(self, obj); // allocate a new hash code
1090 temp = mark.copy_set_hash(hash); // merge the hash code into header
1091 // use (machine word version) atomic operation to install the hash
1092 test = obj->cas_set_mark(temp, mark);
1093 if (test == mark) {
1094 return hash;
1095 }
1096 // If atomic operation failed, we must inflate the header
1097 // into heavy weight monitor. We could add more code here
1098 // for fast path, but it does not worth the complexity.
1099 } else if (mark.has_monitor()) {
1100 ObjectMonitorHandle omh;
1101 if (!omh.save_om_ptr(obj, mark)) {
1102 // Lost a race with async deflation so try again.
1103 assert(AsyncDeflateIdleMonitors, "sanity check");
1104 continue;
1105 }
1106 monitor = omh.om_ptr();
1107 temp = monitor->header();
1108 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1109 hash = temp.hash();
1110 if (hash != 0) {
1111 return hash;
1112 }
1113 // Skip to the following code to reduce code size
1114 } else if (self->is_lock_owned((address)mark.locker())) {
1115 temp = mark.displaced_mark_helper(); // this is a lightweight monitor owned
1116 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1117 hash = temp.hash(); // by current thread, check if the displaced
1118 if (hash != 0) { // header contains hash code
1119 return hash;
1120 }
1121 // WARNING:
1122 // The displaced header in the BasicLock on a thread's stack
1123 // is strictly immutable. It CANNOT be changed in ANY cases.
1124 // So we have to inflate the stack lock into an ObjectMonitor
1125 // even if the current thread owns the lock. The BasicLock on
1126 // a thread's stack can be asynchronously read by other threads
1127 // during an inflate() call so any change to that stack memory
1128 // may not propagate to other threads correctly.
1129 }
1130
1131 // Inflate the monitor to set hash code
1132 ObjectMonitorHandle omh;
1133 inflate(&omh, self, obj, inflate_cause_hash_code);
1134 monitor = omh.om_ptr();
1135 // Load displaced header and check it has hash code
1136 mark = monitor->header();
1137 assert(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT, mark.value());
1138 hash = mark.hash();
1139 if (hash == 0) {
1140 hash = get_next_hash(self, obj);
1141 temp = mark.copy_set_hash(hash); // merge hash code into header
1142 assert(temp.is_neutral(), "invariant: header=" INTPTR_FORMAT, temp.value());
1143 uintptr_t v = Atomic::cmpxchg(temp.value(), (volatile uintptr_t*)monitor->header_addr(), mark.value());
1144 test = markWord(v);
1145 if (test != mark) {
1146 // The only non-deflation update to the ObjectMonitor's
1147 // header/dmw field is to merge in the hash code. If someone
1148 // adds a new usage of the header/dmw field, please update
1149 // this code.
1150 // ObjectMonitor::install_displaced_markword_in_object()
1151 // does mark the header/dmw field as part of async deflation,
1152 // but that protocol cannot happen now due to the
1153 // ObjectMonitorHandle above.
1154 hash = test.hash();
1155 assert(test.is_neutral(), "invariant: header=" INTPTR_FORMAT, test.value());
1156 assert(hash != 0, "Trivial unexpected object/monitor header usage.");
1157 }
1158 }
1159 // We finally get the hash
1160 return hash;
1161 }
1162 }
1163
1164 // Deprecated -- use FastHashCode() instead.
1165
1166 intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) {
1167 return FastHashCode(Thread::current(), obj());
1168 }
1169
1170
1171 bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
1172 Handle h_obj) {
1173 if (UseBiasedLocking) {
1174 BiasedLocking::revoke_and_rebias(h_obj, false, thread);
1175 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1176 }
1177
1178 assert(thread == JavaThread::current(), "Can only be called on current thread");
1179 oop obj = h_obj();
1180
1181 while (true) {
1182 markWord mark = read_stable_mark(obj);
1183
1184 // Uncontended case, header points to stack
1185 if (mark.has_locker()) {
1186 return thread->is_lock_owned((address)mark.locker());
1187 }
1188 // Contended case, header points to ObjectMonitor (tagged pointer)
1189 if (mark.has_monitor()) {
1190 ObjectMonitorHandle omh;
1191 if (!omh.save_om_ptr(obj, mark)) {
1192 // Lost a race with async deflation so try again.
1193 assert(AsyncDeflateIdleMonitors, "sanity check");
1194 continue;
1195 }
1196 bool ret_code = omh.om_ptr()->is_entered(thread) != 0;
1197 return ret_code;
1198 }
1199 // Unlocked case, header in place
1200 assert(mark.is_neutral(), "sanity check");
1201 return false;
1202 }
1203 }
1204
1205 // Be aware of this method could revoke bias of the lock object.
1206 // This method queries the ownership of the lock handle specified by 'h_obj'.
1207 // If the current thread owns the lock, it returns owner_self. If no
1208 // thread owns the lock, it returns owner_none. Otherwise, it will return
1209 // owner_other.
1210 ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership
1211 (JavaThread *self, Handle h_obj) {
1212 // The caller must beware this method can revoke bias, and
1213 // revocation can result in a safepoint.
1214 assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
1215 assert(self->thread_state() != _thread_blocked, "invariant");
1216
1217 // Possible mark states: neutral, biased, stack-locked, inflated
1218
1219 if (UseBiasedLocking && h_obj()->mark().has_bias_pattern()) {
1220 // CASE: biased
1221 BiasedLocking::revoke_and_rebias(h_obj, false, self);
1222 assert(!h_obj->mark().has_bias_pattern(),
1223 "biases should be revoked by now");
1224 }
1225
1226 assert(self == JavaThread::current(), "Can only be called on current thread");
1227 oop obj = h_obj();
1228
1229 while (true) {
1230 markWord mark = read_stable_mark(obj);
1231
1232 // CASE: stack-locked. Mark points to a BasicLock on the owner's stack.
1233 if (mark.has_locker()) {
1234 return self->is_lock_owned((address)mark.locker()) ?
1235 owner_self : owner_other;
1236 }
1237
1238 // CASE: inflated. Mark (tagged pointer) points to an ObjectMonitor.
1239 // The Object:ObjectMonitor relationship is stable as long as we're
1240 // not at a safepoint and AsyncDeflateIdleMonitors is false.
1241 if (mark.has_monitor()) {
1242 ObjectMonitorHandle omh;
1243 if (!omh.save_om_ptr(obj, mark)) {
1244 // Lost a race with async deflation so try again.
1245 assert(AsyncDeflateIdleMonitors, "sanity check");
1246 continue;
1247 }
1248 ObjectMonitor* monitor = omh.om_ptr();
1249 void* owner = monitor->_owner;
1250 if (owner == NULL) return owner_none;
1251 return (owner == self ||
1252 self->is_lock_owned((address)owner)) ? owner_self : owner_other;
1253 }
1254
1255 // CASE: neutral
1256 assert(mark.is_neutral(), "sanity check");
1257 return owner_none; // it's unlocked
1258 }
1259 }
1260
1261 // FIXME: jvmti should call this
1262 JavaThread* ObjectSynchronizer::get_lock_owner(ThreadsList * t_list, Handle h_obj) {
1263 if (UseBiasedLocking) {
1264 if (SafepointSynchronize::is_at_safepoint()) {
1265 BiasedLocking::revoke_at_safepoint(h_obj);
1266 } else {
1267 BiasedLocking::revoke_and_rebias(h_obj, false, JavaThread::current());
1268 }
1269 assert(!h_obj->mark().has_bias_pattern(), "biases should be revoked by now");
1270 }
1271
1272 oop obj = h_obj();
1273
1274 while (true) {
1275 address owner = NULL;
1276 markWord mark = read_stable_mark(obj);
1277
1278 // Uncontended case, header points to stack
1279 if (mark.has_locker()) {
1280 owner = (address) mark.locker();
1281 }
1282
1283 // Contended case, header points to ObjectMonitor (tagged pointer)
1284 else if (mark.has_monitor()) {
1285 ObjectMonitorHandle omh;
1286 if (!omh.save_om_ptr(obj, mark)) {
1287 // Lost a race with async deflation so try again.
1288 assert(AsyncDeflateIdleMonitors, "sanity check");
1289 continue;
1290 }
1291 ObjectMonitor* monitor = omh.om_ptr();
1292 assert(monitor != NULL, "monitor should be non-null");
1293 owner = (address) monitor->owner();
1294 }
1295
1296 if (owner != NULL) {
1297 // owning_thread_from_monitor_owner() may also return NULL here
1298 return Threads::owning_thread_from_monitor_owner(t_list, owner);
1299 }
1300
1301 // Unlocked case, header in place
1302 // Cannot have assertion since this object may have been
1303 // locked by another thread when reaching here.
1304 // assert(mark.is_neutral(), "sanity check");
1305
1306 return NULL;
1307 }
1308 }
1309
1310 // Visitors ...
1311
1312 void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
1313 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
1314 while (block != NULL) {
1315 assert(block->object() == CHAINMARKER, "must be a block header");
1316 for (int i = _BLOCKSIZE - 1; i > 0; i--) {
1317 ObjectMonitor* mid = (ObjectMonitor *)(block + i);
1318 if (mid->is_active()) {
1319 ObjectMonitorHandle omh(mid);
1320
1321 if (mid->object() == NULL ||
1322 (AsyncDeflateIdleMonitors && mid->ref_count() < 0)) {
1323 // Only process with closure if the object is set.
1324 // For async deflation, race here if monitor is not owned!
1325 // The above ref_count bump (in ObjectMonitorHandle ctr)
1326 // will cause subsequent async deflation to skip it.
1327 // However, previous or concurrent async deflation is a race
1328 // so skip this ObjectMonitor if it is being async deflated.
1329 continue;
1330 }
1331 closure->do_monitor(mid);
1332 }
1333 }
1334 // unmarked_next() is not needed with g_block_list (no next field marking).
1335 block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om);
1336 }
1337 }
1338
1339 static bool monitors_used_above_threshold() {
1340 if (OrderAccess::load_acquire(&g_om_population) == 0) {
1341 return false;
1342 }
1343 if (MonitorUsedDeflationThreshold > 0) {
1344 int monitors_used = OrderAccess::load_acquire(&g_om_population) -
1345 OrderAccess::load_acquire(&g_om_free_count);
1346 int monitor_usage = (monitors_used * 100LL) /
1347 OrderAccess::load_acquire(&g_om_population);
1348 return monitor_usage > MonitorUsedDeflationThreshold;
1349 }
1350 return false;
1351 }
1352
1353 // Returns true if MonitorBound is set (> 0) and if the specified
1354 // cnt is > MonitorBound. Otherwise returns false.
1355 static bool is_MonitorBound_exceeded(const int cnt) {
1356 const int mx = MonitorBound;
1357 return mx > 0 && cnt > mx;
1358 }
1359
1360 bool ObjectSynchronizer::is_async_deflation_needed() {
1361 if (!AsyncDeflateIdleMonitors) {
1362 return false;
1363 }
1364 if (is_async_deflation_requested()) {
1365 // Async deflation request.
1366 return true;
1367 }
1368 if (AsyncDeflationInterval > 0 &&
1369 time_since_last_async_deflation_ms() > AsyncDeflationInterval &&
1370 monitors_used_above_threshold()) {
1371 // It's been longer than our specified deflate interval and there
1372 // are too many monitors in use. We don't deflate more frequently
1373 // than AsyncDeflationInterval (unless is_async_deflation_requested)
1374 // in order to not swamp the ServiceThread.
1375 _last_async_deflation_time_ns = os::javaTimeNanos();
1376 return true;
1377 }
1378 if (is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) -
1379 OrderAccess::load_acquire(&g_om_free_count))) {
1380 // Not enough ObjectMonitors on the global free list.
1381 return true;
1382 }
1383 return false;
1384 }
1385
1386 bool ObjectSynchronizer::is_safepoint_deflation_needed() {
1387 if (!AsyncDeflateIdleMonitors) {
1388 if (monitors_used_above_threshold()) {
1389 // Too many monitors in use.
1390 return true;
1391 }
1392 return false;
1393 }
1394 if (is_special_deflation_requested()) {
1395 // For AsyncDeflateIdleMonitors only do a safepoint deflation
1396 // if there is a special deflation request.
1397 return true;
1398 }
1399 return false;
1400 }
1401
1402 jlong ObjectSynchronizer::time_since_last_async_deflation_ms() {
1403 return (os::javaTimeNanos() - _last_async_deflation_time_ns) / (NANOUNITS / MILLIUNITS);
1404 }
1405
1406 void ObjectSynchronizer::oops_do(OopClosure* f) {
1407 // We only scan the global used list here (for moribund threads), and
1408 // the thread-local monitors in Thread::oops_do().
1409 global_used_oops_do(f);
1410 }
1411
1412 void ObjectSynchronizer::global_used_oops_do(OopClosure* f) {
1413 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1414 list_oops_do(OrderAccess::load_acquire(&g_om_in_use_list), OrderAccess::load_acquire(&g_om_in_use_count), f);
1415 }
1416
1417 void ObjectSynchronizer::thread_local_used_oops_do(Thread* thread, OopClosure* f) {
1418 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1419 list_oops_do(OrderAccess::load_acquire(&thread->om_in_use_list), OrderAccess::load_acquire(&thread->om_in_use_count), f);
1420 }
1421
1422 void ObjectSynchronizer::list_oops_do(ObjectMonitor* list, int count, OopClosure* f) {
1423 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
1424 chk_for_list_loop(list, count);
1425 // The oops_do() phase does not overlap with monitor deflation
1426 // so no need to update the ObjectMonitor's ref_count for this
1427 // ObjectMonitor* use.
1428 for (ObjectMonitor* mid = list; mid != NULL; mid = unmarked_next(mid)) {
1429 if (mid->object() != NULL) {
1430 f->do_oop((oop*)mid->object_addr());
1431 }
1432 }
1433 }
1434
1435
1436 // -----------------------------------------------------------------------------
1437 // ObjectMonitor Lifecycle
1438 // -----------------------
1439 // Inflation unlinks monitors from the global g_free_list and
1440 // associates them with objects. Deflation -- which occurs at
1441 // STW-time -- disassociates idle monitors from objects. Such
1442 // scavenged monitors are returned to the g_free_list.
1443 //
1444 // ObjectMonitors reside in type-stable memory (TSM) and are immortal.
1445 //
1446 // Lifecycle:
1447 // -- unassigned and on the global free list
1448 // -- unassigned and on a thread's private om_free_list
1449 // -- assigned to an object. The object is inflated and the mark refers
1450 // to the objectmonitor.
1451
1452
1453 // Constraining monitor pool growth via MonitorBound ...
1454 //
1455 // If MonitorBound is not set (<= 0), MonitorBound checks are disabled.
1456 //
1457 // When safepoint deflation is being used (!AsyncDeflateIdleMonitors):
1458 // The monitor pool is grow-only. We scavenge at STW safepoint-time, but the
1459 // the rate of scavenging is driven primarily by GC. As such, we can find
1460 // an inordinate number of monitors in circulation.
1461 // To avoid that scenario we can artificially induce a STW safepoint
1462 // if the pool appears to be growing past some reasonable bound.
1463 // Generally we favor time in space-time tradeoffs, but as there's no
1464 // natural back-pressure on the # of extant monitors we need to impose some
1465 // type of limit. Beware that if MonitorBound is set to too low a value
1466 // we could just loop. In addition, if MonitorBound is set to a low value
1467 // we'll incur more safepoints, which are harmful to performance.
1468 // See also: GuaranteedSafepointInterval
1469 //
1470 // The current implementation uses asynchronous VM operations.
1471 //
1472 // When safepoint deflation is being used and MonitorBound is set, the
1473 // boundry applies to
1474 // (g_om_population - g_om_free_count)
1475 // i.e., if there are not enough ObjectMonitors on the global free list,
1476 // then a safepoint deflation is induced. Picking a good MonitorBound value
1477 // is non-trivial.
1478 //
1479 // When async deflation is being used:
1480 // The monitor pool is still grow-only. Async deflation is requested
1481 // by a safepoint's cleanup phase or by the ServiceThread at periodic
1482 // intervals when is_async_deflation_needed() returns true. In
1483 // addition to other policies that are checked, if there are not
1484 // enough ObjectMonitors on the global free list, then
1485 // is_async_deflation_needed() will return true. The ServiceThread
1486 // calls deflate_global_idle_monitors_using_JT() and also calls
1487 // deflate_per_thread_idle_monitors_using_JT() as needed.
1488
1489 static void InduceScavenge(Thread* self, const char * Whence) {
1490 assert(!AsyncDeflateIdleMonitors, "is not used by async deflation");
1491
1492 // Induce STW safepoint to trim monitors
1493 // Ultimately, this results in a call to deflate_idle_monitors() in the near future.
1494 // More precisely, trigger an asynchronous STW safepoint as the number
1495 // of active monitors passes the specified threshold.
1496 // TODO: assert thread state is reasonable
1497
1498 if (ForceMonitorScavenge == 0 && Atomic::xchg (1, &ForceMonitorScavenge) == 0) {
1499 // Induce a 'null' safepoint to scavenge monitors
1500 // Must VM_Operation instance be heap allocated as the op will be enqueue and posted
1501 // to the VMthread and have a lifespan longer than that of this activation record.
1502 // The VMThread will delete the op when completed.
1503 VMThread::execute(new VM_ScavengeMonitors());
1504 }
1505 }
1506
1507 ObjectMonitor* ObjectSynchronizer::om_alloc(Thread* self,
1508 const InflateCause cause) {
1509 // A large MAXPRIVATE value reduces both list lock contention
1510 // and list coherency traffic, but also tends to increase the
1511 // number of ObjectMonitors in circulation as well as the STW
1512 // scavenge costs. As usual, we lean toward time in space-time
1513 // tradeoffs.
1514 const int MAXPRIVATE = 1024;
1515
1516 stringStream ss;
1517 for (;;) {
1518 ObjectMonitor* m;
1519
1520 // 1: try to allocate from the thread's local om_free_list.
1521 // Threads will attempt to allocate first from their local list, then
1522 // from the global list, and only after those attempts fail will the
1523 // thread attempt to instantiate new monitors. Thread-local free lists
1524 // improve allocation latency, as well as reducing coherency traffic
1525 // on the shared global list.
1526 m = take_from_start_of_om_free_list(self);
1527 if (m != NULL) {
1528 guarantee(m->object() == NULL, "invariant");
1529 m->set_allocation_state(ObjectMonitor::New);
1530 prepend_to_om_in_use_list(self, m);
1531 return m;
1532 }
1533
1534 // 2: try to allocate from the global g_free_list
1535 // CONSIDER: use muxTry() instead of muxAcquire().
1536 // If the muxTry() fails then drop immediately into case 3.
1537 // If we're using thread-local free lists then try
1538 // to reprovision the caller's free list.
1539 if (OrderAccess::load_acquire(&g_free_list) != NULL) {
1540 // Reprovision the thread's om_free_list.
1541 // Use bulk transfers to reduce the allocation rate and heat
1542 // on various locks.
1543 for (int i = self->om_free_provision; --i >= 0;) {
1544 ObjectMonitor* take = take_from_start_of_g_free_list();
1545 if (take == NULL) {
1546 break; // No more are available.
1547 }
1548 guarantee(take->object() == NULL, "invariant");
1549 if (AsyncDeflateIdleMonitors) {
1550 // We allowed 3 field values to linger during async deflation.
1551 // We clear header and restore ref_count here, but we leave
1552 // owner == DEFLATER_MARKER so the simple C2 ObjectMonitor
1553 // enter optimization can no longer race with async deflation
1554 // and reuse.
1555 take->set_header(markWord::zero());
1556 if (take->ref_count() < 0) {
1557 // Add back max_jint to restore the ref_count field to its
1558 // proper value.
1559 Atomic::add(max_jint, &take->_ref_count);
1560
1561 assert(take->ref_count() >= 0, "must not be negative: ref_count=%d",
1562 take->ref_count());
1563 }
1564 }
1565 take->Recycle();
1566 assert(take->is_free(), "invariant");
1567 om_release(self, take, false);
1568 }
1569 self->om_free_provision += 1 + (self->om_free_provision/2);
1570 if (self->om_free_provision > MAXPRIVATE) self->om_free_provision = MAXPRIVATE;
1571
1572 if (!AsyncDeflateIdleMonitors &&
1573 is_MonitorBound_exceeded(OrderAccess::load_acquire(&g_om_population) -
1574 OrderAccess::load_acquire(&g_om_free_count))) {
1575 // Not enough ObjectMonitors on the global free list.
1576 // We can't safely induce a STW safepoint from om_alloc() as our thread
1577 // state may not be appropriate for such activities and callers may hold
1578 // naked oops, so instead we defer the action.
1579 InduceScavenge(self, "om_alloc");
1580 }
1581 continue;
1582 }
1583
1584 // 3: allocate a block of new ObjectMonitors
1585 // Both the local and global free lists are empty -- resort to malloc().
1586 // In the current implementation ObjectMonitors are TSM - immortal.
1587 // Ideally, we'd write "new ObjectMonitor[_BLOCKSIZE], but we want
1588 // each ObjectMonitor to start at the beginning of a cache line,
1589 // so we use align_up().
1590 // A better solution would be to use C++ placement-new.
1591 // BEWARE: As it stands currently, we don't run the ctors!
1592 assert(_BLOCKSIZE > 1, "invariant");
1593 size_t neededsize = sizeof(PaddedObjectMonitor) * _BLOCKSIZE;
1594 PaddedObjectMonitor* temp;
1595 size_t aligned_size = neededsize + (OM_CACHE_LINE_SIZE - 1);
1596 void* real_malloc_addr = (void*)NEW_C_HEAP_ARRAY(char, aligned_size,
1597 mtInternal);
1598 temp = (PaddedObjectMonitor*)align_up(real_malloc_addr, OM_CACHE_LINE_SIZE);
1599
1600 // NOTE: (almost) no way to recover if allocation failed.
1601 // We might be able to induce a STW safepoint and scavenge enough
1602 // ObjectMonitors to permit progress.
1603 if (temp == NULL) {
1604 vm_exit_out_of_memory(neededsize, OOM_MALLOC_ERROR,
1605 "Allocate ObjectMonitors");
1606 }
1607 (void)memset((void *) temp, 0, neededsize);
1608
1609 // Format the block.
1610 // initialize the linked list, each monitor points to its next
1611 // forming the single linked free list, the very first monitor
1612 // will points to next block, which forms the block list.
1613 // The trick of using the 1st element in the block as g_block_list
1614 // linkage should be reconsidered. A better implementation would
1615 // look like: class Block { Block * next; int N; ObjectMonitor Body [N] ; }
1616
1617 for (int i = 1; i < _BLOCKSIZE; i++) {
1618 OrderAccess::release_store(&temp[i]._next_om, (ObjectMonitor*)&temp[i+1]);
1619 assert(temp[i].is_free(), "invariant");
1620 }
1621
1622 // terminate the last monitor as the end of list
1623 OrderAccess::release_store(&temp[_BLOCKSIZE - 1]._next_om, (ObjectMonitor*)NULL);
1624
1625 // Element [0] is reserved for global list linkage
1626 temp[0].set_object(CHAINMARKER);
1627
1628 // Consider carving out this thread's current request from the
1629 // block in hand. This avoids some lock traffic and redundant
1630 // list activity.
1631
1632 prepend_block_to_lists(temp);
1633 }
1634 }
1635
1636 // Place "m" on the caller's private per-thread om_free_list.
1637 // In practice there's no need to clamp or limit the number of
1638 // monitors on a thread's om_free_list as the only non-allocation time
1639 // we'll call om_release() is to return a monitor to the free list after
1640 // a CAS attempt failed. This doesn't allow unbounded #s of monitors to
1641 // accumulate on a thread's free list.
1642 //
1643 // Key constraint: all ObjectMonitors on a thread's free list and the global
1644 // free list must have their object field set to null. This prevents the
1645 // scavenger -- deflate_monitor_list() or deflate_monitor_list_using_JT()
1646 // -- from reclaiming them while we are trying to release them.
1647
1648 void ObjectSynchronizer::om_release(Thread* self, ObjectMonitor* m,
1649 bool from_per_thread_alloc) {
1650 guarantee(m->header().value() == 0, "invariant");
1651 guarantee(m->object() == NULL, "invariant");
1652 stringStream ss;
1653 guarantee((m->is_busy() | m->_recursions) == 0, "freeing in-use monitor: "
1654 "%s, recursions=" INTPTR_FORMAT, m->is_busy_to_string(&ss),
1655 m->_recursions);
1656 m->set_allocation_state(ObjectMonitor::Free);
1657 // _next_om is used for both per-thread in-use and free lists so
1658 // we have to remove 'm' from the in-use list first (as needed).
1659 if (from_per_thread_alloc) {
1660 // Need to remove 'm' from om_in_use_list.
1661 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
1662 // protocol because async deflation can do list deletions in parallel.
1663 ObjectMonitor* cur_mid_in_use = NULL;
1664 ObjectMonitor* mid = NULL;
1665 ObjectMonitor* next = NULL;
1666 bool extracted = false;
1667
1668 if (!mark_list_head(&self->om_in_use_list, &mid, &next)) {
1669 fatal("thread=" INTPTR_FORMAT " in-use list must not be empty.", p2i(self));
1670 }
1671 while (true) {
1672 if (m == mid) {
1673 // We found 'm' on the per-thread in-use list so try to extract it.
1674 // First try the list head:
1675 if (Atomic::cmpxchg(next, &self->om_in_use_list, mid) != mid) {
1676 // We could not switch the list head to next.
1677 ObjectMonitor* marked_mid = mark_om_ptr(mid);
1678 // Switch cur_mid_in_use's next field to next (which also
1679 // unmarks cur_mid_in_use):
1680 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
1681 if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, marked_mid)
1682 != marked_mid) {
1683 // We could not switch cur_mid_in_use's next field. This
1684 // should not be possible since it was marked so we:
1685 fatal("mid=" INTPTR_FORMAT " must be referred to by the list "
1686 "head: &om_in_use_list=" INTPTR_FORMAT " or by "
1687 "cur_mid_in_use's next field: cur_mid_in_use=" INTPTR_FORMAT
1688 ", next_om=" INTPTR_FORMAT, p2i(mid),
1689 p2i((ObjectMonitor**)&self->om_in_use_list),
1690 p2i(cur_mid_in_use), p2i(cur_mid_in_use->_next_om));
1691 }
1692 }
1693 extracted = true;
1694 Atomic::dec(&self->om_in_use_count);
1695 // Unmark mid, but leave the next value for any lagging list
1696 // walkers. It will get cleaned up when mid is prepended to
1697 // the thread's free list:
1698 set_next(mid, next);
1699 break;
1700 }
1701 if (cur_mid_in_use != NULL) {
1702 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
1703 }
1704 // The next cur_mid_in_use keeps mid's marked next field so
1705 // that it is stable for a possible next field change. It
1706 // cannot be deflated while it is marked.
1707 cur_mid_in_use = mid;
1708 mid = next;
1709 if (mid == NULL) {
1710 // Reached end of the list and didn't find m so:
1711 fatal("must find m=" INTPTR_FORMAT "on om_in_use_list=" INTPTR_FORMAT,
1712 p2i(m), p2i(self->om_in_use_list));
1713 }
1714 // Mark mid's next field so we can possibly extract it:
1715 next = mark_next_loop(mid);
1716 }
1717 }
1718
1719 prepend_to_om_free_list(self, m);
1720 guarantee(m->is_free(), "invariant");
1721 }
1722
1723 // Return ObjectMonitors on a moribund thread's free and in-use
1724 // lists to the appropriate global lists. The ObjectMonitors on the
1725 // per-thread in-use list may still be in use by other threads.
1726 //
1727 // We currently call om_flush() from Threads::remove() before the
1728 // thread has been excised from the thread list and is no longer a
1729 // mutator. This means that om_flush() cannot run concurrently with
1730 // a safepoint and interleave with deflate_idle_monitors(). In
1731 // particular, this ensures that the thread's in-use monitors are
1732 // scanned by a GC safepoint, either via Thread::oops_do() (before
1733 // om_flush() is called) or via ObjectSynchronizer::oops_do() (after
1734 // om_flush() is called).
1735 //
1736 // With AsyncDeflateIdleMonitors, deflate_global_idle_monitors_using_JT()
1737 // and deflate_per_thread_idle_monitors_using_JT() (in another thread) can
1738 // run at the same time as om_flush() so we have to follow a careful
1739 // protocol to prevent list corruption.
1740
1741 void ObjectSynchronizer::om_flush(Thread* self) {
1742 // This function can race with an async deflater thread. Since
1743 // deflation has to process the per-thread in-use list before
1744 // prepending the deflated ObjectMonitors to the global free list,
1745 // we process the per-thread lists in the same order to prevent
1746 // ordering races.
1747 int in_use_count = 0;
1748 ObjectMonitor* in_use_list = NULL;
1749 ObjectMonitor* in_use_tail = NULL;
1750 ObjectMonitor* next = NULL;
1751
1752 // An async deflation thread checks to see if the target thread
1753 // is exiting, but if it has made it past that check before we
1754 // started exiting, then it is racing to get to the in-use list.
1755 if (mark_list_head(&self->om_in_use_list, &in_use_list, &next)) {
1756 chk_for_list_loop(in_use_list, OrderAccess::load_acquire(&self->om_in_use_count));
1757 // At this point, we have marked the in-use list head so an
1758 // async deflation thread cannot come in after us. If an async
1759 // deflation thread is ahead of us, then we'll detect that and
1760 // wait for it to finish its work.
1761 //
1762 // The thread is going away, however the ObjectMonitors on the
1763 // om_in_use_list may still be in-use by other threads. Link
1764 // them to in_use_tail, which will be linked into the global
1765 // in-use list g_om_in_use_list below.
1766 //
1767 // Account for the in-use list head before the loop since it is
1768 // already marked (by this thread):
1769 in_use_tail = in_use_list;
1770 in_use_count++;
1771 for (ObjectMonitor* cur_om = unmarked_next(in_use_list); cur_om != NULL;) {
1772 if (is_next_marked(cur_om)) {
1773 // This next field is marked so there must be an async deflater
1774 // thread ahead of us so we'll give it a chance to finish.
1775 while (is_next_marked(cur_om)) {
1776 os::naked_short_sleep(1);
1777 }
1778 // Refetch the possibly changed next field and try again.
1779 cur_om = unmarked_next(in_use_tail);
1780 continue;
1781 }
1782 if (!cur_om->is_active()) {
1783 // cur_om was deflated and the allocation state was changed
1784 // to Free while it was marked. We happened to see it just
1785 // after it was unmarked (and added to the free list).
1786 // Refetch the possibly changed next field and try again.
1787 cur_om = unmarked_next(in_use_tail);
1788 continue;
1789 }
1790 in_use_tail = cur_om;
1791 in_use_count++;
1792 cur_om = unmarked_next(cur_om);
1793 }
1794 guarantee(in_use_tail != NULL, "invariant");
1795 int l_om_in_use_count = OrderAccess::load_acquire(&self->om_in_use_count);
1796 ADIM_guarantee(l_om_in_use_count == in_use_count, "in-use counts don't "
1797 "match: l_om_in_use_count=%d, in_use_count=%d",
1798 l_om_in_use_count, in_use_count);
1799 // Clear the in-use count before unmarking the in-use list head
1800 // to avoid races:
1801 OrderAccess::release_store(&self->om_in_use_count, 0);
1802 // Clear the in-use list head (which also unmarks it):
1803 OrderAccess::release_store(&self->om_in_use_list, (ObjectMonitor*)NULL);
1804 // Unmark the disconnected list head:
1805 set_next(in_use_list, next);
1806 }
1807
1808 int free_count = 0;
1809 ObjectMonitor* free_list = OrderAccess::load_acquire(&self->om_free_list);
1810 ObjectMonitor* free_tail = NULL;
1811 if (free_list != NULL) {
1812 chk_for_list_loop(free_list, OrderAccess::load_acquire(&self->om_free_count));
1813 // The thread is going away. Set 'free_tail' to the last per-thread free
1814 // monitor which will be linked to g_free_list below.
1815 stringStream ss;
1816 for (ObjectMonitor* s = free_list; s != NULL; s = unmarked_next(s)) {
1817 free_count++;
1818 free_tail = s;
1819 guarantee(s->object() == NULL, "invariant");
1820 guarantee(!s->is_busy(), "must be !is_busy: %s", s->is_busy_to_string(&ss));
1821 }
1822 guarantee(free_tail != NULL, "invariant");
1823 int l_om_free_count = OrderAccess::load_acquire(&self->om_free_count);
1824 ADIM_guarantee(l_om_free_count == free_count, "free counts don't match: "
1825 "l_om_free_count=%d, free_count=%d", l_om_free_count,
1826 free_count);
1827 OrderAccess::release_store(&self->om_free_list, (ObjectMonitor*)NULL);
1828 OrderAccess::release_store(&self->om_free_count, 0);
1829 }
1830
1831 if (free_tail != NULL) {
1832 prepend_list_to_g_free_list(free_list, free_tail, free_count);
1833 }
1834
1835 if (in_use_tail != NULL) {
1836 prepend_list_to_g_om_in_use_list(in_use_list, in_use_tail, in_use_count);
1837 }
1838
1839 LogStreamHandle(Debug, monitorinflation) lsh_debug;
1840 LogStreamHandle(Info, monitorinflation) lsh_info;
1841 LogStream* ls = NULL;
1842 if (log_is_enabled(Debug, monitorinflation)) {
1843 ls = &lsh_debug;
1844 } else if ((free_count != 0 || in_use_count != 0) &&
1845 log_is_enabled(Info, monitorinflation)) {
1846 ls = &lsh_info;
1847 }
1848 if (ls != NULL) {
1849 ls->print_cr("om_flush: jt=" INTPTR_FORMAT ", free_count=%d"
1850 ", in_use_count=%d" ", om_free_provision=%d",
1851 p2i(self), free_count, in_use_count, self->om_free_provision);
1852 }
1853 }
1854
1855 static void post_monitor_inflate_event(EventJavaMonitorInflate* event,
1856 const oop obj,
1857 ObjectSynchronizer::InflateCause cause) {
1858 assert(event != NULL, "invariant");
1859 assert(event->should_commit(), "invariant");
1860 event->set_monitorClass(obj->klass());
1861 event->set_address((uintptr_t)(void*)obj);
1862 event->set_cause((u1)cause);
1863 event->commit();
1864 }
1865
1866 // Fast path code shared by multiple functions
1867 void ObjectSynchronizer::inflate_helper(ObjectMonitorHandle* omh_p, oop obj) {
1868 while (true) {
1869 markWord mark = obj->mark();
1870 if (mark.has_monitor()) {
1871 if (!omh_p->save_om_ptr(obj, mark)) {
1872 // Lost a race with async deflation so try again.
1873 assert(AsyncDeflateIdleMonitors, "sanity check");
1874 continue;
1875 }
1876 ObjectMonitor* monitor = omh_p->om_ptr();
1877 assert(ObjectSynchronizer::verify_objmon_isinpool(monitor), "monitor is invalid");
1878 markWord dmw = monitor->header();
1879 assert(dmw.is_neutral(), "sanity check: header=" INTPTR_FORMAT, dmw.value());
1880 return;
1881 }
1882 inflate(omh_p, Thread::current(), obj, inflate_cause_vm_internal);
1883 return;
1884 }
1885 }
1886
1887 void ObjectSynchronizer::inflate(ObjectMonitorHandle* omh_p, Thread* self,
1888 oop object, const InflateCause cause) {
1889 // Inflate mutates the heap ...
1890 // Relaxing assertion for bug 6320749.
1891 assert(Universe::verify_in_progress() ||
1892 !SafepointSynchronize::is_at_safepoint(), "invariant");
1893
1894 EventJavaMonitorInflate event;
1895
1896 for (;;) {
1897 const markWord mark = object->mark();
1898 assert(!mark.has_bias_pattern(), "invariant");
1899
1900 // The mark can be in one of the following states:
1901 // * Inflated - just return
1902 // * Stack-locked - coerce it to inflated
1903 // * INFLATING - busy wait for conversion to complete
1904 // * Neutral - aggressively inflate the object.
1905 // * BIASED - Illegal. We should never see this
1906
1907 // CASE: inflated
1908 if (mark.has_monitor()) {
1909 if (!omh_p->save_om_ptr(object, mark)) {
1910 // Lost a race with async deflation so try again.
1911 assert(AsyncDeflateIdleMonitors, "sanity check");
1912 continue;
1913 }
1914 ObjectMonitor* inf = omh_p->om_ptr();
1915 markWord dmw = inf->header();
1916 assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
1917 assert(oopDesc::equals((oop) inf->object(), object), "invariant");
1918 assert(ObjectSynchronizer::verify_objmon_isinpool(inf), "monitor is invalid");
1919 return;
1920 }
1921
1922 // CASE: inflation in progress - inflating over a stack-lock.
1923 // Some other thread is converting from stack-locked to inflated.
1924 // Only that thread can complete inflation -- other threads must wait.
1925 // The INFLATING value is transient.
1926 // Currently, we spin/yield/park and poll the markword, waiting for inflation to finish.
1927 // We could always eliminate polling by parking the thread on some auxiliary list.
1928 if (mark == markWord::INFLATING()) {
1929 read_stable_mark(object);
1930 continue;
1931 }
1932
1933 // CASE: stack-locked
1934 // Could be stack-locked either by this thread or by some other thread.
1935 //
1936 // Note that we allocate the objectmonitor speculatively, _before_ attempting
1937 // to install INFLATING into the mark word. We originally installed INFLATING,
1938 // allocated the objectmonitor, and then finally STed the address of the
1939 // objectmonitor into the mark. This was correct, but artificially lengthened
1940 // the interval in which INFLATED appeared in the mark, thus increasing
1941 // the odds of inflation contention.
1942 //
1943 // We now use per-thread private objectmonitor free lists.
1944 // These list are reprovisioned from the global free list outside the
1945 // critical INFLATING...ST interval. A thread can transfer
1946 // multiple objectmonitors en-mass from the global free list to its local free list.
1947 // This reduces coherency traffic and lock contention on the global free list.
1948 // Using such local free lists, it doesn't matter if the om_alloc() call appears
1949 // before or after the CAS(INFLATING) operation.
1950 // See the comments in om_alloc().
1951
1952 LogStreamHandle(Trace, monitorinflation) lsh;
1953
1954 if (mark.has_locker()) {
1955 ObjectMonitor* m = om_alloc(self, cause);
1956 // Optimistically prepare the objectmonitor - anticipate successful CAS
1957 // We do this before the CAS in order to minimize the length of time
1958 // in which INFLATING appears in the mark.
1959 m->Recycle();
1960 m->_Responsible = NULL;
1961 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // Consider: maintain by type/class
1962
1963 markWord cmp = object->cas_set_mark(markWord::INFLATING(), mark);
1964 if (cmp != mark) {
1965 om_release(self, m, true);
1966 continue; // Interference -- just retry
1967 }
1968
1969 // We've successfully installed INFLATING (0) into the mark-word.
1970 // This is the only case where 0 will appear in a mark-word.
1971 // Only the singular thread that successfully swings the mark-word
1972 // to 0 can perform (or more precisely, complete) inflation.
1973 //
1974 // Why do we CAS a 0 into the mark-word instead of just CASing the
1975 // mark-word from the stack-locked value directly to the new inflated state?
1982 // value from the BasicLock on the owner's stack to the ObjectMonitor, all
1983 // the while preserving the hashCode stability invariants. If the owner
1984 // decides to release the lock while the value is 0, the unlock will fail
1985 // and control will eventually pass from slow_exit() to inflate. The owner
1986 // will then spin, waiting for the 0 value to disappear. Put another way,
1987 // the 0 causes the owner to stall if the owner happens to try to
1988 // drop the lock (restoring the header from the BasicLock to the object)
1989 // while inflation is in-progress. This protocol avoids races that might
1990 // would otherwise permit hashCode values to change or "flicker" for an object.
1991 // Critically, while object->mark is 0 mark.displaced_mark_helper() is stable.
1992 // 0 serves as a "BUSY" inflate-in-progress indicator.
1993
1994
1995 // fetch the displaced mark from the owner's stack.
1996 // The owner can't die or unwind past the lock while our INFLATING
1997 // object is in the mark. Furthermore the owner can't complete
1998 // an unlock on the object, either.
1999 markWord dmw = mark.displaced_mark_helper();
2000 // Catch if the object's header is not neutral (not locked and
2001 // not marked is what we care about here).
2002 ADIM_guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
2003
2004 // Setup monitor fields to proper values -- prepare the monitor
2005 m->set_header(dmw);
2006
2007 // Optimization: if the mark.locker stack address is associated
2008 // with this thread we could simply set m->_owner = self.
2009 // Note that a thread can inflate an object
2010 // that it has stack-locked -- as might happen in wait() -- directly
2011 // with CAS. That is, we can avoid the xchg-NULL .... ST idiom.
2012 m->set_owner(mark.locker());
2013 m->set_object(object);
2014 // TODO-FIXME: assert BasicLock->dhw != 0.
2015
2016 omh_p->set_om_ptr(m);
2017 assert(m->is_new(), "freshly allocated monitor must be new");
2018 m->set_allocation_state(ObjectMonitor::Old);
2019
2020 // Must preserve store ordering. The monitor state must
2021 // be stable at the time of publishing the monitor address.
2022 guarantee(object->mark() == markWord::INFLATING(), "invariant");
2023 object->release_set_mark(markWord::encode(m));
2024
2025 // Hopefully the performance counters are allocated on distinct cache lines
2026 // to avoid false sharing on MP systems ...
2027 OM_PERFDATA_OP(Inflations, inc());
2028 if (log_is_enabled(Trace, monitorinflation)) {
2029 ResourceMark rm(self);
2030 lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
2031 INTPTR_FORMAT ", type='%s'", p2i(object),
2032 object->mark().value(), object->klass()->external_name());
2033 }
2034 if (event.should_commit()) {
2035 post_monitor_inflate_event(&event, object, cause);
2036 }
2037 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
2038 return;
2039 }
2040
2041 // CASE: neutral
2042 // TODO-FIXME: for entry we currently inflate and then try to CAS _owner.
2043 // If we know we're inflating for entry it's better to inflate by swinging a
2044 // pre-locked ObjectMonitor pointer into the object header. A successful
2045 // CAS inflates the object *and* confers ownership to the inflating thread.
2046 // In the current implementation we use a 2-step mechanism where we CAS()
2047 // to inflate and then CAS() again to try to swing _owner from NULL to self.
2048 // An inflateTry() method that we could call from fast_enter() and slow_enter()
2049 // would be useful.
2050
2051 // Catch if the object's header is not neutral (not locked and
2052 // not marked is what we care about here).
2053 ADIM_guarantee(mark.is_neutral(), "invariant: header=" INTPTR_FORMAT,mark.value());
2054 ObjectMonitor* m = om_alloc(self, cause);
2055 // prepare m for installation - set monitor to initial state
2056 m->Recycle();
2057 m->set_header(mark);
2058 // If we leave _owner == DEFLATER_MARKER here, then the simple C2
2059 // ObjectMonitor enter optimization can no longer race with async
2060 // deflation and reuse.
2061 m->set_object(object);
2062 m->_Responsible = NULL;
2063 m->_SpinDuration = ObjectMonitor::Knob_SpinLimit; // consider: keep metastats by type/class
2064
2065 omh_p->set_om_ptr(m);
2066 assert(m->is_new(), "freshly allocated monitor must be new");
2067 m->set_allocation_state(ObjectMonitor::Old);
2068
2069 if (object->cas_set_mark(markWord::encode(m), mark) != mark) {
2070 guarantee(!m->owner_is_DEFLATER_MARKER() || m->ref_count() >= 0,
2071 "race between deflation and om_release() with m=" INTPTR_FORMAT
2072 ", _owner=" INTPTR_FORMAT ", ref_count=%d", p2i(m),
2073 p2i(m->_owner), m->ref_count());
2074 m->set_header(markWord::zero());
2075 m->set_object(NULL);
2076 m->Recycle();
2077 omh_p->set_om_ptr(NULL);
2078 // om_release() will reset the allocation state
2079 om_release(self, m, true);
2080 m = NULL;
2081 continue;
2082 // interference - the markword changed - just retry.
2083 // The state-transitions are one-way, so there's no chance of
2084 // live-lock -- "Inflated" is an absorbing state.
2085 }
2086
2087 // Hopefully the performance counters are allocated on distinct
2088 // cache lines to avoid false sharing on MP systems ...
2089 OM_PERFDATA_OP(Inflations, inc());
2090 if (log_is_enabled(Trace, monitorinflation)) {
2091 ResourceMark rm(self);
2092 lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
2093 INTPTR_FORMAT ", type='%s'", p2i(object),
2094 object->mark().value(), object->klass()->external_name());
2095 }
2096 if (event.should_commit()) {
2097 post_monitor_inflate_event(&event, object, cause);
2098 }
2099 ADIM_guarantee(!m->is_free(), "inflated monitor to be returned cannot be free");
2100 return;
2101 }
2102 }
2103
2104
2105 // We maintain a list of in-use monitors for each thread.
2106 //
2107 // For safepoint based deflation:
2108 // deflate_thread_local_monitors() scans a single thread's in-use list, while
2109 // deflate_idle_monitors() scans only a global list of in-use monitors which
2110 // is populated only as a thread dies (see om_flush()).
2111 //
2112 // These operations are called at all safepoints, immediately after mutators
2113 // are stopped, but before any objects have moved. Collectively they traverse
2114 // the population of in-use monitors, deflating where possible. The scavenged
2115 // monitors are returned to the global monitor free list.
2116 //
2117 // Beware that we scavenge at *every* stop-the-world point. Having a large
2118 // number of monitors in-use could negatively impact performance. We also want
2119 // to minimize the total # of monitors in circulation, as they incur a small
2120 // footprint penalty.
2121 //
2122 // Perversely, the heap size -- and thus the STW safepoint rate --
2123 // typically drives the scavenge rate. Large heaps can mean infrequent GC,
2124 // which in turn can mean large(r) numbers of ObjectMonitors in circulation.
2125 // This is an unfortunate aspect of this design.
2126 //
2127 // For async deflation:
2128 // If a special deflation request is made, then the safepoint based
2129 // deflation mechanism is used. Otherwise, an async deflation request
2130 // is registered with the ServiceThread and it is notified.
2131
2132 void ObjectSynchronizer::do_safepoint_work(DeflateMonitorCounters* counters) {
2133 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2134
2135 // The per-thread in-use lists are handled in
2136 // ParallelSPCleanupThreadClosure::do_thread().
2137
2138 if (!AsyncDeflateIdleMonitors || is_special_deflation_requested()) {
2139 // Use the older mechanism for the global in-use list or if a
2140 // special deflation has been requested before the safepoint.
2141 ObjectSynchronizer::deflate_idle_monitors(counters);
2142 return;
2143 }
2144
2145 log_debug(monitorinflation)("requesting async deflation of idle monitors.");
2146 // Request deflation of idle monitors by the ServiceThread:
2147 set_is_async_deflation_requested(true);
2148 MonitorLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
2149 ml.notify_all();
2150 }
2151
2152 // Deflate a single monitor if not in-use
2153 // Return true if deflated, false if in-use
2154 bool ObjectSynchronizer::deflate_monitor(ObjectMonitor* mid, oop obj,
2155 ObjectMonitor** free_head_p,
2156 ObjectMonitor** free_tail_p) {
2157 bool deflated;
2158 // Normal case ... The monitor is associated with obj.
2159 const markWord mark = obj->mark();
2160 guarantee(mark == markWord::encode(mid), "should match: mark="
2161 INTPTR_FORMAT ", encoded mid=" INTPTR_FORMAT, mark.value(),
2162 markWord::encode(mid).value());
2163 // Make sure that mark.monitor() and markWord::encode() agree:
2164 guarantee(mark.monitor() == mid, "should match: monitor()=" INTPTR_FORMAT
2165 ", mid=" INTPTR_FORMAT, p2i(mark.monitor()), p2i(mid));
2166 const markWord dmw = mid->header();
2167 guarantee(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
2168
2169 if (mid->is_busy() || mid->ref_count() != 0) {
2170 // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor*
2171 // is in use so no deflation.
2172 deflated = false;
2173 } else {
2174 // Deflate the monitor if it is no longer being used
2175 // It's idle - scavenge and return to the global free list
2176 // plain old deflation ...
2177 if (log_is_enabled(Trace, monitorinflation)) {
2178 ResourceMark rm;
2179 log_trace(monitorinflation)("deflate_monitor: "
2180 "object=" INTPTR_FORMAT ", mark="
2181 INTPTR_FORMAT ", type='%s'", p2i(obj),
2182 mark.value(), obj->klass()->external_name());
2183 }
2184
2185 // Restore the header back to obj
2186 obj->release_set_mark(dmw);
2187 if (AsyncDeflateIdleMonitors) {
2188 // clear() expects the owner field to be NULL and we won't race
2189 // with the simple C2 ObjectMonitor enter optimization since
2190 // we're at a safepoint.
2191 mid->set_owner(NULL);
2192 }
2193 mid->clear();
2194
2195 assert(mid->object() == NULL, "invariant: object=" INTPTR_FORMAT,
2196 p2i(mid->object()));
2197 assert(mid->is_free(), "invariant");
2198
2199 // Move the deflated ObjectMonitor to the working free list
2200 // defined by free_head_p and free_tail_p. No races on this list
2201 // so no need for load_acquire() or store_release().
2202 if (*free_head_p == NULL) *free_head_p = mid;
2203 if (*free_tail_p != NULL) {
2204 // We append to the list so the caller can use mid->_next_om
2205 // to fix the linkages in its context.
2206 ObjectMonitor* prevtail = *free_tail_p;
2207 // Should have been cleaned up by the caller:
2208 // Note: Should not have to mark prevtail here since we're at a
2209 // safepoint and ObjectMonitors on the local free list should
2210 // not be accessed in parallel.
2211 assert(prevtail->_next_om == NULL, "must be NULL: _next_om="
2212 INTPTR_FORMAT, p2i(prevtail->_next_om));
2213 set_next(prevtail, mid);
2214 }
2215 *free_tail_p = mid;
2216 // At this point, mid->_next_om still refers to its current
2217 // value and another ObjectMonitor's _next_om field still
2218 // refers to this ObjectMonitor. Those linkages have to be
2219 // cleaned up by the caller who has the complete context.
2220 deflated = true;
2221 }
2222 return deflated;
2223 }
2224
2225 // Deflate the specified ObjectMonitor if not in-use using a JavaThread.
2226 // Returns true if it was deflated and false otherwise.
2227 //
2228 // The async deflation protocol sets owner to DEFLATER_MARKER and
2229 // makes ref_count negative as signals to contending threads that
2230 // an async deflation is in progress. There are a number of checks
2231 // as part of the protocol to make sure that the calling thread has
2232 // not lost the race to a contending thread or to a thread that just
2233 // wants to use the ObjectMonitor*.
2234 //
2235 // The ObjectMonitor has been successfully async deflated when:
2236 // (owner == DEFLATER_MARKER && ref_count < 0)
2237 // Contending threads or ObjectMonitor* using threads that see those
2238 // values know to retry their operation.
2239 //
2240 bool ObjectSynchronizer::deflate_monitor_using_JT(ObjectMonitor* mid,
2241 ObjectMonitor** free_head_p,
2242 ObjectMonitor** free_tail_p) {
2243 assert(AsyncDeflateIdleMonitors, "sanity check");
2244 assert(Thread::current()->is_Java_thread(), "precondition");
2245 // A newly allocated ObjectMonitor should not be seen here so we
2246 // avoid an endless inflate/deflate cycle.
2247 assert(mid->is_old(), "must be old: allocation_state=%d",
2248 (int) mid->allocation_state());
2249
2250 if (mid->is_busy() || mid->ref_count() != 0) {
2251 // Easy checks are first - the ObjectMonitor is busy or ObjectMonitor*
2252 // is in use so no deflation.
2253 return false;
2254 }
2255
2256 if (Atomic::replace_if_null(DEFLATER_MARKER, &(mid->_owner))) {
2257 // ObjectMonitor is not owned by another thread. Our setting
2258 // owner to DEFLATER_MARKER forces any contending thread through
2259 // the slow path. This is just the first part of the async
2260 // deflation dance.
2261
2262 if (mid->_contentions != 0 || mid->_waiters != 0) {
2263 // Another thread has raced to enter the ObjectMonitor after
2264 // mid->is_busy() above or has already entered and waited on
2265 // it which makes it busy so no deflation. Restore owner to
2266 // NULL if it is still DEFLATER_MARKER.
2267 Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER);
2268 return false;
2269 }
2270
2271 if (Atomic::cmpxchg(-max_jint, &mid->_ref_count, (jint)0) == 0) {
2272 // Make ref_count negative to force any contending threads or
2273 // ObjectMonitor* using threads to retry. This is the second
2274 // part of the async deflation dance.
2275
2276 if (mid->owner_is_DEFLATER_MARKER()) {
2277 // If owner is still DEFLATER_MARKER, then we have successfully
2278 // signaled any contending threads to retry. If it is not, then we
2279 // have lost the race to an entering thread and the ObjectMonitor
2280 // is now busy. This is the third and final part of the async
2281 // deflation dance.
2282 // Note: This owner check solves the ABA problem with ref_count
2283 // where another thread acquired the ObjectMonitor, finished
2284 // using it and restored the ref_count to zero.
2285
2286 // Sanity checks for the races:
2287 guarantee(mid->_contentions == 0, "must be 0: contentions=%d",
2288 mid->_contentions);
2289 guarantee(mid->_waiters == 0, "must be 0: waiters=%d", mid->_waiters);
2290 guarantee(mid->_cxq == NULL, "must be no contending threads: cxq="
2291 INTPTR_FORMAT, p2i(mid->_cxq));
2292 guarantee(mid->_EntryList == NULL,
2293 "must be no entering threads: EntryList=" INTPTR_FORMAT,
2294 p2i(mid->_EntryList));
2295
2296 const oop obj = (oop) mid->object();
2297 if (log_is_enabled(Trace, monitorinflation)) {
2298 ResourceMark rm;
2299 log_trace(monitorinflation)("deflate_monitor_using_JT: "
2300 "object=" INTPTR_FORMAT ", mark="
2301 INTPTR_FORMAT ", type='%s'",
2302 p2i(obj), obj->mark().value(),
2303 obj->klass()->external_name());
2304 }
2305
2306 // Install the old mark word if nobody else has already done it.
2307 mid->install_displaced_markword_in_object(obj);
2308 mid->clear_using_JT();
2309
2310 assert(mid->object() == NULL, "must be NULL: object=" INTPTR_FORMAT,
2311 p2i(mid->object()));
2312 assert(mid->is_free(), "must be free: allocation_state=%d",
2313 (int) mid->allocation_state());
2314
2315 // Move the deflated ObjectMonitor to the working free list
2316 // defined by free_head_p and free_tail_p. No races on this list
2317 // so no need for load_acquire() or store_release().
2318 if (*free_head_p == NULL) {
2319 // First one on the list.
2320 *free_head_p = mid;
2321 }
2322 if (*free_tail_p != NULL) {
2323 // We append to the list so the caller can use mid->_next_om
2324 // to fix the linkages in its context.
2325 ObjectMonitor* prevtail = *free_tail_p;
2326 // Should have been cleaned up by the caller:
2327 ObjectMonitor* next = mark_next_loop(prevtail);
2328 assert(unmarked_next(prevtail) == NULL, "must be NULL: _next_om="
2329 INTPTR_FORMAT, p2i(unmarked_next(prevtail)));
2330 set_next(prevtail, mid); // prevtail now points to mid (and is unmarked)
2331 }
2332 *free_tail_p = mid;
2333
2334 // At this point, mid->_next_om still refers to its current
2335 // value and another ObjectMonitor's _next_om field still
2336 // refers to this ObjectMonitor. Those linkages have to be
2337 // cleaned up by the caller who has the complete context.
2338
2339 // We leave owner == DEFLATER_MARKER and ref_count < 0
2340 // to force any racing threads to retry.
2341 return true; // Success, ObjectMonitor has been deflated.
2342 }
2343
2344 // The owner was changed from DEFLATER_MARKER so we lost the
2345 // race since the ObjectMonitor is now busy.
2346
2347 // Add back max_jint to restore the ref_count field to its
2348 // proper value (which may not be what we saw above):
2349 Atomic::add(max_jint, &mid->_ref_count);
2350
2351 assert(mid->ref_count() >= 0, "must not be negative: ref_count=%d",
2352 mid->ref_count());
2353 return false;
2354 }
2355
2356 // The ref_count was no longer 0 so we lost the race since the
2357 // ObjectMonitor is now busy or the ObjectMonitor* is now is use.
2358 // Restore owner to NULL if it is still DEFLATER_MARKER:
2359 Atomic::cmpxchg((void*)NULL, &mid->_owner, DEFLATER_MARKER);
2360 }
2361
2362 // The owner field is no longer NULL so we lost the race since the
2363 // ObjectMonitor is now busy.
2364 return false;
2365 }
2366
2367 // Walk a given monitor list, and deflate idle monitors.
2368 // The given list could be a per-thread list or a global list.
2369 //
2370 // In the case of parallel processing of thread local monitor lists,
2371 // work is done by Threads::parallel_threads_do() which ensures that
2372 // each Java thread is processed by exactly one worker thread, and
2373 // thus avoid conflicts that would arise when worker threads would
2374 // process the same monitor lists concurrently.
2375 //
2376 // See also ParallelSPCleanupTask and
2377 // SafepointSynchronize::do_cleanup_tasks() in safepoint.cpp and
2378 // Threads::parallel_java_threads_do() in thread.cpp.
2379 int ObjectSynchronizer::deflate_monitor_list(ObjectMonitor* volatile * list_p,
2380 int volatile * count_p,
2381 ObjectMonitor** free_head_p,
2382 ObjectMonitor** free_tail_p) {
2383 ObjectMonitor* cur_mid_in_use = NULL;
2384 ObjectMonitor* mid = NULL;
2385 ObjectMonitor* next = NULL;
2386 int deflated_count = 0;
2387
2388 // We use the simpler mark-mid-as-we-go protocol since there are no
2389 // parallel list deletions since we are at a safepoint.
2390 if (!mark_list_head(list_p, &mid, &next)) {
2391 return 0; // The list is empty so nothing to deflate.
2392 }
2393
2394 while (true) {
2395 oop obj = (oop) mid->object();
2396 if (obj != NULL && deflate_monitor(mid, obj, free_head_p, free_tail_p)) {
2397 // Deflation succeeded and already updated free_head_p and
2398 // free_tail_p as needed. Finish the move to the local free list
2399 // by unlinking mid from the global or per-thread in-use list.
2400 if (Atomic::cmpxchg(next, list_p, mid) != mid) {
2401 // We could not switch the list head to next.
2402 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
2403 if (Atomic::cmpxchg(next, &cur_mid_in_use->_next_om, mid) != mid) {
2404 // deflate_monitor_list() is called at a safepoint so the
2405 // global or per-thread in-use list should not be modified
2406 // in parallel so we:
2407 fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: "
2408 "list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: "
2409 "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT,
2410 p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use),
2411 p2i(cur_mid_in_use->_next_om));
2412 }
2413 }
2414 // At this point mid is disconnected from the in-use list so
2415 // its marked next field no longer has any effects.
2416 deflated_count++;
2417 Atomic::dec(count_p);
2418 chk_for_list_loop(OrderAccess::load_acquire(list_p),
2419 OrderAccess::load_acquire(count_p));
2420 chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p),
2421 OrderAccess::load_acquire(count_p));
2422 // mid is current tail in the free_head_p list so NULL terminate it
2423 // (which also unmarks it):
2424 set_next(mid, NULL);
2425
2426 // All the list management is done so move on to the next one:
2427 mid = next;
2428 } else {
2429 set_next(mid, next); // unmark next field
2430
2431 // All the list management is done so move on to the next one:
2432 cur_mid_in_use = mid;
2433 mid = next;
2434 }
2435 if (mid == NULL) {
2436 break; // Reached end of the list so nothing more to deflate.
2437 }
2438 // Mark mid's next field so we can possibly deflate it:
2439 next = mark_next_loop(mid);
2440 }
2441 return deflated_count;
2442 }
2443
2444 // Walk a given ObjectMonitor list and deflate idle ObjectMonitors using
2445 // a JavaThread. Returns the number of deflated ObjectMonitors. The given
2446 // list could be a per-thread in-use list or the global in-use list.
2447 // If a safepoint has started, then we save state via saved_mid_in_use_p
2448 // and return to the caller to honor the safepoint.
2449 //
2450 int ObjectSynchronizer::deflate_monitor_list_using_JT(ObjectMonitor* volatile * list_p,
2451 int volatile * count_p,
2452 ObjectMonitor** free_head_p,
2453 ObjectMonitor** free_tail_p,
2454 ObjectMonitor** saved_mid_in_use_p) {
2455 assert(AsyncDeflateIdleMonitors, "sanity check");
2456 assert(Thread::current()->is_Java_thread(), "precondition");
2457
2458 ObjectMonitor* cur_mid_in_use = NULL;
2459 ObjectMonitor* mid = NULL;
2460 ObjectMonitor* next = NULL;
2461 ObjectMonitor* next_next = NULL;
2462 int deflated_count = 0;
2463
2464 // We use the more complicated mark-cur_mid_in_use-and-mid-as-we-go
2465 // protocol because om_release() can do list deletions in parallel.
2466 // We also mark-next-next-as-we-go to prevent an om_flush() that is
2467 // behind this thread from passing us.
2468 if (*saved_mid_in_use_p == NULL) {
2469 // No saved state so start at the beginning.
2470 // Mark the list head's next field so we can possibly deflate it:
2471 if (!mark_list_head(list_p, &mid, &next)) {
2472 return 0; // The list is empty so nothing to deflate.
2473 }
2474 } else {
2475 // We're restarting after a safepoint so restore the necessary state
2476 // before we resume.
2477 cur_mid_in_use = *saved_mid_in_use_p;
2478 // Mark cur_mid_in_use's next field so we can possibly update its
2479 // next field to extract a deflated ObjectMonitor.
2480 mid = mark_next_loop(cur_mid_in_use);
2481 if (mid == NULL) {
2482 set_next(cur_mid_in_use, NULL); // unmark next field
2483 *saved_mid_in_use_p = NULL;
2484 return 0; // The remainder is empty so nothing more to deflate.
2485 }
2486 // Mark mid's next field so we can possibly deflate it:
2487 next = mark_next_loop(mid);
2488 }
2489
2490 while (true) {
2491 // The current mid's next field is marked at this point. If we have
2492 // a cur_mid_in_use, then its next field is also marked at this point.
2493
2494 if (next != NULL) {
2495 // We mark the next -> next field so that an om_flush()
2496 // thread that is behind us cannot pass us when we
2497 // unmark the current mid's next field.
2498 next_next = mark_next_loop(next);
2499 }
2500
2501 // Only try to deflate if there is an associated Java object and if
2502 // mid is old (is not newly allocated and is not newly freed).
2503 if (mid->object() != NULL && mid->is_old() &&
2504 deflate_monitor_using_JT(mid, free_head_p, free_tail_p)) {
2505 // Deflation succeeded and already updated free_head_p and
2506 // free_tail_p as needed. Finish the move to the local free list
2507 // by unlinking mid from the global or per-thread in-use list.
2508 if (Atomic::cmpxchg(next, list_p, mid) != mid) {
2509 // We could not switch the list head to next.
2510 ObjectMonitor* marked_mid = mark_om_ptr(mid);
2511 ObjectMonitor* marked_next = mark_om_ptr(next);
2512 // Switch cur_mid_in_use's next field to marked next:
2513 ADIM_guarantee(cur_mid_in_use != NULL, "must not be NULL");
2514 if (Atomic::cmpxchg(marked_next, &cur_mid_in_use->_next_om,
2515 marked_mid) != marked_mid) {
2516 // We could not switch cur_mid_in_use's next field. This
2517 // should not be possible since it was marked so we:
2518 fatal("mid=" INTPTR_FORMAT " must be referred to by the list head: "
2519 "&list_p=" INTPTR_FORMAT " or by cur_mid_in_use's next field: "
2520 "cur_mid_in_use=" INTPTR_FORMAT ", next_om=" INTPTR_FORMAT,
2521 p2i(mid), p2i((ObjectMonitor**)list_p), p2i(cur_mid_in_use),
2522 p2i(cur_mid_in_use->_next_om));
2523 }
2524 }
2525 // At this point mid is disconnected from the in-use list so
2526 // its marked next field no longer has any effects.
2527 deflated_count++;
2528 Atomic::dec(count_p);
2529 chk_for_list_loop(OrderAccess::load_acquire(list_p),
2530 OrderAccess::load_acquire(count_p));
2531 chk_om_not_on_list(mid, OrderAccess::load_acquire(list_p),
2532 OrderAccess::load_acquire(count_p));
2533 // mid is current tail in the free_head_p list so NULL terminate it
2534 // (which also unmarks it):
2535 set_next(mid, NULL);
2536
2537 // All the list management is done so move on to the next one:
2538 mid = next; // mid keeps non-NULL next's marked next field
2539 next = next_next;
2540 } else {
2541 // mid is considered in-use if it does not have an associated
2542 // Java object or mid is not old or deflation did not succeed.
2543 // A mid->is_new() node can be seen here when it is freshly
2544 // returned by om_alloc() (and skips the deflation code path).
2545 // A mid->is_old() node can be seen here when deflation failed.
2546 // A mid->is_free() node can be seen here when a fresh node from
2547 // om_alloc() is released by om_release() due to losing the race
2548 // in inflate().
2549
2550 // All the list management is done so move on to the next one:
2551 if (cur_mid_in_use != NULL) {
2552 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2553 }
2554 // The next cur_mid_in_use keeps mid's marked next field so
2555 // that it is stable for a possible next field change. It
2556 // cannot be modified by om_release() while it is marked.
2557 cur_mid_in_use = mid;
2558 mid = next; // mid keeps non-NULL next's marked next field
2559 next = next_next;
2560
2561 if (SafepointSynchronize::is_synchronizing() &&
2562 cur_mid_in_use != OrderAccess::load_acquire(list_p) &&
2563 cur_mid_in_use->is_old()) {
2564 // If a safepoint has started and cur_mid_in_use is not the list
2565 // head and is old, then it is safe to use as saved state. Return
2566 // to the caller before blocking.
2567 *saved_mid_in_use_p = cur_mid_in_use;
2568 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2569 if (mid != NULL) {
2570 set_next(mid, next); // umark mid
2571 }
2572 return deflated_count;
2573 }
2574 }
2575 if (mid == NULL) {
2576 if (cur_mid_in_use != NULL) {
2577 set_next(cur_mid_in_use, mid); // umark cur_mid_in_use
2578 }
2579 break; // Reached end of the list so nothing more to deflate.
2580 }
2581
2582 // The current mid's next field is marked at this point. If we have
2583 // a cur_mid_in_use, then its next field is also marked at this point.
2584 }
2585 // We finished the list without a safepoint starting so there's
2586 // no need to save state.
2587 *saved_mid_in_use_p = NULL;
2588 return deflated_count;
2589 }
2590
2591 void ObjectSynchronizer::prepare_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2592 OrderAccess::release_store(&counters->n_in_use, 0); // currently associated with objects
2593 OrderAccess::release_store(&counters->n_in_circulation, 0); // extant
2594 OrderAccess::release_store(&counters->n_scavenged, 0); // reclaimed (global and per-thread)
2595 OrderAccess::release_store(&counters->per_thread_scavenged, 0); // per-thread scavenge total
2596 counters->per_thread_times = 0.0; // per-thread scavenge times
2597 }
2598
2599 void ObjectSynchronizer::deflate_idle_monitors(DeflateMonitorCounters* counters) {
2600 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2601
2602 if (AsyncDeflateIdleMonitors) {
2603 // Nothing to do when global idle ObjectMonitors are deflated using
2604 // a JavaThread unless a special deflation has been requested.
2605 if (!is_special_deflation_requested()) {
2606 return;
2607 }
2608 }
2609
2610 bool deflated = false;
2611
2612 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2613 ObjectMonitor* free_tail_p = NULL;
2614 elapsedTimer timer;
2615
2616 if (log_is_enabled(Info, monitorinflation)) {
2617 timer.start();
2618 }
2619
2620 // Note: the thread-local monitors lists get deflated in
2621 // a separate pass. See deflate_thread_local_monitors().
2622
2623 // For moribund threads, scan g_om_in_use_list
2624 int deflated_count = 0;
2625 if (OrderAccess::load_acquire(&g_om_in_use_list) != NULL) {
2626 // Update n_in_circulation before g_om_in_use_count is updated by deflation.
2627 Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_circulation);
2628
2629 deflated_count = deflate_monitor_list(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p);
2630 Atomic::add(OrderAccess::load_acquire(&g_om_in_use_count), &counters->n_in_use);
2631 }
2632
2633 if (free_head_p != NULL) {
2634 // Move the deflated ObjectMonitors back to the global free list.
2635 // No races on the working free list so no need for load_acquire().
2636 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2637 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2638 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2639 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2640 Atomic::add(deflated_count, &counters->n_scavenged);
2641 }
2642 timer.stop();
2643
2644 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2645 LogStreamHandle(Info, monitorinflation) lsh_info;
2646 LogStream* ls = NULL;
2647 if (log_is_enabled(Debug, monitorinflation)) {
2648 ls = &lsh_debug;
2649 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2650 ls = &lsh_info;
2651 }
2652 if (ls != NULL) {
2653 ls->print_cr("deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2654 }
2655 }
2656
2657 // Deflate global idle ObjectMonitors using a JavaThread.
2658 //
2659 void ObjectSynchronizer::deflate_global_idle_monitors_using_JT() {
2660 assert(AsyncDeflateIdleMonitors, "sanity check");
2661 assert(Thread::current()->is_Java_thread(), "precondition");
2662 JavaThread* self = JavaThread::current();
2663
2664 deflate_common_idle_monitors_using_JT(true /* is_global */, self);
2665 }
2666
2667 // Deflate the specified JavaThread's idle ObjectMonitors using a JavaThread.
2668 //
2669 void ObjectSynchronizer::deflate_per_thread_idle_monitors_using_JT(JavaThread* target) {
2670 assert(AsyncDeflateIdleMonitors, "sanity check");
2671 assert(Thread::current()->is_Java_thread(), "precondition");
2672
2673 deflate_common_idle_monitors_using_JT(false /* !is_global */, target);
2674 }
2675
2676 // Deflate global or per-thread idle ObjectMonitors using a JavaThread.
2677 //
2678 void ObjectSynchronizer::deflate_common_idle_monitors_using_JT(bool is_global, JavaThread* target) {
2679 JavaThread* self = JavaThread::current();
2680
2681 int deflated_count = 0;
2682 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged ObjectMonitors
2683 ObjectMonitor* free_tail_p = NULL;
2684 ObjectMonitor* saved_mid_in_use_p = NULL;
2685 elapsedTimer timer;
2686
2687 if (log_is_enabled(Info, monitorinflation)) {
2688 timer.start();
2689 }
2690
2691 if (is_global) {
2692 OM_PERFDATA_OP(MonExtant, set_value(OrderAccess::load_acquire(&g_om_in_use_count)));
2693 } else {
2694 OM_PERFDATA_OP(MonExtant, inc(OrderAccess::load_acquire(&target->om_in_use_count)));
2695 }
2696
2697 do {
2698 int local_deflated_count;
2699 if (is_global) {
2700 local_deflated_count = deflate_monitor_list_using_JT(&g_om_in_use_list, &g_om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2701 } else {
2702 local_deflated_count = deflate_monitor_list_using_JT(&target->om_in_use_list, &target->om_in_use_count, &free_head_p, &free_tail_p, &saved_mid_in_use_p);
2703 }
2704 deflated_count += local_deflated_count;
2705
2706 if (free_head_p != NULL) {
2707 // Move the deflated ObjectMonitors to the global free list.
2708 // No races on the working list so no need for load_acquire().
2709 guarantee(free_tail_p != NULL && local_deflated_count > 0, "free_tail_p=" INTPTR_FORMAT ", local_deflated_count=%d", p2i(free_tail_p), local_deflated_count);
2710 // Note: The target thread can be doing an om_alloc() that
2711 // is trying to prepend an ObjectMonitor on its in-use list
2712 // at the same time that we have deflated the current in-use
2713 // list head and put it on the local free list. prepend_to_common()
2714 // will detect the race and retry which avoids list corruption,
2715 // but the next field in free_tail_p can flicker to marked
2716 // and then unmarked while prepend_to_common() is sorting it
2717 // all out.
2718 assert(unmarked_next(free_tail_p) == NULL, "must be NULL: _next_om="
2719 INTPTR_FORMAT, p2i(unmarked_next(free_tail_p)));
2720
2721 prepend_list_to_g_free_list(free_head_p, free_tail_p, local_deflated_count);
2722
2723 OM_PERFDATA_OP(Deflations, inc(local_deflated_count));
2724 }
2725
2726 if (saved_mid_in_use_p != NULL) {
2727 // deflate_monitor_list_using_JT() detected a safepoint starting.
2728 timer.stop();
2729 {
2730 if (is_global) {
2731 log_debug(monitorinflation)("pausing deflation of global idle monitors for a safepoint.");
2732 } else {
2733 log_debug(monitorinflation)("jt=" INTPTR_FORMAT ": pausing deflation of per-thread idle monitors for a safepoint.", p2i(target));
2734 }
2735 assert(SafepointSynchronize::is_synchronizing(), "sanity check");
2736 ThreadBlockInVM blocker(self);
2737 }
2738 // Prepare for another loop after the safepoint.
2739 free_head_p = NULL;
2740 free_tail_p = NULL;
2741 if (log_is_enabled(Info, monitorinflation)) {
2742 timer.start();
2743 }
2744 }
2745 } while (saved_mid_in_use_p != NULL);
2746 timer.stop();
2747
2748 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2749 LogStreamHandle(Info, monitorinflation) lsh_info;
2750 LogStream* ls = NULL;
2751 if (log_is_enabled(Debug, monitorinflation)) {
2752 ls = &lsh_debug;
2753 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2754 ls = &lsh_info;
2755 }
2756 if (ls != NULL) {
2757 if (is_global) {
2758 ls->print_cr("async-deflating global idle monitors, %3.7f secs, %d monitors", timer.seconds(), deflated_count);
2759 } else {
2760 ls->print_cr("jt=" INTPTR_FORMAT ": async-deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(target), timer.seconds(), deflated_count);
2761 }
2762 }
2763 }
2764
2765 void ObjectSynchronizer::finish_deflate_idle_monitors(DeflateMonitorCounters* counters) {
2766 // Report the cumulative time for deflating each thread's idle
2767 // monitors. Note: if the work is split among more than one
2768 // worker thread, then the reported time will likely be more
2769 // than a beginning to end measurement of the phase.
2770 // Note: AsyncDeflateIdleMonitors only deflates per-thread idle
2771 // monitors at a safepoint when a special deflation has been requested.
2772 log_info(safepoint, cleanup)("deflating per-thread idle monitors, %3.7f secs, monitors=%d",
2773 counters->per_thread_times,
2774 OrderAccess::load_acquire(&counters->per_thread_scavenged));
2775
2776 bool needs_special_deflation = is_special_deflation_requested();
2777 if (!AsyncDeflateIdleMonitors || needs_special_deflation) {
2778 // AsyncDeflateIdleMonitors does not use these counters unless
2779 // there is a special deflation request.
2780
2781 OM_PERFDATA_OP(Deflations, inc(counters->n_scavenged));
2782 OM_PERFDATA_OP(MonExtant, set_value(counters->n_in_circulation));
2783 }
2784
2785 if (log_is_enabled(Debug, monitorinflation)) {
2786 // exit_globals()'s call to audit_and_print_stats() is done
2787 // at the Info level.
2788 ObjectSynchronizer::audit_and_print_stats(false /* on_exit */);
2789 } else if (log_is_enabled(Info, monitorinflation)) {
2790 log_info(monitorinflation)("g_om_population=%d, g_om_in_use_count=%d, "
2791 "g_om_free_count=%d",
2792 OrderAccess::load_acquire(&g_om_population),
2793 OrderAccess::load_acquire(&g_om_in_use_count),
2794 OrderAccess::load_acquire(&g_om_free_count));
2795 }
2796
2797 ForceMonitorScavenge = 0; // Reset
2798 GVars.stw_random = os::random();
2799 GVars.stw_cycle++;
2800 if (needs_special_deflation) {
2801 set_is_special_deflation_requested(false); // special deflation is done
2802 }
2803 }
2804
2805 void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread, DeflateMonitorCounters* counters) {
2806 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2807
2808 if (AsyncDeflateIdleMonitors && !is_special_deflation_requested()) {
2809 // Nothing to do if a special deflation has NOT been requested.
2810 return;
2811 }
2812
2813 ObjectMonitor* free_head_p = NULL; // Local SLL of scavenged monitors
2814 ObjectMonitor* free_tail_p = NULL;
2815 elapsedTimer timer;
2816
2817 if (log_is_enabled(Info, safepoint, cleanup) ||
2818 log_is_enabled(Info, monitorinflation)) {
2819 timer.start();
2820 }
2821
2822 // Update n_in_circulation before om_in_use_count is updated by deflation.
2823 Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_circulation);
2824
2825 int deflated_count = deflate_monitor_list(&thread->om_in_use_list, &thread->om_in_use_count, &free_head_p, &free_tail_p);
2826 Atomic::add(OrderAccess::load_acquire(&thread->om_in_use_count), &counters->n_in_use);
2827
2828 if (free_head_p != NULL) {
2829 // Move the deflated ObjectMonitors back to the global free list.
2830 // No races on the working list so no need for load_acquire().
2831 guarantee(free_tail_p != NULL && deflated_count > 0, "invariant");
2832 assert(free_tail_p->_next_om == NULL, "must be NULL: _next_om="
2833 INTPTR_FORMAT, p2i(free_tail_p->_next_om));
2834 prepend_list_to_g_free_list(free_head_p, free_tail_p, deflated_count);
2835 Atomic::add(deflated_count, &counters->n_scavenged);
2836 Atomic::add(deflated_count, &counters->per_thread_scavenged);
2837 }
2838
2839 timer.stop();
2840 // Safepoint logging cares about cumulative per_thread_times and
2841 // we'll capture most of the cost, but not the muxRelease() which
2842 // should be cheap.
2843 counters->per_thread_times += timer.seconds();
2844
2845 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2846 LogStreamHandle(Info, monitorinflation) lsh_info;
2847 LogStream* ls = NULL;
2848 if (log_is_enabled(Debug, monitorinflation)) {
2849 ls = &lsh_debug;
2850 } else if (deflated_count != 0 && log_is_enabled(Info, monitorinflation)) {
2851 ls = &lsh_info;
2852 }
2853 if (ls != NULL) {
2854 ls->print_cr("jt=" INTPTR_FORMAT ": deflating per-thread idle monitors, %3.7f secs, %d monitors", p2i(thread), timer.seconds(), deflated_count);
2855 }
2856 }
2857
2858 // Monitor cleanup on JavaThread::exit
2859
2860 // Iterate through monitor cache and attempt to release thread's monitors
2861 // Gives up on a particular monitor if an exception occurs, but continues
2862 // the overall iteration, swallowing the exception.
2863 class ReleaseJavaMonitorsClosure: public MonitorClosure {
2864 private:
2875
2876 // Release all inflated monitors owned by THREAD. Lightweight monitors are
2877 // ignored. This is meant to be called during JNI thread detach which assumes
2878 // all remaining monitors are heavyweight. All exceptions are swallowed.
2879 // Scanning the extant monitor list can be time consuming.
2880 // A simple optimization is to add a per-thread flag that indicates a thread
2881 // called jni_monitorenter() during its lifetime.
2882 //
2883 // Instead of No_Savepoint_Verifier it might be cheaper to
2884 // use an idiom of the form:
2885 // auto int tmp = SafepointSynchronize::_safepoint_counter ;
2886 // <code that must not run at safepoint>
2887 // guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
2888 // Since the tests are extremely cheap we could leave them enabled
2889 // for normal product builds.
2890
2891 void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
2892 assert(THREAD == JavaThread::current(), "must be current Java thread");
2893 NoSafepointVerifier nsv;
2894 ReleaseJavaMonitorsClosure rjmc(THREAD);
2895 ObjectSynchronizer::monitors_iterate(&rjmc);
2896 THREAD->clear_pending_exception();
2897 }
2898
2899 const char* ObjectSynchronizer::inflate_cause_name(const InflateCause cause) {
2900 switch (cause) {
2901 case inflate_cause_vm_internal: return "VM Internal";
2902 case inflate_cause_monitor_enter: return "Monitor Enter";
2903 case inflate_cause_wait: return "Monitor Wait";
2904 case inflate_cause_notify: return "Monitor Notify";
2905 case inflate_cause_hash_code: return "Monitor Hash Code";
2906 case inflate_cause_jni_enter: return "JNI Monitor Enter";
2907 case inflate_cause_jni_exit: return "JNI Monitor Exit";
2908 default:
2909 ShouldNotReachHere();
2910 }
2911 return "Unknown";
2912 }
2913
2914 //------------------------------------------------------------------------------
2915 // Debugging code
2929 u_char* ObjectSynchronizer::get_gvars_stw_random_addr() {
2930 return (u_char*)&GVars.stw_random;
2931 }
2932
2933 void ObjectSynchronizer::audit_and_print_stats(bool on_exit) {
2934 assert(on_exit || SafepointSynchronize::is_at_safepoint(), "invariant");
2935
2936 LogStreamHandle(Debug, monitorinflation) lsh_debug;
2937 LogStreamHandle(Info, monitorinflation) lsh_info;
2938 LogStreamHandle(Trace, monitorinflation) lsh_trace;
2939 LogStream* ls = NULL;
2940 if (log_is_enabled(Trace, monitorinflation)) {
2941 ls = &lsh_trace;
2942 } else if (log_is_enabled(Debug, monitorinflation)) {
2943 ls = &lsh_debug;
2944 } else if (log_is_enabled(Info, monitorinflation)) {
2945 ls = &lsh_info;
2946 }
2947 assert(ls != NULL, "sanity check");
2948
2949 // Log counts for the global and per-thread monitor lists:
2950 int chk_om_population = log_monitor_list_counts(ls);
2951 int error_cnt = 0;
2952
2953 ls->print_cr("Checking global lists:");
2954
2955 // Check g_om_population:
2956 if (OrderAccess::load_acquire(&g_om_population) == chk_om_population) {
2957 ls->print_cr("g_om_population=%d equals chk_om_population=%d",
2958 OrderAccess::load_acquire(&g_om_population),
2959 chk_om_population);
2960 } else {
2961 ls->print_cr("ERROR: g_om_population=%d is not equal to "
2962 "chk_om_population=%d",
2963 OrderAccess::load_acquire(&g_om_population),
2964 chk_om_population);
2965 error_cnt++;
2966 }
2967
2968 // Check g_om_in_use_list and g_om_in_use_count:
2969 chk_global_in_use_list_and_count(ls, &error_cnt);
2970
2971 // Check g_free_list and g_om_free_count:
2972 chk_global_free_list_and_count(ls, &error_cnt);
2973
2974 ls->print_cr("Checking per-thread lists:");
2975
2976 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
2977 // Check om_in_use_list and om_in_use_count:
2978 chk_per_thread_in_use_list_and_count(jt, ls, &error_cnt);
2979
2980 // Check om_free_list and om_free_count:
2981 chk_per_thread_free_list_and_count(jt, ls, &error_cnt);
2982 }
2983
2984 if (error_cnt == 0) {
2985 ls->print_cr("No errors found in monitor list checks.");
2986 } else {
2987 log_error(monitorinflation)("found monitor list errors: error_cnt=%d", error_cnt);
2988 }
2989
2990 if ((on_exit && log_is_enabled(Info, monitorinflation)) ||
2991 (!on_exit && log_is_enabled(Trace, monitorinflation))) {
2992 // When exiting this log output is at the Info level. When called
2993 // at a safepoint, this log output is at the Trace level since
2994 // there can be a lot of it.
2995 log_in_use_monitor_details(ls);
2996 }
2997
2998 ls->flush();
2999
3000 guarantee(error_cnt == 0, "ERROR: found monitor list errors: error_cnt=%d", error_cnt);
3001 }
3002
3003 // Check a free monitor entry; log any errors.
3004 void ObjectSynchronizer::chk_free_entry(JavaThread* jt, ObjectMonitor* n,
3005 outputStream * out, int *error_cnt_p) {
3006 stringStream ss;
3007 if (n->is_busy()) {
3008 if (jt != NULL) {
3009 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3010 ": free per-thread monitor must not be busy: %s", p2i(jt),
3011 p2i(n), n->is_busy_to_string(&ss));
3012 } else {
3013 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3014 "must not be busy: %s", p2i(n), n->is_busy_to_string(&ss));
3015 }
3016 *error_cnt_p = *error_cnt_p + 1;
3017 }
3018 if (n->header().value() != 0) {
3019 if (jt != NULL) {
3020 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3021 ": free per-thread monitor must have NULL _header "
3022 "field: _header=" INTPTR_FORMAT, p2i(jt), p2i(n),
3023 n->header().value());
3024 *error_cnt_p = *error_cnt_p + 1;
3025 } else if (!AsyncDeflateIdleMonitors) {
3026 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3027 "must have NULL _header field: _header=" INTPTR_FORMAT,
3028 p2i(n), n->header().value());
3029 *error_cnt_p = *error_cnt_p + 1;
3030 }
3031 }
3032 if (n->object() != NULL) {
3033 if (jt != NULL) {
3034 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3035 ": free per-thread monitor must have NULL _object "
3036 "field: _object=" INTPTR_FORMAT, p2i(jt), p2i(n),
3037 p2i(n->object()));
3038 } else {
3039 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": free global monitor "
3040 "must have NULL _object field: _object=" INTPTR_FORMAT,
3041 p2i(n), p2i(n->object()));
3042 }
3043 *error_cnt_p = *error_cnt_p + 1;
3044 }
3045 }
3046
3047 // Check the global free list and count; log the results of the checks.
3048 void ObjectSynchronizer::chk_global_free_list_and_count(outputStream * out,
3049 int *error_cnt_p) {
3050 int chk_om_free_count = 0;
3051 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_free_list); n != NULL; n = unmarked_next(n)) {
3052 chk_free_entry(NULL /* jt */, n, out, error_cnt_p);
3053 chk_om_free_count++;
3054 }
3055 if (OrderAccess::load_acquire(&g_om_free_count) == chk_om_free_count) {
3056 out->print_cr("g_om_free_count=%d equals chk_om_free_count=%d",
3057 OrderAccess::load_acquire(&g_om_free_count),
3058 chk_om_free_count);
3059 } else {
3060 // With lock free access to g_free_list, it is possible for an
3061 // ObjectMonitor to be prepended to g_free_list after we started
3062 // calculating chk_om_free_count so g_om_free_count may not
3063 // match anymore.
3064 out->print_cr("WARNING: g_om_free_count=%d is not equal to "
3065 "chk_om_free_count=%d",
3066 OrderAccess::load_acquire(&g_om_free_count),
3067 chk_om_free_count);
3068 }
3069 }
3070
3071 // Check the global in-use list and count; log the results of the checks.
3072 void ObjectSynchronizer::chk_global_in_use_list_and_count(outputStream * out,
3073 int *error_cnt_p) {
3074 int chk_om_in_use_count = 0;
3075 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) {
3076 chk_in_use_entry(NULL /* jt */, n, out, error_cnt_p);
3077 chk_om_in_use_count++;
3078 }
3079 if (OrderAccess::load_acquire(&g_om_in_use_count) == chk_om_in_use_count) {
3080 out->print_cr("g_om_in_use_count=%d equals chk_om_in_use_count=%d",
3081 OrderAccess::load_acquire(&g_om_in_use_count),
3082 chk_om_in_use_count);
3083 } else {
3084 out->print_cr("ERROR: g_om_in_use_count=%d is not equal to chk_om_in_use_count=%d",
3085 OrderAccess::load_acquire(&g_om_in_use_count),
3086 chk_om_in_use_count);
3087 *error_cnt_p = *error_cnt_p + 1;
3088 }
3089 }
3090
3091 // Check an in-use monitor entry; log any errors.
3092 void ObjectSynchronizer::chk_in_use_entry(JavaThread* jt, ObjectMonitor* n,
3093 outputStream * out, int *error_cnt_p) {
3094 if (n->header().value() == 0) {
3095 if (jt != NULL) {
3096 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3097 ": in-use per-thread monitor must have non-NULL _header "
3098 "field.", p2i(jt), p2i(n));
3099 } else {
3100 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global monitor "
3101 "must have non-NULL _header field.", p2i(n));
3102 }
3103 *error_cnt_p = *error_cnt_p + 1;
3104 }
3105 if (n->object() == NULL) {
3106 if (jt != NULL) {
3135 out->print_cr("ERROR: jt=" INTPTR_FORMAT ", monitor=" INTPTR_FORMAT
3136 ": in-use per-thread monitor's object does not refer "
3137 "to the same monitor: obj=" INTPTR_FORMAT ", mark="
3138 INTPTR_FORMAT ", obj_mon=" INTPTR_FORMAT, p2i(jt),
3139 p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3140 } else {
3141 out->print_cr("ERROR: monitor=" INTPTR_FORMAT ": in-use global "
3142 "monitor's object does not refer to the same monitor: obj="
3143 INTPTR_FORMAT ", mark=" INTPTR_FORMAT ", obj_mon="
3144 INTPTR_FORMAT, p2i(n), p2i(obj), mark.value(), p2i(obj_mon));
3145 }
3146 *error_cnt_p = *error_cnt_p + 1;
3147 }
3148 }
3149
3150 // Check the thread's free list and count; log the results of the checks.
3151 void ObjectSynchronizer::chk_per_thread_free_list_and_count(JavaThread *jt,
3152 outputStream * out,
3153 int *error_cnt_p) {
3154 int chk_om_free_count = 0;
3155 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_free_list); n != NULL; n = unmarked_next(n)) {
3156 chk_free_entry(jt, n, out, error_cnt_p);
3157 chk_om_free_count++;
3158 }
3159 if (OrderAccess::load_acquire(&jt->om_free_count) == chk_om_free_count) {
3160 out->print_cr("jt=" INTPTR_FORMAT ": om_free_count=%d equals "
3161 "chk_om_free_count=%d", p2i(jt),
3162 OrderAccess::load_acquire(&jt->om_free_count),
3163 chk_om_free_count);
3164 } else {
3165 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_free_count=%d is not "
3166 "equal to chk_om_free_count=%d", p2i(jt),
3167 OrderAccess::load_acquire(&jt->om_free_count),
3168 chk_om_free_count);
3169 *error_cnt_p = *error_cnt_p + 1;
3170 }
3171 }
3172
3173 // Check the thread's in-use list and count; log the results of the checks.
3174 void ObjectSynchronizer::chk_per_thread_in_use_list_and_count(JavaThread *jt,
3175 outputStream * out,
3176 int *error_cnt_p) {
3177 int chk_om_in_use_count = 0;
3178 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) {
3179 chk_in_use_entry(jt, n, out, error_cnt_p);
3180 chk_om_in_use_count++;
3181 }
3182 if (OrderAccess::load_acquire(&jt->om_in_use_count) == chk_om_in_use_count) {
3183 out->print_cr("jt=" INTPTR_FORMAT ": om_in_use_count=%d equals "
3184 "chk_om_in_use_count=%d", p2i(jt),
3185 OrderAccess::load_acquire(&jt->om_in_use_count),
3186 chk_om_in_use_count);
3187 } else {
3188 out->print_cr("ERROR: jt=" INTPTR_FORMAT ": om_in_use_count=%d is not "
3189 "equal to chk_om_in_use_count=%d", p2i(jt),
3190 OrderAccess::load_acquire(&jt->om_in_use_count),
3191 chk_om_in_use_count);
3192 *error_cnt_p = *error_cnt_p + 1;
3193 }
3194 }
3195
3196 // Log details about ObjectMonitors on the in-use lists. The 'BHL'
3197 // flags indicate why the entry is in-use, 'object' and 'object type'
3198 // indicate the associated object and its type.
3199 void ObjectSynchronizer::log_in_use_monitor_details(outputStream * out) {
3200 stringStream ss;
3201 if (OrderAccess::load_acquire(&g_om_in_use_count) > 0) {
3202 out->print_cr("In-use global monitor info:");
3203 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3204 out->print_cr("%18s %s %7s %18s %18s",
3205 "monitor", "BHL", "ref_cnt", "object", "object type");
3206 out->print_cr("================== === ======= ================== ==================");
3207 for (ObjectMonitor* n = OrderAccess::load_acquire(&g_om_in_use_list); n != NULL; n = unmarked_next(n)) {
3208 const oop obj = (oop) n->object();
3209 const markWord mark = n->header();
3210 ResourceMark rm;
3211 out->print(INTPTR_FORMAT " %d%d%d %7d " INTPTR_FORMAT " %s",
3212 p2i(n), n->is_busy() != 0, mark.hash() != 0,
3213 n->owner() != NULL, (int)n->ref_count(), p2i(obj),
3214 obj->klass()->external_name());
3215 if (n->is_busy() != 0) {
3216 out->print(" (%s)", n->is_busy_to_string(&ss));
3217 ss.reset();
3218 }
3219 out->cr();
3220 }
3221 }
3222
3223 out->print_cr("In-use per-thread monitor info:");
3224 out->print_cr("(B -> is_busy, H -> has hash code, L -> lock status)");
3225 out->print_cr("%18s %18s %s %7s %18s %18s",
3226 "jt", "monitor", "BHL", "ref_cnt", "object", "object type");
3227 out->print_cr("================== ================== === ======= ================== ==================");
3228 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3229 for (ObjectMonitor* n = OrderAccess::load_acquire(&jt->om_in_use_list); n != NULL; n = unmarked_next(n)) {
3230 const oop obj = (oop) n->object();
3231 const markWord mark = n->header();
3232 ResourceMark rm;
3233 out->print(INTPTR_FORMAT " " INTPTR_FORMAT " %d%d%d %7d "
3234 INTPTR_FORMAT " %s", p2i(jt), p2i(n), n->is_busy() != 0,
3235 mark.hash() != 0, n->owner() != NULL, (int)n->ref_count(),
3236 p2i(obj), obj->klass()->external_name());
3237 if (n->is_busy() != 0) {
3238 out->print(" (%s)", n->is_busy_to_string(&ss));
3239 ss.reset();
3240 }
3241 out->cr();
3242 }
3243 }
3244
3245 out->flush();
3246 }
3247
3248 // Log counts for the global and per-thread monitor lists and return
3249 // the population count.
3250 int ObjectSynchronizer::log_monitor_list_counts(outputStream * out) {
3251 int pop_count = 0;
3252 out->print_cr("%18s %10s %10s %10s",
3253 "Global Lists:", "InUse", "Free", "Total");
3254 out->print_cr("================== ========== ========== ==========");
3255 out->print_cr("%18s %10d %10d %10d", "",
3256 OrderAccess::load_acquire(&g_om_in_use_count),
3257 OrderAccess::load_acquire(&g_om_free_count),
3258 OrderAccess::load_acquire(&g_om_population));
3259 pop_count += OrderAccess::load_acquire(&g_om_in_use_count) +
3260 OrderAccess::load_acquire(&g_om_free_count);
3261
3262 out->print_cr("%18s %10s %10s %10s",
3263 "Per-Thread Lists:", "InUse", "Free", "Provision");
3264 out->print_cr("================== ========== ========== ==========");
3265
3266 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *jt = jtiwh.next(); ) {
3267 out->print_cr(INTPTR_FORMAT " %10d %10d %10d", p2i(jt),
3268 OrderAccess::load_acquire(&jt->om_in_use_count),
3269 OrderAccess::load_acquire(&jt->om_free_count),
3270 jt->om_free_provision);
3271 pop_count += OrderAccess::load_acquire(&jt->om_in_use_count) +
3272 OrderAccess::load_acquire(&jt->om_free_count);
3273 }
3274 return pop_count;
3275 }
3276
3277 #ifndef PRODUCT
3278
3279 // Check if monitor belongs to the monitor cache
3280 // The list is grow-only so it's *relatively* safe to traverse
3281 // the list of extant blocks without taking a lock.
3282
3283 int ObjectSynchronizer::verify_objmon_isinpool(ObjectMonitor *monitor) {
3284 PaddedObjectMonitor* block = OrderAccess::load_acquire(&g_block_list);
3285 while (block != NULL) {
3286 assert(block->object() == CHAINMARKER, "must be a block header");
3287 if (monitor > &block[0] && monitor < &block[_BLOCKSIZE]) {
3288 address mon = (address)monitor;
3289 address blk = (address)block;
3290 size_t diff = mon - blk;
3291 assert((diff % sizeof(PaddedObjectMonitor)) == 0, "must be aligned");
3292 return 1;
3293 }
3294 // unmarked_next() is not needed with g_block_list (no next field marking).
3295 block = (PaddedObjectMonitor*)OrderAccess::load_acquire(&block->_next_om);
3296 }
3297 return 0;
3298 }
3299
3300 #endif
|