< prev index next >

src/hotspot/share/runtime/objectMonitor.cpp

Print this page
rev 51780 : imported patch syncknobs-00-base
rev 51781 : imported patch syncknobs-01-Knob_ReportSettings
rev 51782 : imported patch syncknobs-02-Knob_SpinBackOff
rev 51783 : imported patch syncknobs-03-BackOffMask
rev 51784 : imported patch syncknobs-04-Knob_ExitRelease
rev 51785 : imported patch syncknobs-05-Knob_InlineNotify
rev 51786 : imported patch syncknobs-06-Knob_Verbose
rev 51787 : imported patch syncknobs-07-Knob_VerifyInUse
rev 51788 : imported patch syncknobs-08-Knob_VerifyMatch
rev 51789 : imported patch syncknobs-09-Knob_SpinBase
rev 51790 : imported patch syncknobs-10-Knob_CASPenalty
rev 51791 : imported patch syncknobs-11-Knob_OXPenalty
rev 51792 : imported patch syncknobs-12-Knob_SpinSetSucc
rev 51793 : imported patch syncknobs-13-Knob_SpinEarly
rev 51794 : imported patch syncknobs-14-Knob_SuccEnabled
rev 51795 : imported patch syncknobs-15-Knob_SuccRestrict
rev 51796 : imported patch syncknobs-16-Knob_MaxSpinners
rev 51797 : imported patch syncknobs-17-Knob_SpinAfterFutile
rev 51798 : imported patch syncknobs-18-Knob_OState
rev 51799 : imported patch syncknobs-19-Knob_UsePause
rev 51800 : imported patch syncknobs-20-Knob_ExitPolicy
rev 51801 : imported patch syncknobs-21-Knob_ResetEvent
rev 51802 : imported patch syncknobs-22-Knob_FastHSSEC
rev 51803 : imported patch syncknobs-23-Knob_MoveNotifyee
rev 51804 : imported patch syncknobs-24-Knob_QMode


  84 #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread)                  \
  85   {                                                                        \
  86     if (DTraceMonitorProbes) {                                             \
  87       DTRACE_MONITOR_PROBE_COMMON(obj, thread);                            \
  88       HOTSPOT_MONITOR_##probe(jtid,                                        \
  89                               (uintptr_t)(monitor), bytes, len);           \
  90     }                                                                      \
  91   }
  92 
  93 #else //  ndef DTRACE_ENABLED
  94 
  95 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon)    {;}
  96 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon)          {;}
  97 
  98 #endif // ndef DTRACE_ENABLED
  99 
 100 // Tunables ...
 101 // The knob* variables are effectively final.  Once set they should
 102 // never be modified hence.  Consider using __read_mostly with GCC.
 103 
 104 int ObjectMonitor::Knob_ExitRelease  = 0;
 105 int ObjectMonitor::Knob_InlineNotify = 1;
 106 int ObjectMonitor::Knob_Verbose      = 0;
 107 int ObjectMonitor::Knob_VerifyInUse  = 0;
 108 int ObjectMonitor::Knob_VerifyMatch  = 0;
 109 int ObjectMonitor::Knob_SpinLimit    = 5000;    // derived by an external tool -
 110 
 111 static int Knob_ReportSettings      = 0;
 112 static int Knob_SpinBase            = 0;       // Floor AKA SpinMin
 113 static int Knob_SpinBackOff         = 0;       // spin-loop backoff
 114 static int Knob_CASPenalty          = -1;      // Penalty for failed CAS
 115 static int Knob_OXPenalty           = -1;      // Penalty for observed _owner change
 116 static int Knob_SpinSetSucc         = 1;       // spinners set the _succ field
 117 static int Knob_SpinEarly           = 1;
 118 static int Knob_SuccEnabled         = 1;       // futile wake throttling
 119 static int Knob_SuccRestrict        = 0;       // Limit successors + spinners to at-most-one
 120 static int Knob_MaxSpinners         = -1;      // Should be a function of # CPUs
 121 static int Knob_Bonus               = 100;     // spin success bonus
 122 static int Knob_BonusB              = 100;     // spin success bonus
 123 static int Knob_Penalty             = 200;     // spin failure penalty
 124 static int Knob_Poverty             = 1000;
 125 static int Knob_SpinAfterFutile     = 1;       // Spin after returning from park()
 126 static int Knob_FixedSpin           = 0;
 127 static int Knob_OState              = 3;       // Spinner checks thread state of _owner
 128 static int Knob_UsePause            = 1;
 129 static int Knob_ExitPolicy          = 0;
 130 static int Knob_PreSpin             = 10;      // 20-100 likely better
 131 static int Knob_ResetEvent          = 0;
 132 static int BackOffMask              = 0;
 133 
 134 static int Knob_FastHSSEC           = 0;
 135 static int Knob_MoveNotifyee        = 2;       // notify() - disposition of notifyee
 136 static int Knob_QMode               = 0;       // EntryList-cxq policy - queue discipline
 137 static volatile int InitDone        = 0;
 138 
 139 // -----------------------------------------------------------------------------
 140 // Theory of operations -- Monitors lists, thread residency, etc:
 141 //
 142 // * A thread acquires ownership of a monitor by successfully
 143 //   CAS()ing the _owner field from null to non-null.
 144 //
 145 // * Invariant: A thread appears on at most one monitor list --
 146 //   cxq, EntryList or WaitSet -- at any one time.
 147 //
 148 // * Contending threads "push" themselves onto the cxq with CAS
 149 //   and then spin/park.
 150 //
 151 // * After a contending thread eventually acquires the lock it must
 152 //   dequeue itself from either the EntryList or the cxq.
 153 //
 154 // * The exiting thread identifies and unparks an "heir presumptive"
 155 //   tentative successor thread on the EntryList.  Critically, the
 156 //   exiting thread doesn't unlink the successor thread from the EntryList.


 282   }
 283 
 284   if (Self->is_lock_owned ((address)cur)) {
 285     assert(_recursions == 0, "internal state error");
 286     _recursions = 1;
 287     // Commute owner from a thread-specific on-stack BasicLockObject address to
 288     // a full-fledged "Thread *".
 289     _owner = Self;
 290     return;
 291   }
 292 
 293   // We've encountered genuine contention.
 294   assert(Self->_Stalled == 0, "invariant");
 295   Self->_Stalled = intptr_t(this);
 296 
 297   // Try one round of spinning *before* enqueueing Self
 298   // and before going through the awkward and expensive state
 299   // transitions.  The following spin is strictly optional ...
 300   // Note that if we acquire the monitor from an initial spin
 301   // we forgo posting JVMTI events and firing DTRACE probes.
 302   if (Knob_SpinEarly && TrySpin (Self) > 0) {
 303     assert(_owner == Self, "invariant");
 304     assert(_recursions == 0, "invariant");
 305     assert(((oop)(object()))->mark() == markOopDesc::encode(this), "invariant");
 306     Self->_Stalled = 0;
 307     return;
 308   }
 309 
 310   assert(_owner != Self, "invariant");
 311   assert(_succ != Self, "invariant");
 312   assert(Self->is_Java_thread(), "invariant");
 313   JavaThread * jt = (JavaThread *) Self;
 314   assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
 315   assert(jt->thread_state() != _thread_blocked, "invariant");
 316   assert(this->object() != NULL, "invariant");
 317   assert(_count >= 0, "invariant");
 318 
 319   // Prevent deflation at STW-time.  See deflate_idle_monitors() and is_busy().
 320   // Ensure the object-monitor relationship remains stable while there's contention.
 321   Atomic::inc(&_count);
 322 


 444   assert(Self->is_Java_thread(), "invariant");
 445   assert(((JavaThread *) Self)->thread_state() == _thread_blocked, "invariant");
 446 
 447   // Try the lock - TATAS
 448   if (TryLock (Self) > 0) {
 449     assert(_succ != Self, "invariant");
 450     assert(_owner == Self, "invariant");
 451     assert(_Responsible != Self, "invariant");
 452     return;
 453   }
 454 
 455   DeferredInitialize();
 456 
 457   // We try one round of spinning *before* enqueueing Self.
 458   //
 459   // If the _owner is ready but OFFPROC we could use a YieldTo()
 460   // operation to donate the remainder of this thread's quantum
 461   // to the owner.  This has subtle but beneficial affinity
 462   // effects.
 463 
 464   if (TrySpin (Self) > 0) {
 465     assert(_owner == Self, "invariant");
 466     assert(_succ != Self, "invariant");
 467     assert(_Responsible != Self, "invariant");
 468     return;
 469   }
 470 
 471   // The Spin failed -- Enqueue and park the thread ...
 472   assert(_succ != Self, "invariant");
 473   assert(_owner != Self, "invariant");
 474   assert(_Responsible != Self, "invariant");
 475 
 476   // Enqueue "Self" on ObjectMonitor's _cxq.
 477   //
 478   // Node acts as a proxy for Self.
 479   // As an aside, if were to ever rewrite the synchronization code mostly
 480   // in Java, WaitNodes, ObjectMonitors, and Events would become 1st-class
 481   // Java objects.  This would avoid awkward lifecycle and liveness issues,
 482   // as well as eliminate a subset of ABA issues.
 483   // TODO: eliminate ObjectWaiter and enqueue either Threads or Events.
 484 


 566       Self->_ParkEvent->park();
 567     }
 568 
 569     if (TryLock(Self) > 0) break;
 570 
 571     // The lock is still contested.
 572     // Keep a tally of the # of futile wakeups.
 573     // Note that the counter is not protected by a lock or updated by atomics.
 574     // That is by design - we trade "lossy" counters which are exposed to
 575     // races during updates for a lower probe effect.
 576 
 577     // This PerfData object can be used in parallel with a safepoint.
 578     // See the work around in PerfDataManager::destroy().
 579     OM_PERFDATA_OP(FutileWakeups, inc());
 580     ++nWakeups;
 581 
 582     // Assuming this is not a spurious wakeup we'll normally find _succ == Self.
 583     // We can defer clearing _succ until after the spin completes
 584     // TrySpin() must tolerate being called with _succ == Self.
 585     // Try yet another round of adaptive spinning.
 586     if ((Knob_SpinAfterFutile & 1) && TrySpin(Self) > 0) break;
 587 
 588     // We can find that we were unpark()ed and redesignated _succ while
 589     // we were spinning.  That's harmless.  If we iterate and call park(),
 590     // park() will consume the event and return immediately and we'll
 591     // just spin again.  This pattern can repeat, leaving _succ to simply
 592     // spin on a CPU.  Enable Knob_ResetEvent to clear pending unparks().
 593     // Alternately, we can sample fired() here, and if set, forgo spinning
 594     // in the next iteration.
 595 
 596     if ((Knob_ResetEvent & 1) && Self->_ParkEvent->fired()) {
 597       Self->_ParkEvent->reset();
 598       OrderAccess::fence();
 599     }
 600     if (_succ == Self) _succ = NULL;
 601 
 602     // Invariant: after clearing _succ a thread *must* retry _owner before parking.
 603     OrderAccess::fence();
 604   }
 605 
 606   // Egress :
 607   // Self has acquired the lock -- Unlink Self from the cxq or EntryList.
 608   // Normally we'll find Self on the EntryList .
 609   // From the perspective of the lock owner (this thread), the
 610   // EntryList is stable and cxq is prepend-only.
 611   // The head of cxq is volatile but the interior is stable.
 612   // In addition, Self.TState is stable.
 613 
 614   assert(_owner == Self, "invariant");
 615   assert(object() != NULL, "invariant");
 616   // I'd like to write:
 617   //   guarantee (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
 618   // but as we're at a safepoint that's not safe.
 619 


 658   // To that end, the 1-0 exit() operation must have at least STST|LDST
 659   // "release" barrier semantics.  Specifically, there must be at least a
 660   // STST|LDST barrier in exit() before the ST of null into _owner that drops
 661   // the lock.   The barrier ensures that changes to monitor meta-data and data
 662   // protected by the lock will be visible before we release the lock, and
 663   // therefore before some other thread (CPU) has a chance to acquire the lock.
 664   // See also: http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
 665   //
 666   // Critically, any prior STs to _succ or EntryList must be visible before
 667   // the ST of null into _owner in the *subsequent* (following) corresponding
 668   // monitorexit.  Recall too, that in 1-0 mode monitorexit does not necessarily
 669   // execute a serializing instruction.
 670 
 671   return;
 672 }
 673 
 674 // ReenterI() is a specialized inline form of the latter half of the
 675 // contended slow-path from EnterI().  We use ReenterI() only for
 676 // monitor reentry in wait().
 677 //
 678 // In the future we should reconcile EnterI() and ReenterI(), adding
 679 // Knob_Reset and Knob_SpinAfterFutile support and restructuring the
 680 // loop accordingly.
 681 
 682 void ObjectMonitor::ReenterI(Thread * Self, ObjectWaiter * SelfNode) {
 683   assert(Self != NULL, "invariant");
 684   assert(SelfNode != NULL, "invariant");
 685   assert(SelfNode->_thread == Self, "invariant");
 686   assert(_waiters > 0, "invariant");
 687   assert(((oop)(object()))->mark() == markOopDesc::encode(this), "invariant");
 688   assert(((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant");
 689   JavaThread * jt = (JavaThread *) Self;
 690 
 691   int nWakeups = 0;
 692   for (;;) {
 693     ObjectWaiter::TStates v = SelfNode->TState;
 694     guarantee(v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant");
 695     assert(_owner != Self, "invariant");
 696 
 697     if (TryLock(Self) > 0) break;
 698     if (TrySpin(Self) > 0) break;
 699 
 700     // State transition wrappers around park() ...


 912   if (_recursions != 0) {
 913     _recursions--;        // this is simple recursive enter
 914     return;
 915   }
 916 
 917   // Invariant: after setting Responsible=null an thread must execute
 918   // a MEMBAR or other serializing instruction before fetching EntryList|cxq.
 919   _Responsible = NULL;
 920 
 921 #if INCLUDE_JFR
 922   // get the owner's thread id for the MonitorEnter event
 923   // if it is enabled and the thread isn't suspended
 924   if (not_suspended && EventJavaMonitorEnter::is_enabled()) {
 925     _previous_owner_tid = JFR_THREAD_ID(Self);
 926   }
 927 #endif
 928 
 929   for (;;) {
 930     assert(THREAD == _owner, "invariant");
 931 
 932     if (Knob_ExitPolicy == 0) {
 933       // release semantics: prior loads and stores from within the critical section
 934       // must not float (reorder) past the following store that drops the lock.
 935       // On SPARC that requires MEMBAR #loadstore|#storestore.
 936       // But of course in TSO #loadstore|#storestore is not required.
 937       // I'd like to write one of the following:
 938       // A.  OrderAccess::release() ; _owner = NULL
 939       // B.  OrderAccess::loadstore(); OrderAccess::storestore(); _owner = NULL;
 940       // Unfortunately OrderAccess::release() and OrderAccess::loadstore() both
 941       // store into a _dummy variable.  That store is not needed, but can result
 942       // in massive wasteful coherency traffic on classic SMP systems.
 943       // Instead, I use release_store(), which is implemented as just a simple
 944       // ST on x64, x86 and SPARC.
 945       OrderAccess::release_store(&_owner, (void*)NULL);   // drop the lock
 946       OrderAccess::storeload();                        // See if we need to wake a successor
 947       if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) {
 948         return;
 949       }
 950       // Other threads are blocked trying to acquire the lock.
 951 
 952       // Normally the exiting thread is responsible for ensuring succession,


 971       // Another less appealing alternative would be for the exiting thread
 972       // to drop the lock and then spin briefly to see if a spinner managed
 973       // to acquire the lock.  If so, the exiting thread could exit
 974       // immediately without waking a successor, otherwise the exiting
 975       // thread would need to dequeue and wake a successor.
 976       // (Note that we'd need to make the post-drop spin short, but no
 977       // shorter than the worst-case round-trip cache-line migration time.
 978       // The dropped lock needs to become visible to the spinner, and then
 979       // the acquisition of the lock by the spinner must become visible to
 980       // the exiting thread).
 981 
 982       // It appears that an heir-presumptive (successor) must be made ready.
 983       // Only the current lock owner can manipulate the EntryList or
 984       // drain _cxq, so we need to reacquire the lock.  If we fail
 985       // to reacquire the lock the responsibility for ensuring succession
 986       // falls to the new owner.
 987       //
 988       if (!Atomic::replace_if_null(THREAD, &_owner)) {
 989         return;
 990       }
 991     } else {
 992       if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) {
 993         OrderAccess::release_store(&_owner, (void*)NULL);   // drop the lock
 994         OrderAccess::storeload();
 995         // Ratify the previously observed values.
 996         if (_cxq == NULL || _succ != NULL) {
 997           return;
 998         }
 999 
1000         // inopportune interleaving -- the exiting thread (this thread)
1001         // in the fast-exit path raced an entering thread in the slow-enter
1002         // path.
1003         // We have two choices:
1004         // A.  Try to reacquire the lock.
1005         //     If the CAS() fails return immediately, otherwise
1006         //     we either restart/rerun the exit operation, or simply
1007         //     fall-through into the code below which wakes a successor.
1008         // B.  If the elements forming the EntryList|cxq are TSM
1009         //     we could simply unpark() the lead thread and return
1010         //     without having set _succ.
1011         if (!Atomic::replace_if_null(THREAD, &_owner)) {
1012           return;
1013         }
1014       }
1015     }
1016 
1017     guarantee(_owner == THREAD, "invariant");
1018 
1019     ObjectWaiter * w = NULL;
1020     int QMode = Knob_QMode;
1021 
1022     if (QMode == 2 && _cxq != NULL) {
1023       // QMode == 2 : cxq has precedence over EntryList.
1024       // Try to directly wake a successor from the cxq.
1025       // If successful, the successor will need to unlink itself from cxq.
1026       w = _cxq;
1027       assert(w != NULL, "invariant");
1028       assert(w->TState == ObjectWaiter::TS_CXQ, "Invariant");
1029       ExitEpilog(Self, w);
1030       return;
1031     }
1032 
1033     if (QMode == 3 && _cxq != NULL) {
1034       // Aggressively drain cxq into EntryList at the first opportunity.
1035       // This policy ensure that recently-run threads live at the head of EntryList.
1036       // Drain _cxq into EntryList - bulk transfer.
1037       // First, detach _cxq.
1038       // The following loop is tantamount to: w = swap(&cxq, NULL)
1039       w = _cxq;
1040       for (;;) {
1041         assert(w != NULL, "Invariant");
1042         ObjectWaiter * u = Atomic::cmpxchg((ObjectWaiter*)NULL, &_cxq, w);
1043         if (u == w) break;
1044         w = u;
1045       }
1046       assert(w != NULL, "invariant");
1047 
1048       ObjectWaiter * q = NULL;
1049       ObjectWaiter * p;
1050       for (p = w; p != NULL; p = p->_next) {
1051         guarantee(p->TState == ObjectWaiter::TS_CXQ, "Invariant");
1052         p->TState = ObjectWaiter::TS_ENTER;
1053         p->_prev = q;
1054         q = p;
1055       }
1056 
1057       // Append the RATs to the EntryList
1058       // TODO: organize EntryList as a CDLL so we can locate the tail in constant-time.
1059       ObjectWaiter * Tail;
1060       for (Tail = _EntryList; Tail != NULL && Tail->_next != NULL;
1061            Tail = Tail->_next)
1062         /* empty */;
1063       if (Tail == NULL) {
1064         _EntryList = w;
1065       } else {
1066         Tail->_next = w;
1067         w->_prev = Tail;
1068       }
1069 
1070       // Fall thru into code that tries to wake a successor from EntryList
1071     }
1072 
1073     if (QMode == 4 && _cxq != NULL) {
1074       // Aggressively drain cxq into EntryList at the first opportunity.
1075       // This policy ensure that recently-run threads live at the head of EntryList.
1076 
1077       // Drain _cxq into EntryList - bulk transfer.
1078       // First, detach _cxq.
1079       // The following loop is tantamount to: w = swap(&cxq, NULL)
1080       w = _cxq;
1081       for (;;) {
1082         assert(w != NULL, "Invariant");
1083         ObjectWaiter * u = Atomic::cmpxchg((ObjectWaiter*)NULL, &_cxq, w);
1084         if (u == w) break;
1085         w = u;
1086       }
1087       assert(w != NULL, "invariant");
1088 
1089       ObjectWaiter * q = NULL;
1090       ObjectWaiter * p;
1091       for (p = w; p != NULL; p = p->_next) {
1092         guarantee(p->TState == ObjectWaiter::TS_CXQ, "Invariant");
1093         p->TState = ObjectWaiter::TS_ENTER;
1094         p->_prev = q;
1095         q = p;
1096       }
1097 
1098       // Prepend the RATs to the EntryList
1099       if (_EntryList != NULL) {
1100         q->_next = _EntryList;
1101         _EntryList->_prev = q;
1102       }
1103       _EntryList = w;
1104 
1105       // Fall thru into code that tries to wake a successor from EntryList
1106     }
1107 
1108     w = _EntryList;
1109     if (w != NULL) {
1110       // I'd like to write: guarantee (w->_thread != Self).
1111       // But in practice an exiting thread may find itself on the EntryList.
1112       // Let's say thread T1 calls O.wait().  Wait() enqueues T1 on O's waitset and
1113       // then calls exit().  Exit release the lock by setting O._owner to NULL.
1114       // Let's say T1 then stalls.  T2 acquires O and calls O.notify().  The
1115       // notify() operation moves T1 from O's waitset to O's EntryList. T2 then
1116       // release the lock "O".  T2 resumes immediately after the ST of null into
1117       // _owner, above.  T2 notices that the EntryList is populated, so it
1118       // reacquires the lock and then finds itself on the EntryList.
1119       // Given all that, we have to tolerate the circumstance where "w" is
1120       // associated with Self.
1121       assert(w->TState == ObjectWaiter::TS_ENTER, "invariant");
1122       ExitEpilog(Self, w);
1123       return;
1124     }
1125 
1126     // If we find that both _cxq and EntryList are null then just


1133     // The following loop is tantamount to: w = swap(&cxq, NULL)
1134     for (;;) {
1135       assert(w != NULL, "Invariant");
1136       ObjectWaiter * u = Atomic::cmpxchg((ObjectWaiter*)NULL, &_cxq, w);
1137       if (u == w) break;
1138       w = u;
1139     }
1140 
1141     assert(w != NULL, "invariant");
1142     assert(_EntryList == NULL, "invariant");
1143 
1144     // Convert the LIFO SLL anchored by _cxq into a DLL.
1145     // The list reorganization step operates in O(LENGTH(w)) time.
1146     // It's critical that this step operate quickly as
1147     // "Self" still holds the outer-lock, restricting parallelism
1148     // and effectively lengthening the critical section.
1149     // Invariant: s chases t chases u.
1150     // TODO-FIXME: consider changing EntryList from a DLL to a CDLL so
1151     // we have faster access to the tail.
1152 
1153     if (QMode == 1) {
1154       // QMode == 1 : drain cxq to EntryList, reversing order
1155       // We also reverse the order of the list.
1156       ObjectWaiter * s = NULL;
1157       ObjectWaiter * t = w;
1158       ObjectWaiter * u = NULL;
1159       while (t != NULL) {
1160         guarantee(t->TState == ObjectWaiter::TS_CXQ, "invariant");
1161         t->TState = ObjectWaiter::TS_ENTER;
1162         u = t->_next;
1163         t->_prev = u;
1164         t->_next = s;
1165         s = t;
1166         t = u;
1167       }
1168       _EntryList  = s;
1169       assert(s != NULL, "invariant");
1170     } else {
1171       // QMode == 0 or QMode == 2
1172       _EntryList = w;
1173       ObjectWaiter * q = NULL;
1174       ObjectWaiter * p;
1175       for (p = w; p != NULL; p = p->_next) {
1176         guarantee(p->TState == ObjectWaiter::TS_CXQ, "Invariant");
1177         p->TState = ObjectWaiter::TS_ENTER;
1178         p->_prev = q;
1179         q = p;
1180       }
1181     }
1182 
1183     // In 1-0 mode we need: ST EntryList; MEMBAR #storestore; ST _owner = NULL
1184     // The MEMBAR is satisfied by the release_store() operation in ExitEpilog().
1185 
1186     // See if we can abdicate to a spinner instead of waking a thread.
1187     // A primary goal of the implementation is to reduce the
1188     // context-switch rate.
1189     if (_succ != NULL) continue;
1190 
1191     w = _EntryList;
1192     if (w != NULL) {
1193       guarantee(w->TState == ObjectWaiter::TS_ENTER, "invariant");
1194       ExitEpilog(Self, w);
1195       return;
1196     }
1197   }
1198 }
1199 
1200 // ExitSuspendEquivalent:
1201 // A faster alternate to handle_special_suspend_equivalent_condition()


1209 //
1210 // A.  To ameliorate the problem we might also defer state transitions
1211 //     to as late as possible -- just prior to parking.
1212 //     Given that, we'd call HSSEC after having returned from park(),
1213 //     but before attempting to acquire the monitor.  This is only a
1214 //     partial solution.  It avoids calling HSSEC while holding the
1215 //     monitor (good), but it still increases successor reacquisition latency --
1216 //     the interval between unparking a successor and the time the successor
1217 //     resumes and retries the lock.  See ReenterI(), which defers state transitions.
1218 //     If we use this technique we can also avoid EnterI()-exit() loop
1219 //     in ::enter() where we iteratively drop the lock and then attempt
1220 //     to reacquire it after suspending.
1221 //
1222 // B.  In the future we might fold all the suspend bits into a
1223 //     composite per-thread suspend flag and then update it with CAS().
1224 //     Alternately, a Dekker-like mechanism with multiple variables
1225 //     would suffice:
1226 //       ST Self->_suspend_equivalent = false
1227 //       MEMBAR
1228 //       LD Self_>_suspend_flags
1229 //
1230 // UPDATE 2007-10-6: since I've replaced the native Mutex/Monitor subsystem
1231 // with a more efficient implementation, the need to use "FastHSSEC" has
1232 // decreased. - Dave
1233 
1234 
1235 bool ObjectMonitor::ExitSuspendEquivalent(JavaThread * jSelf) {
1236   const int Mode = Knob_FastHSSEC;
1237   if (Mode && !jSelf->is_external_suspend()) {
1238     assert(jSelf->is_suspend_equivalent(), "invariant");
1239     jSelf->clear_suspend_equivalent();
1240     if (2 == Mode) OrderAccess::storeload();
1241     if (!jSelf->is_external_suspend()) return false;
1242     // We raced a suspension -- fall thru into the slow path
1243     jSelf->set_suspend_equivalent();
1244   }
1245   return jSelf->handle_special_suspend_equivalent_condition();
1246 }
1247 
1248 
1249 void ObjectMonitor::ExitEpilog(Thread * Self, ObjectWaiter * Wakee) {
1250   assert(_owner == Self, "invariant");
1251 
1252   // Exit protocol:
1253   // 1. ST _succ = wakee
1254   // 2. membar #loadstore|#storestore;
1255   // 2. ST _owner = NULL
1256   // 3. unpark(wakee)
1257 
1258   _succ = Knob_SuccEnabled ? Wakee->_thread : NULL;
1259   ParkEvent * Trigger = Wakee->_event;
1260 
1261   // Hygiene -- once we've set _owner = NULL we can't safely dereference Wakee again.
1262   // The thread associated with Wakee may have grabbed the lock and "Wakee" may be
1263   // out-of-scope (non-extant).
1264   Wakee  = NULL;
1265 
1266   // Drop the lock
1267   OrderAccess::release_store(&_owner, (void*)NULL);
1268   OrderAccess::fence();                               // ST _owner vs LD in unpark()
1269 
1270   DTRACE_MONITOR_PROBE(contended__exit, this, object(), Self);
1271   Trigger->unpark();
1272 
1273   // Maintain stats and report events to JVMTI
1274   OM_PERFDATA_OP(Parks, inc());
1275 }
1276 
1277 
1278 // -----------------------------------------------------------------------------


1331 #define CHECK_OWNER()                                                       \
1332   do {                                                                      \
1333     if (THREAD != _owner) {                                                 \
1334       if (THREAD->is_lock_owned((address) _owner)) {                        \
1335         _owner = THREAD;  /* Convert from basiclock addr to Thread addr */  \
1336         _recursions = 0;                                                    \
1337       } else {                                                              \
1338         THROW(vmSymbols::java_lang_IllegalMonitorStateException());         \
1339       }                                                                     \
1340     }                                                                       \
1341   } while (false)
1342 
1343 // check_slow() is a misnomer.  It's called to simply to throw an IMSX exception.
1344 // TODO-FIXME: remove check_slow() -- it's likely dead.
1345 
1346 void ObjectMonitor::check_slow(TRAPS) {
1347   assert(THREAD != _owner && !THREAD->is_lock_owned((address) _owner), "must not be owner");
1348   THROW_MSG(vmSymbols::java_lang_IllegalMonitorStateException(), "current thread not owner");
1349 }
1350 
1351 static int Adjust(volatile int * adr, int dx) {
1352   int v;
1353   for (v = *adr; Atomic::cmpxchg(v + dx, adr, v) != v; v = *adr) /* empty */;
1354   return v;
1355 }
1356 
1357 static void post_monitor_wait_event(EventJavaMonitorWait* event,
1358                                     ObjectMonitor* monitor,
1359                                     jlong notifier_tid,
1360                                     jlong timeout,
1361                                     bool timedout) {
1362   assert(event != NULL, "invariant");
1363   assert(monitor != NULL, "invariant");
1364   event->set_monitorClass(((oop)monitor->object())->klass());
1365   event->set_timeout(timeout);
1366   event->set_address((uintptr_t)monitor->object_addr());
1367   event->set_notifier(notifier_tid);
1368   event->set_timedOut(timedout);
1369   event->commit();
1370 }
1371 
1372 // -----------------------------------------------------------------------------
1373 // Wait/Notify/NotifyAll
1374 //
1375 // Note: a subset of changes to ObjectMonitor::wait()
1376 // will need to be replicated in complete_exit


1582   // check if the notification happened
1583   if (!WasNotified) {
1584     // no, it could be timeout or Thread.interrupt() or both
1585     // check for interrupt event, otherwise it is timeout
1586     if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) {
1587       THROW(vmSymbols::java_lang_InterruptedException());
1588     }
1589   }
1590 
1591   // NOTE: Spurious wake up will be consider as timeout.
1592   // Monitor notify has precedence over thread interrupt.
1593 }
1594 
1595 
1596 // Consider:
1597 // If the lock is cool (cxq == null && succ == null) and we're on an MP system
1598 // then instead of transferring a thread from the WaitSet to the EntryList
1599 // we might just dequeue a thread from the WaitSet and directly unpark() it.
1600 
1601 void ObjectMonitor::INotify(Thread * Self) {
1602   const int policy = Knob_MoveNotifyee;
1603 
1604   Thread::SpinAcquire(&_WaitSetLock, "WaitSet - notify");
1605   ObjectWaiter * iterator = DequeueWaiter();
1606   if (iterator != NULL) {
1607     guarantee(iterator->TState == ObjectWaiter::TS_WAIT, "invariant");
1608     guarantee(iterator->_notified == 0, "invariant");
1609     // Disposition - what might we do with iterator ?
1610     // a.  add it directly to the EntryList - either tail (policy == 1)
1611     //     or head (policy == 0).
1612     // b.  push it onto the front of the _cxq (policy == 2).
1613     // For now we use (b).
1614     if (policy != 4) {
1615       iterator->TState = ObjectWaiter::TS_ENTER;
1616     }
1617     iterator->_notified = 1;
1618     iterator->_notifier_tid = JFR_THREAD_ID(Self);
1619 
1620     ObjectWaiter * list = _EntryList;
1621     if (list != NULL) {
1622       assert(list->_prev == NULL, "invariant");
1623       assert(list->TState == ObjectWaiter::TS_ENTER, "invariant");
1624       assert(list != iterator, "invariant");
1625     }
1626 
1627     if (policy == 0) {       // prepend to EntryList
1628       if (list == NULL) {
1629         iterator->_next = iterator->_prev = NULL;
1630         _EntryList = iterator;
1631       } else {
1632         list->_prev = iterator;
1633         iterator->_next = list;
1634         iterator->_prev = NULL;
1635         _EntryList = iterator;
1636       }
1637     } else if (policy == 1) {      // append to EntryList
1638       if (list == NULL) {
1639         iterator->_next = iterator->_prev = NULL;
1640         _EntryList = iterator;
1641       } else {
1642         // CONSIDER:  finding the tail currently requires a linear-time walk of
1643         // the EntryList.  We can make tail access constant-time by converting to
1644         // a CDLL instead of using our current DLL.
1645         ObjectWaiter * tail;
1646         for (tail = list; tail->_next != NULL; tail = tail->_next) {}
1647         assert(tail != NULL && tail->_next == NULL, "invariant");
1648         tail->_next = iterator;
1649         iterator->_prev = tail;
1650         iterator->_next = NULL;
1651       }
1652     } else if (policy == 2) {      // prepend to cxq
1653       if (list == NULL) {
1654         iterator->_next = iterator->_prev = NULL;
1655         _EntryList = iterator;
1656       } else {
1657         iterator->TState = ObjectWaiter::TS_CXQ;
1658         for (;;) {
1659           ObjectWaiter * front = _cxq;
1660           iterator->_next = front;
1661           if (Atomic::cmpxchg(iterator, &_cxq, front) == front) {
1662             break;
1663           }
1664         }
1665       }
1666     } else if (policy == 3) {      // append to cxq
1667       iterator->TState = ObjectWaiter::TS_CXQ;
1668       for (;;) {
1669         ObjectWaiter * tail = _cxq;
1670         if (tail == NULL) {
1671           iterator->_next = NULL;
1672           if (Atomic::replace_if_null(iterator, &_cxq)) {
1673             break;
1674           }
1675         } else {
1676           while (tail->_next != NULL) tail = tail->_next;
1677           tail->_next = iterator;
1678           iterator->_prev = tail;
1679           iterator->_next = NULL;
1680           break;
1681         }
1682       }
1683     } else {
1684       ParkEvent * ev = iterator->_event;
1685       iterator->TState = ObjectWaiter::TS_RUN;
1686       OrderAccess::fence();
1687       ev->unpark();
1688     }
1689 
1690     // _WaitSetLock protects the wait queue, not the EntryList.  We could
1691     // move the add-to-EntryList operation, above, outside the critical section
1692     // protected by _WaitSetLock.  In practice that's not useful.  With the
1693     // exception of  wait() timeouts and interrupts the monitor owner
1694     // is the only thread that grabs _WaitSetLock.  There's almost no contention
1695     // on _WaitSetLock so it's not profitable to reduce the length of the
1696     // critical section.
1697 
1698     if (policy < 4) {
1699       iterator->wait_reenter_begin(this);
1700     }
1701   }
1702   Thread::SpinRelease(&_WaitSetLock);
1703 }
1704 
1705 // Consider: a not-uncommon synchronization bug is to use notify() when
1706 // notifyAll() is more appropriate, potentially resulting in stranded
1707 // threads; this is one example of a lost wakeup. A useful diagnostic
1708 // option is to force all notify() operations to behave as notifyAll().
1709 //
1710 // Note: We can also detect many such problems with a "minimum wait".
1711 // When the "minimum wait" is set to a small non-zero timeout value
1712 // and the program does not hang whereas it did absent "minimum wait",
1713 // that suggests a lost wakeup bug.
1714 
1715 void ObjectMonitor::notify(TRAPS) {
1716   CHECK_OWNER();
1717   if (_WaitSet == NULL) {
1718     return;
1719   }
1720   DTRACE_MONITOR_PROBE(notify, this, object(), THREAD);
1721   INotify(THREAD);


1837       return 1;
1838     }
1839     SpinPause();
1840   }
1841 
1842   // Admission control - verify preconditions for spinning
1843   //
1844   // We always spin a little bit, just to prevent _SpinDuration == 0 from
1845   // becoming an absorbing state.  Put another way, we spin briefly to
1846   // sample, just in case the system load, parallelism, contention, or lock
1847   // modality changed.
1848   //
1849   // Consider the following alternative:
1850   // Periodically set _SpinDuration = _SpinLimit and try a long/full
1851   // spin attempt.  "Periodically" might mean after a tally of
1852   // the # of failed spin attempts (or iterations) reaches some threshold.
1853   // This takes us into the realm of 1-out-of-N spinning, where we
1854   // hold the duration constant but vary the frequency.
1855 
1856   ctr = _SpinDuration;
1857   if (ctr < Knob_SpinBase) ctr = Knob_SpinBase;
1858   if (ctr <= 0) return 0;
1859 
1860   if (Knob_SuccRestrict && _succ != NULL) return 0;
1861   if (Knob_OState && NotRunnable (Self, (Thread *) _owner)) {
1862     return 0;
1863   }
1864 
1865   int MaxSpin = Knob_MaxSpinners;
1866   if (MaxSpin >= 0) {
1867     if (_Spinner > MaxSpin) {
1868       return 0;
1869     }
1870     // Slightly racy, but benign ...
1871     Adjust(&_Spinner, 1);
1872   }
1873 
1874   // We're good to spin ... spin ingress.
1875   // CONSIDER: use Prefetch::write() to avoid RTS->RTO upgrades
1876   // when preparing to LD...CAS _owner, etc and the CAS is likely
1877   // to succeed.
1878   int hits    = 0;
1879   int msk     = 0;
1880   int caspty  = Knob_CASPenalty;
1881   int oxpty   = Knob_OXPenalty;
1882   int sss     = Knob_SpinSetSucc;
1883   if (sss && _succ == NULL) _succ = Self;
1884   Thread * prv = NULL;
1885 
1886   // There are three ways to exit the following loop:
1887   // 1.  A successful spin where this thread has acquired the lock.
1888   // 2.  Spin failure with prejudice
1889   // 3.  Spin failure without prejudice
1890 
1891   while (--ctr >= 0) {
1892 
1893     // Periodic polling -- Check for pending GC
1894     // Threads may spin while they're unsafe.
1895     // We don't want spinning threads to delay the JVM from reaching
1896     // a stop-the-world safepoint or to steal cycles from GC.
1897     // If we detect a pending safepoint we abort in order that
1898     // (a) this thread, if unsafe, doesn't delay the safepoint, and (b)
1899     // this thread, if safe, doesn't steal cycles from GC.
1900     // This is in keeping with the "no loitering in runtime" rule.
1901     // We periodically check to see if there's a safepoint pending.
1902     if ((ctr & 0xFF) == 0) {
1903       if (SafepointMechanism::poll(Self)) {
1904         goto Abort;           // abrupt spin egress
1905       }
1906       if (Knob_UsePause & 1) SpinPause();
1907     }
1908 
1909     if (Knob_UsePause & 2) SpinPause();
1910 
1911     // Exponential back-off ...  Stay off the bus to reduce coherency traffic.
1912     // This is useful on classic SMP systems, but is of less utility on
1913     // N1-style CMT platforms.
1914     //
1915     // Trade-off: lock acquisition latency vs coherency bandwidth.
1916     // Lock hold times are typically short.  A histogram
1917     // of successful spin attempts shows that we usually acquire
1918     // the lock early in the spin.  That suggests we want to
1919     // sample _owner frequently in the early phase of the spin,
1920     // but then back-off and sample less frequently as the spin
1921     // progresses.  The back-off makes a good citizen on SMP big
1922     // SMP systems.  Oversampling _owner can consume excessive
1923     // coherency bandwidth.  Relatedly, if we _oversample _owner we
1924     // can inadvertently interfere with the the ST m->owner=null.
1925     // executed by the lock owner.
1926     if (ctr & msk) continue;
1927     ++hits;
1928     if ((hits & 0xF) == 0) {
1929       // The 0xF, above, corresponds to the exponent.
1930       // Consider: (msk+1)|msk
1931       msk = ((msk << 2)|3) & BackOffMask;
1932     }
1933 
1934     // Probe _owner with TATAS
1935     // If this thread observes the monitor transition or flicker
1936     // from locked to unlocked to locked, then the odds that this
1937     // thread will acquire the lock in this spin attempt go down
1938     // considerably.  The same argument applies if the CAS fails
1939     // or if we observe _owner change from one non-null value to
1940     // another non-null value.   In such cases we might abort
1941     // the spin without prejudice or apply a "penalty" to the
1942     // spin count-down variable "ctr", reducing it by 100, say.
1943 
1944     Thread * ox = (Thread *) _owner;
1945     if (ox == NULL) {
1946       ox = (Thread*)Atomic::cmpxchg(Self, &_owner, (void*)NULL);
1947       if (ox == NULL) {
1948         // The CAS succeeded -- this thread acquired ownership
1949         // Take care of some bookkeeping to exit spin state.
1950         if (sss && _succ == Self) {
1951           _succ = NULL;
1952         }
1953         if (MaxSpin > 0) Adjust(&_Spinner, -1);
1954 
1955         // Increase _SpinDuration :
1956         // The spin was successful (profitable) so we tend toward
1957         // longer spin attempts in the future.
1958         // CONSIDER: factor "ctr" into the _SpinDuration adjustment.
1959         // If we acquired the lock early in the spin cycle it
1960         // makes sense to increase _SpinDuration proportionally.
1961         // Note that we don't clamp SpinDuration precisely at SpinLimit.
1962         int x = _SpinDuration;
1963         if (x < Knob_SpinLimit) {
1964           if (x < Knob_Poverty) x = Knob_Poverty;
1965           _SpinDuration = x + Knob_Bonus;
1966         }
1967         return 1;
1968       }
1969 
1970       // The CAS failed ... we can take any of the following actions:
1971       // * penalize: ctr -= Knob_CASPenalty
1972       // * exit spin with prejudice -- goto Abort;
1973       // * exit spin without prejudice.
1974       // * Since CAS is high-latency, retry again immediately.
1975       prv = ox;
1976       if (caspty == -2) break;
1977       if (caspty == -1) goto Abort;
1978       ctr -= caspty;
1979       continue;
1980     }
1981 
1982     // Did lock ownership change hands ?
1983     if (ox != prv && prv != NULL) {
1984       if (oxpty == -2) break;
1985       if (oxpty == -1) goto Abort;
1986       ctr -= oxpty;
1987     }
1988     prv = ox;
1989 
1990     // Abort the spin if the owner is not executing.
1991     // The owner must be executing in order to drop the lock.
1992     // Spinning while the owner is OFFPROC is idiocy.
1993     // Consider: ctr -= RunnablePenalty ;
1994     if (Knob_OState && NotRunnable (Self, ox)) {
1995       goto Abort;
1996     }
1997     if (sss && _succ == NULL) _succ = Self;


1998   }
1999 
2000   // Spin failed with prejudice -- reduce _SpinDuration.
2001   // TODO: Use an AIMD-like policy to adjust _SpinDuration.
2002   // AIMD is globally stable.
2003   {
2004     int x = _SpinDuration;
2005     if (x > 0) {
2006       // Consider an AIMD scheme like: x -= (x >> 3) + 100
2007       // This is globally sample and tends to damp the response.
2008       x -= Knob_Penalty;
2009       if (x < 0) x = 0;
2010       _SpinDuration = x;
2011     }
2012   }
2013 
2014  Abort:
2015   if (MaxSpin >= 0) Adjust(&_Spinner, -1);
2016   if (sss && _succ == Self) {
2017     _succ = NULL;
2018     // Invariant: after setting succ=null a contending thread
2019     // must recheck-retry _owner before parking.  This usually happens
2020     // in the normal usage of TrySpin(), but it's safest
2021     // to make TrySpin() as foolproof as possible.
2022     OrderAccess::fence();
2023     if (TryLock(Self) > 0) return 1;
2024   }
2025   return 0;
2026 }
2027 
2028 // NotRunnable() -- informed spinning
2029 //
2030 // Don't bother spinning if the owner is not eligible to drop the lock.
2031 // Spin only if the owner thread is _thread_in_Java or _thread_in_vm.
2032 // The thread must be runnable in order to drop the lock in timely fashion.
2033 // If the _owner is not runnable then spinning will not likely be
2034 // successful (profitable).
2035 //
2036 // Beware -- the thread referenced by _owner could have died


2187     n = PerfDataManager::create_counter(SUN_RT, #n, PerfData::U_Events,  \
2188                                         CHECK);                          \
2189   }
2190 #define NEWPERFVARIABLE(n)                                                \
2191   {                                                                       \
2192     n = PerfDataManager::create_variable(SUN_RT, #n, PerfData::U_Events,  \
2193                                          CHECK);                          \
2194   }
2195     NEWPERFCOUNTER(_sync_Inflations);
2196     NEWPERFCOUNTER(_sync_Deflations);
2197     NEWPERFCOUNTER(_sync_ContendedLockAttempts);
2198     NEWPERFCOUNTER(_sync_FutileWakeups);
2199     NEWPERFCOUNTER(_sync_Parks);
2200     NEWPERFCOUNTER(_sync_Notifications);
2201     NEWPERFVARIABLE(_sync_MonExtant);
2202 #undef NEWPERFCOUNTER
2203 #undef NEWPERFVARIABLE
2204   }
2205 }
2206 
2207 static char * kvGet(char * kvList, const char * Key) {
2208   if (kvList == NULL) return NULL;
2209   size_t n = strlen(Key);
2210   char * Search;
2211   for (Search = kvList; *Search; Search += strlen(Search) + 1) {
2212     if (strncmp (Search, Key, n) == 0) {
2213       if (Search[n] == '=') return Search + n + 1;
2214       if (Search[n] == 0)   return(char *) "1";
2215     }
2216   }
2217   return NULL;
2218 }
2219 
2220 static int kvGetInt(char * kvList, const char * Key, int Default) {
2221   char * v = kvGet(kvList, Key);
2222   int rslt = v ? ::strtol(v, NULL, 0) : Default;
2223   if (Knob_ReportSettings && v != NULL) {
2224     tty->print_cr("INFO: SyncKnob: %s %d(%d)", Key, rslt, Default) ;
2225     tty->flush();
2226   }
2227   return rslt;
2228 }
2229 
2230 void ObjectMonitor::DeferredInitialize() {
2231   if (InitDone > 0) return;
2232   if (Atomic::cmpxchg (-1, &InitDone, 0) != 0) {
2233     while (InitDone != 1) /* empty */;
2234     return;
2235   }
2236 
2237   // One-shot global initialization ...
2238   // The initialization is idempotent, so we don't need locks.
2239   // In the future consider doing this via os::init_2().
2240   // SyncKnobs consist of <Key>=<Value> pairs in the style
2241   // of environment variables.  Start by converting ':' to NUL.
2242 
2243   if (SyncKnobs == NULL) SyncKnobs = "";
2244 
2245   size_t sz = strlen(SyncKnobs);
2246   char * knobs = (char *) os::malloc(sz + 2, mtInternal);
2247   if (knobs == NULL) {
2248     vm_exit_out_of_memory(sz + 2, OOM_MALLOC_ERROR, "Parse SyncKnobs");
2249     guarantee(0, "invariant");
2250   }
2251   strcpy(knobs, SyncKnobs);
2252   knobs[sz+1] = 0;
2253   for (char * p = knobs; *p; p++) {
2254     if (*p == ':') *p = 0;
2255   }
2256 
2257   #define SETKNOB(x) { Knob_##x = kvGetInt(knobs, #x, Knob_##x); }
2258   SETKNOB(ReportSettings);
2259   SETKNOB(ExitRelease);
2260   SETKNOB(InlineNotify);
2261   SETKNOB(Verbose);
2262   SETKNOB(VerifyInUse);
2263   SETKNOB(VerifyMatch);
2264   SETKNOB(FixedSpin);
2265   SETKNOB(SpinLimit);
2266   SETKNOB(SpinBase);
2267   SETKNOB(SpinBackOff);
2268   SETKNOB(CASPenalty);
2269   SETKNOB(OXPenalty);
2270   SETKNOB(SpinSetSucc);
2271   SETKNOB(SuccEnabled);
2272   SETKNOB(SuccRestrict);
2273   SETKNOB(Penalty);
2274   SETKNOB(Bonus);
2275   SETKNOB(BonusB);
2276   SETKNOB(Poverty);
2277   SETKNOB(SpinAfterFutile);
2278   SETKNOB(UsePause);
2279   SETKNOB(SpinEarly);
2280   SETKNOB(OState);
2281   SETKNOB(MaxSpinners);
2282   SETKNOB(PreSpin);
2283   SETKNOB(ExitPolicy);
2284   SETKNOB(QMode);
2285   SETKNOB(ResetEvent);
2286   SETKNOB(MoveNotifyee);
2287   SETKNOB(FastHSSEC);
2288   #undef SETKNOB
2289 
2290   if (os::is_MP()) {
2291     BackOffMask = (1 << Knob_SpinBackOff) - 1;
2292     if (Knob_ReportSettings) {
2293       tty->print_cr("INFO: BackOffMask=0x%X", BackOffMask);
2294     }
2295     // CONSIDER: BackOffMask = ROUNDUP_NEXT_POWER2 (ncpus-1)
2296   } else {
2297     Knob_SpinLimit = 0;
2298     Knob_SpinBase  = 0;
2299     Knob_PreSpin   = 0;
2300     Knob_FixedSpin = -1;
2301   }
2302 
2303   os::free(knobs);
2304   OrderAccess::fence();
2305   InitDone = 1;
2306 }
2307 


  84 #define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread)                  \
  85   {                                                                        \
  86     if (DTraceMonitorProbes) {                                             \
  87       DTRACE_MONITOR_PROBE_COMMON(obj, thread);                            \
  88       HOTSPOT_MONITOR_##probe(jtid,                                        \
  89                               (uintptr_t)(monitor), bytes, len);           \
  90     }                                                                      \
  91   }
  92 
  93 #else //  ndef DTRACE_ENABLED
  94 
  95 #define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon)    {;}
  96 #define DTRACE_MONITOR_PROBE(probe, obj, thread, mon)          {;}
  97 
  98 #endif // ndef DTRACE_ENABLED
  99 
 100 // Tunables ...
 101 // The knob* variables are effectively final.  Once set they should
 102 // never be modified hence.  Consider using __read_mostly with GCC.
 103 





 104 int ObjectMonitor::Knob_SpinLimit    = 5000;    // derived by an external tool -
 105 










 106 static int Knob_Bonus               = 100;     // spin success bonus
 107 static int Knob_BonusB              = 100;     // spin success bonus
 108 static int Knob_Penalty             = 200;     // spin failure penalty
 109 static int Knob_Poverty             = 1000;

 110 static int Knob_FixedSpin           = 0;



 111 static int Knob_PreSpin             = 10;      // 20-100 likely better


 112 



 113 static volatile int InitDone        = 0;
 114 
 115 // -----------------------------------------------------------------------------
 116 // Theory of operations -- Monitors lists, thread residency, etc:
 117 //
 118 // * A thread acquires ownership of a monitor by successfully
 119 //   CAS()ing the _owner field from null to non-null.
 120 //
 121 // * Invariant: A thread appears on at most one monitor list --
 122 //   cxq, EntryList or WaitSet -- at any one time.
 123 //
 124 // * Contending threads "push" themselves onto the cxq with CAS
 125 //   and then spin/park.
 126 //
 127 // * After a contending thread eventually acquires the lock it must
 128 //   dequeue itself from either the EntryList or the cxq.
 129 //
 130 // * The exiting thread identifies and unparks an "heir presumptive"
 131 //   tentative successor thread on the EntryList.  Critically, the
 132 //   exiting thread doesn't unlink the successor thread from the EntryList.


 258   }
 259 
 260   if (Self->is_lock_owned ((address)cur)) {
 261     assert(_recursions == 0, "internal state error");
 262     _recursions = 1;
 263     // Commute owner from a thread-specific on-stack BasicLockObject address to
 264     // a full-fledged "Thread *".
 265     _owner = Self;
 266     return;
 267   }
 268 
 269   // We've encountered genuine contention.
 270   assert(Self->_Stalled == 0, "invariant");
 271   Self->_Stalled = intptr_t(this);
 272 
 273   // Try one round of spinning *before* enqueueing Self
 274   // and before going through the awkward and expensive state
 275   // transitions.  The following spin is strictly optional ...
 276   // Note that if we acquire the monitor from an initial spin
 277   // we forgo posting JVMTI events and firing DTRACE probes.
 278   if (TrySpin(Self) > 0) {
 279     assert(_owner == Self, "invariant");
 280     assert(_recursions == 0, "invariant");
 281     assert(((oop)(object()))->mark() == markOopDesc::encode(this), "invariant");
 282     Self->_Stalled = 0;
 283     return;
 284   }
 285 
 286   assert(_owner != Self, "invariant");
 287   assert(_succ != Self, "invariant");
 288   assert(Self->is_Java_thread(), "invariant");
 289   JavaThread * jt = (JavaThread *) Self;
 290   assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
 291   assert(jt->thread_state() != _thread_blocked, "invariant");
 292   assert(this->object() != NULL, "invariant");
 293   assert(_count >= 0, "invariant");
 294 
 295   // Prevent deflation at STW-time.  See deflate_idle_monitors() and is_busy().
 296   // Ensure the object-monitor relationship remains stable while there's contention.
 297   Atomic::inc(&_count);
 298 


 420   assert(Self->is_Java_thread(), "invariant");
 421   assert(((JavaThread *) Self)->thread_state() == _thread_blocked, "invariant");
 422 
 423   // Try the lock - TATAS
 424   if (TryLock (Self) > 0) {
 425     assert(_succ != Self, "invariant");
 426     assert(_owner == Self, "invariant");
 427     assert(_Responsible != Self, "invariant");
 428     return;
 429   }
 430 
 431   DeferredInitialize();
 432 
 433   // We try one round of spinning *before* enqueueing Self.
 434   //
 435   // If the _owner is ready but OFFPROC we could use a YieldTo()
 436   // operation to donate the remainder of this thread's quantum
 437   // to the owner.  This has subtle but beneficial affinity
 438   // effects.
 439 
 440   if (TrySpin(Self) > 0) {
 441     assert(_owner == Self, "invariant");
 442     assert(_succ != Self, "invariant");
 443     assert(_Responsible != Self, "invariant");
 444     return;
 445   }
 446 
 447   // The Spin failed -- Enqueue and park the thread ...
 448   assert(_succ != Self, "invariant");
 449   assert(_owner != Self, "invariant");
 450   assert(_Responsible != Self, "invariant");
 451 
 452   // Enqueue "Self" on ObjectMonitor's _cxq.
 453   //
 454   // Node acts as a proxy for Self.
 455   // As an aside, if were to ever rewrite the synchronization code mostly
 456   // in Java, WaitNodes, ObjectMonitors, and Events would become 1st-class
 457   // Java objects.  This would avoid awkward lifecycle and liveness issues,
 458   // as well as eliminate a subset of ABA issues.
 459   // TODO: eliminate ObjectWaiter and enqueue either Threads or Events.
 460 


 542       Self->_ParkEvent->park();
 543     }
 544 
 545     if (TryLock(Self) > 0) break;
 546 
 547     // The lock is still contested.
 548     // Keep a tally of the # of futile wakeups.
 549     // Note that the counter is not protected by a lock or updated by atomics.
 550     // That is by design - we trade "lossy" counters which are exposed to
 551     // races during updates for a lower probe effect.
 552 
 553     // This PerfData object can be used in parallel with a safepoint.
 554     // See the work around in PerfDataManager::destroy().
 555     OM_PERFDATA_OP(FutileWakeups, inc());
 556     ++nWakeups;
 557 
 558     // Assuming this is not a spurious wakeup we'll normally find _succ == Self.
 559     // We can defer clearing _succ until after the spin completes
 560     // TrySpin() must tolerate being called with _succ == Self.
 561     // Try yet another round of adaptive spinning.
 562     if (TrySpin(Self) > 0) break;
 563 
 564     // We can find that we were unpark()ed and redesignated _succ while
 565     // we were spinning.  That's harmless.  If we iterate and call park(),
 566     // park() will consume the event and return immediately and we'll
 567     // just spin again.  This pattern can repeat, leaving _succ to simply
 568     // spin on a CPU.


 569 




 570     if (_succ == Self) _succ = NULL;
 571 
 572     // Invariant: after clearing _succ a thread *must* retry _owner before parking.
 573     OrderAccess::fence();
 574   }
 575 
 576   // Egress :
 577   // Self has acquired the lock -- Unlink Self from the cxq or EntryList.
 578   // Normally we'll find Self on the EntryList .
 579   // From the perspective of the lock owner (this thread), the
 580   // EntryList is stable and cxq is prepend-only.
 581   // The head of cxq is volatile but the interior is stable.
 582   // In addition, Self.TState is stable.
 583 
 584   assert(_owner == Self, "invariant");
 585   assert(object() != NULL, "invariant");
 586   // I'd like to write:
 587   //   guarantee (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
 588   // but as we're at a safepoint that's not safe.
 589 


 628   // To that end, the 1-0 exit() operation must have at least STST|LDST
 629   // "release" barrier semantics.  Specifically, there must be at least a
 630   // STST|LDST barrier in exit() before the ST of null into _owner that drops
 631   // the lock.   The barrier ensures that changes to monitor meta-data and data
 632   // protected by the lock will be visible before we release the lock, and
 633   // therefore before some other thread (CPU) has a chance to acquire the lock.
 634   // See also: http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
 635   //
 636   // Critically, any prior STs to _succ or EntryList must be visible before
 637   // the ST of null into _owner in the *subsequent* (following) corresponding
 638   // monitorexit.  Recall too, that in 1-0 mode monitorexit does not necessarily
 639   // execute a serializing instruction.
 640 
 641   return;
 642 }
 643 
 644 // ReenterI() is a specialized inline form of the latter half of the
 645 // contended slow-path from EnterI().  We use ReenterI() only for
 646 // monitor reentry in wait().
 647 //
 648 // In the future we should reconcile EnterI() and ReenterI().


 649 
 650 void ObjectMonitor::ReenterI(Thread * Self, ObjectWaiter * SelfNode) {
 651   assert(Self != NULL, "invariant");
 652   assert(SelfNode != NULL, "invariant");
 653   assert(SelfNode->_thread == Self, "invariant");
 654   assert(_waiters > 0, "invariant");
 655   assert(((oop)(object()))->mark() == markOopDesc::encode(this), "invariant");
 656   assert(((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant");
 657   JavaThread * jt = (JavaThread *) Self;
 658 
 659   int nWakeups = 0;
 660   for (;;) {
 661     ObjectWaiter::TStates v = SelfNode->TState;
 662     guarantee(v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant");
 663     assert(_owner != Self, "invariant");
 664 
 665     if (TryLock(Self) > 0) break;
 666     if (TrySpin(Self) > 0) break;
 667 
 668     // State transition wrappers around park() ...


 880   if (_recursions != 0) {
 881     _recursions--;        // this is simple recursive enter
 882     return;
 883   }
 884 
 885   // Invariant: after setting Responsible=null an thread must execute
 886   // a MEMBAR or other serializing instruction before fetching EntryList|cxq.
 887   _Responsible = NULL;
 888 
 889 #if INCLUDE_JFR
 890   // get the owner's thread id for the MonitorEnter event
 891   // if it is enabled and the thread isn't suspended
 892   if (not_suspended && EventJavaMonitorEnter::is_enabled()) {
 893     _previous_owner_tid = JFR_THREAD_ID(Self);
 894   }
 895 #endif
 896 
 897   for (;;) {
 898     assert(THREAD == _owner, "invariant");
 899 

 900     // release semantics: prior loads and stores from within the critical section
 901     // must not float (reorder) past the following store that drops the lock.
 902     // On SPARC that requires MEMBAR #loadstore|#storestore.
 903     // But of course in TSO #loadstore|#storestore is not required.
 904     // I'd like to write one of the following:
 905     // A.  OrderAccess::release() ; _owner = NULL
 906     // B.  OrderAccess::loadstore(); OrderAccess::storestore(); _owner = NULL;
 907     // Unfortunately OrderAccess::release() and OrderAccess::loadstore() both
 908     // store into a _dummy variable.  That store is not needed, but can result
 909     // in massive wasteful coherency traffic on classic SMP systems.
 910     // Instead, I use release_store(), which is implemented as just a simple
 911     // ST on x64, x86 and SPARC.
 912     OrderAccess::release_store(&_owner, (void*)NULL);   // drop the lock
 913     OrderAccess::storeload();                        // See if we need to wake a successor
 914     if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) {
 915       return;
 916     }
 917     // Other threads are blocked trying to acquire the lock.
 918 
 919     // Normally the exiting thread is responsible for ensuring succession,


 938     // Another less appealing alternative would be for the exiting thread
 939     // to drop the lock and then spin briefly to see if a spinner managed
 940     // to acquire the lock.  If so, the exiting thread could exit
 941     // immediately without waking a successor, otherwise the exiting
 942     // thread would need to dequeue and wake a successor.
 943     // (Note that we'd need to make the post-drop spin short, but no
 944     // shorter than the worst-case round-trip cache-line migration time.
 945     // The dropped lock needs to become visible to the spinner, and then
 946     // the acquisition of the lock by the spinner must become visible to
 947     // the exiting thread).
 948 
 949     // It appears that an heir-presumptive (successor) must be made ready.
 950     // Only the current lock owner can manipulate the EntryList or
 951     // drain _cxq, so we need to reacquire the lock.  If we fail
 952     // to reacquire the lock the responsibility for ensuring succession
 953     // falls to the new owner.
 954     //
 955     if (!Atomic::replace_if_null(THREAD, &_owner)) {
 956       return;
 957     }

























 958 
 959     guarantee(_owner == THREAD, "invariant");
 960 
 961     ObjectWaiter * w = NULL;























































































 962 
 963     w = _EntryList;
 964     if (w != NULL) {
 965       // I'd like to write: guarantee (w->_thread != Self).
 966       // But in practice an exiting thread may find itself on the EntryList.
 967       // Let's say thread T1 calls O.wait().  Wait() enqueues T1 on O's waitset and
 968       // then calls exit().  Exit release the lock by setting O._owner to NULL.
 969       // Let's say T1 then stalls.  T2 acquires O and calls O.notify().  The
 970       // notify() operation moves T1 from O's waitset to O's EntryList. T2 then
 971       // release the lock "O".  T2 resumes immediately after the ST of null into
 972       // _owner, above.  T2 notices that the EntryList is populated, so it
 973       // reacquires the lock and then finds itself on the EntryList.
 974       // Given all that, we have to tolerate the circumstance where "w" is
 975       // associated with Self.
 976       assert(w->TState == ObjectWaiter::TS_ENTER, "invariant");
 977       ExitEpilog(Self, w);
 978       return;
 979     }
 980 
 981     // If we find that both _cxq and EntryList are null then just


 988     // The following loop is tantamount to: w = swap(&cxq, NULL)
 989     for (;;) {
 990       assert(w != NULL, "Invariant");
 991       ObjectWaiter * u = Atomic::cmpxchg((ObjectWaiter*)NULL, &_cxq, w);
 992       if (u == w) break;
 993       w = u;
 994     }
 995 
 996     assert(w != NULL, "invariant");
 997     assert(_EntryList == NULL, "invariant");
 998 
 999     // Convert the LIFO SLL anchored by _cxq into a DLL.
1000     // The list reorganization step operates in O(LENGTH(w)) time.
1001     // It's critical that this step operate quickly as
1002     // "Self" still holds the outer-lock, restricting parallelism
1003     // and effectively lengthening the critical section.
1004     // Invariant: s chases t chases u.
1005     // TODO-FIXME: consider changing EntryList from a DLL to a CDLL so
1006     // we have faster access to the tail.
1007 



















1008     _EntryList = w;
1009     ObjectWaiter * q = NULL;
1010     ObjectWaiter * p;
1011     for (p = w; p != NULL; p = p->_next) {
1012       guarantee(p->TState == ObjectWaiter::TS_CXQ, "Invariant");
1013       p->TState = ObjectWaiter::TS_ENTER;
1014       p->_prev = q;
1015       q = p;
1016     }

1017 
1018     // In 1-0 mode we need: ST EntryList; MEMBAR #storestore; ST _owner = NULL
1019     // The MEMBAR is satisfied by the release_store() operation in ExitEpilog().
1020 
1021     // See if we can abdicate to a spinner instead of waking a thread.
1022     // A primary goal of the implementation is to reduce the
1023     // context-switch rate.
1024     if (_succ != NULL) continue;
1025 
1026     w = _EntryList;
1027     if (w != NULL) {
1028       guarantee(w->TState == ObjectWaiter::TS_ENTER, "invariant");
1029       ExitEpilog(Self, w);
1030       return;
1031     }
1032   }
1033 }
1034 
1035 // ExitSuspendEquivalent:
1036 // A faster alternate to handle_special_suspend_equivalent_condition()


1044 //
1045 // A.  To ameliorate the problem we might also defer state transitions
1046 //     to as late as possible -- just prior to parking.
1047 //     Given that, we'd call HSSEC after having returned from park(),
1048 //     but before attempting to acquire the monitor.  This is only a
1049 //     partial solution.  It avoids calling HSSEC while holding the
1050 //     monitor (good), but it still increases successor reacquisition latency --
1051 //     the interval between unparking a successor and the time the successor
1052 //     resumes and retries the lock.  See ReenterI(), which defers state transitions.
1053 //     If we use this technique we can also avoid EnterI()-exit() loop
1054 //     in ::enter() where we iteratively drop the lock and then attempt
1055 //     to reacquire it after suspending.
1056 //
1057 // B.  In the future we might fold all the suspend bits into a
1058 //     composite per-thread suspend flag and then update it with CAS().
1059 //     Alternately, a Dekker-like mechanism with multiple variables
1060 //     would suffice:
1061 //       ST Self->_suspend_equivalent = false
1062 //       MEMBAR
1063 //       LD Self_>_suspend_flags





1064 
1065 bool ObjectMonitor::ExitSuspendEquivalent(JavaThread * jSelf) {









1066   return jSelf->handle_special_suspend_equivalent_condition();
1067 }
1068 
1069 
1070 void ObjectMonitor::ExitEpilog(Thread * Self, ObjectWaiter * Wakee) {
1071   assert(_owner == Self, "invariant");
1072 
1073   // Exit protocol:
1074   // 1. ST _succ = wakee
1075   // 2. membar #loadstore|#storestore;
1076   // 2. ST _owner = NULL
1077   // 3. unpark(wakee)
1078 
1079   _succ = Wakee->_thread;
1080   ParkEvent * Trigger = Wakee->_event;
1081 
1082   // Hygiene -- once we've set _owner = NULL we can't safely dereference Wakee again.
1083   // The thread associated with Wakee may have grabbed the lock and "Wakee" may be
1084   // out-of-scope (non-extant).
1085   Wakee  = NULL;
1086 
1087   // Drop the lock
1088   OrderAccess::release_store(&_owner, (void*)NULL);
1089   OrderAccess::fence();                               // ST _owner vs LD in unpark()
1090 
1091   DTRACE_MONITOR_PROBE(contended__exit, this, object(), Self);
1092   Trigger->unpark();
1093 
1094   // Maintain stats and report events to JVMTI
1095   OM_PERFDATA_OP(Parks, inc());
1096 }
1097 
1098 
1099 // -----------------------------------------------------------------------------


1152 #define CHECK_OWNER()                                                       \
1153   do {                                                                      \
1154     if (THREAD != _owner) {                                                 \
1155       if (THREAD->is_lock_owned((address) _owner)) {                        \
1156         _owner = THREAD;  /* Convert from basiclock addr to Thread addr */  \
1157         _recursions = 0;                                                    \
1158       } else {                                                              \
1159         THROW(vmSymbols::java_lang_IllegalMonitorStateException());         \
1160       }                                                                     \
1161     }                                                                       \
1162   } while (false)
1163 
1164 // check_slow() is a misnomer.  It's called to simply to throw an IMSX exception.
1165 // TODO-FIXME: remove check_slow() -- it's likely dead.
1166 
1167 void ObjectMonitor::check_slow(TRAPS) {
1168   assert(THREAD != _owner && !THREAD->is_lock_owned((address) _owner), "must not be owner");
1169   THROW_MSG(vmSymbols::java_lang_IllegalMonitorStateException(), "current thread not owner");
1170 }
1171 






1172 static void post_monitor_wait_event(EventJavaMonitorWait* event,
1173                                     ObjectMonitor* monitor,
1174                                     jlong notifier_tid,
1175                                     jlong timeout,
1176                                     bool timedout) {
1177   assert(event != NULL, "invariant");
1178   assert(monitor != NULL, "invariant");
1179   event->set_monitorClass(((oop)monitor->object())->klass());
1180   event->set_timeout(timeout);
1181   event->set_address((uintptr_t)monitor->object_addr());
1182   event->set_notifier(notifier_tid);
1183   event->set_timedOut(timedout);
1184   event->commit();
1185 }
1186 
1187 // -----------------------------------------------------------------------------
1188 // Wait/Notify/NotifyAll
1189 //
1190 // Note: a subset of changes to ObjectMonitor::wait()
1191 // will need to be replicated in complete_exit


1397   // check if the notification happened
1398   if (!WasNotified) {
1399     // no, it could be timeout or Thread.interrupt() or both
1400     // check for interrupt event, otherwise it is timeout
1401     if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) {
1402       THROW(vmSymbols::java_lang_InterruptedException());
1403     }
1404   }
1405 
1406   // NOTE: Spurious wake up will be consider as timeout.
1407   // Monitor notify has precedence over thread interrupt.
1408 }
1409 
1410 
1411 // Consider:
1412 // If the lock is cool (cxq == null && succ == null) and we're on an MP system
1413 // then instead of transferring a thread from the WaitSet to the EntryList
1414 // we might just dequeue a thread from the WaitSet and directly unpark() it.
1415 
1416 void ObjectMonitor::INotify(Thread * Self) {


1417   Thread::SpinAcquire(&_WaitSetLock, "WaitSet - notify");
1418   ObjectWaiter * iterator = DequeueWaiter();
1419   if (iterator != NULL) {
1420     guarantee(iterator->TState == ObjectWaiter::TS_WAIT, "invariant");
1421     guarantee(iterator->_notified == 0, "invariant");
1422     // Disposition - what might we do with iterator ?
1423     // a.  add it directly to the EntryList - either tail (policy == 1)
1424     //     or head (policy == 0).
1425     // b.  push it onto the front of the _cxq (policy == 2).
1426     // For now we use (b).
1427 
1428     iterator->TState = ObjectWaiter::TS_ENTER;
1429 
1430     iterator->_notified = 1;
1431     iterator->_notifier_tid = JFR_THREAD_ID(Self);
1432 
1433     ObjectWaiter * list = _EntryList;
1434     if (list != NULL) {
1435       assert(list->_prev == NULL, "invariant");
1436       assert(list->TState == ObjectWaiter::TS_ENTER, "invariant");
1437       assert(list != iterator, "invariant");
1438     }
1439 
1440     // prepend to cxq

























1441     if (list == NULL) {
1442       iterator->_next = iterator->_prev = NULL;
1443       _EntryList = iterator;
1444     } else {
1445       iterator->TState = ObjectWaiter::TS_CXQ;
1446       for (;;) {
1447         ObjectWaiter * front = _cxq;
1448         iterator->_next = front;
1449         if (Atomic::cmpxchg(iterator, &_cxq, front) == front) {
1450           break;
1451         }
1452       }
1453     }























1454 
1455     // _WaitSetLock protects the wait queue, not the EntryList.  We could
1456     // move the add-to-EntryList operation, above, outside the critical section
1457     // protected by _WaitSetLock.  In practice that's not useful.  With the
1458     // exception of  wait() timeouts and interrupts the monitor owner
1459     // is the only thread that grabs _WaitSetLock.  There's almost no contention
1460     // on _WaitSetLock so it's not profitable to reduce the length of the
1461     // critical section.
1462 

1463     iterator->wait_reenter_begin(this);
1464   }

1465   Thread::SpinRelease(&_WaitSetLock);
1466 }
1467 
1468 // Consider: a not-uncommon synchronization bug is to use notify() when
1469 // notifyAll() is more appropriate, potentially resulting in stranded
1470 // threads; this is one example of a lost wakeup. A useful diagnostic
1471 // option is to force all notify() operations to behave as notifyAll().
1472 //
1473 // Note: We can also detect many such problems with a "minimum wait".
1474 // When the "minimum wait" is set to a small non-zero timeout value
1475 // and the program does not hang whereas it did absent "minimum wait",
1476 // that suggests a lost wakeup bug.
1477 
1478 void ObjectMonitor::notify(TRAPS) {
1479   CHECK_OWNER();
1480   if (_WaitSet == NULL) {
1481     return;
1482   }
1483   DTRACE_MONITOR_PROBE(notify, this, object(), THREAD);
1484   INotify(THREAD);


1600       return 1;
1601     }
1602     SpinPause();
1603   }
1604 
1605   // Admission control - verify preconditions for spinning
1606   //
1607   // We always spin a little bit, just to prevent _SpinDuration == 0 from
1608   // becoming an absorbing state.  Put another way, we spin briefly to
1609   // sample, just in case the system load, parallelism, contention, or lock
1610   // modality changed.
1611   //
1612   // Consider the following alternative:
1613   // Periodically set _SpinDuration = _SpinLimit and try a long/full
1614   // spin attempt.  "Periodically" might mean after a tally of
1615   // the # of failed spin attempts (or iterations) reaches some threshold.
1616   // This takes us into the realm of 1-out-of-N spinning, where we
1617   // hold the duration constant but vary the frequency.
1618 
1619   ctr = _SpinDuration;

1620   if (ctr <= 0) return 0;
1621 
1622   if (NotRunnable(Self, (Thread *) _owner)) {







1623     return 0;
1624   }



1625 
1626   // We're good to spin ... spin ingress.
1627   // CONSIDER: use Prefetch::write() to avoid RTS->RTO upgrades
1628   // when preparing to LD...CAS _owner, etc and the CAS is likely
1629   // to succeed.
1630   if (_succ == NULL) {
1631     _succ = Self;
1632   }



1633   Thread * prv = NULL;
1634 
1635   // There are three ways to exit the following loop:
1636   // 1.  A successful spin where this thread has acquired the lock.
1637   // 2.  Spin failure with prejudice
1638   // 3.  Spin failure without prejudice
1639 
1640   while (--ctr >= 0) {
1641 
1642     // Periodic polling -- Check for pending GC
1643     // Threads may spin while they're unsafe.
1644     // We don't want spinning threads to delay the JVM from reaching
1645     // a stop-the-world safepoint or to steal cycles from GC.
1646     // If we detect a pending safepoint we abort in order that
1647     // (a) this thread, if unsafe, doesn't delay the safepoint, and (b)
1648     // this thread, if safe, doesn't steal cycles from GC.
1649     // This is in keeping with the "no loitering in runtime" rule.
1650     // We periodically check to see if there's a safepoint pending.
1651     if ((ctr & 0xFF) == 0) {
1652       if (SafepointMechanism::poll(Self)) {
1653         goto Abort;           // abrupt spin egress
1654       }
1655       SpinPause();

























1656     }
1657 
1658     // Probe _owner with TATAS
1659     // If this thread observes the monitor transition or flicker
1660     // from locked to unlocked to locked, then the odds that this
1661     // thread will acquire the lock in this spin attempt go down
1662     // considerably.  The same argument applies if the CAS fails
1663     // or if we observe _owner change from one non-null value to
1664     // another non-null value.   In such cases we might abort
1665     // the spin without prejudice or apply a "penalty" to the
1666     // spin count-down variable "ctr", reducing it by 100, say.
1667 
1668     Thread * ox = (Thread *) _owner;
1669     if (ox == NULL) {
1670       ox = (Thread*)Atomic::cmpxchg(Self, &_owner, (void*)NULL);
1671       if (ox == NULL) {
1672         // The CAS succeeded -- this thread acquired ownership
1673         // Take care of some bookkeeping to exit spin state.
1674         if (_succ == Self) {
1675           _succ = NULL;
1676         }

1677 
1678         // Increase _SpinDuration :
1679         // The spin was successful (profitable) so we tend toward
1680         // longer spin attempts in the future.
1681         // CONSIDER: factor "ctr" into the _SpinDuration adjustment.
1682         // If we acquired the lock early in the spin cycle it
1683         // makes sense to increase _SpinDuration proportionally.
1684         // Note that we don't clamp SpinDuration precisely at SpinLimit.
1685         int x = _SpinDuration;
1686         if (x < Knob_SpinLimit) {
1687           if (x < Knob_Poverty) x = Knob_Poverty;
1688           _SpinDuration = x + Knob_Bonus;
1689         }
1690         return 1;
1691       }
1692 
1693       // The CAS failed ... we can take any of the following actions:
1694       // * penalize: ctr -= CASPenalty
1695       // * exit spin with prejudice -- goto Abort;
1696       // * exit spin without prejudice.
1697       // * Since CAS is high-latency, retry again immediately.
1698       prv = ox;
1699       goto Abort;



1700     }
1701 
1702     // Did lock ownership change hands ?
1703     if (ox != prv && prv != NULL) {
1704       goto Abort;


1705     }
1706     prv = ox;
1707 
1708     // Abort the spin if the owner is not executing.
1709     // The owner must be executing in order to drop the lock.
1710     // Spinning while the owner is OFFPROC is idiocy.
1711     // Consider: ctr -= RunnablePenalty ;
1712     if (NotRunnable(Self, ox)) {
1713       goto Abort;
1714     }
1715     if (_succ == NULL) {
1716       _succ = Self;
1717     }
1718   }
1719 
1720   // Spin failed with prejudice -- reduce _SpinDuration.
1721   // TODO: Use an AIMD-like policy to adjust _SpinDuration.
1722   // AIMD is globally stable.
1723   {
1724     int x = _SpinDuration;
1725     if (x > 0) {
1726       // Consider an AIMD scheme like: x -= (x >> 3) + 100
1727       // This is globally sample and tends to damp the response.
1728       x -= Knob_Penalty;
1729       if (x < 0) x = 0;
1730       _SpinDuration = x;
1731     }
1732   }
1733 
1734  Abort:
1735   if (_succ == Self) {

1736     _succ = NULL;
1737     // Invariant: after setting succ=null a contending thread
1738     // must recheck-retry _owner before parking.  This usually happens
1739     // in the normal usage of TrySpin(), but it's safest
1740     // to make TrySpin() as foolproof as possible.
1741     OrderAccess::fence();
1742     if (TryLock(Self) > 0) return 1;
1743   }
1744   return 0;
1745 }
1746 
1747 // NotRunnable() -- informed spinning
1748 //
1749 // Don't bother spinning if the owner is not eligible to drop the lock.
1750 // Spin only if the owner thread is _thread_in_Java or _thread_in_vm.
1751 // The thread must be runnable in order to drop the lock in timely fashion.
1752 // If the _owner is not runnable then spinning will not likely be
1753 // successful (profitable).
1754 //
1755 // Beware -- the thread referenced by _owner could have died


1906     n = PerfDataManager::create_counter(SUN_RT, #n, PerfData::U_Events,  \
1907                                         CHECK);                          \
1908   }
1909 #define NEWPERFVARIABLE(n)                                                \
1910   {                                                                       \
1911     n = PerfDataManager::create_variable(SUN_RT, #n, PerfData::U_Events,  \
1912                                          CHECK);                          \
1913   }
1914     NEWPERFCOUNTER(_sync_Inflations);
1915     NEWPERFCOUNTER(_sync_Deflations);
1916     NEWPERFCOUNTER(_sync_ContendedLockAttempts);
1917     NEWPERFCOUNTER(_sync_FutileWakeups);
1918     NEWPERFCOUNTER(_sync_Parks);
1919     NEWPERFCOUNTER(_sync_Notifications);
1920     NEWPERFVARIABLE(_sync_MonExtant);
1921 #undef NEWPERFCOUNTER
1922 #undef NEWPERFVARIABLE
1923   }
1924 }
1925 























1926 void ObjectMonitor::DeferredInitialize() {
1927   if (InitDone > 0) return;
1928   if (Atomic::cmpxchg (-1, &InitDone, 0) != 0) {
1929     while (InitDone != 1) /* empty */;
1930     return;
1931   }
1932 
1933   // One-shot global initialization ...
1934   // The initialization is idempotent, so we don't need locks.
1935   // In the future consider doing this via os::init_2().




1936 
1937   if (!os::is_MP()) {



















































1938     Knob_SpinLimit = 0;

1939     Knob_PreSpin   = 0;
1940     Knob_FixedSpin = -1;
1941   }
1942 

1943   OrderAccess::fence();
1944   InitDone = 1;
1945 }
1946 
< prev index next >