844 SuspendRetryDelay, bits)) {
845 // Didn't make it so let the caller know.
846 return false;
847 }
848 }
849 // We aren't allowed to wait for the external suspend to complete
850 // so if the other thread isn't externally suspended we need to
851 // let the caller know.
852 else if (!is_ext_suspend_completed_with_lock(bits)) {
853 return false;
854 }
855 }
856
857 return true;
858 }
859
860 // GC Support
861 bool Thread::claim_par_threads_do(uintx claim_token) {
862 uintx token = _threads_do_token;
863 if (token != claim_token) {
864 uintx res = Atomic::cmpxchg(claim_token, &_threads_do_token, token);
865 if (res == token) {
866 return true;
867 }
868 guarantee(res == claim_token, "invariant");
869 }
870 return false;
871 }
872
873 void Thread::oops_do(OopClosure* f, CodeBlobClosure* cf) {
874 active_handles()->oops_do(f);
875 // Do oop for ThreadShadow
876 f->do_oop((oop*)&_pending_exception);
877 handle_area()->oops_do(f);
878
879 // We scan thread local monitor lists here, and the remaining global
880 // monitors in ObjectSynchronizer::oops_do().
881 ObjectSynchronizer::thread_local_used_oops_do(this, f);
882 }
883
884 void Thread::metadata_handles_do(void f(Metadata*)) {
4858 // about native mutex_t or HotSpot Mutex:: latency.
4859 // The mux construct provides a spin-then-block mutual exclusion
4860 // mechanism.
4861 //
4862 // Testing has shown that contention on the ListLock guarding gFreeList
4863 // is common. If we implement ListLock as a simple SpinLock it's common
4864 // for the JVM to devolve to yielding with little progress. This is true
4865 // despite the fact that the critical sections protected by ListLock are
4866 // extremely short.
4867 //
4868 // TODO-FIXME: ListLock should be of type SpinLock.
4869 // We should make this a 1st-class type, integrated into the lock
4870 // hierarchy as leaf-locks. Critically, the SpinLock structure
4871 // should have sufficient padding to avoid false-sharing and excessive
4872 // cache-coherency traffic.
4873
4874
4875 typedef volatile int SpinLockT;
4876
4877 void Thread::SpinAcquire(volatile int * adr, const char * LockName) {
4878 if (Atomic::cmpxchg (1, adr, 0) == 0) {
4879 return; // normal fast-path return
4880 }
4881
4882 // Slow-path : We've encountered contention -- Spin/Yield/Block strategy.
4883 int ctr = 0;
4884 int Yields = 0;
4885 for (;;) {
4886 while (*adr != 0) {
4887 ++ctr;
4888 if ((ctr & 0xFFF) == 0 || !os::is_MP()) {
4889 if (Yields > 5) {
4890 os::naked_short_sleep(1);
4891 } else {
4892 os::naked_yield();
4893 ++Yields;
4894 }
4895 } else {
4896 SpinPause();
4897 }
4898 }
4899 if (Atomic::cmpxchg(1, adr, 0) == 0) return;
4900 }
4901 }
4902
4903 void Thread::SpinRelease(volatile int * adr) {
4904 assert(*adr != 0, "invariant");
4905 OrderAccess::fence(); // guarantee at least release consistency.
4906 // Roach-motel semantics.
4907 // It's safe if subsequent LDs and STs float "up" into the critical section,
4908 // but prior LDs and STs within the critical section can't be allowed
4909 // to reorder or float past the ST that releases the lock.
4910 // Loads and stores in the critical section - which appear in program
4911 // order before the store that releases the lock - must also appear
4912 // before the store that releases the lock in memory visibility order.
4913 // Conceptually we need a #loadstore|#storestore "release" MEMBAR before
4914 // the ST of 0 into the lock-word which releases the lock, so fence
4915 // more than covers this on all platforms.
4916 *adr = 0;
4917 }
4918
4919 // muxAcquire and muxRelease:
4951 // (List,LockByte). Acquire will CAS the full lockword while Release
4952 // will STB 0 into the LockByte. The 1-0 scheme admits stranding, so
4953 // acquiring threads use timers (ParkTimed) to detect and recover from
4954 // the stranding window. Thread/Node structures must be aligned on 256-byte
4955 // boundaries by using placement-new.
4956 // * Augment MCS with advisory back-link fields maintained with CAS().
4957 // Pictorially: LockWord -> T1 <-> T2 <-> T3 <-> ... <-> Tn <-> Owner.
4958 // The validity of the backlinks must be ratified before we trust the value.
4959 // If the backlinks are invalid the exiting thread must back-track through the
4960 // the forward links, which are always trustworthy.
4961 // * Add a successor indication. The LockWord is currently encoded as
4962 // (List, LOCKBIT:1). We could also add a SUCCBIT or an explicit _succ variable
4963 // to provide the usual futile-wakeup optimization.
4964 // See RTStt for details.
4965 //
4966
4967
4968 const intptr_t LOCKBIT = 1;
4969
4970 void Thread::muxAcquire(volatile intptr_t * Lock, const char * LockName) {
4971 intptr_t w = Atomic::cmpxchg(LOCKBIT, Lock, (intptr_t)0);
4972 if (w == 0) return;
4973 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) {
4974 return;
4975 }
4976
4977 ParkEvent * const Self = Thread::current()->_MuxEvent;
4978 assert((intptr_t(Self) & LOCKBIT) == 0, "invariant");
4979 for (;;) {
4980 int its = (os::is_MP() ? 100 : 0) + 1;
4981
4982 // Optional spin phase: spin-then-park strategy
4983 while (--its >= 0) {
4984 w = *Lock;
4985 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) {
4986 return;
4987 }
4988 }
4989
4990 Self->reset();
4991 Self->OnList = intptr_t(Lock);
4992 // The following fence() isn't _strictly necessary as the subsequent
4993 // CAS() both serializes execution and ratifies the fetched *Lock value.
4994 OrderAccess::fence();
4995 for (;;) {
4996 w = *Lock;
4997 if ((w & LOCKBIT) == 0) {
4998 if (Atomic::cmpxchg(w|LOCKBIT, Lock, w) == w) {
4999 Self->OnList = 0; // hygiene - allows stronger asserts
5000 return;
5001 }
5002 continue; // Interference -- *Lock changed -- Just retry
5003 }
5004 assert(w & LOCKBIT, "invariant");
5005 Self->ListNext = (ParkEvent *) (w & ~LOCKBIT);
5006 if (Atomic::cmpxchg(intptr_t(Self)|LOCKBIT, Lock, w) == w) break;
5007 }
5008
5009 while (Self->OnList != 0) {
5010 Self->park();
5011 }
5012 }
5013 }
5014
5015 // Release() must extract a successor from the list and then wake that thread.
5016 // It can "pop" the front of the list or use a detach-modify-reattach (DMR) scheme
5017 // similar to that used by ParkEvent::Allocate() and ::Release(). DMR-based
5018 // Release() would :
5019 // (A) CAS() or swap() null to *Lock, releasing the lock and detaching the list.
5020 // (B) Extract a successor from the private list "in-hand"
5021 // (C) attempt to CAS() the residual back into *Lock over null.
5022 // If there were any newly arrived threads and the CAS() would fail.
5023 // In that case Release() would detach the RATs, re-merge the list in-hand
5024 // with the RATs and repeat as needed. Alternately, Release() might
5025 // detach and extract a successor, but then pass the residual list to the wakee.
5026 // The wakee would be responsible for reattaching and remerging before it
5027 // competed for the lock.
5028 //
5029 // Both "pop" and DMR are immune from ABA corruption -- there can be
5030 // multiple concurrent pushers, but only one popper or detacher.
5031 // This implementation pops from the head of the list. This is unfair,
5032 // but tends to provide excellent throughput as hot threads remain hot.
5033 // (We wake recently run threads first).
5034 //
5035 // All paths through muxRelease() will execute a CAS.
5036 // Release consistency -- We depend on the CAS in muxRelease() to provide full
5037 // bidirectional fence/MEMBAR semantics, ensuring that all prior memory operations
5038 // executed within the critical section are complete and globally visible before the
5039 // store (CAS) to the lock-word that releases the lock becomes globally visible.
5040 void Thread::muxRelease(volatile intptr_t * Lock) {
5041 for (;;) {
5042 const intptr_t w = Atomic::cmpxchg((intptr_t)0, Lock, LOCKBIT);
5043 assert(w & LOCKBIT, "invariant");
5044 if (w == LOCKBIT) return;
5045 ParkEvent * const List = (ParkEvent *) (w & ~LOCKBIT);
5046 assert(List != NULL, "invariant");
5047 assert(List->OnList == intptr_t(Lock), "invariant");
5048 ParkEvent * const nxt = List->ListNext;
5049 guarantee((intptr_t(nxt) & LOCKBIT) == 0, "invariant");
5050
5051 // The following CAS() releases the lock and pops the head element.
5052 // The CAS() also ratifies the previously fetched lock-word value.
5053 if (Atomic::cmpxchg(intptr_t(nxt), Lock, w) != w) {
5054 continue;
5055 }
5056 List->OnList = 0;
5057 OrderAccess::fence();
5058 List->unpark();
5059 return;
5060 }
5061 }
5062
5063
5064 void Threads::verify() {
5065 ALL_JAVA_THREADS(p) {
5066 p->verify();
5067 }
5068 VMThread* thread = VMThread::vm_thread();
5069 if (thread != NULL) thread->verify();
5070 }
|
844 SuspendRetryDelay, bits)) {
845 // Didn't make it so let the caller know.
846 return false;
847 }
848 }
849 // We aren't allowed to wait for the external suspend to complete
850 // so if the other thread isn't externally suspended we need to
851 // let the caller know.
852 else if (!is_ext_suspend_completed_with_lock(bits)) {
853 return false;
854 }
855 }
856
857 return true;
858 }
859
860 // GC Support
861 bool Thread::claim_par_threads_do(uintx claim_token) {
862 uintx token = _threads_do_token;
863 if (token != claim_token) {
864 uintx res = Atomic::cmpxchg(&_threads_do_token, token, claim_token);
865 if (res == token) {
866 return true;
867 }
868 guarantee(res == claim_token, "invariant");
869 }
870 return false;
871 }
872
873 void Thread::oops_do(OopClosure* f, CodeBlobClosure* cf) {
874 active_handles()->oops_do(f);
875 // Do oop for ThreadShadow
876 f->do_oop((oop*)&_pending_exception);
877 handle_area()->oops_do(f);
878
879 // We scan thread local monitor lists here, and the remaining global
880 // monitors in ObjectSynchronizer::oops_do().
881 ObjectSynchronizer::thread_local_used_oops_do(this, f);
882 }
883
884 void Thread::metadata_handles_do(void f(Metadata*)) {
4858 // about native mutex_t or HotSpot Mutex:: latency.
4859 // The mux construct provides a spin-then-block mutual exclusion
4860 // mechanism.
4861 //
4862 // Testing has shown that contention on the ListLock guarding gFreeList
4863 // is common. If we implement ListLock as a simple SpinLock it's common
4864 // for the JVM to devolve to yielding with little progress. This is true
4865 // despite the fact that the critical sections protected by ListLock are
4866 // extremely short.
4867 //
4868 // TODO-FIXME: ListLock should be of type SpinLock.
4869 // We should make this a 1st-class type, integrated into the lock
4870 // hierarchy as leaf-locks. Critically, the SpinLock structure
4871 // should have sufficient padding to avoid false-sharing and excessive
4872 // cache-coherency traffic.
4873
4874
4875 typedef volatile int SpinLockT;
4876
4877 void Thread::SpinAcquire(volatile int * adr, const char * LockName) {
4878 if (Atomic::cmpxchg(adr, 0, 1) == 0) {
4879 return; // normal fast-path return
4880 }
4881
4882 // Slow-path : We've encountered contention -- Spin/Yield/Block strategy.
4883 int ctr = 0;
4884 int Yields = 0;
4885 for (;;) {
4886 while (*adr != 0) {
4887 ++ctr;
4888 if ((ctr & 0xFFF) == 0 || !os::is_MP()) {
4889 if (Yields > 5) {
4890 os::naked_short_sleep(1);
4891 } else {
4892 os::naked_yield();
4893 ++Yields;
4894 }
4895 } else {
4896 SpinPause();
4897 }
4898 }
4899 if (Atomic::cmpxchg(adr, 0, 1) == 0) return;
4900 }
4901 }
4902
4903 void Thread::SpinRelease(volatile int * adr) {
4904 assert(*adr != 0, "invariant");
4905 OrderAccess::fence(); // guarantee at least release consistency.
4906 // Roach-motel semantics.
4907 // It's safe if subsequent LDs and STs float "up" into the critical section,
4908 // but prior LDs and STs within the critical section can't be allowed
4909 // to reorder or float past the ST that releases the lock.
4910 // Loads and stores in the critical section - which appear in program
4911 // order before the store that releases the lock - must also appear
4912 // before the store that releases the lock in memory visibility order.
4913 // Conceptually we need a #loadstore|#storestore "release" MEMBAR before
4914 // the ST of 0 into the lock-word which releases the lock, so fence
4915 // more than covers this on all platforms.
4916 *adr = 0;
4917 }
4918
4919 // muxAcquire and muxRelease:
4951 // (List,LockByte). Acquire will CAS the full lockword while Release
4952 // will STB 0 into the LockByte. The 1-0 scheme admits stranding, so
4953 // acquiring threads use timers (ParkTimed) to detect and recover from
4954 // the stranding window. Thread/Node structures must be aligned on 256-byte
4955 // boundaries by using placement-new.
4956 // * Augment MCS with advisory back-link fields maintained with CAS().
4957 // Pictorially: LockWord -> T1 <-> T2 <-> T3 <-> ... <-> Tn <-> Owner.
4958 // The validity of the backlinks must be ratified before we trust the value.
4959 // If the backlinks are invalid the exiting thread must back-track through the
4960 // the forward links, which are always trustworthy.
4961 // * Add a successor indication. The LockWord is currently encoded as
4962 // (List, LOCKBIT:1). We could also add a SUCCBIT or an explicit _succ variable
4963 // to provide the usual futile-wakeup optimization.
4964 // See RTStt for details.
4965 //
4966
4967
4968 const intptr_t LOCKBIT = 1;
4969
4970 void Thread::muxAcquire(volatile intptr_t * Lock, const char * LockName) {
4971 intptr_t w = Atomic::cmpxchg(Lock, (intptr_t)0, LOCKBIT);
4972 if (w == 0) return;
4973 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg(Lock, w, w|LOCKBIT) == w) {
4974 return;
4975 }
4976
4977 ParkEvent * const Self = Thread::current()->_MuxEvent;
4978 assert((intptr_t(Self) & LOCKBIT) == 0, "invariant");
4979 for (;;) {
4980 int its = (os::is_MP() ? 100 : 0) + 1;
4981
4982 // Optional spin phase: spin-then-park strategy
4983 while (--its >= 0) {
4984 w = *Lock;
4985 if ((w & LOCKBIT) == 0 && Atomic::cmpxchg(Lock, w, w|LOCKBIT) == w) {
4986 return;
4987 }
4988 }
4989
4990 Self->reset();
4991 Self->OnList = intptr_t(Lock);
4992 // The following fence() isn't _strictly necessary as the subsequent
4993 // CAS() both serializes execution and ratifies the fetched *Lock value.
4994 OrderAccess::fence();
4995 for (;;) {
4996 w = *Lock;
4997 if ((w & LOCKBIT) == 0) {
4998 if (Atomic::cmpxchg(Lock, w, w|LOCKBIT) == w) {
4999 Self->OnList = 0; // hygiene - allows stronger asserts
5000 return;
5001 }
5002 continue; // Interference -- *Lock changed -- Just retry
5003 }
5004 assert(w & LOCKBIT, "invariant");
5005 Self->ListNext = (ParkEvent *) (w & ~LOCKBIT);
5006 if (Atomic::cmpxchg(Lock, w, intptr_t(Self)|LOCKBIT) == w) break;
5007 }
5008
5009 while (Self->OnList != 0) {
5010 Self->park();
5011 }
5012 }
5013 }
5014
5015 // Release() must extract a successor from the list and then wake that thread.
5016 // It can "pop" the front of the list or use a detach-modify-reattach (DMR) scheme
5017 // similar to that used by ParkEvent::Allocate() and ::Release(). DMR-based
5018 // Release() would :
5019 // (A) CAS() or swap() null to *Lock, releasing the lock and detaching the list.
5020 // (B) Extract a successor from the private list "in-hand"
5021 // (C) attempt to CAS() the residual back into *Lock over null.
5022 // If there were any newly arrived threads and the CAS() would fail.
5023 // In that case Release() would detach the RATs, re-merge the list in-hand
5024 // with the RATs and repeat as needed. Alternately, Release() might
5025 // detach and extract a successor, but then pass the residual list to the wakee.
5026 // The wakee would be responsible for reattaching and remerging before it
5027 // competed for the lock.
5028 //
5029 // Both "pop" and DMR are immune from ABA corruption -- there can be
5030 // multiple concurrent pushers, but only one popper or detacher.
5031 // This implementation pops from the head of the list. This is unfair,
5032 // but tends to provide excellent throughput as hot threads remain hot.
5033 // (We wake recently run threads first).
5034 //
5035 // All paths through muxRelease() will execute a CAS.
5036 // Release consistency -- We depend on the CAS in muxRelease() to provide full
5037 // bidirectional fence/MEMBAR semantics, ensuring that all prior memory operations
5038 // executed within the critical section are complete and globally visible before the
5039 // store (CAS) to the lock-word that releases the lock becomes globally visible.
5040 void Thread::muxRelease(volatile intptr_t * Lock) {
5041 for (;;) {
5042 const intptr_t w = Atomic::cmpxchg(Lock, LOCKBIT, (intptr_t)0);
5043 assert(w & LOCKBIT, "invariant");
5044 if (w == LOCKBIT) return;
5045 ParkEvent * const List = (ParkEvent *) (w & ~LOCKBIT);
5046 assert(List != NULL, "invariant");
5047 assert(List->OnList == intptr_t(Lock), "invariant");
5048 ParkEvent * const nxt = List->ListNext;
5049 guarantee((intptr_t(nxt) & LOCKBIT) == 0, "invariant");
5050
5051 // The following CAS() releases the lock and pops the head element.
5052 // The CAS() also ratifies the previously fetched lock-word value.
5053 if (Atomic::cmpxchg(Lock, w, intptr_t(nxt)) != w) {
5054 continue;
5055 }
5056 List->OnList = 0;
5057 OrderAccess::fence();
5058 List->unpark();
5059 return;
5060 }
5061 }
5062
5063
5064 void Threads::verify() {
5065 ALL_JAVA_THREADS(p) {
5066 p->verify();
5067 }
5068 VMThread* thread = VMThread::vm_thread();
5069 if (thread != NULL) thread->verify();
5070 }
|