--- old/src/hotspot/share/runtime/threadSMR.cpp 2018-05-01 12:13:23.000000000 -0400 +++ new/src/hotspot/share/runtime/threadSMR.cpp 2018-05-01 12:13:23.000000000 -0400 @@ -378,6 +378,250 @@ } }; +// Closure to determine if the specified JavaThread is found by +// threads_do(). +// +class VerifyHazardPointerThreadClosure : public ThreadClosure { + private: + bool _found; + Thread *_self; + + public: + VerifyHazardPointerThreadClosure(Thread *self) : _found(false), _self(self) {} + + bool found() const { return _found; } + + virtual void do_thread(Thread *thread) { + if (thread == _self) { + _found = true; + } + } +}; + + +// Acquire a stable ThreadsList. +// +ThreadsList *ThreadsSMRSupport::acquire_stable_list(Thread *self, bool is_ThreadsListSetter) { + assert(self != NULL, "sanity check"); + // acquire_stable_list_nested_path() will grab the Threads_lock + // so let's make sure the ThreadsListHandle is in a safe place. + // ThreadsListSetter cannot make this check on this code path. + debug_only(if (!is_ThreadsListSetter && StrictSafepointChecks) self->check_for_valid_safepoint_state(/* potential_vm_operation */ false);) + + if (self->get_threads_hazard_ptr() == NULL) { + // The typical case is first. + return acquire_stable_list_fast_path(self); + } + + // The nested case is rare. + return acquire_stable_list_nested_path(self); +} + +// Fast path (and lock free) way to acquire a stable ThreadsList. +// +ThreadsList *ThreadsSMRSupport::acquire_stable_list_fast_path(Thread *self) { + assert(self != NULL, "sanity check"); + assert(self->get_threads_hazard_ptr() == NULL, "sanity check"); + assert(self->get_nested_threads_hazard_ptr() == NULL, + "cannot have a nested hazard ptr with a NULL regular hazard ptr"); + + ThreadsList* threads; + + // Stable recording of a hazard ptr for SMR. This code does not use + // locks so its use of the _java_thread_list & _threads_hazard_ptr + // fields is racy relative to code that uses those fields with locks. + // OrderAccess and Atomic functions are used to deal with those races. + // + while (true) { + threads = get_java_thread_list(); + + // Publish a tagged hazard ptr to denote that the hazard ptr is not + // yet verified as being stable. Due to the fence after the hazard + // ptr write, it will be sequentially consistent w.r.t. the + // sequentially consistent writes of the ThreadsList, even on + // non-multiple copy atomic machines where stores can be observed + // in different order from different observer threads. + ThreadsList* unverified_threads = Thread::tag_hazard_ptr(threads); + self->set_threads_hazard_ptr(unverified_threads); + + // If _java_thread_list has changed, we have lost a race with + // Threads::add() or Threads::remove() and have to try again. + if (get_java_thread_list() != threads) { + continue; + } + + // We try to remove the tag which will verify the hazard ptr as + // being stable. This exchange can race with a scanning thread + // which might invalidate the tagged hazard ptr to keep it from + // being followed to access JavaThread ptrs. If we lose the race, + // we simply retry. If we win the race, then the stable hazard + // ptr is officially published. + if (self->cmpxchg_threads_hazard_ptr(threads, unverified_threads) == unverified_threads) { + break; + } + } + + // A stable hazard ptr has been published letting other threads know + // that the ThreadsList and the JavaThreads reachable from this list + // are protected and hence they should not be deleted until everyone + // agrees it is safe to do so. + + verify_hazard_pointer_scanned(self, threads); + + return threads; +} + +// Acquire a nested stable ThreadsList; this is rare so it uses +// Threads_lock. +// +ThreadsList *ThreadsSMRSupport::acquire_stable_list_nested_path(Thread *self) { + assert(self != NULL, "sanity check"); + assert(self->get_threads_hazard_ptr() != NULL, + "cannot have a NULL regular hazard ptr when acquiring a nested hazard ptr"); + + // The thread already has a hazard ptr (ThreadsList ref) so we need + // to create a nested ThreadsListHandle with the current ThreadsList + // since it might be different than our current hazard ptr. The need + // for a nested ThreadsListHandle is rare so we do this while holding + // the Threads_lock so we don't race with the scanning code; the code + // is so much simpler this way. + + NestedThreadsList* node; + { + // Only grab the Threads_lock if we don't already own it. + MutexLockerEx ml(Threads_lock->owned_by_self() ? NULL : Threads_lock); + node = new NestedThreadsList(get_java_thread_list()); + // We insert at the front of the list to match up with the delete + // in release_stable_list(). + node->set_next(self->get_nested_threads_hazard_ptr()); + self->set_nested_threads_hazard_ptr(node); + if (EnableThreadSMRStatistics) { + self->inc_nested_threads_hazard_ptr_cnt(); + if (self->nested_threads_hazard_ptr_cnt() > _nested_thread_list_max) { + _nested_thread_list_max = self->nested_threads_hazard_ptr_cnt(); + } + } + } + log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::acquire_stable_list: add NestedThreadsList node containing ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(node->t_list())); + + verify_hazard_pointer_scanned(self, node->t_list()); + + return node->t_list(); +} + +// Release a stable ThreadsList. +// +void ThreadsSMRSupport::release_stable_list(Thread *self) { + assert(self != NULL, "sanity check"); + // release_stable_list_nested_path() will grab the Threads_lock + // so let's make sure the ThreadsListHandle is in a safe place. + debug_only(if (StrictSafepointChecks) self->check_for_valid_safepoint_state(/* potential_vm_operation */ false);) + + if (self->get_nested_threads_hazard_ptr() == NULL) { + // The typical case is first. + release_stable_list_fast_path(self); + return; + } + + // The nested case is rare. + release_stable_list_nested_path(self); +} + +// Fast path way to release a stable ThreadsList. The release portion +// is lock-free, but the wake up portion is not. +// +void ThreadsSMRSupport::release_stable_list_fast_path(Thread *self) { + assert(self != NULL, "sanity check"); + assert(self->get_threads_hazard_ptr() != NULL, "sanity check"); + assert(self->get_nested_threads_hazard_ptr() == NULL, + "cannot have a nested hazard ptr when releasing a regular hazard ptr"); + + // After releasing the hazard ptr, other threads may go ahead and + // free up some memory temporarily used by a ThreadsList snapshot. + self->set_threads_hazard_ptr(NULL); + + // We use double-check locking to reduce traffic on the system + // wide Thread-SMR delete_lock. + if (ThreadsSMRSupport::delete_notify()) { + // An exiting thread might be waiting in smr_delete(); we need to + // check with delete_lock to be sure. + release_stable_list_wake_up((char *) "regular hazard ptr"); + } +} + +// Release a nested stable ThreadsList; this is rare so it uses +// Threads_lock. +// +void ThreadsSMRSupport::release_stable_list_nested_path(Thread *self) { + assert(self != NULL, "sanity check"); + assert(self->get_nested_threads_hazard_ptr() != NULL, "sanity check"); + assert(self->get_threads_hazard_ptr() != NULL, + "must have a regular hazard ptr to have nested hazard ptrs"); + + // We have a nested ThreadsListHandle so we have to release it first. + // The need for a nested ThreadsListHandle is rare so we do this while + // holding the Threads_lock so we don't race with the scanning code; + // the code is so much simpler this way. + + NestedThreadsList *node; + { + // Only grab the Threads_lock if we don't already own it. + MutexLockerEx ml(Threads_lock->owned_by_self() ? NULL : Threads_lock); + // We remove from the front of the list to match up with the insert + // in acquire_stable_list(). + node = self->get_nested_threads_hazard_ptr(); + self->set_nested_threads_hazard_ptr(node->next()); + if (EnableThreadSMRStatistics) { + self->dec_nested_threads_hazard_ptr_cnt(); + } + } + + // An exiting thread might be waiting in smr_delete(); we need to + // check with delete_lock to be sure. + release_stable_list_wake_up((char *) "nested hazard ptr"); + + log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::release_stable_list: delete NestedThreadsList node containing ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(node->t_list())); + + delete node; +} + +// Verify that the stable hazard pointer used to safely keep threads +// alive is scanned by threads_do() which is a key piece of honoring +// the Thread-SMR protocol. +void ThreadsSMRSupport::verify_hazard_pointer_scanned(Thread *self, ThreadsList *threads) { +#ifdef ASSERT + assert(threads != NULL, "threads must not be NULL"); + + // The closure will attempt to verify that the calling thread can + // be found by threads_do() on the specified ThreadsList. If it + // is successful, then the specified ThreadsList was acquired as + // a stable hazard pointer by the calling thread in a way that + // honored the Thread-SMR protocol. + // + // If the calling thread cannot be found by threads_do() and if + // it is not the shutdown thread, then the calling thread is not + // honoring the Thread-SMR ptotocol. This means that the specified + // ThreadsList is not a stable hazard pointer and can be freed + // by another thread from the to-be-deleted list at any time. + // + // Note: The shutdown thread has removed itself from the Threads + // list and is safe to have a waiver from this check because + // VM_Exit::_shutdown_thread is not set until after the VMThread + // has started the final safepoint which holds the Threads_lock + // for the remainder of the VM's life. + // + VerifyHazardPointerThreadClosure cl(self); + threads_do(&cl, threads); + + // If the calling thread is not honoring the Thread-SMR protocol, + // then we will either crash in threads_do() above because 'threads' + // was freed by another thread or we will fail the assert() below. + // In either case, we won't get past this point with a badly placed + // ThreadsListHandle. + + assert(cl.found() || self == VM_Exit::shutdown_thread(), "Acquired a ThreadsList snapshot from a thread not recognized by the Thread-SMR protocol."); +#endif +} // 'entries + 1' so we always have at least one entry. ThreadsList::ThreadsList(int entries) : _length(entries), _threads(NEW_C_HEAP_ARRAY(JavaThread*, entries + 1, mtThread)), _next_list(NULL) { @@ -543,192 +787,12 @@ } } -// Closure to determine if the specified JavaThread is found by -// threads_do(). -// -class VerifyHazardPointerThreadClosure : public ThreadClosure { - private: - bool _found; - Thread *_self; - - public: - VerifyHazardPointerThreadClosure(Thread *self) : _found(false), _self(self) {} - - bool found() const { return _found; } - - virtual void do_thread(Thread *thread) { - if (thread == _self) { - _found = true; - } - } -}; - -// Apply the closure to all threads in the system, with a snapshot of -// all JavaThreads provided by the list parameter. -void ThreadsSMRSupport::threads_do(ThreadClosure *tc, ThreadsList *list) { - list->threads_do(tc); - Threads::non_java_threads_do(tc); -} - -// Apply the closure to all threads in the system. -void ThreadsSMRSupport::threads_do(ThreadClosure *tc) { - threads_do(tc, _java_thread_list); -} - -// Verify that the stable hazard pointer used to safely keep threads -// alive is scanned by threads_do() which is a key piece of honoring -// the Thread-SMR protocol. -void ThreadsSMRSupport::verify_hazard_pointer_scanned(Thread *self, ThreadsList *threads) { -#ifdef ASSERT - assert(threads != NULL, "threads must not be NULL"); - - // The closure will attempt to verify that the calling thread can - // be found by threads_do() on the specified ThreadsList. If it - // is successful, then the specified ThreadsList was acquired as - // a stable hazard pointer by the calling thread in a way that - // honored the Thread-SMR protocol. - // - // If the calling thread cannot be found by threads_do() and if - // it is not the shutdown thread, then the calling thread is not - // honoring the Thread-SMR ptotocol. This means that the specified - // ThreadsList is not a stable hazard pointer and can be freed - // by another thread from the to-be-deleted list at any time. - // - // Note: The shutdown thread has removed itself from the Threads - // list and is safe to have a waiver from this check because - // VM_Exit::_shutdown_thread is not set until after the VMThread - // has started the final safepoint which holds the Threads_lock - // for the remainder of the VM's life. - // - VerifyHazardPointerThreadClosure cl(self); - threads_do(&cl, threads); - - // If the calling thread is not honoring the Thread-SMR protocol, - // then we will either crash in threads_do() above because 'threads' - // was freed by another thread or we will fail the assert() below. - // In either case, we won't get past this point with a badly placed - // ThreadsListHandle. - - assert(cl.found() || self == VM_Exit::shutdown_thread(), "Acquired a ThreadsList snapshot from a thread not recognized by the Thread-SMR protocol."); -#endif -} - void ThreadsListSetter::set() { assert(_target->get_threads_hazard_ptr() == NULL, "hazard ptr should not already be set"); (void) ThreadsSMRSupport::acquire_stable_list(_target, /* is_ThreadsListSetter */ true); _target_needs_release = true; } -// Acquire a stable ThreadsList. -// -ThreadsList *ThreadsSMRSupport::acquire_stable_list(Thread *self, bool is_ThreadsListSetter) { - assert(self != NULL, "sanity check"); - // acquire_stable_list_nested_path() will grab the Threads_lock - // so let's make sure the ThreadsListHandle is in a safe place. - // ThreadsListSetter cannot make this check on this code path. - debug_only(if (!is_ThreadsListSetter && StrictSafepointChecks) self->check_for_valid_safepoint_state(/* potential_vm_operation */ false);) - - if (self->get_threads_hazard_ptr() == NULL) { - // The typical case is first. - return acquire_stable_list_fast_path(self); - } - - // The nested case is rare. - return acquire_stable_list_nested_path(self); -} - -// Fast path (and lock free) way to acquire a stable ThreadsList. -// -ThreadsList *ThreadsSMRSupport::acquire_stable_list_fast_path(Thread *self) { - assert(self != NULL, "sanity check"); - assert(self->get_threads_hazard_ptr() == NULL, "sanity check"); - assert(self->get_nested_threads_hazard_ptr() == NULL, - "cannot have a nested hazard ptr with a NULL regular hazard ptr"); - - ThreadsList* threads; - - // Stable recording of a hazard ptr for SMR. This code does not use - // locks so its use of the _java_thread_list & _threads_hazard_ptr - // fields is racy relative to code that uses those fields with locks. - // OrderAccess and Atomic functions are used to deal with those races. - // - while (true) { - threads = get_java_thread_list(); - - // Publish a tagged hazard ptr to denote that the hazard ptr is not - // yet verified as being stable. Due to the fence after the hazard - // ptr write, it will be sequentially consistent w.r.t. the - // sequentially consistent writes of the ThreadsList, even on - // non-multiple copy atomic machines where stores can be observed - // in different order from different observer threads. - ThreadsList* unverified_threads = Thread::tag_hazard_ptr(threads); - self->set_threads_hazard_ptr(unverified_threads); - - // If _java_thread_list has changed, we have lost a race with - // Threads::add() or Threads::remove() and have to try again. - if (get_java_thread_list() != threads) { - continue; - } - - // We try to remove the tag which will verify the hazard ptr as - // being stable. This exchange can race with a scanning thread - // which might invalidate the tagged hazard ptr to keep it from - // being followed to access JavaThread ptrs. If we lose the race, - // we simply retry. If we win the race, then the stable hazard - // ptr is officially published. - if (self->cmpxchg_threads_hazard_ptr(threads, unverified_threads) == unverified_threads) { - break; - } - } - - // A stable hazard ptr has been published letting other threads know - // that the ThreadsList and the JavaThreads reachable from this list - // are protected and hence they should not be deleted until everyone - // agrees it is safe to do so. - - verify_hazard_pointer_scanned(self, threads); - - return threads; -} - -// Acquire a nested stable ThreadsList; this is rare so it uses -// Threads_lock. -// -ThreadsList *ThreadsSMRSupport::acquire_stable_list_nested_path(Thread *self) { - assert(self != NULL, "sanity check"); - assert(self->get_threads_hazard_ptr() != NULL, - "cannot have a NULL regular hazard ptr when acquiring a nested hazard ptr"); - - // The thread already has a hazard ptr (ThreadsList ref) so we need - // to create a nested ThreadsListHandle with the current ThreadsList - // since it might be different than our current hazard ptr. The need - // for a nested ThreadsListHandle is rare so we do this while holding - // the Threads_lock so we don't race with the scanning code; the code - // is so much simpler this way. - - NestedThreadsList* node; - { - // Only grab the Threads_lock if we don't already own it. - MutexLockerEx ml(Threads_lock->owned_by_self() ? NULL : Threads_lock); - node = new NestedThreadsList(get_java_thread_list()); - // We insert at the front of the list to match up with the delete - // in release_stable_list(). - node->set_next(self->get_nested_threads_hazard_ptr()); - self->set_nested_threads_hazard_ptr(node); - if (EnableThreadSMRStatistics) { - self->inc_nested_threads_hazard_ptr_cnt(); - if (self->nested_threads_hazard_ptr_cnt() > _nested_thread_list_max) { - _nested_thread_list_max = self->nested_threads_hazard_ptr_cnt(); - } - } - } - log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::acquire_stable_list: add NestedThreadsList node containing ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(node->t_list())); - - verify_hazard_pointer_scanned(self, node->t_list()); - - return node->t_list(); -} - void ThreadsSMRSupport::add_thread(JavaThread *thread){ ThreadsList *new_list = ThreadsList::add_thread(ThreadsSMRSupport::get_java_thread_list(), thread); if (EnableThreadSMRStatistics) { @@ -858,82 +922,6 @@ return thread_is_protected; } -// Release a stable ThreadsList. -// -void ThreadsSMRSupport::release_stable_list(Thread *self) { - assert(self != NULL, "sanity check"); - // release_stable_list_nested_path() will grab the Threads_lock - // so let's make sure the ThreadsListHandle is in a safe place. - debug_only(if (StrictSafepointChecks) self->check_for_valid_safepoint_state(/* potential_vm_operation */ false);) - - if (self->get_nested_threads_hazard_ptr() == NULL) { - // The typical case is first. - release_stable_list_fast_path(self); - return; - } - - // The nested case is rare. - release_stable_list_nested_path(self); -} - -// Fast path way to release a stable ThreadsList. The release portion -// is lock-free, but the wake up portion is not. -// -void ThreadsSMRSupport::release_stable_list_fast_path(Thread *self) { - assert(self != NULL, "sanity check"); - assert(self->get_threads_hazard_ptr() != NULL, "sanity check"); - assert(self->get_nested_threads_hazard_ptr() == NULL, - "cannot have a nested hazard ptr when releasing a regular hazard ptr"); - - // After releasing the hazard ptr, other threads may go ahead and - // free up some memory temporarily used by a ThreadsList snapshot. - self->set_threads_hazard_ptr(NULL); - - // We use double-check locking to reduce traffic on the system - // wide Thread-SMR delete_lock. - if (ThreadsSMRSupport::delete_notify()) { - // An exiting thread might be waiting in smr_delete(); we need to - // check with delete_lock to be sure. - release_stable_list_wake_up((char *) "regular hazard ptr"); - } -} - -// Release a nested stable ThreadsList; this is rare so it uses -// Threads_lock. -// -void ThreadsSMRSupport::release_stable_list_nested_path(Thread *self) { - assert(self != NULL, "sanity check"); - assert(self->get_nested_threads_hazard_ptr() != NULL, "sanity check"); - assert(self->get_threads_hazard_ptr() != NULL, - "must have a regular hazard ptr to have nested hazard ptrs"); - - // We have a nested ThreadsListHandle so we have to release it first. - // The need for a nested ThreadsListHandle is rare so we do this while - // holding the Threads_lock so we don't race with the scanning code; - // the code is so much simpler this way. - - NestedThreadsList *node; - { - // Only grab the Threads_lock if we don't already own it. - MutexLockerEx ml(Threads_lock->owned_by_self() ? NULL : Threads_lock); - // We remove from the front of the list to match up with the insert - // in acquire_stable_list(). - node = self->get_nested_threads_hazard_ptr(); - self->set_nested_threads_hazard_ptr(node->next()); - if (EnableThreadSMRStatistics) { - self->dec_nested_threads_hazard_ptr_cnt(); - } - } - - // An exiting thread might be waiting in smr_delete(); we need to - // check with delete_lock to be sure. - release_stable_list_wake_up((char *) "nested hazard ptr"); - - log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::release_stable_list: delete NestedThreadsList node containing ThreadsList=" INTPTR_FORMAT, os::current_thread_id(), p2i(node->t_list())); - - delete node; -} - // Wake up portion of the release stable ThreadsList protocol; // uses the delete_lock(). // @@ -1055,6 +1043,18 @@ log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread=" INTPTR_FORMAT " is deleted.", os::current_thread_id(), p2i(thread)); } +// Apply the closure to all threads in the system, with a snapshot of +// all JavaThreads provided by the list parameter. +void ThreadsSMRSupport::threads_do(ThreadClosure *tc, ThreadsList *list) { + list->threads_do(tc); + Threads::non_java_threads_do(tc); +} + +// Apply the closure to all threads in the system. +void ThreadsSMRSupport::threads_do(ThreadClosure *tc) { + threads_do(tc, _java_thread_list); +} + // Debug, logging, and printing stuff at the end: