# HG changeset patch # User rehn # Date 1545125061 -3600 # Tue Dec 18 10:24:21 2018 +0100 # Node ID 9400e21c121c54b9c97b36925ccf2f50ac2b275d # Parent e84983c2735e01bb9bd952b3300bd80b220891aa 8214271: Fast primitive to wake many threads Reviewed-by: diff --git a/src/hotspot/os/linux/waitBarrier_linux.cpp b/src/hotspot/os/linux/waitBarrier_linux.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/os/linux/waitBarrier_linux.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled/precompiled.hpp" +#include "runtime/orderAccess.hpp" +#include "runtime/os.hpp" +#include "waitBarrier_linux.hpp" +#include +#include + +#define check_with_errno(check_type, cond, msg) \ + do { \ + int err = errno; \ + check_type(cond, "%s; error='%s' (errno=%s)", msg, os::strerror(err), \ + os::errno_name(err)); \ +} while (false) + +#define guarantee_with_errno(cond, msg) check_with_errno(guarantee, cond, msg) + +static int futex(volatile int *uaddr, int futex_op, int val, + const struct timespec *timeout, int *uaddr2, int val3) +{ + return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); +} + +void LinuxWaitBarrier::arm(int barrier_tag) { + assert(_futex_barrier == 0, "Already armed"); + _futex_barrier = barrier_tag; + OrderAccess::fence(); +} + +void LinuxWaitBarrier::disarm() { + assert(_futex_barrier != 0, "Not armed"); + _futex_barrier = 0; + OrderAccess::fence(); +} + +void LinuxWaitBarrier::wake() { + assert(_futex_barrier == 0, "Not disarmed"); + int s = futex(&_futex_barrier, + FUTEX_WAKE, + INT_MAX, /* wake a max of this many threads */ + NULL /* ignored */, + NULL /* ignored */, + 0 /* ignored */); + guarantee_with_errno(s > -1, "futex FUTEX_WAKE"); +} + +void LinuxWaitBarrier::wait(int barrier_tag) { + assert(barrier_tag != 0, "Trying to wait on disarmed value"); + if (barrier_tag == 0 || + barrier_tag != _futex_barrier) { + OrderAccess::fence(); + return; + } + do { + int s = futex(&_futex_barrier, + FUTEX_WAIT, + barrier_tag, /* should be this tag */ + NULL, /* no timeout */ + NULL, /* ignored */ + 0 /* ignored */); + guarantee_with_errno((s == 0) || + (s == -1 && errno == EAGAIN) || + (s == -1 && errno == EINTR), + "futex FUTEX_WAIT"); + // Return value 0, re-check in case of spurious wake-up. + // EINTR and re-check and go back to waiting. + // EAGAIN we already are disarmed, we should pass the check, + // if not re-armed with same tag. + } while (barrier_tag == _futex_barrier); +} diff --git a/src/hotspot/os/linux/waitBarrier_linux.hpp b/src/hotspot/os/linux/waitBarrier_linux.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/os/linux/waitBarrier_linux.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef OS_LINUX_WAITBARRIER_LINUX_HPP +#define OS_LINUX_WAITBARRIER_LINUX_HPP + +#include "memory/allocation.hpp" + +class LinuxWaitBarrier : public CHeapObj { + volatile int _futex_barrier; + + // Prevent copying and assignment of LinuxWaitBarrier instances. + LinuxWaitBarrier(const LinuxWaitBarrier&); + LinuxWaitBarrier& operator=(const LinuxWaitBarrier&); + + public: + LinuxWaitBarrier() : _futex_barrier(0) {}; + ~LinuxWaitBarrier() {}; + + const char* description() { return "futex"; } + + void arm(int barrier_tag); + void disarm(); + void wake(); + void wait(int barrier_tag); +}; + +#endif // OS_LINUX_WAITBARRIER_LINUX_HPP diff --git a/src/hotspot/share/utilities/waitBarrier.hpp b/src/hotspot/share/utilities/waitBarrier.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/utilities/waitBarrier.hpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_UTILITIES_WAITBARRIER_HPP +#define SHARE_UTILITIES_WAITBARRIER_HPP + +#include "memory/allocation.hpp" +#include "utilities/waitBarrier_generic.hpp" + +#if defined(LINUX) +#include "waitBarrier_linux.hpp" +typedef LinuxWaitBarrier WaitBarrierDefault; +#else +typedef GenericWaitBarrier WaitBarrierDefault; +#endif + +// Platform independent WaitBarrier API. +// The WaitBarrier primary objective is to wake threads waiting in wait() fast. +// It can be arm with any int, the tag, that is not 0. +// - After arm() returns any thread calling wait(tag) with the correct tag will be blocked. +// - After disarm() is called any thread calling wait(...) will never block. +// - When wake() returns no threads are blocked any more, if it was disarmed before. +// - After calling disarm() and wake() it my be re-armed immediately with a different tag. +// - Re-arming with the same tag before all threads returned from previously wait +// is implementation defined. They may or may not return from the previously wait(). +// Wake thread: +// - arm(tag) +// - *work* +// - disarm() +// - wake() +// Wait thread: +// - wait(tag) +template +class WaitBarrierType : public CHeapObj { + WaitBarrierImpl _impl; + + // Prevent copying and assignment of WaitBarrier instances. + WaitBarrierType(const WaitBarrierDefault&); + WaitBarrierType& operator=(const WaitBarrierDefault&); + + public: + WaitBarrierType() : _impl() {} + ~WaitBarrierType() {} + + // Returns implementation type. + const char* description() { return _impl.description(); } + + // Guarantees any thread calling wait() with same tag will be blocked. + // Provides a trailing fence. + void arm(int barrier_tag) { _impl.arm(barrier_tag); } + + // Guarantees any thread calling wait() with any tag will not be blocked. + // Provides a trailing fence. + void disarm() { _impl.disarm(); } + + // Guarantees any thread called wait() will be awake when it returns. + // Provides a trailing fence. + void wake() { _impl.wake(); } + + // Guarantees to return if disarm() and wake() is called. + // Provides a trailing fence. + void wait(int barrier_tag) { _impl.wait(barrier_tag); } +}; + +typedef WaitBarrierType WaitBarrier; + +#endif // SHARE_UTILITIES_WAITBARRIER_HPP diff --git a/src/hotspot/share/utilities/waitBarrier_generic.cpp b/src/hotspot/share/utilities/waitBarrier_generic.cpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/utilities/waitBarrier_generic.cpp @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "runtime/atomic.hpp" +#include "runtime/orderAccess.hpp" +#include "runtime/os.hpp" +#include "utilities/waitBarrier_generic.hpp" +#include "utilities/spinYield.hpp" + +void GenericWaitBarrier::arm(int barrier_tag) { + assert(_barrier_tag == 0, "Already armed"); + assert(_waiters == 0, "We left a thread hanging"); + _barrier_tag = barrier_tag; + _waiters = 0; + OrderAccess::fence(); +} + +void GenericWaitBarrier::disarm() { + assert(_barrier_tag != 0, "Not armed"); + _barrier_tag = 0; + OrderAccess::fence(); +} + +int GenericWaitBarrier::wake_if_needed() { + assert(_barrier_tag == 0, "Not disarmed"); + int w = _waiters; + if (w == 0) { + // Load of _barrier_threads in caller must not pass the load of _waiters. + OrderAccess::loadload(); + return 0; + } + assert(w > 0, "Bad counting"); + // We need an exact count and never go below 0. + // Otherwise the semaphore might contain to many posts. + if (Atomic::cmpxchg(w - 1, &_waiters, w) == w) { + _sem_barrier.signal(); + return w - 1; + } + return w; +} + +void GenericWaitBarrier::wake() { + assert(_barrier_tag == 0, "Not disarmed"); + int left; + SpinYield sp; + do { + left = GenericWaitBarrier::wake_if_needed(); + if (left == 0 && _barrier_threads > 0) { + // There is no thread to wake but we still have barrier threads. + sp.wait(); + } + // We must loop here until there is no waiters or potential waiters. + } while (left > 0 || _barrier_threads > 0); + OrderAccess::fence(); +} + +void GenericWaitBarrier::wait(int barrier_tag) { + assert(barrier_tag != 0, "Trying to wait on disarmed value"); + Atomic::add(1, &_barrier_threads); + if (barrier_tag != 0 && barrier_tag == _barrier_tag) { + Atomic::add(1, &_waiters); + _sem_barrier.wait(); + // We help out with posting, but we need to do so before we decrement the + // _barrier_threads otherwise we might wake threads up in next wait. + GenericWaitBarrier::wake_if_needed(); + } + Atomic::add(-1, &_barrier_threads); +} diff --git a/src/hotspot/share/utilities/waitBarrier_generic.hpp b/src/hotspot/share/utilities/waitBarrier_generic.hpp new file mode 100644 --- /dev/null +++ b/src/hotspot/share/utilities/waitBarrier_generic.hpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_UTILITIES_WAITBARRIER_GENERIC_HPP +#define SHARE_UTILITIES_WAITBARRIER_GENERIC_HPP + +#include "memory/allocation.hpp" +#include "runtime/semaphore.hpp" + +// Except for the barrier tag it self, it uses two counter to keep the semaphore +// count correct and not leave any late thread hanging. +class GenericWaitBarrier : public CHeapObj { + volatile int _barrier_tag; + // The number of threads waiting on or about to wait on the semaphore. + volatile int _waiters; + // The number of threads in the wait path, before or after the tag check. + // Which means it can become a waiter. + volatile int _barrier_threads; + Semaphore _sem_barrier; + + // Prevent copying and assignment of GenericWaitBarrier instances. + GenericWaitBarrier(const GenericWaitBarrier&); + GenericWaitBarrier& operator=(const GenericWaitBarrier&); + + int wake_if_needed(); + + public: + GenericWaitBarrier() : _barrier_tag(0), _waiters(0), _barrier_threads(0), _sem_barrier(0) {} + ~GenericWaitBarrier() {} + + const char* description() { return "semaphore"; } + + void arm(int barrier_tag); + void disarm(); + void wake(); + void wait(int barrier_tag); +}; + +#endif // SHARE_UTILITIES_WAITBARRIER_GENERIC_HPP diff --git a/test/hotspot/gtest/utilities/test_waitBarrier.cpp b/test/hotspot/gtest/utilities/test_waitBarrier.cpp new file mode 100644 --- /dev/null +++ b/test/hotspot/gtest/utilities/test_waitBarrier.cpp @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "runtime/atomic.hpp" +#include "runtime/orderAccess.hpp" +#include "runtime/os.hpp" +#include "utilities/spinYield.hpp" +#include "utilities/waitBarrier.hpp" +#include "threadHelper.inline.hpp" + +static volatile int wait_tag = 1; +static volatile int valid_value = 0; + +template +class WBThread : public JavaTestThread { +public: + static volatile bool _exit; + WaitBarrierType* _wait_barrier; + Semaphore* _wrt_start; + volatile int _on_barrier; + + WBThread(Semaphore* post, WaitBarrierType* wb, Semaphore* wrt_start) + : JavaTestThread(post), _wait_barrier(wb), _wrt_start(wrt_start) {}; + virtual ~WBThread(){} + void main_run() { + _wrt_start->signal(); + int vv, tag; + while (!_exit) { + tag = OrderAccess::load_acquire(&wait_tag); + OrderAccess::release_store_fence(&_on_barrier, tag); + _wait_barrier->wait(tag); + vv = OrderAccess::load_acquire(&valid_value); + ASSERT_EQ((vv & 0x1), 0); + } + } +}; + +template +volatile bool WBThread::_exit = false; + +template +class WBArmerThread : public JavaTestThread { +public: + WBArmerThread(Semaphore* post) : JavaTestThread(post) { + }; + virtual ~WBArmerThread(){} + void main_run() { + static const int NUMBER_OF_READERS = 4; + Semaphore post; + Semaphore wrt_start; + WaitBarrierType wb; + + wait_tag = 2; + + WBThread* reader1 = new WBThread(&post, &wb, &wrt_start); + WBThread* reader2 = new WBThread(&post, &wb, &wrt_start); + WBThread* reader3 = new WBThread(&post, &wb, &wrt_start); + WBThread* reader4 = new WBThread(&post, &wb, &wrt_start); + + reader1->doit(); + reader2->doit(); + reader3->doit(); + reader4->doit(); + + int nw = NUMBER_OF_READERS; + while (nw > 0) { + wrt_start.wait(); + --nw; + } + SpinYield sp; + jlong stop_ms = os::javaTimeMillis() + 1000; // 1 seconds max test time + while (stop_ms > os::javaTimeMillis()) { + wb.arm(wait_tag + 1); + OrderAccess::release_store(&wait_tag, wait_tag + 1); + + while (reader1->_on_barrier != wait_tag || + reader2->_on_barrier != wait_tag || + reader3->_on_barrier != wait_tag || + reader4->_on_barrier != wait_tag) { + sp.wait(); + } + OrderAccess::release_store(&valid_value, valid_value + 1); // odd + os::naked_short_sleep(1); + OrderAccess::release_store(&valid_value, valid_value + 1); // even + OrderAccess::storestore(); // Stores in WB must not float up. + wb.disarm(); + wb.wake(); + } + WBThread::_exit = true; + for (int i = 0; i < NUMBER_OF_READERS; i++) { + post.wait(); + } + } +}; + +TEST_VM(WaitBarrier, default_wb) { + WBThread::_exit = false; + mt_test_doer >(); +} + +#if defined(LINUX) +TEST_VM(WaitBarrier, generic_wb) { + WBThread::_exit = false; + mt_test_doer >(); +} +#endif # HG changeset patch # User rehn # Date 1545127243 -3600 # Tue Dec 18 11:00:43 2018 +0100 # Node ID 6d3fcce7e7711a39a1e668e4d410fdcce01b746b # Parent 9400e21c121c54b9c97b36925ccf2f50ac2b275d [mq]: 8214271-3 diff --git a/src/hotspot/os/linux/waitBarrier_linux.cpp b/src/hotspot/os/linux/waitBarrier_linux.cpp --- a/src/hotspot/os/linux/waitBarrier_linux.cpp +++ b/src/hotspot/os/linux/waitBarrier_linux.cpp @@ -38,10 +38,9 @@ #define guarantee_with_errno(cond, msg) check_with_errno(guarantee, cond, msg) -static int futex(volatile int *uaddr, int futex_op, int val, - const struct timespec *timeout, int *uaddr2, int val3) +static int futex(volatile int *addr, int futex_op, int op_arg) { - return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); + return syscall(SYS_futex, addr, futex_op, op_arg, NULL, NULL, 0); } void LinuxWaitBarrier::arm(int barrier_tag) { @@ -59,11 +58,8 @@ void LinuxWaitBarrier::wake() { assert(_futex_barrier == 0, "Not disarmed"); int s = futex(&_futex_barrier, - FUTEX_WAKE, - INT_MAX, /* wake a max of this many threads */ - NULL /* ignored */, - NULL /* ignored */, - 0 /* ignored */); + FUTEX_WAKE_PRIVATE, + INT_MAX /* wake a max of this many threads */); guarantee_with_errno(s > -1, "futex FUTEX_WAKE"); } @@ -76,18 +72,14 @@ } do { int s = futex(&_futex_barrier, - FUTEX_WAIT, - barrier_tag, /* should be this tag */ - NULL, /* no timeout */ - NULL, /* ignored */ - 0 /* ignored */); + FUTEX_WAIT_PRIVATE, + barrier_tag /* should be this tag */); guarantee_with_errno((s == 0) || (s == -1 && errno == EAGAIN) || (s == -1 && errno == EINTR), "futex FUTEX_WAIT"); - // Return value 0, re-check in case of spurious wake-up. - // EINTR and re-check and go back to waiting. - // EAGAIN we already are disarmed, we should pass the check, - // if not re-armed with same tag. + // Return value 0: woken up, but re-check in case of spurious wakeup + // Error EINTR: woken by signal, so re-check and re-wait if necessary. + // Error EAGAIN: we are already disarmed and so will pass the check } while (barrier_tag == _futex_barrier); } diff --git a/src/hotspot/share/utilities/waitBarrier.hpp b/src/hotspot/share/utilities/waitBarrier.hpp --- a/src/hotspot/share/utilities/waitBarrier.hpp +++ b/src/hotspot/share/utilities/waitBarrier.hpp @@ -26,6 +26,8 @@ #define SHARE_UTILITIES_WAITBARRIER_HPP #include "memory/allocation.hpp" +#include "runtime/thread.hpp" +#include "utilities/debug.hpp" #include "utilities/waitBarrier_generic.hpp" #if defined(LINUX) @@ -36,21 +38,38 @@ #endif // Platform independent WaitBarrier API. -// The WaitBarrier primary objective is to wake threads waiting in wait() fast. -// It can be arm with any int, the tag, that is not 0. -// - After arm() returns any thread calling wait(tag) with the correct tag will be blocked. -// - After disarm() is called any thread calling wait(...) will never block. -// - When wake() returns no threads are blocked any more, if it was disarmed before. -// - After calling disarm() and wake() it my be re-armed immediately with a different tag. -// - Re-arming with the same tag before all threads returned from previously wait -// is implementation defined. They may or may not return from the previously wait(). -// Wake thread: -// - arm(tag) -// - *work* -// - disarm() -// - wake() -// Wait thread: -// - wait(tag) +// An armed WaitBarrier prevents threads from advancing until the +// barrier is disarmed and the waiting threads woken. The barrier is +// armed by setting a non-zero value - the tag. +// +// Expected Usage: +// - Arming thread: +// tag = ...; // non-zero value +// barrier.arm(tag); +// +// +// barrier.disarm(); +// barrier.wake(); +// +// - After arm(tag) returns any thread calling wait(tag) will block. +// - After disarm() returns any subsequent calls to wait(tag) will not block. +// - After wake() returns all blocked threads are unblocked and eligible to +// execute again. +// - After calling disarm() and wake() the barrier is ready to be re-armed +// with a new tag. (may not be re-armed with last used tag) +// +// - Waiting threads +// wait(tag); // don't execute following code unless 'safe' +// +// +// - A call to wait(tag) will block if the barrier is armed with the value +// 'tag'; else it will return immediately. +// - A blocked thread is eligible to execute again once the barrier is +// disarmed and wake() has been called. +// +// A primary goal of the WaitBarrier implementation is to wake all waiting +// threads as fast, and as concurrently, as possible. +// template class WaitBarrierType : public CHeapObj { WaitBarrierImpl _impl; @@ -59,8 +78,18 @@ WaitBarrierType(const WaitBarrierDefault&); WaitBarrierType& operator=(const WaitBarrierDefault&); +#ifdef ASSERT + int _last_arm_tag; + Thread* _owner; +#endif + public: - WaitBarrierType() : _impl() {} + WaitBarrierType(Thread* owner) : _impl() { +#ifdef ASSERT + _last_arm_tag = 0; + _owner = owner; +#endif + } ~WaitBarrierType() {} // Returns implementation type. @@ -68,19 +97,35 @@ // Guarantees any thread calling wait() with same tag will be blocked. // Provides a trailing fence. - void arm(int barrier_tag) { _impl.arm(barrier_tag); } + void arm(int barrier_tag) { +#ifdef ASSERT + assert(_last_arm_tag != barrier_tag, "Re-arming with same tag"); + _last_arm_tag = barrier_tag; + assert(_owner == Thread::current(), "Not owner thread"); +#endif + _impl.arm(barrier_tag); + } // Guarantees any thread calling wait() with any tag will not be blocked. // Provides a trailing fence. - void disarm() { _impl.disarm(); } + void disarm() { + assert(_owner == Thread::current(), "Not owner thread"); + _impl.disarm(); + } // Guarantees any thread called wait() will be awake when it returns. // Provides a trailing fence. - void wake() { _impl.wake(); } + void wake() { + assert(_owner == Thread::current(), "Not owner thread"); + _impl.wake(); + } // Guarantees to return if disarm() and wake() is called. // Provides a trailing fence. - void wait(int barrier_tag) { _impl.wait(barrier_tag); } + void wait(int barrier_tag) { + assert(_owner != Thread::current(), "Trying to wait with owner thread"); + _impl.wait(barrier_tag); + } }; typedef WaitBarrierType WaitBarrier; diff --git a/src/hotspot/share/utilities/waitBarrier_generic.cpp b/src/hotspot/share/utilities/waitBarrier_generic.cpp --- a/src/hotspot/share/utilities/waitBarrier_generic.cpp +++ b/src/hotspot/share/utilities/waitBarrier_generic.cpp @@ -52,8 +52,8 @@ return 0; } assert(w > 0, "Bad counting"); - // We need an exact count and never go below 0. - // Otherwise the semaphore might contain to many posts. + // We need an exact count which never goes below zero, + // otherwise the semaphore may be signalled too many times. if (Atomic::cmpxchg(w - 1, &_waiters, w) == w) { _sem_barrier.signal(); return w - 1; @@ -71,7 +71,7 @@ // There is no thread to wake but we still have barrier threads. sp.wait(); } - // We must loop here until there is no waiters or potential waiters. + // We must loop here until there are no waiters or potential waiters. } while (left > 0 || _barrier_threads > 0); OrderAccess::fence(); } diff --git a/src/hotspot/share/utilities/waitBarrier_generic.hpp b/src/hotspot/share/utilities/waitBarrier_generic.hpp --- a/src/hotspot/share/utilities/waitBarrier_generic.hpp +++ b/src/hotspot/share/utilities/waitBarrier_generic.hpp @@ -28,14 +28,14 @@ #include "memory/allocation.hpp" #include "runtime/semaphore.hpp" -// Except for the barrier tag it self, it uses two counter to keep the semaphore -// count correct and not leave any late thread hanging. +// Except for the barrier tag itself, it uses two counters to keep the semaphore +// count correct and not leave any late thread waiting. class GenericWaitBarrier : public CHeapObj { volatile int _barrier_tag; // The number of threads waiting on or about to wait on the semaphore. volatile int _waiters; // The number of threads in the wait path, before or after the tag check. - // Which means it can become a waiter. + // These threads can become waiters. volatile int _barrier_threads; Semaphore _sem_barrier; diff --git a/test/hotspot/gtest/utilities/test_waitBarrier.cpp b/test/hotspot/gtest/utilities/test_waitBarrier.cpp --- a/test/hotspot/gtest/utilities/test_waitBarrier.cpp +++ b/test/hotspot/gtest/utilities/test_waitBarrier.cpp @@ -29,7 +29,7 @@ #include "utilities/waitBarrier.hpp" #include "threadHelper.inline.hpp" -static volatile int wait_tag = 1; +static volatile int wait_tag = 0; static volatile int valid_value = 0; template @@ -46,12 +46,23 @@ void main_run() { _wrt_start->signal(); int vv, tag; + // Similar to how a JavaThread would stop in a safepoint. while (!_exit) { + // Load the published tag. tag = OrderAccess::load_acquire(&wait_tag); - OrderAccess::release_store_fence(&_on_barrier, tag); + // Publish the tag this thread is going to wait for. + OrderAccess::release_store(&_on_barrier, tag); + if (_on_barrier == 0) { + SpinPause(); + continue; + } + OrderAccess::storeload(); // Loads in WB must not float up. + // Wait until we are woken. _wait_barrier->wait(tag); + // Verify that we do not see an invalid value. vv = OrderAccess::load_acquire(&valid_value); ASSERT_EQ((vv & 0x1), 0); + OrderAccess::release_store(&_on_barrier, 0); } } }; @@ -69,9 +80,7 @@ static const int NUMBER_OF_READERS = 4; Semaphore post; Semaphore wrt_start; - WaitBarrierType wb; - - wait_tag = 2; + WaitBarrierType wb(this); WBThread* reader1 = new WBThread(&post, &wb, &wrt_start); WBThread* reader2 = new WBThread(&post, &wb, &wrt_start); @@ -88,24 +97,41 @@ wrt_start.wait(); --nw; } - SpinYield sp; jlong stop_ms = os::javaTimeMillis() + 1000; // 1 seconds max test time + int next_tag = 1; + // Similar to how the VM thread would use a WaitBarrier in a safepoint. while (stop_ms > os::javaTimeMillis()) { - wb.arm(wait_tag + 1); - OrderAccess::release_store(&wait_tag, wait_tag + 1); + // Arm next tag. + wb.arm(next_tag); + // Publish tag. + OrderAccess::release_store_fence(&wait_tag, next_tag); + // Wait until threads picked up new tag. while (reader1->_on_barrier != wait_tag || reader2->_on_barrier != wait_tag || reader3->_on_barrier != wait_tag || reader4->_on_barrier != wait_tag) { - sp.wait(); + SpinPause(); } + + // Set an invalid value. OrderAccess::release_store(&valid_value, valid_value + 1); // odd - os::naked_short_sleep(1); + os::naked_yield(); + // Set a valid value. OrderAccess::release_store(&valid_value, valid_value + 1); // even - OrderAccess::storestore(); // Stores in WB must not float up. + // Publish inactive tag. + OrderAccess::release_store_fence(&wait_tag, 0); // Stores in WB must not float up. wb.disarm(); wb.wake(); + + // Wait until threads done valid_value verification. + while (reader1->_on_barrier != 0 || + reader2->_on_barrier != 0 || + reader3->_on_barrier != 0 || + reader4->_on_barrier != 0) { + SpinPause(); + } + ++next_tag; } WBThread::_exit = true; for (int i = 0; i < NUMBER_OF_READERS; i++) {