# HG changeset patch # User rkennke # Date 1496219064 -7200 # Wed May 31 10:24:24 2017 +0200 # Node ID acac3e23ee922f477ab690fe37569b66736acd2c # Parent 55a34e4962e10c822affe8f89273a87e84cade92 [mq]: 8180932.patch diff --git a/src/share/vm/gc/shared/collectedHeap.hpp b/src/share/vm/gc/shared/collectedHeap.hpp --- a/src/share/vm/gc/shared/collectedHeap.hpp +++ b/src/share/vm/gc/shared/collectedHeap.hpp @@ -604,6 +604,16 @@ // unknown phase. The default implementation returns false. virtual bool request_concurrent_phase(const char* phase); + // Provides a thread pool to SafepointSynchronize to use + // for parallel safepoint cleanup. + // GCs that use a GC worker thread pool may want to share + // it for use during safepoint cleanup. This is only possible + // if the GC can pause and resume concurrent work (e.g. G1 + // concurrent marking) for an intermittent non-GC safepoint. + // If this method returns NULL, SafepointSynchronize will + // create and initialize a separate thread pool. + virtual WorkGang* get_safepoint_workers() { return NULL; } + // Non product verification and debugging. #ifndef PRODUCT // Support for PromotionFailureALot. Return true if it's time to cause a diff --git a/src/share/vm/runtime/globals.hpp b/src/share/vm/runtime/globals.hpp --- a/src/share/vm/runtime/globals.hpp +++ b/src/share/vm/runtime/globals.hpp @@ -1184,6 +1184,12 @@ \ product(bool, MonitorInUseLists, true, "Track Monitors for Deflation") \ \ + experimental(bool, ParallelSafepointCleanup, false, \ + "Enable parallel safepoint cleanup") \ + \ + experimental(uint, ParallelSafepointCleanupThreads, 4, \ + "Number of parallel threads used for safepoint cleanup") \ + \ experimental(intx, SyncFlags, 0, "(Unsafe, Unstable) " \ "Experimental Sync flags") \ \ diff --git a/src/share/vm/runtime/safepoint.cpp b/src/share/vm/runtime/safepoint.cpp --- a/src/share/vm/runtime/safepoint.cpp +++ b/src/share/vm/runtime/safepoint.cpp @@ -71,6 +71,10 @@ // -------------------------------------------------------------------------------------------------- // Implementation of Safepoint begin/end +WorkGang* SafepointSynchronize::_cleanup_workers = NULL; +SubTasksDone* SafepointSynchronize::_cleanup_subtasks = NULL; +uint SafepointSynchronize::_num_cleanup_workers = 0; + SafepointSynchronize::SynchronizeState volatile SafepointSynchronize::_state = SafepointSynchronize::_not_synchronized; volatile int SafepointSynchronize::_waiting_to_block = 0; volatile int SafepointSynchronize::_safepoint_counter = 0; @@ -538,67 +542,131 @@ } } +class ParallelSPCleanupThreadClosure : public ThreadClosure { +private: + CodeBlobClosure* _nmethod_cl; + +public: + ParallelSPCleanupThreadClosure() { + _nmethod_cl = NMethodSweeper::prepare_mark_active_nmethods(); + } + + void do_thread(Thread* thread) { + ObjectSynchronizer::deflate_thread_local_monitors(thread); + if (_nmethod_cl != NULL && thread->is_Java_thread() && + ! thread->is_Code_cache_sweeper_thread()) { + JavaThread* jt = (JavaThread*) thread; + jt->nmethods_do(_nmethod_cl); + } + } +}; + +class ParallelSPCleanupTask : public AbstractGangTask { +private: + SubTasksDone* _subtasks; + ParallelSPCleanupThreadClosure _cleanup_threads_cl; +public: + ParallelSPCleanupTask(SubTasksDone* subtasks) : + AbstractGangTask("Parallel Safepoint Cleanup"), + _cleanup_threads_cl(ParallelSPCleanupThreadClosure()), + _subtasks(subtasks) {} + + void work(uint worker_id) { + // All threads deflate monitors and mark nmethods (if necessary). + Threads::parallel_java_threads_do(&_cleanup_threads_cl); + + if (! _subtasks->is_task_claimed(SafepointSynchronize::SAFEPOINT_CLEANUP_DEFLATE_MONITORS)) { + const char* name = "deflating idle monitors"; + EventSafepointCleanupTask event; + TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); + ObjectSynchronizer::deflate_idle_monitors(); + event_safepoint_cleanup_task_commit(event, name); + } + + if (! _subtasks->is_task_claimed(SafepointSynchronize::SAFEPOINT_CLEANUP_UPDATE_INLINE_CACHES)) { + const char* name = "updating inline caches"; + EventSafepointCleanupTask event; + TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); + InlineCacheBuffer::update_inline_caches(); + event_safepoint_cleanup_task_commit(event, name); + } + + if (! _subtasks->is_task_claimed(SafepointSynchronize::SAFEPOINT_CLEANUP_COMPILATION_POLICY)) { + const char* name = "compilation policy safepoint handler"; + EventSafepointCleanupTask event; + TraceTime timer("compilation policy safepoint handler", TRACETIME_LOG(Info, safepoint, cleanup)); + CompilationPolicy::policy()->do_safepoint_work(); + event_safepoint_cleanup_task_commit(event, name); + } + + if (! _subtasks->is_task_claimed(SafepointSynchronize::SAFEPOINT_CLEANUP_SYMBOL_TABLE_REHASH)) { + if (SymbolTable::needs_rehashing()) { + const char* name = "rehashing symbol table"; + EventSafepointCleanupTask event; + TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); + SymbolTable::rehash_table(); + event_safepoint_cleanup_task_commit(event, name); + } + } + + if (! _subtasks->is_task_claimed(SafepointSynchronize::SAFEPOINT_CLEANUP_STRING_TABLE_REHASH)) { + if (StringTable::needs_rehashing()) { + const char* name = "rehashing string table"; + EventSafepointCleanupTask event; + TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); + StringTable::rehash_table(); + event_safepoint_cleanup_task_commit(event, name); + } + } + + if (! _subtasks->is_task_claimed(SafepointSynchronize::SAFEPOINT_CLEANUP_CLD_PURGE)) { + // CMS delays purging the CLDG until the beginning of the next safepoint and to + // make sure concurrent sweep is done + const char* name = "purging class loader data graph"; + EventSafepointCleanupTask event; + TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); + ClassLoaderDataGraph::purge_if_needed(); + event_safepoint_cleanup_task_commit(event, name); + } + _subtasks->all_tasks_completed(SafepointSynchronize::_num_cleanup_workers); + } +}; + // Various cleaning tasks that should be done periodically at safepoints void SafepointSynchronize::do_cleanup_tasks() { - { - const char* name = "deflating idle monitors"; - EventSafepointCleanupTask event; - TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); - ObjectSynchronizer::deflate_idle_monitors(); - event_safepoint_cleanup_task_commit(event, name); + if (_num_cleanup_workers == 0) { + // Deferred initialization. + if (ParallelSafepointCleanup) { + assert(_cleanup_workers == NULL, "already initialized?"); + // Ask the GC to share its workers. + CollectedHeap* heap = Universe::heap(); + assert(heap != NULL, "heap not initialized yet?"); + _cleanup_workers = heap->get_safepoint_workers(); + if (_cleanup_workers == NULL) { + // If GC doesn't want to share or doesn't even use worker threads, we create our own thread pool. + _cleanup_workers = new WorkGang("Parallel Safepoint Cleanup", ParallelSafepointCleanupThreads, false, false); + _cleanup_workers->initialize_workers(); + } + _num_cleanup_workers = MIN2(ParallelSafepointCleanupThreads, _cleanup_workers->total_workers()); + } else { + _num_cleanup_workers = 1; + } + assert(_num_cleanup_workers != 0, "not initialized?"); + _cleanup_subtasks = new SubTasksDone(SAFEPOINT_CLEANUP_NUM_TASKS); } - { - const char* name = "updating inline caches"; - EventSafepointCleanupTask event; - TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); - InlineCacheBuffer::update_inline_caches(); - event_safepoint_cleanup_task_commit(event, name); - } - { - const char* name = "compilation policy safepoint handler"; - EventSafepointCleanupTask event; - TraceTime timer("compilation policy safepoint handler", TRACETIME_LOG(Info, safepoint, cleanup)); - CompilationPolicy::policy()->do_safepoint_work(); - event_safepoint_cleanup_task_commit(event, name); - } - - { - const char* name = "mark nmethods"; - EventSafepointCleanupTask event; - TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); - NMethodSweeper::mark_active_nmethods(); - event_safepoint_cleanup_task_commit(event, name); - } - - if (SymbolTable::needs_rehashing()) { - const char* name = "rehashing symbol table"; - EventSafepointCleanupTask event; - TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); - SymbolTable::rehash_table(); - event_safepoint_cleanup_task_commit(event, name); - } - - if (StringTable::needs_rehashing()) { - const char* name = "rehashing string table"; - EventSafepointCleanupTask event; - TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); - StringTable::rehash_table(); - event_safepoint_cleanup_task_commit(event, name); - } - - { - // CMS delays purging the CLDG until the beginning of the next safepoint and to - // make sure concurrent sweep is done - const char* name = "purging class loader data graph"; - EventSafepointCleanupTask event; - TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup)); - ClassLoaderDataGraph::purge_if_needed(); - event_safepoint_cleanup_task_commit(event, name); + // Run cleanup serially or in parallel. + ParallelSPCleanupTask cleanup(_cleanup_subtasks); + StrongRootsScope srs(_num_cleanup_workers); + if (ParallelSafepointCleanup) { + assert(_cleanup_workers != NULL, "no cleanup workers?"); + _cleanup_workers->run_task(&cleanup, _num_cleanup_workers); + } else { + assert(_num_cleanup_workers == 1, "only 1 thread"); + cleanup.work(0); } } - bool SafepointSynchronize::safepoint_safe(JavaThread *thread, JavaThreadState state) { switch(state) { case _thread_in_native: diff --git a/src/share/vm/runtime/safepoint.hpp b/src/share/vm/runtime/safepoint.hpp --- a/src/share/vm/runtime/safepoint.hpp +++ b/src/share/vm/runtime/safepoint.hpp @@ -51,6 +51,8 @@ class ThreadSafepointState; class SnippetCache; class nmethod; +class WorkGang; +class SubTasksDone; // // Implements roll-forward to safepoint (safepoint synchronization) @@ -75,6 +77,17 @@ _blocking_timeout = 1 }; + enum SafepointCleanupTasks { + SAFEPOINT_CLEANUP_DEFLATE_MONITORS, + SAFEPOINT_CLEANUP_UPDATE_INLINE_CACHES, + SAFEPOINT_CLEANUP_COMPILATION_POLICY, + SAFEPOINT_CLEANUP_SYMBOL_TABLE_REHASH, + SAFEPOINT_CLEANUP_STRING_TABLE_REHASH, + SAFEPOINT_CLEANUP_CLD_PURGE, + // Leave this one last. + SAFEPOINT_CLEANUP_NUM_TASKS + }; + typedef struct { float _time_stamp; // record when the current safepoint occurs in seconds int _vmop_type; // type of VM operation triggers the safepoint @@ -103,6 +116,7 @@ // race freedom. public: static volatile int _safepoint_counter; + static uint _num_cleanup_workers; private: static long _end_of_last_safepoint; // Time of last safepoint in milliseconds @@ -129,6 +143,10 @@ // For debug long safepoint static void print_safepoint_timeout(SafepointTimeoutReason timeout_reason); + // Parallel cleanup + static WorkGang* _cleanup_workers; + static SubTasksDone* _cleanup_subtasks; + public: // Main entry points diff --git a/src/share/vm/runtime/sweeper.cpp b/src/share/vm/runtime/sweeper.cpp --- a/src/share/vm/runtime/sweeper.cpp +++ b/src/share/vm/runtime/sweeper.cpp @@ -199,11 +199,20 @@ * safepoint. */ void NMethodSweeper::mark_active_nmethods() { + CodeBlobClosure* cl = prepare_mark_active_nmethods(); + if (cl != NULL) { + Threads::nmethods_do(cl); + // TODO: Is this really needed? + OrderAccess::storestore(); + } +} + +CodeBlobClosure* NMethodSweeper::prepare_mark_active_nmethods() { assert(SafepointSynchronize::is_at_safepoint(), "must be executed at a safepoint"); // If we do not want to reclaim not-entrant or zombie methods there is no need // to scan stacks if (!MethodFlushing) { - return; + return NULL; } // Increase time so that we can estimate when to invoke the sweeper again. @@ -231,14 +240,13 @@ if (PrintMethodFlushing) { tty->print_cr("### Sweep: stack traversal %ld", _traversals); } - Threads::nmethods_do(&mark_activation_closure); + return &mark_activation_closure; } else { // Only set hotness counter - Threads::nmethods_do(&set_hotness_closure); + return &set_hotness_closure; } - OrderAccess::storestore(); } /** diff --git a/src/share/vm/runtime/sweeper.hpp b/src/share/vm/runtime/sweeper.hpp --- a/src/share/vm/runtime/sweeper.hpp +++ b/src/share/vm/runtime/sweeper.hpp @@ -30,6 +30,8 @@ #include "code/codeCache.hpp" #include "utilities/ticks.hpp" +class CodeBlobClosure; + // An NmethodSweeper is an incremental cleaner for: // - cleanup inline caches // - reclamation of nmethods @@ -114,6 +116,7 @@ #endif static void mark_active_nmethods(); // Invoked at the end of each safepoint + static CodeBlobClosure* prepare_mark_active_nmethods(); static void sweeper_loop(); static void notify(int code_blob_type); // Possibly start the sweeper thread. static void force_sweep(); diff --git a/src/share/vm/runtime/synchronizer.cpp b/src/share/vm/runtime/synchronizer.cpp --- a/src/share/vm/runtime/synchronizer.cpp +++ b/src/share/vm/runtime/synchronizer.cpp @@ -1695,17 +1695,8 @@ Thread::muxAcquire(&gListLock, "scavenge - return"); if (MonitorInUseLists) { - int inUse = 0; - for (JavaThread* cur = Threads::first(); cur != NULL; cur = cur->next()) { - nInCirculation+= cur->omInUseCount; - int deflated_count = deflate_monitor_list(cur->omInUseList_addr(), &freeHeadp, &freeTailp); - cur->omInUseCount-= deflated_count; - if (ObjectMonitor::Knob_VerifyInUse) { - verifyInUse(cur); - } - nScavenged += deflated_count; - nInuse += cur->omInUseCount; - } + // Note: the thread-local monitors lists get deflated in + // a separate pass. See deflate_thread_local_monitors(). // For moribund threads, scan gOmInUseList if (gOmInUseList) { @@ -1780,6 +1771,33 @@ GVars.stwCycle++; } +void ObjectSynchronizer::deflate_thread_local_monitors(Thread* thread) { + assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); + if (! MonitorInUseLists) return; + + ObjectMonitor * freeHeadp = NULL; // Local SLL of scavenged monitors + ObjectMonitor * freeTailp = NULL; + + int deflated_count = deflate_monitor_list(thread->omInUseList_addr(), &freeHeadp, &freeTailp); + thread->omInUseCount-= deflated_count; + if (ObjectMonitor::Knob_VerifyInUse) { + verifyInUse(thread); + } + + // Move the scavenged monitors back to the global free list. + if (freeHeadp != NULL) { + Thread::muxAcquire(&gListLock, "scavenge - return"); + guarantee(freeTailp != NULL && deflated_count > 0, "invariant"); + assert(freeTailp->FreeNext == NULL, "invariant"); + + gMonitorFreeCount += deflated_count; + // constant-time list splice - prepend scavenged segment to gFreeList + freeTailp->FreeNext = gFreeList; + gFreeList = freeHeadp; + Thread::muxRelease(&gListLock); + } +} + // Monitor cleanup on JavaThread::exit // Iterate through monitor cache and attempt to release thread's monitors diff --git a/src/share/vm/runtime/synchronizer.hpp b/src/share/vm/runtime/synchronizer.hpp --- a/src/share/vm/runtime/synchronizer.hpp +++ b/src/share/vm/runtime/synchronizer.hpp @@ -128,6 +128,8 @@ // Basically we deflate all monitors that are not busy. // An adaptive profile-based deflation policy could be used if needed static void deflate_idle_monitors(); + static void deflate_thread_local_monitors(Thread* thread); + // For a given monitor list: global or per-thread, deflate idle monitors static int deflate_monitor_list(ObjectMonitor** listheadp, ObjectMonitor** freeHeadp, diff --git a/src/share/vm/runtime/thread.cpp b/src/share/vm/runtime/thread.cpp --- a/src/share/vm/runtime/thread.cpp +++ b/src/share/vm/runtime/thread.cpp @@ -3377,6 +3377,15 @@ // If CompilerThreads ever become non-JavaThreads, add them here } +void Threads::parallel_java_threads_do(ThreadClosure* tc) { + int cp = Threads::thread_claim_parity(); + ALL_JAVA_THREADS(p) { + if (p->claim_oops_do(true, cp)) { + tc->do_thread(p); + } + } +} + // The system initialization in the library has three phases. // // Phase 1: java.lang.System class initialization diff --git a/src/share/vm/runtime/thread.hpp b/src/share/vm/runtime/thread.hpp --- a/src/share/vm/runtime/thread.hpp +++ b/src/share/vm/runtime/thread.hpp @@ -2068,6 +2068,7 @@ static bool includes(JavaThread* p); static JavaThread* first() { return _thread_list; } static void threads_do(ThreadClosure* tc); + static void parallel_java_threads_do(ThreadClosure* tc); // Initializes the vm and creates the vm thread static jint create_vm(JavaVMInitArgs* args, bool* canTryAgain);