diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp index bc61cce..134f165 100644 --- a/src/hotspot/share/classfile/classLoaderData.cpp +++ b/src/hotspot/share/classfile/classLoaderData.cpp @@ -1117,14 +1117,14 @@ ClassLoaderData* ClassLoaderDataGraph::add(Handle loader, bool is_unsafe_anonymo } void ClassLoaderDataGraph::cld_do(CLDClosure* cl) { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); + assert_locked_or_safepoint_weak(ClassLoaderDataGraph_lock); for (ClassLoaderData* cld = _head; cld != NULL; cld = cld->_next) { cl->do_cld(cld); } } void ClassLoaderDataGraph::cld_unloading_do(CLDClosure* cl) { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); + assert_locked_or_safepoint_weak(ClassLoaderDataGraph_lock); // Only walk the head until any clds not purged from prior unloading // (CMS doesn't purge right away). for (ClassLoaderData* cld = _unloading; cld != _saved_unloading; cld = cld->next()) { @@ -1134,7 +1134,7 @@ void ClassLoaderDataGraph::cld_unloading_do(CLDClosure* cl) { } void ClassLoaderDataGraph::roots_cld_do(CLDClosure* strong, CLDClosure* weak) { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); + assert_locked_or_safepoint_weak(ClassLoaderDataGraph_lock); for (ClassLoaderData* cld = _head; cld != NULL; cld = cld->_next) { CLDClosure* closure = cld->keep_alive() ? strong : weak; if (closure != NULL) { @@ -1144,7 +1144,7 @@ void ClassLoaderDataGraph::roots_cld_do(CLDClosure* strong, CLDClosure* weak) { } void ClassLoaderDataGraph::always_strong_cld_do(CLDClosure* cl) { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); + assert_locked_or_safepoint_weak(ClassLoaderDataGraph_lock); if (ClassUnloading) { roots_cld_do(cl, NULL); } else { diff --git a/src/hotspot/share/gc/z/zDriver.cpp b/src/hotspot/share/gc/z/zDriver.cpp index a4353f6..11ad024 100644 --- a/src/hotspot/share/gc/z/zDriver.cpp +++ b/src/hotspot/share/gc/z/zDriver.cpp @@ -336,7 +336,7 @@ void ZDriver::run_gc_cycle(GCCause::Cause cause) { // Phase 2: Concurrent Mark { ZStatTimer timer(ZPhaseConcurrentMark); - ZHeap::heap()->mark(); + ZHeap::heap()->mark(true /* initial */); } // Phase 3: Pause Mark End @@ -345,7 +345,7 @@ void ZDriver::run_gc_cycle(GCCause::Cause cause) { while (!vm_operation(&cl)) { // Phase 3.5: Concurrent Mark Continue ZStatTimer timer(ZPhaseConcurrentMarkContinue); - ZHeap::heap()->mark(); + ZHeap::heap()->mark(false /* initial */); } } diff --git a/src/hotspot/share/gc/z/zHeap.cpp b/src/hotspot/share/gc/z/zHeap.cpp index e6d9f19..5a4b804 100644 --- a/src/hotspot/share/gc/z/zHeap.cpp +++ b/src/hotspot/share/gc/z/zHeap.cpp @@ -296,8 +296,8 @@ void ZHeap::mark_start() { ZStatHeap::set_at_mark_start(capacity(), used()); } -void ZHeap::mark() { - _mark.mark(); +void ZHeap::mark(bool initial) { + _mark.mark(initial); } void ZHeap::mark_flush_and_free(Thread* thread) { diff --git a/src/hotspot/share/gc/z/zHeap.hpp b/src/hotspot/share/gc/z/zHeap.hpp index 5ea4b36..c526427 100644 --- a/src/hotspot/share/gc/z/zHeap.hpp +++ b/src/hotspot/share/gc/z/zHeap.hpp @@ -133,7 +133,7 @@ public: bool is_object_strongly_live(uintptr_t addr) const; template void mark_object(uintptr_t addr); void mark_start(); - void mark(); + void mark(bool initial); void mark_flush_and_free(Thread* thread); bool mark_end(); diff --git a/src/hotspot/share/gc/z/zHeapIterator.cpp b/src/hotspot/share/gc/z/zHeapIterator.cpp index 241f5f8..932e0af 100644 --- a/src/hotspot/share/gc/z/zHeapIterator.cpp +++ b/src/hotspot/share/gc/z/zHeapIterator.cpp @@ -185,6 +185,7 @@ bool ZHeapIterator::visit_referents() const { void ZHeapIterator::objects_do(ObjectClosure* cl) { ZHeapIteratorRootOopClosure root_cl(this, cl); ZRootsIterator roots; + ZConcurrentRootsIterator concurrent_roots; // Follow roots. Note that we also visit the JVMTI weak tag map // as if they were strong roots to make sure we visit all tagged @@ -192,4 +193,5 @@ void ZHeapIterator::objects_do(ObjectClosure* cl) { // If we didn't do this the user would have expected to see // ObjectFree events for unreachable objects in the tag map. roots.oops_do(&root_cl, true /* visit_jvmti_weak_export */); + concurrent_roots.oops_do(&root_cl); } diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp index 4f19fc4..0fd2a2e 100644 --- a/src/hotspot/share/gc/z/zMark.cpp +++ b/src/hotspot/share/gc/z/zMark.cpp @@ -582,6 +582,23 @@ void ZMark::work(uint64_t timeout_in_millis) { stacks->free(&_allocator); } +class ZMarkConcurrentRootsTask : public ZTask { +private: + ZMark* const _mark; + ZConcurrentRootsIterator _roots; + +public: + ZMarkConcurrentRootsTask(ZMark* mark) : + ZTask("ZMarkConcurrentRootsTask"), + _mark(mark), + _roots() {} + + virtual void work() { + ZMarkBarrierOopClosure cl; + _roots.oops_do(&cl); + } +}; + class ZMarkTask : public ZTask { private: ZMark* const _mark; @@ -604,7 +621,12 @@ public: } }; -void ZMark::mark() { +void ZMark::mark(bool initial) { + if (initial) { + ZMarkConcurrentRootsTask task(this); + _workers->run_concurrent(&task); + } + ZMarkTask task(this); _workers->run_concurrent(&task); } diff --git a/src/hotspot/share/gc/z/zMark.hpp b/src/hotspot/share/gc/z/zMark.hpp index 727d9d9..727abc7 100644 --- a/src/hotspot/share/gc/z/zMark.hpp +++ b/src/hotspot/share/gc/z/zMark.hpp @@ -108,7 +108,7 @@ public: template void mark_object(uintptr_t addr); void start(); - void mark(); + void mark(bool initial); bool end(); void flush_and_free(); diff --git a/src/hotspot/share/gc/z/zRootsIterator.cpp b/src/hotspot/share/gc/z/zRootsIterator.cpp index 751d268..317c68d 100644 --- a/src/hotspot/share/gc/z/zRootsIterator.cpp +++ b/src/hotspot/share/gc/z/zRootsIterator.cpp @@ -52,16 +52,20 @@ static const ZStatSubPhase ZSubPhasePauseRootsSetup("Pause Roots Setup"); static const ZStatSubPhase ZSubPhasePauseRoots("Pause Roots"); static const ZStatSubPhase ZSubPhasePauseRootsTeardown("Pause Roots Teardown"); static const ZStatSubPhase ZSubPhasePauseRootsUniverse("Pause Roots Universe"); -static const ZStatSubPhase ZSubPhasePauseRootsJNIHandles("Pause Roots JNIHandles"); static const ZStatSubPhase ZSubPhasePauseRootsObjectSynchronizer("Pause Roots ObjectSynchronizer"); static const ZStatSubPhase ZSubPhasePauseRootsManagement("Pause Roots Management"); static const ZStatSubPhase ZSubPhasePauseRootsJVMTIExport("Pause Roots JVMTIExport"); static const ZStatSubPhase ZSubPhasePauseRootsJVMTIWeakExport("Pause Roots JVMTIWeakExport"); static const ZStatSubPhase ZSubPhasePauseRootsSystemDictionary("Pause Roots SystemDictionary"); -static const ZStatSubPhase ZSubPhasePauseRootsClassLoaderDataGraph("Pause Roots ClassLoaderDataGraph"); static const ZStatSubPhase ZSubPhasePauseRootsThreads("Pause Roots Threads"); static const ZStatSubPhase ZSubPhasePauseRootsCodeCache("Pause Roots CodeCache"); +static const ZStatSubPhase ZSubPhaseConcurrentRootsSetup("Concurrent Roots Setup"); +static const ZStatSubPhase ZSubPhaseConcurrentRoots("Concurrent Roots"); +static const ZStatSubPhase ZSubPhaseConcurrentRootsTeardown("Concurrent Roots Teardown"); +static const ZStatSubPhase ZSubPhaseConcurrentRootsJNIHandles("Concurrent Roots JNIHandles"); +static const ZStatSubPhase ZSubPhaseConcurrentRootsClassLoaderDataGraph("Concurrent Roots ClassLoaderDataGraph"); + static const ZStatSubPhase ZSubPhasePauseWeakRootsSetup("Pause Weak Roots Setup"); static const ZStatSubPhase ZSubPhasePauseWeakRoots("Pause Weak Roots"); static const ZStatSubPhase ZSubPhasePauseWeakRootsTeardown("Pause Weak Roots Teardown"); @@ -128,21 +132,17 @@ void ZParallelWeakOopsDo::weak_oops_do(BoolObjectClosure* is_alive, OopClo } ZRootsIterator::ZRootsIterator() : - _jni_handles_iter(JNIHandles::global_handles()), _universe(this), _object_synchronizer(this), _management(this), _jvmti_export(this), _jvmti_weak_export(this), _system_dictionary(this), - _jni_handles(this), - _class_loader_data_graph(this), _threads(this), _code_cache(this) { assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); ZStatTimer timer(ZSubPhasePauseRootsSetup); Threads::change_thread_claim_parity(); - ClassLoaderDataGraph::clear_claimed_marks(); COMPILER2_PRESENT(DerivedPointerTable::clear()); CodeCache::gc_prologue(); ZNMethodTable::gc_prologue(); @@ -163,11 +163,6 @@ void ZRootsIterator::do_universe(OopClosure* cl) { Universe::oops_do(cl); } -void ZRootsIterator::do_jni_handles(OopClosure* cl) { - ZStatTimer timer(ZSubPhasePauseRootsJNIHandles); - _jni_handles_iter.oops_do(cl); -} - void ZRootsIterator::do_object_synchronizer(OopClosure* cl) { ZStatTimer timer(ZSubPhasePauseRootsObjectSynchronizer); ObjectSynchronizer::oops_do(cl); @@ -194,12 +189,6 @@ void ZRootsIterator::do_system_dictionary(OopClosure* cl) { SystemDictionary::oops_do(cl); } -void ZRootsIterator::do_class_loader_data_graph(OopClosure* cl) { - ZStatTimer timer(ZSubPhasePauseRootsClassLoaderDataGraph); - CLDToOopClosure cld_cl(cl); - ClassLoaderDataGraph::cld_do(&cld_cl); -} - class ZRootsIteratorThreadClosure : public ThreadClosure { private: OopClosure* const _cl; @@ -238,8 +227,6 @@ void ZRootsIterator::oops_do(OopClosure* cl, bool visit_jvmti_weak_export) { _management.oops_do(cl); _jvmti_export.oops_do(cl); _system_dictionary.oops_do(cl); - _jni_handles.oops_do(cl); - _class_loader_data_graph.oops_do(cl); _threads.oops_do(cl); _code_cache.oops_do(cl); if (visit_jvmti_weak_export) { @@ -247,6 +234,37 @@ void ZRootsIterator::oops_do(OopClosure* cl, bool visit_jvmti_weak_export) { } } +ZConcurrentRootsIterator::ZConcurrentRootsIterator() : + _jni_handles_iter(JNIHandles::global_handles()), + _jni_handles(this), + _class_loader_data_graph(this) { + ZStatTimer timer(ZSubPhaseConcurrentRootsSetup); + ClassLoaderDataGraph_lock->lock(); + ClassLoaderDataGraph::clear_claimed_marks(); +} + +ZConcurrentRootsIterator::~ZConcurrentRootsIterator() { + ZStatTimer timer(ZSubPhaseConcurrentRootsTeardown); + ClassLoaderDataGraph_lock->unlock(); +} + +void ZConcurrentRootsIterator::do_jni_handles(OopClosure* cl) { + ZStatTimer timer(ZSubPhaseConcurrentRootsJNIHandles); + _jni_handles_iter.oops_do(cl); +} + +void ZConcurrentRootsIterator::do_class_loader_data_graph(OopClosure* cl) { + ZStatTimer timer(ZSubPhaseConcurrentRootsClassLoaderDataGraph); + CLDToOopClosure cld_cl(cl); + ClassLoaderDataGraph::cld_do(&cld_cl); +} + +void ZConcurrentRootsIterator::oops_do(OopClosure* cl) { + ZStatTimer timer(ZSubPhaseConcurrentRoots); + _jni_handles.oops_do(cl); + _class_loader_data_graph.oops_do(cl); +} + ZWeakRootsIterator::ZWeakRootsIterator() : _jvmti_weak_export(this), _jfr_weak(this) { diff --git a/src/hotspot/share/gc/z/zRootsIterator.hpp b/src/hotspot/share/gc/z/zRootsIterator.hpp index 9a759a8..8fcd48f 100644 --- a/src/hotspot/share/gc/z/zRootsIterator.hpp +++ b/src/hotspot/share/gc/z/zRootsIterator.hpp @@ -78,16 +78,12 @@ public: class ZRootsIterator { private: - ZOopStorageIterator _jni_handles_iter; - void do_universe(OopClosure* cl); - void do_jni_handles(OopClosure* cl); void do_object_synchronizer(OopClosure* cl); void do_management(OopClosure* cl); void do_jvmti_export(OopClosure* cl); void do_jvmti_weak_export(OopClosure* cl); void do_system_dictionary(OopClosure* cl); - void do_class_loader_data_graph(OopClosure* cl); void do_threads(OopClosure* cl); void do_code_cache(OopClosure* cl); @@ -97,8 +93,6 @@ private: ZSerialOopsDo _jvmti_export; ZSerialOopsDo _jvmti_weak_export; ZSerialOopsDo _system_dictionary; - ZParallelOopsDo _jni_handles; - ZParallelOopsDo _class_loader_data_graph; ZParallelOopsDo _threads; ZParallelOopsDo _code_cache; @@ -109,6 +103,23 @@ public: void oops_do(OopClosure* cl, bool visit_jvmti_weak_export = false); }; +class ZConcurrentRootsIterator { +private: + ZConcurrentOopStorageIterator _jni_handles_iter; + + void do_jni_handles(OopClosure* cl); + void do_class_loader_data_graph(OopClosure* cl); + + ZParallelOopsDo _jni_handles; + ZParallelOopsDo _class_loader_data_graph; + +public: + ZConcurrentRootsIterator(); + ~ZConcurrentRootsIterator(); + + void oops_do(OopClosure* cl); +}; + class ZWeakRootsIterator { private: void do_jvmti_weak_export(BoolObjectClosure* is_alive, OopClosure* cl); diff --git a/src/hotspot/share/runtime/mutexLocker.cpp b/src/hotspot/share/runtime/mutexLocker.cpp index 88dff46..fcfe143 100644 --- a/src/hotspot/share/runtime/mutexLocker.cpp +++ b/src/hotspot/share/runtime/mutexLocker.cpp @@ -167,6 +167,16 @@ void assert_locked_or_safepoint(const Monitor * lock) { fatal("must own lock %s", lock->name()); } +// a weaker assertion than the above +void assert_locked_or_safepoint_weak(const Monitor * lock) { + if (IgnoreLockingAssertions) return; + assert(lock != NULL, "Need non-NULL lock"); + if (lock->is_locked()) return; + if (SafepointSynchronize::is_at_safepoint()) return; + if (!Universe::is_fully_initialized()) return; + fatal("must own lock %s", lock->name()); +} + // a stronger assertion than the above void assert_lock_strong(const Monitor * lock) { if (IgnoreLockingAssertions) return; diff --git a/src/hotspot/share/runtime/mutexLocker.hpp b/src/hotspot/share/runtime/mutexLocker.hpp index 312800a..c95192a 100644 --- a/src/hotspot/share/runtime/mutexLocker.hpp +++ b/src/hotspot/share/runtime/mutexLocker.hpp @@ -199,9 +199,11 @@ class MutexLocker: StackObj { // for debugging: check that we're already owning this lock (or are at a safepoint) #ifdef ASSERT void assert_locked_or_safepoint(const Monitor * lock); +void assert_locked_or_safepoint_weak(const Monitor * lock); void assert_lock_strong(const Monitor * lock); #else #define assert_locked_or_safepoint(lock) +#define assert_locked_or_safepoint_weak(lock) #define assert_lock_strong(lock) #endif