# HG changeset patch # User zgu # Date 1560954411 14400 # Wed Jun 19 10:26:51 2019 -0400 # Node ID d75431eb402c45e1f22efcbb022888ce552ac8be # Parent e9da3a44a7edc0222e727b83bd758b9e43176cee 8226311: Shenandoah: Concurrent evacuation of OopStorage backed weak roots diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -31,6 +31,7 @@ #include "gc/shared/gcTimer.hpp" #include "gc/shared/referenceProcessor.hpp" #include "gc/shared/referenceProcessorPhaseTimes.hpp" +#include "gc/shared/strongRootsScope.hpp" #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" #include "gc/shenandoah/shenandoahClosures.inline.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1521,7 +1521,8 @@ if (ShenandoahVerify) { if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) { - verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::JNIHandleRoots); + ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots); + verifier()->verify_roots_no_forwarded_except(types); } else { verifier()->verify_roots_no_forwarded(); } @@ -1588,6 +1589,7 @@ class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask { private: ShenandoahJNIHandleRoots _jni_roots; + ShenandoahWeakRoots _weak_roots; public: ShenandoahConcurrentRootsEvacUpdateTask() : @@ -1597,7 +1599,9 @@ void work(uint worker_id) { ShenandoahEvacOOMScope oom; ShenandoahEvacuateUpdateRootsClosure cl; + _jni_roots.oops_do(&cl); + _weak_roots.oops_do(&cl); } }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp @@ -27,16 +27,13 @@ #include "classfile/stringTable.hpp" #include "classfile/systemDictionary.hpp" #include "code/codeCache.hpp" -#include "gc/shenandoah/shenandoahClosures.inline.hpp" #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" -#include "gc/shenandoah/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahStringDedup.hpp" #include "gc/shenandoah/shenandoahTimingTracker.hpp" #include "gc/shenandoah/shenandoahVMOperations.hpp" -#include "gc/shared/weakProcessor.inline.hpp" -#include "memory/allocation.inline.hpp" +#include "jfr/jfr.hpp" #include "memory/iterator.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" @@ -71,6 +68,40 @@ _jvmti_root.oops_do(cl, worker_id); } +ShenandoahWeakSerialRoot::ShenandoahWeakSerialRoot(ShenandoahWeakSerialRoot::WeakOopsDo weak_oops_do, ShenandoahPhaseTimings::GCParPhases phase) : + _weak_oops_do(weak_oops_do), _phase(phase) { +} + +void ShenandoahWeakSerialRoot::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id) { + if (_claimed.try_set()) { + ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times(); + ShenandoahWorkerTimingsTracker timer(worker_times, _phase, worker_id); + _weak_oops_do(is_alive, keep_alive); + } +} + +#if INCLUDE_JVMTI +ShenandoahJVMTIWeakRoot::ShenandoahJVMTIWeakRoot() : + ShenandoahWeakSerialRoot(&JvmtiExport::weak_oops_do, ShenandoahPhaseTimings::JVMTIWeakRoots) { +} +#endif // INCLUDE_JVMTI + +#if INCLUDE_JFR +ShenandoahJFRWeakRoot::ShenandoahJFRWeakRoot() : + ShenandoahWeakSerialRoot(&Jfr::weak_oops_do, ShenandoahPhaseTimings::JFRWeakRoots) { +} +#endif // INCLUDE_JFR + +void ShenandoahSerialWeakRoots::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id) { + JVMTI_ONLY(_jvmti_weak_roots.weak_oops_do(is_alive, keep_alive, worker_id);) + JFR_ONLY(_jfr_weak_roots.weak_oops_do(is_alive, keep_alive, worker_id);) +} + +void ShenandoahSerialWeakRoots::weak_oops_do(OopClosure* cl, uint worker_id) { + AlwaysTrueClosure always_true; + weak_oops_do(&always_true, cl, worker_id); +} + ShenandoahThreadRoots::ShenandoahThreadRoots(bool is_par) : _is_par(is_par) { Threads::change_thread_claim_token(); } @@ -93,17 +124,6 @@ Threads::assert_all_threads_claimed(); } -ShenandoahWeakRoots::ShenandoahWeakRoots(uint n_workers) : - _process_timings(n_workers), - _task(&_process_timings, n_workers) { -} - -ShenandoahWeakRoots::~ShenandoahWeakRoots() { - ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times(); - ShenandoahTimingConverter::weak_processing_timing_to_shenandoah_timing(&_process_timings, - worker_times); -} - ShenandoahStringDedupRoots::ShenandoahStringDedupRoots() { if (ShenandoahStringDedup::is_enabled()) { StringDedup::gc_prologue(false); @@ -147,7 +167,6 @@ ShenandoahRootEvacuator::ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool include_concurrent_roots) : ShenandoahRootProcessor(phase), _thread_roots(n_workers > 1), - _weak_roots(n_workers), _include_concurrent_roots(include_concurrent_roots) { } @@ -157,29 +176,28 @@ AlwaysTrueClosure always_true; _serial_roots.oops_do(oops, worker_id); + _serial_weak_roots.weak_oops_do(oops, worker_id); if (_include_concurrent_roots) { _jni_roots.oops_do(oops, worker_id); + _weak_roots.oops_do(oops, worker_id); } _thread_roots.oops_do(oops, NULL, worker_id); _cld_roots.clds_do(&clds, &clds, worker_id); _code_roots.code_blobs_do(&blobsCl, worker_id); - _weak_roots.oops_do(&always_true, oops, worker_id); _dedup_roots.oops_do(&always_true, oops, worker_id); } ShenandoahRootUpdater::ShenandoahRootUpdater(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool update_code_cache) : ShenandoahRootProcessor(phase), _thread_roots(n_workers > 1), - _weak_roots(n_workers), _update_code_cache(update_code_cache) { } ShenandoahRootAdjuster::ShenandoahRootAdjuster(uint n_workers, ShenandoahPhaseTimings::Phase phase) : ShenandoahRootProcessor(phase), - _thread_roots(n_workers > 1), - _weak_roots(n_workers) { + _thread_roots(n_workers > 1) { assert(ShenandoahHeap::heap()->is_full_gc_in_progress(), "Full GC only"); } @@ -195,6 +213,7 @@ _cld_roots.clds_do(&adjust_cld_closure, NULL, worker_id); _code_roots.code_blobs_do(&adjust_code_closure, worker_id); - _weak_roots.oops_do(&always_true, oops, worker_id); + _serial_weak_roots.weak_oops_do(oops, worker_id); + _weak_roots.oops_do(oops, worker_id); _dedup_roots.oops_do(&always_true, oops, worker_id); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp @@ -29,11 +29,7 @@ #include "gc/shenandoah/shenandoahCodeRoots.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" -#include "gc/shared/strongRootsScope.hpp" -#include "gc/shared/weakProcessor.hpp" -#include "gc/shared/weakProcessorPhaseTimes.hpp" -#include "gc/shared/workgroup.hpp" -#include "memory/allocation.hpp" +#include "gc/shenandoah/shenandoahSharedVariables.hpp" #include "memory/iterator.hpp" class ShenandoahSerialRoot { @@ -61,10 +57,102 @@ void oops_do(OopClosure* cl, uint worker_id); }; +class ShenandoahWeakSerialRoot { + typedef void (*WeakOopsDo)(BoolObjectClosure*, OopClosure*); +private: + ShenandoahSharedFlag _claimed; + const WeakOopsDo _weak_oops_do; + const ShenandoahPhaseTimings::GCParPhases _phase; + +public: + ShenandoahWeakSerialRoot(WeakOopsDo oops_do, ShenandoahPhaseTimings::GCParPhases); + void weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id); +}; + +#if INCLUDE_JVMTI +class ShenandoahJVMTIWeakRoot : public ShenandoahWeakSerialRoot { +public: + ShenandoahJVMTIWeakRoot(); +}; +#endif // INCLUDE_JVMTI + +#if INCLUDE_JFR +class ShenandoahJFRWeakRoot : public ShenandoahWeakSerialRoot { +public: + ShenandoahJFRWeakRoot(); +}; +#endif // INCLUDE_JFR + +class ShenandoahSerialWeakRoots { +private: + JVMTI_ONLY(ShenandoahJVMTIWeakRoot _jvmti_weak_roots;) + JFR_ONLY(ShenandoahJFRWeakRoot _jfr_weak_roots;) +public: + void weak_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, uint worker_id); + void weak_oops_do(OopClosure* cl, uint worker_id); +}; + +template +class ShenandoahWeakRoot { +private: + OopStorage::ParState _itr; + const ShenandoahPhaseTimings::GCParPhases _phase; +public: + ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase); + + template + void oops_do(Closure* cl, uint worker_id); +}; + +template <> +class ShenandoahWeakRoot { +private: + OopStorage::ParState _itr; + const ShenandoahPhaseTimings::GCParPhases _phase; + +public: + ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase); + + template + void weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id); +}; + +template +class ShenandoahWeakRoots { +private: + ShenandoahWeakRoot _jni_roots; + ShenandoahWeakRoot _string_table_roots; + ShenandoahWeakRoot _resolved_method_table_roots; + ShenandoahWeakRoot _vm_roots; + +public: + ShenandoahWeakRoots(); + + template + void oops_do(Closure* cl, uint worker_id = 0); +}; + +template <> +class ShenandoahWeakRoots { +private: + ShenandoahWeakRoot _jni_roots; + ShenandoahWeakRoot _string_table_roots; + ShenandoahWeakRoot _resolved_method_table_roots; + ShenandoahWeakRoot _vm_roots; +public: + ShenandoahWeakRoots(); + + template + void oops_do(Closure* cl, uint worker_id = 0); + + template + void weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id); +}; + template class ShenandoahJNIHandleRoots { private: - OopStorage::ParState _itr; + OopStorage::ParState _itr; public: ShenandoahJNIHandleRoots(); @@ -83,18 +171,6 @@ void threads_do(ThreadClosure* tc, uint worker_id); }; -class ShenandoahWeakRoots { -private: - WeakProcessorPhaseTimes _process_timings; - WeakProcessor::Task _task; -public: - ShenandoahWeakRoots(uint n_workers); - ~ShenandoahWeakRoots(); - - template - void oops_do(IsAlive* is_alive, KeepAlive* keep_alive, uint worker_id); -}; - class ShenandoahStringDedupRoots { public: ShenandoahStringDedupRoots(); @@ -167,7 +243,8 @@ ShenandoahJNIHandleRoots _jni_roots; ShenandoahClassLoaderDataRoots _cld_roots; ShenandoahThreadRoots _thread_roots; - ShenandoahWeakRoots _weak_roots; + ShenandoahSerialWeakRoots _serial_weak_roots; + ShenandoahWeakRoots _weak_roots; ShenandoahStringDedupRoots _dedup_roots; ShenandoahCodeCacheRoots _code_roots; bool _include_concurrent_roots; @@ -185,7 +262,8 @@ ShenandoahJNIHandleRoots _jni_roots; ShenandoahClassLoaderDataRoots _cld_roots; ShenandoahThreadRoots _thread_roots; - ShenandoahWeakRoots _weak_roots; + ShenandoahSerialWeakRoots _serial_weak_roots; + ShenandoahWeakRoots _weak_roots; ShenandoahStringDedupRoots _dedup_roots; ShenandoahCodeCacheRoots _code_roots; const bool _update_code_cache; @@ -204,7 +282,8 @@ ShenandoahJNIHandleRoots _jni_roots; ShenandoahClassLoaderDataRoots _cld_roots; ShenandoahThreadRoots _thread_roots; - ShenandoahWeakRoots _weak_roots; + ShenandoahSerialWeakRoots _serial_weak_roots; + ShenandoahWeakRoots _weak_roots; ShenandoahStringDedupRoots _dedup_roots; ShenandoahCodeCacheRoots _code_roots; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp @@ -25,14 +25,83 @@ #define SHARE_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_INLINE_HPP #include "classfile/classLoaderDataGraph.hpp" +#include "classfile/stringTable.hpp" +#include "classfile/systemDictionary.hpp" #include "gc/shared/oopStorageParState.inline.hpp" #include "gc/shenandoah/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahRootProcessor.hpp" #include "gc/shenandoah/shenandoahTimingTracker.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" +#include "prims/resolvedMethodTable.hpp" #include "memory/resourceArea.hpp" template +inline ShenandoahWeakRoot::ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase) : + _itr(storage), _phase(phase) { +} + +template +template +inline void ShenandoahWeakRoot::oops_do(Closure* cl, uint worker_id) { + if (CONCURRENT) { + _itr.oops_do(cl); + } else { + ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times(); + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ThreadRoots, worker_id); + _itr.oops_do(cl); + } +} + +inline ShenandoahWeakRoot::ShenandoahWeakRoot(OopStorage* storage, ShenandoahPhaseTimings::GCParPhases phase) : + _itr(storage), _phase(phase) { +} + +template +void ShenandoahWeakRoot::weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id) { + ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times(); + ShenandoahWorkerTimingsTracker timer(worker_times, _phase, worker_id); + _itr.weak_oops_do(is_alive, keep_alive); +} + +template +ShenandoahWeakRoots::ShenandoahWeakRoots() : + _jni_roots(JNIHandles::weak_global_handles(), ShenandoahPhaseTimings::JNIWeakRoots), + _string_table_roots(StringTable::weak_storage(), ShenandoahPhaseTimings::StringTableRoots), + _resolved_method_table_roots(ResolvedMethodTable::weak_storage(), ShenandoahPhaseTimings::ResolvedMethodTableRoots), + _vm_roots(SystemDictionary::vm_weak_oop_storage(), ShenandoahPhaseTimings::VMWeakRoots) { +} + +template +template +void ShenandoahWeakRoots::oops_do(Closure* cl, uint worker_id) { + _jni_roots.oops_do(cl, worker_id); + _string_table_roots.oops_do(cl, worker_id); + _resolved_method_table_roots.oops_do(cl, worker_id); + _vm_roots.oops_do(cl, worker_id); +} + +inline ShenandoahWeakRoots::ShenandoahWeakRoots() : + _jni_roots(JNIHandles::weak_global_handles(), ShenandoahPhaseTimings::JNIWeakRoots), + _string_table_roots(StringTable::weak_storage(), ShenandoahPhaseTimings::StringTableRoots), + _resolved_method_table_roots(ResolvedMethodTable::weak_storage(), ShenandoahPhaseTimings::ResolvedMethodTableRoots), + _vm_roots(SystemDictionary::vm_weak_oop_storage(), ShenandoahPhaseTimings::VMWeakRoots) { +} + +template +void ShenandoahWeakRoots::weak_oops_do(IsAliveClosure* is_alive, KeepAliveClosure* keep_alive, uint worker_id) { + _jni_roots.weak_oops_do(is_alive, keep_alive, worker_id); + _string_table_roots.weak_oops_do(is_alive, keep_alive, worker_id); + _resolved_method_table_roots.weak_oops_do(is_alive, keep_alive, worker_id); + _vm_roots.weak_oops_do(is_alive, keep_alive, worker_id); +} + +template +void ShenandoahWeakRoots::oops_do(Closure* cl, uint worker_id) { + AlwaysTrueClosure always_true; + weak_oops_do(&always_true, cl, worker_id); +} + +template ShenandoahJNIHandleRoots::ShenandoahJNIHandleRoots() : _itr(JNIHandles::global_handles()) { } @@ -49,11 +118,6 @@ } } -template -void ShenandoahWeakRoots::oops_do(IsAlive* is_alive, KeepAlive* keep_alive, uint worker_id) { - _task.work(worker_id, is_alive, keep_alive); -} - template ShenandoahCodeCacheRoots::ShenandoahCodeCacheRoots() { nmethod::oops_do_marking_prologue(); @@ -185,7 +249,8 @@ _code_roots.code_blobs_do(&update_blobs, worker_id); } - _weak_roots.oops_do(is_alive, keep_alive, worker_id); + _serial_weak_roots.weak_oops_do(is_alive, keep_alive, worker_id); + _weak_roots.weak_oops_do(is_alive, keep_alive, worker_id); _dedup_roots.oops_do(is_alive, keep_alive, worker_id); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp @@ -54,6 +54,10 @@ return (_types & type) != 0; } +ShenandoahRootVerifier::RootTypes ShenandoahRootVerifier::combine(RootTypes t1, RootTypes t2) { + return static_cast(static_cast(t1) | static_cast(t2)); +} + void ShenandoahRootVerifier::oops_do(OopClosure* oops) { CodeBlobToOopClosure blobs(oops, !CodeBlobToOopClosure::FixRelocations); if (verify(CodeRoots)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp @@ -52,6 +52,8 @@ // Used to seed ShenandoahVerifier, do not honor root type filter void roots_do(OopClosure* cl); void strong_roots_do(OopClosure* cl); + + static RootTypes combine(RootTypes t1, RootTypes t2); private: bool verify(RootTypes type) const; };