# HG changeset patch # User zgu # Date 1556804992 14400 # Thu May 02 09:49:52 2019 -0400 # Node ID be6e9bd61713659758a379d9dc66074b2520f85e # Parent c9b9daf927f8abce8572ddeaf2c8e0a678f99fd6 [backport] 8222992: Shenandoah: Pre-evacuate all roots Reviewed-by: shade diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp @@ -79,4 +79,16 @@ inline void do_oop_work(T* p); }; +#ifdef ASSERT +class ShenandoahAssertNotForwardedClosure : public OopClosure { +private: + template + inline void do_oop_work(T* p); + +public: + inline void do_oop(narrowOop* p); + inline void do_oop(oop* p); +}; +#endif // ASSERT + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp @@ -108,4 +108,18 @@ do_oop_work(p); } +#ifdef ASSERT +template +void ShenandoahAssertNotForwardedClosure::do_oop_work(T* p) { + T o = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + shenandoah_assert_not_forwarded(p, obj); + } +} + +void ShenandoahAssertNotForwardedClosure::do_oop(narrowOop* p) { do_oop_work(p); } +void ShenandoahAssertNotForwardedClosure::do_oop(oop* p) { do_oop_work(p); } +#endif + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCLOSURES_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -670,15 +670,13 @@ // Weak processor API requires us to visit the oops, even if we are not doing // anything to them. void ShenandoahConcurrentMark::weak_roots_work() { - ShenandoahIsAliveSelector is_alive; - - if (_heap->has_forwarded_objects()) { - ShenandoahWeakUpdateClosure cl; - WeakProcessor::weak_oops_do(is_alive.is_alive_closure(), &cl); - } else { - ShenandoahWeakAssertNotForwardedClosure cl; - WeakProcessor::weak_oops_do(is_alive.is_alive_closure(), &cl); - } + OopClosure* keep_alive = &do_nothing_cl; +#ifdef ASSERT + ShenandoahWeakAssertNotForwardedClosure verify_cl; + keep_alive = &verify_cl; +#endif + ShenandoahIsAliveClosure is_alive; + WeakProcessor::weak_oops_do(&is_alive, keep_alive); } void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1510,8 +1510,13 @@ if (!cancelled_gc()) { concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false); - if (has_forwarded_objects()) { - concurrent_mark()->update_roots(ShenandoahPhaseTimings::update_roots); + // Degen may be caused by failed evacuation of roots + if (is_degenerated_gc_in_progress() && has_forwarded_objects()) { + concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots); + } + + if (ShenandoahVerify) { + verifier()->verify_roots_no_forwarded(); } stop_concurrent_marking(); @@ -1563,6 +1568,7 @@ } if (ShenandoahVerify) { + verifier()->verify_roots_no_forwarded(); verifier()->verify_during_evacuation(); } } else { @@ -2214,8 +2220,8 @@ assert(!cancelled_gc(), "Should have been done right before"); concurrent_mark()->update_roots(is_degenerated_gc_in_progress() ? - ShenandoahPhaseTimings::degen_gc_update_roots: - ShenandoahPhaseTimings::final_update_refs_roots); + ShenandoahPhaseTimings::degen_gc_update_roots: + ShenandoahPhaseTimings::final_update_refs_roots); ShenandoahGCPhase final_update_refs(ShenandoahPhaseTimings::final_update_refs_recycle); @@ -2224,6 +2230,7 @@ set_update_refs_in_progress(false); if (ShenandoahVerify) { + verifier()->verify_roots_no_forwarded(); verifier()->verify_after_updaterefs(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp @@ -230,6 +230,7 @@ if (ShenandoahStringDedup::is_enabled()) { StringDedup::gc_epilogue(); } + ShenandoahHeap::heap()->phase_timings()->record_workers_end(_phase); } @@ -237,29 +238,33 @@ CodeBlobClosure* blobs, uint worker_id) { + AlwaysTrueClosure always_true; ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times(); { - CLDToOopClosure clds(oops); - ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CLDGRoots, worker_id); - _cld_iterator.root_cld_do(&clds, &clds); - } - - { bool is_par = n_workers() > 1; ResourceMark rm; ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ThreadRoots, worker_id); - Threads::possibly_parallel_oops_do(is_par, oops, NULL); } + { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CLDGRoots, worker_id); + CLDToOopClosure clds(oops); + _cld_iterator.root_cld_do(&clds, &clds); + } + if (blobs != NULL) { ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CodeCacheRoots, worker_id); _coderoots_cset_iterator.possibly_parallel_blobs_do(blobs); } if (ShenandoahStringDedup::is_enabled()) { - ShenandoahForwardedIsAliveClosure is_alive; - ShenandoahStringDedup::parallel_oops_do(&is_alive, oops, worker_id); + ShenandoahStringDedup::parallel_oops_do(&always_true, oops, worker_id); + } + + { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::StringTableRoots, worker_id); + StringTable::possibly_parallel_oops_do(&_par_state_string, oops); } if (!_evacuation_tasks->is_task_claimed(SHENANDOAH_EVAC_Universe_oops_do)) { @@ -267,10 +272,6 @@ Universe::oops_do(oops); } - if (!_evacuation_tasks->is_task_claimed(SHENANDOAH_RP_PS_JNIHandles_oops_do)) { - ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::JNIRoots, worker_id); - JNIHandles::oops_do(oops); - } if (!_evacuation_tasks->is_task_claimed(SHENANDOAH_EVAC_Management_oops_do)) { ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::ManagementRoots, worker_id); Management::oops_do(oops); @@ -279,8 +280,11 @@ if (!_evacuation_tasks->is_task_claimed(SHENANDOAH_EVAC_jvmti_oops_do)) { ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::JVMTIRoots, worker_id); JvmtiExport::oops_do(oops); - ShenandoahForwardedIsAliveClosure is_alive; - JvmtiExport::weak_oops_do(&is_alive, oops); + } + + if (!_evacuation_tasks->is_task_claimed(SHENANDOAH_RP_PS_JNIHandles_oops_do)) { + ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::JNIRoots, worker_id); + JNIHandles::oops_do(oops); } if (!_evacuation_tasks->is_task_claimed(SHENANDOAH_EVAC_SystemDictionary_oops_do)) { @@ -293,6 +297,9 @@ ObjectSynchronizer::oops_do(oops); } + if (!_evacuation_tasks->is_task_claimed(SHENANDOAH_EVAC_WeakProcessor_oops_do)) { + WeakProcessor::oops_do(oops); + } } uint ShenandoahRootEvacuator::n_workers() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp @@ -111,17 +111,18 @@ SubTasksDone* _evacuation_tasks; StrongRootsScope _srs; ShenandoahPhaseTimings::Phase _phase; + ShenandoahCsetCodeRootsIterator _coderoots_cset_iterator; ParallelCLDRootIterator _cld_iterator; - ShenandoahCsetCodeRootsIterator _coderoots_cset_iterator; OopStorage::ParState _par_state_string; enum Shenandoah_evacuate_roots_tasks { SHENANDOAH_EVAC_Universe_oops_do, - SHENANDOAH_EVAC_JNIHandles_oops_do, SHENANDOAH_EVAC_ObjectSynchronizer_oops_do, SHENANDOAH_EVAC_Management_oops_do, SHENANDOAH_EVAC_SystemDictionary_oops_do, SHENANDOAH_EVAC_jvmti_oops_do, + SHENANDOAH_EVAC_JNIHandles_oops_do, + SHENANDOAH_EVAC_WeakProcessor_oops_do, // Leave this one last. SHENANDOAH_EVAC_NumElements }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as @@ -24,14 +24,13 @@ #ifndef SHARE_VM_GC_SHENANDOAHUTILS_HPP #define SHARE_VM_GC_SHENANDOAHUTILS_HPP -#include "jfr/jfrEvents.hpp" - #include "gc/shared/gcCause.hpp" +#include "gc/shared/vmGCOperations.hpp" #include "gc/shared/isGCActiveMark.hpp" #include "gc/shared/suspendibleThreadSet.hpp" -#include "gc/shared/vmGCOperations.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" +#include "jfr/jfrEvents.hpp" #include "memory/allocation.hpp" #include "runtime/safepoint.hpp" #include "runtime/vmThread.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -919,3 +919,31 @@ _verify_gcstate_stable // full gc cleaned up everything ); } + +class ShenandoahVerifyNoForwared : public OopClosure { +private: + template + void do_oop_work(T* p) { + T o = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + oop fwd = (oop) ShenandoahBrooksPointer::get_raw_unchecked(obj); + if (obj != fwd) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL, + "Verify Roots", "Should not be forwarded", __FILE__, __LINE__); + } + } + } + +public: + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } +}; + +void ShenandoahVerifier::verify_roots_no_forwarded() { + guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens"); + ShenandoahRootProcessor rp(_heap, 1, ShenandoahPhaseTimings::_num_phases); // no need for stats + ShenandoahVerifyNoForwared cl; + rp.process_all_roots_slow(&cl); +} + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -186,6 +186,9 @@ void verify_after_traversal(); void verify_after_degenerated(); void verify_generic(VerifyOption option); + + // Roots should only contain to-space oops + void verify_roots_no_forwarded(); }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHVERIFIER_HPP