# HG changeset patch # User shade # Date 1589391710 -7200 # Wed May 13 19:41:50 2020 +0200 # Node ID 00f7d9bf729844f2d0d18a21a91c77acb862f0fb # Parent f143729ca00ec14a98ea5c7f73acba88da97746e 8240870: Shenandoah: merge evac and update phases Reviewed-by: XXX diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -109,9 +109,17 @@ return (on_weak_ref || unknown) && keep_alive; } -oop ShenandoahBarrierSet::load_reference_barrier_not_null(oop obj) { +oop ShenandoahBarrierSet::load_reference_barrier_not_null(oop obj, oop* load_addr) { if (ShenandoahLoadRefBarrier && _heap->has_forwarded_objects()) { - return load_reference_barrier_impl(obj); + return load_reference_barrier_impl(obj, load_addr); + } else { + return obj; + } +} + +oop ShenandoahBarrierSet::load_reference_barrier_not_null(oop obj, narrowOop* load_addr) { + if (ShenandoahLoadRefBarrier && _heap->has_forwarded_objects()) { + return load_reference_barrier_impl(obj, load_addr); } else { return obj; } @@ -119,13 +127,30 @@ oop ShenandoahBarrierSet::load_reference_barrier(oop obj) { if (obj != NULL) { - return load_reference_barrier_not_null(obj); + return load_reference_barrier_not_null(obj, (oop*)NULL); } else { return obj; } } -oop ShenandoahBarrierSet::load_reference_barrier_impl(oop obj) { +oop ShenandoahBarrierSet::load_reference_barrier(oop obj, oop* load_addr) { + if (obj != NULL) { + return load_reference_barrier_not_null(obj, load_addr); + } else { + return obj; + } +} + +oop ShenandoahBarrierSet::load_reference_barrier(oop obj, narrowOop* load_addr) { + if (obj != NULL) { + return load_reference_barrier_not_null(obj, load_addr); + } else { + return obj; + } +} + +template +oop ShenandoahBarrierSet::load_reference_barrier_impl(oop obj, T* load_addr) { assert(ShenandoahLoadRefBarrier, "should be enabled"); if (!CompressedOops::is_null(obj)) { bool evac_in_progress = _heap->is_evacuation_in_progress(); @@ -135,7 +160,7 @@ obj == fwd) { Thread *t = Thread::current(); ShenandoahEvacOOMScope oom_evac_scope; - return _heap->evacuate_object(obj, t); + return _heap->evacuate_object_recursively(obj, load_addr, t); } else { return fwd; } @@ -203,8 +228,9 @@ } } - oop fwd = load_reference_barrier_not_null(obj); + oop fwd = load_reference_barrier_not_null(obj, load_addr); if (load_addr != NULL && fwd != obj) { + // TODO: superfluous here? LRB updates it for us. // Since we are here and we know the load address, update the reference. ShenandoahHeap::cas_oop(fwd, load_addr, obj); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -94,7 +94,12 @@ inline void enqueue(oop obj); oop load_reference_barrier(oop obj); - oop load_reference_barrier_not_null(oop obj); + + oop load_reference_barrier(oop obj, oop* load_addr); + oop load_reference_barrier(oop obj, narrowOop* load_addr); + + oop load_reference_barrier_not_null(oop obj, oop* load_addr); + oop load_reference_barrier_not_null(oop obj, narrowOop* load_addr); template inline oop load_reference_barrier_mutator(oop obj, T* load_addr); @@ -117,7 +122,8 @@ template inline void arraycopy_work(T* src, size_t count); - oop load_reference_barrier_impl(oop obj); + template + oop load_reference_barrier_impl(oop obj, T* load_addr); template oop load_reference_barrier_native_impl(oop obj, T* load_addr); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -63,7 +63,7 @@ assert(_heap->is_evacuation_in_progress(), "evac should be in progress"); ShenandoahEvacOOMScope scope; - fwd = _heap->evacuate_object(obj, Thread::current()); + fwd = _heap->evacuate_object_recursively(obj, load_addr, Thread::current()); } if (load_addr != NULL && fwd != obj) { @@ -150,7 +150,7 @@ oop value = Raw::oop_load_in_heap(addr); if (value != NULL) { ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); - value = bs->load_reference_barrier_not_null(value); + value = bs->load_reference_barrier_not_null(value, addr); bs->keep_alive_if_weak(value); } return value; @@ -161,7 +161,11 @@ oop value = Raw::oop_load_in_heap_at(base, offset); if (value != NULL) { ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set(); - value = bs->load_reference_barrier_not_null(value); + if (UseCompressedOops) { + value = bs->load_reference_barrier_not_null(value, (narrowOop*)Raw::field_addr(base, offset)); + } else { + value = bs->load_reference_barrier_not_null(value, (oop*)Raw::field_addr(base, offset)); + } bs->keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset), value); } @@ -210,7 +214,7 @@ // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway, // because it must be the previous value. if (res != NULL) { - res = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res); + res = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(res, (oop*)NULL); // TODO: really NULL? bs->satb_enqueue(res); } return res; @@ -238,7 +242,7 @@ // Note: We don't need a keep-alive-barrier here. We already enqueue any loaded reference for SATB anyway, // because it must be the previous value. if (previous != NULL) { - previous = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous); + previous = ShenandoahBarrierSet::barrier_set()->load_reference_barrier_not_null(previous, (oop*)NULL); // TODO: really NULL? bs->satb_enqueue(previous); } return previous; @@ -292,7 +296,7 @@ if (HAS_FWD && cset->is_in(obj)) { oop fwd = resolve_forwarded_not_null(obj); if (EVAC && obj == fwd) { - fwd = _heap->evacuate_object(obj, thread); + fwd = _heap->evacuate_object_recursively(obj, elem_ptr, thread); } assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded"); oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp @@ -49,7 +49,7 @@ if (HAS_FWD && _cset->is_in(obj)) { oop fwd = _bs->resolve_forwarded_not_null(obj); if (EVAC && obj == fwd) { - fwd = _heap->evacuate_object(obj, _thread); + fwd = _heap->evacuate_object_recursively(obj, p,_thread); } assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded"); ShenandoahHeap::cas_oop(fwd, p, o); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp @@ -89,7 +89,8 @@ template void ShenandoahEvacuateUpdateRootsClosure::do_oop_work(T* p) { assert(_heap->is_concurrent_weak_root_in_progress() || - _heap->is_concurrent_strong_root_in_progress(), + _heap->is_concurrent_strong_root_in_progress() || + _heap->is_degenerated_gc_in_progress(), "Only do this in root processing phase"); T o = RawAccess<>::oop_load(p); @@ -100,7 +101,7 @@ shenandoah_assert_marked(p, obj); oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); if (resolved == obj) { - resolved = _heap->evacuate_object(obj, _thread); + resolved = _heap->evacuate_object_recursively(obj, p, _thread); } RawAccess::oop_store(p, resolved); } @@ -132,7 +133,7 @@ shenandoah_assert_marked(p, obj); oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); if (resolved == obj) { - resolved = _heap->evacuate_object(obj, _thread); + resolved = _heap->evacuate_object_recursively(obj, p, _thread); } Atomic::cmpxchg(p, obj, resolved); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -313,31 +313,31 @@ // heuristics says there are no regions to compact, and all the collection comes from immediately // reclaimable regions. // - // ................................................................................................ + // ........................................................................................ // - // (immediate garbage shortcut) Concurrent GC - // /-------------------------------------------\ - // | | - // | | - // | | - // | v - // [START] ----> Conc Mark ----o----> Conc Evac --o--> Conc Update-Refs ---o----> [END] - // | | | ^ - // | (af) | (af) | (af) | - // ..................|....................|.................|..............|....................... - // | | | | - // | | | | Degenerated GC - // v v v | - // STW Mark ----------> STW Evac ----> STW Update-Refs ----->o - // | | | ^ - // | (af) | (af) | (af) | - // ..................|....................|.................|..............|....................... - // | | | | - // | v | | Full GC - // \------------------->o<----------------/ | - // | | - // v | - // Full GC --------------------------/ + // (immediate garbage shortcut) Concurrent GC + // /-----------------------------------\ + // | | + // | | + // | | + // | v + // [START] ----> Conc Mark ----o---> Conc Evac-Update ---------> [END] + // | | ^ + // | (af) | (af) | + // ..................|.....................|.......................|....................... + // | | | + // | | | Degenerated GC + // v v | + // STW Mark ---------> STW Evac-Update ------------->o + // | | ^ + // | (af) | (af) | + // ..................|.....................|.......................|....................... + // | | | + // | v | Full GC + // \-------------------->o | + // | | + // v | + // Full GC ------------------/ // ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -394,16 +394,13 @@ // This may be skipped if there is nothing to evacuate. // If so, evac_in_progress would be unset by collection set preparation code. if (heap->is_evacuation_in_progress()) { - // Concurrently evacuate - heap->entry_evac(); - if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_evac)) return; + heap->vmop_entry_init_evac_update(); - // Perform update-refs phase. - heap->vmop_entry_init_updaterefs(); - heap->entry_updaterefs(); - if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_updaterefs)) return; + // Concurrently evacuate and update the refs + heap->entry_evac_update(); + if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_evac_update)) return; - heap->vmop_entry_final_updaterefs(); + heap->vmop_entry_final_evac_update(); // Update references freed up collection set, kick the cleanup to reclaim the space. heap->entry_cleanup_complete(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -175,9 +175,10 @@ // Record actual allocation size req.set_actual_size(size); - if (req.is_gc_alloc()) { - r->set_update_watermark(r->top()); - } + // FIXME: No need, recursive evacuation handles this for normal? +// if (req.is_gc_alloc()) { +// r->set_update_watermark(r->top()); +// } } if (result == NULL || has_no_alloc_capacity(r)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -924,6 +924,61 @@ return NULL; } +typedef Stack ShenandoahEvacUpdateStack; + +class ShenandoahEvacuateContentsOopClosure : public OopIterateClosure { +private: + ShenandoahHeap* const _heap; + ShenandoahEvacUpdateStack* const _stack; + + template + void do_work(T* p) { + oop fwd = _heap->evac_update_with_forwarded(p); + if (fwd != NULL) { + _stack->push(fwd); + } + } + +public: + ShenandoahEvacuateContentsOopClosure(ShenandoahEvacUpdateStack* stack) : _heap(ShenandoahHeap::heap()), _stack(stack) {} + + void do_oop(oop* p) { do_work(p); } + void do_oop(narrowOop* p) { do_work(p); } + + virtual bool do_metadata() { return false; } + virtual void do_klass(Klass *k) { } + virtual void do_cld(ClassLoaderData *cld) { } +}; + +template +oop ShenandoahHeap::evacuate_object_recursively(oop obj, T* load_addr, Thread* thread) { + assert(load_addr != NULL || !is_in(load_addr), "Should know the on-heap address."); + + oop fwd = obj; + if (in_collection_set(obj)) { + fwd = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); + if (obj == fwd) { + fwd = evacuate_object(obj, thread); + } + // TODO: note, we also filter out-of-heap stores + if (load_addr != NULL && fwd != obj && is_in(load_addr)) { + ShenandoahHeap::cas_oop(fwd, load_addr, obj); + } + } + + Stack stack; + + ShenandoahEvacuateContentsOopClosure cl(&stack); + fwd->oop_iterate(&cl); + + while (!stack.is_empty()) { + oop obj = stack.pop(); + obj->oop_iterate(&cl); + } + + return fwd; +} + class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure { private: ShenandoahHeap* const _heap; @@ -933,25 +988,25 @@ _heap(heap), _thread(Thread::current()) {} void do_object(oop p) { - shenandoah_assert_marked(NULL, p); - if (!p->is_forwarded()) { - _heap->evacuate_object(p, _thread); - } + shenandoah_assert_not_in_cset(NULL, p); + + // fine to pass NULL as source address here, since the object is never in collection set + _heap->evacuate_object_recursively(p, (oop*)NULL, _thread); } }; class ShenandoahEvacuationTask : public AbstractGangTask { private: ShenandoahHeap* const _sh; - ShenandoahCollectionSet* const _cs; + ShenandoahRegionIterator* const _regions; bool _concurrent; public: ShenandoahEvacuationTask(ShenandoahHeap* sh, - ShenandoahCollectionSet* cs, + ShenandoahRegionIterator* regions, bool concurrent) : AbstractGangTask("Parallel Evacuation Task"), _sh(sh), - _cs(cs), + _regions(regions), _concurrent(concurrent) {} @@ -971,18 +1026,31 @@ private: void do_work() { ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh); - ShenandoahHeapRegion* r; - while ((r =_cs->claim_next()) != NULL) { - assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index()); - _sh->marked_object_iterate(r, &cl); - + ShenandoahHeapRegion *r = _regions->next(); + ShenandoahMarkingContext *const ctx = _sh->complete_marking_context(); + while (r != NULL) { + HeapWord* update_watermark = r->get_update_watermark(); + assert (update_watermark >= r->bottom(), "sanity"); + + if (r->is_humongous()) { + if (r->is_humongous_start()) { + HeapWord* bottom = r->bottom(); + oop obj = oop(bottom); + if (update_watermark > bottom && ctx->is_marked(obj)) { + cl.do_object(obj); + } + } + } else if (r->is_active() && !r->is_cset()) { + _sh->marked_object_iterate(r, &cl, update_watermark); + } + r->set_update_watermark(r->bottom()); if (ShenandoahPacing) { - _sh->pacer()->report_evac(r->used() >> LogHeapWordSize); + _sh->pacer()->report_evac_update(pointer_delta(update_watermark, r->bottom())); } - if (_sh->check_cancelled_gc_and_yield(_concurrent)) { break; } + r = _regions->next(); } } }; @@ -1570,12 +1638,11 @@ if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) { ShenandoahCodeRoots::arm_nmethods(); } - evacuate_and_update_roots(); } - if (ShenandoahPacing) { - pacer()->setup_for_evac(); - } + // Need to evac all roots, because we would not visit them in collection set + // with evac-update otherwise. + evacuate_and_update_roots(); if (ShenandoahVerify) { ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::None; @@ -1591,6 +1658,11 @@ verifier()->verify_roots_no_forwarded_except(types); verifier()->verify_during_evacuation(); } + + if (ShenandoahPacing) { + pacer()->setup_for_evac_update(); // TODO: Really? + } + } else { if (ShenandoahVerify) { verifier()->verify_after_concmark(); @@ -1617,20 +1689,16 @@ } } -void ShenandoahHeap::op_conc_evac() { - ShenandoahEvacuationTask task(this, _collection_set, true); +void ShenandoahHeap::op_conc_evac_update() { + ShenandoahEvacuationTask task(this, &_update_refs_iterator,true); workers()->run_task(&task); } -void ShenandoahHeap::op_stw_evac() { - ShenandoahEvacuationTask task(this, _collection_set, false); +void ShenandoahHeap::op_stw_evac_update() { + ShenandoahEvacuationTask task(this, &_update_refs_iterator, false); workers()->run_task(&task); } -void ShenandoahHeap::op_updaterefs() { - update_heap_references(true); -} - void ShenandoahHeap::op_cleanup_early() { free_set()->recycle_trash(); } @@ -1712,7 +1780,7 @@ } else if (_evac_in_progress && _heap->in_collection_set(obj)) { oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); if (resolved == obj) { - resolved = _heap->evacuate_object(obj, _thread); + resolved = _heap->evacuate_object_recursively(obj, p, _thread); } Atomic::cmpxchg(p, obj, resolved); assert(_heap->cancelled_gc() || @@ -1961,11 +2029,18 @@ op_cleanup_early(); - case _degenerated_evac: + case _degenerated_evac_update: // If heuristics thinks we should do the cycle, this flag would be set, // and we can do evacuation. Otherwise, it would be the shortcut cycle. if (is_evacuation_in_progress()) { + if (point == _degenerated_evac_update) { + // TODO: Cannot handle it properly yet, Full GC. + cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc); + op_degenerated_fail(); + return; + } + // Degeneration under oom-evac protocol might have left some objects in // collection set un-evacuated. Restart evacuation from the beginning to // capture all objects. For all the objects that are already evacuated, @@ -1996,26 +2071,15 @@ collection_set()->clear_current_index(); } - op_stw_evac(); + op_init_evac_update(); if (cancelled_gc()) { op_degenerated_fail(); return; } } - // If heuristics thinks we should do the cycle, this flag would be set, - // and we need to do update-refs. Otherwise, it would be the shortcut cycle. if (has_forwarded_objects()) { - op_init_updaterefs(); - if (cancelled_gc()) { - op_degenerated_fail(); - return; - } - } - - case _degenerated_updaterefs: - if (has_forwarded_objects()) { - op_final_updaterefs(); + op_final_evac_update(); if (cancelled_gc()) { op_degenerated_fail(); return; @@ -2437,74 +2501,19 @@ return _verifier; } -template -class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { -private: - T cl; - ShenandoahHeap* _heap; - ShenandoahRegionIterator* _regions; - bool _concurrent; -public: - ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) : - AbstractGangTask("Concurrent Update References Task"), - cl(T()), - _heap(ShenandoahHeap::heap()), - _regions(regions), - _concurrent(concurrent) { - } - - void work(uint worker_id) { - if (_concurrent) { - ShenandoahConcurrentWorkerSession worker_session(worker_id); - ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); - do_work(); - } else { - ShenandoahParallelWorkerSession worker_session(worker_id); - do_work(); - } - } - -private: - void do_work() { - ShenandoahHeapRegion* r = _regions->next(); - ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); - while (r != NULL) { - HeapWord* update_watermark = r->get_update_watermark(); - assert (update_watermark >= r->bottom(), "sanity"); - if (r->is_active() && !r->is_cset()) { - _heap->marked_object_oop_iterate(r, &cl, update_watermark); - } - if (ShenandoahPacing) { - _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom())); - } - if (_heap->check_cancelled_gc_and_yield(_concurrent)) { - return; - } - r = _regions->next(); - } - } -}; - -void ShenandoahHeap::update_heap_references(bool concurrent) { - ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator, concurrent); - workers()->run_task(&task); -} - -void ShenandoahHeap::op_init_updaterefs() { +void ShenandoahHeap::op_init_evac_update() { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint"); - set_evacuation_in_progress(false); - { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_evac_update_retire_gclabs); retire_and_reset_gclabs(); } if (ShenandoahVerify) { - if (!is_degenerated_gc_in_progress()) { + if (!is_degenerated_gc_in_progress() && !cancelled_gc()) { verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots); } - verifier()->verify_before_updaterefs(); + verifier()->verify_before_evac_update(); } set_update_refs_in_progress(true); @@ -2512,7 +2521,7 @@ _update_refs_iterator.reset(); if (ShenandoahPacing) { - pacer()->setup_for_updaterefs(); + pacer()->setup_for_evac_update(); } } @@ -2545,18 +2554,20 @@ bool is_thread_safe() { return true; } }; -void ShenandoahHeap::op_final_updaterefs() { +void ShenandoahHeap::op_final_evac_update() { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint"); + set_evacuation_in_progress(false); + finish_concurrent_unloading(); // Check if there is left-over work, and finish it if (_update_refs_iterator.has_next()) { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac_update_finish_work); // Finish updating references where we left off. clear_cancelled_gc(); - update_heap_references(false); + op_stw_evac_update(); } // Clear cancelled GC, if set. On cancellation path, the block before would handle @@ -2566,14 +2577,14 @@ } assert(!cancelled_gc(), "Should have been done right before"); - if (ShenandoahVerify && !is_degenerated_gc_in_progress()) { + if (ShenandoahVerify && !is_degenerated_gc_in_progress() && !cancelled_gc()) { verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots); } if (is_degenerated_gc_in_progress()) { concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots); } else { - concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots); + concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_evac_update_roots); } // Has to be done before cset is clear @@ -2582,7 +2593,7 @@ } { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_update_region_states); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac_update_update_region_states); ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl; parallel_heap_region_iterate(&cl); @@ -2590,7 +2601,7 @@ } { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac_update_trash_cset); trash_cset_regions(); } @@ -2598,7 +2609,7 @@ set_update_refs_in_progress(false); if (ShenandoahVerify) { - verifier()->verify_after_updaterefs(); + verifier()->verify_after_evac_update(); } if (VerifyAfterGC) { @@ -2606,7 +2617,7 @@ } { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_rebuild_freeset); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac_update_rebuild_freeset); ShenandoahHeapLocker locker(lock()); _free_set->rebuild(); } @@ -2710,21 +2721,21 @@ VMThread::execute(&op); // jump to entry_final_mark under safepoint } -void ShenandoahHeap::vmop_entry_init_updaterefs() { +void ShenandoahHeap::vmop_entry_init_evac_update() { TraceCollectorStats tcs(monitoring_support()->stw_collection_counters()); - ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross); + ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_evac_update_gross); try_inject_alloc_failure(); - VM_ShenandoahInitUpdateRefs op; + VM_ShenandoahInitEvacUpdate op; VMThread::execute(&op); } -void ShenandoahHeap::vmop_entry_final_updaterefs() { +void ShenandoahHeap::vmop_entry_final_evac_update() { TraceCollectorStats tcs(monitoring_support()->stw_collection_counters()); - ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross); + ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_evac_update_gross); try_inject_alloc_failure(); - VM_ShenandoahFinalUpdateRefs op; + VM_ShenandoahFinalEvacUpdate op; VMThread::execute(&op); } @@ -2769,26 +2780,26 @@ op_final_mark(); } -void ShenandoahHeap::entry_init_updaterefs() { - static const char* msg = "Pause Init Update Refs"; - ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs); +void ShenandoahHeap::entry_init_evac_update() { + static const char* msg = "Pause Init Evac-Update"; + ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_evac_update); EventMark em("%s", msg); // No workers used in this phase, no setup required - op_init_updaterefs(); + op_init_evac_update(); } -void ShenandoahHeap::entry_final_updaterefs() { - static const char* msg = "Pause Final Update Refs"; - ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs); +void ShenandoahHeap::entry_final_evac_update() { + static const char* msg = "Pause Final Evac-Update"; + ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_evac_update); EventMark em("%s", msg); ShenandoahWorkerScope scope(workers(), ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(), - "final reference update"); - - op_final_updaterefs(); + "final evac-update"); + + op_final_evac_update(); } void ShenandoahHeap::entry_full(GCCause::Cause cause) { @@ -2833,32 +2844,19 @@ op_mark(); } -void ShenandoahHeap::entry_evac() { +void ShenandoahHeap::entry_evac_update() { TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters()); - static const char* msg = "Concurrent evacuation"; - ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac); + static const char* msg = "Concurrent evac-update"; + ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac_update); EventMark em("%s", msg); ShenandoahWorkerScope scope(workers(), - ShenandoahWorkerPolicy::calc_workers_for_conc_evac(), - "concurrent evacuation"); + ShenandoahWorkerPolicy::calc_workers_for_conc_evac_update(), + "concurrent evacuation/update"); try_inject_alloc_failure(); - op_conc_evac(); -} - -void ShenandoahHeap::entry_updaterefs() { - static const char* msg = "Concurrent update references"; - ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs); - EventMark em("%s", msg); - - ShenandoahWorkerScope scope(workers(), - ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(), - "concurrent reference update"); - - try_inject_alloc_failure(); - op_updaterefs(); + op_conc_evac_update(); } void ShenandoahHeap::entry_weak_roots() { @@ -3095,10 +3093,8 @@ return "Pause Degenerated GC (Outside of Cycle)"; case _degenerated_mark: return "Pause Degenerated GC (Mark)"; - case _degenerated_evac: - return "Pause Degenerated GC (Evacuation)"; - case _degenerated_updaterefs: - return "Pause Degenerated GC (Update Refs)"; + case _degenerated_evac_update: + return "Pause Degenerated GC (Evac Update)"; default: ShouldNotReachHere(); return "ERROR"; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -319,8 +319,7 @@ _degenerated_unset, _degenerated_outside_cycle, _degenerated_mark, - _degenerated_evac, - _degenerated_updaterefs, + _degenerated_evac_update, _DEGENERATED_LIMIT }; @@ -332,10 +331,8 @@ return "Outside of Cycle"; case _degenerated_mark: return "Mark"; - case _degenerated_evac: - return "Evacuation"; - case _degenerated_updaterefs: - return "Update Refs"; + case _degenerated_evac_update: + return "Evac Update"; default: ShouldNotReachHere(); return "ERROR"; @@ -378,8 +375,8 @@ // call the entry method below void vmop_entry_init_mark(); void vmop_entry_final_mark(); - void vmop_entry_init_updaterefs(); - void vmop_entry_final_updaterefs(); + void vmop_entry_init_evac_update(); + void vmop_entry_final_evac_update(); void vmop_entry_full(GCCause::Cause cause); void vmop_degenerated(ShenandoahDegenPoint point); @@ -387,8 +384,8 @@ // and workers for net VM operation void entry_init_mark(); void entry_final_mark(); - void entry_init_updaterefs(); - void entry_final_updaterefs(); + void entry_init_evac_update(); + void entry_final_evac_update(); void entry_full(GCCause::Cause cause); void entry_degenerated(int point); @@ -401,8 +398,7 @@ void entry_class_unloading(); void entry_strong_roots(); void entry_cleanup_early(); - void entry_evac(); - void entry_updaterefs(); + void entry_evac_update(); void entry_cleanup_complete(); void entry_uncommit(double shrink_before); @@ -410,8 +406,8 @@ // Actual work for the phases void op_init_mark(); void op_final_mark(); - void op_init_updaterefs(); - void op_final_updaterefs(); + void op_init_evac_update(); + void op_final_evac_update(); void op_full(GCCause::Cause cause); void op_degenerated(ShenandoahDegenPoint point); void op_degenerated_fail(); @@ -424,9 +420,8 @@ void op_class_unloading(); void op_strong_roots(); void op_cleanup_early(); - void op_conc_evac(); - void op_stw_evac(); - void op_updaterefs(); + void op_conc_evac_update(); + void op_stw_evac_update(); void op_cleanup_complete(); void op_uncommit(double shrink_before); @@ -646,9 +641,6 @@ template inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit); - template - inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit); - void reset_mark_bitmap(); // SATB barriers hooks @@ -688,6 +680,9 @@ // by this thread, or by some other thread. inline oop evacuate_object(oop src, Thread* thread); + template + oop evacuate_object_recursively(oop obj, T* load_addr, Thread* thread); + // Call before/after evacuation. void enter_evacuation(); void leave_evacuation(); @@ -717,7 +712,6 @@ private: void trash_cset_regions(); - void update_heap_references(bool concurrent); // ---------- Testing helpers functions // diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -115,23 +115,21 @@ inline oop ShenandoahHeap::evac_update_with_forwarded(T* p) { T o = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(o)) { - oop heap_oop = CompressedOops::decode_not_null(o); - if (in_collection_set(heap_oop)) { - oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop); - if (forwarded_oop == heap_oop) { - forwarded_oop = evacuate_object(heap_oop, Thread::current()); + oop obj = CompressedOops::decode_not_null(o); + if (in_collection_set(obj)) { + oop fwd = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); + if (fwd == obj) { + fwd = evacuate_object(obj, Thread::current()); } - oop prev = cas_oop(forwarded_oop, p, heap_oop); - if (prev == heap_oop) { - return forwarded_oop; - } else { - return NULL; + if (fwd != obj) { // evac failure otherwise + oop prev = cas_oop(fwd, p, obj); + if (prev == obj) { + return fwd; + } } } - return heap_oop; - } else { - return NULL; } + return NULL; } inline oop ShenandoahHeap::cas_oop(oop n, oop* addr, oop c) { @@ -481,45 +479,6 @@ } } -template -class ShenandoahObjectToOopClosure : public ObjectClosure { - T* _cl; -public: - ShenandoahObjectToOopClosure(T* cl) : _cl(cl) {} - - void do_object(oop obj) { - obj->oop_iterate(_cl); - } -}; - -template -class ShenandoahObjectToOopBoundedClosure : public ObjectClosure { - T* _cl; - MemRegion _bounds; -public: - ShenandoahObjectToOopBoundedClosure(T* cl, HeapWord* bottom, HeapWord* top) : - _cl(cl), _bounds(bottom, top) {} - - void do_object(oop obj) { - obj->oop_iterate(_cl, _bounds); - } -}; - -template -inline void ShenandoahHeap::marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* top) { - if (region->is_humongous()) { - HeapWord* bottom = region->bottom(); - if (top > bottom) { - region = region->humongous_start_region(); - ShenandoahObjectToOopBoundedClosure objs(cl, bottom, top); - marked_object_iterate(region, &objs); - } - } else { - ShenandoahObjectToOopClosure objs(cl); - marked_object_iterate(region, &objs, top); - } -} - inline ShenandoahHeapRegion* const ShenandoahHeap::get_region(size_t region_idx) const { if (region_idx < _num_regions) { return _regions[region_idx]; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPacer.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPacer.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahPacer.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPacer.cpp @@ -67,7 +67,7 @@ size_t taxable = free - non_taxable; double tax = 1.0 * live / taxable; // base tax for available free space - tax *= 3; // mark is phase 1 of 3, claim 1/3 of free for it + tax *= 2; // mark is phase 1 of 2, claim 1/2 of free for it tax *= ShenandoahPacingSurcharge; // additional surcharge to help unclutter heap restart_with(non_taxable, tax); @@ -80,31 +80,7 @@ tax); } -void ShenandoahPacer::setup_for_evac() { - assert(ShenandoahPacing, "Only be here when pacing is enabled"); - - size_t used = _heap->collection_set()->used(); - size_t free = _heap->free_set()->available(); - - size_t non_taxable = free * ShenandoahPacingCycleSlack / 100; - size_t taxable = free - non_taxable; - - double tax = 1.0 * used / taxable; // base tax for available free space - tax *= 2; // evac is phase 2 of 3, claim 1/2 of remaining free - tax = MAX2(1, tax); // never allocate more than GC processes during the phase - tax *= ShenandoahPacingSurcharge; // additional surcharge to help unclutter heap - - restart_with(non_taxable, tax); - - log_info(gc, ergo)("Pacer for Evacuation. Used CSet: " SIZE_FORMAT "%s, Free: " SIZE_FORMAT "%s, " - "Non-Taxable: " SIZE_FORMAT "%s, Alloc Tax Rate: %.1fx", - byte_size_in_proper_unit(used), proper_unit_for_byte_size(used), - byte_size_in_proper_unit(free), proper_unit_for_byte_size(free), - byte_size_in_proper_unit(non_taxable), proper_unit_for_byte_size(non_taxable), - tax); -} - -void ShenandoahPacer::setup_for_updaterefs() { +void ShenandoahPacer::setup_for_evac_update() { assert(ShenandoahPacing, "Only be here when pacing is enabled"); size_t used = _heap->used(); @@ -114,13 +90,13 @@ size_t taxable = free - non_taxable; double tax = 1.0 * used / taxable; // base tax for available free space - tax *= 1; // update-refs is phase 3 of 3, claim the remaining free + tax *= 1; // update-refs is phase 2 of 2, claim the remaining free tax = MAX2(1, tax); // never allocate more than GC processes during the phase tax *= ShenandoahPacingSurcharge; // additional surcharge to help unclutter heap restart_with(non_taxable, tax); - log_info(gc, ergo)("Pacer for Update Refs. Used: " SIZE_FORMAT "%s, Free: " SIZE_FORMAT "%s, " + log_info(gc, ergo)("Pacer for Evac-Update. Used: " SIZE_FORMAT "%s, Free: " SIZE_FORMAT "%s, " "Non-Taxable: " SIZE_FORMAT "%s, Alloc Tax Rate: %.1fx", byte_size_in_proper_unit(used), proper_unit_for_byte_size(used), byte_size_in_proper_unit(free), proper_unit_for_byte_size(free), diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPacer.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPacer.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahPacer.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPacer.hpp @@ -71,15 +71,13 @@ void setup_for_idle(); void setup_for_mark(); - void setup_for_evac(); - void setup_for_updaterefs(); + void setup_for_evac_update(); void setup_for_reset(); void setup_for_preclean(); inline void report_mark(size_t words); - inline void report_evac(size_t words); - inline void report_updaterefs(size_t words); + inline void report_evac_update(size_t words); inline void report_alloc(size_t words); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPacer.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPacer.inline.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahPacer.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPacer.inline.hpp @@ -33,11 +33,7 @@ report_progress_internal(words); } -inline void ShenandoahPacer::report_evac(size_t words) { - report_internal(words); -} - -inline void ShenandoahPacer::report_updaterefs(size_t words) { +inline void ShenandoahPacer::report_evac_update(size_t words) { report_internal(words); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp @@ -97,7 +97,7 @@ case init_evac: case scan_roots: case update_roots: - case final_update_refs_roots: + case final_evac_update_roots: case full_gc_scan_roots: case full_gc_update_roots: case full_gc_adjust_roots: @@ -120,7 +120,7 @@ case scan_roots: case update_roots: case init_evac: - case final_update_refs_roots: + case final_evac_update_roots: case degen_gc_update_roots: case full_gc_scan_roots: case full_gc_update_roots: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -105,22 +105,21 @@ f(conc_class_unload_purge_ec, " Exception Caches") \ f(conc_strong_roots, "Concurrent Strong Roots") \ SHENANDOAH_PAR_PHASE_DO(conc_strong_roots_, " CSR: ", f) \ - f(conc_evac, "Concurrent Evacuation") \ \ - f(init_update_refs_gross, "Pause Init Update Refs (G)") \ - f(init_update_refs, "Pause Init Update Refs (N)") \ - f(init_update_refs_retire_gclabs, " Retire GCLABs") \ + f(init_evac_update_gross, "Pause Init Evac-Update (G)") \ + f(init_evac_update, "Pause Init Evac-Update (N)") \ + f(init_evac_update_retire_gclabs, " Retire GCLABs") \ \ - f(conc_update_refs, "Concurrent Update Refs") \ + f(conc_evac_update, "Concurrent Evac-Update") \ \ - f(final_update_refs_gross, "Pause Final Update Refs (G)") \ - f(final_update_refs, "Pause Final Update Refs (N)") \ - f(final_update_refs_finish_work, " Finish Work") \ - f(final_update_refs_roots, " Update Roots") \ - SHENANDOAH_PAR_PHASE_DO(final_update_, " UR: ", f) \ - f(final_update_refs_update_region_states, " Update Region States") \ - f(final_update_refs_trash_cset, " Trash Collection Set") \ - f(final_update_refs_rebuild_freeset, " Rebuild Free Set") \ + f(final_evac_update_gross, "Pause Final Evac-Update (G)") \ + f(final_evac_update, "Pause Final Evac-Update (N)") \ + f(final_evac_update_finish_work, " Finish Work") \ + f(final_evac_update_roots, " Update Roots") \ + SHENANDOAH_PAR_PHASE_DO(final_evac_update_roots_, " EU: ", f) \ + f(final_evac_update_update_region_states, " Update Region States") \ + f(final_evac_update_trash_cset, " Trash Collection Set") \ + f(final_evac_update_rebuild_freeset, " Rebuild Free Set") \ \ f(conc_cleanup_complete, "Concurrent Cleanup") \ \ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp @@ -151,8 +151,8 @@ VM_Operation::VMOp_Type type = vm_op->type(); return type == VM_Operation::VMOp_ShenandoahInitMark || type == VM_Operation::VMOp_ShenandoahFinalMarkStartEvac || - type == VM_Operation::VMOp_ShenandoahInitUpdateRefs || - type == VM_Operation::VMOp_ShenandoahFinalUpdateRefs || + type == VM_Operation::VMOp_ShenandoahInitEvacUpdate || + type == VM_Operation::VMOp_ShenandoahFinalEvacUpdate || type == VM_Operation::VMOp_ShenandoahFullGC || type == VM_Operation::VMOp_ShenandoahDegeneratedGC; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp @@ -61,12 +61,12 @@ ShenandoahHeap::heap()->entry_degenerated(_point); } -void VM_ShenandoahInitUpdateRefs::doit() { +void VM_ShenandoahInitEvacUpdate::doit() { ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); - ShenandoahHeap::heap()->entry_init_updaterefs(); + ShenandoahHeap::heap()->entry_init_evac_update(); } -void VM_ShenandoahFinalUpdateRefs::doit() { +void VM_ShenandoahFinalEvacUpdate::doit() { ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); - ShenandoahHeap::heap()->entry_final_updaterefs(); + ShenandoahHeap::heap()->entry_final_evac_update(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp @@ -33,8 +33,8 @@ // - VM_ShenandoahInitMark: initiate concurrent marking // - VM_ShenandoahReferenceOperation: // - VM_ShenandoahFinalMarkStartEvac: finish up concurrent marking, and start evacuation -// - VM_ShenandoahInitUpdateRefs: initiate update references -// - VM_ShenandoahFinalUpdateRefs: finish up update references +// - VM_ShenandoahInitEvacUpdate: initiate evac-update references +// - VM_ShenandoahFinalEvacUpdate: finish up evac-update references // - VM_ShenandoahFullGC: do full GC class VM_ShenandoahOperation : public VM_Operation { @@ -89,19 +89,19 @@ virtual void doit(); }; -class VM_ShenandoahInitUpdateRefs: public VM_ShenandoahOperation { +class VM_ShenandoahInitEvacUpdate: public VM_ShenandoahOperation { public: - VM_ShenandoahInitUpdateRefs() : VM_ShenandoahOperation() {}; - VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahInitUpdateRefs; } - const char* name() const { return "Shenandoah Init Update References"; } + VM_ShenandoahInitEvacUpdate() : VM_ShenandoahOperation() {}; + VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahInitEvacUpdate; } + const char* name() const { return "Shenandoah Init Evac-Update"; } virtual void doit(); }; -class VM_ShenandoahFinalUpdateRefs: public VM_ShenandoahOperation { +class VM_ShenandoahFinalEvacUpdate: public VM_ShenandoahOperation { public: - VM_ShenandoahFinalUpdateRefs() : VM_ShenandoahOperation() {}; - VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalUpdateRefs; } - const char* name() const { return "Shenandoah Final Update References"; } + VM_ShenandoahFinalEvacUpdate() : VM_ShenandoahOperation() {}; + VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalEvacUpdate; } + const char* name() const { return "Shenandoah Final Evac-Update"; } virtual void doit(); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -639,6 +639,10 @@ enabled = true; expected = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION; break; + case _verify_gcstate_evac_update: + enabled = true; + expected = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::UPDATEREFS; + break; case _verify_gcstate_stable: enabled = true; expected = ShenandoahHeap::STABLE; @@ -865,22 +869,22 @@ ); } -void ShenandoahVerifier::verify_before_updaterefs() { +void ShenandoahVerifier::verify_before_evac_update() { verify_at_safepoint( - "Before Updating References", + "Before Evac Update", _verify_forwarded_allow, // forwarded references allowed _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well - _verify_cset_forwarded, // all cset refs are fully forwarded + _verify_cset_disable, // all cset refs are fully forwarded _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_notrash, // trash regions have been recycled already - _verify_gcstate_forwarded, // evacuation should have produced some forwarded objects + _verify_gcstate_evacuation, // evacuation should have produced some forwarded objects _verify_all_weak_roots ); } -void ShenandoahVerifier::verify_after_updaterefs() { +void ShenandoahVerifier::verify_after_evac_update() { verify_at_safepoint( - "After Updating References", + "After Evac Update", _verify_forwarded_none, // no forwarded references _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well _verify_cset_none, // no cset references, all updated diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -139,7 +139,10 @@ _verify_gcstate_forwarded, // Evacuation is in progress, some objects are forwarded - _verify_gcstate_evacuation + _verify_gcstate_evacuation, + + // Evac-update is in progress, some objects are forwarded + _verify_gcstate_evac_update } VerifyGCState; typedef enum { @@ -190,8 +193,8 @@ void verify_before_evacuation(); void verify_during_evacuation(); void verify_after_evacuation(); - void verify_before_updaterefs(); - void verify_after_updaterefs(); + void verify_before_evac_update(); + void verify_after_evac_update(); void verify_before_fullgc(); void verify_after_fullgc(); void verify_after_degenerated(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp @@ -30,7 +30,7 @@ uint ShenandoahWorkerPolicy::_prev_par_marking = 0; uint ShenandoahWorkerPolicy::_prev_conc_marking = 0; -uint ShenandoahWorkerPolicy::_prev_conc_evac = 0; +uint ShenandoahWorkerPolicy::_prev_conc_evac_update= 0; uint ShenandoahWorkerPolicy::_prev_conc_root_proc = 0; uint ShenandoahWorkerPolicy::_prev_fullgc = 0; uint ShenandoahWorkerPolicy::_prev_degengc = 0; @@ -73,14 +73,14 @@ return _prev_conc_root_proc; } -// Calculate workers for concurrent evacuation (concurrent GC) -uint ShenandoahWorkerPolicy::calc_workers_for_conc_evac() { - uint active_workers = (_prev_conc_evac == 0) ? ConcGCThreads : _prev_conc_evac; - _prev_conc_evac = +// Calculate workers for concurrent evac-update (concurrent GC) +uint ShenandoahWorkerPolicy::calc_workers_for_conc_evac_update() { + uint active_workers = (_prev_conc_evac_update == 0) ? ConcGCThreads : _prev_conc_evac_update; + _prev_conc_evac_update = WorkerPolicy::calc_active_conc_workers(ConcGCThreads, active_workers, Threads::number_of_non_daemon_threads()); - return _prev_conc_evac; + return _prev_conc_evac_update; } // Calculate workers for parallel fullgc @@ -103,16 +103,6 @@ return _prev_degengc; } -// Calculate workers for concurrent reference update -uint ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref() { - uint active_workers = (_prev_conc_update_ref == 0) ? ConcGCThreads : _prev_conc_update_ref; - _prev_conc_update_ref = - WorkerPolicy::calc_active_conc_workers(ConcGCThreads, - active_workers, - Threads::number_of_non_daemon_threads()); - return _prev_conc_update_ref; -} - // Calculate workers for parallel reference update uint ShenandoahWorkerPolicy::calc_workers_for_final_update_ref() { uint active_workers = (_prev_par_update_ref == 0) ? ParallelGCThreads : _prev_par_update_ref; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp --- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp @@ -32,7 +32,7 @@ static uint _prev_par_marking; static uint _prev_conc_marking; static uint _prev_conc_root_proc; - static uint _prev_conc_evac; + static uint _prev_conc_evac_update; static uint _prev_fullgc; static uint _prev_degengc; static uint _prev_conc_update_ref; @@ -53,8 +53,8 @@ // Calculate workers for concurrent root processing static uint calc_workers_for_conc_root_processing(); - // Calculate workers for concurrent evacuation (concurrent GC) - static uint calc_workers_for_conc_evac(); + // Calculate workers for concurrent evac-update (concurrent GC) + static uint calc_workers_for_conc_evac_update(); // Calculate workers for parallel full gc static uint calc_workers_for_fullgc(); @@ -62,9 +62,6 @@ // Calculate workers for parallel degenerated gc static uint calc_workers_for_stw_degenerated(); - // Calculate workers for concurrent reference update - static uint calc_workers_for_conc_update_ref(); - // Calculate workers for parallel/final reference update static uint calc_workers_for_final_update_ref(); diff --git a/src/hotspot/share/runtime/vmOperations.hpp b/src/hotspot/share/runtime/vmOperations.hpp --- a/src/hotspot/share/runtime/vmOperations.hpp +++ b/src/hotspot/share/runtime/vmOperations.hpp @@ -99,8 +99,8 @@ template(ShenandoahFullGC) \ template(ShenandoahInitMark) \ template(ShenandoahFinalMarkStartEvac) \ - template(ShenandoahInitUpdateRefs) \ - template(ShenandoahFinalUpdateRefs) \ + template(ShenandoahInitEvacUpdate) \ + template(ShenandoahFinalEvacUpdate) \ template(ShenandoahDegeneratedGC) \ template(Exit) \ template(LinuxDllLoad) \