--- old/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp 2020-06-18 12:30:30.381769078 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp 2020-06-18 12:30:29.921756393 +0200 @@ -1044,28 +1044,79 @@ } } +class ShenandoahCheckCleanGCLABClosure : public ThreadClosure { +public: + ShenandoahCheckCleanGCLABClosure() {} + void do_thread(Thread* thread) { + PLAB* gclab = ShenandoahThreadLocalData::gclab(thread); + assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name()); + assert(gclab->words_remaining() == 0, "GCLAB should not need retirement"); + } +}; + class ShenandoahRetireGCLABClosure : public ThreadClosure { +private: + bool const _resize; public: + ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {} void do_thread(Thread* thread) { PLAB* gclab = ShenandoahThreadLocalData::gclab(thread); assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name()); gclab->retire(); + if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) { + ShenandoahThreadLocalData::set_gclab_size(thread, 0); + } } }; -void ShenandoahHeap::make_parsable(bool retire_tlabs) { - if (UseTLAB) { - CollectedHeap::ensure_parsability(retire_tlabs); +void ShenandoahHeap::labs_make_parsable() { + assert(UseTLAB, "Only call with UseTLAB"); + + ShenandoahRetireGCLABClosure cl(false); + + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { + ThreadLocalAllocBuffer& tlab = t->tlab(); + tlab.make_parsable(); + cl.do_thread(t); } - ShenandoahRetireGCLABClosure cl; + + workers()->threads_do(&cl); +} + +void ShenandoahHeap::tlabs_retire(bool resize) { + assert(UseTLAB, "Only call with UseTLAB"); + assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled"); + + ThreadLocalAllocStats stats; + + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { + ThreadLocalAllocBuffer& tlab = t->tlab(); + tlab.retire(&stats); + if (resize) { + tlab.resize(); + } + } + + stats.publish(); + +#ifdef ASSERT + ShenandoahCheckCleanGCLABClosure cl; for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { cl.do_thread(t); } workers()->threads_do(&cl); +#endif } -void ShenandoahHeap::resize_tlabs() { - CollectedHeap::resize_all_tlabs(); +void ShenandoahHeap::gclabs_retire(bool resize) { + assert(UseTLAB, "Only call with UseTLAB"); + assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled"); + + ShenandoahRetireGCLABClosure cl(resize); + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { + cl.do_thread(t); + } + workers()->threads_do(&cl); } class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask { @@ -1122,25 +1173,6 @@ return ShenandoahHeapRegion::max_tlab_size_words(); } -class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure { -public: - void do_thread(Thread* thread) { - PLAB* gclab = ShenandoahThreadLocalData::gclab(thread); - gclab->retire(); - if (ShenandoahThreadLocalData::gclab_size(thread) > 0) { - ShenandoahThreadLocalData::set_gclab_size(thread, 0); - } - } -}; - -void ShenandoahHeap::retire_and_reset_gclabs() { - ShenandoahRetireAndResetGCLABClosure cl; - for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { - cl.do_thread(t); - } - workers()->threads_do(&cl); -} - void ShenandoahHeap::collect(GCCause::Cause cause) { control_thread()->request_gc(cause); } @@ -1173,8 +1205,8 @@ } void ShenandoahHeap::prepare_for_verify() { - if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { - make_parsable(false); + if (SafepointSynchronize::is_at_safepoint() && UseTLAB) { + labs_make_parsable(); } } @@ -1264,7 +1296,7 @@ * This is public API, used in preparation of object_iterate(). * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can - * control, we call SH::make_tlabs_parsable(). + * control, we call SH::tlabs_retire, SH::gclabs_retire. */ void ShenandoahHeap::ensure_parsability(bool retire_tlabs) { // No-op. @@ -1420,10 +1452,13 @@ } set_concurrent_mark_in_progress(true); - // We need to reset all TLABs because we'd lose marks on all objects allocated in them. - { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable); - make_parsable(true); + + // We need to reset all TLABs because they might be below the TAMS, and we need to mark + // the objects in them. Do not let mutators allocate any new objects in their current TLABs. + // It is also a good place to resize the TLAB sizes for future allocations. + if (UseTLAB) { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_manage_tlabs); + tlabs_retire(ResizeTLAB); } { @@ -1437,11 +1472,6 @@ concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots); - if (UseTLAB) { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs); - resize_tlabs(); - } - if (ShenandoahPacing) { pacer()->setup_for_mark(); } @@ -1539,9 +1569,9 @@ // which would be outside the collection set, so no cset writes would happen there. // Weaker one: new allocations would happen past update watermark, and so less work would // be needed for reference updates (would update the large filler instead). - { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::retire_tlabs); - make_parsable(true); + if (UseTLAB) { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_manage_labs); + tlabs_retire(false); } { @@ -1917,10 +1947,6 @@ metrics.snap_before(); full_gc()->do_it(cause); - if (UseTLAB) { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs); - resize_all_tlabs(); - } metrics.snap_after(); @@ -2511,9 +2537,12 @@ set_evacuation_in_progress(false); - { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs); - retire_and_reset_gclabs(); + // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to + // make them parsable for update code to work correctly. Plus, we can compute new sizes + // for future GCLABs here. + if (UseTLAB) { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_manage_gclabs); + gclabs_retire(true); } if (ShenandoahVerify) { --- old/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp 2020-06-18 12:30:31.821808767 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp 2020-06-18 12:30:31.429797964 +0200 @@ -595,7 +595,6 @@ inline HeapWord* allocate_from_gclab(Thread* thread, size_t size); HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size); HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size); - void retire_and_reset_gclabs(); public: HeapWord* allocate_memory(ShenandoahAllocRequest& request); @@ -615,10 +614,11 @@ size_t max_tlab_size() const; size_t tlab_used(Thread* ignored) const; - void resize_tlabs(); + void ensure_parsability(bool retire_labs); - void ensure_parsability(bool retire_tlabs); - void make_parsable(bool retire_tlabs); + void labs_make_parsable(); + void tlabs_retire(bool resize); + void gclabs_retire(bool resize); // ---------- Marking support // --- old/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp 2020-06-18 12:30:33.117844476 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp 2020-06-18 12:30:32.629831029 +0200 @@ -140,7 +140,10 @@ _preserved_marks->init(heap->workers()->active_workers()); } - heap->make_parsable(true); + if (UseTLAB) { + heap->gclabs_retire(true); + heap->tlabs_retire(true); + } OrderAccess::fence(); --- old/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp 2020-06-18 12:30:34.425880497 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp 2020-06-18 12:30:34.005868933 +0200 @@ -60,11 +60,10 @@ \ f(init_mark_gross, "Pause Init Mark (G)") \ f(init_mark, "Pause Init Mark (N)") \ - f(make_parsable, " Make Parsable") \ + f(init_manage_tlabs, " Manage TLABs") \ f(init_update_region_states, " Update Region States") \ f(scan_roots, " Scan Roots") \ SHENANDOAH_PAR_PHASE_DO(scan_, " S: ", f) \ - f(resize_tlabs, " Resize TLABs") \ \ f(conc_mark, "Concurrent Marking") \ f(conc_mark_roots, " Roots ") \ @@ -86,7 +85,7 @@ SHENANDOAH_PAR_PHASE_DO(purge_weak_par_, " WR: ", f) \ f(purge_cldg, " CLDG") \ f(final_update_region_states, " Update Region States") \ - f(retire_tlabs, " Retire TLABs") \ + f(final_manage_labs, " Manage GC/TLABs") \ f(choose_cset, " Choose Collection Set") \ f(final_rebuild_freeset, " Rebuild Free Set") \ f(init_evac, " Initial Evacuation") \ @@ -113,7 +112,7 @@ \ f(init_update_refs_gross, "Pause Init Update Refs (G)") \ f(init_update_refs, "Pause Init Update Refs (N)") \ - f(init_update_refs_retire_gclabs, " Retire GCLABs") \ + f(init_update_refs_manage_gclabs, " Manage GCLABs") \ \ f(conc_update_refs, "Concurrent Update Refs") \ \ @@ -166,7 +165,6 @@ f(full_gc_copy_objects_humong, " Humongous Objects") \ f(full_gc_copy_objects_reset_complete, " Reset Complete Bitmap") \ f(full_gc_copy_objects_rebuild, " Rebuild Region Sets") \ - f(full_gc_resize_tlabs, " Resize TLABs") \ f(full_gc_heapdump_post, " Post Heap Dump") \ \ f(conc_uncommit, "Concurrent Uncommit") \ --- old/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp 2020-06-18 12:30:35.817918816 +0200 +++ new/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp 2020-06-18 12:30:35.373906595 +0200 @@ -690,7 +690,10 @@ } OrderAccess::fence(); - _heap->make_parsable(false); + + if (UseTLAB) { + _heap->labs_make_parsable(); + } // Allocate temporary bitmap for storing marking wavefront: _verification_bit_map->clear();