< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

Print this page
rev 48920 : [backport] Use PLAB for evacuations instead of TLAB
rev 48921 : [backport] Fix PLAB alignment reserve

*** 25,34 **** --- 25,35 ---- #include "memory/allocation.hpp" #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/parallelCleaning.hpp" + #include "gc/shared/plab.hpp" #include "gc/shenandoah/brooksPointer.hpp" #include "gc/shenandoah/shenandoahAllocTracker.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp"
*** 371,380 **** --- 372,383 ---- _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()), _phase_timings(NULL), _alloc_tracker(NULL), _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"), _stw_memory_manager("Shenandoah Pauses", "end of GC pause"), + _mutator_gclab_stats(new PLABStats("Shenandoah mutator GCLAB stats", OldPLABSize, PLABWeight)), + _collector_gclab_stats(new PLABStats("Shenandoah collector GCLAB stats", YoungPLABSize, PLABWeight)), _memory_pool(NULL) { log_info(gc, init)("Parallel GC threads: "UINT32_FORMAT, ParallelGCThreads); log_info(gc, init)("Concurrent GC threads: "UINT32_FORMAT, ConcGCThreads); log_info(gc, init)("Parallel reference processing enabled: %s", BOOL_TO_STR(ParallelRefProcEnabled));
*** 546,572 **** } class ShenandoahInitGCLABClosure : public ThreadClosure { public: void do_thread(Thread* thread) { ! thread->gclab().initialize(true); } }; void ShenandoahHeap::post_initialize() { CollectedHeap::post_initialize(); - if (UseTLAB) { MutexLocker ml(Threads_lock); ShenandoahInitGCLABClosure init_gclabs; Threads::java_threads_do(&init_gclabs); gc_threads_do(&init_gclabs); // gclab can not be initialized early during VM startup, as it can not determinate its max_size. // Now, we will let WorkGang to initialize gclab when new worker is created. _workers->set_initialize_gclab(); - } _scm->initialize(_max_workers); _full_gc->initialize(_gc_timer); ref_processing_init(); --- 549,573 ---- } class ShenandoahInitGCLABClosure : public ThreadClosure { public: void do_thread(Thread* thread) { ! ShenandoahHeap::heap()->initialize_gclab(thread); } }; void ShenandoahHeap::post_initialize() { CollectedHeap::post_initialize(); MutexLocker ml(Threads_lock); ShenandoahInitGCLABClosure init_gclabs; Threads::java_threads_do(&init_gclabs); gc_threads_do(&init_gclabs); // gclab can not be initialized early during VM startup, as it can not determinate its max_size. // Now, we will let WorkGang to initialize gclab when new worker is created. _workers->set_initialize_gclab(); _scm->initialize(_max_workers); _full_gc->initialize(_gc_timer); ref_processing_init();
*** 675,720 **** } HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) { // Retain tlab and allocate object in shared space if // the amount free in the tlab is too large to discard. ! if (thread->gclab().free() > thread->gclab().refill_waste_limit()) { ! thread->gclab().record_slow_allocation(size); ! return NULL; ! } // Discard gclab and allocate a new one. // To minimize fragmentation, the last GCLAB may be smaller than the rest. ! size_t new_gclab_size = thread->gclab().compute_size(size); ! ! thread->gclab().clear_before_allocation(); ! ! if (new_gclab_size == 0) { ! return NULL; } // Allocate a new GCLAB... ! HeapWord* obj = allocate_new_gclab(new_gclab_size); ! if (obj == NULL) { return NULL; } if (ZeroTLAB) { // ..and clear it. ! Copy::zero_to_words(obj, new_gclab_size); } else { // ...and zap just allocated object. #ifdef ASSERT // Skip mangling the space corresponding to the object header to // ensure that the returned space is not considered parsable by // any concurrent GC thread. size_t hdr_size = oopDesc::header_size(); ! Copy::fill_to_words(obj + hdr_size, new_gclab_size - hdr_size, badHeapWordVal); #endif // ASSERT } ! thread->gclab().fill(obj, obj + size, new_gclab_size); ! return obj; } HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) { #ifdef ASSERT log_debug(gc, alloc)("Allocate new tlab, requested size = " SIZE_FORMAT " bytes", word_size * HeapWordSize); --- 676,719 ---- } HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) { // Retain tlab and allocate object in shared space if // the amount free in the tlab is too large to discard. ! PLAB* gclab = thread->gclab(); // Discard gclab and allocate a new one. // To minimize fragmentation, the last GCLAB may be smaller than the rest. ! gclab->retire(); ! // Figure out size of new GCLAB ! size_t new_gclab_size; ! if (thread->is_Java_thread()) { ! new_gclab_size = _mutator_gclab_stats->desired_plab_sz(Threads::number_of_threads()); ! } else { ! new_gclab_size = _collector_gclab_stats->desired_plab_sz(workers()->active_workers()); } // Allocate a new GCLAB... ! HeapWord* gclab_buf = allocate_new_gclab(new_gclab_size); ! if (gclab_buf == NULL) { return NULL; } if (ZeroTLAB) { // ..and clear it. ! Copy::zero_to_words(gclab_buf, new_gclab_size); } else { // ...and zap just allocated object. #ifdef ASSERT // Skip mangling the space corresponding to the object header to // ensure that the returned space is not considered parsable by // any concurrent GC thread. size_t hdr_size = oopDesc::header_size(); ! Copy::fill_to_words(gclab_buf + hdr_size, new_gclab_size - hdr_size, badHeapWordVal); #endif // ASSERT } ! gclab->set_buf(gclab_buf, new_gclab_size); ! return gclab->allocate(size); } HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) { #ifdef ASSERT log_debug(gc, alloc)("Allocate new tlab, requested size = " SIZE_FORMAT " bytes", word_size * HeapWordSize);
*** 1079,1100 **** public: ShenandoahRetireTLABClosure(bool retire) : _retire(retire) {} void do_thread(Thread* thread) { ! assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name()); ! thread->gclab().make_parsable(_retire); } }; void ShenandoahHeap::make_tlabs_parsable(bool retire_tlabs) { if (UseTLAB) { CollectedHeap::ensure_parsability(retire_tlabs); ShenandoahRetireTLABClosure cl(retire_tlabs); Threads::java_threads_do(&cl); gc_threads_do(&cl); - } } class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask { ShenandoahRootEvacuator* _rp; --- 1078,1101 ---- public: ShenandoahRetireTLABClosure(bool retire) : _retire(retire) {} void do_thread(Thread* thread) { ! PLAB* gclab = thread->gclab(); ! if (gclab != NULL) { ! gclab->retire(); ! } } }; void ShenandoahHeap::make_tlabs_parsable(bool retire_tlabs) { if (UseTLAB) { CollectedHeap::ensure_parsability(retire_tlabs); + } ShenandoahRetireTLABClosure cl(retire_tlabs); Threads::java_threads_do(&cl); gc_threads_do(&cl); } class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask { ShenandoahRootEvacuator* _rp;
*** 1202,1240 **** size_t ShenandoahHeap::max_tlab_size() const { return ShenandoahHeapRegion::max_tlab_size_bytes(); } - class ShenandoahResizeGCLABClosure : public ThreadClosure { - public: - void do_thread(Thread* thread) { - assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name()); - thread->gclab().resize(); - } - }; - - void ShenandoahHeap::resize_all_tlabs() { - CollectedHeap::resize_all_tlabs(); - - ShenandoahResizeGCLABClosure cl; - Threads::java_threads_do(&cl); - gc_threads_do(&cl); - } - class ShenandoahAccumulateStatisticsGCLABClosure : public ThreadClosure { public: void do_thread(Thread* thread) { ! assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name()); ! thread->gclab().accumulate_statistics(); ! thread->gclab().initialize_statistics(); } }; void ShenandoahHeap::accumulate_statistics_all_gclabs() { ShenandoahAccumulateStatisticsGCLABClosure cl; Threads::java_threads_do(&cl); gc_threads_do(&cl); } bool ShenandoahHeap::can_elide_tlab_store_barriers() const { return true; } --- 1203,1233 ---- size_t ShenandoahHeap::max_tlab_size() const { return ShenandoahHeapRegion::max_tlab_size_bytes(); } class ShenandoahAccumulateStatisticsGCLABClosure : public ThreadClosure { public: void do_thread(Thread* thread) { ! ShenandoahHeap* heap = ShenandoahHeap::heap(); ! PLAB* gclab = thread->gclab(); ! if (gclab != NULL) { ! if (thread->is_Java_thread()) { ! gclab->flush_and_retire_stats(heap->mutator_gclab_stats()); ! } else { ! gclab->flush_and_retire_stats(heap->collector_gclab_stats()); ! } ! } } }; void ShenandoahHeap::accumulate_statistics_all_gclabs() { ShenandoahAccumulateStatisticsGCLABClosure cl; Threads::java_threads_do(&cl); gc_threads_do(&cl); + _mutator_gclab_stats->adjust_desired_plab_sz(); + _collector_gclab_stats->adjust_desired_plab_sz(); } bool ShenandoahHeap::can_elide_tlab_store_barriers() const { return true; }
*** 1490,1500 **** accumulate_statistics_all_tlabs(); } set_concurrent_mark_in_progress(true); // We need to reset all TLABs because we'd lose marks on all objects allocated in them. ! if (UseTLAB) { ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable); make_tlabs_parsable(true); } { --- 1483,1493 ---- accumulate_statistics_all_tlabs(); } set_concurrent_mark_in_progress(true); // We need to reset all TLABs because we'd lose marks on all objects allocated in them. ! { ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable); make_tlabs_parsable(true); } {
*** 1570,1579 **** --- 1563,1573 ---- } void ShenandoahHeap::op_final_evac() { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); + accumulate_statistics_all_gclabs(); set_evacuation_in_progress(false); if (ShenandoahVerify) { verifier()->verify_after_evacuation(); } }
*** 1682,1691 **** --- 1676,1689 ---- traversal_gc()->final_traversal_collection(); } void ShenandoahHeap::op_full(GCCause::Cause cause) { full_gc()->do_it(cause); + if (UseTLAB) { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs); + resize_all_tlabs(); + } } void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) { // Degenerated GC is STW, but it can also fail. Current mechanics communicates // GC failure via cancelled_concgc() flag. So, if we detect the failure after
*** 2289,2298 **** --- 2287,2297 ---- if (ShenandoahVerify) { verifier()->verify_before_updaterefs(); } + accumulate_statistics_all_gclabs(); set_evacuation_in_progress(false); set_update_refs_in_progress(true); make_tlabs_parsable(true); if (UseShenandoahMatrix) { connection_matrix()->clear_all();
*** 2890,2894 **** --- 2889,2907 ---- } char ShenandoahHeap::gc_state() { return _gc_state.raw_value(); } + + void ShenandoahHeap::initialize_gclab(Thread* thread) { + if (thread->is_Java_thread()) { + thread->set_gclab(new PLAB(OldPLABSize)); + } else { + thread->set_gclab(new PLAB(YoungPLABSize)); + } + } + + void ShenandoahHeap::finalize_mutator_gclab(Thread* thread) { + thread->gclab()->flush_and_retire_stats(ShenandoahHeap::heap()->mutator_gclab_stats()); + delete thread->gclab(); + thread->set_gclab(NULL); + }
< prev index next >