< prev index next >
src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
Print this page
rev 48920 : [backport] Use PLAB for evacuations instead of TLAB
rev 48921 : [backport] Fix PLAB alignment reserve
*** 25,34 ****
--- 25,35 ----
#include "memory/allocation.hpp"
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/parallelCleaning.hpp"
+ #include "gc/shared/plab.hpp"
#include "gc/shenandoah/brooksPointer.hpp"
#include "gc/shenandoah/shenandoahAllocTracker.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
*** 371,380 ****
--- 372,383 ----
_gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
_phase_timings(NULL),
_alloc_tracker(NULL),
_cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
_stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
+ _mutator_gclab_stats(new PLABStats("Shenandoah mutator GCLAB stats", OldPLABSize, PLABWeight)),
+ _collector_gclab_stats(new PLABStats("Shenandoah collector GCLAB stats", YoungPLABSize, PLABWeight)),
_memory_pool(NULL)
{
log_info(gc, init)("Parallel GC threads: "UINT32_FORMAT, ParallelGCThreads);
log_info(gc, init)("Concurrent GC threads: "UINT32_FORMAT, ConcGCThreads);
log_info(gc, init)("Parallel reference processing enabled: %s", BOOL_TO_STR(ParallelRefProcEnabled));
*** 546,572 ****
}
class ShenandoahInitGCLABClosure : public ThreadClosure {
public:
void do_thread(Thread* thread) {
! thread->gclab().initialize(true);
}
};
void ShenandoahHeap::post_initialize() {
CollectedHeap::post_initialize();
- if (UseTLAB) {
MutexLocker ml(Threads_lock);
ShenandoahInitGCLABClosure init_gclabs;
Threads::java_threads_do(&init_gclabs);
gc_threads_do(&init_gclabs);
// gclab can not be initialized early during VM startup, as it can not determinate its max_size.
// Now, we will let WorkGang to initialize gclab when new worker is created.
_workers->set_initialize_gclab();
- }
_scm->initialize(_max_workers);
_full_gc->initialize(_gc_timer);
ref_processing_init();
--- 549,573 ----
}
class ShenandoahInitGCLABClosure : public ThreadClosure {
public:
void do_thread(Thread* thread) {
! ShenandoahHeap::heap()->initialize_gclab(thread);
}
};
void ShenandoahHeap::post_initialize() {
CollectedHeap::post_initialize();
MutexLocker ml(Threads_lock);
ShenandoahInitGCLABClosure init_gclabs;
Threads::java_threads_do(&init_gclabs);
gc_threads_do(&init_gclabs);
// gclab can not be initialized early during VM startup, as it can not determinate its max_size.
// Now, we will let WorkGang to initialize gclab when new worker is created.
_workers->set_initialize_gclab();
_scm->initialize(_max_workers);
_full_gc->initialize(_gc_timer);
ref_processing_init();
*** 675,720 ****
}
HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
// Retain tlab and allocate object in shared space if
// the amount free in the tlab is too large to discard.
! if (thread->gclab().free() > thread->gclab().refill_waste_limit()) {
! thread->gclab().record_slow_allocation(size);
! return NULL;
! }
// Discard gclab and allocate a new one.
// To minimize fragmentation, the last GCLAB may be smaller than the rest.
! size_t new_gclab_size = thread->gclab().compute_size(size);
!
! thread->gclab().clear_before_allocation();
!
! if (new_gclab_size == 0) {
! return NULL;
}
// Allocate a new GCLAB...
! HeapWord* obj = allocate_new_gclab(new_gclab_size);
! if (obj == NULL) {
return NULL;
}
if (ZeroTLAB) {
// ..and clear it.
! Copy::zero_to_words(obj, new_gclab_size);
} else {
// ...and zap just allocated object.
#ifdef ASSERT
// Skip mangling the space corresponding to the object header to
// ensure that the returned space is not considered parsable by
// any concurrent GC thread.
size_t hdr_size = oopDesc::header_size();
! Copy::fill_to_words(obj + hdr_size, new_gclab_size - hdr_size, badHeapWordVal);
#endif // ASSERT
}
! thread->gclab().fill(obj, obj + size, new_gclab_size);
! return obj;
}
HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) {
#ifdef ASSERT
log_debug(gc, alloc)("Allocate new tlab, requested size = " SIZE_FORMAT " bytes", word_size * HeapWordSize);
--- 676,719 ----
}
HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
// Retain tlab and allocate object in shared space if
// the amount free in the tlab is too large to discard.
! PLAB* gclab = thread->gclab();
// Discard gclab and allocate a new one.
// To minimize fragmentation, the last GCLAB may be smaller than the rest.
! gclab->retire();
! // Figure out size of new GCLAB
! size_t new_gclab_size;
! if (thread->is_Java_thread()) {
! new_gclab_size = _mutator_gclab_stats->desired_plab_sz(Threads::number_of_threads());
! } else {
! new_gclab_size = _collector_gclab_stats->desired_plab_sz(workers()->active_workers());
}
// Allocate a new GCLAB...
! HeapWord* gclab_buf = allocate_new_gclab(new_gclab_size);
! if (gclab_buf == NULL) {
return NULL;
}
if (ZeroTLAB) {
// ..and clear it.
! Copy::zero_to_words(gclab_buf, new_gclab_size);
} else {
// ...and zap just allocated object.
#ifdef ASSERT
// Skip mangling the space corresponding to the object header to
// ensure that the returned space is not considered parsable by
// any concurrent GC thread.
size_t hdr_size = oopDesc::header_size();
! Copy::fill_to_words(gclab_buf + hdr_size, new_gclab_size - hdr_size, badHeapWordVal);
#endif // ASSERT
}
! gclab->set_buf(gclab_buf, new_gclab_size);
! return gclab->allocate(size);
}
HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) {
#ifdef ASSERT
log_debug(gc, alloc)("Allocate new tlab, requested size = " SIZE_FORMAT " bytes", word_size * HeapWordSize);
*** 1079,1100 ****
public:
ShenandoahRetireTLABClosure(bool retire) : _retire(retire) {}
void do_thread(Thread* thread) {
! assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name());
! thread->gclab().make_parsable(_retire);
}
};
void ShenandoahHeap::make_tlabs_parsable(bool retire_tlabs) {
if (UseTLAB) {
CollectedHeap::ensure_parsability(retire_tlabs);
ShenandoahRetireTLABClosure cl(retire_tlabs);
Threads::java_threads_do(&cl);
gc_threads_do(&cl);
- }
}
class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
ShenandoahRootEvacuator* _rp;
--- 1078,1101 ----
public:
ShenandoahRetireTLABClosure(bool retire) : _retire(retire) {}
void do_thread(Thread* thread) {
! PLAB* gclab = thread->gclab();
! if (gclab != NULL) {
! gclab->retire();
! }
}
};
void ShenandoahHeap::make_tlabs_parsable(bool retire_tlabs) {
if (UseTLAB) {
CollectedHeap::ensure_parsability(retire_tlabs);
+ }
ShenandoahRetireTLABClosure cl(retire_tlabs);
Threads::java_threads_do(&cl);
gc_threads_do(&cl);
}
class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
ShenandoahRootEvacuator* _rp;
*** 1202,1240 ****
size_t ShenandoahHeap::max_tlab_size() const {
return ShenandoahHeapRegion::max_tlab_size_bytes();
}
- class ShenandoahResizeGCLABClosure : public ThreadClosure {
- public:
- void do_thread(Thread* thread) {
- assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name());
- thread->gclab().resize();
- }
- };
-
- void ShenandoahHeap::resize_all_tlabs() {
- CollectedHeap::resize_all_tlabs();
-
- ShenandoahResizeGCLABClosure cl;
- Threads::java_threads_do(&cl);
- gc_threads_do(&cl);
- }
-
class ShenandoahAccumulateStatisticsGCLABClosure : public ThreadClosure {
public:
void do_thread(Thread* thread) {
! assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name());
! thread->gclab().accumulate_statistics();
! thread->gclab().initialize_statistics();
}
};
void ShenandoahHeap::accumulate_statistics_all_gclabs() {
ShenandoahAccumulateStatisticsGCLABClosure cl;
Threads::java_threads_do(&cl);
gc_threads_do(&cl);
}
bool ShenandoahHeap::can_elide_tlab_store_barriers() const {
return true;
}
--- 1203,1233 ----
size_t ShenandoahHeap::max_tlab_size() const {
return ShenandoahHeapRegion::max_tlab_size_bytes();
}
class ShenandoahAccumulateStatisticsGCLABClosure : public ThreadClosure {
public:
void do_thread(Thread* thread) {
! ShenandoahHeap* heap = ShenandoahHeap::heap();
! PLAB* gclab = thread->gclab();
! if (gclab != NULL) {
! if (thread->is_Java_thread()) {
! gclab->flush_and_retire_stats(heap->mutator_gclab_stats());
! } else {
! gclab->flush_and_retire_stats(heap->collector_gclab_stats());
! }
! }
}
};
void ShenandoahHeap::accumulate_statistics_all_gclabs() {
ShenandoahAccumulateStatisticsGCLABClosure cl;
Threads::java_threads_do(&cl);
gc_threads_do(&cl);
+ _mutator_gclab_stats->adjust_desired_plab_sz();
+ _collector_gclab_stats->adjust_desired_plab_sz();
}
bool ShenandoahHeap::can_elide_tlab_store_barriers() const {
return true;
}
*** 1490,1500 ****
accumulate_statistics_all_tlabs();
}
set_concurrent_mark_in_progress(true);
// We need to reset all TLABs because we'd lose marks on all objects allocated in them.
! if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
make_tlabs_parsable(true);
}
{
--- 1483,1493 ----
accumulate_statistics_all_tlabs();
}
set_concurrent_mark_in_progress(true);
// We need to reset all TLABs because we'd lose marks on all objects allocated in them.
! {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
make_tlabs_parsable(true);
}
{
*** 1570,1579 ****
--- 1563,1573 ----
}
void ShenandoahHeap::op_final_evac() {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
+ accumulate_statistics_all_gclabs();
set_evacuation_in_progress(false);
if (ShenandoahVerify) {
verifier()->verify_after_evacuation();
}
}
*** 1682,1691 ****
--- 1676,1689 ----
traversal_gc()->final_traversal_collection();
}
void ShenandoahHeap::op_full(GCCause::Cause cause) {
full_gc()->do_it(cause);
+ if (UseTLAB) {
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
+ resize_all_tlabs();
+ }
}
void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
// Degenerated GC is STW, but it can also fail. Current mechanics communicates
// GC failure via cancelled_concgc() flag. So, if we detect the failure after
*** 2289,2298 ****
--- 2287,2297 ----
if (ShenandoahVerify) {
verifier()->verify_before_updaterefs();
}
+ accumulate_statistics_all_gclabs();
set_evacuation_in_progress(false);
set_update_refs_in_progress(true);
make_tlabs_parsable(true);
if (UseShenandoahMatrix) {
connection_matrix()->clear_all();
*** 2890,2894 ****
--- 2889,2907 ----
}
char ShenandoahHeap::gc_state() {
return _gc_state.raw_value();
}
+
+ void ShenandoahHeap::initialize_gclab(Thread* thread) {
+ if (thread->is_Java_thread()) {
+ thread->set_gclab(new PLAB(OldPLABSize));
+ } else {
+ thread->set_gclab(new PLAB(YoungPLABSize));
+ }
+ }
+
+ void ShenandoahHeap::finalize_mutator_gclab(Thread* thread) {
+ thread->gclab()->flush_and_retire_stats(ShenandoahHeap::heap()->mutator_gclab_stats());
+ delete thread->gclab();
+ thread->set_gclab(NULL);
+ }
< prev index next >