src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

rev 48920 : [backport] Use PLAB for evacuations instead of TLAB
rev 48921 : [backport] Fix PLAB alignment reserve

@@ -25,10 +25,11 @@
 #include "memory/allocation.hpp"
 
 #include "gc/shared/gcTimer.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/parallelCleaning.hpp"
+#include "gc/shared/plab.hpp"
 
 #include "gc/shenandoah/brooksPointer.hpp"
 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahCollectionSet.hpp"

@@ -371,10 +372,12 @@
   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
   _phase_timings(NULL),
   _alloc_tracker(NULL),
   _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
   _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
+  _mutator_gclab_stats(new PLABStats("Shenandoah mutator GCLAB stats", OldPLABSize, PLABWeight)),
+  _collector_gclab_stats(new PLABStats("Shenandoah collector GCLAB stats", YoungPLABSize, PLABWeight)),
   _memory_pool(NULL)
 {
   log_info(gc, init)("Parallel GC threads: "UINT32_FORMAT, ParallelGCThreads);
   log_info(gc, init)("Concurrent GC threads: "UINT32_FORMAT, ConcGCThreads);
   log_info(gc, init)("Parallel reference processing enabled: %s", BOOL_TO_STR(ParallelRefProcEnabled));

@@ -546,27 +549,25 @@
 }
 
 class ShenandoahInitGCLABClosure : public ThreadClosure {
 public:
   void do_thread(Thread* thread) {
-    thread->gclab().initialize(true);
+    ShenandoahHeap::heap()->initialize_gclab(thread);
   }
 };
 
 void ShenandoahHeap::post_initialize() {
   CollectedHeap::post_initialize();
-  if (UseTLAB) {
     MutexLocker ml(Threads_lock);
 
     ShenandoahInitGCLABClosure init_gclabs;
     Threads::java_threads_do(&init_gclabs);
     gc_threads_do(&init_gclabs);
 
     // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
     // Now, we will let WorkGang to initialize gclab when new worker is created.
     _workers->set_initialize_gclab();
-  }
 
   _scm->initialize(_max_workers);
   _full_gc->initialize(_gc_timer);
 
   ref_processing_init();

@@ -675,46 +676,44 @@
 }
 
 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
   // Retain tlab and allocate object in shared space if
   // the amount free in the tlab is too large to discard.
-  if (thread->gclab().free() > thread->gclab().refill_waste_limit()) {
-    thread->gclab().record_slow_allocation(size);
-    return NULL;
-  }
+  PLAB* gclab = thread->gclab();
 
   // Discard gclab and allocate a new one.
   // To minimize fragmentation, the last GCLAB may be smaller than the rest.
-  size_t new_gclab_size = thread->gclab().compute_size(size);
-
-  thread->gclab().clear_before_allocation();
-
-  if (new_gclab_size == 0) {
-    return NULL;
+  gclab->retire();
+  // Figure out size of new GCLAB
+  size_t new_gclab_size;
+  if (thread->is_Java_thread()) {
+    new_gclab_size = _mutator_gclab_stats->desired_plab_sz(Threads::number_of_threads());
+  } else {
+    new_gclab_size = _collector_gclab_stats->desired_plab_sz(workers()->active_workers());
   }
 
   // Allocate a new GCLAB...
-  HeapWord* obj = allocate_new_gclab(new_gclab_size);
-  if (obj == NULL) {
+  HeapWord* gclab_buf = allocate_new_gclab(new_gclab_size);
+  if (gclab_buf == NULL) {
     return NULL;
   }
 
   if (ZeroTLAB) {
     // ..and clear it.
-    Copy::zero_to_words(obj, new_gclab_size);
+    Copy::zero_to_words(gclab_buf, new_gclab_size);
   } else {
     // ...and zap just allocated object.
 #ifdef ASSERT
     // Skip mangling the space corresponding to the object header to
     // ensure that the returned space is not considered parsable by
     // any concurrent GC thread.
     size_t hdr_size = oopDesc::header_size();
-    Copy::fill_to_words(obj + hdr_size, new_gclab_size - hdr_size, badHeapWordVal);
+    Copy::fill_to_words(gclab_buf + hdr_size, new_gclab_size - hdr_size, badHeapWordVal);
 #endif // ASSERT
   }
-  thread->gclab().fill(obj, obj + size, new_gclab_size);
-  return obj;
+  gclab->set_buf(gclab_buf, new_gclab_size);
+  return gclab->allocate(size);
 }
 
 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) {
 #ifdef ASSERT
   log_debug(gc, alloc)("Allocate new tlab, requested size = " SIZE_FORMAT " bytes", word_size * HeapWordSize);

@@ -1079,22 +1078,24 @@
 
 public:
   ShenandoahRetireTLABClosure(bool retire) : _retire(retire) {}
 
   void do_thread(Thread* thread) {
-    assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name());
-    thread->gclab().make_parsable(_retire);
+    PLAB* gclab = thread->gclab();
+    if (gclab != NULL) {
+      gclab->retire();
+    }
   }
 };
 
 void ShenandoahHeap::make_tlabs_parsable(bool retire_tlabs) {
   if (UseTLAB) {
     CollectedHeap::ensure_parsability(retire_tlabs);
+  }
     ShenandoahRetireTLABClosure cl(retire_tlabs);
     Threads::java_threads_do(&cl);
     gc_threads_do(&cl);
-  }
 }
 
 
 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
   ShenandoahRootEvacuator* _rp;

@@ -1202,39 +1203,31 @@
 
 size_t ShenandoahHeap::max_tlab_size() const {
   return ShenandoahHeapRegion::max_tlab_size_bytes();
 }
 
-class ShenandoahResizeGCLABClosure : public ThreadClosure {
-public:
-  void do_thread(Thread* thread) {
-    assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name());
-    thread->gclab().resize();
-  }
-};
-
-void ShenandoahHeap::resize_all_tlabs() {
-  CollectedHeap::resize_all_tlabs();
-
-  ShenandoahResizeGCLABClosure cl;
-  Threads::java_threads_do(&cl);
-  gc_threads_do(&cl);
-}
-
 class ShenandoahAccumulateStatisticsGCLABClosure : public ThreadClosure {
 public:
   void do_thread(Thread* thread) {
-    assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name());
-    thread->gclab().accumulate_statistics();
-    thread->gclab().initialize_statistics();
+    ShenandoahHeap* heap = ShenandoahHeap::heap();
+    PLAB* gclab = thread->gclab();
+    if (gclab != NULL) {
+      if (thread->is_Java_thread()) {
+        gclab->flush_and_retire_stats(heap->mutator_gclab_stats());
+      } else {
+        gclab->flush_and_retire_stats(heap->collector_gclab_stats());
+      }
+    }
   }
 };
 
 void ShenandoahHeap::accumulate_statistics_all_gclabs() {
   ShenandoahAccumulateStatisticsGCLABClosure cl;
   Threads::java_threads_do(&cl);
   gc_threads_do(&cl);
+  _mutator_gclab_stats->adjust_desired_plab_sz();
+  _collector_gclab_stats->adjust_desired_plab_sz();
 }
 
 bool  ShenandoahHeap::can_elide_tlab_store_barriers() const {
   return true;
 }

@@ -1490,11 +1483,11 @@
     accumulate_statistics_all_tlabs();
   }
 
   set_concurrent_mark_in_progress(true);
   // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
-  if (UseTLAB) {
+  {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
     make_tlabs_parsable(true);
   }
 
   {

@@ -1570,10 +1563,11 @@
 }
 
 void ShenandoahHeap::op_final_evac() {
   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
 
+  accumulate_statistics_all_gclabs();
   set_evacuation_in_progress(false);
   if (ShenandoahVerify) {
     verifier()->verify_after_evacuation();
   }
 }

@@ -1682,10 +1676,14 @@
   traversal_gc()->final_traversal_collection();
 }
 
 void ShenandoahHeap::op_full(GCCause::Cause cause) {
   full_gc()->do_it(cause);
+  if (UseTLAB) {
+    ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
+    resize_all_tlabs();
+  }
 }
 
 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
   // GC failure via cancelled_concgc() flag. So, if we detect the failure after

@@ -2289,10 +2287,11 @@
 
   if (ShenandoahVerify) {
     verifier()->verify_before_updaterefs();
   }
 
+  accumulate_statistics_all_gclabs();
   set_evacuation_in_progress(false);
   set_update_refs_in_progress(true);
   make_tlabs_parsable(true);
   if (UseShenandoahMatrix) {
     connection_matrix()->clear_all();

@@ -2890,5 +2889,19 @@
 }
 
 char ShenandoahHeap::gc_state() {
   return _gc_state.raw_value();
 }
+
+void ShenandoahHeap::initialize_gclab(Thread* thread) {
+  if (thread->is_Java_thread()) {
+    thread->set_gclab(new PLAB(OldPLABSize));
+  } else {
+    thread->set_gclab(new PLAB(YoungPLABSize));
+  }
+}
+
+void ShenandoahHeap::finalize_mutator_gclab(Thread* thread) {
+  thread->gclab()->flush_and_retire_stats(ShenandoahHeap::heap()->mutator_gclab_stats());
+  delete thread->gclab();
+  thread->set_gclab(NULL);
+}