src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp

rev 50928 : Process remaining SATB buffers in final mark/traverse loop instead of separate phase

@@ -211,11 +211,11 @@
   void work(uint worker_id) {
     ShenandoahEvacOOMScope oom_evac_scope;
     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 
     // Drain all outstanding work in queues.
-    traversal_gc->main_loop(worker_id, _terminator, true);
+    traversal_gc->main_loop(worker_id, _terminator);
   }
 };
 
 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
 private:

@@ -241,21 +241,12 @@
     ReferenceProcessor* rp = NULL;
     if (process_refs) {
       rp = _heap->ref_processor();
     }
 
-    // Step 1: Drain outstanding SATB queues.
-    // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
-    ShenandoahTraversalSATBBufferClosure satb_cl(q);
-    {
-      // Process remaining finished SATB buffers.
-      SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
-      while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
-      // Process remaining threads SATB buffers below.
-    }
-
     // Step 1: Process ordinary GC roots.
+    ShenandoahTraversalSATBBufferClosure satb_cl(q);
     if (!_heap->is_degenerated_gc_in_progress()) {
       ShenandoahTraversalClosure roots_cl(q, rp);
       CLDToOopClosure cld_cl(&roots_cl);
       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);

@@ -281,11 +272,11 @@
     {
       ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
       ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
 
       // Step 3: Finally drain all outstanding work in queues.
-      traversal_gc->main_loop(worker_id, _terminator, false);
+      traversal_gc->main_loop(worker_id, _terminator);
     }
 
   }
 };
 

@@ -433,20 +424,11 @@
   }
 
   _root_regions_iterator.reset(_root_regions);
 }
 
-void ShenandoahTraversalGC::main_loop(uint worker_id, ParallelTaskTerminator* terminator, bool do_satb) {
-  if (do_satb) {
-    main_loop_prework<true>(worker_id, terminator);
-  } else {
-    main_loop_prework<false>(worker_id, terminator);
-  }
-}
-
-template <bool DO_SATB>
-void ShenandoahTraversalGC::main_loop_prework(uint w, ParallelTaskTerminator* t) {
+void ShenandoahTraversalGC::main_loop(uint w, ParallelTaskTerminator* t) {
   ShenandoahObjToScanQueue* q = task_queues()->queue(w);
 
   // Initialize live data.
   jushort* ld = get_liveness(w);
   Copy::fill_to_bytes(ld, _heap->num_regions() * sizeof(jushort));

@@ -459,94 +441,94 @@
     if (!_heap->is_degenerated_gc_in_progress()) {
       if (_heap->unload_classes()) {
         if (ShenandoahStringDedup::is_enabled()) {
           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
           ShenandoahTraversalMetadataDedupMatrixClosure cl(q, rp, dq);
-          main_loop_work<ShenandoahTraversalMetadataDedupMatrixClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalMetadataDedupMatrixClosure>(&cl, ld, w, t);
         } else {
           ShenandoahTraversalMetadataMatrixClosure cl(q, rp);
-          main_loop_work<ShenandoahTraversalMetadataMatrixClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalMetadataMatrixClosure>(&cl, ld, w, t);
         }
       } else {
         if (ShenandoahStringDedup::is_enabled()) {
           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
           ShenandoahTraversalDedupMatrixClosure cl(q, rp, dq);
-          main_loop_work<ShenandoahTraversalDedupMatrixClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalDedupMatrixClosure>(&cl, ld, w, t);
         } else {
           ShenandoahTraversalMatrixClosure cl(q, rp);
-          main_loop_work<ShenandoahTraversalMatrixClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalMatrixClosure>(&cl, ld, w, t);
         }
       }
     } else {
       if (_heap->unload_classes()) {
         if (ShenandoahStringDedup::is_enabled()) {
           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
           ShenandoahTraversalMetadataDedupDegenMatrixClosure cl(q, rp, dq);
-          main_loop_work<ShenandoahTraversalMetadataDedupDegenMatrixClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalMetadataDedupDegenMatrixClosure>(&cl, ld, w, t);
         } else {
           ShenandoahTraversalMetadataDegenMatrixClosure cl(q, rp);
-          main_loop_work<ShenandoahTraversalMetadataDegenMatrixClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalMetadataDegenMatrixClosure>(&cl, ld, w, t);
         }
       } else {
         if (ShenandoahStringDedup::is_enabled()) {
           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
           ShenandoahTraversalDedupDegenMatrixClosure cl(q, rp, dq);
-          main_loop_work<ShenandoahTraversalDedupDegenMatrixClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalDedupDegenMatrixClosure>(&cl, ld, w, t);
         } else {
           ShenandoahTraversalDegenMatrixClosure cl(q, rp);
-          main_loop_work<ShenandoahTraversalDegenMatrixClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalDegenMatrixClosure>(&cl, ld, w, t);
         }
       }
     }
   } else {
     if (!_heap->is_degenerated_gc_in_progress()) {
       if (_heap->unload_classes()) {
         if (ShenandoahStringDedup::is_enabled()) {
           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
           ShenandoahTraversalMetadataDedupClosure cl(q, rp, dq);
-          main_loop_work<ShenandoahTraversalMetadataDedupClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalMetadataDedupClosure>(&cl, ld, w, t);
         } else {
           ShenandoahTraversalMetadataClosure cl(q, rp);
-          main_loop_work<ShenandoahTraversalMetadataClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalMetadataClosure>(&cl, ld, w, t);
         }
       } else {
         if (ShenandoahStringDedup::is_enabled()) {
           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
           ShenandoahTraversalDedupClosure cl(q, rp, dq);
-          main_loop_work<ShenandoahTraversalDedupClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalDedupClosure>(&cl, ld, w, t);
         } else {
           ShenandoahTraversalClosure cl(q, rp);
-          main_loop_work<ShenandoahTraversalClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalClosure>(&cl, ld, w, t);
         }
       }
     } else {
       if (_heap->unload_classes()) {
         if (ShenandoahStringDedup::is_enabled()) {
           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
           ShenandoahTraversalMetadataDedupDegenClosure cl(q, rp, dq);
-          main_loop_work<ShenandoahTraversalMetadataDedupDegenClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalMetadataDedupDegenClosure>(&cl, ld, w, t);
         } else {
           ShenandoahTraversalMetadataDegenClosure cl(q, rp);
-          main_loop_work<ShenandoahTraversalMetadataDegenClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalMetadataDegenClosure>(&cl, ld, w, t);
         }
       } else {
         if (ShenandoahStringDedup::is_enabled()) {
           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
           ShenandoahTraversalDedupDegenClosure cl(q, rp, dq);
-          main_loop_work<ShenandoahTraversalDedupDegenClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalDedupDegenClosure>(&cl, ld, w, t);
         } else {
           ShenandoahTraversalDegenClosure cl(q, rp);
-          main_loop_work<ShenandoahTraversalDegenClosure, DO_SATB>(&cl, ld, w, t);
+          main_loop_work<ShenandoahTraversalDegenClosure>(&cl, ld, w, t);
         }
       }
     }
   }
   flush_liveness(w);
 
 }
 
-template <class T, bool DO_SATB>
+template <class T>
 void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, ParallelTaskTerminator* terminator) {
   ShenandoahObjToScanQueueSet* queues = task_queues();
   ShenandoahObjToScanQueue* q = queues->queue(worker_id);
   ShenandoahConcurrentMark* conc_mark = _heap->concurrentMark();
 

@@ -602,15 +584,13 @@
   int seed = 17;
 
   while (true) {
     if (check_and_handle_cancelled_gc(terminator)) return;
 
-    if (DO_SATB) {
       while (satb_mq_set.completed_buffers_num() > 0) {
         satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
       }
-    }
 
     if (_arraycopy_task_queue.length() > 0) {
       process_arraycopy_task(cl);
     }
 

@@ -838,11 +818,11 @@
     ShenandoahHeap* sh = ShenandoahHeap::heap();
     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
     assert(sh->process_references(), "why else would we be here?");
     ParallelTaskTerminator terminator(1, traversal_gc->task_queues());
     shenandoah_assert_rp_isalive_installed();
-    traversal_gc->main_loop((uint) 0, &terminator, false);
+    traversal_gc->main_loop((uint) 0, &terminator);
   }
 };
 
 class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure {
 private:

@@ -988,11 +968,11 @@
     ShenandoahHeap* sh = ShenandoahHeap::heap();
     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
     assert(sh->process_references(), "why else would we be here?");
     shenandoah_assert_rp_isalive_installed();
 
-    traversal_gc->main_loop(_worker_id, _terminator, false);
+    traversal_gc->main_loop(_worker_id, _terminator);
 
     if (_reset_terminator) {
       _terminator->reset_for_reuse();
     }
   }