rev 58201 : 8240216: Shenandoah: remove ShenandoahTerminationTrace
Reviewed-by: XXX

   1 /*
   2  * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 
  27 #include "classfile/classLoaderData.hpp"
  28 #include "classfile/classLoaderDataGraph.hpp"
  29 #include "gc/shared/referenceProcessor.hpp"
  30 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
  31 #include "gc/shared/workgroup.hpp"
  32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  33 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  34 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  35 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  36 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  37 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  39 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  40 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
  41 #include "gc/shenandoah/shenandoahHeuristics.hpp"
  42 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  43 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  44 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  45 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  46 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
  47 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
  48 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
  49 #include "gc/shenandoah/shenandoahUtils.hpp"
  50 #include "gc/shenandoah/shenandoahVerifier.hpp"
  51 
  52 #include "memory/iterator.hpp"
  53 #include "memory/metaspace.hpp"
  54 #include "memory/resourceArea.hpp"
  55 #include "memory/universe.hpp"
  56 
  57 /**
  58  * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm.
  59  * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm
  60  * is incremental-update-based.
  61  *
  62  * NOTE on interaction with TAMS: we want to avoid traversing new objects for
  63  * several reasons:
  64  * - We will not reclaim them in this cycle anyway, because they are not in the
  65  *   cset
  66  * - It makes up for the bulk of work during final-pause
  67  * - It also shortens the concurrent cycle because we don't need to
  68  *   pointlessly traverse through newly allocated objects.
  69  * - As a nice side-effect, it solves the I-U termination problem (mutators
  70  *   cannot outrun the GC by allocating like crazy)
  71  * - It is an easy way to achieve MWF. What MWF does is to also enqueue the
  72  *   target object of stores if it's new. Treating new objects live implicitely
  73  *   achieves the same, but without extra barriers. I think the effect of
  74  *   shortened final-pause (mentioned above) is the main advantage of MWF. In
  75  *   particular, we will not see the head of a completely new long linked list
  76  *   in final-pause and end up traversing huge chunks of the heap there.
  77  * - We don't need to see/update the fields of new objects either, because they
  78  *   are either still null, or anything that's been stored into them has been
  79  *   evacuated+enqueued before (and will thus be treated later).
  80  *
  81  * We achieve this by setting TAMS for each region, and everything allocated
  82  * beyond TAMS will be 'implicitely marked'.
  83  *
  84  * Gotchas:
  85  * - While we want new objects to be implicitely marked, we don't want to count
  86  *   them alive. Otherwise the next cycle wouldn't pick them up and consider
  87  *   them for cset. This means that we need to protect such regions from
  88  *   getting accidentally thrashed at the end of traversal cycle. This is why I
  89  *   keep track of alloc-regions and check is_alloc_region() in the trashing
  90  *   code.
  91  * - We *need* to traverse through evacuated objects. Those objects are
  92  *   pre-existing, and any references in them point to interesting objects that
  93  *   we need to see. We also want to count them as live, because we just
  94  *   determined that they are alive :-) I achieve this by upping TAMS
  95  *   concurrently for every gclab/gc-shared alloc before publishing the
  96  *   evacuated object. This way, the GC threads will not consider such objects
  97  *   implictely marked, and traverse through them as normal.
  98  */
  99 class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure {
 100 private:
 101   ShenandoahObjToScanQueue* _queue;
 102   ShenandoahTraversalGC* _traversal_gc;
 103   ShenandoahHeap* const _heap;
 104 
 105 public:
 106   ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) :
 107     _queue(q),
 108     _heap(ShenandoahHeap::heap())
 109  { }
 110 
 111   void do_buffer(void** buffer, size_t size) {
 112     for (size_t i = 0; i < size; ++i) {
 113       oop* p = (oop*) &buffer[i];
 114       oop obj = RawAccess<>::oop_load(p);
 115       shenandoah_assert_not_forwarded(p, obj);
 116       if (_heap->marking_context()->mark(obj)) {
 117         _queue->push(ShenandoahMarkTask(obj));
 118       }
 119     }
 120   }
 121 };
 122 
 123 class ShenandoahTraversalSATBThreadsClosure : public ThreadClosure {
 124 private:
 125   ShenandoahTraversalSATBBufferClosure* _satb_cl;
 126 
 127 public:
 128   ShenandoahTraversalSATBThreadsClosure(ShenandoahTraversalSATBBufferClosure* satb_cl) :
 129     _satb_cl(satb_cl) {}
 130 
 131   void do_thread(Thread* thread) {
 132     ShenandoahThreadLocalData::satb_mark_queue(thread).apply_closure_and_empty(_satb_cl);
 133   }
 134 };
 135 
 136 // Like CLDToOopClosure, but clears has_modified_oops, so that we can record modified CLDs during traversal
 137 // and remark them later during final-traversal.
 138 class ShenandoahMarkCLDClosure : public CLDClosure {
 139 private:
 140   OopClosure* _cl;
 141 public:
 142   ShenandoahMarkCLDClosure(OopClosure* cl) : _cl(cl) {}
 143   void do_cld(ClassLoaderData* cld) {
 144     cld->oops_do(_cl, ClassLoaderData::_claim_strong, true);
 145   }
 146 };
 147 
 148 // Like CLDToOopClosure, but only process modified CLDs
 149 class ShenandoahRemarkCLDClosure : public CLDClosure {
 150 private:
 151   OopClosure* _cl;
 152 public:
 153   ShenandoahRemarkCLDClosure(OopClosure* cl) : _cl(cl) {}
 154   void do_cld(ClassLoaderData* cld) {
 155     if (cld->has_modified_oops()) {
 156       cld->oops_do(_cl, ClassLoaderData::_claim_strong, true);
 157     }
 158   }
 159 };
 160 
 161 class ShenandoahInitTraversalCollectionTask : public AbstractGangTask {
 162 private:
 163   ShenandoahCSetRootScanner* _rp;
 164   ShenandoahHeap* _heap;
 165   ShenandoahCsetCodeRootsIterator* _cset_coderoots;
 166   ShenandoahStringDedupRoots       _dedup_roots;
 167 
 168 public:
 169   ShenandoahInitTraversalCollectionTask(ShenandoahCSetRootScanner* rp) :
 170     AbstractGangTask("Shenandoah Init Traversal Collection"),
 171     _rp(rp),
 172     _heap(ShenandoahHeap::heap()) {}
 173 
 174   void work(uint worker_id) {
 175     ShenandoahParallelWorkerSession worker_session(worker_id);
 176 
 177     ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues();
 178     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 179 
 180     bool process_refs = _heap->process_references();
 181     bool unload_classes = _heap->unload_classes();
 182     ReferenceProcessor* rp = NULL;
 183     if (process_refs) {
 184       rp = _heap->ref_processor();
 185     }
 186 
 187     // Step 1: Process ordinary GC roots.
 188     {
 189       ShenandoahTraversalRootsClosure roots_cl(q, rp);
 190       ShenandoahMarkCLDClosure cld_cl(&roots_cl);
 191       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
 192       if (unload_classes) {
 193         _rp->roots_do(worker_id, &roots_cl, NULL, &code_cl);
 194       } else {
 195         _rp->roots_do(worker_id, &roots_cl, &cld_cl, &code_cl);
 196       }
 197     }
 198   }
 199 };
 200 
 201 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
 202 private:
 203   TaskTerminator* _terminator;
 204   ShenandoahHeap* _heap;
 205 public:
 206   ShenandoahConcurrentTraversalCollectionTask(TaskTerminator* terminator) :
 207     AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
 208     _terminator(terminator),
 209     _heap(ShenandoahHeap::heap()) {}
 210 
 211   void work(uint worker_id) {
 212     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 213     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 214     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 215 
 216     // Drain all outstanding work in queues.
 217     traversal_gc->main_loop(worker_id, _terminator, true);
 218   }
 219 };
 220 
 221 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
 222 private:
 223   ShenandoahAllRootScanner* _rp;
 224   TaskTerminator*           _terminator;
 225   ShenandoahHeap* _heap;
 226 public:
 227   ShenandoahFinalTraversalCollectionTask(ShenandoahAllRootScanner* rp, TaskTerminator* terminator) :
 228     AbstractGangTask("Shenandoah Final Traversal Collection"),
 229     _rp(rp),
 230     _terminator(terminator),
 231     _heap(ShenandoahHeap::heap()) {}
 232 
 233   void work(uint worker_id) {
 234     ShenandoahParallelWorkerSession worker_session(worker_id);
 235 
 236     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 237 
 238     ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
 239     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 240 
 241     bool process_refs = _heap->process_references();
 242     bool unload_classes = _heap->unload_classes();
 243     ReferenceProcessor* rp = NULL;
 244     if (process_refs) {
 245       rp = _heap->ref_processor();
 246     }
 247 
 248     // Step 0: Drain outstanding SATB queues.
 249     // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
 250     ShenandoahTraversalSATBBufferClosure satb_cl(q);
 251     {
 252       // Process remaining finished SATB buffers.
 253       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 254       while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
 255       // Process remaining threads SATB buffers below.
 256     }
 257 
 258     // Step 1: Process GC roots.
 259     // For oops in code roots, they are marked, evacuated, enqueued for further traversal,
 260     // and the references to the oops are updated during init pause. New nmethods are handled
 261     // in similar way during nmethod-register process. Therefore, we don't need to rescan code
 262     // roots here.
 263     if (!_heap->is_degenerated_gc_in_progress()) {
 264       ShenandoahTraversalRootsClosure roots_cl(q, rp);
 265       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 266       if (unload_classes) {
 267         ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
 268         _rp->strong_roots_do(worker_id, &roots_cl, &remark_cld_cl, NULL, &tc);
 269       } else {
 270         CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
 271         _rp->roots_do(worker_id, &roots_cl, &cld_cl, NULL, &tc);
 272       }
 273     } else {
 274       ShenandoahTraversalDegenClosure roots_cl(q, rp);
 275       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 276       if (unload_classes) {
 277         ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
 278         _rp->strong_roots_do(worker_id, &roots_cl, &remark_cld_cl, NULL, &tc);
 279       } else {
 280         CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
 281         _rp->roots_do(worker_id, &roots_cl, &cld_cl, NULL, &tc);
 282       }
 283     }
 284 
 285     {
 286       ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
 287       ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
 288 
 289       // Step 3: Finally drain all outstanding work in queues.
 290       traversal_gc->main_loop(worker_id, _terminator, false);
 291     }
 292 
 293   }
 294 };
 295 
 296 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
 297   _heap(heap),
 298   _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())),
 299   _traversal_set(ShenandoahHeapRegionSet()) {
 300 
 301   // Traversal does not support concurrent code root scanning
 302   FLAG_SET_DEFAULT(ShenandoahConcurrentScanCodeRoots, false);
 303 
 304   uint num_queues = heap->max_workers();
 305   for (uint i = 0; i < num_queues; ++i) {
 306     ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
 307     task_queue->initialize();
 308     _task_queues->register_queue(i, task_queue);
 309   }
 310 }
 311 
 312 ShenandoahTraversalGC::~ShenandoahTraversalGC() {
 313 }
 314 
 315 void ShenandoahTraversalGC::prepare_regions() {
 316   size_t num_regions = _heap->num_regions();
 317   ShenandoahMarkingContext* const ctx = _heap->marking_context();
 318   for (size_t i = 0; i < num_regions; i++) {
 319     ShenandoahHeapRegion* region = _heap->get_region(i);
 320     if (_heap->is_bitmap_slice_committed(region)) {
 321       if (_traversal_set.is_in(i)) {
 322         ctx->capture_top_at_mark_start(region);
 323         region->clear_live_data();
 324         assert(ctx->is_bitmap_clear_range(region->bottom(), region->end()), "bitmap for traversal regions must be cleared");
 325       } else {
 326         // Everything outside the traversal set is always considered live.
 327         ctx->reset_top_at_mark_start(region);
 328       }
 329     } else {
 330       // FreeSet may contain uncommitted empty regions, once they are recommitted,
 331       // their TAMS may have old values, so reset them here.
 332       ctx->reset_top_at_mark_start(region);
 333     }
 334   }
 335 }
 336 
 337 void ShenandoahTraversalGC::prepare() {
 338   {
 339     ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_make_parsable);
 340     _heap->make_parsable(true);
 341   }
 342 
 343   if (UseTLAB) {
 344     ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_resize_tlabs);
 345     _heap->resize_tlabs();
 346   }
 347 
 348   assert(_heap->marking_context()->is_bitmap_clear(), "need clean mark bitmap");
 349   assert(!_heap->marking_context()->is_complete(), "should not be complete");
 350 
 351   // About to choose the collection set, make sure we know which regions are pinned.
 352   {
 353     ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_prepare_sync_pinned);
 354     _heap->sync_pinned_region_status();
 355   }
 356 
 357   ShenandoahCollectionSet* collection_set = _heap->collection_set();
 358   {
 359     ShenandoahHeapLocker lock(_heap->lock());
 360 
 361     collection_set->clear();
 362     assert(collection_set->count() == 0, "collection set not clear");
 363 
 364     // Find collection set
 365     _heap->heuristics()->choose_collection_set(collection_set);
 366     prepare_regions();
 367 
 368     // Rebuild free set
 369     _heap->free_set()->rebuild();
 370   }
 371 
 372   log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "%s, " SIZE_FORMAT "%s CSet, " SIZE_FORMAT " CSet regions",
 373                      byte_size_in_proper_unit(collection_set->garbage()),   proper_unit_for_byte_size(collection_set->garbage()),
 374                      byte_size_in_proper_unit(collection_set->live_data()), proper_unit_for_byte_size(collection_set->live_data()),
 375                      collection_set->count());
 376 }
 377 
 378 void ShenandoahTraversalGC::init_traversal_collection() {
 379   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC");
 380 
 381   if (ShenandoahVerify) {
 382     _heap->verifier()->verify_before_traversal();
 383   }
 384 
 385   if (VerifyBeforeGC) {
 386     Universe::verify();
 387   }
 388 
 389   {
 390     ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare);
 391     prepare();
 392   }
 393 
 394   _heap->set_concurrent_traversal_in_progress(true);
 395   _heap->set_has_forwarded_objects(true);
 396 
 397   bool process_refs = _heap->process_references();
 398   if (process_refs) {
 399     ReferenceProcessor* rp = _heap->ref_processor();
 400     rp->enable_discovery(true /*verify_no_refs*/);
 401     rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
 402   }
 403 
 404   {
 405     ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::init_traversal_gc_work);
 406     assert(_task_queues->is_empty(), "queues must be empty before traversal GC");
 407     TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
 408 
 409 #if COMPILER2_OR_JVMCI
 410     DerivedPointerTable::clear();
 411 #endif
 412 
 413     {
 414       uint nworkers = _heap->workers()->active_workers();
 415       task_queues()->reserve(nworkers);
 416       ShenandoahCSetRootScanner rp(nworkers, ShenandoahPhaseTimings::init_traversal_gc_work);
 417       ShenandoahInitTraversalCollectionTask traversal_task(&rp);
 418       _heap->workers()->run_task(&traversal_task);
 419     }
 420 
 421 #if COMPILER2_OR_JVMCI
 422     DerivedPointerTable::update_pointers();
 423 #endif
 424   }
 425 
 426   if (ShenandoahPacing) {
 427     _heap->pacer()->setup_for_traversal();
 428   }
 429 }
 430 
 431 void ShenandoahTraversalGC::main_loop(uint w, TaskTerminator* t, bool sts_yield) {
 432   ShenandoahObjToScanQueue* q = task_queues()->queue(w);
 433 
 434   // Initialize live data.
 435   jushort* ld = _heap->get_liveness_cache(w);
 436 
 437   ReferenceProcessor* rp = NULL;
 438   if (_heap->process_references()) {
 439     rp = _heap->ref_processor();
 440   }
 441   {
 442     if (!_heap->is_degenerated_gc_in_progress()) {
 443       if (_heap->unload_classes()) {
 444         if (ShenandoahStringDedup::is_enabled()) {
 445           ShenandoahTraversalMetadataDedupClosure cl(q, rp);
 446           main_loop_work<ShenandoahTraversalMetadataDedupClosure>(&cl, ld, w, t, sts_yield);
 447         } else {
 448           ShenandoahTraversalMetadataClosure cl(q, rp);
 449           main_loop_work<ShenandoahTraversalMetadataClosure>(&cl, ld, w, t, sts_yield);
 450         }
 451       } else {
 452         if (ShenandoahStringDedup::is_enabled()) {
 453           ShenandoahTraversalDedupClosure cl(q, rp);
 454           main_loop_work<ShenandoahTraversalDedupClosure>(&cl, ld, w, t, sts_yield);
 455         } else {
 456           ShenandoahTraversalClosure cl(q, rp);
 457           main_loop_work<ShenandoahTraversalClosure>(&cl, ld, w, t, sts_yield);
 458         }
 459       }
 460     } else {
 461       if (_heap->unload_classes()) {
 462         if (ShenandoahStringDedup::is_enabled()) {
 463           ShenandoahTraversalMetadataDedupDegenClosure cl(q, rp);
 464           main_loop_work<ShenandoahTraversalMetadataDedupDegenClosure>(&cl, ld, w, t, sts_yield);
 465         } else {
 466           ShenandoahTraversalMetadataDegenClosure cl(q, rp);
 467           main_loop_work<ShenandoahTraversalMetadataDegenClosure>(&cl, ld, w, t, sts_yield);
 468         }
 469       } else {
 470         if (ShenandoahStringDedup::is_enabled()) {
 471           ShenandoahTraversalDedupDegenClosure cl(q, rp);
 472           main_loop_work<ShenandoahTraversalDedupDegenClosure>(&cl, ld, w, t, sts_yield);
 473         } else {
 474           ShenandoahTraversalDegenClosure cl(q, rp);
 475           main_loop_work<ShenandoahTraversalDegenClosure>(&cl, ld, w, t, sts_yield);
 476         }
 477       }
 478     }
 479   }
 480 
 481   _heap->flush_liveness_cache(w);
 482 }
 483 
 484 template <class T>
 485 void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, TaskTerminator* terminator, bool sts_yield) {
 486   ShenandoahObjToScanQueueSet* queues = task_queues();
 487   ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 488   ShenandoahConcurrentMark* conc_mark = _heap->concurrent_mark();
 489 
 490   uintx stride = ShenandoahMarkLoopStride;
 491 
 492   ShenandoahMarkTask task;
 493 
 494   // Process outstanding queues, if any.
 495   q = queues->claim_next();
 496   while (q != NULL) {
 497     if (_heap->check_cancelled_gc_and_yield(sts_yield)) {
 498       return;
 499     }
 500 
 501     for (uint i = 0; i < stride; i++) {
 502       if (q->pop(task)) {
 503         conc_mark->do_task<T>(q, cl, live_data, &task);
 504       } else {
 505         assert(q->is_empty(), "Must be empty");
 506         q = queues->claim_next();
 507         break;
 508       }
 509     }
 510   }
 511 
 512   if (check_and_handle_cancelled_gc(terminator, sts_yield)) return;
 513 
 514   // Normal loop.
 515   q = queues->queue(worker_id);
 516 
 517   ShenandoahTraversalSATBBufferClosure drain_satb(q);
 518   SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 519 
 520   while (true) {
 521     if (check_and_handle_cancelled_gc(terminator, sts_yield)) return;
 522 
 523     while (satb_mq_set.completed_buffers_num() > 0) {
 524       satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
 525     }
 526 
 527     uint work = 0;
 528     for (uint i = 0; i < stride; i++) {
 529       if (q->pop(task) ||
 530           queues->steal(worker_id, task)) {
 531         conc_mark->do_task<T>(q, cl, live_data, &task);
 532         work++;
 533       } else {
 534         break;
 535       }
 536     }
 537 
 538     if (work == 0) {
 539       // No more work, try to terminate
 540       ShenandoahSuspendibleThreadSetLeaver stsl(sts_yield && ShenandoahSuspendibleWorkers);
 541       ShenandoahTerminationTimingsTracker term_tracker(worker_id);
 542       ShenandoahTerminatorTerminator tt(_heap);
 543 
 544       if (terminator->offer_termination(&tt)) return;
 545     }
 546   }
 547 }
 548 
 549 bool ShenandoahTraversalGC::check_and_handle_cancelled_gc(TaskTerminator* terminator, bool sts_yield) {
 550   if (_heap->cancelled_gc()) {
 551     return true;
 552   }
 553   return false;
 554 }
 555 
 556 void ShenandoahTraversalGC::concurrent_traversal_collection() {
 557   ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::conc_traversal);
 558   if (!_heap->cancelled_gc()) {
 559     uint nworkers = _heap->workers()->active_workers();
 560     task_queues()->reserve(nworkers);
 561     ShenandoahTerminationTracker tracker(ShenandoahPhaseTimings::conc_traversal_termination);
 562 
 563     TaskTerminator terminator(nworkers, task_queues());
 564     ShenandoahConcurrentTraversalCollectionTask task(&terminator);
 565     _heap->workers()->run_task(&task);
 566   }
 567 
 568   if (!_heap->cancelled_gc() && ShenandoahPreclean && _heap->process_references()) {
 569     preclean_weak_refs();
 570   }
 571 }
 572 
 573 void ShenandoahTraversalGC::final_traversal_collection() {
 574   if (!_heap->cancelled_gc()) {
 575 #if COMPILER2_OR_JVMCI
 576     DerivedPointerTable::clear();
 577 #endif
 578     ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work);
 579     uint nworkers = _heap->workers()->active_workers();
 580     task_queues()->reserve(nworkers);
 581 
 582     // Finish traversal
 583     ShenandoahAllRootScanner rp(nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);
 584     ShenandoahTerminationTracker term(ShenandoahPhaseTimings::final_traversal_gc_termination);
 585 
 586     TaskTerminator terminator(nworkers, task_queues());
 587     ShenandoahFinalTraversalCollectionTask task(&rp, &terminator);
 588     _heap->workers()->run_task(&task);
 589 #if COMPILER2_OR_JVMCI
 590     DerivedPointerTable::update_pointers();
 591 #endif
 592   }
 593 
 594   if (!_heap->cancelled_gc() && _heap->process_references()) {
 595     weak_refs_work();
 596   }
 597 
 598   if (!_heap->cancelled_gc()) {
 599     assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
 600     TASKQUEUE_STATS_ONLY(_task_queues->print_taskqueue_stats());
 601     TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
 602 
 603     // No more marking expected
 604     _heap->set_concurrent_traversal_in_progress(false);
 605     _heap->mark_complete_marking_context();
 606 
 607     // A rare case, TLAB/GCLAB is initialized from an empty region without
 608     // any live data, the region can be trashed and may be uncommitted in later code,
 609     // that results the TLAB/GCLAB not usable. Retire them here.
 610     _heap->make_parsable(true);
 611 
 612     _heap->parallel_cleaning(false);
 613     fixup_roots();
 614 
 615     _heap->set_has_forwarded_objects(false);
 616 
 617     // Resize metaspace
 618     MetaspaceGC::compute_new_size();
 619 
 620     // Need to see that pinned region status is updated: newly pinned regions must not
 621     // be trashed. New unpinned regions should be trashed.
 622     {
 623       ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_sync_pinned);
 624       _heap->sync_pinned_region_status();
 625     }
 626 
 627     // Still good? We can now trash the cset, and make final verification
 628     {
 629       ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);
 630       ShenandoahHeapLocker lock(_heap->lock());
 631 
 632       // Trash everything
 633       // Clear immediate garbage regions.
 634       size_t num_regions = _heap->num_regions();
 635 
 636       ShenandoahHeapRegionSet* traversal_regions = traversal_set();
 637       ShenandoahFreeSet* free_regions = _heap->free_set();
 638       ShenandoahMarkingContext* const ctx = _heap->marking_context();
 639       free_regions->clear();
 640       for (size_t i = 0; i < num_regions; i++) {
 641         ShenandoahHeapRegion* r = _heap->get_region(i);
 642         bool not_allocated = ctx->top_at_mark_start(r) == r->top();
 643 
 644         bool candidate = traversal_regions->is_in(r) && !r->has_live() && not_allocated;
 645         if (r->is_humongous_start() && candidate) {
 646           // Trash humongous.
 647           HeapWord* humongous_obj = r->bottom();
 648           assert(!ctx->is_marked(oop(humongous_obj)), "must not be marked");
 649           r->make_trash_immediate();
 650           while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) {
 651             i++;
 652             r = _heap->get_region(i);
 653             assert(r->is_humongous_continuation(), "must be humongous continuation");
 654             r->make_trash_immediate();
 655           }
 656         } else if (!r->is_empty() && candidate) {
 657           // Trash regular.
 658           assert(!r->is_humongous(), "handled above");
 659           assert(!r->is_trash(), "must not already be trashed");
 660           r->make_trash_immediate();
 661         }
 662       }
 663       _heap->collection_set()->clear();
 664       _heap->free_set()->rebuild();
 665       reset();
 666     }
 667 
 668     assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
 669     assert(!_heap->cancelled_gc(), "must not be cancelled when getting out here");
 670 
 671     if (ShenandoahVerify) {
 672       _heap->verifier()->verify_after_traversal();
 673     }
 674 #ifdef ASSERT
 675     else {
 676       verify_roots_after_gc();
 677     }
 678 #endif
 679 
 680     if (VerifyAfterGC) {
 681       Universe::verify();
 682     }
 683   }
 684 }
 685 
 686 class ShenandoahVerifyAfterGC : public OopClosure {
 687 private:
 688   template <class T>
 689   void do_oop_work(T* p) {
 690     T o = RawAccess<>::oop_load(p);
 691     if (!CompressedOops::is_null(o)) {
 692       oop obj = CompressedOops::decode_not_null(o);
 693       shenandoah_assert_correct(p, obj);
 694       shenandoah_assert_not_in_cset_except(p, obj, ShenandoahHeap::heap()->cancelled_gc());
 695       shenandoah_assert_not_forwarded(p, obj);
 696     }
 697   }
 698 
 699 public:
 700   void do_oop(narrowOop* p) { do_oop_work(p); }
 701   void do_oop(oop* p)       { do_oop_work(p); }
 702 };
 703 
 704 void ShenandoahTraversalGC::verify_roots_after_gc() {
 705   ShenandoahRootVerifier verifier;
 706   ShenandoahVerifyAfterGC cl;
 707   verifier.oops_do(&cl);
 708 }
 709 
 710 class ShenandoahTraversalFixRootsClosure : public OopClosure {
 711 private:
 712   template <class T>
 713   inline void do_oop_work(T* p) {
 714     T o = RawAccess<>::oop_load(p);
 715     if (!CompressedOops::is_null(o)) {
 716       oop obj = CompressedOops::decode_not_null(o);
 717       oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 718       if (obj != forw) {
 719         RawAccess<IS_NOT_NULL>::oop_store(p, forw);
 720       }
 721     }
 722   }
 723 
 724 public:
 725   inline void do_oop(oop* p) { do_oop_work(p); }
 726   inline void do_oop(narrowOop* p) { do_oop_work(p); }
 727 };
 728 
 729 class ShenandoahTraversalFixRootsTask : public AbstractGangTask {
 730 private:
 731   ShenandoahRootUpdater* _rp;
 732 
 733 public:
 734   ShenandoahTraversalFixRootsTask(ShenandoahRootUpdater* rp) :
 735     AbstractGangTask("Shenandoah traversal fix roots"),
 736     _rp(rp) {
 737     assert(ShenandoahHeap::heap()->has_forwarded_objects(), "Must be");
 738   }
 739 
 740   void work(uint worker_id) {
 741     ShenandoahParallelWorkerSession worker_session(worker_id);
 742     ShenandoahTraversalFixRootsClosure cl;
 743     ShenandoahForwardedIsAliveClosure is_alive;
 744     _rp->roots_do(worker_id, &is_alive, &cl);
 745   }
 746 };
 747 
 748 void ShenandoahTraversalGC::fixup_roots() {
 749 #if COMPILER2_OR_JVMCI
 750   DerivedPointerTable::clear();
 751 #endif
 752   ShenandoahRootUpdater rp(_heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots);
 753   ShenandoahTraversalFixRootsTask update_roots_task(&rp);
 754   _heap->workers()->run_task(&update_roots_task);
 755 #if COMPILER2_OR_JVMCI
 756   DerivedPointerTable::update_pointers();
 757 #endif
 758 }
 759 
 760 void ShenandoahTraversalGC::reset() {
 761   _task_queues->clear();
 762 }
 763 
 764 ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() {
 765   return _task_queues;
 766 }
 767 
 768 class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure {
 769 private:
 770   ShenandoahHeap* const _heap;
 771 public:
 772   ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
 773   virtual bool should_return() { return _heap->cancelled_gc(); }
 774 };
 775 
 776 class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure {
 777 public:
 778   void do_void() {
 779     ShenandoahHeap* sh = ShenandoahHeap::heap();
 780     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 781     assert(sh->process_references(), "why else would we be here?");
 782     TaskTerminator terminator(1, traversal_gc->task_queues());
 783     shenandoah_assert_rp_isalive_installed();
 784     traversal_gc->main_loop((uint) 0, &terminator, true);
 785   }
 786 };
 787 
 788 class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure {
 789 private:
 790   ShenandoahObjToScanQueue* _queue;
 791   Thread* _thread;
 792   ShenandoahTraversalGC* _traversal_gc;
 793   ShenandoahMarkingContext* const _mark_context;
 794 
 795   template <class T>
 796   inline void do_oop_work(T* p) {
 797     _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, true /* atomic update */>(p, _thread, _queue, _mark_context);
 798   }
 799 
 800 public:
 801   ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 802     _queue(q), _thread(Thread::current()),
 803     _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 804     _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 805 
 806   void do_oop(narrowOop* p) { do_oop_work(p); }
 807   void do_oop(oop* p)       { do_oop_work(p); }
 808 };
 809 
 810 class ShenandoahTraversalKeepAliveUpdateDegenClosure : public OopClosure {
 811 private:
 812   ShenandoahObjToScanQueue* _queue;
 813   Thread* _thread;
 814   ShenandoahTraversalGC* _traversal_gc;
 815   ShenandoahMarkingContext* const _mark_context;
 816 
 817   template <class T>
 818   inline void do_oop_work(T* p) {
 819     _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */, false /* atomic update */>(p, _thread, _queue, _mark_context);
 820   }
 821 
 822 public:
 823   ShenandoahTraversalKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
 824           _queue(q), _thread(Thread::current()),
 825           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 826           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 827 
 828   void do_oop(narrowOop* p) { do_oop_work(p); }
 829   void do_oop(oop* p)       { do_oop_work(p); }
 830 };
 831 
 832 class ShenandoahTraversalSingleThreadKeepAliveUpdateClosure : public OopClosure {
 833 private:
 834   ShenandoahObjToScanQueue* _queue;
 835   Thread* _thread;
 836   ShenandoahTraversalGC* _traversal_gc;
 837   ShenandoahMarkingContext* const _mark_context;
 838 
 839   template <class T>
 840   inline void do_oop_work(T* p) {
 841     _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, true /* atomic update */>(p, _thread, _queue, _mark_context);
 842   }
 843 
 844 public:
 845   ShenandoahTraversalSingleThreadKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 846           _queue(q), _thread(Thread::current()),
 847           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 848           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 849 
 850   void do_oop(narrowOop* p) { do_oop_work(p); }
 851   void do_oop(oop* p)       { do_oop_work(p); }
 852 };
 853 
 854 class ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure : public OopClosure {
 855 private:
 856   ShenandoahObjToScanQueue* _queue;
 857   Thread* _thread;
 858   ShenandoahTraversalGC* _traversal_gc;
 859   ShenandoahMarkingContext* const _mark_context;
 860 
 861   template <class T>
 862   inline void do_oop_work(T* p) {
 863     _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */, false /* atomic update */>(p, _thread, _queue, _mark_context);
 864   }
 865 
 866 public:
 867   ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
 868           _queue(q), _thread(Thread::current()),
 869           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 870           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 871 
 872   void do_oop(narrowOop* p) { do_oop_work(p); }
 873   void do_oop(oop* p)       { do_oop_work(p); }
 874 };
 875 
 876 class ShenandoahTraversalPrecleanTask : public AbstractGangTask {
 877 private:
 878   ReferenceProcessor* _rp;
 879 
 880 public:
 881   ShenandoahTraversalPrecleanTask(ReferenceProcessor* rp) :
 882           AbstractGangTask("Precleaning task"),
 883           _rp(rp) {}
 884 
 885   void work(uint worker_id) {
 886     assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
 887     ShenandoahParallelWorkerSession worker_session(worker_id);
 888     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 889 
 890     ShenandoahHeap* sh = ShenandoahHeap::heap();
 891 
 892     ShenandoahObjToScanQueue* q = sh->traversal_gc()->task_queues()->queue(worker_id);
 893 
 894     ShenandoahForwardedIsAliveClosure is_alive;
 895     ShenandoahTraversalCancelledGCYieldClosure yield;
 896     ShenandoahTraversalPrecleanCompleteGCClosure complete_gc;
 897     ShenandoahTraversalKeepAliveUpdateClosure keep_alive(q);
 898     ResourceMark rm;
 899     _rp->preclean_discovered_references(&is_alive, &keep_alive,
 900                                         &complete_gc, &yield,
 901                                         NULL);
 902   }
 903 };
 904 
 905 void ShenandoahTraversalGC::preclean_weak_refs() {
 906   // Pre-cleaning weak references before diving into STW makes sense at the
 907   // end of concurrent mark. This will filter out the references which referents
 908   // are alive. Note that ReferenceProcessor already filters out these on reference
 909   // discovery, and the bulk of work is done here. This phase processes leftovers
 910   // that missed the initial filtering, i.e. when referent was marked alive after
 911   // reference was discovered by RP.
 912 
 913   assert(_heap->process_references(), "sanity");
 914   assert(!_heap->is_degenerated_gc_in_progress(), "must be in concurrent non-degenerated phase");
 915 
 916   // Shortcut if no references were discovered to avoid winding up threads.
 917   ReferenceProcessor* rp = _heap->ref_processor();
 918   if (!rp->has_discovered_references()) {
 919     return;
 920   }
 921 
 922   ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
 923 
 924   shenandoah_assert_rp_isalive_not_installed();
 925   ShenandoahForwardedIsAliveClosure is_alive;
 926   ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive);
 927 
 928   assert(task_queues()->is_empty(), "Should be empty");
 929 
 930   // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
 931   // queues and other goodies. When upstream ReferenceProcessor starts supporting
 932   // parallel precleans, we can extend this to more threads.
 933   ShenandoahPushWorkerScope scope(_heap->workers(), 1, /* check_workers = */ false);
 934 
 935   WorkGang* workers = _heap->workers();
 936   uint nworkers = workers->active_workers();
 937   assert(nworkers == 1, "This code uses only a single worker");
 938   task_queues()->reserve(nworkers);
 939 
 940   ShenandoahTraversalPrecleanTask task(rp);
 941   workers->run_task(&task);
 942 
 943   assert(_heap->cancelled_gc() || task_queues()->is_empty(), "Should be empty");
 944 }
 945 
 946 // Weak Reference Closures
 947 class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure {
 948   uint _worker_id;
 949   TaskTerminator* _terminator;
 950   bool _reset_terminator;
 951 
 952 public:
 953   ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, TaskTerminator* t, bool reset_terminator = false):
 954     _worker_id(worker_id),
 955     _terminator(t),
 956     _reset_terminator(reset_terminator) {
 957   }
 958 
 959   void do_void() {
 960     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 961 
 962     ShenandoahHeap* sh = ShenandoahHeap::heap();
 963     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 964     assert(sh->process_references(), "why else would we be here?");
 965     shenandoah_assert_rp_isalive_installed();
 966 
 967     traversal_gc->main_loop(_worker_id, _terminator, false);
 968 
 969     if (_reset_terminator) {
 970       _terminator->reset_for_reuse();
 971     }
 972   }
 973 };
 974 
 975 class ShenandoahTraversalSingleThreadedDrainMarkingStackClosure: public VoidClosure {
 976   uint _worker_id;
 977   TaskTerminator* _terminator;
 978   bool _reset_terminator;
 979 
 980 public:
 981   ShenandoahTraversalSingleThreadedDrainMarkingStackClosure(uint worker_id, TaskTerminator* t, bool reset_terminator = false):
 982           _worker_id(worker_id),
 983           _terminator(t),
 984           _reset_terminator(reset_terminator) {
 985   }
 986 
 987   void do_void() {
 988     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 989 
 990     ShenandoahHeap* sh = ShenandoahHeap::heap();
 991     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 992     assert(sh->process_references(), "why else would we be here?");
 993     shenandoah_assert_rp_isalive_installed();
 994 
 995     traversal_gc->main_loop(_worker_id, _terminator, false);
 996 
 997     if (_reset_terminator) {
 998       _terminator->reset_for_reuse();
 999     }
1000   }
1001 };
1002 
1003 void ShenandoahTraversalGC::weak_refs_work() {
1004   assert(_heap->process_references(), "sanity");
1005 
1006   ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs;
1007 
1008   ShenandoahGCPhase phase(phase_root);
1009 
1010   ReferenceProcessor* rp = _heap->ref_processor();
1011 
1012   // NOTE: We cannot shortcut on has_discovered_references() here, because
1013   // we will miss marking JNI Weak refs then, see implementation in
1014   // ReferenceProcessor::process_discovered_references.
1015   weak_refs_work_doit();
1016 
1017   rp->verify_no_references_recorded();
1018   assert(!rp->discovery_enabled(), "Post condition");
1019 
1020 }
1021 
1022 class ShenandoahTraversalRefProcTaskProxy : public AbstractGangTask {
1023 private:
1024   AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
1025   TaskTerminator* _terminator;
1026 
1027 public:
1028   ShenandoahTraversalRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
1029                                       TaskTerminator* t) :
1030     AbstractGangTask("Process reference objects in parallel"),
1031     _proc_task(proc_task),
1032     _terminator(t) {
1033   }
1034 
1035   void work(uint worker_id) {
1036     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
1037     ShenandoahHeap* heap = ShenandoahHeap::heap();
1038     ShenandoahTraversalDrainMarkingStackClosure complete_gc(worker_id, _terminator);
1039 
1040     ShenandoahForwardedIsAliveClosure is_alive;
1041     if (!heap->is_degenerated_gc_in_progress()) {
1042       ShenandoahTraversalKeepAliveUpdateClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
1043       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
1044     } else {
1045       ShenandoahTraversalKeepAliveUpdateDegenClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
1046       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
1047     }
1048   }
1049 };
1050 
1051 class ShenandoahTraversalRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
1052 private:
1053   WorkGang* _workers;
1054 
1055 public:
1056   ShenandoahTraversalRefProcTaskExecutor(WorkGang* workers) : _workers(workers) {}
1057 
1058   // Executes a task using worker threads.
1059   void execute(ProcessTask& task, uint ergo_workers) {
1060     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
1061 
1062     ShenandoahHeap* heap = ShenandoahHeap::heap();
1063     ShenandoahTraversalGC* traversal_gc = heap->traversal_gc();
1064     ShenandoahPushWorkerQueuesScope scope(_workers,
1065                                           traversal_gc->task_queues(),
1066                                           ergo_workers,
1067                                           /* do_check = */ false);
1068     uint nworkers = _workers->active_workers();
1069     traversal_gc->task_queues()->reserve(nworkers);
1070     TaskTerminator terminator(nworkers, traversal_gc->task_queues());
1071     ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator);
1072     _workers->run_task(&proc_task_proxy);
1073   }
1074 };
1075 
1076 void ShenandoahTraversalGC::weak_refs_work_doit() {
1077   ReferenceProcessor* rp = _heap->ref_processor();
1078 
1079   ShenandoahPhaseTimings::Phase phase_process = ShenandoahPhaseTimings::weakrefs_process;
1080 
1081   shenandoah_assert_rp_isalive_not_installed();
1082   ShenandoahForwardedIsAliveClosure is_alive;
1083   ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive);
1084 
1085   WorkGang* workers = _heap->workers();
1086   uint nworkers = workers->active_workers();
1087 
1088   rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
1089   rp->set_active_mt_degree(nworkers);
1090 
1091   assert(task_queues()->is_empty(), "Should be empty");
1092 
1093   // complete_gc and keep_alive closures instantiated here are only needed for
1094   // single-threaded path in RP. They share the queue 0 for tracking work, which
1095   // simplifies implementation. Since RP may decide to call complete_gc several
1096   // times, we need to be able to reuse the terminator.
1097   uint serial_worker_id = 0;
1098   TaskTerminator terminator(1, task_queues());
1099   ShenandoahTraversalSingleThreadedDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
1100   ShenandoahPushWorkerQueuesScope scope(workers, task_queues(), 1, /* do_check = */ false);
1101 
1102   ShenandoahTraversalRefProcTaskExecutor executor(workers);
1103 
1104   ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
1105   if (!_heap->is_degenerated_gc_in_progress()) {
1106     ShenandoahTraversalSingleThreadKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id));
1107     rp->process_discovered_references(&is_alive, &keep_alive,
1108                                       &complete_gc, &executor,
1109                                       &pt);
1110   } else {
1111     ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure keep_alive(task_queues()->queue(serial_worker_id));
1112     rp->process_discovered_references(&is_alive, &keep_alive,
1113                                       &complete_gc, &executor,
1114                                       &pt);
1115   }
1116 
1117   pt.print_all_references();
1118   assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty");
1119 }
--- EOF ---