rev 58201 : 8240216: Shenandoah: remove ShenandoahTerminationTrace
Reviewed-by: XXX

   1 /*
   2  * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 
  27 #include "classfile/classLoaderData.hpp"
  28 #include "classfile/classLoaderDataGraph.hpp"
  29 #include "gc/shared/referenceProcessor.hpp"
  30 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
  31 #include "gc/shared/workgroup.hpp"
  32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  33 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  34 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  35 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  36 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  37 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  39 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  40 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
  41 #include "gc/shenandoah/shenandoahHeuristics.hpp"
  42 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  43 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  44 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  45 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  46 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
  47 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
  48 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
  49 #include "gc/shenandoah/shenandoahUtils.hpp"
  50 #include "gc/shenandoah/shenandoahVerifier.hpp"
  51 
  52 #include "memory/iterator.hpp"
  53 #include "memory/metaspace.hpp"
  54 #include "memory/resourceArea.hpp"
  55 #include "memory/universe.hpp"
  56 
  57 /**
  58  * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm.
  59  * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm
  60  * is incremental-update-based.
  61  *
  62  * NOTE on interaction with TAMS: we want to avoid traversing new objects for
  63  * several reasons:
  64  * - We will not reclaim them in this cycle anyway, because they are not in the
  65  *   cset
  66  * - It makes up for the bulk of work during final-pause
  67  * - It also shortens the concurrent cycle because we don't need to
  68  *   pointlessly traverse through newly allocated objects.
  69  * - As a nice side-effect, it solves the I-U termination problem (mutators
  70  *   cannot outrun the GC by allocating like crazy)
  71  * - It is an easy way to achieve MWF. What MWF does is to also enqueue the
  72  *   target object of stores if it's new. Treating new objects live implicitely
  73  *   achieves the same, but without extra barriers. I think the effect of
  74  *   shortened final-pause (mentioned above) is the main advantage of MWF. In
  75  *   particular, we will not see the head of a completely new long linked list
  76  *   in final-pause and end up traversing huge chunks of the heap there.
  77  * - We don't need to see/update the fields of new objects either, because they
  78  *   are either still null, or anything that's been stored into them has been
  79  *   evacuated+enqueued before (and will thus be treated later).
  80  *
  81  * We achieve this by setting TAMS for each region, and everything allocated
  82  * beyond TAMS will be 'implicitely marked'.
  83  *
  84  * Gotchas:
  85  * - While we want new objects to be implicitely marked, we don't want to count
  86  *   them alive. Otherwise the next cycle wouldn't pick them up and consider
  87  *   them for cset. This means that we need to protect such regions from
  88  *   getting accidentally thrashed at the end of traversal cycle. This is why I
  89  *   keep track of alloc-regions and check is_alloc_region() in the trashing
  90  *   code.
  91  * - We *need* to traverse through evacuated objects. Those objects are
  92  *   pre-existing, and any references in them point to interesting objects that
  93  *   we need to see. We also want to count them as live, because we just
  94  *   determined that they are alive :-) I achieve this by upping TAMS
  95  *   concurrently for every gclab/gc-shared alloc before publishing the
  96  *   evacuated object. This way, the GC threads will not consider such objects
  97  *   implictely marked, and traverse through them as normal.
  98  */
  99 class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure {
 100 private:
 101   ShenandoahObjToScanQueue* _queue;
 102   ShenandoahTraversalGC* _traversal_gc;
 103   ShenandoahHeap* const _heap;
 104 
 105 public:
 106   ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) :
 107     _queue(q),
 108     _heap(ShenandoahHeap::heap())
 109  { }
 110 
 111   void do_buffer(void** buffer, size_t size) {
 112     for (size_t i = 0; i < size; ++i) {
 113       oop* p = (oop*) &buffer[i];
 114       oop obj = RawAccess<>::oop_load(p);
 115       shenandoah_assert_not_forwarded(p, obj);
 116       if (_heap->marking_context()->mark(obj)) {
 117         _queue->push(ShenandoahMarkTask(obj));
 118       }
 119     }
 120   }
 121 };
 122 
 123 class ShenandoahTraversalSATBThreadsClosure : public ThreadClosure {
 124 private:
 125   ShenandoahTraversalSATBBufferClosure* _satb_cl;
 126 
 127 public:
 128   ShenandoahTraversalSATBThreadsClosure(ShenandoahTraversalSATBBufferClosure* satb_cl) :
 129     _satb_cl(satb_cl) {}
 130 
 131   void do_thread(Thread* thread) {
 132     ShenandoahThreadLocalData::satb_mark_queue(thread).apply_closure_and_empty(_satb_cl);
 133   }
 134 };
 135 
 136 // Like CLDToOopClosure, but clears has_modified_oops, so that we can record modified CLDs during traversal
 137 // and remark them later during final-traversal.
 138 class ShenandoahMarkCLDClosure : public CLDClosure {
 139 private:
 140   OopClosure* _cl;
 141 public:
 142   ShenandoahMarkCLDClosure(OopClosure* cl) : _cl(cl) {}
 143   void do_cld(ClassLoaderData* cld) {
 144     cld->oops_do(_cl, ClassLoaderData::_claim_strong, true);
 145   }
 146 };
 147 
 148 // Like CLDToOopClosure, but only process modified CLDs
 149 class ShenandoahRemarkCLDClosure : public CLDClosure {
 150 private:
 151   OopClosure* _cl;
 152 public:
 153   ShenandoahRemarkCLDClosure(OopClosure* cl) : _cl(cl) {}
 154   void do_cld(ClassLoaderData* cld) {
 155     if (cld->has_modified_oops()) {
 156       cld->oops_do(_cl, ClassLoaderData::_claim_strong, true);
 157     }
 158   }
 159 };
 160 
 161 class ShenandoahInitTraversalCollectionTask : public AbstractGangTask {
 162 private:
 163   ShenandoahCSetRootScanner* _rp;
 164   ShenandoahHeap* _heap;
 165   ShenandoahCsetCodeRootsIterator* _cset_coderoots;
 166   ShenandoahStringDedupRoots       _dedup_roots;
 167 
 168 public:
 169   ShenandoahInitTraversalCollectionTask(ShenandoahCSetRootScanner* rp) :
 170     AbstractGangTask("Shenandoah Init Traversal Collection"),
 171     _rp(rp),
 172     _heap(ShenandoahHeap::heap()) {}
 173 
 174   void work(uint worker_id) {
 175     ShenandoahParallelWorkerSession worker_session(worker_id);
 176 
 177     ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues();
 178     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 179 
 180     bool process_refs = _heap->process_references();
 181     bool unload_classes = _heap->unload_classes();
 182     ReferenceProcessor* rp = NULL;
 183     if (process_refs) {
 184       rp = _heap->ref_processor();
 185     }
 186 
 187     // Step 1: Process ordinary GC roots.
 188     {
 189       ShenandoahTraversalRootsClosure roots_cl(q, rp);
 190       ShenandoahMarkCLDClosure cld_cl(&roots_cl);
 191       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
 192       if (unload_classes) {
 193         _rp->roots_do(worker_id, &roots_cl, NULL, &code_cl);
 194       } else {
 195         _rp->roots_do(worker_id, &roots_cl, &cld_cl, &code_cl);
 196       }
 197     }
 198   }
 199 };
 200 
 201 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
 202 private:
 203   TaskTerminator* _terminator;
 204   ShenandoahHeap* _heap;
 205 public:
 206   ShenandoahConcurrentTraversalCollectionTask(TaskTerminator* terminator) :
 207     AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
 208     _terminator(terminator),
 209     _heap(ShenandoahHeap::heap()) {}
 210 
 211   void work(uint worker_id) {
 212     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 213     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 214     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 215 
 216     // Drain all outstanding work in queues.
 217     traversal_gc->main_loop(worker_id, _terminator, true);
 218   }
 219 };
 220 
 221 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
 222 private:
 223   ShenandoahAllRootScanner* _rp;
 224   TaskTerminator*           _terminator;
 225   ShenandoahHeap* _heap;
 226 public:
 227   ShenandoahFinalTraversalCollectionTask(ShenandoahAllRootScanner* rp, TaskTerminator* terminator) :
 228     AbstractGangTask("Shenandoah Final Traversal Collection"),
 229     _rp(rp),
 230     _terminator(terminator),
 231     _heap(ShenandoahHeap::heap()) {}
 232 
 233   void work(uint worker_id) {
 234     ShenandoahParallelWorkerSession worker_session(worker_id);
 235 
 236     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 237 
 238     ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
 239     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 240 
 241     bool process_refs = _heap->process_references();
 242     bool unload_classes = _heap->unload_classes();
 243     ReferenceProcessor* rp = NULL;
 244     if (process_refs) {
 245       rp = _heap->ref_processor();
 246     }
 247 
 248     // Step 0: Drain outstanding SATB queues.
 249     // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
 250     ShenandoahTraversalSATBBufferClosure satb_cl(q);
 251     {
 252       // Process remaining finished SATB buffers.
 253       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 254       while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
 255       // Process remaining threads SATB buffers below.
 256     }
 257 
 258     // Step 1: Process GC roots.
 259     // For oops in code roots, they are marked, evacuated, enqueued for further traversal,
 260     // and the references to the oops are updated during init pause. New nmethods are handled
 261     // in similar way during nmethod-register process. Therefore, we don't need to rescan code
 262     // roots here.
 263     if (!_heap->is_degenerated_gc_in_progress()) {
 264       ShenandoahTraversalRootsClosure roots_cl(q, rp);
 265       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 266       if (unload_classes) {
 267         ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
 268         _rp->strong_roots_do(worker_id, &roots_cl, &remark_cld_cl, NULL, &tc);
 269       } else {
 270         CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
 271         _rp->roots_do(worker_id, &roots_cl, &cld_cl, NULL, &tc);
 272       }
 273     } else {
 274       ShenandoahTraversalDegenClosure roots_cl(q, rp);
 275       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 276       if (unload_classes) {
 277         ShenandoahRemarkCLDClosure remark_cld_cl(&roots_cl);
 278         _rp->strong_roots_do(worker_id, &roots_cl, &remark_cld_cl, NULL, &tc);
 279       } else {
 280         CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
 281         _rp->roots_do(worker_id, &roots_cl, &cld_cl, NULL, &tc);
 282       }
 283     }
 284 
 285     {
 286       ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
 287       ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
 288 
 289       // Step 3: Finally drain all outstanding work in queues.
 290       traversal_gc->main_loop(worker_id, _terminator, false);
 291     }
 292 
 293   }
 294 };
 295 
 296 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
 297   _heap(heap),
 298   _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())),
 299   _traversal_set(ShenandoahHeapRegionSet()) {
 300 
 301   // Traversal does not support concurrent code root scanning
 302   FLAG_SET_DEFAULT(ShenandoahConcurrentScanCodeRoots, false);
 303 
 304   uint num_queues = heap->max_workers();
 305   for (uint i = 0; i < num_queues; ++i) {
 306     ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
 307     task_queue->initialize();
 308     _task_queues->register_queue(i, task_queue);
 309   }
 310 }
 311 
 312 ShenandoahTraversalGC::~ShenandoahTraversalGC() {
 313 }
 314 
 315 void ShenandoahTraversalGC::prepare_regions() {
 316   size_t num_regions = _heap->num_regions();
 317   ShenandoahMarkingContext* const ctx = _heap->marking_context();
 318   for (size_t i = 0; i < num_regions; i++) {
 319     ShenandoahHeapRegion* region = _heap->get_region(i);
 320     if (_heap->is_bitmap_slice_committed(region)) {
 321       if (_traversal_set.is_in(i)) {
 322         ctx->capture_top_at_mark_start(region);
 323         region->clear_live_data();
 324         assert(ctx->is_bitmap_clear_range(region->bottom(), region->end()), "bitmap for traversal regions must be cleared");
 325       } else {
 326         // Everything outside the traversal set is always considered live.
 327         ctx->reset_top_at_mark_start(region);
 328       }
 329     } else {
 330       // FreeSet may contain uncommitted empty regions, once they are recommitted,
 331       // their TAMS may have old values, so reset them here.
 332       ctx->reset_top_at_mark_start(region);
 333     }
 334   }
 335 }
 336 
 337 void ShenandoahTraversalGC::prepare() {
 338   {
 339     ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_make_parsable);
 340     _heap->make_parsable(true);
 341   }
 342 
 343   if (UseTLAB) {
 344     ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_resize_tlabs);
 345     _heap->resize_tlabs();
 346   }
 347 
 348   assert(_heap->marking_context()->is_bitmap_clear(), "need clean mark bitmap");
 349   assert(!_heap->marking_context()->is_complete(), "should not be complete");
 350 
 351   // About to choose the collection set, make sure we know which regions are pinned.
 352   {
 353     ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_prepare_sync_pinned);
 354     _heap->sync_pinned_region_status();
 355   }
 356 
 357   ShenandoahCollectionSet* collection_set = _heap->collection_set();
 358   {
 359     ShenandoahHeapLocker lock(_heap->lock());
 360 
 361     collection_set->clear();
 362     assert(collection_set->count() == 0, "collection set not clear");
 363 
 364     // Find collection set
 365     _heap->heuristics()->choose_collection_set(collection_set);
 366     prepare_regions();
 367 
 368     // Rebuild free set
 369     _heap->free_set()->rebuild();
 370   }
 371 
 372   log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "%s, " SIZE_FORMAT "%s CSet, " SIZE_FORMAT " CSet regions",
 373                      byte_size_in_proper_unit(collection_set->garbage()),   proper_unit_for_byte_size(collection_set->garbage()),
 374                      byte_size_in_proper_unit(collection_set->live_data()), proper_unit_for_byte_size(collection_set->live_data()),
 375                      collection_set->count());
 376 }
 377 
 378 void ShenandoahTraversalGC::init_traversal_collection() {
 379   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC");
 380 
 381   if (ShenandoahVerify) {
 382     _heap->verifier()->verify_before_traversal();
 383   }
 384 
 385   if (VerifyBeforeGC) {
 386     Universe::verify();
 387   }
 388 
 389   {
 390     ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare);
 391     prepare();
 392   }
 393 
 394   _heap->set_concurrent_traversal_in_progress(true);
 395   _heap->set_has_forwarded_objects(true);
 396 
 397   bool process_refs = _heap->process_references();
 398   if (process_refs) {
 399     ReferenceProcessor* rp = _heap->ref_processor();
 400     rp->enable_discovery(true /*verify_no_refs*/);
 401     rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
 402   }
 403 
 404   {
 405     ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::init_traversal_gc_work);
 406     assert(_task_queues->is_empty(), "queues must be empty before traversal GC");
 407     TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
 408 
 409 #if COMPILER2_OR_JVMCI
 410     DerivedPointerTable::clear();
 411 #endif
 412 
 413     {
 414       uint nworkers = _heap->workers()->active_workers();
 415       task_queues()->reserve(nworkers);
 416       ShenandoahCSetRootScanner rp(nworkers, ShenandoahPhaseTimings::init_traversal_gc_work);
 417       ShenandoahInitTraversalCollectionTask traversal_task(&rp);
 418       _heap->workers()->run_task(&traversal_task);
 419     }
 420 
 421 #if COMPILER2_OR_JVMCI
 422     DerivedPointerTable::update_pointers();
 423 #endif
 424   }
 425 
 426   if (ShenandoahPacing) {
 427     _heap->pacer()->setup_for_traversal();
 428   }
 429 }
 430 
 431 void ShenandoahTraversalGC::main_loop(uint w, TaskTerminator* t, bool sts_yield) {
 432   ShenandoahObjToScanQueue* q = task_queues()->queue(w);
 433 
 434   // Initialize live data.
 435   jushort* ld = _heap->get_liveness_cache(w);
 436 
 437   ReferenceProcessor* rp = NULL;
 438   if (_heap->process_references()) {
 439     rp = _heap->ref_processor();
 440   }
 441   {
 442     if (!_heap->is_degenerated_gc_in_progress()) {
 443       if (_heap->unload_classes()) {
 444         if (ShenandoahStringDedup::is_enabled()) {
 445           ShenandoahTraversalMetadataDedupClosure cl(q, rp);
 446           main_loop_work<ShenandoahTraversalMetadataDedupClosure>(&cl, ld, w, t, sts_yield);
 447         } else {
 448           ShenandoahTraversalMetadataClosure cl(q, rp);
 449           main_loop_work<ShenandoahTraversalMetadataClosure>(&cl, ld, w, t, sts_yield);
 450         }
 451       } else {
 452         if (ShenandoahStringDedup::is_enabled()) {
 453           ShenandoahTraversalDedupClosure cl(q, rp);
 454           main_loop_work<ShenandoahTraversalDedupClosure>(&cl, ld, w, t, sts_yield);
 455         } else {
 456           ShenandoahTraversalClosure cl(q, rp);
 457           main_loop_work<ShenandoahTraversalClosure>(&cl, ld, w, t, sts_yield);
 458         }
 459       }
 460     } else {
 461       if (_heap->unload_classes()) {
 462         if (ShenandoahStringDedup::is_enabled()) {
 463           ShenandoahTraversalMetadataDedupDegenClosure cl(q, rp);
 464           main_loop_work<ShenandoahTraversalMetadataDedupDegenClosure>(&cl, ld, w, t, sts_yield);
 465         } else {
 466           ShenandoahTraversalMetadataDegenClosure cl(q, rp);
 467           main_loop_work<ShenandoahTraversalMetadataDegenClosure>(&cl, ld, w, t, sts_yield);
 468         }
 469       } else {
 470         if (ShenandoahStringDedup::is_enabled()) {
 471           ShenandoahTraversalDedupDegenClosure cl(q, rp);
 472           main_loop_work<ShenandoahTraversalDedupDegenClosure>(&cl, ld, w, t, sts_yield);
 473         } else {
 474           ShenandoahTraversalDegenClosure cl(q, rp);
 475           main_loop_work<ShenandoahTraversalDegenClosure>(&cl, ld, w, t, sts_yield);
 476         }
 477       }
 478     }
 479   }
 480 
 481   _heap->flush_liveness_cache(w);
 482 }
 483 
 484 template <class T>
 485 void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, TaskTerminator* terminator, bool sts_yield) {
 486   ShenandoahObjToScanQueueSet* queues = task_queues();
 487   ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 488   ShenandoahConcurrentMark* conc_mark = _heap->concurrent_mark();
 489 
 490   uintx stride = ShenandoahMarkLoopStride;
 491 
 492   ShenandoahMarkTask task;
 493 
 494   // Process outstanding queues, if any.
 495   q = queues->claim_next();
 496   while (q != NULL) {
 497     if (_heap->check_cancelled_gc_and_yield(sts_yield)) {
 498       return;
 499     }
 500 
 501     for (uint i = 0; i < stride; i++) {
 502       if (q->pop(task)) {
 503         conc_mark->do_task<T>(q, cl, live_data, &task);
 504       } else {
 505         assert(q->is_empty(), "Must be empty");
 506         q = queues->claim_next();
 507         break;
 508       }
 509     }
 510   }
 511 
 512   if (check_and_handle_cancelled_gc(terminator, sts_yield)) return;
 513 
 514   // Normal loop.
 515   q = queues->queue(worker_id);
 516 
 517   ShenandoahTraversalSATBBufferClosure drain_satb(q);
 518   SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 519 
 520   while (true) {
 521     if (check_and_handle_cancelled_gc(terminator, sts_yield)) return;
 522 
 523     while (satb_mq_set.completed_buffers_num() > 0) {
 524       satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
 525     }
 526 
 527     uint work = 0;
 528     for (uint i = 0; i < stride; i++) {
 529       if (q->pop(task) ||
 530           queues->steal(worker_id, task)) {
 531         conc_mark->do_task<T>(q, cl, live_data, &task);
 532         work++;
 533       } else {
 534         break;
 535       }
 536     }
 537 
 538     if (work == 0) {
 539       // No more work, try to terminate
 540       ShenandoahSuspendibleThreadSetLeaver stsl(sts_yield && ShenandoahSuspendibleWorkers);

 541       ShenandoahTerminatorTerminator tt(_heap);
 542 
 543       if (terminator->offer_termination(&tt)) return;
 544     }
 545   }
 546 }
 547 
 548 bool ShenandoahTraversalGC::check_and_handle_cancelled_gc(TaskTerminator* terminator, bool sts_yield) {
 549   if (_heap->cancelled_gc()) {
 550     return true;
 551   }
 552   return false;
 553 }
 554 
 555 void ShenandoahTraversalGC::concurrent_traversal_collection() {
 556   ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::conc_traversal);
 557   if (!_heap->cancelled_gc()) {
 558     uint nworkers = _heap->workers()->active_workers();
 559     task_queues()->reserve(nworkers);

 560 
 561     TaskTerminator terminator(nworkers, task_queues());
 562     ShenandoahConcurrentTraversalCollectionTask task(&terminator);
 563     _heap->workers()->run_task(&task);
 564   }
 565 
 566   if (!_heap->cancelled_gc() && ShenandoahPreclean && _heap->process_references()) {
 567     preclean_weak_refs();
 568   }
 569 }
 570 
 571 void ShenandoahTraversalGC::final_traversal_collection() {
 572   if (!_heap->cancelled_gc()) {
 573 #if COMPILER2_OR_JVMCI
 574     DerivedPointerTable::clear();
 575 #endif
 576     ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work);
 577     uint nworkers = _heap->workers()->active_workers();
 578     task_queues()->reserve(nworkers);
 579 
 580     // Finish traversal
 581     ShenandoahAllRootScanner rp(nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);


 582     TaskTerminator terminator(nworkers, task_queues());
 583     ShenandoahFinalTraversalCollectionTask task(&rp, &terminator);
 584     _heap->workers()->run_task(&task);
 585 #if COMPILER2_OR_JVMCI
 586     DerivedPointerTable::update_pointers();
 587 #endif
 588   }
 589 
 590   if (!_heap->cancelled_gc() && _heap->process_references()) {
 591     weak_refs_work();
 592   }
 593 
 594   if (!_heap->cancelled_gc()) {
 595     assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
 596     TASKQUEUE_STATS_ONLY(_task_queues->print_taskqueue_stats());
 597     TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
 598 
 599     // No more marking expected
 600     _heap->set_concurrent_traversal_in_progress(false);
 601     _heap->mark_complete_marking_context();
 602 
 603     // A rare case, TLAB/GCLAB is initialized from an empty region without
 604     // any live data, the region can be trashed and may be uncommitted in later code,
 605     // that results the TLAB/GCLAB not usable. Retire them here.
 606     _heap->make_parsable(true);
 607 
 608     _heap->parallel_cleaning(false);
 609     fixup_roots();
 610 
 611     _heap->set_has_forwarded_objects(false);
 612 
 613     // Resize metaspace
 614     MetaspaceGC::compute_new_size();
 615 
 616     // Need to see that pinned region status is updated: newly pinned regions must not
 617     // be trashed. New unpinned regions should be trashed.
 618     {
 619       ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_sync_pinned);
 620       _heap->sync_pinned_region_status();
 621     }
 622 
 623     // Still good? We can now trash the cset, and make final verification
 624     {
 625       ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);
 626       ShenandoahHeapLocker lock(_heap->lock());
 627 
 628       // Trash everything
 629       // Clear immediate garbage regions.
 630       size_t num_regions = _heap->num_regions();
 631 
 632       ShenandoahHeapRegionSet* traversal_regions = traversal_set();
 633       ShenandoahFreeSet* free_regions = _heap->free_set();
 634       ShenandoahMarkingContext* const ctx = _heap->marking_context();
 635       free_regions->clear();
 636       for (size_t i = 0; i < num_regions; i++) {
 637         ShenandoahHeapRegion* r = _heap->get_region(i);
 638         bool not_allocated = ctx->top_at_mark_start(r) == r->top();
 639 
 640         bool candidate = traversal_regions->is_in(r) && !r->has_live() && not_allocated;
 641         if (r->is_humongous_start() && candidate) {
 642           // Trash humongous.
 643           HeapWord* humongous_obj = r->bottom();
 644           assert(!ctx->is_marked(oop(humongous_obj)), "must not be marked");
 645           r->make_trash_immediate();
 646           while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) {
 647             i++;
 648             r = _heap->get_region(i);
 649             assert(r->is_humongous_continuation(), "must be humongous continuation");
 650             r->make_trash_immediate();
 651           }
 652         } else if (!r->is_empty() && candidate) {
 653           // Trash regular.
 654           assert(!r->is_humongous(), "handled above");
 655           assert(!r->is_trash(), "must not already be trashed");
 656           r->make_trash_immediate();
 657         }
 658       }
 659       _heap->collection_set()->clear();
 660       _heap->free_set()->rebuild();
 661       reset();
 662     }
 663 
 664     assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
 665     assert(!_heap->cancelled_gc(), "must not be cancelled when getting out here");
 666 
 667     if (ShenandoahVerify) {
 668       _heap->verifier()->verify_after_traversal();
 669     }
 670 #ifdef ASSERT
 671     else {
 672       verify_roots_after_gc();
 673     }
 674 #endif
 675 
 676     if (VerifyAfterGC) {
 677       Universe::verify();
 678     }
 679   }
 680 }
 681 
 682 class ShenandoahVerifyAfterGC : public OopClosure {
 683 private:
 684   template <class T>
 685   void do_oop_work(T* p) {
 686     T o = RawAccess<>::oop_load(p);
 687     if (!CompressedOops::is_null(o)) {
 688       oop obj = CompressedOops::decode_not_null(o);
 689       shenandoah_assert_correct(p, obj);
 690       shenandoah_assert_not_in_cset_except(p, obj, ShenandoahHeap::heap()->cancelled_gc());
 691       shenandoah_assert_not_forwarded(p, obj);
 692     }
 693   }
 694 
 695 public:
 696   void do_oop(narrowOop* p) { do_oop_work(p); }
 697   void do_oop(oop* p)       { do_oop_work(p); }
 698 };
 699 
 700 void ShenandoahTraversalGC::verify_roots_after_gc() {
 701   ShenandoahRootVerifier verifier;
 702   ShenandoahVerifyAfterGC cl;
 703   verifier.oops_do(&cl);
 704 }
 705 
 706 class ShenandoahTraversalFixRootsClosure : public OopClosure {
 707 private:
 708   template <class T>
 709   inline void do_oop_work(T* p) {
 710     T o = RawAccess<>::oop_load(p);
 711     if (!CompressedOops::is_null(o)) {
 712       oop obj = CompressedOops::decode_not_null(o);
 713       oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 714       if (obj != forw) {
 715         RawAccess<IS_NOT_NULL>::oop_store(p, forw);
 716       }
 717     }
 718   }
 719 
 720 public:
 721   inline void do_oop(oop* p) { do_oop_work(p); }
 722   inline void do_oop(narrowOop* p) { do_oop_work(p); }
 723 };
 724 
 725 class ShenandoahTraversalFixRootsTask : public AbstractGangTask {
 726 private:
 727   ShenandoahRootUpdater* _rp;
 728 
 729 public:
 730   ShenandoahTraversalFixRootsTask(ShenandoahRootUpdater* rp) :
 731     AbstractGangTask("Shenandoah traversal fix roots"),
 732     _rp(rp) {
 733     assert(ShenandoahHeap::heap()->has_forwarded_objects(), "Must be");
 734   }
 735 
 736   void work(uint worker_id) {
 737     ShenandoahParallelWorkerSession worker_session(worker_id);
 738     ShenandoahTraversalFixRootsClosure cl;
 739     ShenandoahForwardedIsAliveClosure is_alive;
 740     _rp->roots_do(worker_id, &is_alive, &cl);
 741   }
 742 };
 743 
 744 void ShenandoahTraversalGC::fixup_roots() {
 745 #if COMPILER2_OR_JVMCI
 746   DerivedPointerTable::clear();
 747 #endif
 748   ShenandoahRootUpdater rp(_heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots);
 749   ShenandoahTraversalFixRootsTask update_roots_task(&rp);
 750   _heap->workers()->run_task(&update_roots_task);
 751 #if COMPILER2_OR_JVMCI
 752   DerivedPointerTable::update_pointers();
 753 #endif
 754 }
 755 
 756 void ShenandoahTraversalGC::reset() {
 757   _task_queues->clear();
 758 }
 759 
 760 ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() {
 761   return _task_queues;
 762 }
 763 
 764 class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure {
 765 private:
 766   ShenandoahHeap* const _heap;
 767 public:
 768   ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
 769   virtual bool should_return() { return _heap->cancelled_gc(); }
 770 };
 771 
 772 class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure {
 773 public:
 774   void do_void() {
 775     ShenandoahHeap* sh = ShenandoahHeap::heap();
 776     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 777     assert(sh->process_references(), "why else would we be here?");
 778     TaskTerminator terminator(1, traversal_gc->task_queues());
 779     shenandoah_assert_rp_isalive_installed();
 780     traversal_gc->main_loop((uint) 0, &terminator, true);
 781   }
 782 };
 783 
 784 class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure {
 785 private:
 786   ShenandoahObjToScanQueue* _queue;
 787   Thread* _thread;
 788   ShenandoahTraversalGC* _traversal_gc;
 789   ShenandoahMarkingContext* const _mark_context;
 790 
 791   template <class T>
 792   inline void do_oop_work(T* p) {
 793     _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, true /* atomic update */>(p, _thread, _queue, _mark_context);
 794   }
 795 
 796 public:
 797   ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 798     _queue(q), _thread(Thread::current()),
 799     _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 800     _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 801 
 802   void do_oop(narrowOop* p) { do_oop_work(p); }
 803   void do_oop(oop* p)       { do_oop_work(p); }
 804 };
 805 
 806 class ShenandoahTraversalKeepAliveUpdateDegenClosure : public OopClosure {
 807 private:
 808   ShenandoahObjToScanQueue* _queue;
 809   Thread* _thread;
 810   ShenandoahTraversalGC* _traversal_gc;
 811   ShenandoahMarkingContext* const _mark_context;
 812 
 813   template <class T>
 814   inline void do_oop_work(T* p) {
 815     _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */, false /* atomic update */>(p, _thread, _queue, _mark_context);
 816   }
 817 
 818 public:
 819   ShenandoahTraversalKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
 820           _queue(q), _thread(Thread::current()),
 821           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 822           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 823 
 824   void do_oop(narrowOop* p) { do_oop_work(p); }
 825   void do_oop(oop* p)       { do_oop_work(p); }
 826 };
 827 
 828 class ShenandoahTraversalSingleThreadKeepAliveUpdateClosure : public OopClosure {
 829 private:
 830   ShenandoahObjToScanQueue* _queue;
 831   Thread* _thread;
 832   ShenandoahTraversalGC* _traversal_gc;
 833   ShenandoahMarkingContext* const _mark_context;
 834 
 835   template <class T>
 836   inline void do_oop_work(T* p) {
 837     _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, true /* atomic update */>(p, _thread, _queue, _mark_context);
 838   }
 839 
 840 public:
 841   ShenandoahTraversalSingleThreadKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 842           _queue(q), _thread(Thread::current()),
 843           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 844           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 845 
 846   void do_oop(narrowOop* p) { do_oop_work(p); }
 847   void do_oop(oop* p)       { do_oop_work(p); }
 848 };
 849 
 850 class ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure : public OopClosure {
 851 private:
 852   ShenandoahObjToScanQueue* _queue;
 853   Thread* _thread;
 854   ShenandoahTraversalGC* _traversal_gc;
 855   ShenandoahMarkingContext* const _mark_context;
 856 
 857   template <class T>
 858   inline void do_oop_work(T* p) {
 859     _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */, false /* atomic update */>(p, _thread, _queue, _mark_context);
 860   }
 861 
 862 public:
 863   ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
 864           _queue(q), _thread(Thread::current()),
 865           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 866           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 867 
 868   void do_oop(narrowOop* p) { do_oop_work(p); }
 869   void do_oop(oop* p)       { do_oop_work(p); }
 870 };
 871 
 872 class ShenandoahTraversalPrecleanTask : public AbstractGangTask {
 873 private:
 874   ReferenceProcessor* _rp;
 875 
 876 public:
 877   ShenandoahTraversalPrecleanTask(ReferenceProcessor* rp) :
 878           AbstractGangTask("Precleaning task"),
 879           _rp(rp) {}
 880 
 881   void work(uint worker_id) {
 882     assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
 883     ShenandoahParallelWorkerSession worker_session(worker_id);
 884     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 885 
 886     ShenandoahHeap* sh = ShenandoahHeap::heap();
 887 
 888     ShenandoahObjToScanQueue* q = sh->traversal_gc()->task_queues()->queue(worker_id);
 889 
 890     ShenandoahForwardedIsAliveClosure is_alive;
 891     ShenandoahTraversalCancelledGCYieldClosure yield;
 892     ShenandoahTraversalPrecleanCompleteGCClosure complete_gc;
 893     ShenandoahTraversalKeepAliveUpdateClosure keep_alive(q);
 894     ResourceMark rm;
 895     _rp->preclean_discovered_references(&is_alive, &keep_alive,
 896                                         &complete_gc, &yield,
 897                                         NULL);
 898   }
 899 };
 900 
 901 void ShenandoahTraversalGC::preclean_weak_refs() {
 902   // Pre-cleaning weak references before diving into STW makes sense at the
 903   // end of concurrent mark. This will filter out the references which referents
 904   // are alive. Note that ReferenceProcessor already filters out these on reference
 905   // discovery, and the bulk of work is done here. This phase processes leftovers
 906   // that missed the initial filtering, i.e. when referent was marked alive after
 907   // reference was discovered by RP.
 908 
 909   assert(_heap->process_references(), "sanity");
 910   assert(!_heap->is_degenerated_gc_in_progress(), "must be in concurrent non-degenerated phase");
 911 
 912   // Shortcut if no references were discovered to avoid winding up threads.
 913   ReferenceProcessor* rp = _heap->ref_processor();
 914   if (!rp->has_discovered_references()) {
 915     return;
 916   }
 917 
 918   ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
 919 
 920   shenandoah_assert_rp_isalive_not_installed();
 921   ShenandoahForwardedIsAliveClosure is_alive;
 922   ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive);
 923 
 924   assert(task_queues()->is_empty(), "Should be empty");
 925 
 926   // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
 927   // queues and other goodies. When upstream ReferenceProcessor starts supporting
 928   // parallel precleans, we can extend this to more threads.
 929   ShenandoahPushWorkerScope scope(_heap->workers(), 1, /* check_workers = */ false);
 930 
 931   WorkGang* workers = _heap->workers();
 932   uint nworkers = workers->active_workers();
 933   assert(nworkers == 1, "This code uses only a single worker");
 934   task_queues()->reserve(nworkers);
 935 
 936   ShenandoahTraversalPrecleanTask task(rp);
 937   workers->run_task(&task);
 938 
 939   assert(_heap->cancelled_gc() || task_queues()->is_empty(), "Should be empty");
 940 }
 941 
 942 // Weak Reference Closures
 943 class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure {
 944   uint _worker_id;
 945   TaskTerminator* _terminator;
 946   bool _reset_terminator;
 947 
 948 public:
 949   ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, TaskTerminator* t, bool reset_terminator = false):
 950     _worker_id(worker_id),
 951     _terminator(t),
 952     _reset_terminator(reset_terminator) {
 953   }
 954 
 955   void do_void() {
 956     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 957 
 958     ShenandoahHeap* sh = ShenandoahHeap::heap();
 959     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 960     assert(sh->process_references(), "why else would we be here?");
 961     shenandoah_assert_rp_isalive_installed();
 962 
 963     traversal_gc->main_loop(_worker_id, _terminator, false);
 964 
 965     if (_reset_terminator) {
 966       _terminator->reset_for_reuse();
 967     }
 968   }
 969 };
 970 
 971 class ShenandoahTraversalSingleThreadedDrainMarkingStackClosure: public VoidClosure {
 972   uint _worker_id;
 973   TaskTerminator* _terminator;
 974   bool _reset_terminator;
 975 
 976 public:
 977   ShenandoahTraversalSingleThreadedDrainMarkingStackClosure(uint worker_id, TaskTerminator* t, bool reset_terminator = false):
 978           _worker_id(worker_id),
 979           _terminator(t),
 980           _reset_terminator(reset_terminator) {
 981   }
 982 
 983   void do_void() {
 984     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 985 
 986     ShenandoahHeap* sh = ShenandoahHeap::heap();
 987     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 988     assert(sh->process_references(), "why else would we be here?");
 989     shenandoah_assert_rp_isalive_installed();
 990 
 991     traversal_gc->main_loop(_worker_id, _terminator, false);
 992 
 993     if (_reset_terminator) {
 994       _terminator->reset_for_reuse();
 995     }
 996   }
 997 };
 998 
 999 void ShenandoahTraversalGC::weak_refs_work() {
1000   assert(_heap->process_references(), "sanity");
1001 
1002   ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs;
1003 
1004   ShenandoahGCPhase phase(phase_root);
1005 
1006   ReferenceProcessor* rp = _heap->ref_processor();
1007 
1008   // NOTE: We cannot shortcut on has_discovered_references() here, because
1009   // we will miss marking JNI Weak refs then, see implementation in
1010   // ReferenceProcessor::process_discovered_references.
1011   weak_refs_work_doit();
1012 
1013   rp->verify_no_references_recorded();
1014   assert(!rp->discovery_enabled(), "Post condition");
1015 
1016 }
1017 
1018 class ShenandoahTraversalRefProcTaskProxy : public AbstractGangTask {
1019 private:
1020   AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
1021   TaskTerminator* _terminator;
1022 
1023 public:
1024   ShenandoahTraversalRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
1025                                       TaskTerminator* t) :
1026     AbstractGangTask("Process reference objects in parallel"),
1027     _proc_task(proc_task),
1028     _terminator(t) {
1029   }
1030 
1031   void work(uint worker_id) {
1032     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
1033     ShenandoahHeap* heap = ShenandoahHeap::heap();
1034     ShenandoahTraversalDrainMarkingStackClosure complete_gc(worker_id, _terminator);
1035 
1036     ShenandoahForwardedIsAliveClosure is_alive;
1037     if (!heap->is_degenerated_gc_in_progress()) {
1038       ShenandoahTraversalKeepAliveUpdateClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
1039       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
1040     } else {
1041       ShenandoahTraversalKeepAliveUpdateDegenClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
1042       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
1043     }
1044   }
1045 };
1046 
1047 class ShenandoahTraversalRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
1048 private:
1049   WorkGang* _workers;
1050 
1051 public:
1052   ShenandoahTraversalRefProcTaskExecutor(WorkGang* workers) : _workers(workers) {}
1053 
1054   // Executes a task using worker threads.
1055   void execute(ProcessTask& task, uint ergo_workers) {
1056     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
1057 
1058     ShenandoahHeap* heap = ShenandoahHeap::heap();
1059     ShenandoahTraversalGC* traversal_gc = heap->traversal_gc();
1060     ShenandoahPushWorkerQueuesScope scope(_workers,
1061                                           traversal_gc->task_queues(),
1062                                           ergo_workers,
1063                                           /* do_check = */ false);
1064     uint nworkers = _workers->active_workers();
1065     traversal_gc->task_queues()->reserve(nworkers);
1066     TaskTerminator terminator(nworkers, traversal_gc->task_queues());
1067     ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator);
1068     _workers->run_task(&proc_task_proxy);
1069   }
1070 };
1071 
1072 void ShenandoahTraversalGC::weak_refs_work_doit() {
1073   ReferenceProcessor* rp = _heap->ref_processor();
1074 
1075   ShenandoahPhaseTimings::Phase phase_process = ShenandoahPhaseTimings::weakrefs_process;
1076 
1077   shenandoah_assert_rp_isalive_not_installed();
1078   ShenandoahForwardedIsAliveClosure is_alive;
1079   ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive);
1080 
1081   WorkGang* workers = _heap->workers();
1082   uint nworkers = workers->active_workers();
1083 
1084   rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
1085   rp->set_active_mt_degree(nworkers);
1086 
1087   assert(task_queues()->is_empty(), "Should be empty");
1088 
1089   // complete_gc and keep_alive closures instantiated here are only needed for
1090   // single-threaded path in RP. They share the queue 0 for tracking work, which
1091   // simplifies implementation. Since RP may decide to call complete_gc several
1092   // times, we need to be able to reuse the terminator.
1093   uint serial_worker_id = 0;
1094   TaskTerminator terminator(1, task_queues());
1095   ShenandoahTraversalSingleThreadedDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
1096   ShenandoahPushWorkerQueuesScope scope(workers, task_queues(), 1, /* do_check = */ false);
1097 
1098   ShenandoahTraversalRefProcTaskExecutor executor(workers);
1099 
1100   ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
1101   if (!_heap->is_degenerated_gc_in_progress()) {
1102     ShenandoahTraversalSingleThreadKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id));
1103     rp->process_discovered_references(&is_alive, &keep_alive,
1104                                       &complete_gc, &executor,
1105                                       &pt);
1106   } else {
1107     ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure keep_alive(task_queues()->queue(serial_worker_id));
1108     rp->process_discovered_references(&is_alive, &keep_alive,
1109                                       &complete_gc, &executor,
1110                                       &pt);
1111   }
1112 
1113   pt.print_all_references();
1114   assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty");
1115 }
--- EOF ---