1 /*
   2  * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "classfile/classLoaderData.hpp"
  27 #include "classfile/classLoaderDataGraph.hpp"
  28 #include "gc/shared/referenceProcessor.hpp"
  29 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
  30 #include "gc/shared/workgroup.hpp"
  31 #include "gc/shared/weakProcessor.hpp"
  32 #include "gc/shared/weakProcessor.inline.hpp"
  33 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  34 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  35 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  36 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  37 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  39 #include "gc/shenandoah/shenandoahHeap.hpp"
  40 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  41 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  42 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
  43 #include "gc/shenandoah/shenandoahHeuristics.hpp"
  44 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  45 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  46 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
  47 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  48 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
  49 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
  50 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
  51 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
  52 #include "gc/shenandoah/shenandoahUtils.hpp"
  53 #include "gc/shenandoah/shenandoahVerifier.hpp"
  54 
  55 #include "memory/iterator.hpp"
  56 #include "memory/metaspace.hpp"
  57 #include "memory/resourceArea.hpp"
  58 
  59 /**
  60  * NOTE: We are using the SATB buffer in thread.hpp and satbMarkQueue.hpp, however, it is not an SATB algorithm.
  61  * We're using the buffer as generic oop buffer to enqueue new values in concurrent oop stores, IOW, the algorithm
  62  * is incremental-update-based.
  63  *
  64  * NOTE on interaction with TAMS: we want to avoid traversing new objects for
  65  * several reasons:
  66  * - We will not reclaim them in this cycle anyway, because they are not in the
  67  *   cset
  68  * - It makes up for the bulk of work during final-pause
  69  * - It also shortens the concurrent cycle because we don't need to
  70  *   pointlessly traverse through newly allocated objects.
  71  * - As a nice side-effect, it solves the I-U termination problem (mutators
  72  *   cannot outrun the GC by allocating like crazy)
  73  * - It is an easy way to achieve MWF. What MWF does is to also enqueue the
  74  *   target object of stores if it's new. Treating new objects live implicitely
  75  *   achieves the same, but without extra barriers. I think the effect of
  76  *   shortened final-pause (mentioned above) is the main advantage of MWF. In
  77  *   particular, we will not see the head of a completely new long linked list
  78  *   in final-pause and end up traversing huge chunks of the heap there.
  79  * - We don't need to see/update the fields of new objects either, because they
  80  *   are either still null, or anything that's been stored into them has been
  81  *   evacuated+enqueued before (and will thus be treated later).
  82  *
  83  * We achieve this by setting TAMS for each region, and everything allocated
  84  * beyond TAMS will be 'implicitely marked'.
  85  *
  86  * Gotchas:
  87  * - While we want new objects to be implicitely marked, we don't want to count
  88  *   them alive. Otherwise the next cycle wouldn't pick them up and consider
  89  *   them for cset. This means that we need to protect such regions from
  90  *   getting accidentally thrashed at the end of traversal cycle. This is why I
  91  *   keep track of alloc-regions and check is_alloc_region() in the trashing
  92  *   code.
  93  * - We *need* to traverse through evacuated objects. Those objects are
  94  *   pre-existing, and any references in them point to interesting objects that
  95  *   we need to see. We also want to count them as live, because we just
  96  *   determined that they are alive :-) I achieve this by upping TAMS
  97  *   concurrently for every gclab/gc-shared alloc before publishing the
  98  *   evacuated object. This way, the GC threads will not consider such objects
  99  *   implictely marked, and traverse through them as normal.
 100  */
 101 class ShenandoahTraversalSATBBufferClosure : public SATBBufferClosure {
 102 private:
 103   ShenandoahObjToScanQueue* _queue;
 104   ShenandoahTraversalGC* _traversal_gc;
 105   ShenandoahHeap* const _heap;
 106 
 107 public:
 108   ShenandoahTraversalSATBBufferClosure(ShenandoahObjToScanQueue* q) :
 109     _queue(q),
 110     _heap(ShenandoahHeap::heap())
 111  { }
 112 
 113   void do_buffer(void** buffer, size_t size) {
 114     for (size_t i = 0; i < size; ++i) {
 115       oop* p = (oop*) &buffer[i];
 116       oop obj = RawAccess<>::oop_load(p);
 117       shenandoah_assert_not_forwarded(p, obj);
 118       if (_heap->marking_context()->mark(obj)) {
 119         _queue->push(ShenandoahMarkTask(obj));
 120       }
 121     }
 122   }
 123 };
 124 
 125 class ShenandoahTraversalSATBThreadsClosure : public ThreadClosure {
 126 private:
 127   ShenandoahTraversalSATBBufferClosure* _satb_cl;
 128 
 129 public:
 130   ShenandoahTraversalSATBThreadsClosure(ShenandoahTraversalSATBBufferClosure* satb_cl) :
 131     _satb_cl(satb_cl) {}
 132 
 133   void do_thread(Thread* thread) {
 134     if (thread->is_Java_thread()) {
 135       JavaThread* jt = (JavaThread*)thread;
 136       ShenandoahThreadLocalData::satb_mark_queue(jt).apply_closure_and_empty(_satb_cl);
 137     } else if (thread->is_VM_thread()) {
 138       ShenandoahBarrierSet::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl);
 139     }
 140   }
 141 };
 142 
 143 // Like CLDToOopClosure, but clears has_modified_oops, so that we can record modified CLDs during traversal
 144 // and remark them later during final-traversal.
 145 class ShenandoahMarkCLDClosure : public CLDClosure {
 146 private:
 147   OopClosure* _cl;
 148 public:
 149   ShenandoahMarkCLDClosure(OopClosure* cl) : _cl(cl) {}
 150   void do_cld(ClassLoaderData* cld) {
 151     cld->oops_do(_cl, true, true);
 152   }
 153 };
 154 
 155 // Like CLDToOopClosure, but only process modified CLDs
 156 class ShenandoahRemarkCLDClosure : public CLDClosure {
 157 private:
 158   OopClosure* _cl;
 159 public:
 160   ShenandoahRemarkCLDClosure(OopClosure* cl) : _cl(cl) {}
 161   void do_cld(ClassLoaderData* cld) {
 162     if (cld->has_modified_oops()) {
 163       cld->oops_do(_cl, true, true);
 164     }
 165   }
 166 };
 167 
 168 class ShenandoahInitTraversalCollectionTask : public AbstractGangTask {
 169 private:
 170   ShenandoahRootProcessor* _rp;
 171   ShenandoahHeap* _heap;
 172   ShenandoahCsetCodeRootsIterator* _cset_coderoots;
 173 public:
 174   ShenandoahInitTraversalCollectionTask(ShenandoahRootProcessor* rp, ShenandoahCsetCodeRootsIterator* cset_coderoots) :
 175     AbstractGangTask("Shenandoah Init Traversal Collection"),
 176     _rp(rp),
 177     _heap(ShenandoahHeap::heap()),
 178     _cset_coderoots(cset_coderoots) {}
 179 
 180   void work(uint worker_id) {
 181     ShenandoahParallelWorkerSession worker_session(worker_id);
 182 
 183     ShenandoahEvacOOMScope oom_evac_scope;
 184     ShenandoahObjToScanQueueSet* queues = _heap->traversal_gc()->task_queues();
 185     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 186 
 187     bool process_refs = _heap->process_references();
 188     bool unload_classes = _heap->unload_classes();
 189     ReferenceProcessor* rp = NULL;
 190     if (process_refs) {
 191       rp = _heap->ref_processor();
 192     }
 193 
 194     // Step 1: Process ordinary GC roots.
 195     {
 196       ShenandoahTraversalClosure roots_cl(q, rp);
 197       ShenandoahMarkCLDClosure cld_cl(&roots_cl);
 198       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
 199       if (unload_classes) {
 200         _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, NULL, NULL, worker_id);
 201         // Need to pre-evac code roots here. Otherwise we might see from-space constants.
 202         ShenandoahWorkerTimings* worker_times = _heap->phase_timings()->worker_times();
 203         ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
 204         _cset_coderoots->possibly_parallel_blobs_do(&code_cl);
 205       } else {
 206         _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, NULL, worker_id);
 207       }
 208     }
 209   }
 210 };
 211 
 212 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
 213 private:
 214   ShenandoahTaskTerminator* _terminator;
 215   ShenandoahHeap* _heap;
 216 public:
 217   ShenandoahConcurrentTraversalCollectionTask(ShenandoahTaskTerminator* terminator) :
 218     AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
 219     _terminator(terminator),
 220     _heap(ShenandoahHeap::heap()) {}
 221 
 222   void work(uint worker_id) {
 223     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 224     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 225     ShenandoahEvacOOMScope oom_evac_scope;
 226     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 227 
 228     // Drain all outstanding work in queues.
 229     traversal_gc->main_loop(worker_id, _terminator, true);
 230   }
 231 };
 232 
 233 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
 234 private:
 235   ShenandoahRootProcessor* _rp;
 236   ShenandoahTaskTerminator* _terminator;
 237   ShenandoahHeap* _heap;
 238 public:
 239   ShenandoahFinalTraversalCollectionTask(ShenandoahRootProcessor* rp, ShenandoahTaskTerminator* terminator) :
 240     AbstractGangTask("Shenandoah Final Traversal Collection"),
 241     _rp(rp),
 242     _terminator(terminator),
 243     _heap(ShenandoahHeap::heap()) {}
 244 
 245   void work(uint worker_id) {
 246     ShenandoahParallelWorkerSession worker_session(worker_id);
 247 
 248     ShenandoahEvacOOMScope oom_evac_scope;
 249     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 250 
 251     ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
 252     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 253 
 254     bool process_refs = _heap->process_references();
 255     bool unload_classes = _heap->unload_classes();
 256     ReferenceProcessor* rp = NULL;
 257     if (process_refs) {
 258       rp = _heap->ref_processor();
 259     }
 260 
 261     // Step 0: Drain outstanding SATB queues.
 262     // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
 263     ShenandoahTraversalSATBBufferClosure satb_cl(q);
 264     {
 265       // Process remaining finished SATB buffers.
 266       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 267       while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
 268       // Process remaining threads SATB buffers below.
 269     }
 270 
 271     // Step 1: Process GC roots.
 272     // For oops in code roots, they are marked, evacuated, enqueued for further traversal,
 273     // and the references to the oops are updated during init pause. New nmethods are handled
 274     // in similar way during nmethod-register process. Therefore, we don't need to rescan code
 275     // roots here.
 276     if (!_heap->is_degenerated_gc_in_progress()) {
 277       ShenandoahTraversalClosure roots_cl(q, rp);
 278       CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
 279       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 280       if (unload_classes) {
 281         ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
 282         _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, NULL, &tc, worker_id);
 283       } else {
 284         _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, &tc, worker_id);
 285       }
 286     } else {
 287       ShenandoahTraversalDegenClosure roots_cl(q, rp);
 288       CLDToOopClosure cld_cl(&roots_cl, ClassLoaderData::_claim_strong);
 289       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 290       if (unload_classes) {
 291         ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
 292         _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, NULL, &tc, worker_id);
 293       } else {
 294         _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, NULL, &tc, worker_id);
 295       }
 296     }
 297 
 298     {
 299       ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
 300       ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
 301 
 302       // Step 3: Finally drain all outstanding work in queues.
 303       traversal_gc->main_loop(worker_id, _terminator, false);
 304     }
 305 
 306   }
 307 };
 308 
 309 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
 310   _heap(heap),
 311   _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())),
 312   _traversal_set(ShenandoahHeapRegionSet()) {
 313 
 314   uint num_queues = heap->max_workers();
 315   for (uint i = 0; i < num_queues; ++i) {
 316     ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
 317     task_queue->initialize();
 318     _task_queues->register_queue(i, task_queue);
 319   }
 320 }
 321 
 322 ShenandoahTraversalGC::~ShenandoahTraversalGC() {
 323 }
 324 
 325 void ShenandoahTraversalGC::prepare_regions() {
 326   size_t num_regions = _heap->num_regions();
 327   ShenandoahMarkingContext* const ctx = _heap->marking_context();
 328   for (size_t i = 0; i < num_regions; i++) {
 329     ShenandoahHeapRegion* region = _heap->get_region(i);
 330     if (_heap->is_bitmap_slice_committed(region)) {
 331       if (_traversal_set.is_in(i)) {
 332         ctx->capture_top_at_mark_start(region);
 333         region->clear_live_data();
 334         assert(ctx->is_bitmap_clear_range(region->bottom(), region->end()), "bitmap for traversal regions must be cleared");
 335       } else {
 336         // Everything outside the traversal set is always considered live.
 337         ctx->reset_top_at_mark_start(region);
 338       }
 339     } else {
 340       // FreeSet may contain uncommitted empty regions, once they are recommitted,
 341       // their TAMS may have old values, so reset them here.
 342       ctx->reset_top_at_mark_start(region);
 343     }
 344   }
 345 }
 346 
 347 void ShenandoahTraversalGC::prepare() {
 348   _heap->collection_set()->clear();
 349   assert(_heap->collection_set()->count() == 0, "collection set not clear");
 350 
 351   {
 352     ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_make_parsable);
 353     _heap->make_parsable(true);
 354   }
 355 
 356   if (UseTLAB) {
 357     ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_resize_tlabs);
 358     _heap->resize_tlabs();
 359   }
 360 
 361   assert(_heap->marking_context()->is_bitmap_clear(), "need clean mark bitmap");
 362   assert(!_heap->marking_context()->is_complete(), "should not be complete");
 363 
 364   ShenandoahFreeSet* free_set = _heap->free_set();
 365   ShenandoahCollectionSet* collection_set = _heap->collection_set();
 366 
 367   // Find collection set
 368   _heap->heuristics()->choose_collection_set(collection_set);
 369   prepare_regions();
 370 
 371   // Rebuild free set
 372   free_set->rebuild();
 373 
 374   log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "M, " SIZE_FORMAT "M CSet, " SIZE_FORMAT " CSet regions",
 375                      collection_set->garbage() / M, collection_set->live_data() / M, collection_set->count());
 376 }
 377 
 378 void ShenandoahTraversalGC::init_traversal_collection() {
 379   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "STW traversal GC");
 380 
 381   if (ShenandoahVerify) {
 382     _heap->verifier()->verify_before_traversal();
 383   }
 384 
 385   if (VerifyBeforeGC) {
 386     Universe::verify();
 387   }
 388 
 389   {
 390     ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare);
 391     ShenandoahHeapLocker lock(_heap->lock());
 392     prepare();
 393   }
 394 
 395   _heap->set_concurrent_traversal_in_progress(true);
 396 
 397   bool process_refs = _heap->process_references();
 398   if (process_refs) {
 399     ReferenceProcessor* rp = _heap->ref_processor();
 400     rp->enable_discovery(true /*verify_no_refs*/);
 401     rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
 402   }
 403 
 404   {
 405     ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::init_traversal_gc_work);
 406     assert(_task_queues->is_empty(), "queues must be empty before traversal GC");
 407     TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
 408 
 409 #if defined(COMPILER2) || INCLUDE_JVMCI
 410     DerivedPointerTable::clear();
 411 #endif
 412 
 413     {
 414       uint nworkers = _heap->workers()->active_workers();
 415       task_queues()->reserve(nworkers);
 416       ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::init_traversal_gc_work);
 417 
 418       ShenandoahCsetCodeRootsIterator cset_coderoots = ShenandoahCodeRoots::cset_iterator();
 419 
 420       ShenandoahInitTraversalCollectionTask traversal_task(&rp, &cset_coderoots);
 421       _heap->workers()->run_task(&traversal_task);
 422     }
 423 
 424 #if defined(COMPILER2) || INCLUDE_JVMCI
 425     DerivedPointerTable::update_pointers();
 426 #endif
 427   }
 428 
 429   if (ShenandoahPacing) {
 430     _heap->pacer()->setup_for_traversal();
 431   }
 432 }
 433 
 434 void ShenandoahTraversalGC::main_loop(uint w, ShenandoahTaskTerminator* t, bool sts_yield) {
 435   ShenandoahObjToScanQueue* q = task_queues()->queue(w);
 436 
 437   // Initialize live data.
 438   jushort* ld = _heap->get_liveness_cache(w);
 439 
 440   ReferenceProcessor* rp = NULL;
 441   if (_heap->process_references()) {
 442     rp = _heap->ref_processor();
 443   }
 444   {
 445     if (!_heap->is_degenerated_gc_in_progress()) {
 446       if (_heap->unload_classes()) {
 447         if (ShenandoahStringDedup::is_enabled()) {
 448           ShenandoahTraversalMetadataDedupClosure cl(q, rp);
 449           main_loop_work<ShenandoahTraversalMetadataDedupClosure>(&cl, ld, w, t, sts_yield);
 450         } else {
 451           ShenandoahTraversalMetadataClosure cl(q, rp);
 452           main_loop_work<ShenandoahTraversalMetadataClosure>(&cl, ld, w, t, sts_yield);
 453         }
 454       } else {
 455         if (ShenandoahStringDedup::is_enabled()) {
 456           ShenandoahTraversalDedupClosure cl(q, rp);
 457           main_loop_work<ShenandoahTraversalDedupClosure>(&cl, ld, w, t, sts_yield);
 458         } else {
 459           ShenandoahTraversalClosure cl(q, rp);
 460           main_loop_work<ShenandoahTraversalClosure>(&cl, ld, w, t, sts_yield);
 461         }
 462       }
 463     } else {
 464       if (_heap->unload_classes()) {
 465         if (ShenandoahStringDedup::is_enabled()) {
 466           ShenandoahTraversalMetadataDedupDegenClosure cl(q, rp);
 467           main_loop_work<ShenandoahTraversalMetadataDedupDegenClosure>(&cl, ld, w, t, sts_yield);
 468         } else {
 469           ShenandoahTraversalMetadataDegenClosure cl(q, rp);
 470           main_loop_work<ShenandoahTraversalMetadataDegenClosure>(&cl, ld, w, t, sts_yield);
 471         }
 472       } else {
 473         if (ShenandoahStringDedup::is_enabled()) {
 474           ShenandoahTraversalDedupDegenClosure cl(q, rp);
 475           main_loop_work<ShenandoahTraversalDedupDegenClosure>(&cl, ld, w, t, sts_yield);
 476         } else {
 477           ShenandoahTraversalDegenClosure cl(q, rp);
 478           main_loop_work<ShenandoahTraversalDegenClosure>(&cl, ld, w, t, sts_yield);
 479         }
 480       }
 481     }
 482   }
 483 
 484   _heap->flush_liveness_cache(w);
 485 }
 486 
 487 template <class T>
 488 void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, ShenandoahTaskTerminator* terminator, bool sts_yield) {
 489   ShenandoahObjToScanQueueSet* queues = task_queues();
 490   ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 491   ShenandoahConcurrentMark* conc_mark = _heap->concurrent_mark();
 492 
 493   uintx stride = ShenandoahMarkLoopStride;
 494 
 495   ShenandoahMarkTask task;
 496 
 497   // Process outstanding queues, if any.
 498   q = queues->claim_next();
 499   while (q != NULL) {
 500     if (_heap->check_cancelled_gc_and_yield(sts_yield)) {
 501       ShenandoahCancelledTerminatorTerminator tt;
 502       ShenandoahEvacOOMScopeLeaver oom_scope_leaver;
 503       ShenandoahSuspendibleThreadSetLeaver stsl(sts_yield && ShenandoahSuspendibleWorkers);
 504       while (!terminator->offer_termination(&tt));
 505       return;
 506     }
 507 
 508     for (uint i = 0; i < stride; i++) {
 509       if (q->pop(task)) {
 510         conc_mark->do_task<T>(q, cl, live_data, &task);
 511       } else {
 512         assert(q->is_empty(), "Must be empty");
 513         q = queues->claim_next();
 514         break;
 515       }
 516     }
 517   }
 518 
 519   if (check_and_handle_cancelled_gc(terminator, sts_yield)) return;
 520 
 521   // Normal loop.
 522   q = queues->queue(worker_id);
 523 
 524   ShenandoahTraversalSATBBufferClosure drain_satb(q);
 525   SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 526 
 527   while (true) {
 528     if (check_and_handle_cancelled_gc(terminator, sts_yield)) return;
 529 
 530     while (satb_mq_set.completed_buffers_num() > 0) {
 531       satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
 532     }
 533 
 534     uint work = 0;
 535     for (uint i = 0; i < stride; i++) {
 536       if (q->pop(task) ||
 537           queues->steal(worker_id, task)) {
 538         conc_mark->do_task<T>(q, cl, live_data, &task);
 539         work++;
 540       } else {
 541         break;
 542       }
 543     }
 544 
 545     if (work == 0) {
 546       // No more work, try to terminate
 547       ShenandoahEvacOOMScopeLeaver oom_scope_leaver;
 548       ShenandoahSuspendibleThreadSetLeaver stsl(sts_yield && ShenandoahSuspendibleWorkers);
 549       ShenandoahTerminationTimingsTracker term_tracker(worker_id);
 550       if (terminator->offer_termination()) return;
 551     }
 552   }
 553 }
 554 
 555 bool ShenandoahTraversalGC::check_and_handle_cancelled_gc(ShenandoahTaskTerminator* terminator, bool sts_yield) {
 556   if (_heap->cancelled_gc()) {
 557     ShenandoahCancelledTerminatorTerminator tt;
 558     ShenandoahEvacOOMScopeLeaver oom_scope_leaver;
 559     ShenandoahSuspendibleThreadSetLeaver stsl(sts_yield && ShenandoahSuspendibleWorkers);
 560     while (! terminator->offer_termination(&tt));
 561     return true;
 562   }
 563   return false;
 564 }
 565 
 566 void ShenandoahTraversalGC::concurrent_traversal_collection() {
 567   ClassLoaderDataGraph::clear_claimed_marks();
 568 
 569   ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::conc_traversal);
 570   if (!_heap->cancelled_gc()) {
 571     uint nworkers = _heap->workers()->active_workers();
 572     task_queues()->reserve(nworkers);
 573     ShenandoahTerminationTracker tracker(ShenandoahPhaseTimings::conc_traversal_termination);
 574 
 575     ShenandoahTaskTerminator terminator(nworkers, task_queues());
 576     ShenandoahConcurrentTraversalCollectionTask task(&terminator);
 577     _heap->workers()->run_task(&task);
 578   }
 579 
 580   if (!_heap->cancelled_gc() && ShenandoahPreclean && _heap->process_references()) {
 581     preclean_weak_refs();
 582   }
 583 }
 584 
 585 void ShenandoahTraversalGC::final_traversal_collection() {
 586   _heap->make_parsable(true);
 587 
 588   if (!_heap->cancelled_gc()) {
 589 #if defined(COMPILER2) || INCLUDE_JVMCI
 590     DerivedPointerTable::clear();
 591 #endif
 592     ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::final_traversal_gc_work);
 593     uint nworkers = _heap->workers()->active_workers();
 594     task_queues()->reserve(nworkers);
 595 
 596     // Finish traversal
 597     ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::final_traversal_gc_work);
 598     ShenandoahTerminationTracker term(ShenandoahPhaseTimings::final_traversal_gc_termination);
 599 
 600     ShenandoahTaskTerminator terminator(nworkers, task_queues());
 601     ShenandoahFinalTraversalCollectionTask task(&rp, &terminator);
 602     _heap->workers()->run_task(&task);
 603 #if defined(COMPILER2) || INCLUDE_JVMCI
 604     DerivedPointerTable::update_pointers();
 605 #endif
 606   }
 607 
 608   if (!_heap->cancelled_gc() && _heap->process_references()) {
 609     weak_refs_work();
 610   }
 611 
 612   if (!_heap->cancelled_gc() && _heap->unload_classes()) {
 613     _heap->unload_classes_and_cleanup_tables(false);
 614     fixup_roots();
 615   }
 616 
 617   if (!_heap->cancelled_gc()) {
 618     assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
 619     TASKQUEUE_STATS_ONLY(_task_queues->print_taskqueue_stats());
 620     TASKQUEUE_STATS_ONLY(_task_queues->reset_taskqueue_stats());
 621 
 622     // No more marking expected
 623     _heap->mark_complete_marking_context();
 624 
 625     // Resize metaspace
 626     MetaspaceGC::compute_new_size();
 627 
 628     // Still good? We can now trash the cset, and make final verification
 629     {
 630       ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);
 631       ShenandoahHeapLocker lock(_heap->lock());
 632 
 633       // Trash everything
 634       // Clear immediate garbage regions.
 635       size_t num_regions = _heap->num_regions();
 636 
 637       ShenandoahHeapRegionSet* traversal_regions = traversal_set();
 638       ShenandoahFreeSet* free_regions = _heap->free_set();
 639       ShenandoahMarkingContext* const ctx = _heap->marking_context();
 640       free_regions->clear();
 641       for (size_t i = 0; i < num_regions; i++) {
 642         ShenandoahHeapRegion* r = _heap->get_region(i);
 643         bool not_allocated = ctx->top_at_mark_start(r) == r->top();
 644 
 645         bool candidate = traversal_regions->is_in(r) && !r->has_live() && not_allocated;
 646         if (r->is_humongous_start() && candidate) {
 647           // Trash humongous.
 648           HeapWord* humongous_obj = r->bottom() + ShenandoahBrooksPointer::word_size();
 649           assert(!ctx->is_marked(oop(humongous_obj)), "must not be marked");
 650           r->make_trash_immediate();
 651           while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) {
 652             i++;
 653             r = _heap->get_region(i);
 654             assert(r->is_humongous_continuation(), "must be humongous continuation");
 655             r->make_trash_immediate();
 656           }
 657         } else if (!r->is_empty() && candidate) {
 658           // Trash regular.
 659           assert(!r->is_humongous(), "handled above");
 660           assert(!r->is_trash(), "must not already be trashed");
 661           r->make_trash_immediate();
 662         }
 663       }
 664       _heap->collection_set()->clear();
 665       _heap->free_set()->rebuild();
 666       reset();
 667     }
 668 
 669     assert(_task_queues->is_empty(), "queues must be empty after traversal GC");
 670     _heap->set_concurrent_traversal_in_progress(false);
 671     assert(!_heap->cancelled_gc(), "must not be cancelled when getting out here");
 672 
 673     if (ShenandoahVerify) {
 674       _heap->verifier()->verify_after_traversal();
 675     }
 676 
 677     if (VerifyAfterGC) {
 678       Universe::verify();
 679     }
 680   }
 681 }
 682 
 683 class ShenandoahTraversalFixRootsClosure : public OopClosure {
 684 private:
 685   template <class T>
 686   inline void do_oop_work(T* p) {
 687     T o = RawAccess<>::oop_load(p);
 688     if (!CompressedOops::is_null(o)) {
 689       oop obj = CompressedOops::decode_not_null(o);
 690       oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 691       if (!oopDesc::equals_raw(obj, forw)) {
 692         RawAccess<IS_NOT_NULL>::oop_store(p, forw);
 693       }
 694     }
 695   }
 696 
 697 public:
 698   inline void do_oop(oop* p) { do_oop_work(p); }
 699   inline void do_oop(narrowOop* p) { do_oop_work(p); }
 700 };
 701 
 702 class ShenandoahTraversalFixRootsTask : public AbstractGangTask {
 703 private:
 704   ShenandoahRootProcessor* _rp;
 705 
 706 public:
 707   ShenandoahTraversalFixRootsTask(ShenandoahRootProcessor* rp) :
 708     AbstractGangTask("Shenandoah traversal fix roots"),
 709     _rp(rp) {}
 710 
 711   void work(uint worker_id) {
 712     ShenandoahParallelWorkerSession worker_session(worker_id);
 713     ShenandoahTraversalFixRootsClosure cl;
 714     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
 715     CLDToOopClosure cldCl(&cl, ClassLoaderData::_claim_strong);
 716     _rp->process_all_roots(&cl, &cl, &cldCl, &blobsCl, NULL, worker_id);
 717   }
 718 };
 719 
 720 void ShenandoahTraversalGC::fixup_roots() {
 721 #if defined(COMPILER2) || INCLUDE_JVMCI
 722   DerivedPointerTable::clear();
 723 #endif
 724   ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), ShenandoahPhaseTimings::final_traversal_update_roots);
 725   ShenandoahTraversalFixRootsTask update_roots_task(&rp);
 726   _heap->workers()->run_task(&update_roots_task);
 727 #if defined(COMPILER2) || INCLUDE_JVMCI
 728   DerivedPointerTable::update_pointers();
 729 #endif
 730 }
 731 
 732 void ShenandoahTraversalGC::reset() {
 733   _task_queues->clear();
 734 }
 735 
 736 ShenandoahObjToScanQueueSet* ShenandoahTraversalGC::task_queues() {
 737   return _task_queues;
 738 }
 739 
 740 class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure {
 741 private:
 742   ShenandoahHeap* const _heap;
 743 public:
 744   ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
 745   virtual bool should_return() { return _heap->cancelled_gc(); }
 746 };
 747 
 748 class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure {
 749 public:
 750   void do_void() {
 751     ShenandoahHeap* sh = ShenandoahHeap::heap();
 752     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 753     assert(sh->process_references(), "why else would we be here?");
 754     ShenandoahTaskTerminator terminator(1, traversal_gc->task_queues());
 755     shenandoah_assert_rp_isalive_installed();
 756     traversal_gc->main_loop((uint) 0, &terminator, true);
 757   }
 758 };
 759 
 760 class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure {
 761 private:
 762   ShenandoahObjToScanQueue* _queue;
 763   Thread* _thread;
 764   ShenandoahTraversalGC* _traversal_gc;
 765   ShenandoahMarkingContext* const _mark_context;
 766 
 767   template <class T>
 768   inline void do_oop_work(T* p) {
 769     _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue, _mark_context);
 770   }
 771 
 772 public:
 773   ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 774     _queue(q), _thread(Thread::current()),
 775     _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 776     _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 777 
 778   void do_oop(narrowOop* p) { do_oop_work(p); }
 779   void do_oop(oop* p)       { do_oop_work(p); }
 780 };
 781 
 782 class ShenandoahTraversalWeakUpdateClosure : public OopClosure {
 783 private:
 784   template <class T>
 785   inline void do_oop_work(T* p) {
 786     // Cannot call maybe_update_with_forwarded, because on traversal-degen
 787     // path the collection set is already dropped. Instead, do the unguarded store.
 788     // TODO: This can be fixed after degen-traversal stops dropping cset.
 789     T o = RawAccess<>::oop_load(p);
 790     if (!CompressedOops::is_null(o)) {
 791       oop obj = CompressedOops::decode_not_null(o);
 792       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 793       shenandoah_assert_marked(p, obj);
 794       RawAccess<IS_NOT_NULL>::oop_store(p, obj);
 795     }
 796   }
 797 
 798 public:
 799   ShenandoahTraversalWeakUpdateClosure() {}
 800 
 801   void do_oop(narrowOop* p) { do_oop_work(p); }
 802   void do_oop(oop* p)       { do_oop_work(p); }
 803 };
 804 
 805 class ShenandoahTraversalKeepAliveUpdateDegenClosure : public OopClosure {
 806 private:
 807   ShenandoahObjToScanQueue* _queue;
 808   Thread* _thread;
 809   ShenandoahTraversalGC* _traversal_gc;
 810   ShenandoahMarkingContext* const _mark_context;
 811 
 812   template <class T>
 813   inline void do_oop_work(T* p) {
 814     _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue, _mark_context);
 815   }
 816 
 817 public:
 818   ShenandoahTraversalKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
 819           _queue(q), _thread(Thread::current()),
 820           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 821           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 822 
 823   void do_oop(narrowOop* p) { do_oop_work(p); }
 824   void do_oop(oop* p)       { do_oop_work(p); }
 825 };
 826 
 827 class ShenandoahTraversalSingleThreadKeepAliveUpdateClosure : public OopClosure {
 828 private:
 829   ShenandoahObjToScanQueue* _queue;
 830   Thread* _thread;
 831   ShenandoahTraversalGC* _traversal_gc;
 832   ShenandoahMarkingContext* const _mark_context;
 833 
 834   template <class T>
 835   inline void do_oop_work(T* p) {
 836     ShenandoahEvacOOMScope evac_scope;
 837     _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */>(p, _thread, _queue, _mark_context);
 838   }
 839 
 840 public:
 841   ShenandoahTraversalSingleThreadKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 842           _queue(q), _thread(Thread::current()),
 843           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 844           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 845 
 846   void do_oop(narrowOop* p) { do_oop_work(p); }
 847   void do_oop(oop* p)       { do_oop_work(p); }
 848 };
 849 
 850 class ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure : public OopClosure {
 851 private:
 852   ShenandoahObjToScanQueue* _queue;
 853   Thread* _thread;
 854   ShenandoahTraversalGC* _traversal_gc;
 855   ShenandoahMarkingContext* const _mark_context;
 856 
 857   template <class T>
 858   inline void do_oop_work(T* p) {
 859     ShenandoahEvacOOMScope evac_scope;
 860     _traversal_gc->process_oop<T, false /* string dedup */, true /* degen */>(p, _thread, _queue, _mark_context);
 861   }
 862 
 863 public:
 864   ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure(ShenandoahObjToScanQueue* q) :
 865           _queue(q), _thread(Thread::current()),
 866           _traversal_gc(ShenandoahHeap::heap()->traversal_gc()),
 867           _mark_context(ShenandoahHeap::heap()->marking_context()) {}
 868 
 869   void do_oop(narrowOop* p) { do_oop_work(p); }
 870   void do_oop(oop* p)       { do_oop_work(p); }
 871 };
 872 
 873 class ShenandoahTraversalPrecleanTask : public AbstractGangTask {
 874 private:
 875   ReferenceProcessor* _rp;
 876 
 877 public:
 878   ShenandoahTraversalPrecleanTask(ReferenceProcessor* rp) :
 879           AbstractGangTask("Precleaning task"),
 880           _rp(rp) {}
 881 
 882   void work(uint worker_id) {
 883     assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
 884     ShenandoahParallelWorkerSession worker_session(worker_id);
 885     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 886     ShenandoahEvacOOMScope oom_evac_scope;
 887 
 888     ShenandoahHeap* sh = ShenandoahHeap::heap();
 889 
 890     ShenandoahObjToScanQueue* q = sh->traversal_gc()->task_queues()->queue(worker_id);
 891 
 892     ShenandoahForwardedIsAliveClosure is_alive;
 893     ShenandoahTraversalCancelledGCYieldClosure yield;
 894     ShenandoahTraversalPrecleanCompleteGCClosure complete_gc;
 895     ShenandoahTraversalKeepAliveUpdateClosure keep_alive(q);
 896     ResourceMark rm;
 897     _rp->preclean_discovered_references(&is_alive, &keep_alive,
 898                                         &complete_gc, &yield,
 899                                         NULL);
 900   }
 901 };
 902 
 903 void ShenandoahTraversalGC::preclean_weak_refs() {
 904   // Pre-cleaning weak references before diving into STW makes sense at the
 905   // end of concurrent mark. This will filter out the references which referents
 906   // are alive. Note that ReferenceProcessor already filters out these on reference
 907   // discovery, and the bulk of work is done here. This phase processes leftovers
 908   // that missed the initial filtering, i.e. when referent was marked alive after
 909   // reference was discovered by RP.
 910 
 911   assert(_heap->process_references(), "sanity");
 912   assert(!_heap->is_degenerated_gc_in_progress(), "must be in concurrent non-degenerated phase");
 913 
 914   // Shortcut if no references were discovered to avoid winding up threads.
 915   ReferenceProcessor* rp = _heap->ref_processor();
 916   if (!rp->has_discovered_references()) {
 917     return;
 918   }
 919 
 920   ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
 921 
 922   shenandoah_assert_rp_isalive_not_installed();
 923   ShenandoahForwardedIsAliveClosure is_alive;
 924   ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive);
 925 
 926   assert(task_queues()->is_empty(), "Should be empty");
 927 
 928   // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
 929   // queues and other goodies. When upstream ReferenceProcessor starts supporting
 930   // parallel precleans, we can extend this to more threads.
 931   ShenandoahPushWorkerScope scope(_heap->workers(), 1, /* check_workers = */ false);
 932 
 933   WorkGang* workers = _heap->workers();
 934   uint nworkers = workers->active_workers();
 935   assert(nworkers == 1, "This code uses only a single worker");
 936   task_queues()->reserve(nworkers);
 937 
 938   ShenandoahTraversalPrecleanTask task(rp);
 939   workers->run_task(&task);
 940 
 941   assert(_heap->cancelled_gc() || task_queues()->is_empty(), "Should be empty");
 942 }
 943 
 944 // Weak Reference Closures
 945 class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure {
 946   uint _worker_id;
 947   ShenandoahTaskTerminator* _terminator;
 948   bool _reset_terminator;
 949 
 950 public:
 951   ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false):
 952     _worker_id(worker_id),
 953     _terminator(t),
 954     _reset_terminator(reset_terminator) {
 955   }
 956 
 957   void do_void() {
 958     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 959 
 960     ShenandoahHeap* sh = ShenandoahHeap::heap();
 961     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 962     assert(sh->process_references(), "why else would we be here?");
 963     shenandoah_assert_rp_isalive_installed();
 964 
 965     traversal_gc->main_loop(_worker_id, _terminator, false);
 966 
 967     if (_reset_terminator) {
 968       _terminator->reset_for_reuse();
 969     }
 970   }
 971 };
 972 
 973 class ShenandoahTraversalSingleThreadedDrainMarkingStackClosure: public VoidClosure {
 974   uint _worker_id;
 975   ShenandoahTaskTerminator* _terminator;
 976   bool _reset_terminator;
 977 
 978 public:
 979   ShenandoahTraversalSingleThreadedDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false):
 980           _worker_id(worker_id),
 981           _terminator(t),
 982           _reset_terminator(reset_terminator) {
 983   }
 984 
 985   void do_void() {
 986     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 987 
 988     ShenandoahHeap* sh = ShenandoahHeap::heap();
 989     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 990     assert(sh->process_references(), "why else would we be here?");
 991     shenandoah_assert_rp_isalive_installed();
 992 
 993     ShenandoahEvacOOMScope evac_scope;
 994     traversal_gc->main_loop(_worker_id, _terminator, false);
 995 
 996     if (_reset_terminator) {
 997       _terminator->reset_for_reuse();
 998     }
 999   }
1000 };
1001 
1002 void ShenandoahTraversalGC::weak_refs_work() {
1003   assert(_heap->process_references(), "sanity");
1004 
1005   ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs;
1006 
1007   ShenandoahGCPhase phase(phase_root);
1008 
1009   ReferenceProcessor* rp = _heap->ref_processor();
1010 
1011   // NOTE: We cannot shortcut on has_discovered_references() here, because
1012   // we will miss marking JNI Weak refs then, see implementation in
1013   // ReferenceProcessor::process_discovered_references.
1014   weak_refs_work_doit();
1015 
1016   rp->verify_no_references_recorded();
1017   assert(!rp->discovery_enabled(), "Post condition");
1018 
1019 }
1020 
1021 class ShenandoahTraversalRefProcTaskProxy : public AbstractGangTask {
1022 private:
1023   AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
1024   ShenandoahTaskTerminator* _terminator;
1025 
1026 public:
1027   ShenandoahTraversalRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
1028                                       ShenandoahTaskTerminator* t) :
1029     AbstractGangTask("Process reference objects in parallel"),
1030     _proc_task(proc_task),
1031     _terminator(t) {
1032   }
1033 
1034   void work(uint worker_id) {
1035     ShenandoahEvacOOMScope oom_evac_scope;
1036     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
1037     ShenandoahHeap* heap = ShenandoahHeap::heap();
1038     ShenandoahTraversalDrainMarkingStackClosure complete_gc(worker_id, _terminator);
1039 
1040     ShenandoahForwardedIsAliveClosure is_alive;
1041     if (!heap->is_degenerated_gc_in_progress()) {
1042       ShenandoahTraversalKeepAliveUpdateClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
1043       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
1044     } else {
1045       ShenandoahTraversalKeepAliveUpdateDegenClosure keep_alive(heap->traversal_gc()->task_queues()->queue(worker_id));
1046       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
1047     }
1048   }
1049 };
1050 
1051 class ShenandoahTraversalRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
1052 private:
1053   WorkGang* _workers;
1054 
1055 public:
1056   ShenandoahTraversalRefProcTaskExecutor(WorkGang* workers) : _workers(workers) {}
1057 
1058   // Executes a task using worker threads.
1059   void execute(ProcessTask& task, uint ergo_workers) {
1060     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
1061 
1062     ShenandoahHeap* heap = ShenandoahHeap::heap();
1063     ShenandoahTraversalGC* traversal_gc = heap->traversal_gc();
1064     ShenandoahPushWorkerQueuesScope scope(_workers,
1065                                           traversal_gc->task_queues(),
1066                                           ergo_workers,
1067                                           /* do_check = */ false);
1068     uint nworkers = _workers->active_workers();
1069     traversal_gc->task_queues()->reserve(nworkers);
1070     ShenandoahTaskTerminator terminator(nworkers, traversal_gc->task_queues());
1071     ShenandoahTraversalRefProcTaskProxy proc_task_proxy(task, &terminator);
1072     _workers->run_task(&proc_task_proxy);
1073   }
1074 };
1075 
1076 void ShenandoahTraversalGC::weak_refs_work_doit() {
1077   ReferenceProcessor* rp = _heap->ref_processor();
1078 
1079   ShenandoahPhaseTimings::Phase phase_process = ShenandoahPhaseTimings::weakrefs_process;
1080 
1081   shenandoah_assert_rp_isalive_not_installed();
1082   ShenandoahForwardedIsAliveClosure is_alive;
1083   ReferenceProcessorIsAliveMutator fix_isalive(rp, &is_alive);
1084 
1085   WorkGang* workers = _heap->workers();
1086   uint nworkers = workers->active_workers();
1087 
1088   rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
1089   rp->set_active_mt_degree(nworkers);
1090 
1091   assert(task_queues()->is_empty(), "Should be empty");
1092 
1093   // complete_gc and keep_alive closures instantiated here are only needed for
1094   // single-threaded path in RP. They share the queue 0 for tracking work, which
1095   // simplifies implementation. Since RP may decide to call complete_gc several
1096   // times, we need to be able to reuse the terminator.
1097   uint serial_worker_id = 0;
1098   ShenandoahTaskTerminator terminator(1, task_queues());
1099   ShenandoahTraversalSingleThreadedDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
1100   ShenandoahPushWorkerQueuesScope scope(workers, task_queues(), 1, /* do_check = */ false);
1101 
1102   ShenandoahTraversalRefProcTaskExecutor executor(workers);
1103 
1104   ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
1105   if (!_heap->is_degenerated_gc_in_progress()) {
1106     ShenandoahTraversalSingleThreadKeepAliveUpdateClosure keep_alive(task_queues()->queue(serial_worker_id));
1107     rp->process_discovered_references(&is_alive, &keep_alive,
1108                                       &complete_gc, &executor,
1109                                       &pt);
1110   } else {
1111     ShenandoahTraversalSingleThreadKeepAliveUpdateDegenClosure keep_alive(task_queues()->queue(serial_worker_id));
1112     rp->process_discovered_references(&is_alive, &keep_alive,
1113                                       &complete_gc, &executor,
1114                                       &pt);
1115   }
1116 
1117   {
1118     ShenandoahGCPhase phase(phase_process);
1119     ShenandoahTerminationTracker termination(ShenandoahPhaseTimings::weakrefs_termination);
1120 
1121     // Process leftover weak oops (using parallel version)
1122     ShenandoahTraversalWeakUpdateClosure cl;
1123     WeakProcessor::weak_oops_do(workers, &is_alive, &cl, 1);
1124 
1125     pt.print_all_references();
1126 
1127     assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty");
1128   }
1129 }