rev 59458 : 8245827: Shenandoah: Cleanup Shenandoah code root iterators and root scanner
rev 59459 : 8245880: Shenandoah: should not mark all code roots if class unloading is enabled

   1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 
  27 #include "classfile/symbolTable.hpp"
  28 #include "classfile/systemDictionary.hpp"
  29 #include "code/codeCache.hpp"
  30 
  31 #include "gc/shared/weakProcessor.inline.hpp"
  32 #include "gc/shared/gcTimer.hpp"
  33 #include "gc/shared/gcTrace.hpp"
  34 #include "gc/shared/referenceProcessor.hpp"
  35 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
  36 #include "gc/shared/strongRootsScope.hpp"
  37 
  38 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
  39 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  40 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  41 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  42 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  43 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  44 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  45 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  46 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
  47 #include "gc/shenandoah/shenandoahUtils.hpp"
  48 
  49 #include "memory/iterator.inline.hpp"
  50 #include "memory/metaspace.hpp"
  51 #include "memory/resourceArea.hpp"
  52 #include "oops/oop.inline.hpp"
  53 #include "runtime/handles.inline.hpp"
  54 
  55 template<UpdateRefsMode UPDATE_REFS>
  56 class ShenandoahInitMarkRootsClosure : public OopClosure {
  57 private:
  58   ShenandoahObjToScanQueue* _queue;
  59   ShenandoahHeap* _heap;
  60   ShenandoahMarkingContext* const _mark_context;
  61 
  62   template <class T>
  63   inline void do_oop_work(T* p) {
  64     ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, NO_DEDUP>(p, _heap, _queue, _mark_context);
  65   }
  66 
  67 public:
  68   ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) :
  69     _queue(q),
  70     _heap(ShenandoahHeap::heap()),
  71     _mark_context(_heap->marking_context()) {};
  72 
  73   void do_oop(narrowOop* p) { do_oop_work(p); }
  74   void do_oop(oop* p)       { do_oop_work(p); }
  75 };
  76 
  77 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
  78   MetadataVisitingOopIterateClosure(rp),
  79   _queue(q),
  80   _heap(ShenandoahHeap::heap()),
  81   _mark_context(_heap->marking_context())
  82 { }
  83 
  84 template<UpdateRefsMode UPDATE_REFS>
  85 class ShenandoahInitMarkRootsTask : public AbstractGangTask {
  86 private:
  87   ShenandoahRootScanner* _rp;
  88 public:
  89   ShenandoahInitMarkRootsTask(ShenandoahRootScanner* rp) :
  90     AbstractGangTask("Shenandoah init mark roots task"),
  91     _rp(rp) {
  92   }
  93 
  94   void work(uint worker_id) {
  95     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
  96     ShenandoahParallelWorkerSession worker_session(worker_id);
  97 
  98     ShenandoahHeap* heap = ShenandoahHeap::heap();
  99     ShenandoahObjToScanQueueSet* queues = heap->concurrent_mark()->task_queues();
 100     assert(queues->get_reserved() > worker_id, "Queue has not been reserved for worker id: %d", worker_id);
 101 
 102     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 103 
 104     ShenandoahInitMarkRootsClosure<UPDATE_REFS> mark_cl(q);
 105     do_work(heap, &mark_cl, worker_id);
 106   }
 107 
 108 private:
 109   void do_work(ShenandoahHeap* heap, OopClosure* oops, uint worker_id) {
 110     // The rationale for selecting the roots to scan is as follows:
 111     //   a. With unload_classes = true, we only want to scan the actual strong roots from the
 112     //      code cache. This will allow us to identify the dead classes, unload them, *and*
 113     //      invalidate the relevant code cache blobs. This could be only done together with
 114     //      class unloading.
 115     //   b. With unload_classes = false, we have to nominally retain all the references from code
 116     //      cache, because there could be the case of embedded class/oop in the generated code,
 117     //      which we will never visit during mark. Without code cache invalidation, as in (a),
 118     //      we risk executing that code cache blob, and crashing.
 119     if (heap->unload_classes()) {
 120       _rp->strong_roots_do(worker_id, oops);
 121     } else {
 122       _rp->roots_do(worker_id, oops);
 123     }
 124   }
 125 };
 126 
 127 class ShenandoahUpdateRootsTask : public AbstractGangTask {
 128 private:
 129   ShenandoahRootUpdater*  _root_updater;
 130   bool                    _check_alive;
 131 public:
 132   ShenandoahUpdateRootsTask(ShenandoahRootUpdater* root_updater, bool check_alive) :
 133     AbstractGangTask("Shenandoah update roots task"),
 134     _root_updater(root_updater),
 135     _check_alive(check_alive){
 136   }
 137 
 138   void work(uint worker_id) {
 139     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 140     ShenandoahParallelWorkerSession worker_session(worker_id);
 141 
 142     ShenandoahHeap* heap = ShenandoahHeap::heap();
 143     ShenandoahUpdateRefsClosure cl;
 144     if (_check_alive) {
 145       ShenandoahForwardedIsAliveClosure is_alive;
 146       _root_updater->roots_do<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>(worker_id, &is_alive, &cl);
 147     } else {
 148       AlwaysTrueClosure always_true;;
 149       _root_updater->roots_do<AlwaysTrueClosure, ShenandoahUpdateRefsClosure>(worker_id, &always_true, &cl);
 150     }
 151   }
 152 };
 153 
 154 class ShenandoahConcurrentMarkingTask : public AbstractGangTask {
 155 private:
 156   ShenandoahConcurrentMark* _cm;
 157   TaskTerminator* _terminator;
 158 
 159 public:
 160   ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator) :
 161     AbstractGangTask("Root Region Scan"), _cm(cm), _terminator(terminator) {
 162   }
 163 
 164   void work(uint worker_id) {
 165     ShenandoahHeap* heap = ShenandoahHeap::heap();
 166     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 167     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 168     ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
 169     ReferenceProcessor* rp;
 170     if (heap->process_references()) {
 171       rp = heap->ref_processor();
 172       shenandoah_assert_rp_isalive_installed();
 173     } else {
 174       rp = NULL;
 175     }
 176 

 177     _cm->concurrent_scan_code_roots(worker_id, rp);


 178     _cm->mark_loop(worker_id, _terminator, rp,
 179                    true, // cancellable
 180                    ShenandoahStringDedup::is_enabled()); // perform string dedup
 181   }
 182 };
 183 
 184 class ShenandoahSATBAndRemarkCodeRootsThreadsClosure : public ThreadClosure {
 185 private:
 186   ShenandoahSATBBufferClosure* _satb_cl;
 187   OopClosure*            const _cl;
 188   MarkingCodeBlobClosure*      _code_cl;
 189   uintx _claim_token;
 190 
 191 public:
 192   ShenandoahSATBAndRemarkCodeRootsThreadsClosure(ShenandoahSATBBufferClosure* satb_cl, OopClosure* cl, MarkingCodeBlobClosure* code_cl) :
 193     _satb_cl(satb_cl), _cl(cl), _code_cl(code_cl),
 194     _claim_token(Threads::thread_claim_token()) {}
 195 
 196   void do_thread(Thread* thread) {
 197     if (thread->claim_threads_do(true, _claim_token)) {
 198       ShenandoahThreadLocalData::satb_mark_queue(thread).apply_closure_and_empty(_satb_cl);
 199       if (thread->is_Java_thread()) {
 200         if (_cl != NULL) {
 201           ResourceMark rm;
 202           thread->oops_do(_cl, _code_cl);
 203         } else if (_code_cl != NULL) {
 204           // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
 205           // however the liveness of oops reachable from nmethods have very complex lifecycles:
 206           // * Alive if on the stack of an executing method
 207           // * Weakly reachable otherwise
 208           // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
 209           // live by the SATB invariant but other oops recorded in nmethods may behave differently.
 210           JavaThread* jt = (JavaThread*)thread;
 211           jt->nmethods_do(_code_cl);
 212         }
 213       }
 214     }
 215   }
 216 };
 217 
 218 class ShenandoahFinalMarkingTask : public AbstractGangTask {
 219 private:
 220   ShenandoahConcurrentMark* _cm;
 221   TaskTerminator*           _terminator;
 222   bool _dedup_string;
 223 
 224 public:
 225   ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, TaskTerminator* terminator, bool dedup_string) :
 226     AbstractGangTask("Shenandoah Final Marking"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
 227   }
 228 
 229   void work(uint worker_id) {
 230     ShenandoahHeap* heap = ShenandoahHeap::heap();
 231 
 232     ShenandoahParallelWorkerSession worker_session(worker_id);
 233     ReferenceProcessor* rp;
 234     if (heap->process_references()) {
 235       rp = heap->ref_processor();
 236       shenandoah_assert_rp_isalive_installed();
 237     } else {
 238       rp = NULL;
 239     }
 240 
 241     // First drain remaining SATB buffers.
 242     // Notice that this is not strictly necessary for mark-compact. But since
 243     // it requires a StrongRootsScope around the task, we need to claim the
 244     // threads, and performance-wise it doesn't really matter. Adds about 1ms to
 245     // full-gc.
 246     {
 247       ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
 248 
 249       ShenandoahSATBBufferClosure cl(q);
 250       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 251       while (satb_mq_set.apply_closure_to_completed_buffer(&cl));
 252       bool do_nmethods = heap->unload_classes() && !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading();
 253       if (heap->has_forwarded_objects()) {
 254         ShenandoahMarkResolveRefsClosure resolve_mark_cl(q, rp);
 255         MarkingCodeBlobClosure blobsCl(&resolve_mark_cl, !CodeBlobToOopClosure::FixRelocations);
 256         ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl,
 257                                                           ShenandoahStoreValEnqueueBarrier ? &resolve_mark_cl : NULL,
 258                                                           do_nmethods ? &blobsCl : NULL);
 259         Threads::threads_do(&tc);
 260       } else {
 261         ShenandoahMarkRefsClosure mark_cl(q, rp);
 262         MarkingCodeBlobClosure blobsCl(&mark_cl, !CodeBlobToOopClosure::FixRelocations);
 263         ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl,
 264                                                           ShenandoahStoreValEnqueueBarrier ? &mark_cl : NULL,
 265                                                           do_nmethods ? &blobsCl : NULL);
 266         Threads::threads_do(&tc);
 267       }
 268     }
 269 
 270     if (heap->is_degenerated_gc_in_progress()) {

 271       // Degenerated cycle may bypass concurrent cycle, so code roots might not be scanned,
 272       // let's check here.
 273       _cm->concurrent_scan_code_roots(worker_id, rp);
 274     }
 275 
 276     _cm->mark_loop(worker_id, _terminator, rp,
 277                    false, // not cancellable
 278                    _dedup_string);
 279 
 280     assert(_cm->task_queues()->is_empty(), "Should be empty");
 281   }
 282 };
 283 
 284 void ShenandoahConcurrentMark::mark_roots(ShenandoahPhaseTimings::Phase root_phase) {
 285   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
 286   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 287 
 288   ShenandoahHeap* heap = ShenandoahHeap::heap();
 289 
 290   ShenandoahGCPhase phase(root_phase);
 291 
 292   WorkGang* workers = heap->workers();
 293   uint nworkers = workers->active_workers();
 294 
 295   assert(nworkers <= task_queues()->size(), "Just check");
 296 
 297   ShenandoahRootScanner root_proc(nworkers, root_phase);
 298   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
 299   task_queues()->reserve(nworkers);
 300 
 301   if (heap->has_forwarded_objects()) {
 302     ShenandoahInitMarkRootsTask<RESOLVE> mark_roots(&root_proc);
 303     workers->run_task(&mark_roots);
 304   } else {
 305     // No need to update references, which means the heap is stable.
 306     // Can save time not walking through forwarding pointers.
 307     ShenandoahInitMarkRootsTask<NONE> mark_roots(&root_proc);
 308     workers->run_task(&mark_roots);
 309   }
 310 
 311   clear_claim_codecache();
 312 }
 313 
 314 void ShenandoahConcurrentMark::update_roots(ShenandoahPhaseTimings::Phase root_phase) {
 315   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 316   assert(root_phase == ShenandoahPhaseTimings::full_gc_update_roots ||
 317          root_phase == ShenandoahPhaseTimings::degen_gc_update_roots,
 318          "Only for these phases");
 319 
 320   ShenandoahGCPhase phase(root_phase);
 321 
 322   bool check_alive = root_phase == ShenandoahPhaseTimings::degen_gc_update_roots;
 323 
 324 #if COMPILER2_OR_JVMCI
 325   DerivedPointerTable::clear();
 326 #endif
 327 
 328   uint nworkers = _heap->workers()->active_workers();
 329 
 330   ShenandoahRootUpdater root_updater(nworkers, root_phase);
 331   ShenandoahUpdateRootsTask update_roots(&root_updater, check_alive);
 332   _heap->workers()->run_task(&update_roots);
 333 
 334 #if COMPILER2_OR_JVMCI
 335   DerivedPointerTable::update_pointers();
 336 #endif
 337 }
 338 
 339 class ShenandoahUpdateThreadRootsTask : public AbstractGangTask {
 340 private:
 341   ShenandoahThreadRoots           _thread_roots;
 342   ShenandoahPhaseTimings::Phase   _phase;
 343   ShenandoahGCWorkerPhase         _worker_phase;
 344 public:
 345   ShenandoahUpdateThreadRootsTask(bool is_par, ShenandoahPhaseTimings::Phase phase) :
 346     AbstractGangTask("Shenandoah Update Thread Roots"),
 347     _thread_roots(phase, is_par),
 348     _phase(phase),
 349     _worker_phase(phase) {}
 350 
 351   void work(uint worker_id) {
 352     ShenandoahParallelWorkerSession worker_session(worker_id);
 353     ShenandoahUpdateRefsClosure cl;
 354     _thread_roots.oops_do(&cl, NULL, worker_id);
 355   }
 356 };
 357 
 358 void ShenandoahConcurrentMark::update_thread_roots(ShenandoahPhaseTimings::Phase root_phase) {
 359   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 360 
 361   ShenandoahGCPhase phase(root_phase);
 362 
 363 #if COMPILER2_OR_JVMCI
 364   DerivedPointerTable::clear();
 365 #endif
 366 
 367   WorkGang* workers = _heap->workers();
 368   bool is_par = workers->active_workers() > 1;
 369 
 370   ShenandoahUpdateThreadRootsTask task(is_par, root_phase);
 371   workers->run_task(&task);
 372 
 373 #if COMPILER2_OR_JVMCI
 374   DerivedPointerTable::update_pointers();
 375 #endif
 376 }
 377 
 378 void ShenandoahConcurrentMark::initialize(uint workers) {
 379   _heap = ShenandoahHeap::heap();
 380 
 381   uint num_queues = MAX2(workers, 1U);
 382 
 383   _task_queues = new ShenandoahObjToScanQueueSet((int) num_queues);
 384 
 385   for (uint i = 0; i < num_queues; ++i) {
 386     ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
 387     task_queue->initialize();
 388     _task_queues->register_queue(i, task_queue);
 389   }
 390 }
 391 
 392 void ShenandoahConcurrentMark::concurrent_scan_code_roots(uint worker_id, ReferenceProcessor* rp) {
 393   if (claim_codecache()) {
 394     ShenandoahObjToScanQueue* q = task_queues()->queue(worker_id);
 395     if (!_heap->unload_classes()) {
 396       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 397       // TODO: We can not honor StringDeduplication here, due to lock ranking
 398       // inversion. So, we may miss some deduplication candidates.
 399       if (_heap->has_forwarded_objects()) {
 400         ShenandoahMarkResolveRefsClosure cl(q, rp);
 401         CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
 402         CodeCache::blobs_do(&blobs);
 403       } else {
 404         ShenandoahMarkRefsClosure cl(q, rp);
 405         CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
 406         CodeCache::blobs_do(&blobs);
 407       }
 408     }
 409   }
 410 }
 411 
 412 void ShenandoahConcurrentMark::mark_from_roots() {
 413   WorkGang* workers = _heap->workers();
 414   uint nworkers = workers->active_workers();
 415 
 416   if (_heap->process_references()) {
 417     ReferenceProcessor* rp = _heap->ref_processor();
 418     rp->set_active_mt_degree(nworkers);
 419 
 420     // enable ("weak") refs discovery
 421     rp->enable_discovery(true /*verify_no_refs*/);
 422     rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
 423   }
 424 
 425   shenandoah_assert_rp_isalive_not_installed();
 426   ShenandoahIsAliveSelector is_alive;
 427   ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
 428 
 429   task_queues()->reserve(nworkers);
 430 
 431   {
 432     TaskTerminator terminator(nworkers, task_queues());
 433     ShenandoahConcurrentMarkingTask task(this, &terminator);
 434     workers->run_task(&task);
 435   }
 436 
 437   assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty when not cancelled");
 438 }
 439 
 440 void ShenandoahConcurrentMark::finish_mark_from_roots(bool full_gc) {
 441   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 442 
 443   uint nworkers = _heap->workers()->active_workers();
 444 
 445   // Finally mark everything else we've got in our queues during the previous steps.
 446   // It does two different things for concurrent vs. mark-compact GC:
 447   // - For concurrent GC, it starts with empty task queues, drains the remaining
 448   //   SATB buffers, and then completes the marking closure.
 449   // - For mark-compact GC, it starts out with the task queues seeded by initial
 450   //   root scan, and completes the closure, thus marking through all live objects
 451   // The implementation is the same, so it's shared here.
 452   {
 453     ShenandoahGCPhase phase(full_gc ?
 454                             ShenandoahPhaseTimings::full_gc_mark_finish_queues :
 455                             ShenandoahPhaseTimings::finish_queues);
 456     task_queues()->reserve(nworkers);
 457 
 458     shenandoah_assert_rp_isalive_not_installed();
 459     ShenandoahIsAliveSelector is_alive;
 460     ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
 461 
 462     StrongRootsScope scope(nworkers);
 463     TaskTerminator terminator(nworkers, task_queues());
 464     ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
 465     _heap->workers()->run_task(&task);
 466   }
 467 
 468   assert(task_queues()->is_empty(), "Should be empty");
 469 
 470   // When we're done marking everything, we process weak references.
 471   if (_heap->process_references()) {
 472     weak_refs_work(full_gc);
 473   }
 474 
 475   assert(task_queues()->is_empty(), "Should be empty");
 476   TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
 477   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
 478 }
 479 
 480 // Weak Reference Closures
 481 class ShenandoahCMDrainMarkingStackClosure: public VoidClosure {
 482   uint _worker_id;
 483   TaskTerminator* _terminator;
 484   bool _reset_terminator;
 485 
 486 public:
 487   ShenandoahCMDrainMarkingStackClosure(uint worker_id, TaskTerminator* t, bool reset_terminator = false):
 488     _worker_id(worker_id),
 489     _terminator(t),
 490     _reset_terminator(reset_terminator) {
 491   }
 492 
 493   void do_void() {
 494     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 495 
 496     ShenandoahHeap* sh = ShenandoahHeap::heap();
 497     ShenandoahConcurrentMark* scm = sh->concurrent_mark();
 498     assert(sh->process_references(), "why else would we be here?");
 499     ReferenceProcessor* rp = sh->ref_processor();
 500 
 501     shenandoah_assert_rp_isalive_installed();
 502 
 503     scm->mark_loop(_worker_id, _terminator, rp,
 504                    false,   // not cancellable
 505                    false);  // do not do strdedup
 506 
 507     if (_reset_terminator) {
 508       _terminator->reset_for_reuse();
 509     }
 510   }
 511 };
 512 
 513 class ShenandoahCMKeepAliveClosure : public OopClosure {
 514 private:
 515   ShenandoahObjToScanQueue* _queue;
 516   ShenandoahHeap* _heap;
 517   ShenandoahMarkingContext* const _mark_context;
 518 
 519   template <class T>
 520   inline void do_oop_work(T* p) {
 521     ShenandoahConcurrentMark::mark_through_ref<T, NONE, NO_DEDUP>(p, _heap, _queue, _mark_context);
 522   }
 523 
 524 public:
 525   ShenandoahCMKeepAliveClosure(ShenandoahObjToScanQueue* q) :
 526     _queue(q),
 527     _heap(ShenandoahHeap::heap()),
 528     _mark_context(_heap->marking_context()) {}
 529 
 530   void do_oop(narrowOop* p) { do_oop_work(p); }
 531   void do_oop(oop* p)       { do_oop_work(p); }
 532 };
 533 
 534 class ShenandoahCMKeepAliveUpdateClosure : public OopClosure {
 535 private:
 536   ShenandoahObjToScanQueue* _queue;
 537   ShenandoahHeap* _heap;
 538   ShenandoahMarkingContext* const _mark_context;
 539 
 540   template <class T>
 541   inline void do_oop_work(T* p) {
 542     ShenandoahConcurrentMark::mark_through_ref<T, SIMPLE, NO_DEDUP>(p, _heap, _queue, _mark_context);
 543   }
 544 
 545 public:
 546   ShenandoahCMKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 547     _queue(q),
 548     _heap(ShenandoahHeap::heap()),
 549     _mark_context(_heap->marking_context()) {}
 550 
 551   void do_oop(narrowOop* p) { do_oop_work(p); }
 552   void do_oop(oop* p)       { do_oop_work(p); }
 553 };
 554 
 555 class ShenandoahWeakUpdateClosure : public OopClosure {
 556 private:
 557   ShenandoahHeap* const _heap;
 558 
 559   template <class T>
 560   inline void do_oop_work(T* p) {
 561     oop o = _heap->maybe_update_with_forwarded(p);
 562     shenandoah_assert_marked_except(p, o, o == NULL);
 563   }
 564 
 565 public:
 566   ShenandoahWeakUpdateClosure() : _heap(ShenandoahHeap::heap()) {}
 567 
 568   void do_oop(narrowOop* p) { do_oop_work(p); }
 569   void do_oop(oop* p)       { do_oop_work(p); }
 570 };
 571 
 572 class ShenandoahRefProcTaskProxy : public AbstractGangTask {
 573 private:
 574   AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
 575   TaskTerminator* _terminator;
 576 
 577 public:
 578   ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
 579                              TaskTerminator* t) :
 580     AbstractGangTask("Process reference objects in parallel"),
 581     _proc_task(proc_task),
 582     _terminator(t) {
 583   }
 584 
 585   void work(uint worker_id) {
 586     ResourceMark rm;
 587     HandleMark hm;
 588     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 589     ShenandoahHeap* heap = ShenandoahHeap::heap();
 590     ShenandoahParallelWorkerSession worker_session(worker_id);
 591     ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id, _terminator);
 592     if (heap->has_forwarded_objects()) {
 593       ShenandoahForwardedIsAliveClosure is_alive;
 594       ShenandoahCMKeepAliveUpdateClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
 595       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
 596     } else {
 597       ShenandoahIsAliveClosure is_alive;
 598       ShenandoahCMKeepAliveClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
 599       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
 600     }
 601   }
 602 };
 603 
 604 class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
 605 private:
 606   WorkGang* _workers;
 607 
 608 public:
 609   ShenandoahRefProcTaskExecutor(WorkGang* workers) :
 610     _workers(workers) {
 611   }
 612 
 613   // Executes a task using worker threads.
 614   void execute(ProcessTask& task, uint ergo_workers) {
 615     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 616 
 617     ShenandoahHeap* heap = ShenandoahHeap::heap();
 618     ShenandoahConcurrentMark* cm = heap->concurrent_mark();
 619     ShenandoahPushWorkerQueuesScope scope(_workers, cm->task_queues(),
 620                                           ergo_workers,
 621                                           /* do_check = */ false);
 622     uint nworkers = _workers->active_workers();
 623     cm->task_queues()->reserve(nworkers);
 624     TaskTerminator terminator(nworkers, cm->task_queues());
 625     ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator);
 626     _workers->run_task(&proc_task_proxy);
 627   }
 628 };
 629 
 630 void ShenandoahConcurrentMark::weak_refs_work(bool full_gc) {
 631   assert(_heap->process_references(), "sanity");
 632 
 633   ShenandoahPhaseTimings::Phase phase_root =
 634           full_gc ?
 635           ShenandoahPhaseTimings::full_gc_weakrefs :
 636           ShenandoahPhaseTimings::weakrefs;
 637 
 638   ShenandoahGCPhase phase(phase_root);
 639 
 640   ReferenceProcessor* rp = _heap->ref_processor();
 641 
 642   // NOTE: We cannot shortcut on has_discovered_references() here, because
 643   // we will miss marking JNI Weak refs then, see implementation in
 644   // ReferenceProcessor::process_discovered_references.
 645   weak_refs_work_doit(full_gc);
 646 
 647   rp->verify_no_references_recorded();
 648   assert(!rp->discovery_enabled(), "Post condition");
 649 
 650 }
 651 
 652 void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) {
 653   ReferenceProcessor* rp = _heap->ref_processor();
 654 
 655   ShenandoahPhaseTimings::Phase phase_process =
 656           full_gc ?
 657           ShenandoahPhaseTimings::full_gc_weakrefs_process :
 658           ShenandoahPhaseTimings::weakrefs_process;
 659 
 660   shenandoah_assert_rp_isalive_not_installed();
 661   ShenandoahIsAliveSelector is_alive;
 662   ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
 663 
 664   WorkGang* workers = _heap->workers();
 665   uint nworkers = workers->active_workers();
 666 
 667   rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
 668   rp->set_active_mt_degree(nworkers);
 669 
 670   assert(task_queues()->is_empty(), "Should be empty");
 671 
 672   // complete_gc and keep_alive closures instantiated here are only needed for
 673   // single-threaded path in RP. They share the queue 0 for tracking work, which
 674   // simplifies implementation. Since RP may decide to call complete_gc several
 675   // times, we need to be able to reuse the terminator.
 676   uint serial_worker_id = 0;
 677   TaskTerminator terminator(1, task_queues());
 678   ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
 679 
 680   ShenandoahRefProcTaskExecutor executor(workers);
 681 
 682   ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
 683 
 684   {
 685     // Note: Don't emit JFR event for this phase, to avoid overflow nesting phase level.
 686     // Reference Processor emits 2 levels JFR event, that can get us over the JFR
 687     // event nesting level limits, in case of degenerated GC gets upgraded to
 688     // full GC.
 689     ShenandoahTimingsTracker phase_timing(phase_process);
 690 
 691     if (_heap->has_forwarded_objects()) {
 692       ShenandoahCMKeepAliveUpdateClosure keep_alive(get_queue(serial_worker_id));
 693       const ReferenceProcessorStats& stats =
 694         rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
 695                                           &complete_gc, &executor,
 696                                           &pt);
 697        _heap->tracer()->report_gc_reference_stats(stats);
 698     } else {
 699       ShenandoahCMKeepAliveClosure keep_alive(get_queue(serial_worker_id));
 700       const ReferenceProcessorStats& stats =
 701         rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
 702                                           &complete_gc, &executor,
 703                                           &pt);
 704       _heap->tracer()->report_gc_reference_stats(stats);
 705     }
 706 
 707     pt.print_all_references();
 708 
 709     assert(task_queues()->is_empty(), "Should be empty");
 710   }
 711 }
 712 
 713 class ShenandoahCancelledGCYieldClosure : public YieldClosure {
 714 private:
 715   ShenandoahHeap* const _heap;
 716 public:
 717   ShenandoahCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
 718   virtual bool should_return() { return _heap->cancelled_gc(); }
 719 };
 720 
 721 class ShenandoahPrecleanCompleteGCClosure : public VoidClosure {
 722 public:
 723   void do_void() {
 724     ShenandoahHeap* sh = ShenandoahHeap::heap();
 725     ShenandoahConcurrentMark* scm = sh->concurrent_mark();
 726     assert(sh->process_references(), "why else would we be here?");
 727     TaskTerminator terminator(1, scm->task_queues());
 728 
 729     ReferenceProcessor* rp = sh->ref_processor();
 730     shenandoah_assert_rp_isalive_installed();
 731 
 732     scm->mark_loop(0, &terminator, rp,
 733                    false, // not cancellable
 734                    false); // do not do strdedup
 735   }
 736 };
 737 
 738 class ShenandoahPrecleanTask : public AbstractGangTask {
 739 private:
 740   ReferenceProcessor* _rp;
 741 
 742 public:
 743   ShenandoahPrecleanTask(ReferenceProcessor* rp) :
 744           AbstractGangTask("Precleaning task"),
 745           _rp(rp) {}
 746 
 747   void work(uint worker_id) {
 748     assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
 749     ShenandoahParallelWorkerSession worker_session(worker_id);
 750 
 751     ShenandoahHeap* sh = ShenandoahHeap::heap();
 752     assert(!sh->has_forwarded_objects(), "No forwarded objects expected here");
 753 
 754     ShenandoahObjToScanQueue* q = sh->concurrent_mark()->get_queue(worker_id);
 755 
 756     ShenandoahCancelledGCYieldClosure yield;
 757     ShenandoahPrecleanCompleteGCClosure complete_gc;
 758 
 759     ShenandoahIsAliveClosure is_alive;
 760     ShenandoahCMKeepAliveClosure keep_alive(q);
 761     ResourceMark rm;
 762     _rp->preclean_discovered_references(&is_alive, &keep_alive,
 763                                         &complete_gc, &yield,
 764                                         NULL);
 765   }
 766 };
 767 
 768 void ShenandoahConcurrentMark::preclean_weak_refs() {
 769   // Pre-cleaning weak references before diving into STW makes sense at the
 770   // end of concurrent mark. This will filter out the references which referents
 771   // are alive. Note that ReferenceProcessor already filters out these on reference
 772   // discovery, and the bulk of work is done here. This phase processes leftovers
 773   // that missed the initial filtering, i.e. when referent was marked alive after
 774   // reference was discovered by RP.
 775 
 776   assert(_heap->process_references(), "sanity");
 777 
 778   // Shortcut if no references were discovered to avoid winding up threads.
 779   ReferenceProcessor* rp = _heap->ref_processor();
 780   if (!rp->has_discovered_references()) {
 781     return;
 782   }
 783 
 784   assert(task_queues()->is_empty(), "Should be empty");
 785 
 786   ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
 787 
 788   shenandoah_assert_rp_isalive_not_installed();
 789   ShenandoahIsAliveSelector is_alive;
 790   ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
 791 
 792   // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
 793   // queues and other goodies. When upstream ReferenceProcessor starts supporting
 794   // parallel precleans, we can extend this to more threads.
 795   WorkGang* workers = _heap->workers();
 796   uint nworkers = workers->active_workers();
 797   assert(nworkers == 1, "This code uses only a single worker");
 798   task_queues()->reserve(nworkers);
 799 
 800   ShenandoahPrecleanTask task(rp);
 801   workers->run_task(&task);
 802 
 803   assert(task_queues()->is_empty(), "Should be empty");
 804 }
 805 
 806 void ShenandoahConcurrentMark::cancel() {
 807   // Clean up marking stacks.
 808   ShenandoahObjToScanQueueSet* queues = task_queues();
 809   queues->clear();
 810 
 811   // Cancel SATB buffers.
 812   ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
 813 }
 814 
 815 ShenandoahObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) {
 816   assert(task_queues()->get_reserved() > worker_id, "No reserved queue for worker id: %d", worker_id);
 817   return _task_queues->queue(worker_id);
 818 }
 819 
 820 template <bool CANCELLABLE>
 821 void ShenandoahConcurrentMark::mark_loop_prework(uint w, TaskTerminator *t, ReferenceProcessor *rp,
 822                                                  bool strdedup) {
 823   ShenandoahObjToScanQueue* q = get_queue(w);
 824 
 825   ShenandoahLiveData* ld = _heap->get_liveness_cache(w);
 826 
 827   // TODO: We can clean up this if we figure out how to do templated oop closures that
 828   // play nice with specialized_oop_iterators.
 829   if (_heap->unload_classes()) {
 830     if (_heap->has_forwarded_objects()) {
 831       if (strdedup) {
 832         ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp);
 833         mark_loop_work<ShenandoahMarkUpdateRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 834       } else {
 835         ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp);
 836         mark_loop_work<ShenandoahMarkUpdateRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
 837       }
 838     } else {
 839       if (strdedup) {
 840         ShenandoahMarkRefsMetadataDedupClosure cl(q, rp);
 841         mark_loop_work<ShenandoahMarkRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 842       } else {
 843         ShenandoahMarkRefsMetadataClosure cl(q, rp);
 844         mark_loop_work<ShenandoahMarkRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
 845       }
 846     }
 847   } else {
 848     if (_heap->has_forwarded_objects()) {
 849       if (strdedup) {
 850         ShenandoahMarkUpdateRefsDedupClosure cl(q, rp);
 851         mark_loop_work<ShenandoahMarkUpdateRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 852       } else {
 853         ShenandoahMarkUpdateRefsClosure cl(q, rp);
 854         mark_loop_work<ShenandoahMarkUpdateRefsClosure, CANCELLABLE>(&cl, ld, w, t);
 855       }
 856     } else {
 857       if (strdedup) {
 858         ShenandoahMarkRefsDedupClosure cl(q, rp);
 859         mark_loop_work<ShenandoahMarkRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 860       } else {
 861         ShenandoahMarkRefsClosure cl(q, rp);
 862         mark_loop_work<ShenandoahMarkRefsClosure, CANCELLABLE>(&cl, ld, w, t);
 863       }
 864     }
 865   }
 866 
 867   _heap->flush_liveness_cache(w);
 868 }
 869 
 870 template <class T, bool CANCELLABLE>
 871 void ShenandoahConcurrentMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *terminator) {
 872   uintx stride = ShenandoahMarkLoopStride;
 873 
 874   ShenandoahHeap* heap = ShenandoahHeap::heap();
 875   ShenandoahObjToScanQueueSet* queues = task_queues();
 876   ShenandoahObjToScanQueue* q;
 877   ShenandoahMarkTask t;
 878 
 879   /*
 880    * Process outstanding queues, if any.
 881    *
 882    * There can be more queues than workers. To deal with the imbalance, we claim
 883    * extra queues first. Since marking can push new tasks into the queue associated
 884    * with this worker id, we come back to process this queue in the normal loop.
 885    */
 886   assert(queues->get_reserved() == heap->workers()->active_workers(),
 887          "Need to reserve proper number of queues: reserved: %u, active: %u", queues->get_reserved(), heap->workers()->active_workers());
 888 
 889   q = queues->claim_next();
 890   while (q != NULL) {
 891     if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
 892       return;
 893     }
 894 
 895     for (uint i = 0; i < stride; i++) {
 896       if (q->pop(t)) {
 897         do_task<T>(q, cl, live_data, &t);
 898       } else {
 899         assert(q->is_empty(), "Must be empty");
 900         q = queues->claim_next();
 901         break;
 902       }
 903     }
 904   }
 905   q = get_queue(worker_id);
 906 
 907   ShenandoahSATBBufferClosure drain_satb(q);
 908   SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 909 
 910   /*
 911    * Normal marking loop:
 912    */
 913   while (true) {
 914     if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
 915       return;
 916     }
 917 
 918     while (satb_mq_set.completed_buffers_num() > 0) {
 919       satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
 920     }
 921 
 922     uint work = 0;
 923     for (uint i = 0; i < stride; i++) {
 924       if (q->pop(t) ||
 925           queues->steal(worker_id, t)) {
 926         do_task<T>(q, cl, live_data, &t);
 927         work++;
 928       } else {
 929         break;
 930       }
 931     }
 932 
 933     if (work == 0) {
 934       // No work encountered in current stride, try to terminate.
 935       // Need to leave the STS here otherwise it might block safepoints.
 936       ShenandoahSuspendibleThreadSetLeaver stsl(CANCELLABLE && ShenandoahSuspendibleWorkers);
 937       ShenandoahTerminatorTerminator tt(heap);
 938       if (terminator->offer_termination(&tt)) return;
 939     }
 940   }
 941 }
 942 
 943 bool ShenandoahConcurrentMark::claim_codecache() {
 944   return _claimed_codecache.try_set();
 945 }
 946 
 947 void ShenandoahConcurrentMark::clear_claim_codecache() {
 948   _claimed_codecache.unset();
 949 }
--- EOF ---