rev 55429 : 8226311: Shenandoah: Concurrent evacuation of OopStorage backed weak roots

   1 /*
   2  * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "classfile/symbolTable.hpp"
  27 #include "classfile/systemDictionary.hpp"
  28 #include "code/codeCache.hpp"
  29 
  30 #include "gc/shared/weakProcessor.inline.hpp"
  31 #include "gc/shared/gcTimer.hpp"
  32 #include "gc/shared/referenceProcessor.hpp"
  33 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
  34 #include "gc/shared/strongRootsScope.hpp"
  35 
  36 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
  37 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  38 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  39 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  40 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  41 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  42 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  43 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
  44 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
  45 #include "gc/shenandoah/shenandoahUtils.hpp"
  46 
  47 #include "memory/iterator.inline.hpp"
  48 #include "memory/metaspace.hpp"
  49 #include "memory/resourceArea.hpp"
  50 #include "oops/oop.inline.hpp"
  51 #include "runtime/handles.inline.hpp"
  52 
  53 template<UpdateRefsMode UPDATE_REFS>
  54 class ShenandoahInitMarkRootsClosure : public OopClosure {
  55 private:
  56   ShenandoahObjToScanQueue* _queue;
  57   ShenandoahHeap* _heap;
  58   ShenandoahMarkingContext* const _mark_context;
  59 
  60   template <class T>
  61   inline void do_oop_work(T* p) {
  62     ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, NO_DEDUP>(p, _heap, _queue, _mark_context);
  63   }
  64 
  65 public:
  66   ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) :
  67     _queue(q),
  68     _heap(ShenandoahHeap::heap()),
  69     _mark_context(_heap->marking_context()) {};
  70 
  71   void do_oop(narrowOop* p) { do_oop_work(p); }
  72   void do_oop(oop* p)       { do_oop_work(p); }
  73 };
  74 
  75 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
  76   MetadataVisitingOopIterateClosure(rp),
  77   _queue(q),
  78   _heap(ShenandoahHeap::heap()),
  79   _mark_context(_heap->marking_context())
  80 { }
  81 
  82 template<UpdateRefsMode UPDATE_REFS>
  83 class ShenandoahInitMarkRootsTask : public AbstractGangTask {
  84 private:
  85   ShenandoahAllRootScanner* _rp;
  86   bool _process_refs;
  87 public:
  88   ShenandoahInitMarkRootsTask(ShenandoahAllRootScanner* rp, bool process_refs) :
  89     AbstractGangTask("Shenandoah init mark roots task"),
  90     _rp(rp),
  91     _process_refs(process_refs) {
  92   }
  93 
  94   void work(uint worker_id) {
  95     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
  96     ShenandoahParallelWorkerSession worker_session(worker_id);
  97 
  98     ShenandoahHeap* heap = ShenandoahHeap::heap();
  99     ShenandoahObjToScanQueueSet* queues = heap->concurrent_mark()->task_queues();
 100     assert(queues->get_reserved() > worker_id, "Queue has not been reserved for worker id: %d", worker_id);
 101 
 102     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 103 
 104     ShenandoahInitMarkRootsClosure<UPDATE_REFS> mark_cl(q);
 105     do_work(heap, &mark_cl, worker_id);
 106   }
 107 
 108 private:
 109   void do_work(ShenandoahHeap* heap, OopClosure* oops, uint worker_id) {
 110     // The rationale for selecting the roots to scan is as follows:
 111     //   a. With unload_classes = true, we only want to scan the actual strong roots from the
 112     //      code cache. This will allow us to identify the dead classes, unload them, *and*
 113     //      invalidate the relevant code cache blobs. This could be only done together with
 114     //      class unloading.
 115     //   b. With unload_classes = false, we have to nominally retain all the references from code
 116     //      cache, because there could be the case of embedded class/oop in the generated code,
 117     //      which we will never visit during mark. Without code cache invalidation, as in (a),
 118     //      we risk executing that code cache blob, and crashing.
 119     if (heap->unload_classes()) {
 120       _rp->strong_roots_do(worker_id, oops);
 121     } else {
 122       _rp->roots_do(worker_id, oops);
 123     }
 124   }
 125 };
 126 
 127 class ShenandoahUpdateRootsTask : public AbstractGangTask {
 128 private:
 129   ShenandoahRootUpdater*  _root_updater;
 130 public:
 131   ShenandoahUpdateRootsTask(ShenandoahRootUpdater* root_updater) :
 132     AbstractGangTask("Shenandoah update roots task"),
 133     _root_updater(root_updater) {
 134   }
 135 
 136   void work(uint worker_id) {
 137     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 138     ShenandoahParallelWorkerSession worker_session(worker_id);
 139 
 140     ShenandoahHeap* heap = ShenandoahHeap::heap();
 141     ShenandoahUpdateRefsClosure cl;
 142     AlwaysTrueClosure always_true;
 143     _root_updater->roots_do<AlwaysTrueClosure, ShenandoahUpdateRefsClosure>(worker_id, &always_true, &cl);
 144   }
 145 };
 146 
 147 class ShenandoahConcurrentMarkingTask : public AbstractGangTask {
 148 private:
 149   ShenandoahConcurrentMark* _cm;
 150   ShenandoahTaskTerminator* _terminator;
 151 
 152 public:
 153   ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, ShenandoahTaskTerminator* terminator) :
 154     AbstractGangTask("Root Region Scan"), _cm(cm), _terminator(terminator) {
 155   }
 156 
 157   void work(uint worker_id) {
 158     ShenandoahHeap* heap = ShenandoahHeap::heap();
 159     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 160     ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 161     ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
 162     ReferenceProcessor* rp;
 163     if (heap->process_references()) {
 164       rp = heap->ref_processor();
 165       shenandoah_assert_rp_isalive_installed();
 166     } else {
 167       rp = NULL;
 168     }
 169 
 170     _cm->concurrent_scan_code_roots(worker_id, rp);
 171     _cm->mark_loop(worker_id, _terminator, rp,
 172                    true, // cancellable
 173                    ShenandoahStringDedup::is_enabled()); // perform string dedup
 174   }
 175 };
 176 
 177 class ShenandoahSATBThreadsClosure : public ThreadClosure {
 178 private:
 179   ShenandoahSATBBufferClosure* _satb_cl;
 180   uintx _claim_token;
 181 
 182 public:
 183   ShenandoahSATBThreadsClosure(ShenandoahSATBBufferClosure* satb_cl) :
 184     _satb_cl(satb_cl),
 185     _claim_token(Threads::thread_claim_token()) {}
 186 
 187   void do_thread(Thread* thread) {
 188     if (thread->claim_threads_do(true, _claim_token)) {
 189       ShenandoahThreadLocalData::satb_mark_queue(thread).apply_closure_and_empty(_satb_cl);
 190     }
 191   }
 192 };
 193 
 194 class ShenandoahFinalMarkingTask : public AbstractGangTask {
 195 private:
 196   ShenandoahConcurrentMark* _cm;
 197   ShenandoahTaskTerminator* _terminator;
 198   bool _dedup_string;
 199 
 200 public:
 201   ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, ShenandoahTaskTerminator* terminator, bool dedup_string) :
 202     AbstractGangTask("Shenandoah Final Marking"), _cm(cm), _terminator(terminator), _dedup_string(dedup_string) {
 203   }
 204 
 205   void work(uint worker_id) {
 206     ShenandoahHeap* heap = ShenandoahHeap::heap();
 207 
 208     ShenandoahParallelWorkerSession worker_session(worker_id);
 209     // First drain remaining SATB buffers.
 210     // Notice that this is not strictly necessary for mark-compact. But since
 211     // it requires a StrongRootsScope around the task, we need to claim the
 212     // threads, and performance-wise it doesn't really matter. Adds about 1ms to
 213     // full-gc.
 214     {
 215       ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
 216       ShenandoahSATBBufferClosure cl(q);
 217       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 218       while (satb_mq_set.apply_closure_to_completed_buffer(&cl));
 219       ShenandoahSATBThreadsClosure tc(&cl);
 220       Threads::threads_do(&tc);
 221     }
 222 
 223     ReferenceProcessor* rp;
 224     if (heap->process_references()) {
 225       rp = heap->ref_processor();
 226       shenandoah_assert_rp_isalive_installed();
 227     } else {
 228       rp = NULL;
 229     }
 230 
 231     if (heap->is_degenerated_gc_in_progress()) {
 232       // Degenerated cycle may bypass concurrent cycle, so code roots might not be scanned,
 233       // let's check here.
 234       _cm->concurrent_scan_code_roots(worker_id, rp);
 235     }
 236 
 237     _cm->mark_loop(worker_id, _terminator, rp,
 238                    false, // not cancellable
 239                    _dedup_string);
 240 
 241     assert(_cm->task_queues()->is_empty(), "Should be empty");
 242   }
 243 };
 244 
 245 void ShenandoahConcurrentMark::mark_roots(ShenandoahPhaseTimings::Phase root_phase) {
 246   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
 247   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 248 
 249   ShenandoahHeap* heap = ShenandoahHeap::heap();
 250 
 251   ShenandoahGCPhase phase(root_phase);
 252 
 253   WorkGang* workers = heap->workers();
 254   uint nworkers = workers->active_workers();
 255 
 256   assert(nworkers <= task_queues()->size(), "Just check");
 257 
 258   ShenandoahAllRootScanner root_proc(nworkers, root_phase);
 259   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
 260   task_queues()->reserve(nworkers);
 261 
 262   if (heap->has_forwarded_objects()) {
 263     ShenandoahInitMarkRootsTask<RESOLVE> mark_roots(&root_proc, _heap->process_references());
 264     workers->run_task(&mark_roots);
 265   } else {
 266     // No need to update references, which means the heap is stable.
 267     // Can save time not walking through forwarding pointers.
 268     ShenandoahInitMarkRootsTask<NONE> mark_roots(&root_proc, _heap->process_references());
 269     workers->run_task(&mark_roots);
 270   }
 271 
 272   if (ShenandoahConcurrentScanCodeRoots) {
 273     clear_claim_codecache();
 274   }
 275 }
 276 
 277 void ShenandoahConcurrentMark::update_roots(ShenandoahPhaseTimings::Phase root_phase) {
 278   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 279 
 280   bool update_code_cache = true; // initialize to safer value
 281   switch (root_phase) {
 282     case ShenandoahPhaseTimings::update_roots:
 283     case ShenandoahPhaseTimings::final_update_refs_roots:
 284       update_code_cache = false;
 285       break;
 286     case ShenandoahPhaseTimings::full_gc_roots:
 287     case ShenandoahPhaseTimings::degen_gc_update_roots:
 288       update_code_cache = true;
 289       break;
 290     default:
 291       ShouldNotReachHere();
 292   }
 293 
 294   ShenandoahGCPhase phase(root_phase);
 295 
 296 #if COMPILER2_OR_JVMCI
 297   DerivedPointerTable::clear();
 298 #endif
 299 
 300   uint nworkers = _heap->workers()->active_workers();
 301 
 302   ShenandoahRootUpdater root_updater(nworkers, root_phase, update_code_cache);
 303   ShenandoahUpdateRootsTask update_roots(&root_updater);
 304   _heap->workers()->run_task(&update_roots);
 305 
 306 #if COMPILER2_OR_JVMCI
 307   DerivedPointerTable::update_pointers();
 308 #endif
 309 }
 310 
 311 class ShenandoahUpdateThreadRootsTask : public AbstractGangTask {
 312 private:
 313   ShenandoahThreadRoots           _thread_roots;
 314   ShenandoahPhaseTimings::Phase   _phase;
 315 public:
 316   ShenandoahUpdateThreadRootsTask(bool is_par, ShenandoahPhaseTimings::Phase phase) :
 317     AbstractGangTask("Shenandoah Update Thread Roots"),
 318     _thread_roots(is_par),
 319     _phase(phase) {
 320     ShenandoahHeap::heap()->phase_timings()->record_workers_start(_phase);
 321   }
 322 
 323   ~ShenandoahUpdateThreadRootsTask() {
 324     ShenandoahHeap::heap()->phase_timings()->record_workers_end(_phase);
 325   }
 326   void work(uint worker_id) {
 327     ShenandoahUpdateRefsClosure cl;
 328     _thread_roots.oops_do(&cl, NULL, worker_id);
 329   }
 330 };
 331 
 332 void ShenandoahConcurrentMark::update_thread_roots(ShenandoahPhaseTimings::Phase root_phase) {
 333   WorkGang* workers = _heap->workers();
 334   bool is_par = workers->active_workers() > 1;
 335 #if COMPILER2_OR_JVMCI
 336   DerivedPointerTable::clear();
 337 #endif
 338   ShenandoahUpdateThreadRootsTask task(is_par, root_phase);
 339   workers->run_task(&task);
 340 #if COMPILER2_OR_JVMCI
 341   DerivedPointerTable::update_pointers();
 342 #endif
 343 }
 344 
 345 void ShenandoahConcurrentMark::initialize(uint workers) {
 346   _heap = ShenandoahHeap::heap();
 347 
 348   uint num_queues = MAX2(workers, 1U);
 349 
 350   _task_queues = new ShenandoahObjToScanQueueSet((int) num_queues);
 351 
 352   for (uint i = 0; i < num_queues; ++i) {
 353     ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
 354     task_queue->initialize();
 355     _task_queues->register_queue(i, task_queue);
 356   }
 357 }
 358 
 359 void ShenandoahConcurrentMark::concurrent_scan_code_roots(uint worker_id, ReferenceProcessor* rp) {
 360   if (ShenandoahConcurrentScanCodeRoots && claim_codecache()) {
 361     ShenandoahObjToScanQueue* q = task_queues()->queue(worker_id);
 362     if (!_heap->unload_classes()) {
 363       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 364       // TODO: We can not honor StringDeduplication here, due to lock ranking
 365       // inversion. So, we may miss some deduplication candidates.
 366       if (_heap->has_forwarded_objects()) {
 367         ShenandoahMarkResolveRefsClosure cl(q, rp);
 368         CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
 369         CodeCache::blobs_do(&blobs);
 370       } else {
 371         ShenandoahMarkRefsClosure cl(q, rp);
 372         CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations);
 373         CodeCache::blobs_do(&blobs);
 374       }
 375     }
 376   }
 377 }
 378 
 379 void ShenandoahConcurrentMark::mark_from_roots() {
 380   WorkGang* workers = _heap->workers();
 381   uint nworkers = workers->active_workers();
 382 
 383   ShenandoahGCPhase conc_mark_phase(ShenandoahPhaseTimings::conc_mark);
 384 
 385   if (_heap->process_references()) {
 386     ReferenceProcessor* rp = _heap->ref_processor();
 387     rp->set_active_mt_degree(nworkers);
 388 
 389     // enable ("weak") refs discovery
 390     rp->enable_discovery(true /*verify_no_refs*/);
 391     rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
 392   }
 393 
 394   shenandoah_assert_rp_isalive_not_installed();
 395   ShenandoahIsAliveSelector is_alive;
 396   ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
 397 
 398   task_queues()->reserve(nworkers);
 399 
 400   {
 401     ShenandoahTerminationTracker term(ShenandoahPhaseTimings::conc_termination);
 402     ShenandoahTaskTerminator terminator(nworkers, task_queues());
 403     ShenandoahConcurrentMarkingTask task(this, &terminator);
 404     workers->run_task(&task);
 405   }
 406 
 407   assert(task_queues()->is_empty() || _heap->cancelled_gc(), "Should be empty when not cancelled");
 408 }
 409 
 410 void ShenandoahConcurrentMark::finish_mark_from_roots(bool full_gc) {
 411   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 412 
 413   uint nworkers = _heap->workers()->active_workers();
 414 
 415   // Finally mark everything else we've got in our queues during the previous steps.
 416   // It does two different things for concurrent vs. mark-compact GC:
 417   // - For concurrent GC, it starts with empty task queues, drains the remaining
 418   //   SATB buffers, and then completes the marking closure.
 419   // - For mark-compact GC, it starts out with the task queues seeded by initial
 420   //   root scan, and completes the closure, thus marking through all live objects
 421   // The implementation is the same, so it's shared here.
 422   {
 423     ShenandoahGCPhase phase(full_gc ?
 424                             ShenandoahPhaseTimings::full_gc_mark_finish_queues :
 425                             ShenandoahPhaseTimings::finish_queues);
 426     task_queues()->reserve(nworkers);
 427 
 428     shenandoah_assert_rp_isalive_not_installed();
 429     ShenandoahIsAliveSelector is_alive;
 430     ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
 431 
 432     ShenandoahTerminationTracker termination_tracker(full_gc ?
 433                                                      ShenandoahPhaseTimings::full_gc_mark_termination :
 434                                                      ShenandoahPhaseTimings::termination);
 435 
 436     StrongRootsScope scope(nworkers);
 437     ShenandoahTaskTerminator terminator(nworkers, task_queues());
 438     ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled());
 439     _heap->workers()->run_task(&task);
 440   }
 441 
 442   assert(task_queues()->is_empty(), "Should be empty");
 443 
 444   // When we're done marking everything, we process weak references.
 445   if (_heap->process_references()) {
 446     weak_refs_work(full_gc);
 447   }
 448 
 449   weak_roots_work();
 450 
 451   // And finally finish class unloading
 452   if (_heap->unload_classes()) {
 453     _heap->unload_classes_and_cleanup_tables(full_gc);
 454   } else if (ShenandoahStringDedup::is_enabled()) {
 455     ShenandoahIsAliveSelector alive;
 456     BoolObjectClosure* is_alive = alive.is_alive_closure();
 457     ShenandoahStringDedup::unlink_or_oops_do(is_alive, NULL, false);
 458   }
 459   assert(task_queues()->is_empty(), "Should be empty");
 460   TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
 461   TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
 462 
 463   // Resize Metaspace
 464   MetaspaceGC::compute_new_size();
 465 }
 466 
 467 // Weak Reference Closures
 468 class ShenandoahCMDrainMarkingStackClosure: public VoidClosure {
 469   uint _worker_id;
 470   ShenandoahTaskTerminator* _terminator;
 471   bool _reset_terminator;
 472 
 473 public:
 474   ShenandoahCMDrainMarkingStackClosure(uint worker_id, ShenandoahTaskTerminator* t, bool reset_terminator = false):
 475     _worker_id(worker_id),
 476     _terminator(t),
 477     _reset_terminator(reset_terminator) {
 478   }
 479 
 480   void do_void() {
 481     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 482 
 483     ShenandoahHeap* sh = ShenandoahHeap::heap();
 484     ShenandoahConcurrentMark* scm = sh->concurrent_mark();
 485     assert(sh->process_references(), "why else would we be here?");
 486     ReferenceProcessor* rp = sh->ref_processor();
 487 
 488     shenandoah_assert_rp_isalive_installed();
 489 
 490     scm->mark_loop(_worker_id, _terminator, rp,
 491                    false,   // not cancellable
 492                    false);  // do not do strdedup
 493 
 494     if (_reset_terminator) {
 495       _terminator->reset_for_reuse();
 496     }
 497   }
 498 };
 499 
 500 class ShenandoahCMKeepAliveClosure : public OopClosure {
 501 private:
 502   ShenandoahObjToScanQueue* _queue;
 503   ShenandoahHeap* _heap;
 504   ShenandoahMarkingContext* const _mark_context;
 505 
 506   template <class T>
 507   inline void do_oop_work(T* p) {
 508     ShenandoahConcurrentMark::mark_through_ref<T, NONE, NO_DEDUP>(p, _heap, _queue, _mark_context);
 509   }
 510 
 511 public:
 512   ShenandoahCMKeepAliveClosure(ShenandoahObjToScanQueue* q) :
 513     _queue(q),
 514     _heap(ShenandoahHeap::heap()),
 515     _mark_context(_heap->marking_context()) {}
 516 
 517   void do_oop(narrowOop* p) { do_oop_work(p); }
 518   void do_oop(oop* p)       { do_oop_work(p); }
 519 };
 520 
 521 class ShenandoahCMKeepAliveUpdateClosure : public OopClosure {
 522 private:
 523   ShenandoahObjToScanQueue* _queue;
 524   ShenandoahHeap* _heap;
 525   ShenandoahMarkingContext* const _mark_context;
 526 
 527   template <class T>
 528   inline void do_oop_work(T* p) {
 529     ShenandoahConcurrentMark::mark_through_ref<T, SIMPLE, NO_DEDUP>(p, _heap, _queue, _mark_context);
 530   }
 531 
 532 public:
 533   ShenandoahCMKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 534     _queue(q),
 535     _heap(ShenandoahHeap::heap()),
 536     _mark_context(_heap->marking_context()) {}
 537 
 538   void do_oop(narrowOop* p) { do_oop_work(p); }
 539   void do_oop(oop* p)       { do_oop_work(p); }
 540 };
 541 
 542 class ShenandoahWeakUpdateClosure : public OopClosure {
 543 private:
 544   ShenandoahHeap* const _heap;
 545 
 546   template <class T>
 547   inline void do_oop_work(T* p) {
 548     oop o = _heap->maybe_update_with_forwarded(p);
 549     shenandoah_assert_marked_except(p, o, o == NULL);
 550   }
 551 
 552 public:
 553   ShenandoahWeakUpdateClosure() : _heap(ShenandoahHeap::heap()) {}
 554 
 555   void do_oop(narrowOop* p) { do_oop_work(p); }
 556   void do_oop(oop* p)       { do_oop_work(p); }
 557 };
 558 
 559 class ShenandoahWeakAssertNotForwardedClosure : public OopClosure {
 560 private:
 561   template <class T>
 562   inline void do_oop_work(T* p) {
 563 #ifdef ASSERT
 564     T o = RawAccess<>::oop_load(p);
 565     if (!CompressedOops::is_null(o)) {
 566       oop obj = CompressedOops::decode_not_null(o);
 567       shenandoah_assert_not_forwarded(p, obj);
 568     }
 569 #endif
 570   }
 571 
 572 public:
 573   ShenandoahWeakAssertNotForwardedClosure() {}
 574 
 575   void do_oop(narrowOop* p) { do_oop_work(p); }
 576   void do_oop(oop* p)       { do_oop_work(p); }
 577 };
 578 
 579 class ShenandoahRefProcTaskProxy : public AbstractGangTask {
 580 private:
 581   AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
 582   ShenandoahTaskTerminator* _terminator;
 583 
 584 public:
 585   ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
 586                              ShenandoahTaskTerminator* t) :
 587     AbstractGangTask("Process reference objects in parallel"),
 588     _proc_task(proc_task),
 589     _terminator(t) {
 590   }
 591 
 592   void work(uint worker_id) {
 593     ResourceMark rm;
 594     HandleMark hm;
 595     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 596     ShenandoahHeap* heap = ShenandoahHeap::heap();
 597     ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id, _terminator);
 598     if (heap->has_forwarded_objects()) {
 599       ShenandoahForwardedIsAliveClosure is_alive;
 600       ShenandoahCMKeepAliveUpdateClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
 601       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
 602     } else {
 603       ShenandoahIsAliveClosure is_alive;
 604       ShenandoahCMKeepAliveClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
 605       _proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
 606     }
 607   }
 608 };
 609 
 610 class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
 611 private:
 612   WorkGang* _workers;
 613 
 614 public:
 615   ShenandoahRefProcTaskExecutor(WorkGang* workers) :
 616     _workers(workers) {
 617   }
 618 
 619   // Executes a task using worker threads.
 620   void execute(ProcessTask& task, uint ergo_workers) {
 621     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 622 
 623     ShenandoahHeap* heap = ShenandoahHeap::heap();
 624     ShenandoahConcurrentMark* cm = heap->concurrent_mark();
 625     ShenandoahPushWorkerQueuesScope scope(_workers, cm->task_queues(),
 626                                           ergo_workers,
 627                                           /* do_check = */ false);
 628     uint nworkers = _workers->active_workers();
 629     cm->task_queues()->reserve(nworkers);
 630     ShenandoahTaskTerminator terminator(nworkers, cm->task_queues());
 631     ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator);
 632     _workers->run_task(&proc_task_proxy);
 633   }
 634 };
 635 
 636 void ShenandoahConcurrentMark::weak_refs_work(bool full_gc) {
 637   assert(_heap->process_references(), "sanity");
 638 
 639   ShenandoahPhaseTimings::Phase phase_root =
 640           full_gc ?
 641           ShenandoahPhaseTimings::full_gc_weakrefs :
 642           ShenandoahPhaseTimings::weakrefs;
 643 
 644   ShenandoahGCPhase phase(phase_root);
 645 
 646   ReferenceProcessor* rp = _heap->ref_processor();
 647 
 648   // NOTE: We cannot shortcut on has_discovered_references() here, because
 649   // we will miss marking JNI Weak refs then, see implementation in
 650   // ReferenceProcessor::process_discovered_references.
 651   weak_refs_work_doit(full_gc);
 652 
 653   rp->verify_no_references_recorded();
 654   assert(!rp->discovery_enabled(), "Post condition");
 655 
 656 }
 657 
 658 // Process leftover weak oops: update them, if needed or assert they do not
 659 // need updating otherwise.
 660 // Weak processor API requires us to visit the oops, even if we are not doing
 661 // anything to them.
 662 void ShenandoahConcurrentMark::weak_roots_work() {
 663   WorkGang* workers = _heap->workers();
 664   OopClosure* keep_alive = &do_nothing_cl;
 665 #ifdef ASSERT
 666   ShenandoahWeakAssertNotForwardedClosure verify_cl;
 667   keep_alive = &verify_cl;
 668 #endif
 669   ShenandoahIsAliveClosure is_alive;
 670   WeakProcessor::weak_oops_do(workers, &is_alive, keep_alive, 1);
 671 }
 672 
 673 void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) {
 674   ReferenceProcessor* rp = _heap->ref_processor();
 675 
 676   ShenandoahPhaseTimings::Phase phase_process =
 677           full_gc ?
 678           ShenandoahPhaseTimings::full_gc_weakrefs_process :
 679           ShenandoahPhaseTimings::weakrefs_process;
 680 
 681   ShenandoahPhaseTimings::Phase phase_process_termination =
 682           full_gc ?
 683           ShenandoahPhaseTimings::full_gc_weakrefs_termination :
 684           ShenandoahPhaseTimings::weakrefs_termination;
 685 
 686   shenandoah_assert_rp_isalive_not_installed();
 687   ShenandoahIsAliveSelector is_alive;
 688   ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
 689 
 690   WorkGang* workers = _heap->workers();
 691   uint nworkers = workers->active_workers();
 692 
 693   rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
 694   rp->set_active_mt_degree(nworkers);
 695 
 696   assert(task_queues()->is_empty(), "Should be empty");
 697 
 698   // complete_gc and keep_alive closures instantiated here are only needed for
 699   // single-threaded path in RP. They share the queue 0 for tracking work, which
 700   // simplifies implementation. Since RP may decide to call complete_gc several
 701   // times, we need to be able to reuse the terminator.
 702   uint serial_worker_id = 0;
 703   ShenandoahTaskTerminator terminator(1, task_queues());
 704   ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
 705 
 706   ShenandoahRefProcTaskExecutor executor(workers);
 707 
 708   ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
 709 
 710   {
 711     ShenandoahGCPhase phase(phase_process);
 712     ShenandoahTerminationTracker phase_term(phase_process_termination);
 713 
 714     if (_heap->has_forwarded_objects()) {
 715       ShenandoahCMKeepAliveUpdateClosure keep_alive(get_queue(serial_worker_id));
 716       rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
 717                                         &complete_gc, &executor,
 718                                         &pt);
 719 
 720     } else {
 721       ShenandoahCMKeepAliveClosure keep_alive(get_queue(serial_worker_id));
 722       rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
 723                                         &complete_gc, &executor,
 724                                         &pt);
 725 
 726     }
 727 
 728     pt.print_all_references();
 729 
 730     assert(task_queues()->is_empty(), "Should be empty");
 731   }
 732 }
 733 
 734 class ShenandoahCancelledGCYieldClosure : public YieldClosure {
 735 private:
 736   ShenandoahHeap* const _heap;
 737 public:
 738   ShenandoahCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
 739   virtual bool should_return() { return _heap->cancelled_gc(); }
 740 };
 741 
 742 class ShenandoahPrecleanCompleteGCClosure : public VoidClosure {
 743 public:
 744   void do_void() {
 745     ShenandoahHeap* sh = ShenandoahHeap::heap();
 746     ShenandoahConcurrentMark* scm = sh->concurrent_mark();
 747     assert(sh->process_references(), "why else would we be here?");
 748     ShenandoahTaskTerminator terminator(1, scm->task_queues());
 749 
 750     ReferenceProcessor* rp = sh->ref_processor();
 751     shenandoah_assert_rp_isalive_installed();
 752 
 753     scm->mark_loop(0, &terminator, rp,
 754                    false, // not cancellable
 755                    false); // do not do strdedup
 756   }
 757 };
 758 
 759 class ShenandoahPrecleanKeepAliveUpdateClosure : public OopClosure {
 760 private:
 761   ShenandoahObjToScanQueue* _queue;
 762   ShenandoahHeap* _heap;
 763   ShenandoahMarkingContext* const _mark_context;
 764 
 765   template <class T>
 766   inline void do_oop_work(T* p) {
 767     ShenandoahConcurrentMark::mark_through_ref<T, CONCURRENT, NO_DEDUP>(p, _heap, _queue, _mark_context);
 768   }
 769 
 770 public:
 771   ShenandoahPrecleanKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 772     _queue(q),
 773     _heap(ShenandoahHeap::heap()),
 774     _mark_context(_heap->marking_context()) {}
 775 
 776   void do_oop(narrowOop* p) { do_oop_work(p); }
 777   void do_oop(oop* p)       { do_oop_work(p); }
 778 };
 779 
 780 class ShenandoahPrecleanTask : public AbstractGangTask {
 781 private:
 782   ReferenceProcessor* _rp;
 783 
 784 public:
 785   ShenandoahPrecleanTask(ReferenceProcessor* rp) :
 786           AbstractGangTask("Precleaning task"),
 787           _rp(rp) {}
 788 
 789   void work(uint worker_id) {
 790     assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
 791     ShenandoahParallelWorkerSession worker_session(worker_id);
 792 
 793     ShenandoahHeap* sh = ShenandoahHeap::heap();
 794 
 795     ShenandoahObjToScanQueue* q = sh->concurrent_mark()->get_queue(worker_id);
 796 
 797     ShenandoahCancelledGCYieldClosure yield;
 798     ShenandoahPrecleanCompleteGCClosure complete_gc;
 799 
 800     if (sh->has_forwarded_objects()) {
 801       ShenandoahForwardedIsAliveClosure is_alive;
 802       ShenandoahPrecleanKeepAliveUpdateClosure keep_alive(q);
 803       ResourceMark rm;
 804       _rp->preclean_discovered_references(&is_alive, &keep_alive,
 805                                           &complete_gc, &yield,
 806                                           NULL);
 807     } else {
 808       ShenandoahIsAliveClosure is_alive;
 809       ShenandoahCMKeepAliveClosure keep_alive(q);
 810       ResourceMark rm;
 811       _rp->preclean_discovered_references(&is_alive, &keep_alive,
 812                                           &complete_gc, &yield,
 813                                           NULL);
 814     }
 815   }
 816 };
 817 
 818 void ShenandoahConcurrentMark::preclean_weak_refs() {
 819   // Pre-cleaning weak references before diving into STW makes sense at the
 820   // end of concurrent mark. This will filter out the references which referents
 821   // are alive. Note that ReferenceProcessor already filters out these on reference
 822   // discovery, and the bulk of work is done here. This phase processes leftovers
 823   // that missed the initial filtering, i.e. when referent was marked alive after
 824   // reference was discovered by RP.
 825 
 826   assert(_heap->process_references(), "sanity");
 827 
 828   // Shortcut if no references were discovered to avoid winding up threads.
 829   ReferenceProcessor* rp = _heap->ref_processor();
 830   if (!rp->has_discovered_references()) {
 831     return;
 832   }
 833 
 834   assert(task_queues()->is_empty(), "Should be empty");
 835 
 836   ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
 837 
 838   shenandoah_assert_rp_isalive_not_installed();
 839   ShenandoahIsAliveSelector is_alive;
 840   ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
 841 
 842   // Execute precleaning in the worker thread: it will give us GCLABs, String dedup
 843   // queues and other goodies. When upstream ReferenceProcessor starts supporting
 844   // parallel precleans, we can extend this to more threads.
 845   WorkGang* workers = _heap->workers();
 846   uint nworkers = workers->active_workers();
 847   assert(nworkers == 1, "This code uses only a single worker");
 848   task_queues()->reserve(nworkers);
 849 
 850   ShenandoahPrecleanTask task(rp);
 851   workers->run_task(&task);
 852 
 853   assert(task_queues()->is_empty(), "Should be empty");
 854 }
 855 
 856 void ShenandoahConcurrentMark::cancel() {
 857   // Clean up marking stacks.
 858   ShenandoahObjToScanQueueSet* queues = task_queues();
 859   queues->clear();
 860 
 861   // Cancel SATB buffers.
 862   ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
 863 }
 864 
 865 ShenandoahObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) {
 866   assert(task_queues()->get_reserved() > worker_id, "No reserved queue for worker id: %d", worker_id);
 867   return _task_queues->queue(worker_id);
 868 }
 869 
 870 template <bool CANCELLABLE>
 871 void ShenandoahConcurrentMark::mark_loop_prework(uint w, ShenandoahTaskTerminator *t, ReferenceProcessor *rp,
 872                                                  bool strdedup) {
 873   ShenandoahObjToScanQueue* q = get_queue(w);
 874 
 875   jushort* ld = _heap->get_liveness_cache(w);
 876 
 877   // TODO: We can clean up this if we figure out how to do templated oop closures that
 878   // play nice with specialized_oop_iterators.
 879   if (_heap->unload_classes()) {
 880     if (_heap->has_forwarded_objects()) {
 881       if (strdedup) {
 882         ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp);
 883         mark_loop_work<ShenandoahMarkUpdateRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 884       } else {
 885         ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp);
 886         mark_loop_work<ShenandoahMarkUpdateRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
 887       }
 888     } else {
 889       if (strdedup) {
 890         ShenandoahMarkRefsMetadataDedupClosure cl(q, rp);
 891         mark_loop_work<ShenandoahMarkRefsMetadataDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 892       } else {
 893         ShenandoahMarkRefsMetadataClosure cl(q, rp);
 894         mark_loop_work<ShenandoahMarkRefsMetadataClosure, CANCELLABLE>(&cl, ld, w, t);
 895       }
 896     }
 897   } else {
 898     if (_heap->has_forwarded_objects()) {
 899       if (strdedup) {
 900         ShenandoahMarkUpdateRefsDedupClosure cl(q, rp);
 901         mark_loop_work<ShenandoahMarkUpdateRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 902       } else {
 903         ShenandoahMarkUpdateRefsClosure cl(q, rp);
 904         mark_loop_work<ShenandoahMarkUpdateRefsClosure, CANCELLABLE>(&cl, ld, w, t);
 905       }
 906     } else {
 907       if (strdedup) {
 908         ShenandoahMarkRefsDedupClosure cl(q, rp);
 909         mark_loop_work<ShenandoahMarkRefsDedupClosure, CANCELLABLE>(&cl, ld, w, t);
 910       } else {
 911         ShenandoahMarkRefsClosure cl(q, rp);
 912         mark_loop_work<ShenandoahMarkRefsClosure, CANCELLABLE>(&cl, ld, w, t);
 913       }
 914     }
 915   }
 916 
 917   _heap->flush_liveness_cache(w);
 918 }
 919 
 920 template <class T, bool CANCELLABLE>
 921 void ShenandoahConcurrentMark::mark_loop_work(T* cl, jushort* live_data, uint worker_id, ShenandoahTaskTerminator *terminator) {
 922   uintx stride = ShenandoahMarkLoopStride;
 923 
 924   ShenandoahHeap* heap = ShenandoahHeap::heap();
 925   ShenandoahObjToScanQueueSet* queues = task_queues();
 926   ShenandoahObjToScanQueue* q;
 927   ShenandoahMarkTask t;
 928 
 929   /*
 930    * Process outstanding queues, if any.
 931    *
 932    * There can be more queues than workers. To deal with the imbalance, we claim
 933    * extra queues first. Since marking can push new tasks into the queue associated
 934    * with this worker id, we come back to process this queue in the normal loop.
 935    */
 936   assert(queues->get_reserved() == heap->workers()->active_workers(),
 937          "Need to reserve proper number of queues: reserved: %u, active: %u", queues->get_reserved(), heap->workers()->active_workers());
 938 
 939   q = queues->claim_next();
 940   while (q != NULL) {
 941     if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
 942       return;
 943     }
 944 
 945     for (uint i = 0; i < stride; i++) {
 946       if (q->pop(t)) {
 947         do_task<T>(q, cl, live_data, &t);
 948       } else {
 949         assert(q->is_empty(), "Must be empty");
 950         q = queues->claim_next();
 951         break;
 952       }
 953     }
 954   }
 955   q = get_queue(worker_id);
 956 
 957   ShenandoahSATBBufferClosure drain_satb(q);
 958   SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 959 
 960   /*
 961    * Normal marking loop:
 962    */
 963   while (true) {
 964     if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) {
 965       return;
 966     }
 967 
 968     while (satb_mq_set.completed_buffers_num() > 0) {
 969       satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
 970     }
 971 
 972     uint work = 0;
 973     for (uint i = 0; i < stride; i++) {
 974       if (q->pop(t) ||
 975           queues->steal(worker_id, t)) {
 976         do_task<T>(q, cl, live_data, &t);
 977         work++;
 978       } else {
 979         break;
 980       }
 981     }
 982 
 983     if (work == 0) {
 984       // No work encountered in current stride, try to terminate.
 985       // Need to leave the STS here otherwise it might block safepoints.
 986       ShenandoahSuspendibleThreadSetLeaver stsl(CANCELLABLE && ShenandoahSuspendibleWorkers);
 987       ShenandoahTerminationTimingsTracker term_tracker(worker_id);
 988       ShenandoahTerminatorTerminator tt(heap);
 989       if (terminator->offer_termination(&tt)) return;
 990     }
 991   }
 992 }
 993 
 994 bool ShenandoahConcurrentMark::claim_codecache() {
 995   assert(ShenandoahConcurrentScanCodeRoots, "must not be called otherwise");
 996   return _claimed_codecache.try_set();
 997 }
 998 
 999 void ShenandoahConcurrentMark::clear_claim_codecache() {
1000   assert(ShenandoahConcurrentScanCodeRoots, "must not be called otherwise");
1001   _claimed_codecache.unset();
1002 }
--- EOF ---