1 /* 2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc_implementation/shared/parallelCleaning.hpp" 26 #include "gc_implementation/shenandoah/shenandoahBarrierSet.inline.hpp" 27 #include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp" 28 #include "gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp" 29 #include "gc_implementation/shenandoah/shenandoahOopClosures.inline.hpp" 30 #include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp" 31 #include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp" 32 #include "gc_implementation/shenandoah/shenandoah_specialized_oop_closures.hpp" 33 #include "gc_implementation/shenandoah/shenandoahOopClosures.inline.hpp" 34 #include "gc_implementation/shenandoah/brooksPointer.hpp" 35 #include "gc_implementation/shenandoah/shenandoahUtils.hpp" 36 #include "memory/referenceProcessor.hpp" 37 #include "gc_implementation/shenandoah/shenandoahTaskqueue.hpp" 38 #include "code/codeCache.hpp" 39 #include "classfile/symbolTable.hpp" 40 #include "classfile/systemDictionary.hpp" 41 #include "memory/iterator.inline.hpp" 42 #include "oops/oop.inline.hpp" 43 #include "utilities/taskqueue.hpp" 44 45 template<UpdateRefsMode UPDATE_REFS> 46 class ShenandoahInitMarkRootsClosure : public OopClosure { 47 private: 48 ShenandoahObjToScanQueue* _queue; 49 ShenandoahHeap* _heap; 50 51 template <class T> 52 inline void do_oop_nv(T* p) { 53 ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS>(p, _heap, _queue); 54 } 55 56 public: 57 ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) : 58 _queue(q), _heap(ShenandoahHeap::heap()) {}; 59 60 void do_oop(narrowOop* p) { do_oop_nv(p); } 61 void do_oop(oop* p) { do_oop_nv(p); } 62 }; 63 64 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) : 65 MetadataAwareOopClosure(rp), 66 _queue(q), 67 _heap((ShenandoahHeap*) Universe::heap()) 68 { 69 } 70 71 template<UpdateRefsMode UPDATE_REFS> 72 class ShenandoahInitMarkRootsTask : public AbstractGangTask { 73 private: 74 ShenandoahRootProcessor* _rp; 75 bool _process_refs; 76 public: 77 ShenandoahInitMarkRootsTask(ShenandoahRootProcessor* rp, bool process_refs) : 78 AbstractGangTask("Shenandoah init mark roots task"), 79 _rp(rp), 80 _process_refs(process_refs) { 81 } 82 83 void work(uint worker_id) { 84 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 85 86 ShenandoahHeap* heap = ShenandoahHeap::heap(); 87 ShenandoahObjToScanQueueSet* queues = heap->concurrentMark()->task_queues(); 88 assert(queues->get_reserved() > worker_id, err_msg("Queue has not been reserved for worker id: %d", worker_id)); 89 90 ShenandoahObjToScanQueue* q = queues->queue(worker_id); 91 ShenandoahInitMarkRootsClosure<UPDATE_REFS> mark_cl(q); 92 CLDToOopClosure cldCl(&mark_cl); 93 MarkingCodeBlobClosure blobsCl(&mark_cl, ! CodeBlobToOopClosure::FixRelocations); 94 95 // The rationale for selecting the roots to scan is as follows: 96 // a. With unload_classes = true, we only want to scan the actual strong roots from the 97 // code cache. This will allow us to identify the dead classes, unload them, *and* 98 // invalidate the relevant code cache blobs. This could be only done together with 99 // class unloading. 100 // b. With unload_classes = false, we have to nominally retain all the references from code 101 // cache, because there could be the case of embedded class/oop in the generated code, 102 // which we will never visit during mark. Without code cache invalidation, as in (a), 103 // we risk executing that code cache blob, and crashing. 104 // c. With ShenandoahConcurrentScanCodeRoots, we avoid scanning the entire code cache here, 105 // and instead do that in concurrent phase under the relevant lock. This saves init mark 106 // pause time. 107 108 ResourceMark m; 109 if (heap->concurrentMark()->unload_classes()) { 110 _rp->process_strong_roots(&mark_cl, _process_refs ? NULL : &mark_cl, &cldCl, NULL, &blobsCl, NULL, worker_id); 111 } else { 112 if (ShenandoahConcurrentScanCodeRoots) { 113 CodeBlobClosure* code_blobs = NULL; 114 #ifdef ASSERT 115 ShenandoahAssertToSpaceClosure assert_to_space_oops; 116 CodeBlobToOopClosure assert_to_space(&assert_to_space_oops, !CodeBlobToOopClosure::FixRelocations); 117 // If conc code cache evac is disabled, code cache should have only to-space ptrs. 118 // Otherwise, it should have to-space ptrs only if mark does not update refs. 119 if (!ShenandoahConcurrentEvacCodeRoots && !heap->has_forwarded_objects()) { 120 code_blobs = &assert_to_space; 121 } 122 #endif 123 _rp->process_all_roots(&mark_cl, _process_refs ? NULL : &mark_cl, &cldCl, code_blobs, NULL, worker_id); 124 } else { 125 _rp->process_all_roots(&mark_cl, _process_refs ? NULL : &mark_cl, &cldCl, &blobsCl, NULL, worker_id); 126 } 127 } 128 } 129 }; 130 131 class ShenandoahUpdateRootsTask : public AbstractGangTask { 132 private: 133 ShenandoahRootProcessor* _rp; 134 const bool _update_code_cache; 135 public: 136 ShenandoahUpdateRootsTask(ShenandoahRootProcessor* rp, bool update_code_cache) : 137 AbstractGangTask("Shenandoah update roots task"), 138 _rp(rp), 139 _update_code_cache(update_code_cache) { 140 } 141 142 void work(uint worker_id) { 143 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 144 145 ShenandoahHeap* heap = ShenandoahHeap::heap(); 146 ShenandoahUpdateRefsClosure cl; 147 CLDToOopClosure cldCl(&cl); 148 149 CodeBlobClosure* code_blobs; 150 CodeBlobToOopClosure update_blobs(&cl, CodeBlobToOopClosure::FixRelocations); 151 #ifdef ASSERT 152 ShenandoahAssertToSpaceClosure assert_to_space_oops; 153 CodeBlobToOopClosure assert_to_space(&assert_to_space_oops, !CodeBlobToOopClosure::FixRelocations); 154 #endif 155 if (_update_code_cache) { 156 code_blobs = &update_blobs; 157 } else { 158 code_blobs = 159 DEBUG_ONLY(&assert_to_space) 160 NOT_DEBUG(NULL); 161 } 162 _rp->process_all_roots(&cl, &cl, &cldCl, code_blobs, NULL, worker_id); 163 } 164 }; 165 166 class ShenandoahConcurrentMarkingTask : public AbstractGangTask { 167 private: 168 ShenandoahConcurrentMark* _cm; 169 ParallelTaskTerminator* _terminator; 170 bool _update_refs; 171 172 public: 173 ShenandoahConcurrentMarkingTask(ShenandoahConcurrentMark* cm, ParallelTaskTerminator* terminator, bool update_refs) : 174 AbstractGangTask("Root Region Scan"), _cm(cm), _terminator(terminator), _update_refs(update_refs) { 175 } 176 177 178 void work(uint worker_id) { 179 ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); 180 jushort* live_data = _cm->get_liveness(worker_id); 181 ReferenceProcessor* rp; 182 if (_cm->process_references()) { 183 rp = ShenandoahHeap::heap()->ref_processor(); 184 shenandoah_assert_rp_isalive_installed(); 185 } else { 186 rp = NULL; 187 } 188 189 _cm->concurrent_scan_code_roots(worker_id, rp, _update_refs); 190 _cm->mark_loop(worker_id, _terminator, rp, 191 true, // cancellable 192 true, // drain SATBs as we go 193 true, // count liveness 194 _cm->unload_classes(), 195 _update_refs); 196 } 197 }; 198 199 class ShenandoahFinalMarkingTask : public AbstractGangTask { 200 private: 201 ShenandoahConcurrentMark* _cm; 202 ParallelTaskTerminator* _terminator; 203 bool _update_refs; 204 bool _count_live; 205 bool _unload_classes; 206 207 public: 208 ShenandoahFinalMarkingTask(ShenandoahConcurrentMark* cm, ParallelTaskTerminator* terminator, bool update_refs, bool count_live, bool unload_classes) : 209 AbstractGangTask("Shenandoah Final Marking"), _cm(cm), _terminator(terminator), _update_refs(update_refs), _count_live(count_live), _unload_classes(unload_classes) { 210 } 211 212 void work(uint worker_id) { 213 // First drain remaining SATB buffers. 214 // Notice that this is not strictly necessary for mark-compact. But since 215 // it requires a StrongRootsScope around the task, we need to claim the 216 // threads, and performance-wise it doesn't really matter. Adds about 1ms to 217 // full-gc. 218 _cm->drain_satb_buffers(worker_id, true); 219 220 ReferenceProcessor* rp; 221 if (_cm->process_references()) { 222 rp = ShenandoahHeap::heap()->ref_processor(); 223 shenandoah_assert_rp_isalive_installed(); 224 } else { 225 rp = NULL; 226 } 227 228 // Degenerated cycle may bypass concurrent cycle, so code roots might not be scanned, 229 // let's check here. 230 _cm->concurrent_scan_code_roots(worker_id, rp, _update_refs); 231 _cm->mark_loop(worker_id, _terminator, rp, 232 false, // not cancellable 233 false, // do not drain SATBs, already drained 234 _count_live, 235 _unload_classes, 236 _update_refs); 237 238 assert(_cm->task_queues()->is_empty(), "Should be empty"); 239 } 240 }; 241 242 void ShenandoahConcurrentMark::mark_roots(ShenandoahPhaseTimings::Phase root_phase) { 243 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread"); 244 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 245 246 ShenandoahHeap* heap = ShenandoahHeap::heap(); 247 248 ShenandoahGCPhase phase(root_phase); 249 250 WorkGang* workers = heap->workers(); 251 uint nworkers = workers->active_workers(); 252 253 assert(nworkers <= task_queues()->size(), "Just check"); 254 255 ShenandoahRootProcessor root_proc(heap, nworkers, root_phase); 256 TASKQUEUE_STATS_ONLY(reset_taskqueue_stats()); 257 task_queues()->reserve(nworkers); 258 259 if (heap->has_forwarded_objects()) { 260 ShenandoahInitMarkRootsTask<RESOLVE> mark_roots(&root_proc, process_references()); 261 workers->run_task(&mark_roots); 262 } else { 263 // No need to update references, which means the heap is stable. 264 // Can save time not walking through forwarding pointers. 265 ShenandoahInitMarkRootsTask<NONE> mark_roots(&root_proc, process_references()); 266 workers->run_task(&mark_roots); 267 } 268 269 if (ShenandoahConcurrentScanCodeRoots) { 270 clear_claim_codecache(); 271 } 272 } 273 274 void ShenandoahConcurrentMark::init_mark_roots() { 275 assert(Thread::current()->is_VM_thread(), "can only do this in VMThread"); 276 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 277 278 mark_roots(ShenandoahPhaseTimings::scan_roots); 279 } 280 281 void ShenandoahConcurrentMark::update_roots(ShenandoahPhaseTimings::Phase root_phase) { 282 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 283 284 bool update_code_cache = true; // initialize to safer value 285 switch (root_phase) { 286 case ShenandoahPhaseTimings::update_roots: 287 case ShenandoahPhaseTimings::final_update_refs_roots: 288 // If code cache was evacuated concurrently, we need to update code cache roots. 289 update_code_cache = ShenandoahConcurrentEvacCodeRoots; 290 break; 291 case ShenandoahPhaseTimings::full_gc_roots: 292 update_code_cache = true; 293 break; 294 default: 295 ShouldNotReachHere(); 296 } 297 298 ShenandoahHeap* heap = ShenandoahHeap::heap(); 299 300 ShenandoahGCPhase phase(root_phase); 301 302 COMPILER2_PRESENT(DerivedPointerTable::clear()); 303 304 uint nworkers = heap->workers()->active_workers(); 305 306 ShenandoahRootProcessor root_proc(heap, nworkers, root_phase); 307 ShenandoahUpdateRootsTask update_roots(&root_proc, update_code_cache); 308 heap->workers()->run_task(&update_roots); 309 310 COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); 311 } 312 313 void ShenandoahConcurrentMark::initialize(uint workers) { 314 _heap = ShenandoahHeap::heap(); 315 316 uint num_queues = MAX2(workers, 1U); 317 318 _task_queues = new ShenandoahObjToScanQueueSet((int) num_queues); 319 320 for (uint i = 0; i < num_queues; ++i) { 321 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue(); 322 task_queue->initialize(); 323 _task_queues->register_queue(i, task_queue); 324 } 325 326 JavaThread::satb_mark_queue_set().set_buffer_size(ShenandoahSATBBufferSize); 327 328 size_t num_regions = ShenandoahHeap::heap()->num_regions(); 329 _liveness_local = NEW_C_HEAP_ARRAY(jushort*, workers, mtGC); 330 for (uint worker = 0; worker < workers; worker++) { 331 _liveness_local[worker] = NEW_C_HEAP_ARRAY(jushort, num_regions, mtGC); 332 } 333 } 334 335 void ShenandoahConcurrentMark::concurrent_scan_code_roots(uint worker_id, ReferenceProcessor* rp, bool update_refs) { 336 if (ShenandoahConcurrentScanCodeRoots && claim_codecache()) { 337 ShenandoahObjToScanQueue* q = task_queues()->queue(worker_id); 338 if (!unload_classes()) { 339 MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); 340 if (update_refs) { 341 ShenandoahMarkResolveRefsClosure cl(q, rp); 342 CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations); 343 CodeCache::blobs_do(&blobs); 344 } else { 345 ShenandoahMarkRefsClosure cl(q, rp); 346 CodeBlobToOopClosure blobs(&cl, !CodeBlobToOopClosure::FixRelocations); 347 CodeCache::blobs_do(&blobs); 348 } 349 } 350 } 351 } 352 353 void ShenandoahConcurrentMark::mark_from_roots() { 354 ShenandoahHeap* sh = ShenandoahHeap::heap(); 355 WorkGang* workers = sh->workers(); 356 uint nworkers = workers->active_workers(); 357 358 bool update_refs = sh->has_forwarded_objects(); 359 360 ShenandoahGCPhase conc_mark_phase(ShenandoahPhaseTimings::conc_mark); 361 362 if (process_references()) { 363 ReferenceProcessor* rp = sh->ref_processor(); 364 rp->set_active_mt_degree(nworkers); 365 366 // enable ("weak") refs discovery 367 rp->enable_discovery(true /*verify_no_refs*/, true); 368 rp->setup_policy(sh->is_full_gc_in_progress()); // snapshot the soft ref policy to be used in this cycle 369 } 370 371 shenandoah_assert_rp_isalive_not_installed(); 372 ReferenceProcessorIsAliveMutator fix_isalive(sh->ref_processor(), sh->is_alive_closure()); 373 374 task_queues()->reserve(nworkers); 375 376 if (UseShenandoahOWST) { 377 ShenandoahTaskTerminator terminator(nworkers, task_queues()); 378 ShenandoahConcurrentMarkingTask markingTask = ShenandoahConcurrentMarkingTask(this, &terminator, update_refs); 379 workers->run_task(&markingTask); 380 } else { 381 ParallelTaskTerminator terminator(nworkers, task_queues()); 382 ShenandoahConcurrentMarkingTask markingTask = ShenandoahConcurrentMarkingTask(this, &terminator, update_refs); 383 workers->run_task(&markingTask); 384 } 385 386 assert(task_queues()->is_empty() || sh->cancelled_concgc(), "Should be empty when not cancelled"); 387 if (! sh->cancelled_concgc()) { 388 TASKQUEUE_STATS_ONLY(print_taskqueue_stats()); 389 } 390 391 TASKQUEUE_STATS_ONLY(reset_taskqueue_stats()); 392 } 393 394 void ShenandoahConcurrentMark::finish_mark_from_roots() { 395 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 396 397 ShenandoahHeap* sh = ShenandoahHeap::heap(); 398 399 TASKQUEUE_STATS_ONLY(reset_taskqueue_stats()); 400 401 shared_finish_mark_from_roots(/* full_gc = */ false); 402 403 if (sh->has_forwarded_objects()) { 404 update_roots(ShenandoahPhaseTimings::update_roots); 405 } 406 407 TASKQUEUE_STATS_ONLY(print_taskqueue_stats()); 408 } 409 410 void ShenandoahConcurrentMark::shared_finish_mark_from_roots(bool full_gc) { 411 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 412 413 ShenandoahHeap* sh = ShenandoahHeap::heap(); 414 415 uint nworkers = sh->workers()->active_workers(); 416 417 // Finally mark everything else we've got in our queues during the previous steps. 418 // It does two different things for concurrent vs. mark-compact GC: 419 // - For concurrent GC, it starts with empty task queues, drains the remaining 420 // SATB buffers, and then completes the marking closure. 421 // - For mark-compact GC, it starts out with the task queues seeded by initial 422 // root scan, and completes the closure, thus marking through all live objects 423 // The implementation is the same, so it's shared here. 424 { 425 ShenandoahGCPhase phase(full_gc ? 426 ShenandoahPhaseTimings::full_gc_mark_finish_queues : 427 ShenandoahPhaseTimings::finish_queues); 428 bool count_live = !(ShenandoahNoLivenessFullGC && full_gc); // we do not need liveness data for full GC 429 task_queues()->reserve(nworkers); 430 431 shenandoah_assert_rp_isalive_not_installed(); 432 ReferenceProcessorIsAliveMutator fix_isalive(sh->ref_processor(), sh->is_alive_closure()); 433 434 SharedHeap::StrongRootsScope scope(sh, true); 435 if (UseShenandoahOWST) { 436 ShenandoahTaskTerminator terminator(nworkers, task_queues()); 437 ShenandoahFinalMarkingTask task(this, &terminator, sh->has_forwarded_objects(), count_live, unload_classes()); 438 sh->workers()->run_task(&task); 439 } else { 440 ParallelTaskTerminator terminator(nworkers, task_queues()); 441 ShenandoahFinalMarkingTask task(this, &terminator, sh->has_forwarded_objects(), count_live, unload_classes()); 442 sh->workers()->run_task(&task); 443 } 444 } 445 446 assert(task_queues()->is_empty(), "Should be empty"); 447 448 // When we're done marking everything, we process weak references. 449 if (process_references()) { 450 weak_refs_work(full_gc); 451 } 452 453 // And finally finish class unloading 454 if (unload_classes()) { 455 sh->unload_classes_and_cleanup_tables(full_gc); 456 } 457 458 assert(task_queues()->is_empty(), "Should be empty"); 459 460 } 461 462 class ShenandoahSATBThreadsClosure : public ThreadClosure { 463 ShenandoahSATBBufferClosure* _satb_cl; 464 int _thread_parity; 465 466 public: 467 ShenandoahSATBThreadsClosure(ShenandoahSATBBufferClosure* satb_cl) : 468 _satb_cl(satb_cl), 469 _thread_parity(SharedHeap::heap()->strong_roots_parity()) {} 470 471 void do_thread(Thread* thread) { 472 if (thread->is_Java_thread()) { 473 if (thread->claim_oops_do(true, _thread_parity)) { 474 JavaThread* jt = (JavaThread*)thread; 475 jt->satb_mark_queue().apply_closure_and_empty(_satb_cl); 476 } 477 } else if (thread->is_VM_thread()) { 478 if (thread->claim_oops_do(true, _thread_parity)) { 479 JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(_satb_cl); 480 } 481 } 482 } 483 }; 484 485 void ShenandoahConcurrentMark::drain_satb_buffers(uint worker_id, bool remark) { 486 ShenandoahObjToScanQueue* q = get_queue(worker_id); 487 ShenandoahSATBBufferClosure cl(q); 488 489 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 490 while (satb_mq_set.apply_closure_to_completed_buffer(&cl)); 491 492 if (remark) { 493 ShenandoahSATBThreadsClosure tc(&cl); 494 Threads::threads_do(&tc); 495 } 496 } 497 498 #if TASKQUEUE_STATS 499 void ShenandoahConcurrentMark::print_taskqueue_stats_hdr(outputStream* const st) { 500 st->print_raw_cr("GC Task Stats"); 501 st->print_raw("thr "); TaskQueueStats::print_header(1, st); st->cr(); 502 st->print_raw("--- "); TaskQueueStats::print_header(2, st); st->cr(); 503 } 504 505 void ShenandoahConcurrentMark::print_taskqueue_stats() const { 506 if (! ShenandoahLogTrace) { 507 return; 508 } 509 ResourceMark rm; 510 outputStream* st = gclog_or_tty; 511 print_taskqueue_stats_hdr(st); 512 513 TaskQueueStats totals; 514 const uint n = _task_queues->size(); 515 for (uint i = 0; i < n; ++i) { 516 st->print(UINT32_FORMAT_W(3), i); 517 _task_queues->queue(i)->stats.print(st); 518 st->cr(); 519 totals += _task_queues->queue(i)->stats; 520 } 521 st->print("tot "); totals.print(st); st->cr(); 522 DEBUG_ONLY(totals.verify()); 523 524 } 525 526 void ShenandoahConcurrentMark::reset_taskqueue_stats() { 527 const uint n = task_queues()->size(); 528 for (uint i = 0; i < n; ++i) { 529 task_queues()->queue(i)->stats.reset(); 530 } 531 } 532 #endif // TASKQUEUE_STATS 533 534 // Weak Reference Closures 535 class ShenandoahCMDrainMarkingStackClosure: public VoidClosure { 536 uint _worker_id; 537 ParallelTaskTerminator* _terminator; 538 bool _reset_terminator; 539 540 public: 541 ShenandoahCMDrainMarkingStackClosure(uint worker_id, ParallelTaskTerminator* t, bool reset_terminator = false): 542 _worker_id(worker_id), 543 _terminator(t), 544 _reset_terminator(reset_terminator) { 545 } 546 547 void do_void() { 548 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 549 550 ShenandoahHeap* sh = ShenandoahHeap::heap(); 551 ShenandoahConcurrentMark* scm = sh->concurrentMark(); 552 assert(scm->process_references(), "why else would we be here?"); 553 ReferenceProcessor* rp = sh->ref_processor(); 554 555 shenandoah_assert_rp_isalive_installed(); 556 557 scm->mark_loop(_worker_id, _terminator, rp, 558 false, // not cancellable 559 false, // do not drain SATBs 560 true, // count liveness 561 scm->unload_classes(), 562 sh->has_forwarded_objects()); 563 564 if (_reset_terminator) { 565 _terminator->reset_for_reuse(); 566 } 567 } 568 }; 569 570 571 class ShenandoahCMKeepAliveClosure : public OopClosure { 572 private: 573 ShenandoahObjToScanQueue* _queue; 574 ShenandoahHeap* _heap; 575 576 template <class T> 577 inline void do_oop_nv(T* p) { 578 ShenandoahConcurrentMark::mark_through_ref<T, NONE>(p, _heap, _queue); 579 } 580 581 public: 582 ShenandoahCMKeepAliveClosure(ShenandoahObjToScanQueue* q) : 583 _queue(q), _heap(ShenandoahHeap::heap()) {}; 584 585 void do_oop(narrowOop* p) { do_oop_nv(p); } 586 void do_oop(oop* p) { do_oop_nv(p); } 587 }; 588 589 class ShenandoahCMKeepAliveUpdateClosure : public OopClosure { 590 private: 591 ShenandoahObjToScanQueue* _queue; 592 ShenandoahHeap* _heap; 593 594 template <class T> 595 inline void do_oop_nv(T* p) { 596 ShenandoahConcurrentMark::mark_through_ref<T, SIMPLE>(p, _heap, _queue); 597 } 598 599 public: 600 ShenandoahCMKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) : 601 _queue(q), _heap(ShenandoahHeap::heap()) {}; 602 603 void do_oop(narrowOop* p) { do_oop_nv(p); } 604 void do_oop(oop* p) { do_oop_nv(p); } 605 }; 606 607 class ShenandoahRefProcTaskProxy : public AbstractGangTask { 608 609 private: 610 AbstractRefProcTaskExecutor::ProcessTask& _proc_task; 611 ParallelTaskTerminator* _terminator; 612 public: 613 614 ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task, 615 ParallelTaskTerminator* t) : 616 AbstractGangTask("Process reference objects in parallel"), 617 _proc_task(proc_task), 618 _terminator(t) { 619 } 620 621 void work(uint worker_id) { 622 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 623 ShenandoahHeap* heap = ShenandoahHeap::heap(); 624 ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id, _terminator); 625 if (heap->has_forwarded_objects()) { 626 ShenandoahForwardedIsAliveClosure is_alive; 627 ShenandoahCMKeepAliveUpdateClosure keep_alive(heap->concurrentMark()->get_queue(worker_id)); 628 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc); 629 } else { 630 ShenandoahIsAliveClosure is_alive; 631 ShenandoahCMKeepAliveClosure keep_alive(heap->concurrentMark()->get_queue(worker_id)); 632 _proc_task.work(worker_id, is_alive, keep_alive, complete_gc); 633 } 634 } 635 }; 636 637 class ShenandoahRefEnqueueTaskProxy : public AbstractGangTask { 638 639 private: 640 AbstractRefProcTaskExecutor::EnqueueTask& _enqueue_task; 641 642 public: 643 644 ShenandoahRefEnqueueTaskProxy(AbstractRefProcTaskExecutor::EnqueueTask& enqueue_task) : 645 AbstractGangTask("Enqueue reference objects in parallel"), 646 _enqueue_task(enqueue_task) { 647 } 648 649 void work(uint worker_id) { 650 _enqueue_task.work(worker_id); 651 } 652 }; 653 654 class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor { 655 656 private: 657 WorkGang* _workers; 658 659 public: 660 661 ShenandoahRefProcTaskExecutor(WorkGang* workers) : 662 _workers(workers) { 663 } 664 665 // Executes a task using worker threads. 666 void execute(ProcessTask& task) { 667 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); 668 669 // Shortcut execution if task is empty. 670 // This should be replaced with the generic ReferenceProcessor shortcut, 671 // see JDK-8181214, JDK-8043575, JDK-6938732. 672 if (task.is_empty()) { 673 return; 674 } 675 676 ShenandoahHeap* heap = ShenandoahHeap::heap(); 677 ShenandoahConcurrentMark* cm = heap->concurrentMark(); 678 uint nworkers = _workers->active_workers(); 679 cm->task_queues()->reserve(nworkers); 680 if (UseShenandoahOWST) { 681 ShenandoahTaskTerminator terminator(nworkers, cm->task_queues()); 682 ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator); 683 _workers->run_task(&proc_task_proxy); 684 } else { 685 ParallelTaskTerminator terminator(nworkers, cm->task_queues()); 686 ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator); 687 _workers->run_task(&proc_task_proxy); 688 } 689 } 690 691 void execute(EnqueueTask& task) { 692 ShenandoahRefEnqueueTaskProxy enqueue_task_proxy(task); 693 _workers->run_task(&enqueue_task_proxy); 694 } 695 }; 696 697 698 void ShenandoahConcurrentMark::weak_refs_work(bool full_gc) { 699 assert(process_references(), "sanity"); 700 701 ShenandoahHeap* sh = ShenandoahHeap::heap(); 702 703 ShenandoahPhaseTimings::Phase phase_root = 704 full_gc ? 705 ShenandoahPhaseTimings::full_gc_weakrefs : 706 ShenandoahPhaseTimings::weakrefs; 707 708 ShenandoahGCPhase phase(phase_root); 709 710 ReferenceProcessor* rp = sh->ref_processor(); 711 weak_refs_work_doit(full_gc); 712 713 rp->verify_no_references_recorded(); 714 assert(!rp->discovery_enabled(), "Post condition"); 715 716 } 717 718 void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) { 719 ShenandoahHeap* sh = ShenandoahHeap::heap(); 720 721 ReferenceProcessor* rp = sh->ref_processor(); 722 723 ShenandoahPhaseTimings::Phase phase_process = 724 full_gc ? 725 ShenandoahPhaseTimings::full_gc_weakrefs_process : 726 ShenandoahPhaseTimings::weakrefs_process; 727 728 ShenandoahPhaseTimings::Phase phase_enqueue = 729 full_gc ? 730 ShenandoahPhaseTimings::full_gc_weakrefs_enqueue : 731 ShenandoahPhaseTimings::weakrefs_enqueue; 732 733 shenandoah_assert_rp_isalive_not_installed(); 734 ReferenceProcessorIsAliveMutator fix_isalive(rp, sh->is_alive_closure()); 735 736 WorkGang* workers = sh->workers(); 737 uint nworkers = workers->active_workers(); 738 739 // Setup collector policy for softref cleaning. 740 bool clear_soft_refs = sh->collector_policy()->use_should_clear_all_soft_refs(true /* bogus arg*/); 741 log_develop_debug(gc, ref)("clearing soft refs: %s", BOOL_TO_STR(clear_soft_refs)); 742 rp->setup_policy(clear_soft_refs); 743 rp->set_active_mt_degree(nworkers); 744 745 assert(task_queues()->is_empty(), "Should be empty"); 746 747 // complete_gc and keep_alive closures instantiated here are only needed for 748 // single-threaded path in RP. They share the queue 0 for tracking work, which 749 // simplifies implementation. Since RP may decide to call complete_gc several 750 // times, we need to be able to reuse the terminator. 751 uint serial_worker_id = 0; 752 ParallelTaskTerminator terminator(1, task_queues()); 753 ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true); 754 755 ShenandoahRefProcTaskExecutor executor(workers); 756 757 { 758 ShenandoahGCPhase phase(phase_process); 759 760 if (sh->has_forwarded_objects()) { 761 ShenandoahForwardedIsAliveClosure is_alive; 762 ShenandoahCMKeepAliveUpdateClosure keep_alive(get_queue(serial_worker_id)); 763 rp->process_discovered_references(&is_alive, &keep_alive, 764 &complete_gc, &executor, 765 NULL, sh->shenandoahPolicy()->tracer()->gc_id()); 766 } else { 767 ShenandoahIsAliveClosure is_alive; 768 ShenandoahCMKeepAliveClosure keep_alive(get_queue(serial_worker_id)); 769 rp->process_discovered_references(&is_alive, &keep_alive, 770 &complete_gc, &executor, 771 NULL, sh->shenandoahPolicy()->tracer()->gc_id()); 772 } 773 774 assert(task_queues()->is_empty(), "Should be empty"); 775 } 776 777 { 778 ShenandoahGCPhase phase(phase_enqueue); 779 rp->enqueue_discovered_references(&executor); 780 } 781 } 782 783 class ShenandoahCancelledGCYieldClosure : public YieldClosure { 784 private: 785 ShenandoahHeap* const _heap; 786 public: 787 ShenandoahCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {}; 788 virtual bool should_return() { return _heap->cancelled_concgc(); } 789 }; 790 791 class ShenandoahPrecleanCompleteGCClosure : public VoidClosure { 792 public: 793 void do_void() { 794 ShenandoahHeap* sh = ShenandoahHeap::heap(); 795 ShenandoahConcurrentMark* scm = sh->concurrentMark(); 796 assert(scm->process_references(), "why else would we be here?"); 797 ParallelTaskTerminator terminator(1, scm->task_queues()); 798 799 ReferenceProcessor* rp = sh->ref_processor(); 800 shenandoah_assert_rp_isalive_installed(); 801 802 scm->mark_loop(0, &terminator, rp, 803 false, // not cancellable 804 true, // drain SATBs 805 true, // count liveness 806 scm->unload_classes(), 807 sh->has_forwarded_objects()); 808 } 809 }; 810 811 class ShenandoahPrecleanKeepAliveUpdateClosure : public OopClosure { 812 private: 813 ShenandoahObjToScanQueue* _queue; 814 ShenandoahHeap* _heap; 815 816 template <class T> 817 inline void do_oop_nv(T* p) { 818 ShenandoahConcurrentMark::mark_through_ref<T, CONCURRENT>(p, _heap, _queue); 819 } 820 821 public: 822 ShenandoahPrecleanKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) : 823 _queue(q), _heap(ShenandoahHeap::heap()) {} 824 825 void do_oop(narrowOop* p) { do_oop_nv(p); } 826 void do_oop(oop* p) { do_oop_nv(p); } 827 }; 828 829 void ShenandoahConcurrentMark::preclean_weak_refs() { 830 // Pre-cleaning weak references before diving into STW makes sense at the 831 // end of concurrent mark. This will filter out the references which referents 832 // are alive. Note that ReferenceProcessor already filters out these on reference 833 // discovery, and the bulk of work is done here. This phase processes leftovers 834 // that missed the initial filtering, i.e. when referent was marked alive after 835 // reference was discovered by RP. 836 837 assert(process_references(), "sanity"); 838 839 ShenandoahHeap* sh = ShenandoahHeap::heap(); 840 ReferenceProcessor* rp = sh->ref_processor(); 841 ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false); 842 843 shenandoah_assert_rp_isalive_not_installed(); 844 ReferenceProcessorIsAliveMutator fix_isalive(rp, sh->is_alive_closure()); 845 846 // Interrupt on cancelled GC 847 ShenandoahCancelledGCYieldClosure yield; 848 849 assert(task_queues()->is_empty(), "Should be empty"); 850 851 ShenandoahPrecleanCompleteGCClosure complete_gc; 852 if (sh->has_forwarded_objects()) { 853 ShenandoahForwardedIsAliveClosure is_alive; 854 ShenandoahPrecleanKeepAliveUpdateClosure keep_alive(get_queue(0)); 855 ResourceMark rm; 856 rp->preclean_discovered_references(&is_alive, &keep_alive, 857 &complete_gc, &yield, 858 NULL, sh->shenandoahPolicy()->tracer()->gc_id()); 859 } else { 860 ShenandoahIsAliveClosure is_alive; 861 ShenandoahCMKeepAliveClosure keep_alive(get_queue(0)); 862 ResourceMark rm; 863 rp->preclean_discovered_references(&is_alive, &keep_alive, 864 &complete_gc, &yield, 865 NULL, sh->shenandoahPolicy()->tracer()->gc_id()); 866 } 867 868 assert(task_queues()->is_empty(), "Should be empty"); 869 } 870 871 void ShenandoahConcurrentMark::cancel() { 872 // Clean up marking stacks. 873 ShenandoahObjToScanQueueSet* queues = task_queues(); 874 queues->clear(); 875 876 // Cancel SATB buffers. 877 JavaThread::satb_mark_queue_set().abandon_partial_marking(); 878 } 879 880 ShenandoahObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) { 881 assert(task_queues()->get_reserved() > worker_id, err_msg("No reserved queue for worker id: %d", worker_id)); 882 return _task_queues->queue(worker_id); 883 } 884 885 void ShenandoahConcurrentMark::clear_queue(ShenandoahObjToScanQueue *q) { 886 q->set_empty(); 887 q->overflow_stack()->clear(); 888 q->clear_buffer(); 889 } 890 891 template <bool CANCELLABLE, bool DRAIN_SATB, bool COUNT_LIVENESS> 892 void ShenandoahConcurrentMark::mark_loop_prework(uint w, ParallelTaskTerminator *t, ReferenceProcessor *rp, bool class_unload, bool update_refs) { 893 ShenandoahObjToScanQueue* q = get_queue(w); 894 895 jushort* ld; 896 if (COUNT_LIVENESS) { 897 ld = get_liveness(w); 898 Copy::fill_to_bytes(ld, _heap->num_regions() * sizeof(jushort)); 899 } else { 900 ld = NULL; 901 } 902 903 // TODO: We can clean up this if we figure out how to do templated oop closures that 904 // play nice with specialized_oop_iterators. 905 if (class_unload) { 906 if (update_refs) { 907 ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp); 908 mark_loop_work<ShenandoahMarkUpdateRefsMetadataClosure, CANCELLABLE, DRAIN_SATB, COUNT_LIVENESS>(&cl, ld, w, t); 909 } else { 910 ShenandoahMarkRefsMetadataClosure cl(q, rp); 911 mark_loop_work<ShenandoahMarkRefsMetadataClosure, CANCELLABLE, DRAIN_SATB, COUNT_LIVENESS>(&cl, ld, w, t); 912 } 913 } else { 914 if (update_refs) { 915 ShenandoahMarkUpdateRefsClosure cl(q, rp); 916 mark_loop_work<ShenandoahMarkUpdateRefsClosure, CANCELLABLE, DRAIN_SATB, COUNT_LIVENESS>(&cl, ld, w, t); 917 } else { 918 ShenandoahMarkRefsClosure cl(q, rp); 919 mark_loop_work<ShenandoahMarkRefsClosure, CANCELLABLE, DRAIN_SATB, COUNT_LIVENESS>(&cl, ld, w, t); 920 } 921 } 922 923 if (COUNT_LIVENESS) { 924 for (uint i = 0; i < _heap->num_regions(); i++) { 925 ShenandoahHeapRegion* r = _heap->get_region(i); 926 jushort live = ld[i]; 927 if (live > 0) { 928 r->increase_live_data_gc_words(live); 929 } 930 } 931 } 932 } 933 934 template <class T, bool CANCELLABLE, bool DRAIN_SATB, bool COUNT_LIVENESS> 935 void ShenandoahConcurrentMark::mark_loop_work(T* cl, jushort* live_data, uint worker_id, ParallelTaskTerminator *terminator) { 936 int seed = 17; 937 uintx stride = CANCELLABLE ? ShenandoahMarkLoopStride : 1; 938 939 ShenandoahHeap* heap = ShenandoahHeap::heap(); 940 ShenandoahObjToScanQueueSet* queues = task_queues(); 941 ShenandoahObjToScanQueue* q; 942 ShenandoahMarkTask t; 943 944 /* 945 * Process outstanding queues, if any. 946 * 947 * There can be more queues than workers. To deal with the imbalance, we claim 948 * extra queues first. Since marking can push new tasks into the queue associated 949 * with this worker id, we come back to process this queue in the normal loop. 950 */ 951 assert(queues->get_reserved() == heap->workers()->active_workers(), 952 "Need to reserve proper number of queues"); 953 954 q = queues->claim_next(); 955 while (q != NULL) { 956 if (CANCELLABLE && heap->cancelled_concgc()) { 957 ShenandoahCancelledTerminatorTerminator tt; 958 while (!terminator->offer_termination(&tt)); 959 return; 960 } 961 962 for (uint i = 0; i < stride; i++) { 963 if (try_queue(q, t)) { 964 do_task<T, COUNT_LIVENESS>(q, cl, live_data, &t); 965 } else { 966 assert(q->is_empty(), "Must be empty"); 967 q = queues->claim_next(); 968 break; 969 } 970 } 971 } 972 973 q = get_queue(worker_id); 974 975 /* 976 * Normal marking loop: 977 */ 978 while (true) { 979 if (CANCELLABLE && heap->cancelled_concgc()) { 980 ShenandoahCancelledTerminatorTerminator tt; 981 while (!terminator->offer_termination(&tt)); 982 return; 983 } 984 985 for (uint i = 0; i < stride; i++) { 986 if (try_queue(q, t) || 987 (DRAIN_SATB && try_draining_satb_buffer(q, t)) || 988 queues->steal(worker_id, &seed, t)) { 989 do_task<T, COUNT_LIVENESS>(q, cl, live_data, &t); 990 } else { 991 if (terminator->offer_termination()) return; 992 } 993 } 994 } 995 } 996 997 bool ShenandoahConcurrentMark::process_references() const { 998 return _heap->process_references(); 999 } 1000 1001 bool ShenandoahConcurrentMark::unload_classes() const { 1002 return _heap->unload_classes(); 1003 } 1004 1005 bool ShenandoahConcurrentMark::claim_codecache() { 1006 assert(ShenandoahConcurrentScanCodeRoots, "must not be called otherwise"); 1007 return _claimed_codecache.try_set(); 1008 } 1009 1010 void ShenandoahConcurrentMark::clear_claim_codecache() { 1011 assert(ShenandoahConcurrentScanCodeRoots, "must not be called otherwise"); 1012 _claimed_codecache.unset(); 1013 } 1014 1015 jushort* ShenandoahConcurrentMark::get_liveness(uint worker_id) { 1016 return _liveness_local[worker_id]; 1017 }