< prev index next >

src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp

Print this page
rev 50928 : Process remaining SATB buffers in final mark/traverse loop instead of separate phase


 196       }
 197     }
 198   }
 199 };
 200 
 201 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
 202 private:
 203   ParallelTaskTerminator* _terminator;
 204   ShenandoahHeap* _heap;
 205 public:
 206   ShenandoahConcurrentTraversalCollectionTask(ParallelTaskTerminator* terminator) :
 207     AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
 208     _terminator(terminator),
 209     _heap(ShenandoahHeap::heap()) {}
 210 
 211   void work(uint worker_id) {
 212     ShenandoahEvacOOMScope oom_evac_scope;
 213     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 214 
 215     // Drain all outstanding work in queues.
 216     traversal_gc->main_loop(worker_id, _terminator, true);
 217   }
 218 };
 219 
 220 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
 221 private:
 222   ShenandoahRootProcessor* _rp;
 223   ParallelTaskTerminator* _terminator;
 224   ShenandoahHeap* _heap;
 225 public:
 226   ShenandoahFinalTraversalCollectionTask(ShenandoahRootProcessor* rp, ParallelTaskTerminator* terminator) :
 227     AbstractGangTask("Shenandoah Final Traversal Collection"),
 228     _rp(rp),
 229     _terminator(terminator),
 230     _heap(ShenandoahHeap::heap()) {}
 231 
 232   void work(uint worker_id) {
 233     ShenandoahEvacOOMScope oom_evac_scope;
 234     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 235 
 236     ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
 237     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 238 
 239     bool process_refs = _heap->process_references();
 240     bool unload_classes = _heap->unload_classes();
 241     ReferenceProcessor* rp = NULL;
 242     if (process_refs) {
 243       rp = _heap->ref_processor();
 244     }
 245 
 246     // Step 1: Drain outstanding SATB queues.
 247     // NOTE: we piggy-back draining of remaining thread SATB buffers on the final root scan below.
 248     ShenandoahTraversalSATBBufferClosure satb_cl(q);
 249     {
 250       // Process remaining finished SATB buffers.
 251       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 252       while (satb_mq_set.apply_closure_to_completed_buffer(&satb_cl));
 253       // Process remaining threads SATB buffers below.
 254     }
 255 
 256     // Step 1: Process ordinary GC roots.

 257     if (!_heap->is_degenerated_gc_in_progress()) {
 258       ShenandoahTraversalClosure roots_cl(q, rp);
 259       CLDToOopClosure cld_cl(&roots_cl);
 260       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
 261       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 262       if (unload_classes) {
 263         ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
 264         _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, &code_cl, &tc, worker_id);
 265       } else {
 266         _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, &tc, worker_id);
 267       }
 268     } else {
 269       ShenandoahTraversalDegenClosure roots_cl(q, rp);
 270       CLDToOopClosure cld_cl(&roots_cl);
 271       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
 272       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 273       if (unload_classes) {
 274         ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
 275         _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, &code_cl, &tc, worker_id);
 276       } else {
 277         _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, &tc, worker_id);
 278       }
 279     }
 280 
 281     {
 282       ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
 283       ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
 284 
 285       // Step 3: Finally drain all outstanding work in queues.
 286       traversal_gc->main_loop(worker_id, _terminator, false);
 287     }
 288 
 289   }
 290 };
 291 
 292 void ShenandoahTraversalGC::flush_liveness(uint worker_id) {
 293   jushort* ld = get_liveness(worker_id);
 294   for (uint i = 0; i < _heap->num_regions(); i++) {
 295     ShenandoahHeapRegion* r = _heap->get_region(i);
 296     jushort live = ld[i];
 297     if (live > 0) {
 298       r->increase_live_data_gc_words(live);
 299       ld[i] = 0;
 300     }
 301   }
 302 }
 303 
 304 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
 305   _heap(heap),
 306   _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())),


 418         _heap->workers()->run_task(&traversal_task);
 419       } else {
 420         ParallelTaskTerminator terminator(nworkers, task_queues());
 421         ShenandoahInitTraversalCollectionTask traversal_task(&rp);
 422         _heap->workers()->run_task(&traversal_task);
 423       }
 424     }
 425 
 426 #if defined(COMPILER2) || INCLUDE_JVMCI
 427     DerivedPointerTable::update_pointers();
 428 #endif
 429   }
 430 
 431   if (ShenandoahPacing) {
 432     _heap->pacer()->setup_for_traversal();
 433   }
 434 
 435   _root_regions_iterator.reset(_root_regions);
 436 }
 437 
 438 void ShenandoahTraversalGC::main_loop(uint worker_id, ParallelTaskTerminator* terminator, bool do_satb) {
 439   if (do_satb) {
 440     main_loop_prework<true>(worker_id, terminator);
 441   } else {
 442     main_loop_prework<false>(worker_id, terminator);
 443   }
 444 }
 445 
 446 template <bool DO_SATB>
 447 void ShenandoahTraversalGC::main_loop_prework(uint w, ParallelTaskTerminator* t) {
 448   ShenandoahObjToScanQueue* q = task_queues()->queue(w);
 449 
 450   // Initialize live data.
 451   jushort* ld = get_liveness(w);
 452   Copy::fill_to_bytes(ld, _heap->num_regions() * sizeof(jushort));
 453 
 454   ReferenceProcessor* rp = NULL;
 455   if (_heap->process_references()) {
 456     rp = _heap->ref_processor();
 457   }
 458   if (UseShenandoahMatrix) {
 459     if (!_heap->is_degenerated_gc_in_progress()) {
 460       if (_heap->unload_classes()) {
 461         if (ShenandoahStringDedup::is_enabled()) {
 462           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 463           ShenandoahTraversalMetadataDedupMatrixClosure cl(q, rp, dq);
 464           main_loop_work<ShenandoahTraversalMetadataDedupMatrixClosure, DO_SATB>(&cl, ld, w, t);
 465         } else {
 466           ShenandoahTraversalMetadataMatrixClosure cl(q, rp);
 467           main_loop_work<ShenandoahTraversalMetadataMatrixClosure, DO_SATB>(&cl, ld, w, t);
 468         }
 469       } else {
 470         if (ShenandoahStringDedup::is_enabled()) {
 471           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 472           ShenandoahTraversalDedupMatrixClosure cl(q, rp, dq);
 473           main_loop_work<ShenandoahTraversalDedupMatrixClosure, DO_SATB>(&cl, ld, w, t);
 474         } else {
 475           ShenandoahTraversalMatrixClosure cl(q, rp);
 476           main_loop_work<ShenandoahTraversalMatrixClosure, DO_SATB>(&cl, ld, w, t);
 477         }
 478       }
 479     } else {
 480       if (_heap->unload_classes()) {
 481         if (ShenandoahStringDedup::is_enabled()) {
 482           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 483           ShenandoahTraversalMetadataDedupDegenMatrixClosure cl(q, rp, dq);
 484           main_loop_work<ShenandoahTraversalMetadataDedupDegenMatrixClosure, DO_SATB>(&cl, ld, w, t);
 485         } else {
 486           ShenandoahTraversalMetadataDegenMatrixClosure cl(q, rp);
 487           main_loop_work<ShenandoahTraversalMetadataDegenMatrixClosure, DO_SATB>(&cl, ld, w, t);
 488         }
 489       } else {
 490         if (ShenandoahStringDedup::is_enabled()) {
 491           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 492           ShenandoahTraversalDedupDegenMatrixClosure cl(q, rp, dq);
 493           main_loop_work<ShenandoahTraversalDedupDegenMatrixClosure, DO_SATB>(&cl, ld, w, t);
 494         } else {
 495           ShenandoahTraversalDegenMatrixClosure cl(q, rp);
 496           main_loop_work<ShenandoahTraversalDegenMatrixClosure, DO_SATB>(&cl, ld, w, t);
 497         }
 498       }
 499     }
 500   } else {
 501     if (!_heap->is_degenerated_gc_in_progress()) {
 502       if (_heap->unload_classes()) {
 503         if (ShenandoahStringDedup::is_enabled()) {
 504           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 505           ShenandoahTraversalMetadataDedupClosure cl(q, rp, dq);
 506           main_loop_work<ShenandoahTraversalMetadataDedupClosure, DO_SATB>(&cl, ld, w, t);
 507         } else {
 508           ShenandoahTraversalMetadataClosure cl(q, rp);
 509           main_loop_work<ShenandoahTraversalMetadataClosure, DO_SATB>(&cl, ld, w, t);
 510         }
 511       } else {
 512         if (ShenandoahStringDedup::is_enabled()) {
 513           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 514           ShenandoahTraversalDedupClosure cl(q, rp, dq);
 515           main_loop_work<ShenandoahTraversalDedupClosure, DO_SATB>(&cl, ld, w, t);
 516         } else {
 517           ShenandoahTraversalClosure cl(q, rp);
 518           main_loop_work<ShenandoahTraversalClosure, DO_SATB>(&cl, ld, w, t);
 519         }
 520       }
 521     } else {
 522       if (_heap->unload_classes()) {
 523         if (ShenandoahStringDedup::is_enabled()) {
 524           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 525           ShenandoahTraversalMetadataDedupDegenClosure cl(q, rp, dq);
 526           main_loop_work<ShenandoahTraversalMetadataDedupDegenClosure, DO_SATB>(&cl, ld, w, t);
 527         } else {
 528           ShenandoahTraversalMetadataDegenClosure cl(q, rp);
 529           main_loop_work<ShenandoahTraversalMetadataDegenClosure, DO_SATB>(&cl, ld, w, t);
 530         }
 531       } else {
 532         if (ShenandoahStringDedup::is_enabled()) {
 533           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 534           ShenandoahTraversalDedupDegenClosure cl(q, rp, dq);
 535           main_loop_work<ShenandoahTraversalDedupDegenClosure, DO_SATB>(&cl, ld, w, t);
 536         } else {
 537           ShenandoahTraversalDegenClosure cl(q, rp);
 538           main_loop_work<ShenandoahTraversalDegenClosure, DO_SATB>(&cl, ld, w, t);
 539         }
 540       }
 541     }
 542   }
 543   flush_liveness(w);
 544 
 545 }
 546 
 547 template <class T, bool DO_SATB>
 548 void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, ParallelTaskTerminator* terminator) {
 549   ShenandoahObjToScanQueueSet* queues = task_queues();
 550   ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 551   ShenandoahConcurrentMark* conc_mark = _heap->concurrentMark();
 552 
 553   uintx stride = ShenandoahMarkLoopStride;
 554 
 555   ShenandoahMarkTask task;
 556 
 557   // Process outstanding queues, if any.
 558   q = queues->claim_next();
 559   while (q != NULL) {
 560     if (_heap->check_cancelled_gc_and_yield()) {
 561       ShenandoahCancelledTerminatorTerminator tt;
 562       ShenandoahEvacOOMScopeLeaver oom_scope_leaver;
 563       while (!terminator->offer_termination(&tt));
 564       return;
 565     }
 566 
 567     for (uint i = 0; i < stride; i++) {


 587     if (ShenandoahPacing) {
 588       _heap->pacer()->report_partial(r->get_live_data_words());
 589     }
 590     if (check_and_handle_cancelled_gc(terminator)) return;
 591     r = _root_regions_iterator.claim_next();
 592   }
 593 
 594   if (check_and_handle_cancelled_gc(terminator)) return;
 595 
 596   // Normal loop.
 597   q = queues->queue(worker_id);
 598 
 599   ShenandoahTraversalSATBBufferClosure drain_satb(q);
 600   SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 601 
 602   int seed = 17;
 603 
 604   while (true) {
 605     if (check_and_handle_cancelled_gc(terminator)) return;
 606 
 607     if (DO_SATB) {
 608       while (satb_mq_set.completed_buffers_num() > 0) {
 609         satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
 610       }
 611     }
 612 
 613     if (_arraycopy_task_queue.length() > 0) {
 614       process_arraycopy_task(cl);
 615     }
 616 
 617     uint work = 0;
 618     for (uint i = 0; i < stride; i++) {
 619       if (q->pop_buffer(task) ||
 620           q->pop_local(task) ||
 621           q->pop_overflow(task) ||
 622           queues->steal(worker_id, &seed, task)) {
 623         conc_mark->do_task<T>(q, cl, live_data, &task);
 624         work++;
 625       } else {
 626         break;
 627       }
 628     }
 629 
 630     if (work == 0 &&
 631         _arraycopy_task_queue.length() == 0) {


 823 jushort* ShenandoahTraversalGC::get_liveness(uint worker_id) {
 824   return _liveness_local[worker_id];
 825 }
 826 
 827 class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure {
 828 private:
 829   ShenandoahHeap* const _heap;
 830 public:
 831   ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
 832   virtual bool should_return() { return _heap->cancelled_gc(); }
 833 };
 834 
 835 class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure {
 836 public:
 837   void do_void() {
 838     ShenandoahHeap* sh = ShenandoahHeap::heap();
 839     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 840     assert(sh->process_references(), "why else would we be here?");
 841     ParallelTaskTerminator terminator(1, traversal_gc->task_queues());
 842     shenandoah_assert_rp_isalive_installed();
 843     traversal_gc->main_loop((uint) 0, &terminator, false);
 844   }
 845 };
 846 
 847 class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure {
 848 private:
 849   ShenandoahObjToScanQueue* _queue;
 850   Thread* _thread;
 851   ShenandoahTraversalGC* _traversal_gc;
 852   template <class T>
 853   inline void do_oop_nv(T* p) {
 854     _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, false /* matrix */>(p, _thread, _queue, NULL);
 855   }
 856 
 857 public:
 858   ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 859     _queue(q), _thread(Thread::current()),
 860     _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {}
 861 
 862   void do_oop(narrowOop* p) { do_oop_nv(p); }
 863   void do_oop(oop* p)       { do_oop_nv(p); }


 973 class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure {
 974   uint _worker_id;
 975   ParallelTaskTerminator* _terminator;
 976   bool _reset_terminator;
 977 
 978 public:
 979   ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, ParallelTaskTerminator* t, bool reset_terminator = false):
 980     _worker_id(worker_id),
 981     _terminator(t),
 982     _reset_terminator(reset_terminator) {
 983   }
 984 
 985   void do_void() {
 986     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 987 
 988     ShenandoahHeap* sh = ShenandoahHeap::heap();
 989     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 990     assert(sh->process_references(), "why else would we be here?");
 991     shenandoah_assert_rp_isalive_installed();
 992 
 993     traversal_gc->main_loop(_worker_id, _terminator, false);
 994 
 995     if (_reset_terminator) {
 996       _terminator->reset_for_reuse();
 997     }
 998   }
 999 };
1000 
1001 void ShenandoahTraversalGC::weak_refs_work() {
1002   assert(_heap->process_references(), "sanity");
1003 
1004   ShenandoahHeap* sh = ShenandoahHeap::heap();
1005 
1006   ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs;
1007 
1008   ShenandoahGCPhase phase(phase_root);
1009 
1010   ReferenceProcessor* rp = sh->ref_processor();
1011 
1012   // NOTE: We cannot shortcut on has_discovered_references() here, because
1013   // we will miss marking JNI Weak refs then, see implementation in




 196       }
 197     }
 198   }
 199 };
 200 
 201 class ShenandoahConcurrentTraversalCollectionTask : public AbstractGangTask {
 202 private:
 203   ParallelTaskTerminator* _terminator;
 204   ShenandoahHeap* _heap;
 205 public:
 206   ShenandoahConcurrentTraversalCollectionTask(ParallelTaskTerminator* terminator) :
 207     AbstractGangTask("Shenandoah Concurrent Traversal Collection"),
 208     _terminator(terminator),
 209     _heap(ShenandoahHeap::heap()) {}
 210 
 211   void work(uint worker_id) {
 212     ShenandoahEvacOOMScope oom_evac_scope;
 213     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 214 
 215     // Drain all outstanding work in queues.
 216     traversal_gc->main_loop(worker_id, _terminator);
 217   }
 218 };
 219 
 220 class ShenandoahFinalTraversalCollectionTask : public AbstractGangTask {
 221 private:
 222   ShenandoahRootProcessor* _rp;
 223   ParallelTaskTerminator* _terminator;
 224   ShenandoahHeap* _heap;
 225 public:
 226   ShenandoahFinalTraversalCollectionTask(ShenandoahRootProcessor* rp, ParallelTaskTerminator* terminator) :
 227     AbstractGangTask("Shenandoah Final Traversal Collection"),
 228     _rp(rp),
 229     _terminator(terminator),
 230     _heap(ShenandoahHeap::heap()) {}
 231 
 232   void work(uint worker_id) {
 233     ShenandoahEvacOOMScope oom_evac_scope;
 234     ShenandoahTraversalGC* traversal_gc = _heap->traversal_gc();
 235 
 236     ShenandoahObjToScanQueueSet* queues = traversal_gc->task_queues();
 237     ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 238 
 239     bool process_refs = _heap->process_references();
 240     bool unload_classes = _heap->unload_classes();
 241     ReferenceProcessor* rp = NULL;
 242     if (process_refs) {
 243       rp = _heap->ref_processor();
 244     }
 245 










 246     // Step 1: Process ordinary GC roots.
 247     ShenandoahTraversalSATBBufferClosure satb_cl(q);
 248     if (!_heap->is_degenerated_gc_in_progress()) {
 249       ShenandoahTraversalClosure roots_cl(q, rp);
 250       CLDToOopClosure cld_cl(&roots_cl);
 251       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
 252       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 253       if (unload_classes) {
 254         ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
 255         _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, &code_cl, &tc, worker_id);
 256       } else {
 257         _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, &tc, worker_id);
 258       }
 259     } else {
 260       ShenandoahTraversalDegenClosure roots_cl(q, rp);
 261       CLDToOopClosure cld_cl(&roots_cl);
 262       MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations);
 263       ShenandoahTraversalSATBThreadsClosure tc(&satb_cl);
 264       if (unload_classes) {
 265         ShenandoahRemarkCLDClosure weak_cld_cl(&roots_cl);
 266         _rp->process_strong_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &weak_cld_cl, &code_cl, &tc, worker_id);
 267       } else {
 268         _rp->process_all_roots(&roots_cl, process_refs ? NULL : &roots_cl, &cld_cl, &code_cl, &tc, worker_id);
 269       }
 270     }
 271 
 272     {
 273       ShenandoahWorkerTimings *worker_times = _heap->phase_timings()->worker_times();
 274       ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::FinishQueues, worker_id);
 275 
 276       // Step 3: Finally drain all outstanding work in queues.
 277       traversal_gc->main_loop(worker_id, _terminator);
 278     }
 279 
 280   }
 281 };
 282 
 283 void ShenandoahTraversalGC::flush_liveness(uint worker_id) {
 284   jushort* ld = get_liveness(worker_id);
 285   for (uint i = 0; i < _heap->num_regions(); i++) {
 286     ShenandoahHeapRegion* r = _heap->get_region(i);
 287     jushort live = ld[i];
 288     if (live > 0) {
 289       r->increase_live_data_gc_words(live);
 290       ld[i] = 0;
 291     }
 292   }
 293 }
 294 
 295 ShenandoahTraversalGC::ShenandoahTraversalGC(ShenandoahHeap* heap, size_t num_regions) :
 296   _heap(heap),
 297   _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())),


 409         _heap->workers()->run_task(&traversal_task);
 410       } else {
 411         ParallelTaskTerminator terminator(nworkers, task_queues());
 412         ShenandoahInitTraversalCollectionTask traversal_task(&rp);
 413         _heap->workers()->run_task(&traversal_task);
 414       }
 415     }
 416 
 417 #if defined(COMPILER2) || INCLUDE_JVMCI
 418     DerivedPointerTable::update_pointers();
 419 #endif
 420   }
 421 
 422   if (ShenandoahPacing) {
 423     _heap->pacer()->setup_for_traversal();
 424   }
 425 
 426   _root_regions_iterator.reset(_root_regions);
 427 }
 428 
 429 void ShenandoahTraversalGC::main_loop(uint w, ParallelTaskTerminator* t) {









 430   ShenandoahObjToScanQueue* q = task_queues()->queue(w);
 431 
 432   // Initialize live data.
 433   jushort* ld = get_liveness(w);
 434   Copy::fill_to_bytes(ld, _heap->num_regions() * sizeof(jushort));
 435 
 436   ReferenceProcessor* rp = NULL;
 437   if (_heap->process_references()) {
 438     rp = _heap->ref_processor();
 439   }
 440   if (UseShenandoahMatrix) {
 441     if (!_heap->is_degenerated_gc_in_progress()) {
 442       if (_heap->unload_classes()) {
 443         if (ShenandoahStringDedup::is_enabled()) {
 444           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 445           ShenandoahTraversalMetadataDedupMatrixClosure cl(q, rp, dq);
 446           main_loop_work<ShenandoahTraversalMetadataDedupMatrixClosure>(&cl, ld, w, t);
 447         } else {
 448           ShenandoahTraversalMetadataMatrixClosure cl(q, rp);
 449           main_loop_work<ShenandoahTraversalMetadataMatrixClosure>(&cl, ld, w, t);
 450         }
 451       } else {
 452         if (ShenandoahStringDedup::is_enabled()) {
 453           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 454           ShenandoahTraversalDedupMatrixClosure cl(q, rp, dq);
 455           main_loop_work<ShenandoahTraversalDedupMatrixClosure>(&cl, ld, w, t);
 456         } else {
 457           ShenandoahTraversalMatrixClosure cl(q, rp);
 458           main_loop_work<ShenandoahTraversalMatrixClosure>(&cl, ld, w, t);
 459         }
 460       }
 461     } else {
 462       if (_heap->unload_classes()) {
 463         if (ShenandoahStringDedup::is_enabled()) {
 464           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 465           ShenandoahTraversalMetadataDedupDegenMatrixClosure cl(q, rp, dq);
 466           main_loop_work<ShenandoahTraversalMetadataDedupDegenMatrixClosure>(&cl, ld, w, t);
 467         } else {
 468           ShenandoahTraversalMetadataDegenMatrixClosure cl(q, rp);
 469           main_loop_work<ShenandoahTraversalMetadataDegenMatrixClosure>(&cl, ld, w, t);
 470         }
 471       } else {
 472         if (ShenandoahStringDedup::is_enabled()) {
 473           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 474           ShenandoahTraversalDedupDegenMatrixClosure cl(q, rp, dq);
 475           main_loop_work<ShenandoahTraversalDedupDegenMatrixClosure>(&cl, ld, w, t);
 476         } else {
 477           ShenandoahTraversalDegenMatrixClosure cl(q, rp);
 478           main_loop_work<ShenandoahTraversalDegenMatrixClosure>(&cl, ld, w, t);
 479         }
 480       }
 481     }
 482   } else {
 483     if (!_heap->is_degenerated_gc_in_progress()) {
 484       if (_heap->unload_classes()) {
 485         if (ShenandoahStringDedup::is_enabled()) {
 486           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 487           ShenandoahTraversalMetadataDedupClosure cl(q, rp, dq);
 488           main_loop_work<ShenandoahTraversalMetadataDedupClosure>(&cl, ld, w, t);
 489         } else {
 490           ShenandoahTraversalMetadataClosure cl(q, rp);
 491           main_loop_work<ShenandoahTraversalMetadataClosure>(&cl, ld, w, t);
 492         }
 493       } else {
 494         if (ShenandoahStringDedup::is_enabled()) {
 495           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 496           ShenandoahTraversalDedupClosure cl(q, rp, dq);
 497           main_loop_work<ShenandoahTraversalDedupClosure>(&cl, ld, w, t);
 498         } else {
 499           ShenandoahTraversalClosure cl(q, rp);
 500           main_loop_work<ShenandoahTraversalClosure>(&cl, ld, w, t);
 501         }
 502       }
 503     } else {
 504       if (_heap->unload_classes()) {
 505         if (ShenandoahStringDedup::is_enabled()) {
 506           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 507           ShenandoahTraversalMetadataDedupDegenClosure cl(q, rp, dq);
 508           main_loop_work<ShenandoahTraversalMetadataDedupDegenClosure>(&cl, ld, w, t);
 509         } else {
 510           ShenandoahTraversalMetadataDegenClosure cl(q, rp);
 511           main_loop_work<ShenandoahTraversalMetadataDegenClosure>(&cl, ld, w, t);
 512         }
 513       } else {
 514         if (ShenandoahStringDedup::is_enabled()) {
 515           ShenandoahStrDedupQueue* dq = ShenandoahStringDedup::queue(w);
 516           ShenandoahTraversalDedupDegenClosure cl(q, rp, dq);
 517           main_loop_work<ShenandoahTraversalDedupDegenClosure>(&cl, ld, w, t);
 518         } else {
 519           ShenandoahTraversalDegenClosure cl(q, rp);
 520           main_loop_work<ShenandoahTraversalDegenClosure>(&cl, ld, w, t);
 521         }
 522       }
 523     }
 524   }
 525   flush_liveness(w);
 526 
 527 }
 528 
 529 template <class T>
 530 void ShenandoahTraversalGC::main_loop_work(T* cl, jushort* live_data, uint worker_id, ParallelTaskTerminator* terminator) {
 531   ShenandoahObjToScanQueueSet* queues = task_queues();
 532   ShenandoahObjToScanQueue* q = queues->queue(worker_id);
 533   ShenandoahConcurrentMark* conc_mark = _heap->concurrentMark();
 534 
 535   uintx stride = ShenandoahMarkLoopStride;
 536 
 537   ShenandoahMarkTask task;
 538 
 539   // Process outstanding queues, if any.
 540   q = queues->claim_next();
 541   while (q != NULL) {
 542     if (_heap->check_cancelled_gc_and_yield()) {
 543       ShenandoahCancelledTerminatorTerminator tt;
 544       ShenandoahEvacOOMScopeLeaver oom_scope_leaver;
 545       while (!terminator->offer_termination(&tt));
 546       return;
 547     }
 548 
 549     for (uint i = 0; i < stride; i++) {


 569     if (ShenandoahPacing) {
 570       _heap->pacer()->report_partial(r->get_live_data_words());
 571     }
 572     if (check_and_handle_cancelled_gc(terminator)) return;
 573     r = _root_regions_iterator.claim_next();
 574   }
 575 
 576   if (check_and_handle_cancelled_gc(terminator)) return;
 577 
 578   // Normal loop.
 579   q = queues->queue(worker_id);
 580 
 581   ShenandoahTraversalSATBBufferClosure drain_satb(q);
 582   SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
 583 
 584   int seed = 17;
 585 
 586   while (true) {
 587     if (check_and_handle_cancelled_gc(terminator)) return;
 588 

 589     while (satb_mq_set.completed_buffers_num() > 0) {
 590       satb_mq_set.apply_closure_to_completed_buffer(&drain_satb);
 591     }

 592 
 593     if (_arraycopy_task_queue.length() > 0) {
 594       process_arraycopy_task(cl);
 595     }
 596 
 597     uint work = 0;
 598     for (uint i = 0; i < stride; i++) {
 599       if (q->pop_buffer(task) ||
 600           q->pop_local(task) ||
 601           q->pop_overflow(task) ||
 602           queues->steal(worker_id, &seed, task)) {
 603         conc_mark->do_task<T>(q, cl, live_data, &task);
 604         work++;
 605       } else {
 606         break;
 607       }
 608     }
 609 
 610     if (work == 0 &&
 611         _arraycopy_task_queue.length() == 0) {


 803 jushort* ShenandoahTraversalGC::get_liveness(uint worker_id) {
 804   return _liveness_local[worker_id];
 805 }
 806 
 807 class ShenandoahTraversalCancelledGCYieldClosure : public YieldClosure {
 808 private:
 809   ShenandoahHeap* const _heap;
 810 public:
 811   ShenandoahTraversalCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
 812   virtual bool should_return() { return _heap->cancelled_gc(); }
 813 };
 814 
 815 class ShenandoahTraversalPrecleanCompleteGCClosure : public VoidClosure {
 816 public:
 817   void do_void() {
 818     ShenandoahHeap* sh = ShenandoahHeap::heap();
 819     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 820     assert(sh->process_references(), "why else would we be here?");
 821     ParallelTaskTerminator terminator(1, traversal_gc->task_queues());
 822     shenandoah_assert_rp_isalive_installed();
 823     traversal_gc->main_loop((uint) 0, &terminator);
 824   }
 825 };
 826 
 827 class ShenandoahTraversalKeepAliveUpdateClosure : public OopClosure {
 828 private:
 829   ShenandoahObjToScanQueue* _queue;
 830   Thread* _thread;
 831   ShenandoahTraversalGC* _traversal_gc;
 832   template <class T>
 833   inline void do_oop_nv(T* p) {
 834     _traversal_gc->process_oop<T, false /* string dedup */, false /* degen */, false /* matrix */>(p, _thread, _queue, NULL);
 835   }
 836 
 837 public:
 838   ShenandoahTraversalKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
 839     _queue(q), _thread(Thread::current()),
 840     _traversal_gc(ShenandoahHeap::heap()->traversal_gc()) {}
 841 
 842   void do_oop(narrowOop* p) { do_oop_nv(p); }
 843   void do_oop(oop* p)       { do_oop_nv(p); }


 953 class ShenandoahTraversalDrainMarkingStackClosure: public VoidClosure {
 954   uint _worker_id;
 955   ParallelTaskTerminator* _terminator;
 956   bool _reset_terminator;
 957 
 958 public:
 959   ShenandoahTraversalDrainMarkingStackClosure(uint worker_id, ParallelTaskTerminator* t, bool reset_terminator = false):
 960     _worker_id(worker_id),
 961     _terminator(t),
 962     _reset_terminator(reset_terminator) {
 963   }
 964 
 965   void do_void() {
 966     assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
 967 
 968     ShenandoahHeap* sh = ShenandoahHeap::heap();
 969     ShenandoahTraversalGC* traversal_gc = sh->traversal_gc();
 970     assert(sh->process_references(), "why else would we be here?");
 971     shenandoah_assert_rp_isalive_installed();
 972 
 973     traversal_gc->main_loop(_worker_id, _terminator);
 974 
 975     if (_reset_terminator) {
 976       _terminator->reset_for_reuse();
 977     }
 978   }
 979 };
 980 
 981 void ShenandoahTraversalGC::weak_refs_work() {
 982   assert(_heap->process_references(), "sanity");
 983 
 984   ShenandoahHeap* sh = ShenandoahHeap::heap();
 985 
 986   ShenandoahPhaseTimings::Phase phase_root = ShenandoahPhaseTimings::weakrefs;
 987 
 988   ShenandoahGCPhase phase(phase_root);
 989 
 990   ReferenceProcessor* rp = sh->ref_processor();
 991 
 992   // NOTE: We cannot shortcut on has_discovered_references() here, because
 993   // we will miss marking JNI Weak refs then, see implementation in


< prev index next >