269 workers->run_task(&cl, num_workers);
270
271 #ifndef PRODUCT
272 // Need to synchronize with concurrent cleanup since it needs to
273 // finish its card table clearing before we can verify.
274 G1CollectedHeap::heap()->wait_while_free_regions_coming();
275 G1CollectedHeap::heap()->verifier()->verify_card_table_cleanup();
276 #endif
277 }
278 };
279
280 G1RemSet::G1RemSet(G1CollectedHeap* g1,
281 CardTableModRefBS* ct_bs,
282 G1HotCardCache* hot_card_cache) :
283 _g1(g1),
284 _scan_state(new G1RemSetScanState()),
285 _conc_refine_cards(0),
286 _ct_bs(ct_bs),
287 _g1p(_g1->g1_policy()),
288 _hot_card_cache(hot_card_cache),
289 _prev_period_summary(),
290 _into_cset_dirty_card_queue_set(false)
291 {
292 if (log_is_enabled(Trace, gc, remset)) {
293 _prev_period_summary.initialize(this);
294 }
295 // Initialize the card queue set used to hold cards containing
296 // references into the collection set.
297 _into_cset_dirty_card_queue_set.initialize(NULL, // Should never be called by the Java code
298 DirtyCardQ_CBL_mon,
299 DirtyCardQ_FL_lock,
300 -1, // never trigger processing
301 -1, // no limit on length
302 Shared_DirtyCardQ_lock,
303 &JavaThread::dirty_card_queue_set());
304 }
305
306 G1RemSet::~G1RemSet() {
307 if (_scan_state != NULL) {
308 delete _scan_state;
309 }
310 }
311
312 uint G1RemSet::num_par_rem_sets() {
313 return MAX2(DirtyCardQueueSet::num_par_ids() + ConcurrentG1Refine::thread_num(), ParallelGCThreads);
314 }
315
316 void G1RemSet::initialize(size_t capacity, uint max_regions) {
317 G1FromCardCache::initialize(num_par_rem_sets(), max_regions);
318 _scan_state->initialize(max_regions);
319 {
320 GCTraceTime(Debug, gc, marking)("Initialize Card Live Data");
321 _card_live_data.initialize(capacity, max_regions);
322 }
323 if (G1PretouchAuxiliaryMemory) {
418 uint worker_i) {
419 double rs_time_start = os::elapsedTime();
420
421 G1ScanObjsDuringScanRSClosure scan_cl(_g1, pss);
422 G1ScanRSForRegionClosure cl(_scan_state, &scan_cl, heap_region_codeblobs, worker_i);
423 _g1->collection_set_iterate_from(&cl, worker_i);
424
425 double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) -
426 cl.strong_code_root_scan_time_sec();
427
428 G1GCPhaseTimes* p = _g1p->phase_times();
429
430 p->record_time_secs(G1GCPhaseTimes::ScanRS, worker_i, scan_rs_time_sec);
431 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_scanned(), G1GCPhaseTimes::ScannedCards);
432 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_claimed(), G1GCPhaseTimes::ClaimedCards);
433 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_skipped(), G1GCPhaseTimes::SkippedCards);
434
435 p->record_time_secs(G1GCPhaseTimes::CodeRoots, worker_i, cl.strong_code_root_scan_time_sec());
436 }
437
438 // Closure used for updating RSets and recording references that
439 // point into the collection set. Only called during an
440 // evacuation pause.
441 class G1RefineCardClosure: public CardTableEntryClosure {
442 G1RemSet* _g1rs;
443 DirtyCardQueue* _into_cset_dcq;
444 G1ScanObjsDuringUpdateRSClosure* _update_rs_cl;
445 public:
446 G1RefineCardClosure(G1CollectedHeap* g1h,
447 DirtyCardQueue* into_cset_dcq,
448 G1ScanObjsDuringUpdateRSClosure* update_rs_cl) :
449 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq), _update_rs_cl(update_rs_cl)
450 {}
451
452 bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
453 // The only time we care about recording cards that
454 // contain references that point into the collection set
455 // is during RSet updating within an evacuation pause.
456 // In this case worker_i should be the id of a GC worker thread.
457 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
458
459 if (_g1rs->refine_card_during_gc(card_ptr, _update_rs_cl)) {
460 // 'card_ptr' contains references that point into the collection
461 // set. We need to record the card in the DCQS
462 // (_into_cset_dirty_card_queue_set)
463 // that's used for that purpose.
464 //
465 // Enqueue the card
466 _into_cset_dcq->enqueue(card_ptr);
467 }
468 return true;
469 }
470 };
471
472 void G1RemSet::update_rem_set(DirtyCardQueue* into_cset_dcq,
473 G1ParScanThreadState* pss,
474 uint worker_i) {
475 G1ScanObjsDuringUpdateRSClosure update_rs_cl(_g1, pss, worker_i);
476 G1RefineCardClosure refine_card_cl(_g1, into_cset_dcq, &update_rs_cl);
477
478 G1GCParPhaseTimesTracker x(_g1p->phase_times(), G1GCPhaseTimes::UpdateRS, worker_i);
479 if (G1HotCardCache::default_use_cache()) {
480 // Apply the closure to the entries of the hot card cache.
481 G1GCParPhaseTimesTracker y(_g1p->phase_times(), G1GCPhaseTimes::ScanHCC, worker_i);
482 _g1->iterate_hcc_closure(&refine_card_cl, worker_i);
483 }
484 // Apply the closure to all remaining log entries.
485 _g1->iterate_dirty_card_closure(&refine_card_cl, worker_i);
486 }
487
488 void G1RemSet::cleanupHRRS() {
489 HeapRegionRemSet::cleanup();
490 }
491
492 void G1RemSet::oops_into_collection_set_do(G1ParScanThreadState* pss,
493 CodeBlobClosure* heap_region_codeblobs,
494 uint worker_i) {
495 // A DirtyCardQueue that is used to hold cards containing references
496 // that point into the collection set. This DCQ is associated with a
497 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
498 // circumstances (i.e. the pause successfully completes), these cards
499 // are just discarded (there's no need to update the RSets of regions
500 // that were in the collection set - after the pause these regions
501 // are wholly 'free' of live objects. In the event of an evacuation
502 // failure the cards/buffers in this queue set are passed to the
503 // DirtyCardQueueSet that is used to manage RSet updates
504 DirtyCardQueue into_cset_dcq(&_into_cset_dirty_card_queue_set);
505
506 update_rem_set(&into_cset_dcq, pss, worker_i);
507 scan_rem_set(pss, heap_region_codeblobs, worker_i);;
508 }
509
510 void G1RemSet::prepare_for_oops_into_collection_set_do() {
511 _g1->set_refine_cte_cl_concurrency(false);
512 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
513 dcqs.concatenate_logs();
514
515 _scan_state->reset();
516 }
517
518 void G1RemSet::cleanup_after_oops_into_collection_set_do() {
519 G1GCPhaseTimes* phase_times = _g1->g1_policy()->phase_times();
520 // Cleanup after copy
521 _g1->set_refine_cte_cl_concurrency(true);
522
523 // Set all cards back to clean.
524 double start = os::elapsedTime();
525 _scan_state->clear_card_table(_g1->workers());
526 phase_times->record_clear_ct_time((os::elapsedTime() - start) * 1000.0);
527
528 DirtyCardQueueSet& into_cset_dcqs = _into_cset_dirty_card_queue_set;
529
530 if (_g1->evacuation_failed()) {
531 double restore_remembered_set_start = os::elapsedTime();
532
533 // Restore remembered sets for the regions pointing into the collection set.
534 // We just need to transfer the completed buffers from the DirtyCardQueueSet
535 // used to hold cards that contain references that point into the collection set
536 // to the DCQS used to hold the deferred RS updates.
537 _g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs);
538 phase_times->record_evac_fail_restore_remsets((os::elapsedTime() - restore_remembered_set_start) * 1000.0);
539 }
540
541 // Free any completed buffers in the DirtyCardQueueSet used to hold cards
542 // which contain references that point into the collection.
543 _into_cset_dirty_card_queue_set.clear();
544 assert(_into_cset_dirty_card_queue_set.completed_buffers_num() == 0,
545 "all buffers should be freed");
546 _into_cset_dirty_card_queue_set.clear_n_completed_buffers();
547 }
548
549 class G1ScrubRSClosure: public HeapRegionClosure {
550 G1CollectedHeap* _g1h;
551 G1CardLiveData* _live_data;
552 public:
553 G1ScrubRSClosure(G1CardLiveData* live_data) :
554 _g1h(G1CollectedHeap::heap()),
555 _live_data(live_data) { }
556
557 bool doHeapRegion(HeapRegion* r) {
558 if (!r->is_continues_humongous()) {
559 r->rem_set()->scrub(_live_data);
560 }
561 return false;
562 }
563 };
564
565 void G1RemSet::scrub(uint worker_num, HeapRegionClaimer *hrclaimer) {
566 G1ScrubRSClosure scrub_cl(&_card_live_data);
711 // part of the heap (e.g. a partially allocated object) while
712 // processing a stale card. Despite the card being stale, redirty
713 // and re-enqueue, because we've already cleaned the card. Without
714 // this we could incorrectly discard a non-stale card.
715 if (!card_processed) {
716 // The card might have gotten re-dirtied and re-enqueued while we
717 // worked. (In fact, it's pretty likely.)
718 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
719 *card_ptr = CardTableModRefBS::dirty_card_val();
720 MutexLockerEx x(Shared_DirtyCardQ_lock,
721 Mutex::_no_safepoint_check_flag);
722 DirtyCardQueue* sdcq =
723 JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
724 sdcq->enqueue(card_ptr);
725 }
726 } else {
727 _conc_refine_cards++;
728 }
729 }
730
731 bool G1RemSet::refine_card_during_gc(jbyte* card_ptr,
732 G1ScanObjsDuringUpdateRSClosure* update_rs_cl) {
733 assert(_g1->is_gc_active(), "Only call during GC");
734
735 check_card_ptr(card_ptr, _ct_bs);
736
737 // If the card is no longer dirty, nothing to do. This covers cards that were already
738 // scanned as parts of the remembered sets.
739 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
740 // No need to return that this card contains refs that point
741 // into the collection set.
742 return false;
743 }
744
745 // During GC we can immediately clean the card since we will not re-enqueue stale
746 // cards as we know they can be disregarded.
747 *card_ptr = CardTableModRefBS::clean_card_val();
748
749 // Construct the region representing the card.
750 HeapWord* card_start = _ct_bs->addr_for(card_ptr);
751 // And find the region containing it.
752 HeapRegion* r = _g1->heap_region_containing(card_start);
753
754 HeapWord* scan_limit = _scan_state->scan_top(r->hrm_index());
755 if (scan_limit <= card_start) {
756 // If the card starts above the area in the region containing objects to scan, skip it.
757 return false;
758 }
759
760 // Don't use addr_for(card_ptr + 1) which can ask for
761 // a card beyond the heap.
762 HeapWord* card_end = card_start + CardTableModRefBS::card_size_in_words;
763 MemRegion dirty_region(card_start, MIN2(scan_limit, card_end));
764 assert(!dirty_region.is_empty(), "sanity");
765
766 update_rs_cl->set_region(r);
767 update_rs_cl->reset_has_refs_into_cset();
768
769 bool card_processed = r->oops_on_card_seq_iterate_careful<true>(dirty_region, update_rs_cl);
770 assert(card_processed, "must be");
771 _conc_refine_cards++;
772
773 return update_rs_cl->has_refs_into_cset();
774 }
775
776 void G1RemSet::print_periodic_summary_info(const char* header, uint period_count) {
777 if ((G1SummarizeRSetStatsPeriod > 0) && log_is_enabled(Trace, gc, remset) &&
778 (period_count % G1SummarizeRSetStatsPeriod == 0)) {
779
780 if (!_prev_period_summary.initialized()) {
781 _prev_period_summary.initialize(this);
782 }
783
784 G1RemSetSummary current;
785 current.initialize(this);
786 _prev_period_summary.subtract_from(¤t);
787
788 Log(gc, remset) log;
789 log.trace("%s", header);
790 ResourceMark rm;
791 _prev_period_summary.print_on(log.trace_stream());
792
793 _prev_period_summary.set(¤t);
|
269 workers->run_task(&cl, num_workers);
270
271 #ifndef PRODUCT
272 // Need to synchronize with concurrent cleanup since it needs to
273 // finish its card table clearing before we can verify.
274 G1CollectedHeap::heap()->wait_while_free_regions_coming();
275 G1CollectedHeap::heap()->verifier()->verify_card_table_cleanup();
276 #endif
277 }
278 };
279
280 G1RemSet::G1RemSet(G1CollectedHeap* g1,
281 CardTableModRefBS* ct_bs,
282 G1HotCardCache* hot_card_cache) :
283 _g1(g1),
284 _scan_state(new G1RemSetScanState()),
285 _conc_refine_cards(0),
286 _ct_bs(ct_bs),
287 _g1p(_g1->g1_policy()),
288 _hot_card_cache(hot_card_cache),
289 _prev_period_summary()
290 {
291 if (log_is_enabled(Trace, gc, remset)) {
292 _prev_period_summary.initialize(this);
293 }
294 }
295
296 G1RemSet::~G1RemSet() {
297 if (_scan_state != NULL) {
298 delete _scan_state;
299 }
300 }
301
302 uint G1RemSet::num_par_rem_sets() {
303 return MAX2(DirtyCardQueueSet::num_par_ids() + ConcurrentG1Refine::thread_num(), ParallelGCThreads);
304 }
305
306 void G1RemSet::initialize(size_t capacity, uint max_regions) {
307 G1FromCardCache::initialize(num_par_rem_sets(), max_regions);
308 _scan_state->initialize(max_regions);
309 {
310 GCTraceTime(Debug, gc, marking)("Initialize Card Live Data");
311 _card_live_data.initialize(capacity, max_regions);
312 }
313 if (G1PretouchAuxiliaryMemory) {
408 uint worker_i) {
409 double rs_time_start = os::elapsedTime();
410
411 G1ScanObjsDuringScanRSClosure scan_cl(_g1, pss);
412 G1ScanRSForRegionClosure cl(_scan_state, &scan_cl, heap_region_codeblobs, worker_i);
413 _g1->collection_set_iterate_from(&cl, worker_i);
414
415 double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) -
416 cl.strong_code_root_scan_time_sec();
417
418 G1GCPhaseTimes* p = _g1p->phase_times();
419
420 p->record_time_secs(G1GCPhaseTimes::ScanRS, worker_i, scan_rs_time_sec);
421 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_scanned(), G1GCPhaseTimes::ScannedCards);
422 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_claimed(), G1GCPhaseTimes::ClaimedCards);
423 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_skipped(), G1GCPhaseTimes::SkippedCards);
424
425 p->record_time_secs(G1GCPhaseTimes::CodeRoots, worker_i, cl.strong_code_root_scan_time_sec());
426 }
427
428 // Closure used for updating rem sets. Only called during an evacuation pause.
429 class G1RefineCardClosure: public CardTableEntryClosure {
430 G1RemSet* _g1rs;
431 G1ScanObjsDuringUpdateRSClosure* _update_rs_cl;
432 public:
433 G1RefineCardClosure(G1CollectedHeap* g1h, G1ScanObjsDuringUpdateRSClosure* update_rs_cl) :
434 _g1rs(g1h->g1_rem_set()), _update_rs_cl(update_rs_cl)
435 {}
436
437 bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
438 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
439
440 _g1rs->refine_card_during_gc(card_ptr, _update_rs_cl);
441 return true;
442 }
443 };
444
445 void G1RemSet::update_rem_set(G1ParScanThreadState* pss, uint worker_i) {
446 G1ScanObjsDuringUpdateRSClosure update_rs_cl(_g1, pss, worker_i);
447 G1RefineCardClosure refine_card_cl(_g1, &update_rs_cl);
448
449 G1GCParPhaseTimesTracker x(_g1p->phase_times(), G1GCPhaseTimes::UpdateRS, worker_i);
450 if (G1HotCardCache::default_use_cache()) {
451 // Apply the closure to the entries of the hot card cache.
452 G1GCParPhaseTimesTracker y(_g1p->phase_times(), G1GCPhaseTimes::ScanHCC, worker_i);
453 _g1->iterate_hcc_closure(&refine_card_cl, worker_i);
454 }
455 // Apply the closure to all remaining log entries.
456 _g1->iterate_dirty_card_closure(&refine_card_cl, worker_i);
457 }
458
459 void G1RemSet::cleanupHRRS() {
460 HeapRegionRemSet::cleanup();
461 }
462
463 void G1RemSet::oops_into_collection_set_do(G1ParScanThreadState* pss,
464 CodeBlobClosure* heap_region_codeblobs,
465 uint worker_i) {
466 update_rem_set(pss, worker_i);
467 scan_rem_set(pss, heap_region_codeblobs, worker_i);;
468 }
469
470 void G1RemSet::prepare_for_oops_into_collection_set_do() {
471 _g1->set_refine_cte_cl_concurrency(false);
472 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
473 dcqs.concatenate_logs();
474
475 _scan_state->reset();
476 }
477
478 void G1RemSet::cleanup_after_oops_into_collection_set_do() {
479 G1GCPhaseTimes* phase_times = _g1->g1_policy()->phase_times();
480 // Cleanup after copy
481 _g1->set_refine_cte_cl_concurrency(true);
482
483 // Set all cards back to clean.
484 double start = os::elapsedTime();
485 _scan_state->clear_card_table(_g1->workers());
486 phase_times->record_clear_ct_time((os::elapsedTime() - start) * 1000.0);
487 }
488
489 class G1ScrubRSClosure: public HeapRegionClosure {
490 G1CollectedHeap* _g1h;
491 G1CardLiveData* _live_data;
492 public:
493 G1ScrubRSClosure(G1CardLiveData* live_data) :
494 _g1h(G1CollectedHeap::heap()),
495 _live_data(live_data) { }
496
497 bool doHeapRegion(HeapRegion* r) {
498 if (!r->is_continues_humongous()) {
499 r->rem_set()->scrub(_live_data);
500 }
501 return false;
502 }
503 };
504
505 void G1RemSet::scrub(uint worker_num, HeapRegionClaimer *hrclaimer) {
506 G1ScrubRSClosure scrub_cl(&_card_live_data);
651 // part of the heap (e.g. a partially allocated object) while
652 // processing a stale card. Despite the card being stale, redirty
653 // and re-enqueue, because we've already cleaned the card. Without
654 // this we could incorrectly discard a non-stale card.
655 if (!card_processed) {
656 // The card might have gotten re-dirtied and re-enqueued while we
657 // worked. (In fact, it's pretty likely.)
658 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
659 *card_ptr = CardTableModRefBS::dirty_card_val();
660 MutexLockerEx x(Shared_DirtyCardQ_lock,
661 Mutex::_no_safepoint_check_flag);
662 DirtyCardQueue* sdcq =
663 JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
664 sdcq->enqueue(card_ptr);
665 }
666 } else {
667 _conc_refine_cards++;
668 }
669 }
670
671 void G1RemSet::refine_card_during_gc(jbyte* card_ptr,
672 G1ScanObjsDuringUpdateRSClosure* update_rs_cl) {
673 assert(_g1->is_gc_active(), "Only call during GC");
674
675 check_card_ptr(card_ptr, _ct_bs);
676
677 // If the card is no longer dirty, nothing to do. This covers cards that were already
678 // scanned as parts of the remembered sets.
679 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
680 return;
681 }
682
683 // During GC we can immediately clean the card since we will not re-enqueue stale
684 // cards as we know they can be disregarded.
685 *card_ptr = CardTableModRefBS::clean_card_val();
686
687 // Construct the region representing the card.
688 HeapWord* card_start = _ct_bs->addr_for(card_ptr);
689 // And find the region containing it.
690 HeapRegion* r = _g1->heap_region_containing(card_start);
691
692 HeapWord* scan_limit = _scan_state->scan_top(r->hrm_index());
693 if (scan_limit <= card_start) {
694 // If the card starts above the area in the region containing objects to scan, skip it.
695 return;
696 }
697
698 // Don't use addr_for(card_ptr + 1) which can ask for
699 // a card beyond the heap.
700 HeapWord* card_end = card_start + CardTableModRefBS::card_size_in_words;
701 MemRegion dirty_region(card_start, MIN2(scan_limit, card_end));
702 assert(!dirty_region.is_empty(), "sanity");
703
704 update_rs_cl->set_region(r);
705 bool card_processed = r->oops_on_card_seq_iterate_careful<true>(dirty_region, update_rs_cl);
706 assert(card_processed, "must be");
707
708 _conc_refine_cards++;
709
710 return;
711 }
712
713 void G1RemSet::print_periodic_summary_info(const char* header, uint period_count) {
714 if ((G1SummarizeRSetStatsPeriod > 0) && log_is_enabled(Trace, gc, remset) &&
715 (period_count % G1SummarizeRSetStatsPeriod == 0)) {
716
717 if (!_prev_period_summary.initialized()) {
718 _prev_period_summary.initialize(this);
719 }
720
721 G1RemSetSummary current;
722 current.initialize(this);
723 _prev_period_summary.subtract_from(¤t);
724
725 Log(gc, remset) log;
726 log.trace("%s", header);
727 ResourceMark rm;
728 _prev_period_summary.print_on(log.trace_stream());
729
730 _prev_period_summary.set(¤t);
|