318 G1FromCardCache::initialize(num_par_rem_sets(), max_regions);
319 _scan_state->initialize(max_regions);
320 {
321 GCTraceTime(Debug, gc, marking)("Initialize Card Live Data");
322 _card_live_data.initialize(capacity, max_regions);
323 }
324 if (G1PretouchAuxiliaryMemory) {
325 GCTraceTime(Debug, gc, marking)("Pre-Touch Card Live Data");
326 _card_live_data.pretouch();
327 }
328 }
329
330 G1ScanRSClosure::G1ScanRSClosure(G1RemSetScanState* scan_state,
331 G1ParPushHeapRSClosure* push_heap_cl,
332 CodeBlobClosure* code_root_cl,
333 uint worker_i) :
334 _scan_state(scan_state),
335 _push_heap_cl(push_heap_cl),
336 _code_root_cl(code_root_cl),
337 _strong_code_root_scan_time_sec(0.0),
338 _cards(0),
339 _cards_done(0),
340 _worker_i(worker_i) {
341 _g1h = G1CollectedHeap::heap();
342 _bot = _g1h->bot();
343 _ct_bs = _g1h->g1_barrier_set();
344 _block_size = MAX2<size_t>(G1RSetScanBlockSize, 1);
345 }
346
347 void G1ScanRSClosure::scan_card(size_t index, HeapWord* card_start, HeapRegion *r) {
348 MemRegion card_region(card_start, BOTConstants::N_words);
349 MemRegion pre_gc_allocated(r->bottom(), _scan_state->scan_top(r->hrm_index()));
350 MemRegion mr = pre_gc_allocated.intersection(card_region);
351 if (!mr.is_empty() && !_ct_bs->is_card_claimed(index)) {
352 // We make the card as "claimed" lazily (so races are possible
353 // but they're benign), which reduces the number of duplicate
354 // scans (the rsets of the regions in the cset can intersect).
355 _ct_bs->set_card_claimed(index);
356 _push_heap_cl->set_region(r);
357 r->oops_on_card_seq_iterate_careful<true>(mr, _push_heap_cl);
358 _cards_done++;
359 }
360 }
361
362 void G1ScanRSClosure::scan_strong_code_roots(HeapRegion* r) {
363 double scan_start = os::elapsedTime();
364 r->strong_code_roots_do(_code_root_cl);
365 _strong_code_root_scan_time_sec += (os::elapsedTime() - scan_start);
366 }
367
368 bool G1ScanRSClosure::doHeapRegion(HeapRegion* r) {
369 assert(r->in_collection_set(), "should only be called on elements of CS.");
370 uint region_idx = r->hrm_index();
371
372 if (_scan_state->iter_is_complete(region_idx)) {
373 return false;
374 }
375 if (_scan_state->claim_iter(region_idx)) {
376 // If we ever free the collection set concurrently, we should also
377 // clear the card table concurrently therefore we won't need to
378 // add regions of the collection set to the dirty cards region.
379 _scan_state->add_dirty_region(region_idx);
380 }
381
382 HeapRegionRemSetIterator iter(r->rem_set());
383 size_t card_index;
384
385 // We claim cards in block so as to reduce the contention. The block size is determined by
386 // the G1RSetScanBlockSize parameter.
387 size_t claimed_card_block = _scan_state->iter_claimed_next(region_idx, _block_size);
388 for (size_t current_card = 0; iter.has_next(card_index); current_card++) {
389 if (current_card >= claimed_card_block + _block_size) {
390 claimed_card_block = _scan_state->iter_claimed_next(region_idx, _block_size);
391 }
392 if (current_card < claimed_card_block) {
393 continue;
394 }
395 HeapWord* card_start = _g1h->bot()->address_for_index(card_index);
396
397 HeapRegion* card_region = _g1h->heap_region_containing(card_start);
398 _cards++;
399
400 _scan_state->add_dirty_region(card_region->hrm_index());
401
402 // If the card is dirty, then we will scan it during updateRS.
403 if (!card_region->in_collection_set() &&
404 !_ct_bs->is_card_dirty(card_index)) {
405 scan_card(card_index, card_start, card_region);
406 }
407 }
408 if (_scan_state->set_iter_complete(region_idx)) {
409 // Scan the strong code root list attached to the current region
410 scan_strong_code_roots(r);
411 }
412 return false;
413 }
414
415 size_t G1RemSet::scan_rem_set(G1ParPushHeapRSClosure* oops_in_heap_closure,
416 CodeBlobClosure* heap_region_codeblobs,
417 uint worker_i) {
418 double rs_time_start = os::elapsedTime();
419
420 G1ScanRSClosure cl(_scan_state, oops_in_heap_closure, heap_region_codeblobs, worker_i);
421 _g1->collection_set_iterate_from(&cl, worker_i);
422
423 double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) -
424 cl.strong_code_root_scan_time_sec();
425
426 _g1p->phase_times()->record_time_secs(G1GCPhaseTimes::ScanRS, worker_i, scan_rs_time_sec);
427 _g1p->phase_times()->record_time_secs(G1GCPhaseTimes::CodeRoots, worker_i, cl.strong_code_root_scan_time_sec());
428
429 return cl.cards_done();
430 }
431
432 // Closure used for updating RSets and recording references that
433 // point into the collection set. Only called during an
434 // evacuation pause.
435
436 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure {
437 G1RemSet* _g1rs;
438 DirtyCardQueue* _into_cset_dcq;
439 G1ParPushHeapRSClosure* _cl;
440 public:
441 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h,
442 DirtyCardQueue* into_cset_dcq,
443 G1ParPushHeapRSClosure* cl) :
444 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq), _cl(cl)
445 {}
446
447 bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
448 // The only time we care about recording cards that
449 // contain references that point into the collection set
467
468 void G1RemSet::update_rem_set(DirtyCardQueue* into_cset_dcq,
469 G1ParPushHeapRSClosure* oops_in_heap_closure,
470 uint worker_i) {
471 RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq, oops_in_heap_closure);
472
473 G1GCParPhaseTimesTracker x(_g1p->phase_times(), G1GCPhaseTimes::UpdateRS, worker_i);
474 if (G1HotCardCache::default_use_cache()) {
475 // Apply the closure to the entries of the hot card cache.
476 G1GCParPhaseTimesTracker y(_g1p->phase_times(), G1GCPhaseTimes::ScanHCC, worker_i);
477 _g1->iterate_hcc_closure(&into_cset_update_rs_cl, worker_i);
478 }
479 // Apply the closure to all remaining log entries.
480 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, worker_i);
481 }
482
483 void G1RemSet::cleanupHRRS() {
484 HeapRegionRemSet::cleanup();
485 }
486
487 size_t G1RemSet::oops_into_collection_set_do(G1ParPushHeapRSClosure* cl,
488 CodeBlobClosure* heap_region_codeblobs,
489 uint worker_i) {
490 // A DirtyCardQueue that is used to hold cards containing references
491 // that point into the collection set. This DCQ is associated with a
492 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
493 // circumstances (i.e. the pause successfully completes), these cards
494 // are just discarded (there's no need to update the RSets of regions
495 // that were in the collection set - after the pause these regions
496 // are wholly 'free' of live objects. In the event of an evacuation
497 // failure the cards/buffers in this queue set are passed to the
498 // DirtyCardQueueSet that is used to manage RSet updates
499 DirtyCardQueue into_cset_dcq(&_into_cset_dirty_card_queue_set);
500
501 update_rem_set(&into_cset_dcq, cl, worker_i);
502 return scan_rem_set(cl, heap_region_codeblobs, worker_i);;
503 }
504
505 void G1RemSet::prepare_for_oops_into_collection_set_do() {
506 _g1->set_refine_cte_cl_concurrency(false);
507 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
508 dcqs.concatenate_logs();
509
510 _scan_state->reset();
511 }
512
513 void G1RemSet::cleanup_after_oops_into_collection_set_do() {
514 G1GCPhaseTimes* phase_times = _g1->g1_policy()->phase_times();
515 // Cleanup after copy
516 _g1->set_refine_cte_cl_concurrency(true);
517
518 // Set all cards back to clean.
519 double start = os::elapsedTime();
520 _scan_state->clear_card_table(_g1->workers());
521 phase_times->record_clear_ct_time((os::elapsedTime() - start) * 1000.0);
522
|
318 G1FromCardCache::initialize(num_par_rem_sets(), max_regions);
319 _scan_state->initialize(max_regions);
320 {
321 GCTraceTime(Debug, gc, marking)("Initialize Card Live Data");
322 _card_live_data.initialize(capacity, max_regions);
323 }
324 if (G1PretouchAuxiliaryMemory) {
325 GCTraceTime(Debug, gc, marking)("Pre-Touch Card Live Data");
326 _card_live_data.pretouch();
327 }
328 }
329
330 G1ScanRSClosure::G1ScanRSClosure(G1RemSetScanState* scan_state,
331 G1ParPushHeapRSClosure* push_heap_cl,
332 CodeBlobClosure* code_root_cl,
333 uint worker_i) :
334 _scan_state(scan_state),
335 _push_heap_cl(push_heap_cl),
336 _code_root_cl(code_root_cl),
337 _strong_code_root_scan_time_sec(0.0),
338 _cards_claimed(0),
339 _cards_scanned(0),
340 _cards_skipped(0),
341 _worker_i(worker_i) {
342 _g1h = G1CollectedHeap::heap();
343 _bot = _g1h->bot();
344 _ct_bs = _g1h->g1_barrier_set();
345 _block_size = MAX2<size_t>(G1RSetScanBlockSize, 1);
346 }
347
348 void G1ScanRSClosure::scan_card(size_t index, HeapWord* card_start, HeapRegion *r) {
349 MemRegion card_region(card_start, BOTConstants::N_words);
350 MemRegion pre_gc_allocated(r->bottom(), _scan_state->scan_top(r->hrm_index()));
351 MemRegion mr = pre_gc_allocated.intersection(card_region);
352 if (!mr.is_empty() && !_ct_bs->is_card_claimed(index)) {
353 // We make the card as "claimed" lazily (so races are possible
354 // but they're benign), which reduces the number of duplicate
355 // scans (the rsets of the regions in the cset can intersect).
356 _ct_bs->set_card_claimed(index);
357 _push_heap_cl->set_region(r);
358 r->oops_on_card_seq_iterate_careful<true>(mr, _push_heap_cl);
359 _cards_scanned++;
360 }
361 }
362
363 void G1ScanRSClosure::scan_strong_code_roots(HeapRegion* r) {
364 double scan_start = os::elapsedTime();
365 r->strong_code_roots_do(_code_root_cl);
366 _strong_code_root_scan_time_sec += (os::elapsedTime() - scan_start);
367 }
368
369 bool G1ScanRSClosure::doHeapRegion(HeapRegion* r) {
370 assert(r->in_collection_set(), "should only be called on elements of CS.");
371 uint region_idx = r->hrm_index();
372
373 if (_scan_state->iter_is_complete(region_idx)) {
374 return false;
375 }
376 if (_scan_state->claim_iter(region_idx)) {
377 // If we ever free the collection set concurrently, we should also
378 // clear the card table concurrently therefore we won't need to
379 // add regions of the collection set to the dirty cards region.
380 _scan_state->add_dirty_region(region_idx);
381 }
382
383 HeapRegionRemSetIterator iter(r->rem_set());
384 size_t card_index;
385
386 // We claim cards in block so as to reduce the contention. The block size is determined by
387 // the G1RSetScanBlockSize parameter.
388 size_t claimed_card_block = _scan_state->iter_claimed_next(region_idx, _block_size);
389 for (size_t current_card = 0; iter.has_next(card_index); current_card++) {
390 if (current_card >= claimed_card_block + _block_size) {
391 claimed_card_block = _scan_state->iter_claimed_next(region_idx, _block_size);
392 }
393 if (current_card < claimed_card_block) {
394 _cards_skipped++;
395 continue;
396 }
397 HeapWord* card_start = _g1h->bot()->address_for_index(card_index);
398
399 HeapRegion* card_region = _g1h->heap_region_containing(card_start);
400 _cards_claimed++;
401
402 _scan_state->add_dirty_region(card_region->hrm_index());
403
404 // If the card is dirty, then we will scan it during updateRS.
405 if (!card_region->in_collection_set() &&
406 !_ct_bs->is_card_dirty(card_index)) {
407 scan_card(card_index, card_start, card_region);
408 }
409 }
410 if (_scan_state->set_iter_complete(region_idx)) {
411 // Scan the strong code root list attached to the current region
412 scan_strong_code_roots(r);
413 }
414 return false;
415 }
416
417 void G1RemSet::scan_rem_set(G1ParPushHeapRSClosure* oops_in_heap_closure,
418 CodeBlobClosure* heap_region_codeblobs,
419 uint worker_i) {
420 double rs_time_start = os::elapsedTime();
421
422 G1ScanRSClosure cl(_scan_state, oops_in_heap_closure, heap_region_codeblobs, worker_i);
423 _g1->collection_set_iterate_from(&cl, worker_i);
424
425 double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) -
426 cl.strong_code_root_scan_time_sec();
427
428 G1GCPhaseTimes* p = _g1p->phase_times();
429
430 p->record_time_secs(G1GCPhaseTimes::ScanRS, worker_i, scan_rs_time_sec);
431 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_scanned(), G1GCPhaseTimes::ScannedCards);
432 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_claimed(), G1GCPhaseTimes::ClaimedCards);
433 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_skipped(), G1GCPhaseTimes::SkippedCards);
434
435 p->record_time_secs(G1GCPhaseTimes::CodeRoots, worker_i, cl.strong_code_root_scan_time_sec());
436 }
437
438 // Closure used for updating RSets and recording references that
439 // point into the collection set. Only called during an
440 // evacuation pause.
441
442 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure {
443 G1RemSet* _g1rs;
444 DirtyCardQueue* _into_cset_dcq;
445 G1ParPushHeapRSClosure* _cl;
446 public:
447 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h,
448 DirtyCardQueue* into_cset_dcq,
449 G1ParPushHeapRSClosure* cl) :
450 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq), _cl(cl)
451 {}
452
453 bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
454 // The only time we care about recording cards that
455 // contain references that point into the collection set
473
474 void G1RemSet::update_rem_set(DirtyCardQueue* into_cset_dcq,
475 G1ParPushHeapRSClosure* oops_in_heap_closure,
476 uint worker_i) {
477 RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq, oops_in_heap_closure);
478
479 G1GCParPhaseTimesTracker x(_g1p->phase_times(), G1GCPhaseTimes::UpdateRS, worker_i);
480 if (G1HotCardCache::default_use_cache()) {
481 // Apply the closure to the entries of the hot card cache.
482 G1GCParPhaseTimesTracker y(_g1p->phase_times(), G1GCPhaseTimes::ScanHCC, worker_i);
483 _g1->iterate_hcc_closure(&into_cset_update_rs_cl, worker_i);
484 }
485 // Apply the closure to all remaining log entries.
486 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, worker_i);
487 }
488
489 void G1RemSet::cleanupHRRS() {
490 HeapRegionRemSet::cleanup();
491 }
492
493 void G1RemSet::oops_into_collection_set_do(G1ParPushHeapRSClosure* cl,
494 CodeBlobClosure* heap_region_codeblobs,
495 uint worker_i) {
496 // A DirtyCardQueue that is used to hold cards containing references
497 // that point into the collection set. This DCQ is associated with a
498 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
499 // circumstances (i.e. the pause successfully completes), these cards
500 // are just discarded (there's no need to update the RSets of regions
501 // that were in the collection set - after the pause these regions
502 // are wholly 'free' of live objects. In the event of an evacuation
503 // failure the cards/buffers in this queue set are passed to the
504 // DirtyCardQueueSet that is used to manage RSet updates
505 DirtyCardQueue into_cset_dcq(&_into_cset_dirty_card_queue_set);
506
507 update_rem_set(&into_cset_dcq, cl, worker_i);
508 scan_rem_set(cl, heap_region_codeblobs, worker_i);;
509 }
510
511 void G1RemSet::prepare_for_oops_into_collection_set_do() {
512 _g1->set_refine_cte_cl_concurrency(false);
513 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
514 dcqs.concatenate_logs();
515
516 _scan_state->reset();
517 }
518
519 void G1RemSet::cleanup_after_oops_into_collection_set_do() {
520 G1GCPhaseTimes* phase_times = _g1->g1_policy()->phase_times();
521 // Cleanup after copy
522 _g1->set_refine_cte_cl_concurrency(true);
523
524 // Set all cards back to clean.
525 double start = os::elapsedTime();
526 _scan_state->clear_card_table(_g1->workers());
527 phase_times->record_clear_ct_time((os::elapsedTime() - start) * 1000.0);
528
|