317 G1FromCardCache::initialize(num_par_rem_sets(), max_regions);
318 _scan_state->initialize(max_regions);
319 {
320 GCTraceTime(Debug, gc, marking)("Initialize Card Live Data");
321 _card_live_data.initialize(capacity, max_regions);
322 }
323 if (G1PretouchAuxiliaryMemory) {
324 GCTraceTime(Debug, gc, marking)("Pre-Touch Card Live Data");
325 _card_live_data.pretouch();
326 }
327 }
328
329 G1ScanRSClosure::G1ScanRSClosure(G1RemSetScanState* scan_state,
330 G1ParPushHeapRSClosure* push_heap_cl,
331 CodeBlobClosure* code_root_cl,
332 uint worker_i) :
333 _scan_state(scan_state),
334 _push_heap_cl(push_heap_cl),
335 _code_root_cl(code_root_cl),
336 _strong_code_root_scan_time_sec(0.0),
337 _cards(0),
338 _cards_done(0),
339 _worker_i(worker_i) {
340 _g1h = G1CollectedHeap::heap();
341 _bot = _g1h->bot();
342 _ct_bs = _g1h->g1_barrier_set();
343 _block_size = MAX2<size_t>(G1RSetScanBlockSize, 1);
344 }
345
346 void G1ScanRSClosure::scan_card(size_t index, HeapWord* card_start, HeapRegion *r) {
347 MemRegion card_region(card_start, BOTConstants::N_words);
348 MemRegion pre_gc_allocated(r->bottom(), _scan_state->scan_top(r->hrm_index()));
349 MemRegion mr = pre_gc_allocated.intersection(card_region);
350 if (!mr.is_empty() && !_ct_bs->is_card_claimed(index)) {
351 // We make the card as "claimed" lazily (so races are possible
352 // but they're benign), which reduces the number of duplicate
353 // scans (the rsets of the regions in the cset can intersect).
354 _ct_bs->set_card_claimed(index);
355 _push_heap_cl->set_region(r);
356 r->oops_on_card_seq_iterate_careful<true>(mr, _push_heap_cl);
357 _cards_done++;
358 }
359 }
360
361 void G1ScanRSClosure::scan_strong_code_roots(HeapRegion* r) {
362 double scan_start = os::elapsedTime();
363 r->strong_code_roots_do(_code_root_cl);
364 _strong_code_root_scan_time_sec += (os::elapsedTime() - scan_start);
365 }
366
367 bool G1ScanRSClosure::doHeapRegion(HeapRegion* r) {
368 assert(r->in_collection_set(), "should only be called on elements of CS.");
369 uint region_idx = r->hrm_index();
370
371 if (_scan_state->iter_is_complete(region_idx)) {
372 return false;
373 }
374 if (_scan_state->claim_iter(region_idx)) {
375 // If we ever free the collection set concurrently, we should also
376 // clear the card table concurrently therefore we won't need to
377 // add regions of the collection set to the dirty cards region.
378 _scan_state->add_dirty_region(region_idx);
379 }
380
381 HeapRegionRemSetIterator iter(r->rem_set());
382 size_t card_index;
383
384 // We claim cards in block so as to reduce the contention. The block size is determined by
385 // the G1RSetScanBlockSize parameter.
386 size_t claimed_card_block = _scan_state->iter_claimed_next(region_idx, _block_size);
387 for (size_t current_card = 0; iter.has_next(card_index); current_card++) {
388 if (current_card >= claimed_card_block + _block_size) {
389 claimed_card_block = _scan_state->iter_claimed_next(region_idx, _block_size);
390 }
391 if (current_card < claimed_card_block) {
392 continue;
393 }
394 HeapWord* card_start = _g1h->bot()->address_for_index(card_index);
395
396 HeapRegion* card_region = _g1h->heap_region_containing(card_start);
397 _cards++;
398
399 _scan_state->add_dirty_region(card_region->hrm_index());
400
401 // If the card is dirty, then we will scan it during updateRS.
402 if (!card_region->in_collection_set() &&
403 !_ct_bs->is_card_dirty(card_index)) {
404 scan_card(card_index, card_start, card_region);
405 }
406 }
407 if (_scan_state->set_iter_complete(region_idx)) {
408 // Scan the strong code root list attached to the current region
409 scan_strong_code_roots(r);
410 }
411 return false;
412 }
413
414 size_t G1RemSet::scan_rem_set(G1ParPushHeapRSClosure* oops_in_heap_closure,
415 CodeBlobClosure* heap_region_codeblobs,
416 uint worker_i) {
417 double rs_time_start = os::elapsedTime();
418
419 G1ScanRSClosure cl(_scan_state, oops_in_heap_closure, heap_region_codeblobs, worker_i);
420 _g1->collection_set_iterate_from(&cl, worker_i);
421
422 double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) -
423 cl.strong_code_root_scan_time_sec();
424
425 _g1p->phase_times()->record_time_secs(G1GCPhaseTimes::ScanRS, worker_i, scan_rs_time_sec);
426 _g1p->phase_times()->record_time_secs(G1GCPhaseTimes::CodeRoots, worker_i, cl.strong_code_root_scan_time_sec());
427
428 return cl.cards_done();
429 }
430
431 // Closure used for updating RSets and recording references that
432 // point into the collection set. Only called during an
433 // evacuation pause.
434
435 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure {
436 G1RemSet* _g1rs;
437 DirtyCardQueue* _into_cset_dcq;
438 G1ParPushHeapRSClosure* _cl;
439 public:
440 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h,
441 DirtyCardQueue* into_cset_dcq,
442 G1ParPushHeapRSClosure* cl) :
443 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq), _cl(cl)
444 {}
445
446 bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
447 // The only time we care about recording cards that
448 // contain references that point into the collection set
466
467 void G1RemSet::update_rem_set(DirtyCardQueue* into_cset_dcq,
468 G1ParPushHeapRSClosure* oops_in_heap_closure,
469 uint worker_i) {
470 RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq, oops_in_heap_closure);
471
472 G1GCParPhaseTimesTracker x(_g1p->phase_times(), G1GCPhaseTimes::UpdateRS, worker_i);
473 if (G1HotCardCache::default_use_cache()) {
474 // Apply the closure to the entries of the hot card cache.
475 G1GCParPhaseTimesTracker y(_g1p->phase_times(), G1GCPhaseTimes::ScanHCC, worker_i);
476 _g1->iterate_hcc_closure(&into_cset_update_rs_cl, worker_i);
477 }
478 // Apply the closure to all remaining log entries.
479 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, worker_i);
480 }
481
482 void G1RemSet::cleanupHRRS() {
483 HeapRegionRemSet::cleanup();
484 }
485
486 size_t G1RemSet::oops_into_collection_set_do(G1ParPushHeapRSClosure* cl,
487 CodeBlobClosure* heap_region_codeblobs,
488 uint worker_i) {
489 // A DirtyCardQueue that is used to hold cards containing references
490 // that point into the collection set. This DCQ is associated with a
491 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
492 // circumstances (i.e. the pause successfully completes), these cards
493 // are just discarded (there's no need to update the RSets of regions
494 // that were in the collection set - after the pause these regions
495 // are wholly 'free' of live objects. In the event of an evacuation
496 // failure the cards/buffers in this queue set are passed to the
497 // DirtyCardQueueSet that is used to manage RSet updates
498 DirtyCardQueue into_cset_dcq(&_into_cset_dirty_card_queue_set);
499
500 update_rem_set(&into_cset_dcq, cl, worker_i);
501 return scan_rem_set(cl, heap_region_codeblobs, worker_i);;
502 }
503
504 void G1RemSet::prepare_for_oops_into_collection_set_do() {
505 _g1->set_refine_cte_cl_concurrency(false);
506 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
507 dcqs.concatenate_logs();
508
509 _scan_state->reset();
510 }
511
512 void G1RemSet::cleanup_after_oops_into_collection_set_do() {
513 G1GCPhaseTimes* phase_times = _g1->g1_policy()->phase_times();
514 // Cleanup after copy
515 _g1->set_refine_cte_cl_concurrency(true);
516
517 // Set all cards back to clean.
518 double start = os::elapsedTime();
519 _scan_state->clear_card_table(_g1->workers());
520 phase_times->record_clear_ct_time((os::elapsedTime() - start) * 1000.0);
521
|
317 G1FromCardCache::initialize(num_par_rem_sets(), max_regions);
318 _scan_state->initialize(max_regions);
319 {
320 GCTraceTime(Debug, gc, marking)("Initialize Card Live Data");
321 _card_live_data.initialize(capacity, max_regions);
322 }
323 if (G1PretouchAuxiliaryMemory) {
324 GCTraceTime(Debug, gc, marking)("Pre-Touch Card Live Data");
325 _card_live_data.pretouch();
326 }
327 }
328
329 G1ScanRSClosure::G1ScanRSClosure(G1RemSetScanState* scan_state,
330 G1ParPushHeapRSClosure* push_heap_cl,
331 CodeBlobClosure* code_root_cl,
332 uint worker_i) :
333 _scan_state(scan_state),
334 _push_heap_cl(push_heap_cl),
335 _code_root_cl(code_root_cl),
336 _strong_code_root_scan_time_sec(0.0),
337 _cards_claimed(0),
338 _cards_scanned(0),
339 _cards_skipped(0),
340 _worker_i(worker_i) {
341 _g1h = G1CollectedHeap::heap();
342 _bot = _g1h->bot();
343 _ct_bs = _g1h->g1_barrier_set();
344 _block_size = MAX2<size_t>(G1RSetScanBlockSize, 1);
345 }
346
347 void G1ScanRSClosure::scan_card(size_t index, HeapWord* card_start, HeapRegion *r) {
348 MemRegion card_region(card_start, BOTConstants::N_words);
349 MemRegion pre_gc_allocated(r->bottom(), _scan_state->scan_top(r->hrm_index()));
350 MemRegion mr = pre_gc_allocated.intersection(card_region);
351 if (!mr.is_empty() && !_ct_bs->is_card_claimed(index)) {
352 // We make the card as "claimed" lazily (so races are possible
353 // but they're benign), which reduces the number of duplicate
354 // scans (the rsets of the regions in the cset can intersect).
355 _ct_bs->set_card_claimed(index);
356 _push_heap_cl->set_region(r);
357 r->oops_on_card_seq_iterate_careful<true>(mr, _push_heap_cl);
358 _cards_scanned++;
359 }
360 }
361
362 void G1ScanRSClosure::scan_strong_code_roots(HeapRegion* r) {
363 double scan_start = os::elapsedTime();
364 r->strong_code_roots_do(_code_root_cl);
365 _strong_code_root_scan_time_sec += (os::elapsedTime() - scan_start);
366 }
367
368 bool G1ScanRSClosure::doHeapRegion(HeapRegion* r) {
369 assert(r->in_collection_set(), "should only be called on elements of CS.");
370 uint region_idx = r->hrm_index();
371
372 if (_scan_state->iter_is_complete(region_idx)) {
373 return false;
374 }
375 if (_scan_state->claim_iter(region_idx)) {
376 // If we ever free the collection set concurrently, we should also
377 // clear the card table concurrently therefore we won't need to
378 // add regions of the collection set to the dirty cards region.
379 _scan_state->add_dirty_region(region_idx);
380 }
381
382 HeapRegionRemSetIterator iter(r->rem_set());
383 size_t card_index;
384
385 // We claim cards in block so as to reduce the contention. The block size is determined by
386 // the G1RSetScanBlockSize parameter.
387 size_t claimed_card_block = _scan_state->iter_claimed_next(region_idx, _block_size);
388 for (size_t current_card = 0; iter.has_next(card_index); current_card++) {
389 if (current_card >= claimed_card_block + _block_size) {
390 claimed_card_block = _scan_state->iter_claimed_next(region_idx, _block_size);
391 }
392 if (current_card < claimed_card_block) {
393 _cards_skipped++;
394 continue;
395 }
396 HeapWord* card_start = _g1h->bot()->address_for_index(card_index);
397
398 HeapRegion* card_region = _g1h->heap_region_containing(card_start);
399 _cards_claimed++;
400
401 _scan_state->add_dirty_region(card_region->hrm_index());
402
403 // If the card is dirty, then we will scan it during updateRS.
404 if (!card_region->in_collection_set() &&
405 !_ct_bs->is_card_dirty(card_index)) {
406 scan_card(card_index, card_start, card_region);
407 }
408 }
409 if (_scan_state->set_iter_complete(region_idx)) {
410 // Scan the strong code root list attached to the current region
411 scan_strong_code_roots(r);
412 }
413 return false;
414 }
415
416 void G1RemSet::scan_rem_set(G1ParPushHeapRSClosure* oops_in_heap_closure,
417 CodeBlobClosure* heap_region_codeblobs,
418 uint worker_i) {
419 double rs_time_start = os::elapsedTime();
420
421 G1ScanRSClosure cl(_scan_state, oops_in_heap_closure, heap_region_codeblobs, worker_i);
422 _g1->collection_set_iterate_from(&cl, worker_i);
423
424 double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) -
425 cl.strong_code_root_scan_time_sec();
426
427 G1GCPhaseTimes* p = _g1p->phase_times();
428
429 p->record_time_secs(G1GCPhaseTimes::ScanRS, worker_i, scan_rs_time_sec);
430 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_scanned(), G1GCPhaseTimes::ScannedCards);
431 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_claimed(), G1GCPhaseTimes::ClaimedCards);
432 p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_skipped(), G1GCPhaseTimes::SkippedCards);
433
434 p->record_time_secs(G1GCPhaseTimes::CodeRoots, worker_i, cl.strong_code_root_scan_time_sec());
435 }
436
437 // Closure used for updating RSets and recording references that
438 // point into the collection set. Only called during an
439 // evacuation pause.
440
441 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure {
442 G1RemSet* _g1rs;
443 DirtyCardQueue* _into_cset_dcq;
444 G1ParPushHeapRSClosure* _cl;
445 public:
446 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h,
447 DirtyCardQueue* into_cset_dcq,
448 G1ParPushHeapRSClosure* cl) :
449 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq), _cl(cl)
450 {}
451
452 bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
453 // The only time we care about recording cards that
454 // contain references that point into the collection set
472
473 void G1RemSet::update_rem_set(DirtyCardQueue* into_cset_dcq,
474 G1ParPushHeapRSClosure* oops_in_heap_closure,
475 uint worker_i) {
476 RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq, oops_in_heap_closure);
477
478 G1GCParPhaseTimesTracker x(_g1p->phase_times(), G1GCPhaseTimes::UpdateRS, worker_i);
479 if (G1HotCardCache::default_use_cache()) {
480 // Apply the closure to the entries of the hot card cache.
481 G1GCParPhaseTimesTracker y(_g1p->phase_times(), G1GCPhaseTimes::ScanHCC, worker_i);
482 _g1->iterate_hcc_closure(&into_cset_update_rs_cl, worker_i);
483 }
484 // Apply the closure to all remaining log entries.
485 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, worker_i);
486 }
487
488 void G1RemSet::cleanupHRRS() {
489 HeapRegionRemSet::cleanup();
490 }
491
492 void G1RemSet::oops_into_collection_set_do(G1ParPushHeapRSClosure* cl,
493 CodeBlobClosure* heap_region_codeblobs,
494 uint worker_i) {
495 // A DirtyCardQueue that is used to hold cards containing references
496 // that point into the collection set. This DCQ is associated with a
497 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
498 // circumstances (i.e. the pause successfully completes), these cards
499 // are just discarded (there's no need to update the RSets of regions
500 // that were in the collection set - after the pause these regions
501 // are wholly 'free' of live objects. In the event of an evacuation
502 // failure the cards/buffers in this queue set are passed to the
503 // DirtyCardQueueSet that is used to manage RSet updates
504 DirtyCardQueue into_cset_dcq(&_into_cset_dirty_card_queue_set);
505
506 update_rem_set(&into_cset_dcq, cl, worker_i);
507 scan_rem_set(cl, heap_region_codeblobs, worker_i);;
508 }
509
510 void G1RemSet::prepare_for_oops_into_collection_set_do() {
511 _g1->set_refine_cte_cl_concurrency(false);
512 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
513 dcqs.concatenate_logs();
514
515 _scan_state->reset();
516 }
517
518 void G1RemSet::cleanup_after_oops_into_collection_set_do() {
519 G1GCPhaseTimes* phase_times = _g1->g1_policy()->phase_times();
520 // Cleanup after copy
521 _g1->set_refine_cte_cl_concurrency(true);
522
523 // Set all cards back to clean.
524 double start = os::elapsedTime();
525 _scan_state->clear_card_table(_g1->workers());
526 phase_times->record_clear_ct_time((os::elapsedTime() - start) * 1000.0);
527
|