443 bool check_for_refs_into_cset) {
444 assert(_g1->is_in_exact(_ct_bs->addr_for(card_ptr)),
445 err_msg("Card at " PTR_FORMAT " index " SIZE_FORMAT " representing heap at " PTR_FORMAT " (%u) must be in committed heap",
446 p2i(card_ptr),
447 _ct_bs->index_for(_ct_bs->addr_for(card_ptr)),
448 _ct_bs->addr_for(card_ptr),
449 _g1->addr_to_region(_ct_bs->addr_for(card_ptr))));
450
451 // If the card is no longer dirty, nothing to do.
452 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
453 // No need to return that this card contains refs that point
454 // into the collection set.
455 return false;
456 }
457
458 // Construct the region representing the card.
459 HeapWord* start = _ct_bs->addr_for(card_ptr);
460 // And find the region containing it.
461 HeapRegion* r = _g1->heap_region_containing(start);
462
463 // Why do we have to check here whether a card is on a young region,
464 // given that we dirty young regions and, as a result, the
465 // post-barrier is supposed to filter them out and never to enqueue
466 // them? When we allocate a new region as the "allocation region" we
467 // actually dirty its cards after we release the lock, since card
468 // dirtying while holding the lock was a performance bottleneck. So,
469 // as a result, it is possible for other threads to actually
470 // allocate objects in the region (after the acquire the lock)
471 // before all the cards on the region are dirtied. This is unlikely,
472 // and it doesn't happen often, but it can happen. So, the extra
473 // check below filters out those cards.
474 if (r->is_young()) {
475 return false;
476 }
477
478 // While we are processing RSet buffers during the collection, we
479 // actually don't want to scan any cards on the collection set,
480 // since we don't want to update remebered sets with entries that
481 // point into the collection set, given that live objects from the
482 // collection set are about to move and such entries will be stale
483 // very soon. This change also deals with a reliability issue which
484 // involves scanning a card in the collection set and coming across
485 // an array that was being chunked and looking malformed. Note,
486 // however, that if evacuation fails, we have to scan any objects
487 // that were not moved and create any missing entries.
488 if (r->in_collection_set()) {
489 return false;
490 }
491
492 // The result from the hot card cache insert call is either:
493 // * pointer to the current card
494 // (implying that the current card is not 'hot'),
495 // * null
496 // (meaning we had inserted the card ptr into the "hot" card cache,
497 // which had some headroom),
498 // * a pointer to a "hot" card that was evicted from the "hot" cache.
499 //
500
501 G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
502 if (hot_card_cache->use_cache()) {
503 assert(!check_for_refs_into_cset, "sanity");
504 assert(!SafepointSynchronize::is_at_safepoint(), "sanity");
505
506 card_ptr = hot_card_cache->insert(card_ptr);
507 if (card_ptr == NULL) {
508 // There was no eviction. Nothing to do.
509 return false;
510 }
511
512 start = _ct_bs->addr_for(card_ptr);
513 r = _g1->heap_region_containing(start);
514
515 // Checking whether the region we got back from the cache
516 // is young here is inappropriate. The region could have been
517 // freed, reallocated and tagged as young while in the cache.
518 // Hence we could see its young type change at any time.
519 }
520
521 // Don't use addr_for(card_ptr + 1) which can ask for
522 // a card beyond the heap. This is not safe without a perm
523 // gen at the upper end of the heap.
524 HeapWord* end = start + CardTableModRefBS::card_size_in_words;
525 MemRegion dirtyRegion(start, end);
526
527 #if CARD_REPEAT_HISTO
528 init_ct_freq_table(_g1->max_capacity());
529 ct_freq_note_card(_ct_bs->index_for(start));
530 #endif
531
532 G1ParPushHeapRSClosure* oops_in_heap_closure = NULL;
533 if (check_for_refs_into_cset) {
534 // ConcurrentG1RefineThreads have worker numbers larger than what
535 // _cset_rs_update_cl[] is set up to handle. But those threads should
536 // only be active outside of a collection which means that when they
537 // reach here they should have check_for_refs_into_cset == false.
538 assert((size_t)worker_i < n_workers(), "index of worker larger than _cset_rs_update_cl[].length");
539 oops_in_heap_closure = _cset_rs_update_cl[worker_i];
540 }
541 G1UpdateRSOrPushRefOopClosure update_rs_oop_cl(_g1,
542 _g1->g1_rem_set(),
543 oops_in_heap_closure,
544 check_for_refs_into_cset,
545 worker_i);
546 update_rs_oop_cl.set_from(r);
547
548 G1TriggerClosure trigger_cl;
549 FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl);
550 G1InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl);
551 G1Mux2Closure mux(&invoke_cl, &update_rs_oop_cl);
552
553 FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r,
554 (check_for_refs_into_cset ?
555 (OopClosure*)&mux :
556 (OopClosure*)&update_rs_oop_cl));
557
558 // The region for the current card may be a young region. The
559 // current card may have been a card that was evicted from the
560 // card cache. When the card was inserted into the cache, we had
561 // determined that its region was non-young. While in the cache,
562 // the region may have been freed during a cleanup pause, reallocated
563 // and tagged as young.
564 //
565 // We wish to filter out cards for such a region but the current
566 // thread, if we're running concurrently, may "see" the young type
567 // change at any time (so an earlier "is_young" check may pass or
568 // fail arbitrarily). We tell the iteration code to perform this
569 // filtering when it has been determined that there has been an actual
570 // allocation in this region and making it safe to check the young type.
571
572 bool card_processed =
573 r->oops_on_card_seq_iterate_careful(dirtyRegion,
574 &filter_then_update_rs_oop_cl,
575 card_ptr);
576
577 // If unable to process the card then we encountered an unparsable
578 // part of the heap (e.g. a partially allocated object) while
579 // processing a stale card. Despite the card being stale, redirty
580 // and re-enqueue, because we've already cleaned the card. Without
581 // this we could incorrectly discard a non-stale card.
582 if (!card_processed) {
583 assert(!_g1->is_gc_active(), "Unparsable heap during GC");
584 // The card might have gotten re-dirtied and re-enqueued while we
585 // worked. (In fact, it's pretty likely.)
586 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
587 *card_ptr = CardTableModRefBS::dirty_card_val();
588 MutexLockerEx x(Shared_DirtyCardQ_lock,
589 Mutex::_no_safepoint_check_flag);
590 DirtyCardQueue* sdcq =
591 JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
592 sdcq->enqueue(card_ptr);
593 }
594 } else {
595 _conc_refine_cards++;
|
443 bool check_for_refs_into_cset) {
444 assert(_g1->is_in_exact(_ct_bs->addr_for(card_ptr)),
445 err_msg("Card at " PTR_FORMAT " index " SIZE_FORMAT " representing heap at " PTR_FORMAT " (%u) must be in committed heap",
446 p2i(card_ptr),
447 _ct_bs->index_for(_ct_bs->addr_for(card_ptr)),
448 _ct_bs->addr_for(card_ptr),
449 _g1->addr_to_region(_ct_bs->addr_for(card_ptr))));
450
451 // If the card is no longer dirty, nothing to do.
452 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
453 // No need to return that this card contains refs that point
454 // into the collection set.
455 return false;
456 }
457
458 // Construct the region representing the card.
459 HeapWord* start = _ct_bs->addr_for(card_ptr);
460 // And find the region containing it.
461 HeapRegion* r = _g1->heap_region_containing(start);
462
463 // This check is needed for some uncommon cases where we should
464 // ignore the card.
465 //
466 // The region could be young. Cards for young regions are
467 // distinctly marked (set to g1_young_gen), so the post-barrier will
468 // filter them out. However, that marking is performed
469 // concurrently. A write to a young object could occur before the
470 // card has been marked young, slipping past the filter.
471 //
472 // The card could be stale, because the region has been freed since
473 // the card was recorded. In this case the region type could be
474 // anything. If (still) free or (reallocated) young, just ignore
475 // it. If (reallocated) old or humongous, the later card trimming
476 // and additional checks in iteration may detect staleness. At
477 // worst, we end up processing a stale card unnecessarily.
478 //
479 // In the normal (non-stale) case, the synchronization between the
480 // enqueueing of the card and processing it here will have ensured
481 // we see the up-to-date region type here.
482 if (!r->is_old_or_humongous()) {
483 return false;
484 }
485
486 // While we are processing RSet buffers during the collection, we
487 // actually don't want to scan any cards on the collection set,
488 // since we don't want to update remebered sets with entries that
489 // point into the collection set, given that live objects from the
490 // collection set are about to move and such entries will be stale
491 // very soon. This change also deals with a reliability issue which
492 // involves scanning a card in the collection set and coming across
493 // an array that was being chunked and looking malformed. Note,
494 // however, that if evacuation fails, we have to scan any objects
495 // that were not moved and create any missing entries.
496 if (r->in_collection_set()) {
497 return false;
498 }
499
500 // The result from the hot card cache insert call is either:
501 // * pointer to the current card
502 // (implying that the current card is not 'hot'),
503 // * null
504 // (meaning we had inserted the card ptr into the "hot" card cache,
505 // which had some headroom),
506 // * a pointer to a "hot" card that was evicted from the "hot" cache.
507 //
508
509 G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
510 if (hot_card_cache->use_cache()) {
511 assert(!check_for_refs_into_cset, "sanity");
512 assert(!SafepointSynchronize::is_at_safepoint(), "sanity");
513
514 const jbyte* orig_card_ptr = card_ptr;
515 card_ptr = hot_card_cache->insert(card_ptr);
516 if (card_ptr == NULL) {
517 // There was no eviction. Nothing to do.
518 return false;
519 } else if (card_ptr != orig_card_ptr) {
520 // Original card was inserted and an old card was evicted.
521 start = _ct_bs->addr_for(card_ptr);
522 r = _g1->heap_region_containing(start);
523
524 // Check whether the region formerly in the cache should be
525 // ignored, as discussed earlier for the original card. The
526 // region could have been freed while in the cache. The cset is
527 // not relevant here, since we're in concurrent phase.
528 if (!r->is_old_or_humongous()) {
529 return false;
530 }
531 } // Else we still have the original card.
532 }
533
534 // Trim the region designated by the card to what's been allocated
535 // in the region. The card could be stale, or the card could cover
536 // (part of) an object at the end of the allocated space and extend
537 // beyond the end of allocation.
538 HeapWord* scan_limit;
539 if (_g1->is_gc_active()) {
540 // If we're in a STW GC, then a card might be in a GC alloc region
541 // and extend onto a GC LAB, which may not be parsable. Stop such
542 // at the "scan_top" of the region.
543 scan_limit = r->scan_top();
544 } else {
545 // Non-humongous objects are only allocated in the old-gen during
546 // GC, so if region is old then top is stable. Humongous object
547 // allocation sets top last; if top has not yet been set, this is
548 // a stale card and we'll end up with an empty intersection. If
549 // this is not a stale card, the synchronization between the
550 // enqueuing of the card and processing it here will have ensured
551 // we see the up-to-date top here.
552 scan_limit = r->top();
553 }
554 if (scan_limit <= start) {
555 // If the trimmed region is empty, the card must be stale.
556 return false;
557 }
558
559 // Okay to clean and process the card now. There are still some
560 // stale card cases that may be detected by iteration and dealt with
561 // as iteration failure.
562 *const_cast<volatile jbyte*>(card_ptr) = CardTableModRefBS::clean_card_val();
563
564 // This fence serves two purposes. First, the card must be cleaned
565 // before processing the contents. Second, we can't proceed with
566 // processing until after the read of top, for synchronization with
567 // possibly concurrent humongous object allocation. It's okay that
568 // reading top and reading type were racy wrto each other. We need
569 // both set, in any order, to proceed.
570 OrderAccess::fence();
571
572 // Don't use addr_for(card_ptr + 1) which can ask for
573 // a card beyond the heap.
574 HeapWord* end = start + CardTableModRefBS::card_size_in_words;
575 MemRegion dirty_region(start, MIN2(scan_limit, end));
576 assert(!dirty_region.is_empty(), "sanity");
577
578 #if CARD_REPEAT_HISTO
579 init_ct_freq_table(_g1->max_capacity());
580 ct_freq_note_card(_ct_bs->index_for(start));
581 #endif
582
583 G1ParPushHeapRSClosure* oops_in_heap_closure = NULL;
584 if (check_for_refs_into_cset) {
585 // ConcurrentG1RefineThreads have worker numbers larger than what
586 // _cset_rs_update_cl[] is set up to handle. But those threads should
587 // only be active outside of a collection which means that when they
588 // reach here they should have check_for_refs_into_cset == false.
589 assert((size_t)worker_i < n_workers(), "index of worker larger than _cset_rs_update_cl[].length");
590 oops_in_heap_closure = _cset_rs_update_cl[worker_i];
591 }
592 G1UpdateRSOrPushRefOopClosure update_rs_oop_cl(_g1,
593 _g1->g1_rem_set(),
594 oops_in_heap_closure,
595 check_for_refs_into_cset,
596 worker_i);
597 update_rs_oop_cl.set_from(r);
598
599 G1TriggerClosure trigger_cl;
600 FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl);
601 G1InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl);
602 G1Mux2Closure mux(&invoke_cl, &update_rs_oop_cl);
603
604 FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r,
605 (check_for_refs_into_cset ?
606 (OopClosure*)&mux :
607 (OopClosure*)&update_rs_oop_cl));
608
609 bool card_processed =
610 r->oops_on_card_seq_iterate_careful(dirty_region,
611 &filter_then_update_rs_oop_cl);
612
613 // If unable to process the card then we encountered an unparsable
614 // part of the heap (e.g. a partially allocated object) while
615 // processing a stale card. Despite the card being stale, redirty
616 // and re-enqueue, because we've already cleaned the card. Without
617 // this we could incorrectly discard a non-stale card.
618 if (!card_processed) {
619 assert(!_g1->is_gc_active(), "Unparsable heap during GC");
620 // The card might have gotten re-dirtied and re-enqueued while we
621 // worked. (In fact, it's pretty likely.)
622 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
623 *card_ptr = CardTableModRefBS::dirty_card_val();
624 MutexLockerEx x(Shared_DirtyCardQ_lock,
625 Mutex::_no_safepoint_check_flag);
626 DirtyCardQueue* sdcq =
627 JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
628 sdcq->enqueue(card_ptr);
629 }
630 } else {
631 _conc_refine_cards++;
|