< prev index next >

src/share/vm/gc_implementation/g1/g1RemSet.cpp

Print this page
rev 9374 : 8259659: Missing memory fences between memory allocation and refinement
Summary: Refactored to have needed barrier
Reviewed-by: tschatzl, ehelin


 443                            bool check_for_refs_into_cset) {
 444   assert(_g1->is_in_exact(_ct_bs->addr_for(card_ptr)),
 445          err_msg("Card at " PTR_FORMAT " index " SIZE_FORMAT " representing heap at " PTR_FORMAT " (%u) must be in committed heap",
 446                  p2i(card_ptr),
 447                  _ct_bs->index_for(_ct_bs->addr_for(card_ptr)),
 448                  _ct_bs->addr_for(card_ptr),
 449                  _g1->addr_to_region(_ct_bs->addr_for(card_ptr))));
 450 
 451   // If the card is no longer dirty, nothing to do.
 452   if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
 453     // No need to return that this card contains refs that point
 454     // into the collection set.
 455     return false;
 456   }
 457 
 458   // Construct the region representing the card.
 459   HeapWord* start = _ct_bs->addr_for(card_ptr);
 460   // And find the region containing it.
 461   HeapRegion* r = _g1->heap_region_containing(start);
 462 
 463   // Why do we have to check here whether a card is on a young region,
 464   // given that we dirty young regions and, as a result, the
 465   // post-barrier is supposed to filter them out and never to enqueue
 466   // them? When we allocate a new region as the "allocation region" we
 467   // actually dirty its cards after we release the lock, since card
 468   // dirtying while holding the lock was a performance bottleneck. So,
 469   // as a result, it is possible for other threads to actually
 470   // allocate objects in the region (after the acquire the lock)
 471   // before all the cards on the region are dirtied. This is unlikely,
 472   // and it doesn't happen often, but it can happen. So, the extra
 473   // check below filters out those cards.
 474   if (r->is_young()) {








 475     return false;
 476   }
 477 
 478   // While we are processing RSet buffers during the collection, we
 479   // actually don't want to scan any cards on the collection set,
 480   // since we don't want to update remebered sets with entries that
 481   // point into the collection set, given that live objects from the
 482   // collection set are about to move and such entries will be stale
 483   // very soon. This change also deals with a reliability issue which
 484   // involves scanning a card in the collection set and coming across
 485   // an array that was being chunked and looking malformed. Note,
 486   // however, that if evacuation fails, we have to scan any objects
 487   // that were not moved and create any missing entries.
 488   if (r->in_collection_set()) {
 489     return false;
 490   }
 491 
 492   // The result from the hot card cache insert call is either:
 493   //   * pointer to the current card
 494   //     (implying that the current card is not 'hot'),
 495   //   * null
 496   //     (meaning we had inserted the card ptr into the "hot" card cache,
 497   //     which had some headroom),
 498   //   * a pointer to a "hot" card that was evicted from the "hot" cache.
 499   //
 500 
 501   G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
 502   if (hot_card_cache->use_cache()) {
 503     assert(!check_for_refs_into_cset, "sanity");
 504     assert(!SafepointSynchronize::is_at_safepoint(), "sanity");
 505 

 506     card_ptr = hot_card_cache->insert(card_ptr);
 507     if (card_ptr == NULL) {
 508       // There was no eviction. Nothing to do.
 509       return false;
 510     }
 511 
 512     start = _ct_bs->addr_for(card_ptr);
 513     r = _g1->heap_region_containing(start);
 514 
 515     // Checking whether the region we got back from the cache
 516     // is young here is inappropriate. The region could have been
 517     // freed, reallocated and tagged as young while in the cache.
 518     // Hence we could see its young type change at any time.





























 519   }
 520 













 521   // Don't use addr_for(card_ptr + 1) which can ask for
 522   // a card beyond the heap.  This is not safe without a perm
 523   // gen at the upper end of the heap.
 524   HeapWord* end   = start + CardTableModRefBS::card_size_in_words;
 525   MemRegion dirtyRegion(start, end);

 526 
 527 #if CARD_REPEAT_HISTO
 528   init_ct_freq_table(_g1->max_capacity());
 529   ct_freq_note_card(_ct_bs->index_for(start));
 530 #endif
 531 
 532   G1ParPushHeapRSClosure* oops_in_heap_closure = NULL;
 533   if (check_for_refs_into_cset) {
 534     // ConcurrentG1RefineThreads have worker numbers larger than what
 535     // _cset_rs_update_cl[] is set up to handle. But those threads should
 536     // only be active outside of a collection which means that when they
 537     // reach here they should have check_for_refs_into_cset == false.
 538     assert((size_t)worker_i < n_workers(), "index of worker larger than _cset_rs_update_cl[].length");
 539     oops_in_heap_closure = _cset_rs_update_cl[worker_i];
 540   }
 541   G1UpdateRSOrPushRefOopClosure update_rs_oop_cl(_g1,
 542                                                  _g1->g1_rem_set(),
 543                                                  oops_in_heap_closure,
 544                                                  check_for_refs_into_cset,
 545                                                  worker_i);
 546   update_rs_oop_cl.set_from(r);
 547 
 548   G1TriggerClosure trigger_cl;
 549   FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl);
 550   G1InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl);
 551   G1Mux2Closure mux(&invoke_cl, &update_rs_oop_cl);
 552 
 553   FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r,
 554                         (check_for_refs_into_cset ?
 555                                 (OopClosure*)&mux :
 556                                 (OopClosure*)&update_rs_oop_cl));
 557 
 558   // The region for the current card may be a young region. The
 559   // current card may have been a card that was evicted from the
 560   // card cache. When the card was inserted into the cache, we had
 561   // determined that its region was non-young. While in the cache,
 562   // the region may have been freed during a cleanup pause, reallocated
 563   // and tagged as young.
 564   //
 565   // We wish to filter out cards for such a region but the current
 566   // thread, if we're running concurrently, may "see" the young type
 567   // change at any time (so an earlier "is_young" check may pass or
 568   // fail arbitrarily). We tell the iteration code to perform this
 569   // filtering when it has been determined that there has been an actual
 570   // allocation in this region and making it safe to check the young type.
 571 
 572   bool card_processed =
 573     r->oops_on_card_seq_iterate_careful(dirtyRegion,
 574                                         &filter_then_update_rs_oop_cl,
 575                                         card_ptr);
 576 
 577   // If unable to process the card then we encountered an unparsable
 578   // part of the heap (e.g. a partially allocated object) while
 579   // processing a stale card.  Despite the card being stale, redirty
 580   // and re-enqueue, because we've already cleaned the card.  Without
 581   // this we could incorrectly discard a non-stale card.
 582   if (!card_processed) {
 583     assert(!_g1->is_gc_active(), "Unparsable heap during GC");
 584     // The card might have gotten re-dirtied and re-enqueued while we
 585     // worked.  (In fact, it's pretty likely.)
 586     if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
 587       *card_ptr = CardTableModRefBS::dirty_card_val();
 588       MutexLockerEx x(Shared_DirtyCardQ_lock,
 589                       Mutex::_no_safepoint_check_flag);
 590       DirtyCardQueue* sdcq =
 591         JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
 592       sdcq->enqueue(card_ptr);
 593     }
 594   } else {
 595     _conc_refine_cards++;




 443                            bool check_for_refs_into_cset) {
 444   assert(_g1->is_in_exact(_ct_bs->addr_for(card_ptr)),
 445          err_msg("Card at " PTR_FORMAT " index " SIZE_FORMAT " representing heap at " PTR_FORMAT " (%u) must be in committed heap",
 446                  p2i(card_ptr),
 447                  _ct_bs->index_for(_ct_bs->addr_for(card_ptr)),
 448                  _ct_bs->addr_for(card_ptr),
 449                  _g1->addr_to_region(_ct_bs->addr_for(card_ptr))));
 450 
 451   // If the card is no longer dirty, nothing to do.
 452   if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
 453     // No need to return that this card contains refs that point
 454     // into the collection set.
 455     return false;
 456   }
 457 
 458   // Construct the region representing the card.
 459   HeapWord* start = _ct_bs->addr_for(card_ptr);
 460   // And find the region containing it.
 461   HeapRegion* r = _g1->heap_region_containing(start);
 462 
 463   // This check is needed for some uncommon cases where we should
 464   // ignore the card.
 465   //
 466   // The region could be young.  Cards for young regions are
 467   // distinctly marked (set to g1_young_gen), so the post-barrier will
 468   // filter them out.  However, that marking is performed
 469   // concurrently.  A write to a young object could occur before the
 470   // card has been marked young, slipping past the filter.
 471   //
 472   // The card could be stale, because the region has been freed since
 473   // the card was recorded. In this case the region type could be
 474   // anything.  If (still) free or (reallocated) young, just ignore
 475   // it.  If (reallocated) old or humongous, the later card trimming
 476   // and additional checks in iteration may detect staleness.  At
 477   // worst, we end up processing a stale card unnecessarily.
 478   //
 479   // In the normal (non-stale) case, the synchronization between the
 480   // enqueueing of the card and processing it here will have ensured
 481   // we see the up-to-date region type here.
 482   if (!r->is_old_or_humongous()) {
 483     return false;
 484   }
 485 
 486   // While we are processing RSet buffers during the collection, we
 487   // actually don't want to scan any cards on the collection set,
 488   // since we don't want to update remebered sets with entries that
 489   // point into the collection set, given that live objects from the
 490   // collection set are about to move and such entries will be stale
 491   // very soon. This change also deals with a reliability issue which
 492   // involves scanning a card in the collection set and coming across
 493   // an array that was being chunked and looking malformed. Note,
 494   // however, that if evacuation fails, we have to scan any objects
 495   // that were not moved and create any missing entries.
 496   if (r->in_collection_set()) {
 497     return false;
 498   }
 499 
 500   // The result from the hot card cache insert call is either:
 501   //   * pointer to the current card
 502   //     (implying that the current card is not 'hot'),
 503   //   * null
 504   //     (meaning we had inserted the card ptr into the "hot" card cache,
 505   //     which had some headroom),
 506   //   * a pointer to a "hot" card that was evicted from the "hot" cache.
 507   //
 508 
 509   G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
 510   if (hot_card_cache->use_cache()) {
 511     assert(!check_for_refs_into_cset, "sanity");
 512     assert(!SafepointSynchronize::is_at_safepoint(), "sanity");
 513 
 514     const jbyte* orig_card_ptr = card_ptr;
 515     card_ptr = hot_card_cache->insert(card_ptr);
 516     if (card_ptr == NULL) {
 517       // There was no eviction. Nothing to do.
 518       return false;
 519     } else if (card_ptr != orig_card_ptr) {
 520       // Original card was inserted and an old card was evicted.
 521       start = _ct_bs->addr_for(card_ptr);
 522       r = _g1->heap_region_containing(start);
 523 
 524       // Check whether the region formerly in the cache should be
 525       // ignored, as discussed earlier for the original card.  The
 526       // region could have been freed while in the cache.  The cset is
 527       // not relevant here, since we're in concurrent phase.
 528       if (!r->is_old_or_humongous()) {
 529         return false;
 530       }
 531     } // Else we still have the original card.
 532   }
 533 
 534   // Trim the region designated by the card to what's been allocated
 535   // in the region.  The card could be stale, or the card could cover
 536   // (part of) an object at the end of the allocated space and extend
 537   // beyond the end of allocation.
 538   HeapWord* scan_limit;
 539   if (_g1->is_gc_active()) {
 540     // If we're in a STW GC, then a card might be in a GC alloc region
 541     // and extend onto a GC LAB, which may not be parsable.  Stop such
 542     // at the "scan_top" of the region.
 543     scan_limit = r->scan_top();
 544   } else {
 545     // Non-humongous objects are only allocated in the old-gen during
 546     // GC, so if region is old then top is stable.  Humongous object
 547     // allocation sets top last; if top has not yet been set, this is
 548     // a stale card and we'll end up with an empty intersection.  If
 549     // this is not a stale card, the synchronization between the
 550     // enqueuing of the card and processing it here will have ensured
 551     // we see the up-to-date top here.
 552     scan_limit = r->top();
 553   }
 554   if (scan_limit <= start) {
 555     // If the trimmed region is empty, the card must be stale.
 556     return false;
 557   }
 558 
 559   // Okay to clean and process the card now.  There are still some
 560   // stale card cases that may be detected by iteration and dealt with
 561   // as iteration failure.
 562   *const_cast<volatile jbyte*>(card_ptr) = CardTableModRefBS::clean_card_val();
 563 
 564   // This fence serves two purposes.  First, the card must be cleaned
 565   // before processing the contents.  Second, we can't proceed with
 566   // processing until after the read of top, for synchronization with
 567   // possibly concurrent humongous object allocation.  It's okay that
 568   // reading top and reading type were racy wrto each other.  We need
 569   // both set, in any order, to proceed.
 570   OrderAccess::fence();
 571 
 572   // Don't use addr_for(card_ptr + 1) which can ask for
 573   // a card beyond the heap.

 574   HeapWord* end = start + CardTableModRefBS::card_size_in_words;
 575   MemRegion dirty_region(start, MIN2(scan_limit, end));
 576   assert(!dirty_region.is_empty(), "sanity");
 577 
 578 #if CARD_REPEAT_HISTO
 579   init_ct_freq_table(_g1->max_capacity());
 580   ct_freq_note_card(_ct_bs->index_for(start));
 581 #endif
 582 
 583   G1ParPushHeapRSClosure* oops_in_heap_closure = NULL;
 584   if (check_for_refs_into_cset) {
 585     // ConcurrentG1RefineThreads have worker numbers larger than what
 586     // _cset_rs_update_cl[] is set up to handle. But those threads should
 587     // only be active outside of a collection which means that when they
 588     // reach here they should have check_for_refs_into_cset == false.
 589     assert((size_t)worker_i < n_workers(), "index of worker larger than _cset_rs_update_cl[].length");
 590     oops_in_heap_closure = _cset_rs_update_cl[worker_i];
 591   }
 592   G1UpdateRSOrPushRefOopClosure update_rs_oop_cl(_g1,
 593                                                  _g1->g1_rem_set(),
 594                                                  oops_in_heap_closure,
 595                                                  check_for_refs_into_cset,
 596                                                  worker_i);
 597   update_rs_oop_cl.set_from(r);
 598 
 599   G1TriggerClosure trigger_cl;
 600   FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl);
 601   G1InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl);
 602   G1Mux2Closure mux(&invoke_cl, &update_rs_oop_cl);
 603 
 604   FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r,
 605                         (check_for_refs_into_cset ?
 606                                 (OopClosure*)&mux :
 607                                 (OopClosure*)&update_rs_oop_cl));
 608 














 609   bool card_processed =
 610     r->oops_on_card_seq_iterate_careful(dirty_region,
 611                                         &filter_then_update_rs_oop_cl);

 612 
 613   // If unable to process the card then we encountered an unparsable
 614   // part of the heap (e.g. a partially allocated object) while
 615   // processing a stale card.  Despite the card being stale, redirty
 616   // and re-enqueue, because we've already cleaned the card.  Without
 617   // this we could incorrectly discard a non-stale card.
 618   if (!card_processed) {
 619     assert(!_g1->is_gc_active(), "Unparsable heap during GC");
 620     // The card might have gotten re-dirtied and re-enqueued while we
 621     // worked.  (In fact, it's pretty likely.)
 622     if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
 623       *card_ptr = CardTableModRefBS::dirty_card_val();
 624       MutexLockerEx x(Shared_DirtyCardQ_lock,
 625                       Mutex::_no_safepoint_check_flag);
 626       DirtyCardQueue* sdcq =
 627         JavaThread::dirty_card_queue_set().shared_dirty_card_queue();
 628       sdcq->enqueue(card_ptr);
 629     }
 630   } else {
 631     _conc_refine_cards++;


< prev index next >