Print this page
rev 2896 : 6484965: G1: piggy-back liveness accounting phase on marking
Summary: Remove the separate counting phase of concurrent marking by tracking the amount of marked bytes and the cards spanned by marked objects in marking task/worker thread local data structures, which are updated as individual objects are marked.
Reviewed-by: brutisso

Split Close
Expand all
Collapse all
          --- old/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp
          +++ new/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp
↓ open down ↓ 20 lines elided ↑ open up ↑
  21   21   * questions.
  22   22   *
  23   23   */
  24   24  
  25   25  #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP
  26   26  #define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP
  27   27  
  28   28  #include "gc_implementation/g1/concurrentMark.hpp"
  29   29  #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
  30   30  
       31 +// Counts the given memory region in the given task/worker
       32 +// counting data structures.
       33 +inline void ConcurrentMark::count_region(MemRegion mr, HeapRegion* hr,
       34 +                                         size_t* marked_bytes_array,
       35 +                                         BitMap* task_card_bm) {
       36 +  G1CollectedHeap* g1h = _g1h;
       37 +  HeapWord* start = mr.start();
       38 +  HeapWord* last = mr.last();
       39 +  size_t region_size = mr.byte_size();
       40 +  size_t index = hr->hrs_index();
       41 +
       42 +  assert(!hr->continuesHumongous(), "should not be HC region");
       43 +  assert(hr == g1h->heap_region_containing(start), "sanity");
       44 +  assert(hr == g1h->heap_region_containing(mr.last()), "sanity");
       45 +  assert(marked_bytes_array != NULL, "pre-condition");
       46 +  assert(task_card_bm != NULL, "pre-condition");
       47 +
       48 +  // Add to the task local marked bytes for this region.
       49 +  marked_bytes_array[index] += region_size;
       50 +
       51 +  // Below, the term "card num" means the result of shifting an address
       52 +  // by the card shift -- address 0 corresponds to card number 0.  One
       53 +  // must subtract the card num of the bottom of the heap to obtain a
       54 +  // card table index.
       55 +
       56 +  intptr_t start_card_num = intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
       57 +  intptr_t last_card_num  = intptr_t(uintptr_t(last) >> CardTableModRefBS::card_shift);
       58 +
       59 +  BitMap::idx_t start_idx = start_card_num - heap_bottom_card_num();
       60 +  BitMap::idx_t last_idx = last_card_num - heap_bottom_card_num();
       61 +  
       62 +  // The card bitmap is task/worker specific => no need to use 'par' routines.
       63 +  // Set bits in the inclusive bit range [start_idx, last_idx].
       64 +  for (BitMap::idx_t i = start_idx; i <= last_idx; i += 1) {
       65 +    task_card_bm->set_bit(i);
       66 +  }
       67 +}
       68 +
       69 +// Counts the given memory region in the ask/worker counting
       70 +// data structures for the given worker id.
       71 +inline void ConcurrentMark::count_region(MemRegion mr, int worker_i) {
       72 +  size_t* marked_bytes_array = count_marked_bytes_array_for(worker_i);
       73 +  BitMap* task_card_bm = count_card_bitmap_for(worker_i);
       74 +  HeapWord* addr = mr.start();
       75 +  HeapRegion* hr = _g1h->heap_region_containing(addr);
       76 +  count_region(mr, hr, marked_bytes_array, task_card_bm);
       77 +}
       78 +
       79 +// Counts the given object in the given task/worker counting data structures.
       80 +inline void ConcurrentMark::count_object(oop obj,
       81 +                                         HeapRegion* hr,
       82 +                                         size_t* marked_bytes_array,
       83 +                                         BitMap* task_card_bm) {
       84 +  MemRegion mr((HeapWord*)obj, obj->size());
       85 +  count_region(mr, hr, marked_bytes_array, task_card_bm);
       86 +}
       87 +
       88 +// Counts the given object in the task/worker counting data
       89 +// structures for the given worker id.
       90 +inline void ConcurrentMark::count_object(oop obj, HeapRegion* hr, int worker_i) {
       91 +  size_t* marked_bytes_array = count_marked_bytes_array_for(worker_i);
       92 +  BitMap* task_card_bm = count_card_bitmap_for(worker_i);
       93 +  HeapWord* addr = (HeapWord*) obj;
       94 +  count_object(obj, hr, marked_bytes_array, task_card_bm);
       95 +}
       96 +
       97 +// Attempts to mark the given object and, if successful, counts
       98 +// the object in the given task/worker counting structures.
       99 +inline bool ConcurrentMark::par_mark_and_count(oop obj,
      100 +                                               HeapRegion* hr,
      101 +                                               size_t* marked_bytes_array,
      102 +                                               BitMap* task_card_bm) {
      103 +  HeapWord* addr = (HeapWord*)obj;
      104 +  if (_nextMarkBitMap->parMark(addr)) {
      105 +    // Update the task specific count data for the object.
      106 +    count_object(obj, hr, marked_bytes_array, task_card_bm);
      107 +    return true;
      108 +  }
      109 +  return false;
      110 +}
      111 +
      112 +// Attempts to mark the given object and, if successful, counts
      113 +// the object in the task/worker counting structures for the
      114 +// given worker id.
      115 +inline bool ConcurrentMark::par_mark_and_count(oop obj,
      116 +                                               HeapRegion* hr,
      117 +                                               int worker_i) {
      118 +  HeapWord* addr = (HeapWord*)obj;
      119 +  if (_nextMarkBitMap->parMark(addr)) {
      120 +    // Update the task specific count data for the object.
      121 +    count_object(obj, hr, worker_i);
      122 +    return true;
      123 +  }
      124 +  return false;
      125 +}
      126 +
      127 +// As above - but we don't know the heap region containing the
      128 +// object and so have to supply it.
      129 +inline bool ConcurrentMark::par_mark_and_count(oop obj, int worker_i) {
      130 +  HeapWord* addr = (HeapWord*)obj;
      131 +  HeapRegion* hr = _g1h->heap_region_containing(addr);
      132 +  return par_mark_and_count(obj, hr, worker_i);
      133 +}
      134 +
      135 +// Unconditionally mark the given object, and unconditinally count
      136 +// the object in the counting structures for worker id 0.
      137 +// Should *not* be called from parallel code.
      138 +inline bool ConcurrentMark::mark_and_count(oop obj, HeapRegion* hr) {
      139 +  HeapWord* addr = (HeapWord*)obj;
      140 +  _nextMarkBitMap->mark(addr);
      141 +  // Update the task specific count data for the object.
      142 +  count_object(obj, hr, 0 /* worker_i */);
      143 +  return true;
      144 +}
      145 +
      146 +// As above - but we don't have the heap region containing the
      147 +// object, so we have to supply it.
      148 +inline bool ConcurrentMark::mark_and_count(oop obj) {
      149 +  HeapWord* addr = (HeapWord*)obj;
      150 +  HeapRegion* hr = _g1h->heap_region_containing(addr);
      151 +  return mark_and_count(obj, hr);
      152 +}
      153 +
  31  154  inline void CMTask::push(oop obj) {
  32  155    HeapWord* objAddr = (HeapWord*) obj;
  33  156    assert(_g1h->is_in_g1_reserved(objAddr), "invariant");
  34  157    assert(!_g1h->is_on_master_free_list(
  35  158                _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant");
  36  159    assert(!_g1h->is_obj_ill(obj), "invariant");
  37  160    assert(_nextMarkBitMap->isMarked(objAddr), "invariant");
  38  161  
  39  162    if (_cm->verbose_high()) {
  40  163      gclog_or_tty->print_cr("[%d] pushing "PTR_FORMAT, _task_id, (void*) obj);
↓ open down ↓ 36 lines elided ↑ open up ↑
  77  200  inline void CMTask::deal_with_reference(oop obj) {
  78  201    if (_cm->verbose_high()) {
  79  202      gclog_or_tty->print_cr("[%d] we're dealing with reference = "PTR_FORMAT,
  80  203                             _task_id, (void*) obj);
  81  204    }
  82  205  
  83  206    ++_refs_reached;
  84  207  
  85  208    HeapWord* objAddr = (HeapWord*) obj;
  86  209    assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
  87      - if (_g1h->is_in_g1_reserved(objAddr)) {
      210 +  if (_g1h->is_in_g1_reserved(objAddr)) {
  88  211      assert(obj != NULL, "null check is implicit");
  89  212      if (!_nextMarkBitMap->isMarked(objAddr)) {
  90  213        // Only get the containing region if the object is not marked on the
  91  214        // bitmap (otherwise, it's a waste of time since we won't do
  92  215        // anything with it).
  93  216        HeapRegion* hr = _g1h->heap_region_containing_raw(obj);
  94  217        if (!hr->obj_allocated_since_next_marking(obj)) {
  95  218          if (_cm->verbose_high()) {
  96  219            gclog_or_tty->print_cr("[%d] "PTR_FORMAT" is not considered marked",
  97  220                                   _task_id, (void*) obj);
  98  221          }
  99  222  
 100  223          // we need to mark it first
 101      -        if (_nextMarkBitMap->parMark(objAddr)) {
      224 +        if (_cm->par_mark_and_count(obj, hr, _marked_bytes_array, _card_bm)) {
 102  225            // No OrderAccess:store_load() is needed. It is implicit in the
 103      -          // CAS done in parMark(objAddr) above
      226 +          // CAS done in CMBitMap::parMark() call in the routine above.
 104  227            HeapWord* global_finger = _cm->finger();
 105  228  
 106  229  #if _CHECK_BOTH_FINGERS_
 107  230            // we will check both the local and global fingers
 108  231  
 109  232            if (_finger != NULL && objAddr < _finger) {
 110  233              if (_cm->verbose_high()) {
 111  234                gclog_or_tty->print_cr("[%d] below the local finger ("PTR_FORMAT"), "
 112  235                                       "pushing it", _task_id, _finger);
 113  236              }
↓ open down ↓ 43 lines elided ↑ open up ↑
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX