--- old/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp 2011-12-20 11:11:09.808248531 -0800 +++ new/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp 2011-12-20 11:11:09.564464612 -0800 @@ -28,6 +28,129 @@ #include "gc_implementation/g1/concurrentMark.hpp" #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" +// Counts the given memory region in the given task/worker +// counting data structures. +inline void ConcurrentMark::count_region(MemRegion mr, HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm) { + G1CollectedHeap* g1h = _g1h; + HeapWord* start = mr.start(); + HeapWord* last = mr.last(); + size_t region_size = mr.byte_size(); + size_t index = hr->hrs_index(); + + assert(!hr->continuesHumongous(), "should not be HC region"); + assert(hr == g1h->heap_region_containing(start), "sanity"); + assert(hr == g1h->heap_region_containing(mr.last()), "sanity"); + assert(marked_bytes_array != NULL, "pre-condition"); + assert(task_card_bm != NULL, "pre-condition"); + + // Add to the task local marked bytes for this region. + marked_bytes_array[index] += region_size; + + // Below, the term "card num" means the result of shifting an address + // by the card shift -- address 0 corresponds to card number 0. One + // must subtract the card num of the bottom of the heap to obtain a + // card table index. + + intptr_t start_card_num = intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift); + intptr_t last_card_num = intptr_t(uintptr_t(last) >> CardTableModRefBS::card_shift); + + BitMap::idx_t start_idx = start_card_num - heap_bottom_card_num(); + BitMap::idx_t last_idx = last_card_num - heap_bottom_card_num(); + + // The card bitmap is task/worker specific => no need to use 'par' routines. + // Set bits in the inclusive bit range [start_idx, last_idx]. + for (BitMap::idx_t i = start_idx; i <= last_idx; i += 1) { + task_card_bm->set_bit(i); + } +} + +// Counts the given memory region in the ask/worker counting +// data structures for the given worker id. +inline void ConcurrentMark::count_region(MemRegion mr, int worker_i) { + size_t* marked_bytes_array = count_marked_bytes_array_for(worker_i); + BitMap* task_card_bm = count_card_bitmap_for(worker_i); + HeapWord* addr = mr.start(); + HeapRegion* hr = _g1h->heap_region_containing(addr); + count_region(mr, hr, marked_bytes_array, task_card_bm); +} + +// Counts the given object in the given task/worker counting data structures. +inline void ConcurrentMark::count_object(oop obj, + HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm) { + MemRegion mr((HeapWord*)obj, obj->size()); + count_region(mr, hr, marked_bytes_array, task_card_bm); +} + +// Counts the given object in the task/worker counting data +// structures for the given worker id. +inline void ConcurrentMark::count_object(oop obj, HeapRegion* hr, int worker_i) { + size_t* marked_bytes_array = count_marked_bytes_array_for(worker_i); + BitMap* task_card_bm = count_card_bitmap_for(worker_i); + HeapWord* addr = (HeapWord*) obj; + count_object(obj, hr, marked_bytes_array, task_card_bm); +} + +// Attempts to mark the given object and, if successful, counts +// the object in the given task/worker counting structures. +inline bool ConcurrentMark::par_mark_and_count(oop obj, + HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm) { + HeapWord* addr = (HeapWord*)obj; + if (_nextMarkBitMap->parMark(addr)) { + // Update the task specific count data for the object. + count_object(obj, hr, marked_bytes_array, task_card_bm); + return true; + } + return false; +} + +// Attempts to mark the given object and, if successful, counts +// the object in the task/worker counting structures for the +// given worker id. +inline bool ConcurrentMark::par_mark_and_count(oop obj, + HeapRegion* hr, + int worker_i) { + HeapWord* addr = (HeapWord*)obj; + if (_nextMarkBitMap->parMark(addr)) { + // Update the task specific count data for the object. + count_object(obj, hr, worker_i); + return true; + } + return false; +} + +// As above - but we don't know the heap region containing the +// object and so have to supply it. +inline bool ConcurrentMark::par_mark_and_count(oop obj, int worker_i) { + HeapWord* addr = (HeapWord*)obj; + HeapRegion* hr = _g1h->heap_region_containing(addr); + return par_mark_and_count(obj, hr, worker_i); +} + +// Unconditionally mark the given object, and unconditinally count +// the object in the counting structures for worker id 0. +// Should *not* be called from parallel code. +inline bool ConcurrentMark::mark_and_count(oop obj, HeapRegion* hr) { + HeapWord* addr = (HeapWord*)obj; + _nextMarkBitMap->mark(addr); + // Update the task specific count data for the object. + count_object(obj, hr, 0 /* worker_i */); + return true; +} + +// As above - but we don't have the heap region containing the +// object, so we have to supply it. +inline bool ConcurrentMark::mark_and_count(oop obj) { + HeapWord* addr = (HeapWord*)obj; + HeapRegion* hr = _g1h->heap_region_containing(addr); + return mark_and_count(obj, hr); +} + inline void CMTask::push(oop obj) { HeapWord* objAddr = (HeapWord*) obj; assert(_g1h->is_in_g1_reserved(objAddr), "invariant"); @@ -84,7 +207,7 @@ HeapWord* objAddr = (HeapWord*) obj; assert(obj->is_oop_or_null(true /* ignore mark word */), "Error"); - if (_g1h->is_in_g1_reserved(objAddr)) { + if (_g1h->is_in_g1_reserved(objAddr)) { assert(obj != NULL, "null check is implicit"); if (!_nextMarkBitMap->isMarked(objAddr)) { // Only get the containing region if the object is not marked on the @@ -98,9 +221,9 @@ } // we need to mark it first - if (_nextMarkBitMap->parMark(objAddr)) { + if (_cm->par_mark_and_count(obj, hr, _marked_bytes_array, _card_bm)) { // No OrderAccess:store_load() is needed. It is implicit in the - // CAS done in parMark(objAddr) above + // CAS done in CMBitMap::parMark() call in the routine above. HeapWord* global_finger = _cm->finger(); #if _CHECK_BOTH_FINGERS_