--- old/src/share/vm/gc/g1/g1ConcurrentMark.cpp 2016-03-18 13:05:50.585143917 +0100 +++ new/src/share/vm/gc/g1/g1ConcurrentMark.cpp 2016-03-18 13:05:50.489141051 +0100 @@ -521,12 +521,6 @@ _accum_task_vtime[i] = 0.0; } - // Calculate the card number for the bottom of the heap. Used - // in biasing indexes into the accounting card bitmaps. - _heap_bottom_card_num = - intptr_t(uintptr_t(_g1h->reserved_region().start()) >> - CardTableModRefBS::card_shift); - // so that the call below can read a sensible value _heap_start = g1h->reserved_region().start(); set_non_marking_state(); @@ -1131,32 +1125,27 @@ _gc_tracer_cm->report_object_count_after_gc(&is_alive); } -// Base class of the closures that finalize and verify the -// liveness count data. -class G1LiveDataClosureBase: public HeapRegionClosure { -protected: - G1ConcurrentMark* _cm; - +// Helper class that provides functionality to generate the Live Data Count +// information. +class G1LiveDataHelper VALUE_OBJ_CLASS_SPEC { +private: BitMap* _region_bm; BitMap* _card_bm; - // Takes a region that's not empty (i.e., it has at least one - // live object in it and sets its corresponding bit on the region - // bitmap to 1. - void set_bit_for_region(HeapRegion* hr) { - BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index(); - _region_bm->par_at_put(index, true); - } - - // Utility routine to set an exclusive range of cards on the given - // bitmap. - inline void set_card_bitmap_range(BitMap* card_bm, + // The card number of the bottom of the G1 heap. Used for converting addresses + // to bitmap indices quickly. + BitMap::idx_t _heap_card_bias; + + // Utility routine to set an exclusive range of bits on the given + // bitmap, optimized for very small ranges. + // There must be at least one bit to set. + inline void set_card_bitmap_range(BitMap* bm, BitMap::idx_t start_idx, BitMap::idx_t end_idx) { // Set the exclusive bit range [start_idx, end_idx). - assert((end_idx - start_idx) > 0, "at least one card"); - assert(end_idx <= card_bm->size(), "sanity"); + assert((end_idx - start_idx) > 0, "at least one bit"); + assert(end_idx <= bm->size(), "sanity"); // For small ranges use a simple loop; otherwise use set_range or // use par_at_put_range (if parallel). The range is made up of the @@ -1164,16 +1153,22 @@ // allow up to object sizes up to 4K to be handled using the loop. if ((end_idx - start_idx) <= 8) { for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) { - card_bm->set_bit(i); + bm->set_bit(i); } } else { - card_bm->set_range(start_idx, end_idx); + bm->set_range(start_idx, end_idx); } } + // We cache the last mark set. This avoids setting the same bit multiple times. + // This is particularly interesting for dense bitmaps, as this avoids doing + // lots of work most of the time. + BitMap::idx_t _last_marked_bit_idx; + + // Mark the card liveness bitmap for the object spanning from start to end. void mark_card_bitmap_range(HeapWord* start, HeapWord* end) { - BitMap::idx_t start_idx = _cm->card_live_bitmap_index_for(start); - BitMap::idx_t end_idx = _cm->card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size)); + BitMap::idx_t start_idx = card_live_bitmap_index_for(start); + BitMap::idx_t end_idx = card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size)); assert((end_idx - start_idx) > 0, "Trying to mark zero sized range."); @@ -1189,16 +1184,34 @@ _last_marked_bit_idx = end_idx - 1; } - // We cache the last mark set. This avoids setting the same bit multiple times. - // This is particularly interesting for dense bitmaps, as this avoids doing - // any work most of the time. - BitMap::idx_t _last_marked_bit_idx; - void reset_mark_cache() { _last_marked_bit_idx = (BitMap::idx_t)-1; } - void mark_allocated_since_marking(HeapRegion* hr) { +public: + // Returns the index in the per-card liveness count bitmap + // for the given address + inline BitMap::idx_t card_live_bitmap_index_for(HeapWord* addr) { + // Below, the term "card num" means the result of shifting an address + // by the card shift -- address 0 corresponds to card number 0. One + // must subtract the card num of the bottom of the heap to obtain a + // card table index. + BitMap::idx_t card_num = (BitMap::idx_t)(uintptr_t(addr) >> CardTableModRefBS::card_shift); + return card_num - _heap_card_bias; + } + + // Takes a region that's not empty (i.e., it has at least one + // live object in it and sets its corresponding bit on the region + // bitmap to 1. + void set_bit_for_region(HeapRegion* hr) { + BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index(); + _region_bm->par_at_put(index, true); + } + + // Mark the range of bits covered by allocations done since the last marking + // in the given heap region, i.e. from NTAMS to top of the given region. + // Returns if there has been some allocation in this region since the last marking. + bool mark_allocated_since_marking(HeapRegion* hr) { reset_mark_cache(); HeapWord* ntams = hr->next_top_at_mark_start(); @@ -1209,12 +1222,17 @@ // Mark the allocated-since-marking portion... if (ntams < top) { mark_card_bitmap_range(ntams, top); - // This definitely means the region has live objects. - set_bit_for_region(hr); + return true; + } else { + return false; } } - bool mark_marked_during_marking(HeapRegion* hr, bool may_suspend, size_t* total_bytes_marked) { + // Mark the range of bits covered by live objects on the mark bitmap between + // bottom and NTAMS of the given region. + // Returns the number of live bytes marked within that area for the given + // heap region. + size_t mark_marked_during_marking(G1CMBitMap* mark_bitmap, HeapRegion* hr) { reset_mark_cache(); size_t marked_bytes = 0; @@ -1223,26 +1241,11 @@ HeapWord* start = hr->bottom(); if (ntams <= start) { - // Empty region (at the start of marking). Nothing to do. - // hr->add_to_marked_bytes(0); - *total_bytes_marked = marked_bytes; - return false; - } else if (hr->is_starts_humongous()) { - // Humongous object: distribute the marked bytes across the humongous object. - do { - mark_card_bitmap_range(start, hr->top()); - - marked_bytes += pointer_delta(hr->top(), start, 1); - hr->add_to_marked_bytes(marked_bytes); - - hr = G1CollectedHeap::heap()->next_region_in_humongous(hr); - } while (hr != NULL); - *total_bytes_marked = marked_bytes; - return false; - } else if (hr->is_continues_humongous()) { - // Humongous continues regions were handled during processing of the start region. - *total_bytes_marked = marked_bytes; - return false; + // Skip empty regions. + return 0; + } else if (hr->is_humongous()) { + mark_card_bitmap_range(start, hr->top()); + return pointer_delta(hr->top(), start, 1); } assert(start <= hr->end() && start <= ntams && ntams <= hr->end(), @@ -1250,9 +1253,8 @@ "start: " PTR_FORMAT ", ntams: " PTR_FORMAT ", end: " PTR_FORMAT, p2i(start), p2i(ntams), p2i(hr->end())); - G1CMBitMap* bitmap = _cm->nextMarkBitMap(); // Find the first marked object at or after "start". - start = bitmap->getNextMarkedWordAddress(start, ntams); + start = mark_bitmap->getNextMarkedWordAddress(start, ntams); while (start < ntams) { oop obj = oop(start); int obj_sz = obj->size(); @@ -1266,28 +1268,23 @@ marked_bytes += (size_t)obj_sz * HeapWordSize; // Find the next marked object after this one. - start = bitmap->getNextMarkedWordAddress(obj_end, ntams); + start = mark_bitmap->getNextMarkedWordAddress(obj_end, ntams); } - // Update the marked bytes for this region. - hr->add_to_marked_bytes(marked_bytes); - *total_bytes_marked = marked_bytes; - - // Abort iteration if after yielding the marking has been aborted. - if (may_suspend && _cm->do_yield_check() && _cm->has_aborted()) { - return true; - } - // Next heap region - return false; + return marked_bytes; } -public: - G1LiveDataClosureBase(G1CollectedHeap* g1h, - BitMap* region_bm, - BitMap* card_bm): - _cm(g1h->concurrent_mark()), + G1LiveDataHelper(BitMap* region_bm, + BitMap* card_bm): _region_bm(region_bm), - _card_bm(card_bm) { } + _card_bm(card_bm) { + //assert(region_bm != NULL, ""); + assert(card_bm != NULL, ""); + // Calculate the card number for the bottom of the heap. Used + // in biasing indexes into the accounting card bitmaps. + _heap_card_bias = + (BitMap::idx_t)(uintptr_t(G1CollectedHeap::heap()->reserved_region().start()) >> CardTableModRefBS::card_shift); + } }; // Heap region closure used for verifying the live count data @@ -1295,46 +1292,40 @@ // the remark pause. This closure is applied to the heap // regions during the STW cleanup pause. class G1VerifyLiveDataHRClosure: public HeapRegionClosure { - // Calculates the # live objects per region. - class G1VerifyLiveDataClosure: public G1LiveDataClosureBase { - size_t _region_marked_bytes; - - public: - G1VerifyLiveDataClosure(G1CollectedHeap* g1h, - BitMap* region_bm, - BitMap* card_bm) : - G1LiveDataClosureBase(g1h, region_bm, card_bm), - _region_marked_bytes(0) { } - - bool doHeapRegion(HeapRegion* hr) { - mark_marked_during_marking(hr, false, &_region_marked_bytes); - mark_allocated_since_marking(hr); - return false; - } - - size_t region_marked_bytes() const { return _region_marked_bytes; } - }; - +private: G1CollectedHeap* _g1h; - G1ConcurrentMark* _cm; - G1VerifyLiveDataClosure _calc_cl; - BitMap* _act_region_bm; // Region BM to be verified - BitMap* _act_card_bm; // Card BM to be verified + G1CMBitMap* _mark_bitmap; + G1LiveDataHelper _calc_helper; + + BitMap* _act_region_bm; // Region BM to be verified + BitMap* _act_card_bm; // Card BM to be verified BitMap* _exp_region_bm; // Expected Region BM values BitMap* _exp_card_bm; // Expected card BM values int _failures; + // Updates the live data count for the given heap region and returns the number + // of bytes marked. + size_t create_live_data_count(HeapRegion* hr) { + size_t bytes_marked = _calc_helper.mark_marked_during_marking(_mark_bitmap, hr); + bool allocated_since_marking = _calc_helper.mark_allocated_since_marking(hr); + if (allocated_since_marking || bytes_marked > 0) { + _calc_helper.set_bit_for_region(hr); + } + return bytes_marked; + } + public: G1VerifyLiveDataHRClosure(G1CollectedHeap* g1h, + G1CMBitMap* mark_bitmap, BitMap* act_region_bm, BitMap* act_card_bm, BitMap* exp_region_bm, BitMap* exp_card_bm) : _g1h(g1h), - _cm(g1h->concurrent_mark()), - _calc_cl(g1h, exp_region_bm, exp_card_bm), + _mark_bitmap(mark_bitmap), + _calc_helper(exp_region_bm, exp_card_bm), _act_region_bm(act_region_bm), _act_card_bm(act_card_bm), _exp_region_bm(exp_region_bm), @@ -1346,35 +1337,16 @@ bool doHeapRegion(HeapRegion* hr) { int failures = 0; - // Call the G1VerifyLiveDataClosure to walk the marking bitmap for - // this region and set the corresponding bits in the expected region - // and card bitmaps. - bool res = _calc_cl.doHeapRegion(hr); - assert(!res, "Should be completed."); - - // Verify the marked bytes for this region. - size_t exp_marked_bytes = _calc_cl.region_marked_bytes(); + // Walk the marking bitmap for this region and set the corresponding bits + // in the expected region and card bitmaps. + size_t exp_marked_bytes = create_live_data_count(hr); size_t act_marked_bytes = hr->next_marked_bytes(); + // Verify the marked bytes for this region. - if (exp_marked_bytes > act_marked_bytes) { - if (hr->is_starts_humongous()) { - // For start_humongous regions, the size of the whole object will be - // in exp_marked_bytes. - HeapRegion* region = hr; - int num_regions; - for (num_regions = 0; region != NULL; num_regions++) { - region = _g1h->next_region_in_humongous(region); - } - if ((num_regions-1) * HeapRegion::GrainBytes >= exp_marked_bytes) { - failures += 1; - } else if (num_regions * HeapRegion::GrainBytes < exp_marked_bytes) { - failures += 1; - } - } else { - // We're not OK if expected marked bytes > actual marked bytes. It means - // we have missed accounting some objects during the actual marking. - failures += 1; - } + if (exp_marked_bytes != act_marked_bytes) { + failures += 1; + } else if (exp_marked_bytes > HeapRegion::GrainBytes) { + failures += 1; } // Verify the bit, for this region, in the actual and expected @@ -1393,8 +1365,8 @@ // region match. We have an error if we have a set bit in the expected // bit map and the corresponding bit in the actual bitmap is not set. - BitMap::idx_t start_idx = _cm->card_live_bitmap_index_for(hr->bottom()); - BitMap::idx_t end_idx = _cm->card_live_bitmap_index_for(hr->top()); + BitMap::idx_t start_idx = _calc_helper.card_live_bitmap_index_for(hr->bottom()); + BitMap::idx_t end_idx = _calc_helper.card_live_bitmap_index_for(hr->top()); for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) { expected = _exp_card_bm->at(i); @@ -1416,6 +1388,7 @@ class G1VerifyLiveDataTask: public AbstractGangTask { protected: G1CollectedHeap* _g1h; + G1CMBitMap* _mark_bitmap; BitMap* _actual_region_bm; BitMap* _actual_card_bm; @@ -1428,11 +1401,13 @@ public: G1VerifyLiveDataTask(G1CollectedHeap* g1h, + G1CMBitMap* bitmap, BitMap* region_bm, BitMap* card_bm, uint n_workers) : AbstractGangTask("G1 verify final counting"), _g1h(g1h), + _mark_bitmap(bitmap), _actual_region_bm(region_bm), _actual_card_bm(card_bm), _expected_region_bm(region_bm->size(), true /* in_resource_area */), @@ -1444,6 +1419,7 @@ void work(uint worker_id) { G1VerifyLiveDataHRClosure cl(_g1h, + _mark_bitmap, _actual_region_bm, _actual_card_bm, &_expected_region_bm, @@ -1462,45 +1438,47 @@ // (which contains the implicitly live objects) in the // card liveness bitmap. Also sets the bit for each region // containing live data, in the region liveness bitmap. - class G1FinalizeCountDataClosure: public G1LiveDataClosureBase { + class G1FinalizeCountDataClosure: public HeapRegionClosure { + private: + G1LiveDataHelper _helper; public: - G1FinalizeCountDataClosure(G1CollectedHeap* g1h, + G1FinalizeCountDataClosure(G1CMBitMap* bitmap, BitMap* region_bm, BitMap* card_bm) : - G1LiveDataClosureBase(g1h, region_bm, card_bm) { } + HeapRegionClosure(), + _helper(region_bm, card_bm) { } bool doHeapRegion(HeapRegion* hr) { - mark_allocated_since_marking(hr); - // Set the bit for the region if it contains live data - if (hr->next_marked_bytes() > 0) { - set_bit_for_region(hr); + bool allocated_since_marking = _helper.mark_allocated_since_marking(hr); + if (allocated_since_marking || hr->next_marked_bytes() > 0) { + _helper.set_bit_for_region(hr); } - return false; } }; - G1CollectedHeap* _g1h; + G1CMBitMap* _bitmap; + BitMap* _actual_region_bm; BitMap* _actual_card_bm; HeapRegionClaimer _hr_claimer; public: - G1FinalizeLiveDataTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm, uint n_workers) : + G1FinalizeLiveDataTask(G1CMBitMap* bitmap, BitMap* region_bm, BitMap* card_bm, uint n_workers) : AbstractGangTask("G1 final counting"), - _g1h(g1h), + _bitmap(bitmap), _actual_region_bm(region_bm), _actual_card_bm(card_bm), _hr_claimer(n_workers) { } void work(uint worker_id) { - G1FinalizeCountDataClosure cl(_g1h, + G1FinalizeCountDataClosure cl(_bitmap, _actual_region_bm, _actual_card_bm); - _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer); + G1CollectedHeap::heap()->heap_region_par_iterate(&cl, worker_id, &_hr_claimer); } }; @@ -1636,7 +1614,7 @@ { // Finalize the live data. - G1FinalizeLiveDataTask cl(g1h, + G1FinalizeLiveDataTask cl(_nextMarkBitMap, &_region_live_bm, &_card_live_bm, g1h->workers()->active_workers()); @@ -1647,7 +1625,8 @@ // Verify that the liveness count data created concurrently matches one created // during this safepoint. ResourceMark rm; - G1VerifyLiveDataTask cl(g1h, + G1VerifyLiveDataTask cl(G1CollectedHeap::heap(), + _nextMarkBitMap, &_region_live_bm, &_card_live_bm, g1h->workers()->active_workers()); @@ -2432,22 +2411,38 @@ class G1CreateLiveDataTask: public AbstractGangTask { // Aggregate the counting data that was constructed concurrently // with marking. - class G1CreateLiveDataHRClosure: public G1LiveDataClosureBase { + class G1CreateLiveDataHRClosure: public HeapRegionClosure { + G1LiveDataHelper _helper; + + G1CMBitMap* _mark_bitmap; + + G1ConcurrentMark* _cm; public: - G1CreateLiveDataHRClosure(G1CollectedHeap* g1h, - BitMap* cm_card_bm) - : G1LiveDataClosureBase(g1h, NULL, cm_card_bm) { } + G1CreateLiveDataHRClosure(G1ConcurrentMark* cm, + G1CMBitMap* mark_bitmap, + BitMap* cm_card_bm) : + HeapRegionClosure(), + _helper(NULL, cm_card_bm), + _mark_bitmap(mark_bitmap), + _cm(cm) { } bool doHeapRegion(HeapRegion* hr) { - size_t temp; - return mark_marked_during_marking(hr, true, &temp); + size_t marked_bytes = _helper.mark_marked_during_marking(_mark_bitmap, hr); + if (marked_bytes > 0) { + hr->add_to_marked_bytes(marked_bytes); + } + + if (_cm->do_yield_check() && _cm->has_aborted()) { + return true; + } + return false; } }; G1CollectedHeap* _g1h; G1ConcurrentMark* _cm; BitMap* _cm_card_bm; - HeapRegionClaimer _hrclaimer; + HeapRegionClaimer _hr_claimer; public: G1CreateLiveDataTask(G1CollectedHeap* g1h, @@ -2456,14 +2451,14 @@ AbstractGangTask("Create Live Data"), _g1h(g1h), _cm_card_bm(cm_card_bm), - _hrclaimer(n_workers) { + _hr_claimer(n_workers) { } void work(uint worker_id) { SuspendibleThreadSetJoiner sts_join; - G1CreateLiveDataHRClosure cl(_g1h, _cm_card_bm); - _g1h->heap_region_par_iterate(&cl, worker_id, &_hrclaimer); + G1CreateLiveDataHRClosure cl(_g1h->concurrent_mark(), _g1h->concurrent_mark()->nextMarkBitMap(), _cm_card_bm); + _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer); } };