50 free_large_bitmap(_live_cards, _live_cards_size_in_bits);
51 free_large_bitmap(_live_regions, _live_regions_size_in_bits);
52 }
53
54 G1CardLiveData::bm_word_t* G1CardLiveData::allocate_large_bitmap(size_t size_in_bits) {
55 size_t size_in_words = BitMap::calc_size_in_words(size_in_bits);
56
57 bm_word_t* map = MmapArrayAllocator<bm_word_t, mtGC>::allocate(size_in_words);
58
59 return map;
60 }
61
62 void G1CardLiveData::free_large_bitmap(bm_word_t* bitmap, size_t size_in_bits) {
63 MmapArrayAllocator<bm_word_t, mtGC>::free(bitmap, BitMap::calc_size_in_words(size_in_bits));
64 }
65
66 void G1CardLiveData::initialize(size_t max_capacity, uint num_max_regions) {
67 assert(max_capacity % num_max_regions == 0,
68 "Given capacity must be evenly divisible by region size.");
69 size_t region_size = max_capacity / num_max_regions;
70 assert(region_size % (G1SATBCardTableModRefBS::card_size * BitsPerWord) == 0,
71 "Region size must be evenly divisible by area covered by a single word.");
72 _max_capacity = max_capacity;
73 _cards_per_region = region_size / G1SATBCardTableModRefBS::card_size;
74
75 _live_regions_size_in_bits = live_region_bitmap_size_in_bits();
76 _live_regions = allocate_large_bitmap(_live_regions_size_in_bits);
77 _live_cards_size_in_bits = live_card_bitmap_size_in_bits();
78 _live_cards = allocate_large_bitmap(_live_cards_size_in_bits);
79 }
80
81 void G1CardLiveData::pretouch() {
82 live_cards_bm().pretouch();
83 live_regions_bm().pretouch();
84 }
85
86 size_t G1CardLiveData::live_region_bitmap_size_in_bits() const {
87 return _max_capacity / (_cards_per_region << G1SATBCardTableModRefBS::card_shift);
88 }
89
90 size_t G1CardLiveData::live_card_bitmap_size_in_bits() const {
91 return _max_capacity >> G1SATBCardTableModRefBS::card_shift;
92 }
93
94 // Helper class that provides functionality to generate the Live Data Count
95 // information.
96 class G1CardLiveDataHelper VALUE_OBJ_CLASS_SPEC {
97 private:
98 BitMapView _region_bm;
99 BitMapView _card_bm;
100
101 // The card number of the bottom of the G1 heap.
102 // Used in biasing indices into accounting card bitmaps.
103 BitMap::idx_t _heap_card_bias;
104
105 // Utility routine to set an exclusive range of bits on the given
106 // bitmap, optimized for very small ranges.
107 // There must be at least one bit to set.
108 void set_card_bitmap_range(BitMap::idx_t start_idx,
109 BitMap::idx_t end_idx) {
110
111 // Set the exclusive bit range [start_idx, end_idx).
114 // For small ranges use a simple loop; otherwise use set_range.
115 // The range is made up of the cards that are spanned by an object/mem
116 // region so 8 cards will allow up to object sizes up to 4K to be handled
117 // using the loop.
118 if ((end_idx - start_idx) <= 8) {
119 for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) {
120 _card_bm.set_bit(i);
121 }
122 } else {
123 _card_bm.set_range(start_idx, end_idx);
124 }
125 }
126
127 // We cache the last mark set. This avoids setting the same bit multiple times.
128 // This is particularly interesting for dense bitmaps, as this avoids doing
129 // lots of work most of the time.
130 BitMap::idx_t _last_marked_bit_idx;
131
132 void clear_card_bitmap_range(HeapWord* start, HeapWord* end) {
133 BitMap::idx_t start_idx = card_live_bitmap_index_for(start);
134 BitMap::idx_t end_idx = card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size));
135
136 _card_bm.clear_range(start_idx, end_idx);
137 }
138
139 // Mark the card liveness bitmap for the object spanning from start to end.
140 void mark_card_bitmap_range(HeapWord* start, HeapWord* end) {
141 BitMap::idx_t start_idx = card_live_bitmap_index_for(start);
142 BitMap::idx_t end_idx = card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size));
143
144 assert((end_idx - start_idx) > 0, "Trying to mark zero sized range.");
145
146 if (start_idx == _last_marked_bit_idx) {
147 start_idx++;
148 }
149 if (start_idx == end_idx) {
150 return;
151 }
152
153 // Set the bits in the card bitmap for the cards spanned by this object.
154 set_card_bitmap_range(start_idx, end_idx);
155 _last_marked_bit_idx = end_idx - 1;
156 }
157
158 void reset_mark_cache() {
159 _last_marked_bit_idx = (BitMap::idx_t)-1;
160 }
161
162 public:
163 // Returns the index in the per-card liveness count bitmap
164 // for the given address
165 inline BitMap::idx_t card_live_bitmap_index_for(HeapWord* addr) {
166 // Below, the term "card num" means the result of shifting an address
167 // by the card shift -- address 0 corresponds to card number 0. One
168 // must subtract the card num of the bottom of the heap to obtain a
169 // card table index.
170 BitMap::idx_t card_num = uintptr_t(addr) >> CardTableModRefBS::card_shift;
171 return card_num - _heap_card_bias;
172 }
173
174 // Takes a region that's not empty (i.e., it has at least one
175 // live object in it and sets its corresponding bit on the region
176 // bitmap to 1.
177 void set_bit_for_region(HeapRegion* hr) {
178 _region_bm.par_set_bit(hr->hrm_index());
179 }
180
181 void reset_live_data(HeapRegion* hr) {
182 clear_card_bitmap_range(hr->next_top_at_mark_start(), hr->end());
183 }
184
185 // Mark the range of bits covered by allocations done since the last marking
186 // in the given heap region, i.e. from NTAMS to top of the given region.
187 // Returns if there has been some allocation in this region since the last marking.
188 bool mark_allocated_since_marking(HeapRegion* hr) {
189 reset_mark_cache();
190
244 assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere.");
245
246 mark_card_bitmap_range(start, obj_end);
247
248 // Add the size of this object to the number of marked bytes.
249 marked_bytes += obj_size * HeapWordSize;
250
251 // Find the next marked object after this one.
252 start = mark_bitmap->getNextMarkedWordAddress(obj_end, ntams);
253 }
254
255 return marked_bytes;
256 }
257
258 G1CardLiveDataHelper(G1CardLiveData* live_data, HeapWord* base_address) :
259 _region_bm(live_data->live_regions_bm()),
260 _card_bm(live_data->live_cards_bm()) {
261 // Calculate the card number for the bottom of the heap. Used
262 // in biasing indexes into the accounting card bitmaps.
263 _heap_card_bias =
264 uintptr_t(base_address) >> CardTableModRefBS::card_shift;
265 }
266 };
267
268 class G1CreateCardLiveDataTask: public AbstractGangTask {
269 // Aggregate the counting data that was constructed concurrently
270 // with marking.
271 class G1CreateLiveDataClosure : public HeapRegionClosure {
272 G1CardLiveDataHelper _helper;
273
274 G1CMBitMap* _mark_bitmap;
275
276 G1ConcurrentMark* _cm;
277 public:
278 G1CreateLiveDataClosure(G1CollectedHeap* g1h,
279 G1ConcurrentMark* cm,
280 G1CMBitMap* mark_bitmap,
281 G1CardLiveData* live_data) :
282 HeapRegionClosure(),
283 _helper(live_data, g1h->reserved_region().start()),
284 _mark_bitmap(mark_bitmap),
|
50 free_large_bitmap(_live_cards, _live_cards_size_in_bits);
51 free_large_bitmap(_live_regions, _live_regions_size_in_bits);
52 }
53
54 G1CardLiveData::bm_word_t* G1CardLiveData::allocate_large_bitmap(size_t size_in_bits) {
55 size_t size_in_words = BitMap::calc_size_in_words(size_in_bits);
56
57 bm_word_t* map = MmapArrayAllocator<bm_word_t, mtGC>::allocate(size_in_words);
58
59 return map;
60 }
61
62 void G1CardLiveData::free_large_bitmap(bm_word_t* bitmap, size_t size_in_bits) {
63 MmapArrayAllocator<bm_word_t, mtGC>::free(bitmap, BitMap::calc_size_in_words(size_in_bits));
64 }
65
66 void G1CardLiveData::initialize(size_t max_capacity, uint num_max_regions) {
67 assert(max_capacity % num_max_regions == 0,
68 "Given capacity must be evenly divisible by region size.");
69 size_t region_size = max_capacity / num_max_regions;
70 assert(region_size % (G1CardTable::card_size * BitsPerWord) == 0,
71 "Region size must be evenly divisible by area covered by a single word.");
72 _max_capacity = max_capacity;
73 _cards_per_region = region_size / G1CardTable::card_size;
74
75 _live_regions_size_in_bits = live_region_bitmap_size_in_bits();
76 _live_regions = allocate_large_bitmap(_live_regions_size_in_bits);
77 _live_cards_size_in_bits = live_card_bitmap_size_in_bits();
78 _live_cards = allocate_large_bitmap(_live_cards_size_in_bits);
79 }
80
81 void G1CardLiveData::pretouch() {
82 live_cards_bm().pretouch();
83 live_regions_bm().pretouch();
84 }
85
86 size_t G1CardLiveData::live_region_bitmap_size_in_bits() const {
87 return _max_capacity / (_cards_per_region << G1CardTable::card_shift);
88 }
89
90 size_t G1CardLiveData::live_card_bitmap_size_in_bits() const {
91 return _max_capacity >> G1CardTable::card_shift;
92 }
93
94 // Helper class that provides functionality to generate the Live Data Count
95 // information.
96 class G1CardLiveDataHelper VALUE_OBJ_CLASS_SPEC {
97 private:
98 BitMapView _region_bm;
99 BitMapView _card_bm;
100
101 // The card number of the bottom of the G1 heap.
102 // Used in biasing indices into accounting card bitmaps.
103 BitMap::idx_t _heap_card_bias;
104
105 // Utility routine to set an exclusive range of bits on the given
106 // bitmap, optimized for very small ranges.
107 // There must be at least one bit to set.
108 void set_card_bitmap_range(BitMap::idx_t start_idx,
109 BitMap::idx_t end_idx) {
110
111 // Set the exclusive bit range [start_idx, end_idx).
114 // For small ranges use a simple loop; otherwise use set_range.
115 // The range is made up of the cards that are spanned by an object/mem
116 // region so 8 cards will allow up to object sizes up to 4K to be handled
117 // using the loop.
118 if ((end_idx - start_idx) <= 8) {
119 for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) {
120 _card_bm.set_bit(i);
121 }
122 } else {
123 _card_bm.set_range(start_idx, end_idx);
124 }
125 }
126
127 // We cache the last mark set. This avoids setting the same bit multiple times.
128 // This is particularly interesting for dense bitmaps, as this avoids doing
129 // lots of work most of the time.
130 BitMap::idx_t _last_marked_bit_idx;
131
132 void clear_card_bitmap_range(HeapWord* start, HeapWord* end) {
133 BitMap::idx_t start_idx = card_live_bitmap_index_for(start);
134 BitMap::idx_t end_idx = card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, G1CardTable::card_size));
135
136 _card_bm.clear_range(start_idx, end_idx);
137 }
138
139 // Mark the card liveness bitmap for the object spanning from start to end.
140 void mark_card_bitmap_range(HeapWord* start, HeapWord* end) {
141 BitMap::idx_t start_idx = card_live_bitmap_index_for(start);
142 BitMap::idx_t end_idx = card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, G1CardTable::card_size));
143
144 assert((end_idx - start_idx) > 0, "Trying to mark zero sized range.");
145
146 if (start_idx == _last_marked_bit_idx) {
147 start_idx++;
148 }
149 if (start_idx == end_idx) {
150 return;
151 }
152
153 // Set the bits in the card bitmap for the cards spanned by this object.
154 set_card_bitmap_range(start_idx, end_idx);
155 _last_marked_bit_idx = end_idx - 1;
156 }
157
158 void reset_mark_cache() {
159 _last_marked_bit_idx = (BitMap::idx_t)-1;
160 }
161
162 public:
163 // Returns the index in the per-card liveness count bitmap
164 // for the given address
165 inline BitMap::idx_t card_live_bitmap_index_for(HeapWord* addr) {
166 // Below, the term "card num" means the result of shifting an address
167 // by the card shift -- address 0 corresponds to card number 0. One
168 // must subtract the card num of the bottom of the heap to obtain a
169 // card table index.
170 BitMap::idx_t card_num = uintptr_t(addr) >> G1CardTable::card_shift;
171 return card_num - _heap_card_bias;
172 }
173
174 // Takes a region that's not empty (i.e., it has at least one
175 // live object in it and sets its corresponding bit on the region
176 // bitmap to 1.
177 void set_bit_for_region(HeapRegion* hr) {
178 _region_bm.par_set_bit(hr->hrm_index());
179 }
180
181 void reset_live_data(HeapRegion* hr) {
182 clear_card_bitmap_range(hr->next_top_at_mark_start(), hr->end());
183 }
184
185 // Mark the range of bits covered by allocations done since the last marking
186 // in the given heap region, i.e. from NTAMS to top of the given region.
187 // Returns if there has been some allocation in this region since the last marking.
188 bool mark_allocated_since_marking(HeapRegion* hr) {
189 reset_mark_cache();
190
244 assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere.");
245
246 mark_card_bitmap_range(start, obj_end);
247
248 // Add the size of this object to the number of marked bytes.
249 marked_bytes += obj_size * HeapWordSize;
250
251 // Find the next marked object after this one.
252 start = mark_bitmap->getNextMarkedWordAddress(obj_end, ntams);
253 }
254
255 return marked_bytes;
256 }
257
258 G1CardLiveDataHelper(G1CardLiveData* live_data, HeapWord* base_address) :
259 _region_bm(live_data->live_regions_bm()),
260 _card_bm(live_data->live_cards_bm()) {
261 // Calculate the card number for the bottom of the heap. Used
262 // in biasing indexes into the accounting card bitmaps.
263 _heap_card_bias =
264 uintptr_t(base_address) >> G1CardTable::card_shift;
265 }
266 };
267
268 class G1CreateCardLiveDataTask: public AbstractGangTask {
269 // Aggregate the counting data that was constructed concurrently
270 // with marking.
271 class G1CreateLiveDataClosure : public HeapRegionClosure {
272 G1CardLiveDataHelper _helper;
273
274 G1CMBitMap* _mark_bitmap;
275
276 G1ConcurrentMark* _cm;
277 public:
278 G1CreateLiveDataClosure(G1CollectedHeap* g1h,
279 G1ConcurrentMark* cm,
280 G1CMBitMap* mark_bitmap,
281 G1CardLiveData* live_data) :
282 HeapRegionClosure(),
283 _helper(live_data, g1h->reserved_region().start()),
284 _mark_bitmap(mark_bitmap),
|