99 static const G1RemsetIterState Unclaimed = 0; // The remembered set has not been scanned yet.
100 static const G1RemsetIterState Claimed = 1; // The remembered set is currently being scanned.
101 static const G1RemsetIterState Complete = 2; // The remembered set has been completely scanned.
102
103 G1RemsetIterState volatile* _iter_states;
104 // The current location where the next thread should continue scanning in a region's
105 // remembered set.
106 size_t volatile* _iter_claims;
107
108 // Temporary buffer holding the regions we used to store remembered set scan duplicate
109 // information. These are also called "dirty". Valid entries are from [0.._cur_dirty_region)
110 uint* _dirty_region_buffer;
111
112 typedef jbyte IsDirtyRegionState;
113 static const IsDirtyRegionState Clean = 0;
114 static const IsDirtyRegionState Dirty = 1;
115 // Holds a flag for every region whether it is in the _dirty_region_buffer already
116 // to avoid duplicates. Uses jbyte since there are no atomic instructions for bools.
117 IsDirtyRegionState* _in_dirty_region_buffer;
118 size_t _cur_dirty_region;
119 public:
120 G1RemSetScanState() :
121 _max_regions(0),
122 _iter_states(NULL),
123 _iter_claims(NULL),
124 _dirty_region_buffer(NULL),
125 _in_dirty_region_buffer(NULL),
126 _cur_dirty_region(0) {
127
128 }
129
130 ~G1RemSetScanState() {
131 if (_iter_states != NULL) {
132 FREE_C_HEAP_ARRAY(G1RemsetIterState, _iter_states);
133 }
134 if (_iter_claims != NULL) {
135 FREE_C_HEAP_ARRAY(size_t, _iter_claims);
136 }
137 if (_dirty_region_buffer != NULL) {
138 FREE_C_HEAP_ARRAY(uint, _dirty_region_buffer);
139 }
140 if (_in_dirty_region_buffer != NULL) {
141 FREE_C_HEAP_ARRAY(IsDirtyRegionState, _in_dirty_region_buffer);
142 }
143 }
144
145 void initialize(uint max_regions) {
146 assert(_iter_states == NULL, "Must not be initialized twice");
147 assert(_iter_claims == NULL, "Must not be initialized twice");
148 _max_regions = max_regions;
149 _iter_states = NEW_C_HEAP_ARRAY(G1RemsetIterState, max_regions, mtGC);
150 _iter_claims = NEW_C_HEAP_ARRAY(size_t, max_regions, mtGC);
151 _dirty_region_buffer = NEW_C_HEAP_ARRAY(uint, max_regions, mtGC);
152 _in_dirty_region_buffer = NEW_C_HEAP_ARRAY(IsDirtyRegionState, max_regions, mtGC);
153 }
154
155 void reset() {
156 for (uint i = 0; i < _max_regions; i++) {
157 _iter_states[i] = Unclaimed;
158 }
159 memset((void*)_iter_claims, 0, _max_regions * sizeof(size_t));
160 memset(_in_dirty_region_buffer, Clean, _max_regions * sizeof(IsDirtyRegionState));
161 _cur_dirty_region = 0;
162 }
163
164 // Attempt to claim the remembered set of the region for iteration. Returns true
165 // if this call caused the transition from Unclaimed to Claimed.
166 inline bool claim_iter(uint region) {
167 assert(region < _max_regions, "Tried to access invalid region %u", region);
168 if (_iter_states[region] != Unclaimed) {
169 return false;
170 }
171 jint res = Atomic::cmpxchg(Claimed, (jint*)(&_iter_states[region]), Unclaimed);
172 return (res == Unclaimed);
173 }
174
175 // Try to atomically sets the iteration state to "complete". Returns true for the
176 // thread that caused the transition.
177 inline bool set_iter_complete(uint region) {
178 if (iter_is_complete(region)) {
195 }
196
197 // Claim the next block of cards within the remembered set of the region with
198 // step size.
199 inline size_t iter_claimed_next(uint region, size_t step) {
200 return Atomic::add(step, &_iter_claims[region]) - step;
201 }
202
203 void add_dirty_region(uint region) {
204 if (_in_dirty_region_buffer[region] == Dirty) {
205 return;
206 }
207
208 bool marked_as_dirty = Atomic::cmpxchg(Dirty, &_in_dirty_region_buffer[region], Clean) == Clean;
209 if (marked_as_dirty) {
210 size_t allocated = Atomic::add(1, &_cur_dirty_region) - 1;
211 _dirty_region_buffer[allocated] = region;
212 }
213 }
214
215 // Clear the card table of "dirty" regions.
216 void clear_card_table(WorkGang* workers) {
217 if (_cur_dirty_region == 0) {
218 return;
219 }
220
221 size_t const num_chunks = align_size_up(_cur_dirty_region * HeapRegion::CardsPerRegion, G1ClearCardTableTask::chunk_size()) / G1ClearCardTableTask::chunk_size();
222 uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers());
223 size_t const chunk_length = G1ClearCardTableTask::chunk_size() / HeapRegion::CardsPerRegion;
224
225 // Iterate over the dirty cards region list.
226 G1ClearCardTableTask cl(G1CollectedHeap::heap(), _dirty_region_buffer, _cur_dirty_region, chunk_length);
227
228 log_debug(gc, ergo)("Running %s using %u workers for " SIZE_FORMAT " "
229 "units of work for " SIZE_FORMAT " regions.",
230 cl.name(), num_workers, num_chunks, _cur_dirty_region);
231 workers->run_task(&cl, num_workers);
232
233 #ifndef PRODUCT
234 // Need to synchronize with concurrent cleanup since it needs to
290
291 G1ScanRSClosure::G1ScanRSClosure(G1RemSetScanState* scan_state,
292 G1ParPushHeapRSClosure* push_heap_cl,
293 CodeBlobClosure* code_root_cl,
294 uint worker_i) :
295 _scan_state(scan_state),
296 _push_heap_cl(push_heap_cl),
297 _code_root_cl(code_root_cl),
298 _strong_code_root_scan_time_sec(0.0),
299 _cards(0),
300 _cards_done(0),
301 _worker_i(worker_i) {
302 _g1h = G1CollectedHeap::heap();
303 _bot = _g1h->bot();
304 _ct_bs = _g1h->g1_barrier_set();
305 _block_size = MAX2<size_t>(G1RSetScanBlockSize, 1);
306 }
307
308 void G1ScanRSClosure::scan_card(size_t index, HeapWord* card_start, HeapRegion *r) {
309 MemRegion card_region(card_start, BOTConstants::N_words);
310 MemRegion pre_gc_allocated(r->bottom(), r->scan_top());
311 MemRegion mr = pre_gc_allocated.intersection(card_region);
312 if (!mr.is_empty() && !_ct_bs->is_card_claimed(index)) {
313 // We make the card as "claimed" lazily (so races are possible
314 // but they're benign), which reduces the number of duplicate
315 // scans (the rsets of the regions in the cset can intersect).
316 _ct_bs->set_card_claimed(index);
317 _push_heap_cl->set_region(r);
318 r->oops_on_card_seq_iterate_careful<true>(mr, _push_heap_cl);
319 _cards_done++;
320 }
321 }
322
323 void G1ScanRSClosure::scan_strong_code_roots(HeapRegion* r) {
324 double scan_start = os::elapsedTime();
325 r->strong_code_roots_do(_code_root_cl);
326 _strong_code_root_scan_time_sec += (os::elapsedTime() - scan_start);
327 }
328
329 bool G1ScanRSClosure::doHeapRegion(HeapRegion* r) {
330 assert(r->in_collection_set(), "should only be called on elements of CS.");
698 bool G1RemSet::refine_card_during_gc(jbyte* card_ptr,
699 uint worker_i,
700 G1ParPushHeapRSClosure* oops_in_heap_closure) {
701 assert(_g1->is_gc_active(), "Only call during GC");
702
703 check_card_ptr(card_ptr, _ct_bs);
704
705 // If the card is no longer dirty, nothing to do. This covers cards that were already
706 // scanned as parts of the remembered sets.
707 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
708 // No need to return that this card contains refs that point
709 // into the collection set.
710 return false;
711 }
712
713 // Construct the region representing the card.
714 HeapWord* start = _ct_bs->addr_for(card_ptr);
715 // And find the region containing it.
716 HeapRegion* r = _g1->heap_region_containing(start);
717
718 // This check is needed for some uncommon cases where we should
719 // ignore the card.
720 //
721 // The region could be young. Cards for young regions are
722 // distinctly marked (set to g1_young_gen), so the post-barrier will
723 // filter them out. However, that marking is performed
724 // concurrently. A write to a young object could occur before the
725 // card has been marked young, slipping past the filter.
726 //
727 // The card could be stale, because the region has been freed since
728 // the card was recorded. In this case the region type could be
729 // anything. If (still) free or (reallocated) young, just ignore
730 // it. If (reallocated) old or humongous, the later card trimming
731 // and additional checks in iteration may detect staleness. At
732 // worst, we end up processing a stale card unnecessarily.
733 //
734 // In the normal (non-stale) case, the synchronization between the
735 // enqueueing of the card and processing it here will have ensured
736 // we see the up-to-date region type here.
737 if (!r->is_old_or_humongous()) {
738 return false;
739 }
740
741 // While we are processing RSet buffers during the collection, we
742 // actually don't want to scan any cards on the collection set,
743 // since we don't want to update remembered sets with entries that
744 // point into the collection set, given that live objects from the
745 // collection set are about to move and such entries will be stale
746 // very soon. This change also deals with a reliability issue which
747 // involves scanning a card in the collection set and coming across
748 // an array that was being chunked and looking malformed. Note,
749 // however, that if evacuation fails, we have to scan any objects
750 // that were not moved and create any missing entries.
751 if (r->in_collection_set()) {
752 return false;
753 }
754
755 // Trim the region designated by the card to what's been allocated
756 // in the region. The card could be stale, or the card could cover
757 // (part of) an object at the end of the allocated space and extend
758 // beyond the end of allocation.
759
760 // If we're in a STW GC, then a card might be in a GC alloc region
761 // and extend onto a GC LAB, which may not be parsable. Stop such
762 // at the "scan_top" of the region.
763 HeapWord* scan_limit = r->scan_top();
764
765 if (scan_limit <= start) {
766 // If the trimmed region is empty, the card must be stale.
767 return false;
768 }
769
770 // Okay to clean and process the card now. There are still some
771 // stale card cases that may be detected by iteration and dealt with
772 // as iteration failure.
773 *const_cast<volatile jbyte*>(card_ptr) = CardTableModRefBS::clean_card_val();
774
775 // Don't use addr_for(card_ptr + 1) which can ask for
776 // a card beyond the heap.
777 HeapWord* end = start + CardTableModRefBS::card_size_in_words;
778 MemRegion dirty_region(start, MIN2(scan_limit, end));
779 assert(!dirty_region.is_empty(), "sanity");
780
781 G1UpdateRSOrPushRefOopClosure update_rs_oop_cl(_g1,
782 oops_in_heap_closure,
783 true,
784 worker_i);
785 update_rs_oop_cl.set_from(r);
786
|
99 static const G1RemsetIterState Unclaimed = 0; // The remembered set has not been scanned yet.
100 static const G1RemsetIterState Claimed = 1; // The remembered set is currently being scanned.
101 static const G1RemsetIterState Complete = 2; // The remembered set has been completely scanned.
102
103 G1RemsetIterState volatile* _iter_states;
104 // The current location where the next thread should continue scanning in a region's
105 // remembered set.
106 size_t volatile* _iter_claims;
107
108 // Temporary buffer holding the regions we used to store remembered set scan duplicate
109 // information. These are also called "dirty". Valid entries are from [0.._cur_dirty_region)
110 uint* _dirty_region_buffer;
111
112 typedef jbyte IsDirtyRegionState;
113 static const IsDirtyRegionState Clean = 0;
114 static const IsDirtyRegionState Dirty = 1;
115 // Holds a flag for every region whether it is in the _dirty_region_buffer already
116 // to avoid duplicates. Uses jbyte since there are no atomic instructions for bools.
117 IsDirtyRegionState* _in_dirty_region_buffer;
118 size_t _cur_dirty_region;
119
120 // Creates a snapshot of the current _top values at the start of collection to
121 // filter out card marks that we do not want to scan.
122 class G1ResetScanTopClosure : public HeapRegionClosure {
123 private:
124 HeapWord** _scan_top;
125 public:
126 G1ResetScanTopClosure(HeapWord** scan_top) : _scan_top(scan_top) { }
127
128 virtual bool doHeapRegion(HeapRegion* r) {
129 uint hrm_index = r->hrm_index();
130 if (!r->in_collection_set() && r->is_old_or_humongous()) {
131 _scan_top[hrm_index] = r->top();
132 } else {
133 _scan_top[hrm_index] = r->bottom();
134 }
135 return false;
136 }
137 };
138
139 // For each region, contains the maximum top() value to be used during this garbage
140 // collection. Subsumes common checks like filtering out everything but old and
141 // humongous regions outside the collection set.
142 // This is valid because we are not interested in scanning stray remembered set
143 // entries from free or archive regions.
144 HeapWord** _scan_top;
145 public:
146 G1RemSetScanState() :
147 _max_regions(0),
148 _iter_states(NULL),
149 _iter_claims(NULL),
150 _dirty_region_buffer(NULL),
151 _in_dirty_region_buffer(NULL),
152 _cur_dirty_region(0),
153 _scan_top(NULL) {
154 }
155
156 ~G1RemSetScanState() {
157 if (_iter_states != NULL) {
158 FREE_C_HEAP_ARRAY(G1RemsetIterState, _iter_states);
159 }
160 if (_iter_claims != NULL) {
161 FREE_C_HEAP_ARRAY(size_t, _iter_claims);
162 }
163 if (_dirty_region_buffer != NULL) {
164 FREE_C_HEAP_ARRAY(uint, _dirty_region_buffer);
165 }
166 if (_in_dirty_region_buffer != NULL) {
167 FREE_C_HEAP_ARRAY(IsDirtyRegionState, _in_dirty_region_buffer);
168 }
169 if (_scan_top != NULL) {
170 FREE_C_HEAP_ARRAY(HeapWord*, _scan_top);
171 }
172 }
173
174 void initialize(uint max_regions) {
175 assert(_iter_states == NULL, "Must not be initialized twice");
176 assert(_iter_claims == NULL, "Must not be initialized twice");
177 _max_regions = max_regions;
178 _iter_states = NEW_C_HEAP_ARRAY(G1RemsetIterState, max_regions, mtGC);
179 _iter_claims = NEW_C_HEAP_ARRAY(size_t, max_regions, mtGC);
180 _dirty_region_buffer = NEW_C_HEAP_ARRAY(uint, max_regions, mtGC);
181 _in_dirty_region_buffer = NEW_C_HEAP_ARRAY(IsDirtyRegionState, max_regions, mtGC);
182 _scan_top = NEW_C_HEAP_ARRAY(HeapWord*, max_regions, mtGC);
183 }
184
185 void reset() {
186 for (uint i = 0; i < _max_regions; i++) {
187 _iter_states[i] = Unclaimed;
188 }
189
190 memset(_scan_top, 0, _max_regions * sizeof(HeapWord*));
191 G1ResetScanTopClosure cl(_scan_top);
192 G1CollectedHeap::heap()->heap_region_iterate(&cl);
193
194 memset((void*)_iter_claims, 0, _max_regions * sizeof(size_t));
195 memset(_in_dirty_region_buffer, Clean, _max_regions * sizeof(IsDirtyRegionState));
196 _cur_dirty_region = 0;
197 }
198
199 // Attempt to claim the remembered set of the region for iteration. Returns true
200 // if this call caused the transition from Unclaimed to Claimed.
201 inline bool claim_iter(uint region) {
202 assert(region < _max_regions, "Tried to access invalid region %u", region);
203 if (_iter_states[region] != Unclaimed) {
204 return false;
205 }
206 jint res = Atomic::cmpxchg(Claimed, (jint*)(&_iter_states[region]), Unclaimed);
207 return (res == Unclaimed);
208 }
209
210 // Try to atomically sets the iteration state to "complete". Returns true for the
211 // thread that caused the transition.
212 inline bool set_iter_complete(uint region) {
213 if (iter_is_complete(region)) {
230 }
231
232 // Claim the next block of cards within the remembered set of the region with
233 // step size.
234 inline size_t iter_claimed_next(uint region, size_t step) {
235 return Atomic::add(step, &_iter_claims[region]) - step;
236 }
237
238 void add_dirty_region(uint region) {
239 if (_in_dirty_region_buffer[region] == Dirty) {
240 return;
241 }
242
243 bool marked_as_dirty = Atomic::cmpxchg(Dirty, &_in_dirty_region_buffer[region], Clean) == Clean;
244 if (marked_as_dirty) {
245 size_t allocated = Atomic::add(1, &_cur_dirty_region) - 1;
246 _dirty_region_buffer[allocated] = region;
247 }
248 }
249
250 HeapWord* scan_top(uint region_idx) const {
251 return _scan_top[region_idx];
252 }
253
254 // Clear the card table of "dirty" regions.
255 void clear_card_table(WorkGang* workers) {
256 if (_cur_dirty_region == 0) {
257 return;
258 }
259
260 size_t const num_chunks = align_size_up(_cur_dirty_region * HeapRegion::CardsPerRegion, G1ClearCardTableTask::chunk_size()) / G1ClearCardTableTask::chunk_size();
261 uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers());
262 size_t const chunk_length = G1ClearCardTableTask::chunk_size() / HeapRegion::CardsPerRegion;
263
264 // Iterate over the dirty cards region list.
265 G1ClearCardTableTask cl(G1CollectedHeap::heap(), _dirty_region_buffer, _cur_dirty_region, chunk_length);
266
267 log_debug(gc, ergo)("Running %s using %u workers for " SIZE_FORMAT " "
268 "units of work for " SIZE_FORMAT " regions.",
269 cl.name(), num_workers, num_chunks, _cur_dirty_region);
270 workers->run_task(&cl, num_workers);
271
272 #ifndef PRODUCT
273 // Need to synchronize with concurrent cleanup since it needs to
329
330 G1ScanRSClosure::G1ScanRSClosure(G1RemSetScanState* scan_state,
331 G1ParPushHeapRSClosure* push_heap_cl,
332 CodeBlobClosure* code_root_cl,
333 uint worker_i) :
334 _scan_state(scan_state),
335 _push_heap_cl(push_heap_cl),
336 _code_root_cl(code_root_cl),
337 _strong_code_root_scan_time_sec(0.0),
338 _cards(0),
339 _cards_done(0),
340 _worker_i(worker_i) {
341 _g1h = G1CollectedHeap::heap();
342 _bot = _g1h->bot();
343 _ct_bs = _g1h->g1_barrier_set();
344 _block_size = MAX2<size_t>(G1RSetScanBlockSize, 1);
345 }
346
347 void G1ScanRSClosure::scan_card(size_t index, HeapWord* card_start, HeapRegion *r) {
348 MemRegion card_region(card_start, BOTConstants::N_words);
349 MemRegion pre_gc_allocated(r->bottom(), _scan_state->scan_top(r->hrm_index()));
350 MemRegion mr = pre_gc_allocated.intersection(card_region);
351 if (!mr.is_empty() && !_ct_bs->is_card_claimed(index)) {
352 // We make the card as "claimed" lazily (so races are possible
353 // but they're benign), which reduces the number of duplicate
354 // scans (the rsets of the regions in the cset can intersect).
355 _ct_bs->set_card_claimed(index);
356 _push_heap_cl->set_region(r);
357 r->oops_on_card_seq_iterate_careful<true>(mr, _push_heap_cl);
358 _cards_done++;
359 }
360 }
361
362 void G1ScanRSClosure::scan_strong_code_roots(HeapRegion* r) {
363 double scan_start = os::elapsedTime();
364 r->strong_code_roots_do(_code_root_cl);
365 _strong_code_root_scan_time_sec += (os::elapsedTime() - scan_start);
366 }
367
368 bool G1ScanRSClosure::doHeapRegion(HeapRegion* r) {
369 assert(r->in_collection_set(), "should only be called on elements of CS.");
737 bool G1RemSet::refine_card_during_gc(jbyte* card_ptr,
738 uint worker_i,
739 G1ParPushHeapRSClosure* oops_in_heap_closure) {
740 assert(_g1->is_gc_active(), "Only call during GC");
741
742 check_card_ptr(card_ptr, _ct_bs);
743
744 // If the card is no longer dirty, nothing to do. This covers cards that were already
745 // scanned as parts of the remembered sets.
746 if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
747 // No need to return that this card contains refs that point
748 // into the collection set.
749 return false;
750 }
751
752 // Construct the region representing the card.
753 HeapWord* start = _ct_bs->addr_for(card_ptr);
754 // And find the region containing it.
755 HeapRegion* r = _g1->heap_region_containing(start);
756
757 HeapWord* scan_limit = _scan_state->scan_top(r->hrm_index());
758 if (scan_limit <= start) {
759 return false;
760 }
761
762 // Okay to clean and process the card now. There are still some
763 // stale card cases that may be detected by iteration and dealt with
764 // as iteration failure.
765 *const_cast<volatile jbyte*>(card_ptr) = CardTableModRefBS::clean_card_val();
766
767 // Don't use addr_for(card_ptr + 1) which can ask for
768 // a card beyond the heap.
769 HeapWord* end = start + CardTableModRefBS::card_size_in_words;
770 MemRegion dirty_region(start, MIN2(scan_limit, end));
771 assert(!dirty_region.is_empty(), "sanity");
772
773 G1UpdateRSOrPushRefOopClosure update_rs_oop_cl(_g1,
774 oops_in_heap_closure,
775 true,
776 worker_i);
777 update_rs_oop_cl.set_from(r);
778
|