87 }
88
89 G1RemSet::~G1RemSet() {
90 delete _seq_task;
91 for (uint i = 0; i < n_workers(); i++) {
92 assert(_cset_rs_update_cl[i] == NULL, "it should be");
93 }
94 FREE_C_HEAP_ARRAY(OopsInHeapRegionClosure*, _cset_rs_update_cl, mtGC);
95 }
96
97 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) {
98 if (_g1->is_in_g1_reserved(mr.start())) {
99 _n += (int) ((mr.byte_size() / CardTableModRefBS::card_size));
100 if (_start_first == NULL) _start_first = mr.start();
101 }
102 }
103
104 class ScanRSClosure : public HeapRegionClosure {
105 size_t _cards_done, _cards;
106 G1CollectedHeap* _g1h;
107 OopsInHeapRegionClosure* _oc;
108 G1BlockOffsetSharedArray* _bot_shared;
109 CardTableModRefBS *_ct_bs;
110 int _worker_i;
111 int _block_size;
112 bool _try_claimed;
113 public:
114 ScanRSClosure(OopsInHeapRegionClosure* oc, int worker_i) :
115 _oc(oc),
116 _cards(0),
117 _cards_done(0),
118 _worker_i(worker_i),
119 _try_claimed(false)
120 {
121 _g1h = G1CollectedHeap::heap();
122 _bot_shared = _g1h->bot_shared();
123 _ct_bs = (CardTableModRefBS*) (_g1h->barrier_set());
124 _block_size = MAX2<int>(G1RSetScanBlockSize, 1);
125 }
126
127 void set_try_claimed() { _try_claimed = true; }
128
129 void scanCard(size_t index, HeapRegion *r) {
130 // Stack allocate the DirtyCardToOopClosure instance
131 HeapRegionDCTOC cl(_g1h, r, _oc,
132 CardTableModRefBS::Precise,
133 HeapRegionDCTOC::IntoCSFilterKind);
134
135 // Set the "from" region in the closure.
143 // We make the card as "claimed" lazily (so races are possible
144 // but they're benign), which reduces the number of duplicate
145 // scans (the rsets of the regions in the cset can intersect).
146 _ct_bs->set_card_claimed(index);
147 _cards_done++;
148 cl.do_MemRegion(mr);
149 }
150 }
151
152 void printCard(HeapRegion* card_region, size_t card_index,
153 HeapWord* card_start) {
154 gclog_or_tty->print_cr("T %d Region [" PTR_FORMAT ", " PTR_FORMAT ") "
155 "RS names card %p: "
156 "[" PTR_FORMAT ", " PTR_FORMAT ")",
157 _worker_i,
158 card_region->bottom(), card_region->end(),
159 card_index,
160 card_start, card_start + G1BlockOffsetSharedArray::N_words);
161 }
162
163 bool doHeapRegion(HeapRegion* r) {
164 assert(r->in_collection_set(), "should only be called on elements of CS.");
165 HeapRegionRemSet* hrrs = r->rem_set();
166 if (hrrs->iter_is_complete()) return false; // All done.
167 if (!_try_claimed && !hrrs->claim_iter()) return false;
168 // If we ever free the collection set concurrently, we should also
169 // clear the card table concurrently therefore we won't need to
170 // add regions of the collection set to the dirty cards region.
171 _g1h->push_dirty_cards_region(r);
172 // If we didn't return above, then
173 // _try_claimed || r->claim_iter()
174 // is true: either we're supposed to work on claimed-but-not-complete
175 // regions, or we successfully claimed the region.
176 HeapRegionRemSetIterator iter(hrrs);
177 size_t card_index;
178
179 // We claim cards in block so as to recude the contention. The block size is determined by
180 // the G1RSetScanBlockSize parameter.
181 size_t jump_to_card = hrrs->iter_claimed_next(_block_size);
182 for (size_t current_card = 0; iter.has_next(card_index); current_card++) {
183 if (current_card >= jump_to_card + _block_size) {
184 jump_to_card = hrrs->iter_claimed_next(_block_size);
185 }
186 if (current_card < jump_to_card) continue;
187 HeapWord* card_start = _g1h->bot_shared()->address_for_index(card_index);
188 #if 0
189 gclog_or_tty->print("Rem set iteration yielded card [" PTR_FORMAT ", " PTR_FORMAT ").\n",
190 card_start, card_start + CardTableModRefBS::card_size_in_words);
191 #endif
192
193 HeapRegion* card_region = _g1h->heap_region_containing(card_start);
194 assert(card_region != NULL, "Yielding cards not in the heap?");
195 _cards++;
196
197 if (!card_region->is_on_dirty_cards_region_list()) {
198 _g1h->push_dirty_cards_region(card_region);
199 }
200
201 // If the card is dirty, then we will scan it during updateRS.
202 if (!card_region->in_collection_set() &&
203 !_ct_bs->is_card_dirty(card_index)) {
204 scanCard(card_index, card_region);
205 }
206 }
207 if (!_try_claimed) {
208 hrrs->set_iter_complete();
209 }
210 return false;
211 }
212 size_t cards_done() { return _cards_done;}
213 size_t cards_looked_up() { return _cards;}
214 };
215
216 void G1RemSet::scanRS(OopsInHeapRegionClosure* oc, int worker_i) {
217 double rs_time_start = os::elapsedTime();
218 HeapRegion *startRegion = _g1->start_cset_region_for_worker(worker_i);
219
220 ScanRSClosure scanRScl(oc, worker_i);
221
222 _g1->collection_set_iterate_from(startRegion, &scanRScl);
223 scanRScl.set_try_claimed();
224 _g1->collection_set_iterate_from(startRegion, &scanRScl);
225
226 double scan_rs_time_sec = os::elapsedTime() - rs_time_start;
227
228 assert( _cards_scanned != NULL, "invariant" );
229 _cards_scanned[worker_i] = scanRScl.cards_done();
230
231 _g1p->phase_times()->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0);
232 }
233
234 // Closure used for updating RSets and recording references that
235 // point into the collection set. Only called during an
236 // evacuation pause.
237
238 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure {
239 G1RemSet* _g1rs;
240 DirtyCardQueue* _into_cset_dcq;
241 public:
242 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h,
243 DirtyCardQueue* into_cset_dcq) :
244 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq)
245 {}
246 bool do_card_ptr(jbyte* card_ptr, int worker_i) {
247 // The only time we care about recording cards that
248 // contain references that point into the collection set
249 // is during RSet updating within an evacuation pause.
250 // In this case worker_i should be the id of a GC worker thread.
251 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
271
272 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, into_cset_dcq, false, worker_i);
273
274 // Now there should be no dirty cards.
275 if (G1RSLogCheckCardTable) {
276 CountNonCleanMemRegionClosure cl(_g1);
277 _ct_bs->mod_card_iterate(&cl);
278 // XXX This isn't true any more: keeping cards of young regions
279 // marked dirty broke it. Need some reasonable fix.
280 guarantee(cl.n() == 0, "Card table should be clean.");
281 }
282
283 _g1p->phase_times()->record_update_rs_time(worker_i, (os::elapsedTime() - start) * 1000.0);
284 }
285
286 void G1RemSet::cleanupHRRS() {
287 HeapRegionRemSet::cleanup();
288 }
289
290 void G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
291 int worker_i) {
292 #if CARD_REPEAT_HISTO
293 ct_freq_update_histo_and_reset();
294 #endif
295
296 // We cache the value of 'oc' closure into the appropriate slot in the
297 // _cset_rs_update_cl for this worker
298 assert(worker_i < (int)n_workers(), "sanity");
299 _cset_rs_update_cl[worker_i] = oc;
300
301 // A DirtyCardQueue that is used to hold cards containing references
302 // that point into the collection set. This DCQ is associated with a
303 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
304 // circumstances (i.e. the pause successfully completes), these cards
305 // are just discarded (there's no need to update the RSets of regions
306 // that were in the collection set - after the pause these regions
307 // are wholly 'free' of live objects. In the event of an evacuation
308 // failure the cards/buffers in this queue set are:
309 // * passed to the DirtyCardQueueSet that is used to manage deferred
310 // RSet updates, or
311 // * scanned for references that point into the collection set
312 // and the RSet of the corresponding region in the collection set
313 // is updated immediately.
314 DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set());
315
316 assert((ParallelGCThreads > 0) || worker_i == 0, "invariant");
317
318 // The two flags below were introduced temporarily to serialize
319 // the updating and scanning of remembered sets. There are some
320 // race conditions when these two operations are done in parallel
321 // and they are causing failures. When we resolve said race
322 // conditions, we'll revert back to parallel remembered set
323 // updating and scanning. See CRs 6677707 and 6677708.
324 if (G1UseParallelRSetUpdating || (worker_i == 0)) {
325 updateRS(&into_cset_dcq, worker_i);
326 } else {
327 _g1p->phase_times()->record_update_rs_processed_buffers(worker_i, 0);
328 _g1p->phase_times()->record_update_rs_time(worker_i, 0.0);
329 }
330 if (G1UseParallelRSetScanning || (worker_i == 0)) {
331 scanRS(oc, worker_i);
332 } else {
333 _g1p->phase_times()->record_scan_rs_time(worker_i, 0.0);
334 }
335
336 // We now clear the cached values of _cset_rs_update_cl for this worker
337 _cset_rs_update_cl[worker_i] = NULL;
338 }
339
340 void G1RemSet::prepare_for_oops_into_collection_set_do() {
341 cleanupHRRS();
342 ConcurrentG1Refine* cg1r = _g1->concurrent_g1_refine();
343 _g1->set_refine_cte_cl_concurrency(false);
344 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
345 dcqs.concatenate_logs();
346
347 if (G1CollectedHeap::use_parallel_gc_threads()) {
348 // Don't set the number of workers here. It will be set
349 // when the task is run
350 // _seq_task->set_n_termination((int)n_workers());
351 }
|
87 }
88
89 G1RemSet::~G1RemSet() {
90 delete _seq_task;
91 for (uint i = 0; i < n_workers(); i++) {
92 assert(_cset_rs_update_cl[i] == NULL, "it should be");
93 }
94 FREE_C_HEAP_ARRAY(OopsInHeapRegionClosure*, _cset_rs_update_cl, mtGC);
95 }
96
97 void CountNonCleanMemRegionClosure::do_MemRegion(MemRegion mr) {
98 if (_g1->is_in_g1_reserved(mr.start())) {
99 _n += (int) ((mr.byte_size() / CardTableModRefBS::card_size));
100 if (_start_first == NULL) _start_first = mr.start();
101 }
102 }
103
104 class ScanRSClosure : public HeapRegionClosure {
105 size_t _cards_done, _cards;
106 G1CollectedHeap* _g1h;
107
108 OopsInHeapRegionClosure* _oc;
109 CodeBlobToOopClosure* _code_root_cl;
110
111 G1BlockOffsetSharedArray* _bot_shared;
112 CardTableModRefBS *_ct_bs;
113
114 double _strong_code_root_scan_time_sec;
115 int _worker_i;
116 int _block_size;
117 bool _try_claimed;
118
119 public:
120 ScanRSClosure(OopsInHeapRegionClosure* oc,
121 CodeBlobToOopClosure* code_root_cl,
122 int worker_i) :
123 _oc(oc),
124 _code_root_cl(code_root_cl),
125 _strong_code_root_scan_time_sec(0.0),
126 _cards(0),
127 _cards_done(0),
128 _worker_i(worker_i),
129 _try_claimed(false)
130 {
131 _g1h = G1CollectedHeap::heap();
132 _bot_shared = _g1h->bot_shared();
133 _ct_bs = (CardTableModRefBS*) (_g1h->barrier_set());
134 _block_size = MAX2<int>(G1RSetScanBlockSize, 1);
135 }
136
137 void set_try_claimed() { _try_claimed = true; }
138
139 void scanCard(size_t index, HeapRegion *r) {
140 // Stack allocate the DirtyCardToOopClosure instance
141 HeapRegionDCTOC cl(_g1h, r, _oc,
142 CardTableModRefBS::Precise,
143 HeapRegionDCTOC::IntoCSFilterKind);
144
145 // Set the "from" region in the closure.
153 // We make the card as "claimed" lazily (so races are possible
154 // but they're benign), which reduces the number of duplicate
155 // scans (the rsets of the regions in the cset can intersect).
156 _ct_bs->set_card_claimed(index);
157 _cards_done++;
158 cl.do_MemRegion(mr);
159 }
160 }
161
162 void printCard(HeapRegion* card_region, size_t card_index,
163 HeapWord* card_start) {
164 gclog_or_tty->print_cr("T %d Region [" PTR_FORMAT ", " PTR_FORMAT ") "
165 "RS names card %p: "
166 "[" PTR_FORMAT ", " PTR_FORMAT ")",
167 _worker_i,
168 card_region->bottom(), card_region->end(),
169 card_index,
170 card_start, card_start + G1BlockOffsetSharedArray::N_words);
171 }
172
173 void scan_strong_code_roots(HeapRegion* r) {
174 double scan_start = os::elapsedTime();
175 r->strong_code_roots_do(_code_root_cl);
176 _strong_code_root_scan_time_sec += (os::elapsedTime() - scan_start);
177 }
178
179 bool doHeapRegion(HeapRegion* r) {
180 assert(r->in_collection_set(), "should only be called on elements of CS.");
181 HeapRegionRemSet* hrrs = r->rem_set();
182 if (hrrs->iter_is_complete()) return false; // All done.
183 if (!_try_claimed && !hrrs->claim_iter()) return false;
184 // If we ever free the collection set concurrently, we should also
185 // clear the card table concurrently therefore we won't need to
186 // add regions of the collection set to the dirty cards region.
187 _g1h->push_dirty_cards_region(r);
188 // If we didn't return above, then
189 // _try_claimed || r->claim_iter()
190 // is true: either we're supposed to work on claimed-but-not-complete
191 // regions, or we successfully claimed the region.
192
193 HeapRegionRemSetIterator iter(hrrs);
194 size_t card_index;
195
196 // We claim cards in block so as to recude the contention. The block size is determined by
197 // the G1RSetScanBlockSize parameter.
198 size_t jump_to_card = hrrs->iter_claimed_next(_block_size);
199 for (size_t current_card = 0; iter.has_next(card_index); current_card++) {
200 if (current_card >= jump_to_card + _block_size) {
201 jump_to_card = hrrs->iter_claimed_next(_block_size);
202 }
203 if (current_card < jump_to_card) continue;
204 HeapWord* card_start = _g1h->bot_shared()->address_for_index(card_index);
205 #if 0
206 gclog_or_tty->print("Rem set iteration yielded card [" PTR_FORMAT ", " PTR_FORMAT ").\n",
207 card_start, card_start + CardTableModRefBS::card_size_in_words);
208 #endif
209
210 HeapRegion* card_region = _g1h->heap_region_containing(card_start);
211 assert(card_region != NULL, "Yielding cards not in the heap?");
212 _cards++;
213
214 if (!card_region->is_on_dirty_cards_region_list()) {
215 _g1h->push_dirty_cards_region(card_region);
216 }
217
218 // If the card is dirty, then we will scan it during updateRS.
219 if (!card_region->in_collection_set() &&
220 !_ct_bs->is_card_dirty(card_index)) {
221 scanCard(card_index, card_region);
222 }
223 }
224 if (!_try_claimed) {
225 // Scan the strong code root list attached to the current region
226 scan_strong_code_roots(r);
227
228 hrrs->set_iter_complete();
229 }
230 return false;
231 }
232
233 double strong_code_root_scan_time_sec() {
234 return _strong_code_root_scan_time_sec;
235 }
236
237 size_t cards_done() { return _cards_done;}
238 size_t cards_looked_up() { return _cards;}
239 };
240
241 void G1RemSet::scanRS(OopsInHeapRegionClosure* oc,
242 CodeBlobToOopClosure* code_root_cl,
243 int worker_i) {
244 double rs_time_start = os::elapsedTime();
245 HeapRegion *startRegion = _g1->start_cset_region_for_worker(worker_i);
246
247 ScanRSClosure scanRScl(oc, code_root_cl, worker_i);
248
249 _g1->collection_set_iterate_from(startRegion, &scanRScl);
250 scanRScl.set_try_claimed();
251 _g1->collection_set_iterate_from(startRegion, &scanRScl);
252
253 double scan_rs_time_sec = (os::elapsedTime() - rs_time_start)
254 - scanRScl.strong_code_root_scan_time_sec();
255
256 assert(_cards_scanned != NULL, "invariant");
257 _cards_scanned[worker_i] = scanRScl.cards_done();
258
259 _g1p->phase_times()->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0);
260 _g1p->phase_times()->record_strong_code_root_scan_time(worker_i,
261 scanRScl.strong_code_root_scan_time_sec() * 1000.0);
262 }
263
264 // Closure used for updating RSets and recording references that
265 // point into the collection set. Only called during an
266 // evacuation pause.
267
268 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure {
269 G1RemSet* _g1rs;
270 DirtyCardQueue* _into_cset_dcq;
271 public:
272 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h,
273 DirtyCardQueue* into_cset_dcq) :
274 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq)
275 {}
276 bool do_card_ptr(jbyte* card_ptr, int worker_i) {
277 // The only time we care about recording cards that
278 // contain references that point into the collection set
279 // is during RSet updating within an evacuation pause.
280 // In this case worker_i should be the id of a GC worker thread.
281 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
301
302 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, into_cset_dcq, false, worker_i);
303
304 // Now there should be no dirty cards.
305 if (G1RSLogCheckCardTable) {
306 CountNonCleanMemRegionClosure cl(_g1);
307 _ct_bs->mod_card_iterate(&cl);
308 // XXX This isn't true any more: keeping cards of young regions
309 // marked dirty broke it. Need some reasonable fix.
310 guarantee(cl.n() == 0, "Card table should be clean.");
311 }
312
313 _g1p->phase_times()->record_update_rs_time(worker_i, (os::elapsedTime() - start) * 1000.0);
314 }
315
316 void G1RemSet::cleanupHRRS() {
317 HeapRegionRemSet::cleanup();
318 }
319
320 void G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
321 CodeBlobToOopClosure* code_root_cl,
322 int worker_i) {
323 #if CARD_REPEAT_HISTO
324 ct_freq_update_histo_and_reset();
325 #endif
326
327 // We cache the value of 'oc' closure into the appropriate slot in the
328 // _cset_rs_update_cl for this worker
329 assert(worker_i < (int)n_workers(), "sanity");
330 _cset_rs_update_cl[worker_i] = oc;
331
332 // A DirtyCardQueue that is used to hold cards containing references
333 // that point into the collection set. This DCQ is associated with a
334 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
335 // circumstances (i.e. the pause successfully completes), these cards
336 // are just discarded (there's no need to update the RSets of regions
337 // that were in the collection set - after the pause these regions
338 // are wholly 'free' of live objects. In the event of an evacuation
339 // failure the cards/buffers in this queue set are:
340 // * passed to the DirtyCardQueueSet that is used to manage deferred
341 // RSet updates, or
342 // * scanned for references that point into the collection set
343 // and the RSet of the corresponding region in the collection set
344 // is updated immediately.
345 DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set());
346
347 assert((ParallelGCThreads > 0) || worker_i == 0, "invariant");
348
349 // The two flags below were introduced temporarily to serialize
350 // the updating and scanning of remembered sets. There are some
351 // race conditions when these two operations are done in parallel
352 // and they are causing failures. When we resolve said race
353 // conditions, we'll revert back to parallel remembered set
354 // updating and scanning. See CRs 6677707 and 6677708.
355 if (G1UseParallelRSetUpdating || (worker_i == 0)) {
356 updateRS(&into_cset_dcq, worker_i);
357 } else {
358 _g1p->phase_times()->record_update_rs_processed_buffers(worker_i, 0);
359 _g1p->phase_times()->record_update_rs_time(worker_i, 0.0);
360 }
361 if (G1UseParallelRSetScanning || (worker_i == 0)) {
362 scanRS(oc, code_root_cl, worker_i);
363 } else {
364 _g1p->phase_times()->record_scan_rs_time(worker_i, 0.0);
365 }
366
367 // We now clear the cached values of _cset_rs_update_cl for this worker
368 _cset_rs_update_cl[worker_i] = NULL;
369 }
370
371 void G1RemSet::prepare_for_oops_into_collection_set_do() {
372 cleanupHRRS();
373 ConcurrentG1Refine* cg1r = _g1->concurrent_g1_refine();
374 _g1->set_refine_cte_cl_concurrency(false);
375 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
376 dcqs.concatenate_logs();
377
378 if (G1CollectedHeap::use_parallel_gc_threads()) {
379 // Don't set the number of workers here. It will be set
380 // when the task is run
381 // _seq_task->set_n_termination((int)n_workers());
382 }
|