28 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
29 #include "gc/g1/g1CollectedHeap.inline.hpp"
30 #include "gc/g1/g1CollectorPolicy.hpp"
31 #include "gc/g1/g1GCPhaseTimes.hpp"
32 #include "gc/g1/g1HotCardCache.hpp"
33 #include "gc/g1/g1OopClosures.inline.hpp"
34 #include "gc/g1/g1RemSet.inline.hpp"
35 #include "gc/g1/heapRegionManager.inline.hpp"
36 #include "gc/g1/heapRegionRemSet.hpp"
37 #include "memory/iterator.hpp"
38 #include "oops/oop.inline.hpp"
39 #include "utilities/globalDefinitions.hpp"
40 #include "utilities/intHisto.hpp"
41 #include "utilities/stack.inline.hpp"
42
43 G1RemSet::G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
44 : _g1(g1), _conc_refine_cards(0),
45 _ct_bs(ct_bs), _g1p(_g1->g1_policy()),
46 _cg1r(g1->concurrent_g1_refine()),
47 _cset_rs_update_cl(NULL),
48 _prev_period_summary()
49 {
50 _cset_rs_update_cl = NEW_C_HEAP_ARRAY(G1ParPushHeapRSClosure*, n_workers(), mtGC);
51 for (uint i = 0; i < n_workers(); i++) {
52 _cset_rs_update_cl[i] = NULL;
53 }
54 if (G1SummarizeRSetStats) {
55 _prev_period_summary.initialize(this);
56 }
57 }
58
59 G1RemSet::~G1RemSet() {
60 for (uint i = 0; i < n_workers(); i++) {
61 assert(_cset_rs_update_cl[i] == NULL, "it should be");
62 }
63 FREE_C_HEAP_ARRAY(G1ParPushHeapRSClosure*, _cset_rs_update_cl);
64 }
65
66 class ScanRSClosure : public HeapRegionClosure {
67 size_t _cards_done, _cards;
68 G1CollectedHeap* _g1h;
69
70 G1ParPushHeapRSClosure* _oc;
71 CodeBlobClosure* _code_root_cl;
72
73 G1BlockOffsetSharedArray* _bot_shared;
74 G1SATBCardTableModRefBS *_ct_bs;
75
76 double _strong_code_root_scan_time_sec;
225 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure {
226 G1RemSet* _g1rs;
227 DirtyCardQueue* _into_cset_dcq;
228 public:
229 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h,
230 DirtyCardQueue* into_cset_dcq) :
231 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq)
232 {}
233
234 bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
235 // The only time we care about recording cards that
236 // contain references that point into the collection set
237 // is during RSet updating within an evacuation pause.
238 // In this case worker_i should be the id of a GC worker thread.
239 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
240 assert(worker_i < ParallelGCThreads, "should be a GC worker");
241
242 if (_g1rs->refine_card(card_ptr, worker_i, true)) {
243 // 'card_ptr' contains references that point into the collection
244 // set. We need to record the card in the DCQS
245 // (G1CollectedHeap::into_cset_dirty_card_queue_set())
246 // that's used for that purpose.
247 //
248 // Enqueue the card
249 _into_cset_dcq->enqueue(card_ptr);
250 }
251 return true;
252 }
253 };
254
255 void G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, uint worker_i) {
256 RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq);
257
258 G1GCParPhaseTimesTracker x(_g1p->phase_times(), G1GCPhaseTimes::UpdateRS, worker_i);
259 {
260 // Apply the closure to the entries of the hot card cache.
261 G1GCParPhaseTimesTracker y(_g1p->phase_times(), G1GCPhaseTimes::ScanHCC, worker_i);
262 _g1->iterate_hcc_closure(&into_cset_update_rs_cl, worker_i);
263 }
264 // Apply the closure to all remaining log entries.
265 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, worker_i);
269 HeapRegionRemSet::cleanup();
270 }
271
272 size_t G1RemSet::oops_into_collection_set_do(G1ParPushHeapRSClosure* oc,
273 CodeBlobClosure* heap_region_codeblobs,
274 uint worker_i) {
275 // We cache the value of 'oc' closure into the appropriate slot in the
276 // _cset_rs_update_cl for this worker
277 assert(worker_i < n_workers(), "sanity");
278 _cset_rs_update_cl[worker_i] = oc;
279
280 // A DirtyCardQueue that is used to hold cards containing references
281 // that point into the collection set. This DCQ is associated with a
282 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
283 // circumstances (i.e. the pause successfully completes), these cards
284 // are just discarded (there's no need to update the RSets of regions
285 // that were in the collection set - after the pause these regions
286 // are wholly 'free' of live objects. In the event of an evacuation
287 // failure the cards/buffers in this queue set are passed to the
288 // DirtyCardQueueSet that is used to manage RSet updates
289 DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set());
290
291 updateRS(&into_cset_dcq, worker_i);
292 size_t cards_scanned = scanRS(oc, heap_region_codeblobs, worker_i);
293
294 // We now clear the cached values of _cset_rs_update_cl for this worker
295 _cset_rs_update_cl[worker_i] = NULL;
296 return cards_scanned;
297 }
298
299 void G1RemSet::prepare_for_oops_into_collection_set_do() {
300 cleanupHRRS();
301 _g1->set_refine_cte_cl_concurrency(false);
302 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
303 dcqs.concatenate_logs();
304 }
305
306 void G1RemSet::cleanup_after_oops_into_collection_set_do() {
307 // Cleanup after copy
308 _g1->set_refine_cte_cl_concurrency(true);
309 // Set all cards back to clean.
310 _g1->cleanUpCardTable();
311
312 DirtyCardQueueSet& into_cset_dcqs = _g1->into_cset_dirty_card_queue_set();
313 int into_cset_n_buffers = into_cset_dcqs.completed_buffers_num();
314
315 if (_g1->evacuation_failed()) {
316 double restore_remembered_set_start = os::elapsedTime();
317
318 // Restore remembered sets for the regions pointing into the collection set.
319 // We just need to transfer the completed buffers from the DirtyCardQueueSet
320 // used to hold cards that contain references that point into the collection set
321 // to the DCQS used to hold the deferred RS updates.
322 _g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs);
323 _g1->g1_policy()->phase_times()->record_evac_fail_restore_remsets((os::elapsedTime() - restore_remembered_set_start) * 1000.0);
324 }
325
326 // Free any completed buffers in the DirtyCardQueueSet used to hold cards
327 // which contain references that point into the collection.
328 _g1->into_cset_dirty_card_queue_set().clear();
329 assert(_g1->into_cset_dirty_card_queue_set().completed_buffers_num() == 0,
330 "all buffers should be freed");
331 _g1->into_cset_dirty_card_queue_set().clear_n_completed_buffers();
332 }
333
334 class ScrubRSClosure: public HeapRegionClosure {
335 G1CollectedHeap* _g1h;
336 BitMap* _region_bm;
337 BitMap* _card_bm;
338 CardTableModRefBS* _ctbs;
339 public:
340 ScrubRSClosure(BitMap* region_bm, BitMap* card_bm) :
341 _g1h(G1CollectedHeap::heap()),
342 _region_bm(region_bm), _card_bm(card_bm),
343 _ctbs(_g1h->g1_barrier_set()) {}
344
345 bool doHeapRegion(HeapRegion* r) {
346 if (!r->is_continues_humongous()) {
347 r->rem_set()->scrub(_ctbs, _region_bm, _card_bm);
348 }
349 return false;
350 }
351 };
569 }
570
571 summary->print_on(gclog_or_tty);
572 }
573
574 void G1RemSet::prepare_for_verify() {
575 if (G1HRRSFlushLogBuffersOnVerify &&
576 (VerifyBeforeGC || VerifyAfterGC)
577 && (!_g1->collector_state()->full_collection() || G1VerifyRSetsDuringFullGC)) {
578 cleanupHRRS();
579 _g1->set_refine_cte_cl_concurrency(false);
580 if (SafepointSynchronize::is_at_safepoint()) {
581 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
582 dcqs.concatenate_logs();
583 }
584
585 G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
586 bool use_hot_card_cache = hot_card_cache->use_cache();
587 hot_card_cache->set_use_cache(false);
588
589 DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set());
590 updateRS(&into_cset_dcq, 0);
591 _g1->into_cset_dirty_card_queue_set().clear();
592
593 hot_card_cache->set_use_cache(use_hot_card_cache);
594 assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed");
595 }
596 }
|
28 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
29 #include "gc/g1/g1CollectedHeap.inline.hpp"
30 #include "gc/g1/g1CollectorPolicy.hpp"
31 #include "gc/g1/g1GCPhaseTimes.hpp"
32 #include "gc/g1/g1HotCardCache.hpp"
33 #include "gc/g1/g1OopClosures.inline.hpp"
34 #include "gc/g1/g1RemSet.inline.hpp"
35 #include "gc/g1/heapRegionManager.inline.hpp"
36 #include "gc/g1/heapRegionRemSet.hpp"
37 #include "memory/iterator.hpp"
38 #include "oops/oop.inline.hpp"
39 #include "utilities/globalDefinitions.hpp"
40 #include "utilities/intHisto.hpp"
41 #include "utilities/stack.inline.hpp"
42
43 G1RemSet::G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
44 : _g1(g1), _conc_refine_cards(0),
45 _ct_bs(ct_bs), _g1p(_g1->g1_policy()),
46 _cg1r(g1->concurrent_g1_refine()),
47 _cset_rs_update_cl(NULL),
48 _prev_period_summary(),
49 _into_cset_dirty_card_queue_set(false)
50 {
51 _cset_rs_update_cl = NEW_C_HEAP_ARRAY(G1ParPushHeapRSClosure*, n_workers(), mtGC);
52 for (uint i = 0; i < n_workers(); i++) {
53 _cset_rs_update_cl[i] = NULL;
54 }
55 if (G1SummarizeRSetStats) {
56 _prev_period_summary.initialize(this);
57 }
58 // Initialize the card queue set used to hold cards containing
59 // references into the collection set.
60 _into_cset_dirty_card_queue_set.initialize(NULL, // Should never be called by the Java code
61 DirtyCardQ_CBL_mon,
62 DirtyCardQ_FL_lock,
63 -1, // never trigger processing
64 -1, // no limit on length
65 Shared_DirtyCardQ_lock,
66 &JavaThread::dirty_card_queue_set());
67 }
68
69 G1RemSet::~G1RemSet() {
70 for (uint i = 0; i < n_workers(); i++) {
71 assert(_cset_rs_update_cl[i] == NULL, "it should be");
72 }
73 FREE_C_HEAP_ARRAY(G1ParPushHeapRSClosure*, _cset_rs_update_cl);
74 }
75
76 class ScanRSClosure : public HeapRegionClosure {
77 size_t _cards_done, _cards;
78 G1CollectedHeap* _g1h;
79
80 G1ParPushHeapRSClosure* _oc;
81 CodeBlobClosure* _code_root_cl;
82
83 G1BlockOffsetSharedArray* _bot_shared;
84 G1SATBCardTableModRefBS *_ct_bs;
85
86 double _strong_code_root_scan_time_sec;
235 class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure {
236 G1RemSet* _g1rs;
237 DirtyCardQueue* _into_cset_dcq;
238 public:
239 RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h,
240 DirtyCardQueue* into_cset_dcq) :
241 _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq)
242 {}
243
244 bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
245 // The only time we care about recording cards that
246 // contain references that point into the collection set
247 // is during RSet updating within an evacuation pause.
248 // In this case worker_i should be the id of a GC worker thread.
249 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
250 assert(worker_i < ParallelGCThreads, "should be a GC worker");
251
252 if (_g1rs->refine_card(card_ptr, worker_i, true)) {
253 // 'card_ptr' contains references that point into the collection
254 // set. We need to record the card in the DCQS
255 // (_into_cset_dirty_card_queue_set)
256 // that's used for that purpose.
257 //
258 // Enqueue the card
259 _into_cset_dcq->enqueue(card_ptr);
260 }
261 return true;
262 }
263 };
264
265 void G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, uint worker_i) {
266 RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq);
267
268 G1GCParPhaseTimesTracker x(_g1p->phase_times(), G1GCPhaseTimes::UpdateRS, worker_i);
269 {
270 // Apply the closure to the entries of the hot card cache.
271 G1GCParPhaseTimesTracker y(_g1p->phase_times(), G1GCPhaseTimes::ScanHCC, worker_i);
272 _g1->iterate_hcc_closure(&into_cset_update_rs_cl, worker_i);
273 }
274 // Apply the closure to all remaining log entries.
275 _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, worker_i);
279 HeapRegionRemSet::cleanup();
280 }
281
282 size_t G1RemSet::oops_into_collection_set_do(G1ParPushHeapRSClosure* oc,
283 CodeBlobClosure* heap_region_codeblobs,
284 uint worker_i) {
285 // We cache the value of 'oc' closure into the appropriate slot in the
286 // _cset_rs_update_cl for this worker
287 assert(worker_i < n_workers(), "sanity");
288 _cset_rs_update_cl[worker_i] = oc;
289
290 // A DirtyCardQueue that is used to hold cards containing references
291 // that point into the collection set. This DCQ is associated with a
292 // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal
293 // circumstances (i.e. the pause successfully completes), these cards
294 // are just discarded (there's no need to update the RSets of regions
295 // that were in the collection set - after the pause these regions
296 // are wholly 'free' of live objects. In the event of an evacuation
297 // failure the cards/buffers in this queue set are passed to the
298 // DirtyCardQueueSet that is used to manage RSet updates
299 DirtyCardQueue into_cset_dcq(&_into_cset_dirty_card_queue_set);
300
301 updateRS(&into_cset_dcq, worker_i);
302 size_t cards_scanned = scanRS(oc, heap_region_codeblobs, worker_i);
303
304 // We now clear the cached values of _cset_rs_update_cl for this worker
305 _cset_rs_update_cl[worker_i] = NULL;
306 return cards_scanned;
307 }
308
309 void G1RemSet::prepare_for_oops_into_collection_set_do() {
310 cleanupHRRS();
311 _g1->set_refine_cte_cl_concurrency(false);
312 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
313 dcqs.concatenate_logs();
314 }
315
316 void G1RemSet::cleanup_after_oops_into_collection_set_do() {
317 // Cleanup after copy
318 _g1->set_refine_cte_cl_concurrency(true);
319 // Set all cards back to clean.
320 _g1->cleanUpCardTable();
321
322 DirtyCardQueueSet& into_cset_dcqs = _into_cset_dirty_card_queue_set;
323 int into_cset_n_buffers = into_cset_dcqs.completed_buffers_num();
324
325 if (_g1->evacuation_failed()) {
326 double restore_remembered_set_start = os::elapsedTime();
327
328 // Restore remembered sets for the regions pointing into the collection set.
329 // We just need to transfer the completed buffers from the DirtyCardQueueSet
330 // used to hold cards that contain references that point into the collection set
331 // to the DCQS used to hold the deferred RS updates.
332 _g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs);
333 _g1->g1_policy()->phase_times()->record_evac_fail_restore_remsets((os::elapsedTime() - restore_remembered_set_start) * 1000.0);
334 }
335
336 // Free any completed buffers in the DirtyCardQueueSet used to hold cards
337 // which contain references that point into the collection.
338 _into_cset_dirty_card_queue_set.clear();
339 assert(_into_cset_dirty_card_queue_set.completed_buffers_num() == 0,
340 "all buffers should be freed");
341 _into_cset_dirty_card_queue_set.clear_n_completed_buffers();
342 }
343
344 class ScrubRSClosure: public HeapRegionClosure {
345 G1CollectedHeap* _g1h;
346 BitMap* _region_bm;
347 BitMap* _card_bm;
348 CardTableModRefBS* _ctbs;
349 public:
350 ScrubRSClosure(BitMap* region_bm, BitMap* card_bm) :
351 _g1h(G1CollectedHeap::heap()),
352 _region_bm(region_bm), _card_bm(card_bm),
353 _ctbs(_g1h->g1_barrier_set()) {}
354
355 bool doHeapRegion(HeapRegion* r) {
356 if (!r->is_continues_humongous()) {
357 r->rem_set()->scrub(_ctbs, _region_bm, _card_bm);
358 }
359 return false;
360 }
361 };
579 }
580
581 summary->print_on(gclog_or_tty);
582 }
583
584 void G1RemSet::prepare_for_verify() {
585 if (G1HRRSFlushLogBuffersOnVerify &&
586 (VerifyBeforeGC || VerifyAfterGC)
587 && (!_g1->collector_state()->full_collection() || G1VerifyRSetsDuringFullGC)) {
588 cleanupHRRS();
589 _g1->set_refine_cte_cl_concurrency(false);
590 if (SafepointSynchronize::is_at_safepoint()) {
591 DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
592 dcqs.concatenate_logs();
593 }
594
595 G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
596 bool use_hot_card_cache = hot_card_cache->use_cache();
597 hot_card_cache->set_use_cache(false);
598
599 DirtyCardQueue into_cset_dcq(&_into_cset_dirty_card_queue_set);
600 updateRS(&into_cset_dcq, 0);
601 _into_cset_dirty_card_queue_set.clear();
602
603 hot_card_cache->set_use_cache(use_hot_card_cache);
604 assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed");
605 }
606 }
|