179
180 bool marked_as_dirty = Atomic::cmpxchg(true, &_contains[region], false) == false;
181 if (marked_as_dirty) {
182 uint allocated = Atomic::add(1u, &_cur_idx) - 1;
183 _buffer[allocated] = region;
184 }
185 }
186
187 // Creates the union of this and the other G1DirtyRegions.
188 void merge(const G1DirtyRegions* other) {
189 for (uint i = 0; i < other->size(); i++) {
190 uint region = other->at(i);
191 if (!_contains[region]) {
192 _buffer[_cur_idx++] = region;
193 _contains[region] = true;
194 }
195 }
196 }
197 };
198
199 // Creates a snapshot of the current _top values at the start of collection to
200 // filter out card marks that we do not want to scan.
201 class G1ResetScanTopClosure : public HeapRegionClosure {
202 G1RemSetScanState* _scan_state;
203
204 public:
205 G1ResetScanTopClosure(G1RemSetScanState* scan_state) : _scan_state(scan_state) { }
206
207 virtual bool do_heap_region(HeapRegion* r) {
208 uint hrm_index = r->hrm_index();
209 if (r->in_collection_set()) {
210 // Young regions had their card table marked as young at their allocation;
211 // we need to make sure that these marks are cleared at the end of GC, *but*
212 // they should not be scanned for cards.
213 // So directly add them to the "all_dirty_regions".
214 // Same for regions in the (initial) collection set: they may contain cards from
215 // the log buffers, make sure they are cleaned.
216 _scan_state->add_all_dirty_region(hrm_index);
217 } else if (r->is_old_or_humongous_or_archive()) {
218 _scan_state->set_scan_top(hrm_index, r->top());
219 }
220 return false;
221 }
222 };
223 // For each region, contains the maximum top() value to be used during this garbage
224 // collection. Subsumes common checks like filtering out everything but old and
225 // humongous regions outside the collection set.
226 // This is valid because we are not interested in scanning stray remembered set
227 // entries from free or archive regions.
228 HeapWord** _scan_top;
229
230 class G1ClearCardTableTask : public AbstractGangTask {
231 G1CollectedHeap* _g1h;
232 G1DirtyRegions* _regions;
233 uint _chunk_length;
234
235 uint volatile _cur_dirty_regions;
236
237 G1RemSetScanState* _scan_state;
238
239 public:
240 G1ClearCardTableTask(G1CollectedHeap* g1h,
241 G1DirtyRegions* regions,
242 uint chunk_length,
311 ~G1RemSetScanState() {
312 FREE_C_HEAP_ARRAY(G1RemsetIterState, _collection_set_iter_state);
313 FREE_C_HEAP_ARRAY(uint, _card_table_scan_state);
314 FREE_C_HEAP_ARRAY(bool, _region_scan_chunks);
315 FREE_C_HEAP_ARRAY(HeapWord*, _scan_top);
316 }
317
318 void initialize(size_t max_regions) {
319 assert(_collection_set_iter_state == NULL, "Must not be initialized twice");
320 _max_regions = max_regions;
321 _collection_set_iter_state = NEW_C_HEAP_ARRAY(G1RemsetIterState, max_regions, mtGC);
322 _card_table_scan_state = NEW_C_HEAP_ARRAY(uint, max_regions, mtGC);
323 _num_total_scan_chunks = max_regions * _scan_chunks_per_region;
324 _region_scan_chunks = NEW_C_HEAP_ARRAY(bool, _num_total_scan_chunks, mtGC);
325
326 _scan_chunks_shift = (uint8_t)log2_intptr(HeapRegion::CardsPerRegion / _scan_chunks_per_region);
327 _scan_top = NEW_C_HEAP_ARRAY(HeapWord*, max_regions, mtGC);
328 }
329
330 void prepare() {
331 for (size_t i = 0; i < _max_regions; i++) {
332 _collection_set_iter_state[i] = false;
333 clear_scan_top((uint)i);
334 }
335
336 _all_dirty_regions = new G1DirtyRegions(_max_regions);
337 _next_dirty_regions = new G1DirtyRegions(_max_regions);
338
339 G1ResetScanTopClosure cl(this);
340 G1CollectedHeap::heap()->heap_region_iterate(&cl);
341 }
342
343 void prepare_for_merge_heap_roots() {
344 _all_dirty_regions->merge(_next_dirty_regions);
345
346 _next_dirty_regions->reset();
347 for (size_t i = 0; i < _max_regions; i++) {
348 _card_table_scan_state[i] = 0;
349 }
350
351 ::memset(_region_scan_chunks, false, _num_total_scan_chunks * sizeof(*_region_scan_chunks));
352 }
353
354 // Returns whether the given region contains cards we need to scan. The remembered
355 // set and other sources may contain cards that
356 // - are in uncommitted regions
357 // - are located in the collection set
358 // - are located in free regions
359 // as we do not clean up remembered sets before merging heap roots.
360 bool contains_cards_to_process(uint const region_idx) const {
413 }
414
415 G1CollectedHeap* g1h = G1CollectedHeap::heap();
416
417 WorkGang* workers = g1h->workers();
418 uint const max_workers = workers->active_workers();
419
420 uint const start_pos = num_regions * worker_id / max_workers;
421 uint cur = start_pos;
422
423 do {
424 bool result = cl->do_heap_region(g1h->region_at(_next_dirty_regions->at(cur)));
425 guarantee(!result, "Not allowed to ask for early termination.");
426 cur++;
427 if (cur == _next_dirty_regions->size()) {
428 cur = 0;
429 }
430 } while (cur != start_pos);
431 }
432
433 // Attempt to claim the given region in the collection set for iteration. Returns true
434 // if this call caused the transition from Unclaimed to Claimed.
435 inline bool claim_collection_set_region(uint region) {
436 assert(region < _max_regions, "Tried to access invalid region %u", region);
437 if (_collection_set_iter_state[region]) {
438 return false;
439 }
440 return !Atomic::cmpxchg(true, &_collection_set_iter_state[region], false);
441 }
442
443 bool has_cards_to_scan(uint region) {
444 assert(region < _max_regions, "Tried to access invalid region %u", region);
445 return _card_table_scan_state[region] < HeapRegion::CardsPerRegion;
446 }
447
448 uint claim_cards_to_scan(uint region, uint increment) {
449 assert(region < _max_regions, "Tried to access invalid region %u", region);
450 return Atomic::add(increment, &_card_table_scan_state[region]) - increment;
451 }
452
892 G1GCPhaseTimes::GCParPhases coderoots_phase,
893 G1GCPhaseTimes::GCParPhases objcopy_phase) {
894 G1ScanCollectionSetRegionClosure cl(_scan_state, pss, worker_id, scan_phase, coderoots_phase);
895 _g1h->collection_set_iterate_increment_from(&cl, worker_id);
896
897 G1GCPhaseTimes* p = _g1h->phase_times();
898
899 p->record_or_add_time_secs(scan_phase, worker_id, cl.rem_set_opt_root_scan_time().seconds());
900 p->record_or_add_time_secs(scan_phase, worker_id, cl.rem_set_opt_trim_partially_time().seconds());
901
902 p->record_or_add_time_secs(coderoots_phase, worker_id, cl.strong_code_root_scan_time().seconds());
903 p->add_time_secs(objcopy_phase, worker_id, cl.strong_code_root_trim_partially_time().seconds());
904
905 // At this time we record some metrics only for the evacuations after the initial one.
906 if (scan_phase == G1GCPhaseTimes::OptScanHR) {
907 p->record_or_add_thread_work_item(scan_phase, worker_id, cl.opt_refs_scanned(), G1GCPhaseTimes::ScanHRScannedOptRefs);
908 p->record_or_add_thread_work_item(scan_phase, worker_id, cl.opt_refs_memory_used(), G1GCPhaseTimes::ScanHRUsedMemory);
909 }
910 }
911
912 void G1RemSet::prepare_for_scan_heap_roots() {
913 G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
914 dcqs.concatenate_logs();
915
916 _scan_state->prepare();
917 }
918
919 class G1MergeHeapRootsTask : public AbstractGangTask {
920
921 // Visitor for remembered sets, dropping entries onto the card table.
922 class G1MergeCardSetClosure : public HeapRegionClosure {
923 G1RemSetScanState* _scan_state;
924 G1CardTable* _ct;
925
926 uint _merged_sparse;
927 uint _merged_fine;
928 uint _merged_coarse;
929
930 // Returns if the region contains cards we need to scan. If so, remember that
931 // region in the current set of dirty regions.
1219 }
1220
1221 WorkGang* workers = g1h->workers();
1222 size_t const increment_length = g1h->collection_set()->increment_length();
1223
1224 uint const num_workers = initial_evacuation ? workers->active_workers() :
1225 MIN2(workers->active_workers(), (uint)increment_length);
1226
1227 {
1228 G1MergeHeapRootsTask cl(_scan_state, num_workers, initial_evacuation);
1229 log_debug(gc, ergo)("Running %s using %u workers for " SIZE_FORMAT " regions",
1230 cl.name(), num_workers, increment_length);
1231 workers->run_task(&cl, num_workers);
1232 }
1233
1234 if (log_is_enabled(Debug, gc, remset)) {
1235 print_merge_heap_roots_stats();
1236 }
1237 }
1238
1239 void G1RemSet::prepare_for_scan_heap_roots(uint region_idx) {
1240 _scan_state->clear_scan_top(region_idx);
1241 }
1242
1243 void G1RemSet::cleanup_after_scan_heap_roots() {
1244 G1GCPhaseTimes* phase_times = _g1h->phase_times();
1245
1246 // Set all cards back to clean.
1247 double start = os::elapsedTime();
1248 _scan_state->cleanup(_g1h->workers());
1249 phase_times->record_clear_ct_time((os::elapsedTime() - start) * 1000.0);
1250 }
1251
1252 inline void check_card_ptr(CardTable::CardValue* card_ptr, G1CardTable* ct) {
1253 #ifdef ASSERT
1254 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1255 assert(g1h->is_in_exact(ct->addr_for(card_ptr)),
1256 "Card at " PTR_FORMAT " index " SIZE_FORMAT " representing heap at " PTR_FORMAT " (%u) must be in committed heap",
1257 p2i(card_ptr),
1258 ct->index_for(ct->addr_for(card_ptr)),
1259 p2i(ct->addr_for(card_ptr)),
|
179
180 bool marked_as_dirty = Atomic::cmpxchg(true, &_contains[region], false) == false;
181 if (marked_as_dirty) {
182 uint allocated = Atomic::add(1u, &_cur_idx) - 1;
183 _buffer[allocated] = region;
184 }
185 }
186
187 // Creates the union of this and the other G1DirtyRegions.
188 void merge(const G1DirtyRegions* other) {
189 for (uint i = 0; i < other->size(); i++) {
190 uint region = other->at(i);
191 if (!_contains[region]) {
192 _buffer[_cur_idx++] = region;
193 _contains[region] = true;
194 }
195 }
196 }
197 };
198
199 // For each region, contains the maximum top() value to be used during this garbage
200 // collection. Subsumes common checks like filtering out everything but old and
201 // humongous regions outside the collection set.
202 // This is valid because we are not interested in scanning stray remembered set
203 // entries from free or archive regions.
204 HeapWord** _scan_top;
205
206 class G1ClearCardTableTask : public AbstractGangTask {
207 G1CollectedHeap* _g1h;
208 G1DirtyRegions* _regions;
209 uint _chunk_length;
210
211 uint volatile _cur_dirty_regions;
212
213 G1RemSetScanState* _scan_state;
214
215 public:
216 G1ClearCardTableTask(G1CollectedHeap* g1h,
217 G1DirtyRegions* regions,
218 uint chunk_length,
287 ~G1RemSetScanState() {
288 FREE_C_HEAP_ARRAY(G1RemsetIterState, _collection_set_iter_state);
289 FREE_C_HEAP_ARRAY(uint, _card_table_scan_state);
290 FREE_C_HEAP_ARRAY(bool, _region_scan_chunks);
291 FREE_C_HEAP_ARRAY(HeapWord*, _scan_top);
292 }
293
294 void initialize(size_t max_regions) {
295 assert(_collection_set_iter_state == NULL, "Must not be initialized twice");
296 _max_regions = max_regions;
297 _collection_set_iter_state = NEW_C_HEAP_ARRAY(G1RemsetIterState, max_regions, mtGC);
298 _card_table_scan_state = NEW_C_HEAP_ARRAY(uint, max_regions, mtGC);
299 _num_total_scan_chunks = max_regions * _scan_chunks_per_region;
300 _region_scan_chunks = NEW_C_HEAP_ARRAY(bool, _num_total_scan_chunks, mtGC);
301
302 _scan_chunks_shift = (uint8_t)log2_intptr(HeapRegion::CardsPerRegion / _scan_chunks_per_region);
303 _scan_top = NEW_C_HEAP_ARRAY(HeapWord*, max_regions, mtGC);
304 }
305
306 void prepare() {
307 _all_dirty_regions = new G1DirtyRegions(_max_regions);
308 _next_dirty_regions = new G1DirtyRegions(_max_regions);
309 }
310
311 void prepare_for_merge_heap_roots() {
312 _all_dirty_regions->merge(_next_dirty_regions);
313
314 _next_dirty_regions->reset();
315 for (size_t i = 0; i < _max_regions; i++) {
316 _card_table_scan_state[i] = 0;
317 }
318
319 ::memset(_region_scan_chunks, false, _num_total_scan_chunks * sizeof(*_region_scan_chunks));
320 }
321
322 // Returns whether the given region contains cards we need to scan. The remembered
323 // set and other sources may contain cards that
324 // - are in uncommitted regions
325 // - are located in the collection set
326 // - are located in free regions
327 // as we do not clean up remembered sets before merging heap roots.
328 bool contains_cards_to_process(uint const region_idx) const {
381 }
382
383 G1CollectedHeap* g1h = G1CollectedHeap::heap();
384
385 WorkGang* workers = g1h->workers();
386 uint const max_workers = workers->active_workers();
387
388 uint const start_pos = num_regions * worker_id / max_workers;
389 uint cur = start_pos;
390
391 do {
392 bool result = cl->do_heap_region(g1h->region_at(_next_dirty_regions->at(cur)));
393 guarantee(!result, "Not allowed to ask for early termination.");
394 cur++;
395 if (cur == _next_dirty_regions->size()) {
396 cur = 0;
397 }
398 } while (cur != start_pos);
399 }
400
401 void reset_region_claim(uint region_idx) {
402 _collection_set_iter_state[region_idx] = false;
403 }
404
405 // Attempt to claim the given region in the collection set for iteration. Returns true
406 // if this call caused the transition from Unclaimed to Claimed.
407 inline bool claim_collection_set_region(uint region) {
408 assert(region < _max_regions, "Tried to access invalid region %u", region);
409 if (_collection_set_iter_state[region]) {
410 return false;
411 }
412 return !Atomic::cmpxchg(true, &_collection_set_iter_state[region], false);
413 }
414
415 bool has_cards_to_scan(uint region) {
416 assert(region < _max_regions, "Tried to access invalid region %u", region);
417 return _card_table_scan_state[region] < HeapRegion::CardsPerRegion;
418 }
419
420 uint claim_cards_to_scan(uint region, uint increment) {
421 assert(region < _max_regions, "Tried to access invalid region %u", region);
422 return Atomic::add(increment, &_card_table_scan_state[region]) - increment;
423 }
424
864 G1GCPhaseTimes::GCParPhases coderoots_phase,
865 G1GCPhaseTimes::GCParPhases objcopy_phase) {
866 G1ScanCollectionSetRegionClosure cl(_scan_state, pss, worker_id, scan_phase, coderoots_phase);
867 _g1h->collection_set_iterate_increment_from(&cl, worker_id);
868
869 G1GCPhaseTimes* p = _g1h->phase_times();
870
871 p->record_or_add_time_secs(scan_phase, worker_id, cl.rem_set_opt_root_scan_time().seconds());
872 p->record_or_add_time_secs(scan_phase, worker_id, cl.rem_set_opt_trim_partially_time().seconds());
873
874 p->record_or_add_time_secs(coderoots_phase, worker_id, cl.strong_code_root_scan_time().seconds());
875 p->add_time_secs(objcopy_phase, worker_id, cl.strong_code_root_trim_partially_time().seconds());
876
877 // At this time we record some metrics only for the evacuations after the initial one.
878 if (scan_phase == G1GCPhaseTimes::OptScanHR) {
879 p->record_or_add_thread_work_item(scan_phase, worker_id, cl.opt_refs_scanned(), G1GCPhaseTimes::ScanHRScannedOptRefs);
880 p->record_or_add_thread_work_item(scan_phase, worker_id, cl.opt_refs_memory_used(), G1GCPhaseTimes::ScanHRUsedMemory);
881 }
882 }
883
884 void G1RemSet::prepare_region_for_scan(HeapRegion* region) {
885 uint hrm_index = region->hrm_index();
886
887 _scan_state->reset_region_claim(hrm_index);
888 if (region->in_collection_set()) {
889 // Young regions had their card table marked as young at their allocation;
890 // we need to make sure that these marks are cleared at the end of GC, *but*
891 // they should not be scanned for cards.
892 // So directly add them to the "all_dirty_regions".
893 // Same for regions in the (initial) collection set: they may contain cards from
894 // the log buffers, make sure they are cleaned.
895 _scan_state->clear_scan_top(hrm_index);
896 _scan_state->add_all_dirty_region(hrm_index);
897 } else if (region->is_old_or_humongous_or_archive()) {
898 _scan_state->set_scan_top(hrm_index, region->top());
899 } else {
900 assert(region->is_free(), "Should only be free region at this point %s", region->get_type_str());
901 }
902 }
903
904 void G1RemSet::prepare_for_scan_heap_roots() {
905 G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
906 dcqs.concatenate_logs();
907
908 _scan_state->prepare();
909 }
910
911 class G1MergeHeapRootsTask : public AbstractGangTask {
912
913 // Visitor for remembered sets, dropping entries onto the card table.
914 class G1MergeCardSetClosure : public HeapRegionClosure {
915 G1RemSetScanState* _scan_state;
916 G1CardTable* _ct;
917
918 uint _merged_sparse;
919 uint _merged_fine;
920 uint _merged_coarse;
921
922 // Returns if the region contains cards we need to scan. If so, remember that
923 // region in the current set of dirty regions.
1211 }
1212
1213 WorkGang* workers = g1h->workers();
1214 size_t const increment_length = g1h->collection_set()->increment_length();
1215
1216 uint const num_workers = initial_evacuation ? workers->active_workers() :
1217 MIN2(workers->active_workers(), (uint)increment_length);
1218
1219 {
1220 G1MergeHeapRootsTask cl(_scan_state, num_workers, initial_evacuation);
1221 log_debug(gc, ergo)("Running %s using %u workers for " SIZE_FORMAT " regions",
1222 cl.name(), num_workers, increment_length);
1223 workers->run_task(&cl, num_workers);
1224 }
1225
1226 if (log_is_enabled(Debug, gc, remset)) {
1227 print_merge_heap_roots_stats();
1228 }
1229 }
1230
1231 void G1RemSet::exclude_region_from_scan(uint region_idx) {
1232 _scan_state->clear_scan_top(region_idx);
1233 }
1234
1235 void G1RemSet::cleanup_after_scan_heap_roots() {
1236 G1GCPhaseTimes* phase_times = _g1h->phase_times();
1237
1238 // Set all cards back to clean.
1239 double start = os::elapsedTime();
1240 _scan_state->cleanup(_g1h->workers());
1241 phase_times->record_clear_ct_time((os::elapsedTime() - start) * 1000.0);
1242 }
1243
1244 inline void check_card_ptr(CardTable::CardValue* card_ptr, G1CardTable* ct) {
1245 #ifdef ASSERT
1246 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1247 assert(g1h->is_in_exact(ct->addr_for(card_ptr)),
1248 "Card at " PTR_FORMAT " index " SIZE_FORMAT " representing heap at " PTR_FORMAT " (%u) must be in committed heap",
1249 p2i(card_ptr),
1250 ct->index_for(ct->addr_for(card_ptr)),
1251 p2i(ct->addr_for(card_ptr)),
|