31 #include "gc/g1/g1CollectorPolicy.hpp"
32 #include "gc/g1/g1CollectorState.hpp"
33 #include "gc/g1/g1ConcurrentMark.inline.hpp"
34 #include "gc/g1/g1HeapVerifier.hpp"
35 #include "gc/g1/g1OopClosures.inline.hpp"
36 #include "gc/g1/g1StringDedup.hpp"
37 #include "gc/g1/heapRegion.inline.hpp"
38 #include "gc/g1/heapRegionRemSet.hpp"
39 #include "gc/g1/heapRegionSet.inline.hpp"
40 #include "gc/g1/suspendibleThreadSet.hpp"
41 #include "gc/shared/gcId.hpp"
42 #include "gc/shared/gcTimer.hpp"
43 #include "gc/shared/gcTrace.hpp"
44 #include "gc/shared/gcTraceTime.inline.hpp"
45 #include "gc/shared/genOopClosures.inline.hpp"
46 #include "gc/shared/referencePolicy.hpp"
47 #include "gc/shared/strongRootsScope.hpp"
48 #include "gc/shared/taskqueue.inline.hpp"
49 #include "gc/shared/vmGCOperations.hpp"
50 #include "logging/log.hpp"
51 #include "memory/allocation.hpp"
52 #include "memory/resourceArea.hpp"
53 #include "oops/oop.inline.hpp"
54 #include "runtime/atomic.inline.hpp"
55 #include "runtime/handles.inline.hpp"
56 #include "runtime/java.hpp"
57 #include "runtime/prefetch.inline.hpp"
58 #include "services/memTracker.hpp"
59
60 // Concurrent marking bit map wrapper
61
62 G1CMBitMapRO::G1CMBitMapRO(int shifter) :
63 _bm(),
64 _shifter(shifter) {
65 _bmStartWord = 0;
66 _bmWordSize = 0;
67 }
68
69 HeapWord* G1CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
70 const HeapWord* limit) const {
338 while (scan_in_progress()) {
339 RootRegionScan_lock->wait(Mutex::_no_safepoint_check_flag);
340 }
341 }
342 return true;
343 }
344
345 uint G1ConcurrentMark::scale_parallel_threads(uint n_par_threads) {
346 return MAX2((n_par_threads + 2) / 4, 1U);
347 }
348
349 G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h, G1RegionToSpaceMapper* prev_bitmap_storage, G1RegionToSpaceMapper* next_bitmap_storage) :
350 _g1h(g1h),
351 _markBitMap1(),
352 _markBitMap2(),
353 _parallel_marking_threads(0),
354 _max_parallel_marking_threads(0),
355 _sleep_factor(0.0),
356 _marking_task_overhead(1.0),
357 _cleanup_list("Cleanup List"),
358 _region_bm((BitMap::idx_t)(g1h->max_regions()), false /* in_resource_area*/),
359 _card_bm((g1h->reserved_region().byte_size() + CardTableModRefBS::card_size - 1) >>
360 CardTableModRefBS::card_shift,
361 false /* in_resource_area*/),
362
363 _prevMarkBitMap(&_markBitMap1),
364 _nextMarkBitMap(&_markBitMap2),
365
366 _markStack(this),
367 // _finger set in set_non_marking_state
368
369 _max_worker_id(ParallelGCThreads),
370 // _active_tasks set in set_non_marking_state
371 // _tasks set inside the constructor
372 _task_queues(new G1CMTaskQueueSet((int) _max_worker_id)),
373 _terminator(ParallelTaskTerminator((int) _max_worker_id, _task_queues)),
374
375 _has_overflown(false),
376 _concurrent(false),
377 _has_aborted(false),
378 _restart_for_overflow(false),
379 _concurrent_marking_in_progress(false),
380 _concurrent_phase_status(ConcPhaseNotStarted),
381
382 // _verbose_level set below
383
384 _init_times(),
385 _remark_times(), _remark_mark_times(), _remark_weak_ref_times(),
386 _cleanup_times(),
387 _total_counting_time(0.0),
388 _total_rs_scrub_time(0.0),
389
390 _parallel_workers(NULL),
391
392 _count_card_bitmaps(NULL),
393 _count_marked_bytes(NULL),
394 _completed_initialization(false) {
395
396 _markBitMap1.initialize(g1h->reserved_region(), prev_bitmap_storage);
397 _markBitMap2.initialize(g1h->reserved_region(), next_bitmap_storage);
398
399 // Create & start a ConcurrentMark thread.
400 _cmThread = new ConcurrentMarkThread(this);
401 assert(cmThread() != NULL, "CM Thread should have been created");
402 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
403 if (_cmThread->osthread() == NULL) {
404 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
405 }
406
407 assert(CGC_lock != NULL, "Where's the CGC_lock?");
408 assert(_markBitMap1.covers(g1h->reserved_region()), "_markBitMap1 inconsistency");
409 assert(_markBitMap2.covers(g1h->reserved_region()), "_markBitMap2 inconsistency");
410
411 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
412 satb_qs.set_buffer_size(G1SATBBufferSize);
413
484 "must be between 1 and " SIZE_FORMAT,
485 MarkStackSize, MarkStackSizeMax);
486 return;
487 }
488 } else if (FLAG_IS_CMDLINE(MarkStackSizeMax)) {
489 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
490 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT ")"
491 " or for MarkStackSizeMax (" SIZE_FORMAT ")",
492 MarkStackSize, MarkStackSizeMax);
493 return;
494 }
495 }
496 }
497 }
498
499 if (!_markStack.allocate(MarkStackSize)) {
500 log_warning(gc)("Failed to allocate CM marking stack");
501 return;
502 }
503
504 _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_worker_id, mtGC);
505 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_worker_id, mtGC);
506
507 _count_card_bitmaps = NEW_C_HEAP_ARRAY(BitMap, _max_worker_id, mtGC);
508 _count_marked_bytes = NEW_C_HEAP_ARRAY(size_t*, _max_worker_id, mtGC);
509
510 BitMap::idx_t card_bm_size = _card_bm.size();
511
512 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
513 _active_tasks = _max_worker_id;
514
515 uint max_regions = _g1h->max_regions();
516 for (uint i = 0; i < _max_worker_id; ++i) {
517 G1CMTaskQueue* task_queue = new G1CMTaskQueue();
518 task_queue->initialize();
519 _task_queues->register_queue(i, task_queue);
520
521 _count_card_bitmaps[i] = BitMap(card_bm_size, false);
522 _count_marked_bytes[i] = NEW_C_HEAP_ARRAY(size_t, max_regions, mtGC);
523
524 _tasks[i] = new G1CMTask(i, this,
525 _count_marked_bytes[i],
526 &_count_card_bitmaps[i],
527 task_queue, _task_queues);
528
529 _accum_task_vtime[i] = 0.0;
530 }
531
532 // Calculate the card number for the bottom of the heap. Used
533 // in biasing indexes into the accounting card bitmaps.
534 _heap_bottom_card_num =
535 intptr_t(uintptr_t(_g1h->reserved_region().start()) >>
536 CardTableModRefBS::card_shift);
537
538 // Clear all the liveness counting data
539 clear_all_count_data();
540
541 // so that the call below can read a sensible value
542 _heap_start = g1h->reserved_region().start();
543 set_non_marking_state();
544 _completed_initialization = true;
545 }
546
547 void G1ConcurrentMark::reset() {
548 // Starting values for these two. This should be called in a STW
549 // phase.
550 MemRegion reserved = _g1h->g1_reserved();
551 _heap_start = reserved.start();
552 _heap_end = reserved.end();
553
554 // Separated the asserts so that we know which one fires.
555 assert(_heap_start != NULL, "heap bounds should look ok");
556 assert(_heap_end != NULL, "heap bounds should look ok");
557 assert(_heap_start < _heap_end, "heap bounds should look ok");
558
559 // Reset all the marking data structures and any necessary flags
560 reset_marking_state();
698 assert(may_yield || SafepointSynchronize::is_at_safepoint(), "Non-yielding bitmap clear only allowed at safepoint.");
699
700 G1ClearBitMapTask task(bitmap, this, workers->active_workers(), may_yield);
701 workers->run_task(&task);
702 guarantee(!may_yield || task.is_complete(), "Must have completed iteration when not yielding.");
703 }
704
705 void G1ConcurrentMark::cleanup_for_next_mark() {
706 // Make sure that the concurrent mark thread looks to still be in
707 // the current cycle.
708 guarantee(cmThread()->during_cycle(), "invariant");
709
710 // We are finishing up the current cycle by clearing the next
711 // marking bitmap and getting it ready for the next cycle. During
712 // this time no other cycle can start. So, let's make sure that this
713 // is the case.
714 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
715
716 clear_bitmap(_nextMarkBitMap, _parallel_workers, true);
717
718 // Clear the liveness counting data. If the marking has been aborted, the abort()
719 // call already did that.
720 if (!has_aborted()) {
721 clear_all_count_data();
722 }
723
724 // Repeat the asserts from above.
725 guarantee(cmThread()->during_cycle(), "invariant");
726 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
727 }
728
729 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) {
730 assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint.");
731 clear_bitmap((G1CMBitMap*)_prevMarkBitMap, workers, false);
732 }
733
734 class CheckBitmapClearHRClosure : public HeapRegionClosure {
735 G1CMBitMap* _bitmap;
736 bool _error;
737 public:
738 CheckBitmapClearHRClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) {
739 }
740
741 virtual bool doHeapRegion(HeapRegion* r) {
1109
1110 double mark_work_end = os::elapsedTime();
1111
1112 weakRefsWork(clear_all_soft_refs);
1113
1114 if (has_overflown()) {
1115 // Oops. We overflowed. Restart concurrent marking.
1116 _restart_for_overflow = true;
1117
1118 // Verify the heap w.r.t. the previous marking bitmap.
1119 if (VerifyDuringGC) {
1120 HandleMark hm; // handle scope
1121 g1h->prepare_for_verify();
1122 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (overflow)");
1123 }
1124
1125 // Clear the marking state because we will be restarting
1126 // marking due to overflowing the global mark stack.
1127 reset_marking_state();
1128 } else {
1129 {
1130 GCTraceTime(Debug, gc) trace("Aggregate Data", g1h->gc_timer_cm());
1131
1132 // Aggregate the per-task counting data that we have accumulated
1133 // while marking.
1134 aggregate_count_data();
1135 }
1136
1137 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1138 // We're done with marking.
1139 // This is the end of the marking cycle, we're expected all
1140 // threads to have SATB queues with active set to true.
1141 satb_mq_set.set_active_all_threads(false, /* new active value */
1142 true /* expected_active */);
1143
1144 if (VerifyDuringGC) {
1145 HandleMark hm; // handle scope
1146 g1h->prepare_for_verify();
1147 Universe::verify(VerifyOption_G1UseNextMarking, "During GC (after)");
1148 }
1149 g1h->verifier()->check_bitmaps("Remark End");
1150 assert(!restart_for_overflow(), "sanity");
1151 // Completely reset the marking state since marking completed
1152 set_non_marking_state();
1153 }
1154
1155 // Expand the marking stack, if we have to and if we can.
1156 if (_markStack.should_expand()) {
1157 _markStack.expand();
1158 }
1159
1160 // Statistics
1161 double now = os::elapsedTime();
1162 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1163 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1164 _remark_times.add((now - start) * 1000.0);
1165
1166 g1p->record_concurrent_mark_remark_end();
1167
1168 G1CMIsAliveClosure is_alive(g1h);
1169 g1h->gc_tracer_cm()->report_object_count_after_gc(&is_alive);
1170 }
1171
1172 // Base class of the closures that finalize and verify the
1173 // liveness counting data.
1174 class G1CMCountDataClosureBase: public HeapRegionClosure {
1175 protected:
1176 G1CollectedHeap* _g1h;
1177 G1ConcurrentMark* _cm;
1178 CardTableModRefBS* _ct_bs;
1179
1180 BitMap* _region_bm;
1181 BitMap* _card_bm;
1182
1183 // Takes a region that's not empty (i.e., it has at least one
1184 // live object in it and sets its corresponding bit on the region
1185 // bitmap to 1.
1186 void set_bit_for_region(HeapRegion* hr) {
1187 BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index();
1188 _region_bm->par_at_put(index, true);
1189 }
1190
1191 public:
1192 G1CMCountDataClosureBase(G1CollectedHeap* g1h,
1193 BitMap* region_bm, BitMap* card_bm):
1194 _g1h(g1h), _cm(g1h->concurrent_mark()),
1195 _ct_bs(barrier_set_cast<CardTableModRefBS>(g1h->barrier_set())),
1196 _region_bm(region_bm), _card_bm(card_bm) { }
1197 };
1198
1199 // Closure that calculates the # live objects per region. Used
1200 // for verification purposes during the cleanup pause.
1201 class CalcLiveObjectsClosure: public G1CMCountDataClosureBase {
1202 G1CMBitMapRO* _bm;
1203 size_t _region_marked_bytes;
1204
1205 public:
1206 CalcLiveObjectsClosure(G1CMBitMapRO *bm, G1CollectedHeap* g1h,
1207 BitMap* region_bm, BitMap* card_bm) :
1208 G1CMCountDataClosureBase(g1h, region_bm, card_bm),
1209 _bm(bm), _region_marked_bytes(0) { }
1210
1211 bool doHeapRegion(HeapRegion* hr) {
1212 HeapWord* ntams = hr->next_top_at_mark_start();
1213 HeapWord* start = hr->bottom();
1214
1215 assert(start <= hr->end() && start <= ntams && ntams <= hr->end(),
1216 "Preconditions not met - "
1217 "start: " PTR_FORMAT ", ntams: " PTR_FORMAT ", end: " PTR_FORMAT,
1218 p2i(start), p2i(ntams), p2i(hr->end()));
1219
1220 // Find the first marked object at or after "start".
1221 start = _bm->getNextMarkedWordAddress(start, ntams);
1222
1223 size_t marked_bytes = 0;
1224
1225 while (start < ntams) {
1226 oop obj = oop(start);
1227 int obj_sz = obj->size();
1228 HeapWord* obj_end = start + obj_sz;
1229
1230 BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start);
1231 BitMap::idx_t end_idx = _cm->card_bitmap_index_for(obj_end);
1232
1233 // Note: if we're looking at the last region in heap - obj_end
1234 // could be actually just beyond the end of the heap; end_idx
1235 // will then correspond to a (non-existent) card that is also
1236 // just beyond the heap.
1237 if (_g1h->is_in_g1_reserved(obj_end) && !_ct_bs->is_card_aligned(obj_end)) {
1238 // end of object is not card aligned - increment to cover
1239 // all the cards spanned by the object
1240 end_idx += 1;
1241 }
1242
1243 // Set the bits in the card BM for the cards spanned by this object.
1244 _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */);
1245
1246 // Add the size of this object to the number of marked bytes.
1247 marked_bytes += (size_t)obj_sz * HeapWordSize;
1248
1249 // This will happen if we are handling a humongous object that spans
1250 // several heap regions.
1251 if (obj_end > hr->end()) {
1252 break;
1253 }
1254 // Find the next marked object after this one.
1255 start = _bm->getNextMarkedWordAddress(obj_end, ntams);
1256 }
1257
1258 // Mark the allocated-since-marking portion...
1259 HeapWord* top = hr->top();
1260 if (ntams < top) {
1261 BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams);
1262 BitMap::idx_t end_idx = _cm->card_bitmap_index_for(top);
1263
1264 // Note: if we're looking at the last region in heap - top
1265 // could be actually just beyond the end of the heap; end_idx
1266 // will then correspond to a (non-existent) card that is also
1267 // just beyond the heap.
1268 if (_g1h->is_in_g1_reserved(top) && !_ct_bs->is_card_aligned(top)) {
1269 // end of object is not card aligned - increment to cover
1270 // all the cards spanned by the object
1271 end_idx += 1;
1272 }
1273 _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */);
1274
1275 // This definitely means the region has live objects.
1276 set_bit_for_region(hr);
1277 }
1278
1279 // Update the live region bitmap.
1280 if (marked_bytes > 0) {
1281 set_bit_for_region(hr);
1282 }
1283
1284 // Set the marked bytes for the current region so that
1285 // it can be queried by a calling verification routine
1286 _region_marked_bytes = marked_bytes;
1287
1288 return false;
1289 }
1290
1291 size_t region_marked_bytes() const { return _region_marked_bytes; }
1292 };
1293
1294 // Heap region closure used for verifying the counting data
1295 // that was accumulated concurrently and aggregated during
1296 // the remark pause. This closure is applied to the heap
1297 // regions during the STW cleanup pause.
1298
1299 class VerifyLiveObjectDataHRClosure: public HeapRegionClosure {
1300 G1CollectedHeap* _g1h;
1301 G1ConcurrentMark* _cm;
1302 CalcLiveObjectsClosure _calc_cl;
1303 BitMap* _region_bm; // Region BM to be verified
1304 BitMap* _card_bm; // Card BM to be verified
1305
1306 BitMap* _exp_region_bm; // Expected Region BM values
1307 BitMap* _exp_card_bm; // Expected card BM values
1308
1309 int _failures;
1310
1311 public:
1312 VerifyLiveObjectDataHRClosure(G1CollectedHeap* g1h,
1313 BitMap* region_bm,
1314 BitMap* card_bm,
1315 BitMap* exp_region_bm,
1316 BitMap* exp_card_bm) :
1317 _g1h(g1h), _cm(g1h->concurrent_mark()),
1318 _calc_cl(_cm->nextMarkBitMap(), g1h, exp_region_bm, exp_card_bm),
1319 _region_bm(region_bm), _card_bm(card_bm),
1320 _exp_region_bm(exp_region_bm), _exp_card_bm(exp_card_bm),
1321 _failures(0) { }
1322
1323 int failures() const { return _failures; }
1324
1325 bool doHeapRegion(HeapRegion* hr) {
1326 int failures = 0;
1327
1328 // Call the CalcLiveObjectsClosure to walk the marking bitmap for
1329 // this region and set the corresponding bits in the expected region
1330 // and card bitmaps.
1331 bool res = _calc_cl.doHeapRegion(hr);
1332 assert(res == false, "should be continuing");
1333
1334 // Verify the marked bytes for this region.
1335 size_t exp_marked_bytes = _calc_cl.region_marked_bytes();
1336 size_t act_marked_bytes = hr->next_marked_bytes();
1337
1338 if (exp_marked_bytes > act_marked_bytes) {
1339 if (hr->is_starts_humongous()) {
1340 // For start_humongous regions, the size of the whole object will be
1341 // in exp_marked_bytes.
1342 HeapRegion* region = hr;
1343 int num_regions;
1344 for (num_regions = 0; region != NULL; num_regions++) {
1345 region = _g1h->next_region_in_humongous(region);
1346 }
1347 if ((num_regions-1) * HeapRegion::GrainBytes >= exp_marked_bytes) {
1348 failures += 1;
1349 } else if (num_regions * HeapRegion::GrainBytes < exp_marked_bytes) {
1350 failures += 1;
1351 }
1352 } else {
1353 // We're not OK if expected marked bytes > actual marked bytes. It means
1354 // we have missed accounting some objects during the actual marking.
1355 failures += 1;
1356 }
1357 }
1358
1359 // Verify the bit, for this region, in the actual and expected
1360 // (which was just calculated) region bit maps.
1361 // We're not OK if the bit in the calculated expected region
1362 // bitmap is set and the bit in the actual region bitmap is not.
1363 BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index();
1364
1365 bool expected = _exp_region_bm->at(index);
1366 bool actual = _region_bm->at(index);
1367 if (expected && !actual) {
1368 failures += 1;
1369 }
1370
1371 // Verify that the card bit maps for the cards spanned by the current
1372 // region match. We have an error if we have a set bit in the expected
1373 // bit map and the corresponding bit in the actual bitmap is not set.
1374
1375 BitMap::idx_t start_idx = _cm->card_bitmap_index_for(hr->bottom());
1376 BitMap::idx_t end_idx = _cm->card_bitmap_index_for(hr->top());
1377
1378 for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) {
1379 expected = _exp_card_bm->at(i);
1380 actual = _card_bm->at(i);
1381
1382 if (expected && !actual) {
1383 failures += 1;
1384 }
1385 }
1386
1387 _failures += failures;
1388
1389 // We could stop iteration over the heap when we
1390 // find the first violating region by returning true.
1391 return false;
1392 }
1393 };
1394
1395 class G1ParVerifyFinalCountTask: public AbstractGangTask {
1396 protected:
1397 G1CollectedHeap* _g1h;
1398 G1ConcurrentMark* _cm;
1399 BitMap* _actual_region_bm;
1400 BitMap* _actual_card_bm;
1401
1402 uint _n_workers;
1403
1404 BitMap* _expected_region_bm;
1405 BitMap* _expected_card_bm;
1406
1407 int _failures;
1408
1409 HeapRegionClaimer _hrclaimer;
1410
1411 public:
1412 G1ParVerifyFinalCountTask(G1CollectedHeap* g1h,
1413 BitMap* region_bm, BitMap* card_bm,
1414 BitMap* expected_region_bm, BitMap* expected_card_bm)
1415 : AbstractGangTask("G1 verify final counting"),
1416 _g1h(g1h), _cm(_g1h->concurrent_mark()),
1417 _actual_region_bm(region_bm), _actual_card_bm(card_bm),
1418 _expected_region_bm(expected_region_bm), _expected_card_bm(expected_card_bm),
1419 _failures(0),
1420 _n_workers(_g1h->workers()->active_workers()), _hrclaimer(_n_workers) {
1421 assert(VerifyDuringGC, "don't call this otherwise");
1422 assert(_expected_card_bm->size() == _actual_card_bm->size(), "sanity");
1423 assert(_expected_region_bm->size() == _actual_region_bm->size(), "sanity");
1424 }
1425
1426 void work(uint worker_id) {
1427 assert(worker_id < _n_workers, "invariant");
1428
1429 VerifyLiveObjectDataHRClosure verify_cl(_g1h,
1430 _actual_region_bm, _actual_card_bm,
1431 _expected_region_bm,
1432 _expected_card_bm);
1433
1434 _g1h->heap_region_par_iterate(&verify_cl, worker_id, &_hrclaimer);
1435
1436 Atomic::add(verify_cl.failures(), &_failures);
1437 }
1438
1439 int failures() const { return _failures; }
1440 };
1441
1442 // Closure that finalizes the liveness counting data.
1443 // Used during the cleanup pause.
1444 // Sets the bits corresponding to the interval [NTAMS, top]
1445 // (which contains the implicitly live objects) in the
1446 // card liveness bitmap. Also sets the bit for each region,
1447 // containing live data, in the region liveness bitmap.
1448
1449 class FinalCountDataUpdateClosure: public G1CMCountDataClosureBase {
1450 public:
1451 FinalCountDataUpdateClosure(G1CollectedHeap* g1h,
1452 BitMap* region_bm,
1453 BitMap* card_bm) :
1454 G1CMCountDataClosureBase(g1h, region_bm, card_bm) { }
1455
1456 bool doHeapRegion(HeapRegion* hr) {
1457 HeapWord* ntams = hr->next_top_at_mark_start();
1458 HeapWord* top = hr->top();
1459
1460 assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions.");
1461
1462 // Mark the allocated-since-marking portion...
1463 if (ntams < top) {
1464 // This definitely means the region has live objects.
1465 set_bit_for_region(hr);
1466
1467 // Now set the bits in the card bitmap for [ntams, top)
1468 BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams);
1469 BitMap::idx_t end_idx = _cm->card_bitmap_index_for(top);
1470
1471 // Note: if we're looking at the last region in heap - top
1472 // could be actually just beyond the end of the heap; end_idx
1473 // will then correspond to a (non-existent) card that is also
1474 // just beyond the heap.
1475 if (_g1h->is_in_g1_reserved(top) && !_ct_bs->is_card_aligned(top)) {
1476 // end of object is not card aligned - increment to cover
1477 // all the cards spanned by the object
1478 end_idx += 1;
1479 }
1480
1481 assert(end_idx <= _card_bm->size(),
1482 "oob: end_idx= " SIZE_FORMAT ", bitmap size= " SIZE_FORMAT,
1483 end_idx, _card_bm->size());
1484 assert(start_idx < _card_bm->size(),
1485 "oob: start_idx= " SIZE_FORMAT ", bitmap size= " SIZE_FORMAT,
1486 start_idx, _card_bm->size());
1487
1488 _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */);
1489 }
1490
1491 // Set the bit for the region if it contains live data
1492 if (hr->next_marked_bytes() > 0) {
1493 set_bit_for_region(hr);
1494 }
1495
1496 return false;
1497 }
1498 };
1499
1500 class G1ParFinalCountTask: public AbstractGangTask {
1501 protected:
1502 G1CollectedHeap* _g1h;
1503 G1ConcurrentMark* _cm;
1504 BitMap* _actual_region_bm;
1505 BitMap* _actual_card_bm;
1506
1507 uint _n_workers;
1508 HeapRegionClaimer _hrclaimer;
1509
1510 public:
1511 G1ParFinalCountTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm)
1512 : AbstractGangTask("G1 final counting"),
1513 _g1h(g1h), _cm(_g1h->concurrent_mark()),
1514 _actual_region_bm(region_bm), _actual_card_bm(card_bm),
1515 _n_workers(_g1h->workers()->active_workers()), _hrclaimer(_n_workers) {
1516 }
1517
1518 void work(uint worker_id) {
1519 assert(worker_id < _n_workers, "invariant");
1520
1521 FinalCountDataUpdateClosure final_update_cl(_g1h,
1522 _actual_region_bm,
1523 _actual_card_bm);
1524
1525 _g1h->heap_region_par_iterate(&final_update_cl, worker_id, &_hrclaimer);
1526 }
1527 };
1528
1529 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1530 G1CollectedHeap* _g1;
1531 size_t _freed_bytes;
1532 FreeRegionList* _local_cleanup_list;
1533 uint _old_regions_removed;
1534 uint _humongous_regions_removed;
1535 HRRSCleanupTask* _hrrs_cleanup_task;
1536
1537 public:
1538 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1539 FreeRegionList* local_cleanup_list,
1540 HRRSCleanupTask* hrrs_cleanup_task) :
1541 _g1(g1),
1542 _freed_bytes(0),
1543 _local_cleanup_list(local_cleanup_list),
1544 _old_regions_removed(0),
1545 _humongous_regions_removed(0),
1639 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1640 return;
1641 }
1642
1643 g1h->verifier()->verify_region_sets_optional();
1644
1645 if (VerifyDuringGC) {
1646 HandleMark hm; // handle scope
1647 g1h->prepare_for_verify();
1648 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)");
1649 }
1650 g1h->verifier()->check_bitmaps("Cleanup Start");
1651
1652 G1CollectorPolicy* g1p = g1h->g1_policy();
1653 g1p->record_concurrent_mark_cleanup_start();
1654
1655 double start = os::elapsedTime();
1656
1657 HeapRegionRemSet::reset_for_cleanup_tasks();
1658
1659 // Do counting once more with the world stopped for good measure.
1660 G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
1661
1662 g1h->workers()->run_task(&g1_par_count_task);
1663
1664 if (VerifyDuringGC) {
1665 // Verify that the counting data accumulated during marking matches
1666 // that calculated by walking the marking bitmap.
1667
1668 // Bitmaps to hold expected values
1669 BitMap expected_region_bm(_region_bm.size(), true);
1670 BitMap expected_card_bm(_card_bm.size(), true);
1671
1672 G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
1673 &_region_bm,
1674 &_card_bm,
1675 &expected_region_bm,
1676 &expected_card_bm);
1677
1678 g1h->workers()->run_task(&g1_par_verify_task);
1679
1680 guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
1681 }
1682
1683 size_t start_used_bytes = g1h->used();
1684 g1h->collector_state()->set_mark_in_progress(false);
1685
1686 double count_end = os::elapsedTime();
1687 double this_final_counting_time = (count_end - start);
1688 _total_counting_time += this_final_counting_time;
1689
1690 if (log_is_enabled(Trace, gc, liveness)) {
1691 G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1692 _g1h->heap_region_iterate(&cl);
1693 }
1694
1695 // Install newly created mark bitMap as "prev".
1696 swapMarkBitMaps();
1697
1698 g1h->reset_gc_time_stamp();
1699
1700 uint n_workers = _g1h->workers()->active_workers();
1701
1702 // Note end of marking in all heap regions.
1703 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1704 g1h->workers()->run_task(&g1_par_note_end_task);
1705 g1h->check_gc_time_stamps();
1706
1707 if (!cleanup_list_is_empty()) {
1708 // The cleanup list is not empty, so we'll have to process it
1709 // concurrently. Notify anyone else that might be wanting free
1710 // regions that there will be more free regions coming soon.
1711 g1h->set_free_regions_coming();
1712 }
1713
1714 // call below, since it affects the metric by which we sort the heap
1715 // regions.
1716 if (G1ScrubRemSets) {
1717 double rs_scrub_start = os::elapsedTime();
1718 g1h->scrub_rem_set(&_region_bm, &_card_bm);
1719 _total_rs_scrub_time += (os::elapsedTime() - rs_scrub_start);
1720 }
1721
1722 // this will also free any regions totally full of garbage objects,
1723 // and sort the regions.
1724 g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1725
1726 // Statistics.
1727 double end = os::elapsedTime();
1728 _cleanup_times.add((end - start) * 1000.0);
1729
1730 // Clean up will have freed any regions completely full of garbage.
1731 // Update the soft reference policy with the new heap occupancy.
1732 Universe::update_heap_info_at_gc();
1733
1734 if (VerifyDuringGC) {
1735 HandleMark hm; // handle scope
1736 g1h->prepare_for_verify();
1737 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (after)");
1738 }
2150
2151 {
2152 GCTraceTime(Trace, gc) trace("Parallel Unloading", g1h->gc_timer_cm());
2153 weakRefsWorkParallelPart(&g1_is_alive, purged_classes);
2154 }
2155 }
2156
2157 if (G1StringDedup::is_enabled()) {
2158 GCTraceTime(Trace, gc) trace("String Deduplication Unlink", g1h->gc_timer_cm());
2159 G1StringDedup::unlink(&g1_is_alive);
2160 }
2161 }
2162 }
2163
2164 void G1ConcurrentMark::swapMarkBitMaps() {
2165 G1CMBitMapRO* temp = _prevMarkBitMap;
2166 _prevMarkBitMap = (G1CMBitMapRO*)_nextMarkBitMap;
2167 _nextMarkBitMap = (G1CMBitMap*) temp;
2168 }
2169
2170 // Closure for marking entries in SATB buffers.
2171 class G1CMSATBBufferClosure : public SATBBufferClosure {
2172 private:
2173 G1CMTask* _task;
2174 G1CollectedHeap* _g1h;
2175
2176 // This is very similar to G1CMTask::deal_with_reference, but with
2177 // more relaxed requirements for the argument, so this must be more
2178 // circumspect about treating the argument as an object.
2179 void do_entry(void* entry) const {
2180 _task->increment_refs_reached();
2181 HeapRegion* hr = _g1h->heap_region_containing(entry);
2182 if (entry < hr->next_top_at_mark_start()) {
2183 // Until we get here, we don't know whether entry refers to a valid
2184 // object; it could instead have been a stale reference.
2185 oop obj = static_cast<oop>(entry);
2186 assert(obj->is_oop(true /* ignore mark word */),
2187 "Invalid oop in SATB buffer: " PTR_FORMAT, p2i(obj));
2188 _task->make_reference_grey(obj, hr);
2189 }
2190 }
2191
2192 public:
2193 G1CMSATBBufferClosure(G1CMTask* task, G1CollectedHeap* g1h)
2194 : _task(task), _g1h(g1h) { }
2195
2196 virtual void do_buffer(void** buffer, size_t size) {
2197 for (size_t i = 0; i < size; ++i) {
2198 do_entry(buffer[i]);
2199 }
2200 }
2201 };
2202
2203 class G1RemarkThreadsClosure : public ThreadClosure {
2204 G1CMSATBBufferClosure _cm_satb_cl;
2205 G1CMOopClosure _cm_cl;
2206 MarkingCodeBlobClosure _code_cl;
2207 int _thread_parity;
2208
2410 p2i(global_finger), HR_FORMAT_PARAMS(global_hr));
2411 }
2412
2413 // Verify the task fingers
2414 assert(parallel_marking_threads() <= _max_worker_id, "sanity");
2415 for (uint i = 0; i < parallel_marking_threads(); ++i) {
2416 G1CMTask* task = _tasks[i];
2417 HeapWord* task_finger = task->finger();
2418 if (task_finger != NULL && task_finger < _heap_end) {
2419 // See above note on the global finger verification.
2420 HeapRegion* task_hr = _g1h->heap_region_containing(task_finger);
2421 guarantee(task_hr == NULL || task_finger == task_hr->bottom() ||
2422 !task_hr->in_collection_set(),
2423 "task finger: " PTR_FORMAT " region: " HR_FORMAT,
2424 p2i(task_finger), HR_FORMAT_PARAMS(task_hr));
2425 }
2426 }
2427 }
2428 #endif // PRODUCT
2429
2430 // Aggregate the counting data that was constructed concurrently
2431 // with marking.
2432 class AggregateCountDataHRClosure: public HeapRegionClosure {
2433 G1CollectedHeap* _g1h;
2434 G1ConcurrentMark* _cm;
2435 CardTableModRefBS* _ct_bs;
2436 BitMap* _cm_card_bm;
2437 uint _max_worker_id;
2438
2439 public:
2440 AggregateCountDataHRClosure(G1CollectedHeap* g1h,
2441 BitMap* cm_card_bm,
2442 uint max_worker_id) :
2443 _g1h(g1h), _cm(g1h->concurrent_mark()),
2444 _ct_bs(barrier_set_cast<CardTableModRefBS>(g1h->barrier_set())),
2445 _cm_card_bm(cm_card_bm), _max_worker_id(max_worker_id) { }
2446
2447 bool doHeapRegion(HeapRegion* hr) {
2448 HeapWord* start = hr->bottom();
2449 HeapWord* limit = hr->next_top_at_mark_start();
2450 HeapWord* end = hr->end();
2451
2452 assert(start <= limit && limit <= hr->top() && hr->top() <= hr->end(),
2453 "Preconditions not met - "
2454 "start: " PTR_FORMAT ", limit: " PTR_FORMAT ", "
2455 "top: " PTR_FORMAT ", end: " PTR_FORMAT,
2456 p2i(start), p2i(limit), p2i(hr->top()), p2i(hr->end()));
2457
2458 assert(hr->next_marked_bytes() == 0, "Precondition");
2459
2460 if (start == limit) {
2461 // NTAMS of this region has not been set so nothing to do.
2462 return false;
2463 }
2464
2465 // 'start' should be in the heap.
2466 assert(_g1h->is_in_g1_reserved(start) && _ct_bs->is_card_aligned(start), "sanity");
2467 // 'end' *may* be just beyond the end of the heap (if hr is the last region)
2468 assert(!_g1h->is_in_g1_reserved(end) || _ct_bs->is_card_aligned(end), "sanity");
2469
2470 BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start);
2471 BitMap::idx_t limit_idx = _cm->card_bitmap_index_for(limit);
2472 BitMap::idx_t end_idx = _cm->card_bitmap_index_for(end);
2473
2474 // If ntams is not card aligned then we bump card bitmap index
2475 // for limit so that we get the all the cards spanned by
2476 // the object ending at ntams.
2477 // Note: if this is the last region in the heap then ntams
2478 // could be actually just beyond the end of the the heap;
2479 // limit_idx will then correspond to a (non-existent) card
2480 // that is also outside the heap.
2481 if (_g1h->is_in_g1_reserved(limit) && !_ct_bs->is_card_aligned(limit)) {
2482 limit_idx += 1;
2483 }
2484
2485 assert(limit_idx <= end_idx, "or else use atomics");
2486
2487 // Aggregate the "stripe" in the count data associated with hr.
2488 uint hrm_index = hr->hrm_index();
2489 size_t marked_bytes = 0;
2490
2491 for (uint i = 0; i < _max_worker_id; i += 1) {
2492 size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i);
2493 BitMap* task_card_bm = _cm->count_card_bitmap_for(i);
2494
2495 // Fetch the marked_bytes in this region for task i and
2496 // add it to the running total for this region.
2497 marked_bytes += marked_bytes_array[hrm_index];
2498
2499 // Now union the bitmaps[0,max_worker_id)[start_idx..limit_idx)
2500 // into the global card bitmap.
2501 BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx);
2502
2503 while (scan_idx < limit_idx) {
2504 assert(task_card_bm->at(scan_idx) == true, "should be");
2505 _cm_card_bm->set_bit(scan_idx);
2506 assert(_cm_card_bm->at(scan_idx) == true, "should be");
2507
2508 // BitMap::get_next_one_offset() can handle the case when
2509 // its left_offset parameter is greater than its right_offset
2510 // parameter. It does, however, have an early exit if
2511 // left_offset == right_offset. So let's limit the value
2512 // passed in for left offset here.
2513 BitMap::idx_t next_idx = MIN2(scan_idx + 1, limit_idx);
2514 scan_idx = task_card_bm->get_next_one_offset(next_idx, limit_idx);
2515 }
2516 }
2517
2518 // Update the marked bytes for this region.
2519 hr->add_to_marked_bytes(marked_bytes);
2520
2521 // Next heap region
2522 return false;
2523 }
2524 };
2525
2526 class G1AggregateCountDataTask: public AbstractGangTask {
2527 protected:
2528 G1CollectedHeap* _g1h;
2529 G1ConcurrentMark* _cm;
2530 BitMap* _cm_card_bm;
2531 uint _max_worker_id;
2532 uint _active_workers;
2533 HeapRegionClaimer _hrclaimer;
2534
2535 public:
2536 G1AggregateCountDataTask(G1CollectedHeap* g1h,
2537 G1ConcurrentMark* cm,
2538 BitMap* cm_card_bm,
2539 uint max_worker_id,
2540 uint n_workers) :
2541 AbstractGangTask("Count Aggregation"),
2542 _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm),
2543 _max_worker_id(max_worker_id),
2544 _active_workers(n_workers),
2545 _hrclaimer(_active_workers) {
2546 }
2547
2548 void work(uint worker_id) {
2549 AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_worker_id);
2550
2551 _g1h->heap_region_par_iterate(&cl, worker_id, &_hrclaimer);
2552 }
2553 };
2554
2555
2556 void G1ConcurrentMark::aggregate_count_data() {
2557 uint n_workers = _g1h->workers()->active_workers();
2558
2559 G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm,
2560 _max_worker_id, n_workers);
2561
2562 _g1h->workers()->run_task(&g1_par_agg_task);
2563 }
2564
2565 // Clear the per-worker arrays used to store the per-region counting data
2566 void G1ConcurrentMark::clear_all_count_data() {
2567 // Clear the global card bitmap - it will be filled during
2568 // liveness count aggregation (during remark) and the
2569 // final counting task.
2570 _card_bm.clear();
2571
2572 // Clear the global region bitmap - it will be filled as part
2573 // of the final counting task.
2574 _region_bm.clear();
2575
2576 uint max_regions = _g1h->max_regions();
2577 assert(_max_worker_id > 0, "uninitialized");
2578
2579 for (uint i = 0; i < _max_worker_id; i += 1) {
2580 BitMap* task_card_bm = count_card_bitmap_for(i);
2581 size_t* marked_bytes_array = count_marked_bytes_array_for(i);
2582
2583 assert(task_card_bm->size() == _card_bm.size(), "size mismatch");
2584 assert(marked_bytes_array != NULL, "uninitialized");
2585
2586 memset(marked_bytes_array, 0, (size_t) max_regions * sizeof(size_t));
2587 task_card_bm->clear();
2588 }
2589 }
2590
2591 void G1ConcurrentMark::print_stats() {
2592 if (!log_is_enabled(Debug, gc, stats)) {
2593 return;
2594 }
2595 log_debug(gc, stats)("---------------------------------------------------------------------");
2596 for (size_t i = 0; i < _active_tasks; ++i) {
2597 _tasks[i]->print_stats();
2598 log_debug(gc, stats)("---------------------------------------------------------------------");
2599 }
2600 }
2601
2602 // abandon current marking iteration due to a Full GC
2603 void G1ConcurrentMark::abort() {
2604 if (!cmThread()->during_cycle() || _has_aborted) {
2605 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
2606 return;
2607 }
2608
2609 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
2610 // concurrent bitmap clearing.
2611 clear_bitmap(_nextMarkBitMap, _g1h->workers(), false);
2612
2613 // Note we cannot clear the previous marking bitmap here
2614 // since VerifyDuringGC verifies the objects marked during
2615 // a full GC against the previous bitmap.
2616
2617 // Clear the liveness counting data
2618 clear_all_count_data();
2619 // Empty mark stack
2620 reset_marking_state();
2621 for (uint i = 0; i < _max_worker_id; ++i) {
2622 _tasks[i]->clear_region_fields();
2623 }
2624 _first_overflow_barrier_sync.abort();
2625 _second_overflow_barrier_sync.abort();
2626 _has_aborted = true;
2627
2628 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2629 satb_mq_set.abandon_partial_marking();
2630 // This can be called either during or outside marking, we'll read
2631 // the expected_active value from the SATB queue set.
2632 satb_mq_set.set_active_all_threads(
2633 false, /* new active value */
2634 satb_mq_set.is_active() /* expected_active */);
2635
2636 _g1h->trace_heap_after_concurrent_cycle();
2637
2638 _g1h->register_concurrent_cycle_end();
2646 log_trace(gc, marking)("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2647 prefix, ns.sd(), ns.maximum());
2648 }
2649 }
2650
2651 void G1ConcurrentMark::print_summary_info() {
2652 LogHandle(gc, marking) log;
2653 if (!log.is_trace()) {
2654 return;
2655 }
2656
2657 log.trace(" Concurrent marking:");
2658 print_ms_time_info(" ", "init marks", _init_times);
2659 print_ms_time_info(" ", "remarks", _remark_times);
2660 {
2661 print_ms_time_info(" ", "final marks", _remark_mark_times);
2662 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2663
2664 }
2665 print_ms_time_info(" ", "cleanups", _cleanup_times);
2666 log.trace(" Final counting total time = %8.2f s (avg = %8.2f ms).",
2667 _total_counting_time, (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2668 if (G1ScrubRemSets) {
2669 log.trace(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
2670 _total_rs_scrub_time, (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2671 }
2672 log.trace(" Total stop_world time = %8.2f s.",
2673 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2674 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2675 cmThread()->vtime_accum(), cmThread()->vtime_mark_accum());
2676 }
2677
2678 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2679 _parallel_workers->print_worker_threads_on(st);
2680 }
2681
2682 void G1ConcurrentMark::print_on_error(outputStream* st) const {
2683 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2684 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2685 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2686 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
3485 }
3486
3487 // We clear the local state of this task...
3488 clear_region_fields();
3489
3490 if (!is_serial) {
3491 // ...and enter the second barrier.
3492 _cm->enter_second_sync_barrier(_worker_id);
3493 }
3494 // At this point, if we're during the concurrent phase of
3495 // marking, everything has been re-initialized and we're
3496 // ready to restart.
3497 }
3498 }
3499
3500 _claimed = false;
3501 }
3502
3503 G1CMTask::G1CMTask(uint worker_id,
3504 G1ConcurrentMark* cm,
3505 size_t* marked_bytes,
3506 BitMap* card_bm,
3507 G1CMTaskQueue* task_queue,
3508 G1CMTaskQueueSet* task_queues)
3509 : _g1h(G1CollectedHeap::heap()),
3510 _worker_id(worker_id), _cm(cm),
3511 _claimed(false),
3512 _nextMarkBitMap(NULL), _hash_seed(17),
3513 _task_queue(task_queue),
3514 _task_queues(task_queues),
3515 _cm_oop_closure(NULL),
3516 _marked_bytes_array(marked_bytes),
3517 _card_bm(card_bm) {
3518 guarantee(task_queue != NULL, "invariant");
3519 guarantee(task_queues != NULL, "invariant");
3520
3521 _marking_step_diffs_ms.add(0.5);
3522 }
3523
3524 // These are formatting macros that are used below to ensure
3525 // consistent formatting. The *_H_* versions are used to format the
3526 // header for a particular value and they should be kept consistent
3527 // with the corresponding macro. Also note that most of the macros add
3528 // the necessary white space (as a prefix) which makes them a bit
3529 // easier to compose.
3530
3531 // All the output lines are prefixed with this string to be able to
3532 // identify them easily in a large log file.
3533 #define G1PPRL_LINE_PREFIX "###"
3534
3535 #define G1PPRL_ADDR_BASE_FORMAT " " PTR_FORMAT "-" PTR_FORMAT
3536 #ifdef _LP64
3537 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
|
31 #include "gc/g1/g1CollectorPolicy.hpp"
32 #include "gc/g1/g1CollectorState.hpp"
33 #include "gc/g1/g1ConcurrentMark.inline.hpp"
34 #include "gc/g1/g1HeapVerifier.hpp"
35 #include "gc/g1/g1OopClosures.inline.hpp"
36 #include "gc/g1/g1StringDedup.hpp"
37 #include "gc/g1/heapRegion.inline.hpp"
38 #include "gc/g1/heapRegionRemSet.hpp"
39 #include "gc/g1/heapRegionSet.inline.hpp"
40 #include "gc/g1/suspendibleThreadSet.hpp"
41 #include "gc/shared/gcId.hpp"
42 #include "gc/shared/gcTimer.hpp"
43 #include "gc/shared/gcTrace.hpp"
44 #include "gc/shared/gcTraceTime.inline.hpp"
45 #include "gc/shared/genOopClosures.inline.hpp"
46 #include "gc/shared/referencePolicy.hpp"
47 #include "gc/shared/strongRootsScope.hpp"
48 #include "gc/shared/taskqueue.inline.hpp"
49 #include "gc/shared/vmGCOperations.hpp"
50 #include "logging/log.hpp"
51 #include "logging/logTag.hpp"
52 #include "memory/allocation.hpp"
53 #include "memory/resourceArea.hpp"
54 #include "oops/oop.inline.hpp"
55 #include "runtime/atomic.inline.hpp"
56 #include "runtime/handles.inline.hpp"
57 #include "runtime/java.hpp"
58 #include "runtime/prefetch.inline.hpp"
59 #include "services/memTracker.hpp"
60
61 // Concurrent marking bit map wrapper
62
63 G1CMBitMapRO::G1CMBitMapRO(int shifter) :
64 _bm(),
65 _shifter(shifter) {
66 _bmStartWord = 0;
67 _bmWordSize = 0;
68 }
69
70 HeapWord* G1CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
71 const HeapWord* limit) const {
339 while (scan_in_progress()) {
340 RootRegionScan_lock->wait(Mutex::_no_safepoint_check_flag);
341 }
342 }
343 return true;
344 }
345
346 uint G1ConcurrentMark::scale_parallel_threads(uint n_par_threads) {
347 return MAX2((n_par_threads + 2) / 4, 1U);
348 }
349
350 G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h, G1RegionToSpaceMapper* prev_bitmap_storage, G1RegionToSpaceMapper* next_bitmap_storage) :
351 _g1h(g1h),
352 _markBitMap1(),
353 _markBitMap2(),
354 _parallel_marking_threads(0),
355 _max_parallel_marking_threads(0),
356 _sleep_factor(0.0),
357 _marking_task_overhead(1.0),
358 _cleanup_list("Cleanup List"),
359 _region_live_bm(),
360 _card_live_bm(),
361
362 _prevMarkBitMap(&_markBitMap1),
363 _nextMarkBitMap(&_markBitMap2),
364
365 _markStack(this),
366 // _finger set in set_non_marking_state
367
368 _max_worker_id(ParallelGCThreads),
369 // _active_tasks set in set_non_marking_state
370 // _tasks set inside the constructor
371 _task_queues(new G1CMTaskQueueSet((int) _max_worker_id)),
372 _terminator(ParallelTaskTerminator((int) _max_worker_id, _task_queues)),
373
374 _has_overflown(false),
375 _concurrent(false),
376 _has_aborted(false),
377 _restart_for_overflow(false),
378 _concurrent_marking_in_progress(false),
379 _concurrent_phase_status(ConcPhaseNotStarted),
380
381 // _verbose_level set below
382
383 _init_times(),
384 _remark_times(), _remark_mark_times(), _remark_weak_ref_times(),
385 _cleanup_times(),
386 _total_counting_time(0.0),
387 _total_rs_scrub_time(0.0),
388
389 _parallel_workers(NULL),
390
391 _completed_initialization(false) {
392
393 _markBitMap1.initialize(g1h->reserved_region(), prev_bitmap_storage);
394 _markBitMap2.initialize(g1h->reserved_region(), next_bitmap_storage);
395
396 // Create & start a ConcurrentMark thread.
397 _cmThread = new ConcurrentMarkThread(this);
398 assert(cmThread() != NULL, "CM Thread should have been created");
399 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
400 if (_cmThread->osthread() == NULL) {
401 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
402 }
403
404 assert(CGC_lock != NULL, "Where's the CGC_lock?");
405 assert(_markBitMap1.covers(g1h->reserved_region()), "_markBitMap1 inconsistency");
406 assert(_markBitMap2.covers(g1h->reserved_region()), "_markBitMap2 inconsistency");
407
408 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
409 satb_qs.set_buffer_size(G1SATBBufferSize);
410
481 "must be between 1 and " SIZE_FORMAT,
482 MarkStackSize, MarkStackSizeMax);
483 return;
484 }
485 } else if (FLAG_IS_CMDLINE(MarkStackSizeMax)) {
486 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
487 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT ")"
488 " or for MarkStackSizeMax (" SIZE_FORMAT ")",
489 MarkStackSize, MarkStackSizeMax);
490 return;
491 }
492 }
493 }
494 }
495
496 if (!_markStack.allocate(MarkStackSize)) {
497 log_warning(gc)("Failed to allocate CM marking stack");
498 return;
499 }
500
501 allocate_internal_bitmaps();
502
503 if (G1PretouchAuxiliaryMemory) {
504 pretouch_internal_bitmaps();
505 }
506
507 _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_worker_id, mtGC);
508 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_worker_id, mtGC);
509
510 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
511 _active_tasks = _max_worker_id;
512
513 for (uint i = 0; i < _max_worker_id; ++i) {
514 G1CMTaskQueue* task_queue = new G1CMTaskQueue();
515 task_queue->initialize();
516 _task_queues->register_queue(i, task_queue);
517
518 _tasks[i] = new G1CMTask(i, this, task_queue, _task_queues);
519
520 _accum_task_vtime[i] = 0.0;
521 }
522
523 // Calculate the card number for the bottom of the heap. Used
524 // in biasing indexes into the accounting card bitmaps.
525 _heap_bottom_card_num =
526 intptr_t(uintptr_t(_g1h->reserved_region().start()) >>
527 CardTableModRefBS::card_shift);
528
529 // so that the call below can read a sensible value
530 _heap_start = g1h->reserved_region().start();
531 set_non_marking_state();
532 _completed_initialization = true;
533 }
534
535 void G1ConcurrentMark::reset() {
536 // Starting values for these two. This should be called in a STW
537 // phase.
538 MemRegion reserved = _g1h->g1_reserved();
539 _heap_start = reserved.start();
540 _heap_end = reserved.end();
541
542 // Separated the asserts so that we know which one fires.
543 assert(_heap_start != NULL, "heap bounds should look ok");
544 assert(_heap_end != NULL, "heap bounds should look ok");
545 assert(_heap_start < _heap_end, "heap bounds should look ok");
546
547 // Reset all the marking data structures and any necessary flags
548 reset_marking_state();
686 assert(may_yield || SafepointSynchronize::is_at_safepoint(), "Non-yielding bitmap clear only allowed at safepoint.");
687
688 G1ClearBitMapTask task(bitmap, this, workers->active_workers(), may_yield);
689 workers->run_task(&task);
690 guarantee(!may_yield || task.is_complete(), "Must have completed iteration when not yielding.");
691 }
692
693 void G1ConcurrentMark::cleanup_for_next_mark() {
694 // Make sure that the concurrent mark thread looks to still be in
695 // the current cycle.
696 guarantee(cmThread()->during_cycle(), "invariant");
697
698 // We are finishing up the current cycle by clearing the next
699 // marking bitmap and getting it ready for the next cycle. During
700 // this time no other cycle can start. So, let's make sure that this
701 // is the case.
702 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
703
704 clear_bitmap(_nextMarkBitMap, _parallel_workers, true);
705
706 // Clear the live count data. If the marking has been aborted, the abort()
707 // call already did that.
708 if (!has_aborted()) {
709 clear_all_live_data(_parallel_workers);
710 DEBUG_ONLY(verify_all_live_data());
711 }
712
713 // Repeat the asserts from above.
714 guarantee(cmThread()->during_cycle(), "invariant");
715 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
716 }
717
718 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) {
719 assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint.");
720 clear_bitmap((G1CMBitMap*)_prevMarkBitMap, workers, false);
721 }
722
723 class CheckBitmapClearHRClosure : public HeapRegionClosure {
724 G1CMBitMap* _bitmap;
725 bool _error;
726 public:
727 CheckBitmapClearHRClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) {
728 }
729
730 virtual bool doHeapRegion(HeapRegion* r) {
1098
1099 double mark_work_end = os::elapsedTime();
1100
1101 weakRefsWork(clear_all_soft_refs);
1102
1103 if (has_overflown()) {
1104 // Oops. We overflowed. Restart concurrent marking.
1105 _restart_for_overflow = true;
1106
1107 // Verify the heap w.r.t. the previous marking bitmap.
1108 if (VerifyDuringGC) {
1109 HandleMark hm; // handle scope
1110 g1h->prepare_for_verify();
1111 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (overflow)");
1112 }
1113
1114 // Clear the marking state because we will be restarting
1115 // marking due to overflowing the global mark stack.
1116 reset_marking_state();
1117 } else {
1118 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1119 // We're done with marking.
1120 // This is the end of the marking cycle, we're expected all
1121 // threads to have SATB queues with active set to true.
1122 satb_mq_set.set_active_all_threads(false, /* new active value */
1123 true /* expected_active */);
1124
1125 if (VerifyDuringGC) {
1126 HandleMark hm; // handle scope
1127 g1h->prepare_for_verify();
1128 Universe::verify(VerifyOption_G1UseNextMarking, "During GC (after)");
1129 }
1130 g1h->verifier()->check_bitmaps("Remark End");
1131 assert(!restart_for_overflow(), "sanity");
1132 // Completely reset the marking state since marking completed
1133 set_non_marking_state();
1134 }
1135
1136 // Expand the marking stack, if we have to and if we can.
1137 if (_markStack.should_expand()) {
1138 _markStack.expand();
1139 }
1140
1141 // Statistics
1142 double now = os::elapsedTime();
1143 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1144 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1145 _remark_times.add((now - start) * 1000.0);
1146
1147 g1p->record_concurrent_mark_remark_end();
1148
1149 G1CMIsAliveClosure is_alive(g1h);
1150 g1h->gc_tracer_cm()->report_object_count_after_gc(&is_alive);
1151 }
1152
1153 // Base class of the closures that finalize and verify the
1154 // liveness count data.
1155 class G1LiveDataClosureBase: public HeapRegionClosure {
1156 protected:
1157 G1ConcurrentMark* _cm;
1158
1159 BitMap* _region_bm;
1160 BitMap* _card_bm;
1161
1162 // Takes a region that's not empty (i.e., it has at least one
1163 // live object in it and sets its corresponding bit on the region
1164 // bitmap to 1.
1165 void set_bit_for_region(HeapRegion* hr) {
1166 BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index();
1167 _region_bm->par_at_put(index, true);
1168 }
1169
1170 // Utility routine to set an exclusive range of cards on the given
1171 // bitmap.
1172 inline void set_card_bitmap_range(BitMap* card_bm,
1173 BitMap::idx_t start_idx,
1174 BitMap::idx_t end_idx) {
1175
1176 // Set the exclusive bit range [start_idx, end_idx).
1177 assert((end_idx - start_idx) > 0, "at least one card");
1178 assert(end_idx <= card_bm->size(), "sanity");
1179
1180 // For small ranges use a simple loop; otherwise use set_range or
1181 // use par_at_put_range (if parallel). The range is made up of the
1182 // cards that are spanned by an object/mem region so 8 cards will
1183 // allow up to object sizes up to 4K to be handled using the loop.
1184 if ((end_idx - start_idx) <= 8) {
1185 for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) {
1186 card_bm->set_bit(i);
1187 }
1188 } else {
1189 card_bm->set_range(start_idx, end_idx);
1190 }
1191 }
1192
1193 void mark_card_bitmap_range(HeapWord* start, HeapWord* end) {
1194 BitMap::idx_t start_idx = _cm->card_live_bitmap_index_for(start);
1195 BitMap::idx_t end_idx = _cm->card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size));
1196
1197 assert((end_idx - start_idx) > 0, "Trying to mark zero sized range.");
1198
1199 if (start_idx == _last_marked_bit_idx) {
1200 start_idx++;
1201 }
1202 if (start_idx == end_idx) {
1203 return;
1204 }
1205
1206 // Set the bits in the card bitmap for the cards spanned by this object.
1207 set_card_bitmap_range(_card_bm, start_idx, end_idx);
1208 _last_marked_bit_idx = end_idx - 1;
1209 }
1210
1211 // We cache the last mark set. This avoids setting the same bit multiple times.
1212 // This is particularly interesting for dense bitmaps, as this avoids doing
1213 // any work most of the time.
1214 BitMap::idx_t _last_marked_bit_idx;
1215
1216 void reset_mark_cache() {
1217 _last_marked_bit_idx = (BitMap::idx_t)-1;
1218 }
1219
1220 void mark_allocated_since_marking(HeapRegion* hr) {
1221 reset_mark_cache();
1222
1223 HeapWord* ntams = hr->next_top_at_mark_start();
1224 HeapWord* top = hr->top();
1225
1226 assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions.");
1227
1228 // Mark the allocated-since-marking portion...
1229 if (ntams < top) {
1230 mark_card_bitmap_range(ntams, top);
1231 // This definitely means the region has live objects.
1232 set_bit_for_region(hr);
1233 }
1234 }
1235
1236 bool mark_marked_during_marking(HeapRegion* hr, bool may_suspend, size_t* total_bytes_marked) {
1237 reset_mark_cache();
1238
1239 size_t marked_bytes = 0;
1240
1241 HeapWord* ntams = hr->next_top_at_mark_start();
1242 HeapWord* start = hr->bottom();
1243
1244 if (ntams <= start) {
1245 // Empty region (at the start of marking). Nothing to do.
1246 // hr->add_to_marked_bytes(0);
1247 *total_bytes_marked = marked_bytes;
1248 return false;
1249 } else if (hr->is_starts_humongous()) {
1250 // Humongous object: distribute the marked bytes across the humongous object.
1251 do {
1252 mark_card_bitmap_range(start, hr->top());
1253
1254 marked_bytes += pointer_delta(hr->top(), start, 1);
1255 hr->add_to_marked_bytes(marked_bytes);
1256
1257 hr = G1CollectedHeap::heap()->next_region_in_humongous(hr);
1258 } while (hr != NULL);
1259 *total_bytes_marked = marked_bytes;
1260 return false;
1261 } else if (hr->is_continues_humongous()) {
1262 // Humongous continues regions were handled during processing of the start region.
1263 *total_bytes_marked = marked_bytes;
1264 return false;
1265 }
1266
1267 assert(start <= hr->end() && start <= ntams && ntams <= hr->end(),
1268 "Preconditions not met - "
1269 "start: " PTR_FORMAT ", ntams: " PTR_FORMAT ", end: " PTR_FORMAT,
1270 p2i(start), p2i(ntams), p2i(hr->end()));
1271
1272 G1CMBitMap* bitmap = _cm->nextMarkBitMap();
1273 // Find the first marked object at or after "start".
1274 start = bitmap->getNextMarkedWordAddress(start, ntams);
1275 while (start < ntams) {
1276 oop obj = oop(start);
1277 int obj_sz = obj->size();
1278 HeapWord* obj_end = start + obj_sz;
1279
1280 assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere.");
1281
1282 mark_card_bitmap_range(start, obj_end);
1283
1284 // Add the size of this object to the number of marked bytes.
1285 marked_bytes += (size_t)obj_sz * HeapWordSize;
1286
1287 // Find the next marked object after this one.
1288 start = bitmap->getNextMarkedWordAddress(obj_end, ntams);
1289 }
1290
1291 // Update the marked bytes for this region.
1292 hr->add_to_marked_bytes(marked_bytes);
1293 *total_bytes_marked = marked_bytes;
1294
1295 // Abort iteration if after yielding the marking has been aborted.
1296 if (may_suspend && _cm->do_yield_check() && _cm->has_aborted()) {
1297 return true;
1298 }
1299 // Next heap region
1300 return false;
1301 }
1302
1303 public:
1304 G1LiveDataClosureBase(G1CollectedHeap* g1h,
1305 BitMap* region_bm,
1306 BitMap* card_bm):
1307 _cm(g1h->concurrent_mark()),
1308 _region_bm(region_bm),
1309 _card_bm(card_bm) { }
1310 };
1311
1312 // Heap region closure used for verifying the live count data
1313 // that was created concurrently and finalized during
1314 // the remark pause. This closure is applied to the heap
1315 // regions during the STW cleanup pause.
1316 class G1VerifyLiveDataHRClosure: public HeapRegionClosure {
1317 // Calculates the # live objects per region.
1318 class G1VerifyLiveDataClosure: public G1LiveDataClosureBase {
1319 size_t _region_marked_bytes;
1320
1321 public:
1322 G1VerifyLiveDataClosure(G1CollectedHeap* g1h,
1323 BitMap* region_bm,
1324 BitMap* card_bm) :
1325 G1LiveDataClosureBase(g1h, region_bm, card_bm),
1326 _region_marked_bytes(0) { }
1327
1328 bool doHeapRegion(HeapRegion* hr) {
1329 mark_marked_during_marking(hr, false, &_region_marked_bytes);
1330 mark_allocated_since_marking(hr);
1331 return false;
1332 }
1333
1334 size_t region_marked_bytes() const { return _region_marked_bytes; }
1335 };
1336
1337 G1CollectedHeap* _g1h;
1338 G1ConcurrentMark* _cm;
1339 G1VerifyLiveDataClosure _calc_cl;
1340 BitMap* _act_region_bm; // Region BM to be verified
1341 BitMap* _act_card_bm; // Card BM to be verified
1342
1343 BitMap* _exp_region_bm; // Expected Region BM values
1344 BitMap* _exp_card_bm; // Expected card BM values
1345
1346 int _failures;
1347
1348 public:
1349 G1VerifyLiveDataHRClosure(G1CollectedHeap* g1h,
1350 BitMap* act_region_bm,
1351 BitMap* act_card_bm,
1352 BitMap* exp_region_bm,
1353 BitMap* exp_card_bm) :
1354 _g1h(g1h),
1355 _cm(g1h->concurrent_mark()),
1356 _calc_cl(g1h, exp_region_bm, exp_card_bm),
1357 _act_region_bm(act_region_bm),
1358 _act_card_bm(act_card_bm),
1359 _exp_region_bm(exp_region_bm),
1360 _exp_card_bm(exp_card_bm),
1361 _failures(0) { }
1362
1363 int failures() const { return _failures; }
1364
1365 bool doHeapRegion(HeapRegion* hr) {
1366 int failures = 0;
1367
1368 // Call the G1VerifyLiveDataClosure to walk the marking bitmap for
1369 // this region and set the corresponding bits in the expected region
1370 // and card bitmaps.
1371 bool res = _calc_cl.doHeapRegion(hr);
1372 assert(!res, "Should be completed.");
1373
1374 // Verify the marked bytes for this region.
1375 size_t exp_marked_bytes = _calc_cl.region_marked_bytes();
1376 size_t act_marked_bytes = hr->next_marked_bytes();
1377
1378 if (exp_marked_bytes > act_marked_bytes) {
1379 if (hr->is_starts_humongous()) {
1380 // For start_humongous regions, the size of the whole object will be
1381 // in exp_marked_bytes.
1382 HeapRegion* region = hr;
1383 int num_regions;
1384 for (num_regions = 0; region != NULL; num_regions++) {
1385 region = _g1h->next_region_in_humongous(region);
1386 }
1387 if ((num_regions-1) * HeapRegion::GrainBytes >= exp_marked_bytes) {
1388 failures += 1;
1389 } else if (num_regions * HeapRegion::GrainBytes < exp_marked_bytes) {
1390 failures += 1;
1391 }
1392 } else {
1393 // We're not OK if expected marked bytes > actual marked bytes. It means
1394 // we have missed accounting some objects during the actual marking.
1395 failures += 1;
1396 }
1397 }
1398
1399 // Verify the bit, for this region, in the actual and expected
1400 // (which was just calculated) region bit maps.
1401 // We're not OK if the bit in the calculated expected region
1402 // bitmap is set and the bit in the actual region bitmap is not.
1403 BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index();
1404
1405 bool expected = _exp_region_bm->at(index);
1406 bool actual = _act_region_bm->at(index);
1407 if (expected && !actual) {
1408 failures += 1;
1409 }
1410
1411 // Verify that the card bit maps for the cards spanned by the current
1412 // region match. We have an error if we have a set bit in the expected
1413 // bit map and the corresponding bit in the actual bitmap is not set.
1414
1415 BitMap::idx_t start_idx = _cm->card_live_bitmap_index_for(hr->bottom());
1416 BitMap::idx_t end_idx = _cm->card_live_bitmap_index_for(hr->top());
1417
1418 for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) {
1419 expected = _exp_card_bm->at(i);
1420 actual = _act_card_bm->at(i);
1421
1422 if (expected && !actual) {
1423 failures += 1;
1424 }
1425 }
1426
1427 _failures += failures;
1428
1429 // We could stop iteration over the heap when we
1430 // find the first violating region by returning true.
1431 return false;
1432 }
1433 };
1434
1435 class G1VerifyLiveDataTask: public AbstractGangTask {
1436 protected:
1437 G1CollectedHeap* _g1h;
1438 BitMap* _actual_region_bm;
1439 BitMap* _actual_card_bm;
1440
1441 BitMap _expected_region_bm;
1442 BitMap _expected_card_bm;
1443
1444 int _failures;
1445
1446 HeapRegionClaimer _hr_claimer;
1447
1448 public:
1449 G1VerifyLiveDataTask(G1CollectedHeap* g1h,
1450 BitMap* region_bm,
1451 BitMap* card_bm,
1452 uint n_workers)
1453 : AbstractGangTask("G1 verify final counting"),
1454 _g1h(g1h),
1455 _actual_region_bm(region_bm),
1456 _actual_card_bm(card_bm),
1457 _expected_region_bm(region_bm->size(), true /* in_resource_area */),
1458 _expected_card_bm(card_bm->size(), true /* in_resource_area */),
1459 _failures(0),
1460 _hr_claimer(n_workers) {
1461 assert(VerifyDuringGC, "don't call this otherwise");
1462 }
1463
1464 void work(uint worker_id) {
1465 G1VerifyLiveDataHRClosure cl(_g1h,
1466 _actual_region_bm,
1467 _actual_card_bm,
1468 &_expected_region_bm,
1469 &_expected_card_bm);
1470 _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
1471
1472 Atomic::add(cl.failures(), &_failures);
1473 }
1474
1475 int failures() const { return _failures; }
1476 };
1477
1478 class G1FinalizeLiveDataTask: public AbstractGangTask {
1479 // Finalizes the liveness counting data.
1480 // Sets the bits corresponding to the interval [NTAMS, top]
1481 // (which contains the implicitly live objects) in the
1482 // card liveness bitmap. Also sets the bit for each region
1483 // containing live data, in the region liveness bitmap.
1484 class G1FinalizeCountDataClosure: public G1LiveDataClosureBase {
1485 public:
1486 G1FinalizeCountDataClosure(G1CollectedHeap* g1h,
1487 BitMap* region_bm,
1488 BitMap* card_bm) :
1489 G1LiveDataClosureBase(g1h, region_bm, card_bm) { }
1490
1491 bool doHeapRegion(HeapRegion* hr) {
1492 mark_allocated_since_marking(hr);
1493 // Set the bit for the region if it contains live data
1494 if (hr->next_marked_bytes() > 0) {
1495 set_bit_for_region(hr);
1496 }
1497
1498 return false;
1499 }
1500 };
1501
1502 G1CollectedHeap* _g1h;
1503 BitMap* _actual_region_bm;
1504 BitMap* _actual_card_bm;
1505
1506 HeapRegionClaimer _hr_claimer;
1507
1508 public:
1509 G1FinalizeLiveDataTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm, uint n_workers) :
1510 AbstractGangTask("G1 final counting"),
1511 _g1h(g1h),
1512 _actual_region_bm(region_bm),
1513 _actual_card_bm(card_bm),
1514 _hr_claimer(n_workers) {
1515 }
1516
1517 void work(uint worker_id) {
1518 G1FinalizeCountDataClosure cl(_g1h,
1519 _actual_region_bm,
1520 _actual_card_bm);
1521
1522 _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
1523 }
1524 };
1525
1526 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1527 G1CollectedHeap* _g1;
1528 size_t _freed_bytes;
1529 FreeRegionList* _local_cleanup_list;
1530 uint _old_regions_removed;
1531 uint _humongous_regions_removed;
1532 HRRSCleanupTask* _hrrs_cleanup_task;
1533
1534 public:
1535 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1536 FreeRegionList* local_cleanup_list,
1537 HRRSCleanupTask* hrrs_cleanup_task) :
1538 _g1(g1),
1539 _freed_bytes(0),
1540 _local_cleanup_list(local_cleanup_list),
1541 _old_regions_removed(0),
1542 _humongous_regions_removed(0),
1636 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1637 return;
1638 }
1639
1640 g1h->verifier()->verify_region_sets_optional();
1641
1642 if (VerifyDuringGC) {
1643 HandleMark hm; // handle scope
1644 g1h->prepare_for_verify();
1645 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)");
1646 }
1647 g1h->verifier()->check_bitmaps("Cleanup Start");
1648
1649 G1CollectorPolicy* g1p = g1h->g1_policy();
1650 g1p->record_concurrent_mark_cleanup_start();
1651
1652 double start = os::elapsedTime();
1653
1654 HeapRegionRemSet::reset_for_cleanup_tasks();
1655
1656 {
1657 // Finalize the live data.
1658 G1FinalizeLiveDataTask cl(g1h,
1659 &_region_live_bm,
1660 &_card_live_bm,
1661 g1h->workers()->active_workers());
1662 g1h->workers()->run_task(&cl);
1663 }
1664
1665 if (VerifyDuringGC) {
1666 // Verify that the liveness count data created concurrently matches one created
1667 // during this safepoint.
1668 ResourceMark rm;
1669 G1VerifyLiveDataTask cl(g1h,
1670 &_region_live_bm,
1671 &_card_live_bm,
1672 g1h->workers()->active_workers());
1673 g1h->workers()->run_task(&cl);
1674
1675 guarantee(cl.failures() == 0, "Unexpected accounting failures");
1676 }
1677
1678 g1h->collector_state()->set_mark_in_progress(false);
1679
1680 double count_end = os::elapsedTime();
1681 double this_final_counting_time = (count_end - start);
1682 _total_counting_time += this_final_counting_time;
1683
1684 if (log_is_enabled(Trace, gc, liveness)) {
1685 G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1686 _g1h->heap_region_iterate(&cl);
1687 }
1688
1689 // Install newly created mark bitMap as "prev".
1690 swapMarkBitMaps();
1691
1692 g1h->reset_gc_time_stamp();
1693
1694 uint n_workers = _g1h->workers()->active_workers();
1695
1696 // Note end of marking in all heap regions.
1697 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1698 g1h->workers()->run_task(&g1_par_note_end_task);
1699 g1h->check_gc_time_stamps();
1700
1701 if (!cleanup_list_is_empty()) {
1702 // The cleanup list is not empty, so we'll have to process it
1703 // concurrently. Notify anyone else that might be wanting free
1704 // regions that there will be more free regions coming soon.
1705 g1h->set_free_regions_coming();
1706 }
1707
1708 // call below, since it affects the metric by which we sort the heap
1709 // regions.
1710 if (G1ScrubRemSets) {
1711 double rs_scrub_start = os::elapsedTime();
1712 g1h->scrub_rem_set(&_region_live_bm, &_card_live_bm);
1713 _total_rs_scrub_time += (os::elapsedTime() - rs_scrub_start);
1714 }
1715
1716 // this will also free any regions totally full of garbage objects,
1717 // and sort the regions.
1718 g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1719
1720 // Statistics.
1721 double end = os::elapsedTime();
1722 _cleanup_times.add((end - start) * 1000.0);
1723
1724 // Clean up will have freed any regions completely full of garbage.
1725 // Update the soft reference policy with the new heap occupancy.
1726 Universe::update_heap_info_at_gc();
1727
1728 if (VerifyDuringGC) {
1729 HandleMark hm; // handle scope
1730 g1h->prepare_for_verify();
1731 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (after)");
1732 }
2144
2145 {
2146 GCTraceTime(Trace, gc) trace("Parallel Unloading", g1h->gc_timer_cm());
2147 weakRefsWorkParallelPart(&g1_is_alive, purged_classes);
2148 }
2149 }
2150
2151 if (G1StringDedup::is_enabled()) {
2152 GCTraceTime(Trace, gc) trace("String Deduplication Unlink", g1h->gc_timer_cm());
2153 G1StringDedup::unlink(&g1_is_alive);
2154 }
2155 }
2156 }
2157
2158 void G1ConcurrentMark::swapMarkBitMaps() {
2159 G1CMBitMapRO* temp = _prevMarkBitMap;
2160 _prevMarkBitMap = (G1CMBitMapRO*)_nextMarkBitMap;
2161 _nextMarkBitMap = (G1CMBitMap*) temp;
2162 }
2163
2164 BitMap G1ConcurrentMark::allocate_large_bitmap(BitMap::idx_t size_in_bits) {
2165 size_t size_in_words = BitMap::size_in_words(size_in_bits);
2166
2167 BitMap::bm_word_t* map = MmapArrayAllocator<BitMap::bm_word_t, mtGC>::allocate(size_in_words);
2168
2169 return BitMap(map, size_in_bits);
2170 }
2171
2172 void G1ConcurrentMark::allocate_internal_bitmaps() {
2173 double start_time = os::elapsedTime();
2174
2175 _region_live_bm = allocate_large_bitmap(_g1h->max_regions());
2176
2177 guarantee(_g1h->max_capacity() % CardTableModRefBS::card_size == 0,
2178 "Heap capacity must be aligned to card size.");
2179 _card_live_bm = allocate_large_bitmap(_g1h->max_capacity() / CardTableModRefBS::card_size);
2180
2181 log_debug(gc, marking)("Allocating internal bitmaps took %1.2f seconds.", os::elapsedTime() - start_time);
2182 }
2183
2184 void G1ConcurrentMark::pretouch_internal_bitmaps() {
2185 double start_time = os::elapsedTime();
2186
2187 _region_live_bm.pretouch();
2188 _card_live_bm.pretouch();
2189
2190 log_debug(gc, marking)("Pre-touching internal bitmaps took %1.2f seconds.", os::elapsedTime() - start_time);
2191 }
2192
2193 // Closure for marking entries in SATB buffers.
2194 class G1CMSATBBufferClosure : public SATBBufferClosure {
2195 private:
2196 G1CMTask* _task;
2197 G1CollectedHeap* _g1h;
2198
2199 // This is very similar to G1CMTask::deal_with_reference, but with
2200 // more relaxed requirements for the argument, so this must be more
2201 // circumspect about treating the argument as an object.
2202 void do_entry(void* entry) const {
2203 _task->increment_refs_reached();
2204 HeapRegion* hr = _g1h->heap_region_containing(entry);
2205 if (entry < hr->next_top_at_mark_start()) {
2206 // Until we get here, we don't know whether entry refers to a valid
2207 // object; it could instead have been a stale reference.
2208 oop obj = static_cast<oop>(entry);
2209 assert(obj->is_oop(true /* ignore mark word */),
2210 "Invalid oop in SATB buffer: " PTR_FORMAT, p2i(obj));
2211 _task->make_reference_grey(obj);
2212 }
2213 }
2214
2215 public:
2216 G1CMSATBBufferClosure(G1CMTask* task, G1CollectedHeap* g1h)
2217 : _task(task), _g1h(g1h) { }
2218
2219 virtual void do_buffer(void** buffer, size_t size) {
2220 for (size_t i = 0; i < size; ++i) {
2221 do_entry(buffer[i]);
2222 }
2223 }
2224 };
2225
2226 class G1RemarkThreadsClosure : public ThreadClosure {
2227 G1CMSATBBufferClosure _cm_satb_cl;
2228 G1CMOopClosure _cm_cl;
2229 MarkingCodeBlobClosure _code_cl;
2230 int _thread_parity;
2231
2433 p2i(global_finger), HR_FORMAT_PARAMS(global_hr));
2434 }
2435
2436 // Verify the task fingers
2437 assert(parallel_marking_threads() <= _max_worker_id, "sanity");
2438 for (uint i = 0; i < parallel_marking_threads(); ++i) {
2439 G1CMTask* task = _tasks[i];
2440 HeapWord* task_finger = task->finger();
2441 if (task_finger != NULL && task_finger < _heap_end) {
2442 // See above note on the global finger verification.
2443 HeapRegion* task_hr = _g1h->heap_region_containing(task_finger);
2444 guarantee(task_hr == NULL || task_finger == task_hr->bottom() ||
2445 !task_hr->in_collection_set(),
2446 "task finger: " PTR_FORMAT " region: " HR_FORMAT,
2447 p2i(task_finger), HR_FORMAT_PARAMS(task_hr));
2448 }
2449 }
2450 }
2451 #endif // PRODUCT
2452
2453 class G1CreateLiveDataTask: public AbstractGangTask {
2454 // Aggregate the counting data that was constructed concurrently
2455 // with marking.
2456 class G1CreateLiveDataHRClosure: public G1LiveDataClosureBase {
2457 public:
2458 G1CreateLiveDataHRClosure(G1CollectedHeap* g1h,
2459 BitMap* cm_card_bm)
2460 : G1LiveDataClosureBase(g1h, NULL, cm_card_bm) { }
2461
2462 bool doHeapRegion(HeapRegion* hr) {
2463 size_t temp;
2464 return mark_marked_during_marking(hr, true, &temp);
2465 }
2466 };
2467
2468 G1CollectedHeap* _g1h;
2469 G1ConcurrentMark* _cm;
2470 BitMap* _cm_card_bm;
2471 HeapRegionClaimer _hrclaimer;
2472
2473 public:
2474 G1CreateLiveDataTask(G1CollectedHeap* g1h,
2475 BitMap* cm_card_bm,
2476 uint n_workers) :
2477 AbstractGangTask("Create Live Data"),
2478 _g1h(g1h),
2479 _cm_card_bm(cm_card_bm),
2480 _hrclaimer(n_workers) {
2481 }
2482
2483 void work(uint worker_id) {
2484 SuspendibleThreadSetJoiner sts_join;
2485
2486 G1CreateLiveDataHRClosure cl(_g1h, _cm_card_bm);
2487 _g1h->heap_region_par_iterate(&cl, worker_id, &_hrclaimer);
2488 }
2489 };
2490
2491
2492 void G1ConcurrentMark::create_live_data() {
2493 uint n_workers = _parallel_workers->active_workers();
2494
2495 G1CreateLiveDataTask cl(_g1h,
2496 &_card_live_bm,
2497 n_workers);
2498 _parallel_workers->run_task(&cl);
2499 }
2500
2501 class G1ClearAllLiveDataTask : public AbstractGangTask {
2502 BitMap* _bitmap;
2503 size_t _num_tasks;
2504 size_t _cur_task;
2505 public:
2506 G1ClearAllLiveDataTask(BitMap* bitmap, size_t num_tasks) :
2507 AbstractGangTask("Clear All Live Data"),
2508 _bitmap(bitmap),
2509 _num_tasks(num_tasks),
2510 _cur_task(0) {
2511 }
2512
2513 virtual void work(uint worker_id) {
2514 while (true) {
2515 size_t to_process = Atomic::add(1, &_cur_task) - 1;
2516 if (to_process >= _num_tasks) {
2517 break;
2518 }
2519
2520 BitMap::idx_t start = M * BitsPerByte * to_process;
2521 BitMap::idx_t end = MIN2(start + M * BitsPerByte, _bitmap->size());
2522 _bitmap->clear_range(start, end);
2523 }
2524 }
2525 };
2526
2527 void G1ConcurrentMark::clear_all_live_data(WorkGang* workers) {
2528 double start_time = os::elapsedTime();
2529
2530 guarantee(Universe::is_fully_initialized(), "Should not call this during initialization.");
2531
2532 size_t const num_chunks = align_size_up(_card_live_bm.size_in_words() * HeapWordSize, M) / M;
2533
2534 G1ClearAllLiveDataTask cl(&_card_live_bm, num_chunks);
2535 workers->run_task(&cl);
2536
2537 // The region live bitmap is always very small, even for huge heaps. Clear
2538 // directly.
2539 _region_live_bm.clear();
2540
2541
2542 log_debug(gc, marking)("Clear Live Data took %.3fms", (os::elapsedTime() - start_time) * 1000.0);
2543 }
2544
2545 void G1ConcurrentMark::verify_all_live_data() {
2546 assert(_card_live_bm.count_one_bits() == 0, "Master card bitmap not clear");
2547 assert(_region_live_bm.count_one_bits() == 0, "Master region bitmap not clear");
2548 }
2549
2550 void G1ConcurrentMark::print_stats() {
2551 if (!log_is_enabled(Debug, gc, stats)) {
2552 return;
2553 }
2554 log_debug(gc, stats)("---------------------------------------------------------------------");
2555 for (size_t i = 0; i < _active_tasks; ++i) {
2556 _tasks[i]->print_stats();
2557 log_debug(gc, stats)("---------------------------------------------------------------------");
2558 }
2559 }
2560
2561 void G1ConcurrentMark::abort() {
2562 if (!cmThread()->during_cycle() || _has_aborted) {
2563 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
2564 return;
2565 }
2566
2567 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
2568 // concurrent bitmap clearing.
2569 clear_bitmap(_nextMarkBitMap, _g1h->workers(), false);
2570
2571 // Note we cannot clear the previous marking bitmap here
2572 // since VerifyDuringGC verifies the objects marked during
2573 // a full GC against the previous bitmap.
2574
2575 clear_all_live_data(_g1h->workers());
2576 DEBUG_ONLY(verify_all_live_data());
2577 // Empty mark stack
2578 reset_marking_state();
2579 for (uint i = 0; i < _max_worker_id; ++i) {
2580 _tasks[i]->clear_region_fields();
2581 }
2582 _first_overflow_barrier_sync.abort();
2583 _second_overflow_barrier_sync.abort();
2584 _has_aborted = true;
2585
2586 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2587 satb_mq_set.abandon_partial_marking();
2588 // This can be called either during or outside marking, we'll read
2589 // the expected_active value from the SATB queue set.
2590 satb_mq_set.set_active_all_threads(
2591 false, /* new active value */
2592 satb_mq_set.is_active() /* expected_active */);
2593
2594 _g1h->trace_heap_after_concurrent_cycle();
2595
2596 _g1h->register_concurrent_cycle_end();
2604 log_trace(gc, marking)("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2605 prefix, ns.sd(), ns.maximum());
2606 }
2607 }
2608
2609 void G1ConcurrentMark::print_summary_info() {
2610 LogHandle(gc, marking) log;
2611 if (!log.is_trace()) {
2612 return;
2613 }
2614
2615 log.trace(" Concurrent marking:");
2616 print_ms_time_info(" ", "init marks", _init_times);
2617 print_ms_time_info(" ", "remarks", _remark_times);
2618 {
2619 print_ms_time_info(" ", "final marks", _remark_mark_times);
2620 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2621
2622 }
2623 print_ms_time_info(" ", "cleanups", _cleanup_times);
2624 log.trace(" Finalize live data total time = %8.2f s (avg = %8.2f ms).",
2625 _total_counting_time, (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2626 if (G1ScrubRemSets) {
2627 log.trace(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
2628 _total_rs_scrub_time, (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2629 }
2630 log.trace(" Total stop_world time = %8.2f s.",
2631 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2632 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2633 cmThread()->vtime_accum(), cmThread()->vtime_mark_accum());
2634 }
2635
2636 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2637 _parallel_workers->print_worker_threads_on(st);
2638 }
2639
2640 void G1ConcurrentMark::print_on_error(outputStream* st) const {
2641 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2642 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2643 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2644 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
3443 }
3444
3445 // We clear the local state of this task...
3446 clear_region_fields();
3447
3448 if (!is_serial) {
3449 // ...and enter the second barrier.
3450 _cm->enter_second_sync_barrier(_worker_id);
3451 }
3452 // At this point, if we're during the concurrent phase of
3453 // marking, everything has been re-initialized and we're
3454 // ready to restart.
3455 }
3456 }
3457
3458 _claimed = false;
3459 }
3460
3461 G1CMTask::G1CMTask(uint worker_id,
3462 G1ConcurrentMark* cm,
3463 G1CMTaskQueue* task_queue,
3464 G1CMTaskQueueSet* task_queues)
3465 : _g1h(G1CollectedHeap::heap()),
3466 _worker_id(worker_id), _cm(cm),
3467 _claimed(false),
3468 _nextMarkBitMap(NULL), _hash_seed(17),
3469 _task_queue(task_queue),
3470 _task_queues(task_queues),
3471 _cm_oop_closure(NULL) {
3472 guarantee(task_queue != NULL, "invariant");
3473 guarantee(task_queues != NULL, "invariant");
3474
3475 _marking_step_diffs_ms.add(0.5);
3476 }
3477
3478 // These are formatting macros that are used below to ensure
3479 // consistent formatting. The *_H_* versions are used to format the
3480 // header for a particular value and they should be kept consistent
3481 // with the corresponding macro. Also note that most of the macros add
3482 // the necessary white space (as a prefix) which makes them a bit
3483 // easier to compose.
3484
3485 // All the output lines are prefixed with this string to be able to
3486 // identify them easily in a large log file.
3487 #define G1PPRL_LINE_PREFIX "###"
3488
3489 #define G1PPRL_ADDR_BASE_FORMAT " " PTR_FORMAT "-" PTR_FORMAT
3490 #ifdef _LP64
3491 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
|