504 if (G1PretouchAuxiliaryMemory) {
505 pretouch_internal_bitmaps();
506 }
507
508 _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_worker_id, mtGC);
509 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_worker_id, mtGC);
510
511 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
512 _active_tasks = _max_worker_id;
513
514 for (uint i = 0; i < _max_worker_id; ++i) {
515 G1CMTaskQueue* task_queue = new G1CMTaskQueue();
516 task_queue->initialize();
517 _task_queues->register_queue(i, task_queue);
518
519 _tasks[i] = new G1CMTask(i, this, task_queue, _task_queues);
520
521 _accum_task_vtime[i] = 0.0;
522 }
523
524 // Calculate the card number for the bottom of the heap. Used
525 // in biasing indexes into the accounting card bitmaps.
526 _heap_bottom_card_num =
527 intptr_t(uintptr_t(_g1h->reserved_region().start()) >>
528 CardTableModRefBS::card_shift);
529
530 // so that the call below can read a sensible value
531 _heap_start = g1h->reserved_region().start();
532 set_non_marking_state();
533 _completed_initialization = true;
534 }
535
536 void G1ConcurrentMark::reset() {
537 // Starting values for these two. This should be called in a STW
538 // phase.
539 MemRegion reserved = _g1h->g1_reserved();
540 _heap_start = reserved.start();
541 _heap_end = reserved.end();
542
543 // Separated the asserts so that we know which one fires.
544 assert(_heap_start != NULL, "heap bounds should look ok");
545 assert(_heap_end != NULL, "heap bounds should look ok");
546 assert(_heap_start < _heap_end, "heap bounds should look ok");
547
548 // Reset all the marking data structures and any necessary flags
549 reset_marking_state();
1114 set_non_marking_state();
1115 }
1116
1117 // Expand the marking stack, if we have to and if we can.
1118 if (_markStack.should_expand()) {
1119 _markStack.expand();
1120 }
1121
1122 // Statistics
1123 double now = os::elapsedTime();
1124 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1125 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1126 _remark_times.add((now - start) * 1000.0);
1127
1128 g1p->record_concurrent_mark_remark_end();
1129
1130 G1CMIsAliveClosure is_alive(g1h);
1131 _gc_tracer_cm->report_object_count_after_gc(&is_alive);
1132 }
1133
1134 // Base class of the closures that finalize and verify the
1135 // liveness count data.
1136 class G1LiveDataClosureBase: public HeapRegionClosure {
1137 protected:
1138 G1ConcurrentMark* _cm;
1139
1140 BitMap* _region_bm;
1141 BitMap* _card_bm;
1142
1143 // Takes a region that's not empty (i.e., it has at least one
1144 // live object in it and sets its corresponding bit on the region
1145 // bitmap to 1.
1146 void set_bit_for_region(HeapRegion* hr) {
1147 BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index();
1148 _region_bm->par_at_put(index, true);
1149 }
1150
1151 // Utility routine to set an exclusive range of cards on the given
1152 // bitmap.
1153 inline void set_card_bitmap_range(BitMap* card_bm,
1154 BitMap::idx_t start_idx,
1155 BitMap::idx_t end_idx) {
1156
1157 // Set the exclusive bit range [start_idx, end_idx).
1158 assert((end_idx - start_idx) > 0, "at least one card");
1159 assert(end_idx <= card_bm->size(), "sanity");
1160
1161 // For small ranges use a simple loop; otherwise use set_range or
1162 // use par_at_put_range (if parallel). The range is made up of the
1163 // cards that are spanned by an object/mem region so 8 cards will
1164 // allow up to object sizes up to 4K to be handled using the loop.
1165 if ((end_idx - start_idx) <= 8) {
1166 for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) {
1167 card_bm->set_bit(i);
1168 }
1169 } else {
1170 card_bm->set_range(start_idx, end_idx);
1171 }
1172 }
1173
1174 void mark_card_bitmap_range(HeapWord* start, HeapWord* end) {
1175 BitMap::idx_t start_idx = _cm->card_live_bitmap_index_for(start);
1176 BitMap::idx_t end_idx = _cm->card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size));
1177
1178 assert((end_idx - start_idx) > 0, "Trying to mark zero sized range.");
1179
1180 if (start_idx == _last_marked_bit_idx) {
1181 start_idx++;
1182 }
1183 if (start_idx == end_idx) {
1184 return;
1185 }
1186
1187 // Set the bits in the card bitmap for the cards spanned by this object.
1188 set_card_bitmap_range(_card_bm, start_idx, end_idx);
1189 _last_marked_bit_idx = end_idx - 1;
1190 }
1191
1192 // We cache the last mark set. This avoids setting the same bit multiple times.
1193 // This is particularly interesting for dense bitmaps, as this avoids doing
1194 // any work most of the time.
1195 BitMap::idx_t _last_marked_bit_idx;
1196
1197 void reset_mark_cache() {
1198 _last_marked_bit_idx = (BitMap::idx_t)-1;
1199 }
1200
1201 void mark_allocated_since_marking(HeapRegion* hr) {
1202 reset_mark_cache();
1203
1204 HeapWord* ntams = hr->next_top_at_mark_start();
1205 HeapWord* top = hr->top();
1206
1207 assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions.");
1208
1209 // Mark the allocated-since-marking portion...
1210 if (ntams < top) {
1211 mark_card_bitmap_range(ntams, top);
1212 // This definitely means the region has live objects.
1213 set_bit_for_region(hr);
1214 }
1215 }
1216
1217 bool mark_marked_during_marking(HeapRegion* hr, bool may_suspend, size_t* total_bytes_marked) {
1218 reset_mark_cache();
1219
1220 size_t marked_bytes = 0;
1221
1222 HeapWord* ntams = hr->next_top_at_mark_start();
1223 HeapWord* start = hr->bottom();
1224
1225 if (ntams <= start) {
1226 // Empty region (at the start of marking). Nothing to do.
1227 // hr->add_to_marked_bytes(0);
1228 *total_bytes_marked = marked_bytes;
1229 return false;
1230 } else if (hr->is_starts_humongous()) {
1231 // Humongous object: distribute the marked bytes across the humongous object.
1232 do {
1233 mark_card_bitmap_range(start, hr->top());
1234
1235 marked_bytes += pointer_delta(hr->top(), start, 1);
1236 hr->add_to_marked_bytes(marked_bytes);
1237
1238 hr = G1CollectedHeap::heap()->next_region_in_humongous(hr);
1239 } while (hr != NULL);
1240 *total_bytes_marked = marked_bytes;
1241 return false;
1242 } else if (hr->is_continues_humongous()) {
1243 // Humongous continues regions were handled during processing of the start region.
1244 *total_bytes_marked = marked_bytes;
1245 return false;
1246 }
1247
1248 assert(start <= hr->end() && start <= ntams && ntams <= hr->end(),
1249 "Preconditions not met - "
1250 "start: " PTR_FORMAT ", ntams: " PTR_FORMAT ", end: " PTR_FORMAT,
1251 p2i(start), p2i(ntams), p2i(hr->end()));
1252
1253 G1CMBitMap* bitmap = _cm->nextMarkBitMap();
1254 // Find the first marked object at or after "start".
1255 start = bitmap->getNextMarkedWordAddress(start, ntams);
1256 while (start < ntams) {
1257 oop obj = oop(start);
1258 int obj_sz = obj->size();
1259 HeapWord* obj_end = start + obj_sz;
1260
1261 assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere.");
1262
1263 mark_card_bitmap_range(start, obj_end);
1264
1265 // Add the size of this object to the number of marked bytes.
1266 marked_bytes += (size_t)obj_sz * HeapWordSize;
1267
1268 // Find the next marked object after this one.
1269 start = bitmap->getNextMarkedWordAddress(obj_end, ntams);
1270 }
1271
1272 // Update the marked bytes for this region.
1273 hr->add_to_marked_bytes(marked_bytes);
1274 *total_bytes_marked = marked_bytes;
1275
1276 // Abort iteration if after yielding the marking has been aborted.
1277 if (may_suspend && _cm->do_yield_check() && _cm->has_aborted()) {
1278 return true;
1279 }
1280 // Next heap region
1281 return false;
1282 }
1283
1284 public:
1285 G1LiveDataClosureBase(G1CollectedHeap* g1h,
1286 BitMap* region_bm,
1287 BitMap* card_bm):
1288 _cm(g1h->concurrent_mark()),
1289 _region_bm(region_bm),
1290 _card_bm(card_bm) { }
1291 };
1292
1293 // Heap region closure used for verifying the live count data
1294 // that was created concurrently and finalized during
1295 // the remark pause. This closure is applied to the heap
1296 // regions during the STW cleanup pause.
1297 class G1VerifyLiveDataHRClosure: public HeapRegionClosure {
1298 // Calculates the # live objects per region.
1299 class G1VerifyLiveDataClosure: public G1LiveDataClosureBase {
1300 size_t _region_marked_bytes;
1301
1302 public:
1303 G1VerifyLiveDataClosure(G1CollectedHeap* g1h,
1304 BitMap* region_bm,
1305 BitMap* card_bm) :
1306 G1LiveDataClosureBase(g1h, region_bm, card_bm),
1307 _region_marked_bytes(0) { }
1308
1309 bool doHeapRegion(HeapRegion* hr) {
1310 mark_marked_during_marking(hr, false, &_region_marked_bytes);
1311 mark_allocated_since_marking(hr);
1312 return false;
1313 }
1314
1315 size_t region_marked_bytes() const { return _region_marked_bytes; }
1316 };
1317
1318 G1CollectedHeap* _g1h;
1319 G1ConcurrentMark* _cm;
1320 G1VerifyLiveDataClosure _calc_cl;
1321 BitMap* _act_region_bm; // Region BM to be verified
1322 BitMap* _act_card_bm; // Card BM to be verified
1323
1324 BitMap* _exp_region_bm; // Expected Region BM values
1325 BitMap* _exp_card_bm; // Expected card BM values
1326
1327 int _failures;
1328
1329 public:
1330 G1VerifyLiveDataHRClosure(G1CollectedHeap* g1h,
1331 BitMap* act_region_bm,
1332 BitMap* act_card_bm,
1333 BitMap* exp_region_bm,
1334 BitMap* exp_card_bm) :
1335 _g1h(g1h),
1336 _cm(g1h->concurrent_mark()),
1337 _calc_cl(g1h, exp_region_bm, exp_card_bm),
1338 _act_region_bm(act_region_bm),
1339 _act_card_bm(act_card_bm),
1340 _exp_region_bm(exp_region_bm),
1341 _exp_card_bm(exp_card_bm),
1342 _failures(0) { }
1343
1344 int failures() const { return _failures; }
1345
1346 bool doHeapRegion(HeapRegion* hr) {
1347 int failures = 0;
1348
1349 // Call the G1VerifyLiveDataClosure to walk the marking bitmap for
1350 // this region and set the corresponding bits in the expected region
1351 // and card bitmaps.
1352 bool res = _calc_cl.doHeapRegion(hr);
1353 assert(!res, "Should be completed.");
1354
1355 // Verify the marked bytes for this region.
1356 size_t exp_marked_bytes = _calc_cl.region_marked_bytes();
1357 size_t act_marked_bytes = hr->next_marked_bytes();
1358
1359 if (exp_marked_bytes > act_marked_bytes) {
1360 if (hr->is_starts_humongous()) {
1361 // For start_humongous regions, the size of the whole object will be
1362 // in exp_marked_bytes.
1363 HeapRegion* region = hr;
1364 int num_regions;
1365 for (num_regions = 0; region != NULL; num_regions++) {
1366 region = _g1h->next_region_in_humongous(region);
1367 }
1368 if ((num_regions-1) * HeapRegion::GrainBytes >= exp_marked_bytes) {
1369 failures += 1;
1370 } else if (num_regions * HeapRegion::GrainBytes < exp_marked_bytes) {
1371 failures += 1;
1372 }
1373 } else {
1374 // We're not OK if expected marked bytes > actual marked bytes. It means
1375 // we have missed accounting some objects during the actual marking.
1376 failures += 1;
1377 }
1378 }
1379
1380 // Verify the bit, for this region, in the actual and expected
1381 // (which was just calculated) region bit maps.
1382 // We're not OK if the bit in the calculated expected region
1383 // bitmap is set and the bit in the actual region bitmap is not.
1384 BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index();
1385
1386 bool expected = _exp_region_bm->at(index);
1387 bool actual = _act_region_bm->at(index);
1388 if (expected && !actual) {
1389 failures += 1;
1390 }
1391
1392 // Verify that the card bit maps for the cards spanned by the current
1393 // region match. We have an error if we have a set bit in the expected
1394 // bit map and the corresponding bit in the actual bitmap is not set.
1395
1396 BitMap::idx_t start_idx = _cm->card_live_bitmap_index_for(hr->bottom());
1397 BitMap::idx_t end_idx = _cm->card_live_bitmap_index_for(hr->top());
1398
1399 for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) {
1400 expected = _exp_card_bm->at(i);
1401 actual = _act_card_bm->at(i);
1402
1403 if (expected && !actual) {
1404 failures += 1;
1405 }
1406 }
1407
1408 _failures += failures;
1409
1410 // We could stop iteration over the heap when we
1411 // find the first violating region by returning true.
1412 return false;
1413 }
1414 };
1415
1416 class G1VerifyLiveDataTask: public AbstractGangTask {
1417 protected:
1418 G1CollectedHeap* _g1h;
1419 BitMap* _actual_region_bm;
1420 BitMap* _actual_card_bm;
1421
1422 BitMap _expected_region_bm;
1423 BitMap _expected_card_bm;
1424
1425 int _failures;
1426
1427 HeapRegionClaimer _hr_claimer;
1428
1429 public:
1430 G1VerifyLiveDataTask(G1CollectedHeap* g1h,
1431 BitMap* region_bm,
1432 BitMap* card_bm,
1433 uint n_workers)
1434 : AbstractGangTask("G1 verify final counting"),
1435 _g1h(g1h),
1436 _actual_region_bm(region_bm),
1437 _actual_card_bm(card_bm),
1438 _expected_region_bm(region_bm->size(), true /* in_resource_area */),
1439 _expected_card_bm(card_bm->size(), true /* in_resource_area */),
1440 _failures(0),
1441 _hr_claimer(n_workers) {
1442 assert(VerifyDuringGC, "don't call this otherwise");
1443 }
1444
1445 void work(uint worker_id) {
1446 G1VerifyLiveDataHRClosure cl(_g1h,
1447 _actual_region_bm,
1448 _actual_card_bm,
1449 &_expected_region_bm,
1450 &_expected_card_bm);
1451 _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
1452
1453 Atomic::add(cl.failures(), &_failures);
1454 }
1455
1456 int failures() const { return _failures; }
1457 };
1458
1459 class G1FinalizeLiveDataTask: public AbstractGangTask {
1460 // Finalizes the liveness counting data.
1461 // Sets the bits corresponding to the interval [NTAMS, top]
1462 // (which contains the implicitly live objects) in the
1463 // card liveness bitmap. Also sets the bit for each region
1464 // containing live data, in the region liveness bitmap.
1465 class G1FinalizeCountDataClosure: public G1LiveDataClosureBase {
1466 public:
1467 G1FinalizeCountDataClosure(G1CollectedHeap* g1h,
1468 BitMap* region_bm,
1469 BitMap* card_bm) :
1470 G1LiveDataClosureBase(g1h, region_bm, card_bm) { }
1471
1472 bool doHeapRegion(HeapRegion* hr) {
1473 mark_allocated_since_marking(hr);
1474 // Set the bit for the region if it contains live data
1475 if (hr->next_marked_bytes() > 0) {
1476 set_bit_for_region(hr);
1477 }
1478
1479 return false;
1480 }
1481 };
1482
1483 G1CollectedHeap* _g1h;
1484 BitMap* _actual_region_bm;
1485 BitMap* _actual_card_bm;
1486
1487 HeapRegionClaimer _hr_claimer;
1488
1489 public:
1490 G1FinalizeLiveDataTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm, uint n_workers) :
1491 AbstractGangTask("G1 final counting"),
1492 _g1h(g1h),
1493 _actual_region_bm(region_bm),
1494 _actual_card_bm(card_bm),
1495 _hr_claimer(n_workers) {
1496 }
1497
1498 void work(uint worker_id) {
1499 G1FinalizeCountDataClosure cl(_g1h,
1500 _actual_region_bm,
1501 _actual_card_bm);
1502
1503 _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
1504 }
1505 };
1506
1507 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1508 G1CollectedHeap* _g1;
1509 size_t _freed_bytes;
1510 FreeRegionList* _local_cleanup_list;
1511 uint _old_regions_removed;
1512 uint _humongous_regions_removed;
1513 HRRSCleanupTask* _hrrs_cleanup_task;
1514
1515 public:
1516 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1517 FreeRegionList* local_cleanup_list,
1518 HRRSCleanupTask* hrrs_cleanup_task) :
1519 _g1(g1),
1520 _freed_bytes(0),
1521 _local_cleanup_list(local_cleanup_list),
1522 _old_regions_removed(0),
1523 _humongous_regions_removed(0),
1619 }
1620
1621 g1h->verifier()->verify_region_sets_optional();
1622
1623 if (VerifyDuringGC) {
1624 HandleMark hm; // handle scope
1625 g1h->prepare_for_verify();
1626 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)");
1627 }
1628 g1h->verifier()->check_bitmaps("Cleanup Start");
1629
1630 G1CollectorPolicy* g1p = g1h->g1_policy();
1631 g1p->record_concurrent_mark_cleanup_start();
1632
1633 double start = os::elapsedTime();
1634
1635 HeapRegionRemSet::reset_for_cleanup_tasks();
1636
1637 {
1638 // Finalize the live data.
1639 G1FinalizeLiveDataTask cl(g1h,
1640 &_region_live_bm,
1641 &_card_live_bm,
1642 g1h->workers()->active_workers());
1643 g1h->workers()->run_task(&cl);
1644 }
1645
1646 if (VerifyDuringGC) {
1647 // Verify that the liveness count data created concurrently matches one created
1648 // during this safepoint.
1649 ResourceMark rm;
1650 G1VerifyLiveDataTask cl(g1h,
1651 &_region_live_bm,
1652 &_card_live_bm,
1653 g1h->workers()->active_workers());
1654 g1h->workers()->run_task(&cl);
1655
1656 guarantee(cl.failures() == 0, "Unexpected accounting failures");
1657 }
1658
1659 g1h->collector_state()->set_mark_in_progress(false);
1660
1661 double count_end = os::elapsedTime();
1662 double this_final_counting_time = (count_end - start);
1663 _total_counting_time += this_final_counting_time;
1664
1665 if (log_is_enabled(Trace, gc, liveness)) {
1666 G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1667 _g1h->heap_region_iterate(&cl);
1668 }
1669
1670 // Install newly created mark bitMap as "prev".
2415 // Verify the task fingers
2416 assert(parallel_marking_threads() <= _max_worker_id, "sanity");
2417 for (uint i = 0; i < parallel_marking_threads(); ++i) {
2418 G1CMTask* task = _tasks[i];
2419 HeapWord* task_finger = task->finger();
2420 if (task_finger != NULL && task_finger < _heap_end) {
2421 // See above note on the global finger verification.
2422 HeapRegion* task_hr = _g1h->heap_region_containing(task_finger);
2423 guarantee(task_hr == NULL || task_finger == task_hr->bottom() ||
2424 !task_hr->in_collection_set(),
2425 "task finger: " PTR_FORMAT " region: " HR_FORMAT,
2426 p2i(task_finger), HR_FORMAT_PARAMS(task_hr));
2427 }
2428 }
2429 }
2430 #endif // PRODUCT
2431
2432 class G1CreateLiveDataTask: public AbstractGangTask {
2433 // Aggregate the counting data that was constructed concurrently
2434 // with marking.
2435 class G1CreateLiveDataHRClosure: public G1LiveDataClosureBase {
2436 public:
2437 G1CreateLiveDataHRClosure(G1CollectedHeap* g1h,
2438 BitMap* cm_card_bm)
2439 : G1LiveDataClosureBase(g1h, NULL, cm_card_bm) { }
2440
2441 bool doHeapRegion(HeapRegion* hr) {
2442 size_t temp;
2443 return mark_marked_during_marking(hr, true, &temp);
2444 }
2445 };
2446
2447 G1CollectedHeap* _g1h;
2448 G1ConcurrentMark* _cm;
2449 BitMap* _cm_card_bm;
2450 HeapRegionClaimer _hrclaimer;
2451
2452 public:
2453 G1CreateLiveDataTask(G1CollectedHeap* g1h,
2454 BitMap* cm_card_bm,
2455 uint n_workers) :
2456 AbstractGangTask("Create Live Data"),
2457 _g1h(g1h),
2458 _cm_card_bm(cm_card_bm),
2459 _hrclaimer(n_workers) {
2460 }
2461
2462 void work(uint worker_id) {
2463 SuspendibleThreadSetJoiner sts_join;
2464
2465 G1CreateLiveDataHRClosure cl(_g1h, _cm_card_bm);
2466 _g1h->heap_region_par_iterate(&cl, worker_id, &_hrclaimer);
2467 }
2468 };
2469
2470
2471 void G1ConcurrentMark::create_live_data() {
2472 uint n_workers = _parallel_workers->active_workers();
2473
2474 G1CreateLiveDataTask cl(_g1h,
2475 &_card_live_bm,
2476 n_workers);
2477 _parallel_workers->run_task(&cl);
2478 }
2479
2480 class G1ClearAllLiveDataTask : public AbstractGangTask {
2481 BitMap* _bitmap;
2482 size_t _num_tasks;
2483 size_t _cur_task;
2484 public:
2485 G1ClearAllLiveDataTask(BitMap* bitmap, size_t num_tasks) :
2486 AbstractGangTask("Clear All Live Data"),
|
504 if (G1PretouchAuxiliaryMemory) {
505 pretouch_internal_bitmaps();
506 }
507
508 _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_worker_id, mtGC);
509 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_worker_id, mtGC);
510
511 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
512 _active_tasks = _max_worker_id;
513
514 for (uint i = 0; i < _max_worker_id; ++i) {
515 G1CMTaskQueue* task_queue = new G1CMTaskQueue();
516 task_queue->initialize();
517 _task_queues->register_queue(i, task_queue);
518
519 _tasks[i] = new G1CMTask(i, this, task_queue, _task_queues);
520
521 _accum_task_vtime[i] = 0.0;
522 }
523
524 // so that the call below can read a sensible value
525 _heap_start = g1h->reserved_region().start();
526 set_non_marking_state();
527 _completed_initialization = true;
528 }
529
530 void G1ConcurrentMark::reset() {
531 // Starting values for these two. This should be called in a STW
532 // phase.
533 MemRegion reserved = _g1h->g1_reserved();
534 _heap_start = reserved.start();
535 _heap_end = reserved.end();
536
537 // Separated the asserts so that we know which one fires.
538 assert(_heap_start != NULL, "heap bounds should look ok");
539 assert(_heap_end != NULL, "heap bounds should look ok");
540 assert(_heap_start < _heap_end, "heap bounds should look ok");
541
542 // Reset all the marking data structures and any necessary flags
543 reset_marking_state();
1108 set_non_marking_state();
1109 }
1110
1111 // Expand the marking stack, if we have to and if we can.
1112 if (_markStack.should_expand()) {
1113 _markStack.expand();
1114 }
1115
1116 // Statistics
1117 double now = os::elapsedTime();
1118 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1119 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1120 _remark_times.add((now - start) * 1000.0);
1121
1122 g1p->record_concurrent_mark_remark_end();
1123
1124 G1CMIsAliveClosure is_alive(g1h);
1125 _gc_tracer_cm->report_object_count_after_gc(&is_alive);
1126 }
1127
1128 // Helper class that provides functionality to generate the Live Data Count
1129 // information.
1130 class G1LiveDataHelper VALUE_OBJ_CLASS_SPEC {
1131 private:
1132 BitMap* _region_bm;
1133 BitMap* _card_bm;
1134
1135 // The card number of the bottom of the G1 heap. Used for converting addresses
1136 // to bitmap indices quickly.
1137 BitMap::idx_t _heap_card_bias;
1138
1139 // Utility routine to set an exclusive range of bits on the given
1140 // bitmap, optimized for very small ranges.
1141 // There must be at least one bit to set.
1142 inline void set_card_bitmap_range(BitMap* bm,
1143 BitMap::idx_t start_idx,
1144 BitMap::idx_t end_idx) {
1145
1146 // Set the exclusive bit range [start_idx, end_idx).
1147 assert((end_idx - start_idx) > 0, "at least one bit");
1148 assert(end_idx <= bm->size(), "sanity");
1149
1150 // For small ranges use a simple loop; otherwise use set_range or
1151 // use par_at_put_range (if parallel). The range is made up of the
1152 // cards that are spanned by an object/mem region so 8 cards will
1153 // allow up to object sizes up to 4K to be handled using the loop.
1154 if ((end_idx - start_idx) <= 8) {
1155 for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) {
1156 bm->set_bit(i);
1157 }
1158 } else {
1159 bm->set_range(start_idx, end_idx);
1160 }
1161 }
1162
1163 // We cache the last mark set. This avoids setting the same bit multiple times.
1164 // This is particularly interesting for dense bitmaps, as this avoids doing
1165 // lots of work most of the time.
1166 BitMap::idx_t _last_marked_bit_idx;
1167
1168 // Mark the card liveness bitmap for the object spanning from start to end.
1169 void mark_card_bitmap_range(HeapWord* start, HeapWord* end) {
1170 BitMap::idx_t start_idx = card_live_bitmap_index_for(start);
1171 BitMap::idx_t end_idx = card_live_bitmap_index_for((HeapWord*)align_ptr_up(end, CardTableModRefBS::card_size));
1172
1173 assert((end_idx - start_idx) > 0, "Trying to mark zero sized range.");
1174
1175 if (start_idx == _last_marked_bit_idx) {
1176 start_idx++;
1177 }
1178 if (start_idx == end_idx) {
1179 return;
1180 }
1181
1182 // Set the bits in the card bitmap for the cards spanned by this object.
1183 set_card_bitmap_range(_card_bm, start_idx, end_idx);
1184 _last_marked_bit_idx = end_idx - 1;
1185 }
1186
1187 void reset_mark_cache() {
1188 _last_marked_bit_idx = (BitMap::idx_t)-1;
1189 }
1190
1191 public:
1192 // Returns the index in the per-card liveness count bitmap
1193 // for the given address
1194 inline BitMap::idx_t card_live_bitmap_index_for(HeapWord* addr) {
1195 // Below, the term "card num" means the result of shifting an address
1196 // by the card shift -- address 0 corresponds to card number 0. One
1197 // must subtract the card num of the bottom of the heap to obtain a
1198 // card table index.
1199 BitMap::idx_t card_num = (BitMap::idx_t)(uintptr_t(addr) >> CardTableModRefBS::card_shift);
1200 return card_num - _heap_card_bias;
1201 }
1202
1203 // Takes a region that's not empty (i.e., it has at least one
1204 // live object in it and sets its corresponding bit on the region
1205 // bitmap to 1.
1206 void set_bit_for_region(HeapRegion* hr) {
1207 BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index();
1208 _region_bm->par_at_put(index, true);
1209 }
1210
1211 // Mark the range of bits covered by allocations done since the last marking
1212 // in the given heap region, i.e. from NTAMS to top of the given region.
1213 // Returns if there has been some allocation in this region since the last marking.
1214 bool mark_allocated_since_marking(HeapRegion* hr) {
1215 reset_mark_cache();
1216
1217 HeapWord* ntams = hr->next_top_at_mark_start();
1218 HeapWord* top = hr->top();
1219
1220 assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions.");
1221
1222 // Mark the allocated-since-marking portion...
1223 if (ntams < top) {
1224 mark_card_bitmap_range(ntams, top);
1225 return true;
1226 } else {
1227 return false;
1228 }
1229 }
1230
1231 // Mark the range of bits covered by live objects on the mark bitmap between
1232 // bottom and NTAMS of the given region.
1233 // Returns the number of live bytes marked within that area for the given
1234 // heap region.
1235 size_t mark_marked_during_marking(G1CMBitMap* mark_bitmap, HeapRegion* hr) {
1236 reset_mark_cache();
1237
1238 size_t marked_bytes = 0;
1239
1240 HeapWord* ntams = hr->next_top_at_mark_start();
1241 HeapWord* start = hr->bottom();
1242
1243 if (ntams <= start) {
1244 // Skip empty regions.
1245 return 0;
1246 } else if (hr->is_humongous()) {
1247 mark_card_bitmap_range(start, hr->top());
1248 return pointer_delta(hr->top(), start, 1);
1249 }
1250
1251 assert(start <= hr->end() && start <= ntams && ntams <= hr->end(),
1252 "Preconditions not met - "
1253 "start: " PTR_FORMAT ", ntams: " PTR_FORMAT ", end: " PTR_FORMAT,
1254 p2i(start), p2i(ntams), p2i(hr->end()));
1255
1256 // Find the first marked object at or after "start".
1257 start = mark_bitmap->getNextMarkedWordAddress(start, ntams);
1258 while (start < ntams) {
1259 oop obj = oop(start);
1260 int obj_sz = obj->size();
1261 HeapWord* obj_end = start + obj_sz;
1262
1263 assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere.");
1264
1265 mark_card_bitmap_range(start, obj_end);
1266
1267 // Add the size of this object to the number of marked bytes.
1268 marked_bytes += (size_t)obj_sz * HeapWordSize;
1269
1270 // Find the next marked object after this one.
1271 start = mark_bitmap->getNextMarkedWordAddress(obj_end, ntams);
1272 }
1273
1274 return marked_bytes;
1275 }
1276
1277 G1LiveDataHelper(BitMap* region_bm,
1278 BitMap* card_bm):
1279 _region_bm(region_bm),
1280 _card_bm(card_bm) {
1281 //assert(region_bm != NULL, "");
1282 assert(card_bm != NULL, "");
1283 // Calculate the card number for the bottom of the heap. Used
1284 // in biasing indexes into the accounting card bitmaps.
1285 _heap_card_bias =
1286 (BitMap::idx_t)(uintptr_t(G1CollectedHeap::heap()->reserved_region().start()) >> CardTableModRefBS::card_shift);
1287 }
1288 };
1289
1290 // Heap region closure used for verifying the live count data
1291 // that was created concurrently and finalized during
1292 // the remark pause. This closure is applied to the heap
1293 // regions during the STW cleanup pause.
1294 class G1VerifyLiveDataHRClosure: public HeapRegionClosure {
1295 private:
1296 G1CollectedHeap* _g1h;
1297 G1CMBitMap* _mark_bitmap;
1298 G1LiveDataHelper _calc_helper;
1299
1300 BitMap* _act_region_bm; // Region BM to be verified
1301 BitMap* _act_card_bm; // Card BM to be verified
1302
1303 BitMap* _exp_region_bm; // Expected Region BM values
1304 BitMap* _exp_card_bm; // Expected card BM values
1305
1306 int _failures;
1307
1308 // Updates the live data count for the given heap region and returns the number
1309 // of bytes marked.
1310 size_t create_live_data_count(HeapRegion* hr) {
1311 size_t bytes_marked = _calc_helper.mark_marked_during_marking(_mark_bitmap, hr);
1312 bool allocated_since_marking = _calc_helper.mark_allocated_since_marking(hr);
1313 if (allocated_since_marking || bytes_marked > 0) {
1314 _calc_helper.set_bit_for_region(hr);
1315 }
1316 return bytes_marked;
1317 }
1318
1319 public:
1320 G1VerifyLiveDataHRClosure(G1CollectedHeap* g1h,
1321 G1CMBitMap* mark_bitmap,
1322 BitMap* act_region_bm,
1323 BitMap* act_card_bm,
1324 BitMap* exp_region_bm,
1325 BitMap* exp_card_bm) :
1326 _g1h(g1h),
1327 _mark_bitmap(mark_bitmap),
1328 _calc_helper(exp_region_bm, exp_card_bm),
1329 _act_region_bm(act_region_bm),
1330 _act_card_bm(act_card_bm),
1331 _exp_region_bm(exp_region_bm),
1332 _exp_card_bm(exp_card_bm),
1333 _failures(0) { }
1334
1335 int failures() const { return _failures; }
1336
1337 bool doHeapRegion(HeapRegion* hr) {
1338 int failures = 0;
1339
1340 // Walk the marking bitmap for this region and set the corresponding bits
1341 // in the expected region and card bitmaps.
1342 size_t exp_marked_bytes = create_live_data_count(hr);
1343 size_t act_marked_bytes = hr->next_marked_bytes();
1344 // Verify the marked bytes for this region.
1345
1346 if (exp_marked_bytes != act_marked_bytes) {
1347 failures += 1;
1348 } else if (exp_marked_bytes > HeapRegion::GrainBytes) {
1349 failures += 1;
1350 }
1351
1352 // Verify the bit, for this region, in the actual and expected
1353 // (which was just calculated) region bit maps.
1354 // We're not OK if the bit in the calculated expected region
1355 // bitmap is set and the bit in the actual region bitmap is not.
1356 BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index();
1357
1358 bool expected = _exp_region_bm->at(index);
1359 bool actual = _act_region_bm->at(index);
1360 if (expected && !actual) {
1361 failures += 1;
1362 }
1363
1364 // Verify that the card bit maps for the cards spanned by the current
1365 // region match. We have an error if we have a set bit in the expected
1366 // bit map and the corresponding bit in the actual bitmap is not set.
1367
1368 BitMap::idx_t start_idx = _calc_helper.card_live_bitmap_index_for(hr->bottom());
1369 BitMap::idx_t end_idx = _calc_helper.card_live_bitmap_index_for(hr->top());
1370
1371 for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) {
1372 expected = _exp_card_bm->at(i);
1373 actual = _act_card_bm->at(i);
1374
1375 if (expected && !actual) {
1376 failures += 1;
1377 }
1378 }
1379
1380 _failures += failures;
1381
1382 // We could stop iteration over the heap when we
1383 // find the first violating region by returning true.
1384 return false;
1385 }
1386 };
1387
1388 class G1VerifyLiveDataTask: public AbstractGangTask {
1389 protected:
1390 G1CollectedHeap* _g1h;
1391 G1CMBitMap* _mark_bitmap;
1392 BitMap* _actual_region_bm;
1393 BitMap* _actual_card_bm;
1394
1395 BitMap _expected_region_bm;
1396 BitMap _expected_card_bm;
1397
1398 int _failures;
1399
1400 HeapRegionClaimer _hr_claimer;
1401
1402 public:
1403 G1VerifyLiveDataTask(G1CollectedHeap* g1h,
1404 G1CMBitMap* bitmap,
1405 BitMap* region_bm,
1406 BitMap* card_bm,
1407 uint n_workers)
1408 : AbstractGangTask("G1 verify final counting"),
1409 _g1h(g1h),
1410 _mark_bitmap(bitmap),
1411 _actual_region_bm(region_bm),
1412 _actual_card_bm(card_bm),
1413 _expected_region_bm(region_bm->size(), true /* in_resource_area */),
1414 _expected_card_bm(card_bm->size(), true /* in_resource_area */),
1415 _failures(0),
1416 _hr_claimer(n_workers) {
1417 assert(VerifyDuringGC, "don't call this otherwise");
1418 }
1419
1420 void work(uint worker_id) {
1421 G1VerifyLiveDataHRClosure cl(_g1h,
1422 _mark_bitmap,
1423 _actual_region_bm,
1424 _actual_card_bm,
1425 &_expected_region_bm,
1426 &_expected_card_bm);
1427 _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
1428
1429 Atomic::add(cl.failures(), &_failures);
1430 }
1431
1432 int failures() const { return _failures; }
1433 };
1434
1435 class G1FinalizeLiveDataTask: public AbstractGangTask {
1436 // Finalizes the liveness counting data.
1437 // Sets the bits corresponding to the interval [NTAMS, top]
1438 // (which contains the implicitly live objects) in the
1439 // card liveness bitmap. Also sets the bit for each region
1440 // containing live data, in the region liveness bitmap.
1441 class G1FinalizeCountDataClosure: public HeapRegionClosure {
1442 private:
1443 G1LiveDataHelper _helper;
1444 public:
1445 G1FinalizeCountDataClosure(G1CMBitMap* bitmap,
1446 BitMap* region_bm,
1447 BitMap* card_bm) :
1448 HeapRegionClosure(),
1449 _helper(region_bm, card_bm) { }
1450
1451 bool doHeapRegion(HeapRegion* hr) {
1452 bool allocated_since_marking = _helper.mark_allocated_since_marking(hr);
1453 if (allocated_since_marking || hr->next_marked_bytes() > 0) {
1454 _helper.set_bit_for_region(hr);
1455 }
1456 return false;
1457 }
1458 };
1459
1460 G1CMBitMap* _bitmap;
1461
1462 BitMap* _actual_region_bm;
1463 BitMap* _actual_card_bm;
1464
1465 HeapRegionClaimer _hr_claimer;
1466
1467 public:
1468 G1FinalizeLiveDataTask(G1CMBitMap* bitmap, BitMap* region_bm, BitMap* card_bm, uint n_workers) :
1469 AbstractGangTask("G1 final counting"),
1470 _bitmap(bitmap),
1471 _actual_region_bm(region_bm),
1472 _actual_card_bm(card_bm),
1473 _hr_claimer(n_workers) {
1474 }
1475
1476 void work(uint worker_id) {
1477 G1FinalizeCountDataClosure cl(_bitmap,
1478 _actual_region_bm,
1479 _actual_card_bm);
1480
1481 G1CollectedHeap::heap()->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
1482 }
1483 };
1484
1485 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1486 G1CollectedHeap* _g1;
1487 size_t _freed_bytes;
1488 FreeRegionList* _local_cleanup_list;
1489 uint _old_regions_removed;
1490 uint _humongous_regions_removed;
1491 HRRSCleanupTask* _hrrs_cleanup_task;
1492
1493 public:
1494 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1495 FreeRegionList* local_cleanup_list,
1496 HRRSCleanupTask* hrrs_cleanup_task) :
1497 _g1(g1),
1498 _freed_bytes(0),
1499 _local_cleanup_list(local_cleanup_list),
1500 _old_regions_removed(0),
1501 _humongous_regions_removed(0),
1597 }
1598
1599 g1h->verifier()->verify_region_sets_optional();
1600
1601 if (VerifyDuringGC) {
1602 HandleMark hm; // handle scope
1603 g1h->prepare_for_verify();
1604 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)");
1605 }
1606 g1h->verifier()->check_bitmaps("Cleanup Start");
1607
1608 G1CollectorPolicy* g1p = g1h->g1_policy();
1609 g1p->record_concurrent_mark_cleanup_start();
1610
1611 double start = os::elapsedTime();
1612
1613 HeapRegionRemSet::reset_for_cleanup_tasks();
1614
1615 {
1616 // Finalize the live data.
1617 G1FinalizeLiveDataTask cl(_nextMarkBitMap,
1618 &_region_live_bm,
1619 &_card_live_bm,
1620 g1h->workers()->active_workers());
1621 g1h->workers()->run_task(&cl);
1622 }
1623
1624 if (VerifyDuringGC) {
1625 // Verify that the liveness count data created concurrently matches one created
1626 // during this safepoint.
1627 ResourceMark rm;
1628 G1VerifyLiveDataTask cl(G1CollectedHeap::heap(),
1629 _nextMarkBitMap,
1630 &_region_live_bm,
1631 &_card_live_bm,
1632 g1h->workers()->active_workers());
1633 g1h->workers()->run_task(&cl);
1634
1635 guarantee(cl.failures() == 0, "Unexpected accounting failures");
1636 }
1637
1638 g1h->collector_state()->set_mark_in_progress(false);
1639
1640 double count_end = os::elapsedTime();
1641 double this_final_counting_time = (count_end - start);
1642 _total_counting_time += this_final_counting_time;
1643
1644 if (log_is_enabled(Trace, gc, liveness)) {
1645 G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1646 _g1h->heap_region_iterate(&cl);
1647 }
1648
1649 // Install newly created mark bitMap as "prev".
2394 // Verify the task fingers
2395 assert(parallel_marking_threads() <= _max_worker_id, "sanity");
2396 for (uint i = 0; i < parallel_marking_threads(); ++i) {
2397 G1CMTask* task = _tasks[i];
2398 HeapWord* task_finger = task->finger();
2399 if (task_finger != NULL && task_finger < _heap_end) {
2400 // See above note on the global finger verification.
2401 HeapRegion* task_hr = _g1h->heap_region_containing(task_finger);
2402 guarantee(task_hr == NULL || task_finger == task_hr->bottom() ||
2403 !task_hr->in_collection_set(),
2404 "task finger: " PTR_FORMAT " region: " HR_FORMAT,
2405 p2i(task_finger), HR_FORMAT_PARAMS(task_hr));
2406 }
2407 }
2408 }
2409 #endif // PRODUCT
2410
2411 class G1CreateLiveDataTask: public AbstractGangTask {
2412 // Aggregate the counting data that was constructed concurrently
2413 // with marking.
2414 class G1CreateLiveDataHRClosure: public HeapRegionClosure {
2415 G1LiveDataHelper _helper;
2416
2417 G1CMBitMap* _mark_bitmap;
2418
2419 G1ConcurrentMark* _cm;
2420 public:
2421 G1CreateLiveDataHRClosure(G1ConcurrentMark* cm,
2422 G1CMBitMap* mark_bitmap,
2423 BitMap* cm_card_bm) :
2424 HeapRegionClosure(),
2425 _helper(NULL, cm_card_bm),
2426 _mark_bitmap(mark_bitmap),
2427 _cm(cm) { }
2428
2429 bool doHeapRegion(HeapRegion* hr) {
2430 size_t marked_bytes = _helper.mark_marked_during_marking(_mark_bitmap, hr);
2431 if (marked_bytes > 0) {
2432 hr->add_to_marked_bytes(marked_bytes);
2433 }
2434
2435 if (_cm->do_yield_check() && _cm->has_aborted()) {
2436 return true;
2437 }
2438 return false;
2439 }
2440 };
2441
2442 G1CollectedHeap* _g1h;
2443 G1ConcurrentMark* _cm;
2444 BitMap* _cm_card_bm;
2445 HeapRegionClaimer _hr_claimer;
2446
2447 public:
2448 G1CreateLiveDataTask(G1CollectedHeap* g1h,
2449 BitMap* cm_card_bm,
2450 uint n_workers) :
2451 AbstractGangTask("Create Live Data"),
2452 _g1h(g1h),
2453 _cm_card_bm(cm_card_bm),
2454 _hr_claimer(n_workers) {
2455 }
2456
2457 void work(uint worker_id) {
2458 SuspendibleThreadSetJoiner sts_join;
2459
2460 G1CreateLiveDataHRClosure cl(_g1h->concurrent_mark(), _g1h->concurrent_mark()->nextMarkBitMap(), _cm_card_bm);
2461 _g1h->heap_region_par_iterate(&cl, worker_id, &_hr_claimer);
2462 }
2463 };
2464
2465
2466 void G1ConcurrentMark::create_live_data() {
2467 uint n_workers = _parallel_workers->active_workers();
2468
2469 G1CreateLiveDataTask cl(_g1h,
2470 &_card_live_bm,
2471 n_workers);
2472 _parallel_workers->run_task(&cl);
2473 }
2474
2475 class G1ClearAllLiveDataTask : public AbstractGangTask {
2476 BitMap* _bitmap;
2477 size_t _num_tasks;
2478 size_t _cur_task;
2479 public:
2480 G1ClearAllLiveDataTask(BitMap* bitmap, size_t num_tasks) :
2481 AbstractGangTask("Clear All Live Data"),
|