src/share/vm/gc_implementation/g1/concurrentMark.cpp

Print this page
rev 4008 : 8001985: G1: Backport fix for 7200261 to hsx24
Summary: The automatic backport of the fix for 7200261 did not apply cleanly to hsx24 - there were two rejected hunks that had to be fixed up by hand.
Reviewed-by:


1173   }
1174 
1175 #if VERIFY_OBJS_PROCESSED
1176   _scan_obj_cl.objs_processed = 0;
1177   ThreadLocalObjQueue::objs_enqueued = 0;
1178 #endif
1179 
1180   // Statistics
1181   double now = os::elapsedTime();
1182   _remark_mark_times.add((mark_work_end - start) * 1000.0);
1183   _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1184   _remark_times.add((now - start) * 1000.0);
1185 
1186   g1p->record_concurrent_mark_remark_end();
1187 }
1188 
1189 // Base class of the closures that finalize and verify the
1190 // liveness counting data.
1191 class CMCountDataClosureBase: public HeapRegionClosure {
1192 protected:

1193   ConcurrentMark* _cm;


1194   BitMap* _region_bm;
1195   BitMap* _card_bm;
1196 
1197   void set_card_bitmap_range(BitMap::idx_t start_idx, BitMap::idx_t last_idx) {
1198     assert(start_idx <= last_idx, "sanity");
1199 
1200     // Set the inclusive bit range [start_idx, last_idx].
1201     // For small ranges (up to 8 cards) use a simple loop; otherwise
1202     // use par_at_put_range.
1203     if ((last_idx - start_idx) < 8) {
1204       for (BitMap::idx_t i = start_idx; i <= last_idx; i += 1) {
1205         _card_bm->par_set_bit(i);
1206       }
1207     } else {
1208       assert(last_idx < _card_bm->size(), "sanity");
1209       // Note BitMap::par_at_put_range() is exclusive.
1210       _card_bm->par_at_put_range(start_idx, last_idx+1, true);
1211     }
1212   }
1213 
1214   // It takes a region that's not empty (i.e., it has at least one
1215   // live object in it and sets its corresponding bit on the region
1216   // bitmap to 1. If the region is "starts humongous" it will also set
1217   // to 1 the bits on the region bitmap that correspond to its
1218   // associated "continues humongous" regions.
1219   void set_bit_for_region(HeapRegion* hr) {
1220     assert(!hr->continuesHumongous(), "should have filtered those out");
1221 
1222     BitMap::idx_t index = (BitMap::idx_t) hr->hrs_index();
1223     if (!hr->startsHumongous()) {
1224       // Normal (non-humongous) case: just set the bit.
1225       _region_bm->par_at_put(index, true);
1226     } else {
1227       // Starts humongous case: calculate how many regions are part of
1228       // this humongous region and then set the bit range.
1229       BitMap::idx_t end_index = (BitMap::idx_t) hr->last_hc_index();
1230       _region_bm->par_at_put_range(index, end_index, true);
1231     }
1232   }
1233 
1234 public:
1235   CMCountDataClosureBase(ConcurrentMark *cm,
1236                          BitMap* region_bm, BitMap* card_bm):
1237     _cm(cm), _region_bm(region_bm), _card_bm(card_bm) { }


1238 };
1239 
1240 // Closure that calculates the # live objects per region. Used
1241 // for verification purposes during the cleanup pause.
1242 class CalcLiveObjectsClosure: public CMCountDataClosureBase {
1243   CMBitMapRO* _bm;
1244   size_t _region_marked_bytes;
1245 
1246 public:
1247   CalcLiveObjectsClosure(CMBitMapRO *bm, ConcurrentMark *cm,
1248                          BitMap* region_bm, BitMap* card_bm) :
1249     CMCountDataClosureBase(cm, region_bm, card_bm),
1250     _bm(bm), _region_marked_bytes(0) { }
1251 
1252   bool doHeapRegion(HeapRegion* hr) {
1253 
1254     if (hr->continuesHumongous()) {
1255       // We will ignore these here and process them when their
1256       // associated "starts humongous" region is processed (see
1257       // set_bit_for_heap_region()). Note that we cannot rely on their
1258       // associated "starts humongous" region to have their bit set to
1259       // 1 since, due to the region chunking in the parallel region
1260       // iteration, a "continues humongous" region might be visited
1261       // before its associated "starts humongous".
1262       return false;
1263     }
1264 
1265     HeapWord* nextTop = hr->next_top_at_mark_start();
1266     HeapWord* start   = hr->bottom();
1267 
1268     assert(start <= hr->end() && start <= nextTop && nextTop <= hr->end(),
1269            err_msg("Preconditions not met - "
1270                    "start: "PTR_FORMAT", nextTop: "PTR_FORMAT", end: "PTR_FORMAT,
1271                    start, nextTop, hr->end()));
1272 
1273     // Find the first marked object at or after "start".
1274     start = _bm->getNextMarkedWordAddress(start, nextTop);
1275 
1276     size_t marked_bytes = 0;
1277 
1278     while (start < nextTop) {
1279       oop obj = oop(start);
1280       int obj_sz = obj->size();
1281       HeapWord* obj_last = start + obj_sz - 1;
1282 
1283       BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start);
1284       BitMap::idx_t last_idx = _cm->card_bitmap_index_for(obj_last);










1285 
1286       // Set the bits in the card BM for this object (inclusive).
1287       set_card_bitmap_range(start_idx, last_idx);
1288 
1289       // Add the size of this object to the number of marked bytes.
1290       marked_bytes += (size_t)obj_sz * HeapWordSize;
1291 
1292       // Find the next marked object after this one.
1293       start = _bm->getNextMarkedWordAddress(obj_last + 1, nextTop);
1294     }
1295 
1296     // Mark the allocated-since-marking portion...
1297     HeapWord* top = hr->top();
1298     if (nextTop < top) {
1299       BitMap::idx_t start_idx = _cm->card_bitmap_index_for(nextTop);
1300       BitMap::idx_t last_idx = _cm->card_bitmap_index_for(top - 1);
1301 
1302       set_card_bitmap_range(start_idx, last_idx);









1303 
1304       // This definitely means the region has live objects.
1305       set_bit_for_region(hr);
1306     }
1307 
1308     // Update the live region bitmap.
1309     if (marked_bytes > 0) {
1310       set_bit_for_region(hr);
1311     }
1312 
1313     // Set the marked bytes for the current region so that
1314     // it can be queried by a calling verificiation routine
1315     _region_marked_bytes = marked_bytes;
1316 
1317     return false;
1318   }
1319 
1320   size_t region_marked_bytes() const { return _region_marked_bytes; }
1321 };
1322 
1323 // Heap region closure used for verifying the counting data
1324 // that was accumulated concurrently and aggregated during
1325 // the remark pause. This closure is applied to the heap
1326 // regions during the STW cleanup pause.
1327 
1328 class VerifyLiveObjectDataHRClosure: public HeapRegionClosure {

1329   ConcurrentMark* _cm;
1330   CalcLiveObjectsClosure _calc_cl;
1331   BitMap* _region_bm;   // Region BM to be verified
1332   BitMap* _card_bm;     // Card BM to be verified
1333   bool _verbose;        // verbose output?
1334 
1335   BitMap* _exp_region_bm; // Expected Region BM values
1336   BitMap* _exp_card_bm;   // Expected card BM values
1337 
1338   int _failures;
1339 
1340 public:
1341   VerifyLiveObjectDataHRClosure(ConcurrentMark* cm,
1342                                 BitMap* region_bm,
1343                                 BitMap* card_bm,
1344                                 BitMap* exp_region_bm,
1345                                 BitMap* exp_card_bm,
1346                                 bool verbose) :
1347     _cm(cm),
1348     _calc_cl(_cm->nextMarkBitMap(), _cm, exp_region_bm, exp_card_bm),
1349     _region_bm(region_bm), _card_bm(card_bm), _verbose(verbose),
1350     _exp_region_bm(exp_region_bm), _exp_card_bm(exp_card_bm),
1351     _failures(0) { }
1352 
1353   int failures() const { return _failures; }
1354 
1355   bool doHeapRegion(HeapRegion* hr) {
1356     if (hr->continuesHumongous()) {
1357       // We will ignore these here and process them when their
1358       // associated "starts humongous" region is processed (see
1359       // set_bit_for_heap_region()). Note that we cannot rely on their
1360       // associated "starts humongous" region to have their bit set to
1361       // 1 since, due to the region chunking in the parallel region
1362       // iteration, a "continues humongous" region might be visited
1363       // before its associated "starts humongous".
1364       return false;
1365     }
1366 
1367     int failures = 0;
1368 


1475 
1476     // Use the value already set as the number of active threads
1477     // in the call to run_task().
1478     if (G1CollectedHeap::use_parallel_gc_threads()) {
1479       assert( _g1h->workers()->active_workers() > 0,
1480         "Should have been previously set");
1481       _n_workers = _g1h->workers()->active_workers();
1482     } else {
1483       _n_workers = 1;
1484     }
1485 
1486     assert(_expected_card_bm->size() == _actual_card_bm->size(), "sanity");
1487     assert(_expected_region_bm->size() == _actual_region_bm->size(), "sanity");
1488 
1489     _verbose = _cm->verbose_medium();
1490   }
1491 
1492   void work(uint worker_id) {
1493     assert(worker_id < _n_workers, "invariant");
1494 
1495     VerifyLiveObjectDataHRClosure verify_cl(_cm,
1496                                             _actual_region_bm, _actual_card_bm,
1497                                             _expected_region_bm,
1498                                             _expected_card_bm,
1499                                             _verbose);
1500 
1501     if (G1CollectedHeap::use_parallel_gc_threads()) {
1502       _g1h->heap_region_par_iterate_chunked(&verify_cl,
1503                                             worker_id,
1504                                             _n_workers,
1505                                             HeapRegion::VerifyCountClaimValue);
1506     } else {
1507       _g1h->heap_region_iterate(&verify_cl);
1508     }
1509 
1510     Atomic::add(verify_cl.failures(), &_failures);
1511   }
1512 
1513   int failures() const { return _failures; }
1514 };
1515 
1516 // Closure that finalizes the liveness counting data.
1517 // Used during the cleanup pause.
1518 // Sets the bits corresponding to the interval [NTAMS, top]
1519 // (which contains the implicitly live objects) in the
1520 // card liveness bitmap. Also sets the bit for each region,
1521 // containing live data, in the region liveness bitmap.
1522 
1523 class FinalCountDataUpdateClosure: public CMCountDataClosureBase {
1524  public:
1525   FinalCountDataUpdateClosure(ConcurrentMark* cm,
1526                               BitMap* region_bm,
1527                               BitMap* card_bm) :
1528     CMCountDataClosureBase(cm, region_bm, card_bm) { }
1529 
1530   bool doHeapRegion(HeapRegion* hr) {
1531 
1532     if (hr->continuesHumongous()) {
1533       // We will ignore these here and process them when their
1534       // associated "starts humongous" region is processed (see
1535       // set_bit_for_heap_region()). Note that we cannot rely on their
1536       // associated "starts humongous" region to have their bit set to
1537       // 1 since, due to the region chunking in the parallel region
1538       // iteration, a "continues humongous" region might be visited
1539       // before its associated "starts humongous".
1540       return false;
1541     }
1542 
1543     HeapWord* ntams = hr->next_top_at_mark_start();
1544     HeapWord* top   = hr->top();
1545 
1546     assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions.");
1547 
1548     // Mark the allocated-since-marking portion...
1549     if (ntams < top) {
1550       // This definitely means the region has live objects.
1551       set_bit_for_region(hr);
1552     }
1553 
1554     // Now set the bits for [ntams, top]
1555     BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams);
1556     BitMap::idx_t last_idx = _cm->card_bitmap_index_for(top);
1557     set_card_bitmap_range(start_idx, last_idx);



















1558 
1559     // Set the bit for the region if it contains live data
1560     if (hr->next_marked_bytes() > 0) {
1561       set_bit_for_region(hr);
1562     }
1563 
1564     return false;
1565   }
1566 };
1567 
1568 class G1ParFinalCountTask: public AbstractGangTask {
1569 protected:
1570   G1CollectedHeap* _g1h;
1571   ConcurrentMark* _cm;
1572   BitMap* _actual_region_bm;
1573   BitMap* _actual_card_bm;
1574 
1575   uint    _n_workers;
1576 
1577 public:
1578   G1ParFinalCountTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm)
1579     : AbstractGangTask("G1 final counting"),
1580       _g1h(g1h), _cm(_g1h->concurrent_mark()),
1581       _actual_region_bm(region_bm), _actual_card_bm(card_bm),
1582       _n_workers(0) {
1583     // Use the value already set as the number of active threads
1584     // in the call to run_task().
1585     if (G1CollectedHeap::use_parallel_gc_threads()) {
1586       assert( _g1h->workers()->active_workers() > 0,
1587         "Should have been previously set");
1588       _n_workers = _g1h->workers()->active_workers();
1589     } else {
1590       _n_workers = 1;
1591     }
1592   }
1593 
1594   void work(uint worker_id) {
1595     assert(worker_id < _n_workers, "invariant");
1596 
1597     FinalCountDataUpdateClosure final_update_cl(_cm,
1598                                                 _actual_region_bm,
1599                                                 _actual_card_bm);
1600 
1601     if (G1CollectedHeap::use_parallel_gc_threads()) {
1602       _g1h->heap_region_par_iterate_chunked(&final_update_cl,
1603                                             worker_id,
1604                                             _n_workers,
1605                                             HeapRegion::FinalCountClaimValue);
1606     } else {
1607       _g1h->heap_region_iterate(&final_update_cl);
1608     }
1609   }
1610 };
1611 
1612 class G1ParNoteEndTask;
1613 
1614 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1615   G1CollectedHeap* _g1;
1616   int _worker_num;
1617   size_t _max_live_bytes;


2817 
2818 void ConcurrentMark::clear_marking_state(bool clear_overflow) {
2819   _markStack.setEmpty();
2820   _markStack.clear_overflow();
2821   if (clear_overflow) {
2822     clear_has_overflown();
2823   } else {
2824     assert(has_overflown(), "pre-condition");
2825   }
2826   _finger = _heap_start;
2827 
2828   for (int i = 0; i < (int)_max_task_num; ++i) {
2829     OopTaskQueue* queue = _task_queues->queue(i);
2830     queue->set_empty();
2831   }
2832 }
2833 
2834 // Aggregate the counting data that was constructed concurrently
2835 // with marking.
2836 class AggregateCountDataHRClosure: public HeapRegionClosure {

2837   ConcurrentMark* _cm;

2838   BitMap* _cm_card_bm;
2839   size_t _max_task_num;
2840 
2841  public:
2842   AggregateCountDataHRClosure(ConcurrentMark *cm,
2843                               BitMap* cm_card_bm,
2844                               size_t max_task_num) :
2845     _cm(cm), _cm_card_bm(cm_card_bm),
2846     _max_task_num(max_task_num) { }
2847 
2848   bool is_card_aligned(HeapWord* p) {
2849     return ((uintptr_t(p) & (CardTableModRefBS::card_size - 1)) == 0);
2850   }
2851 
2852   bool doHeapRegion(HeapRegion* hr) {
2853     if (hr->continuesHumongous()) {
2854       // We will ignore these here and process them when their
2855       // associated "starts humongous" region is processed.
2856       // Note that we cannot rely on their associated
2857       // "starts humongous" region to have their bit set to 1
2858       // since, due to the region chunking in the parallel region
2859       // iteration, a "continues humongous" region might be visited
2860       // before its associated "starts humongous".
2861       return false;
2862     }
2863 
2864     HeapWord* start = hr->bottom();
2865     HeapWord* limit = hr->next_top_at_mark_start();
2866     HeapWord* end = hr->end();
2867 
2868     assert(start <= limit && limit <= hr->top() && hr->top() <= hr->end(),
2869            err_msg("Preconditions not met - "
2870                    "start: "PTR_FORMAT", limit: "PTR_FORMAT", "
2871                    "top: "PTR_FORMAT", end: "PTR_FORMAT,
2872                    start, limit, hr->top(), hr->end()));
2873 
2874     assert(hr->next_marked_bytes() == 0, "Precondition");
2875 
2876     if (start == limit) {
2877       // NTAMS of this region has not been set so nothing to do.
2878       return false;
2879     }
2880 
2881     assert(is_card_aligned(start), "sanity");
2882     assert(is_card_aligned(end), "sanity");


2883 
2884     BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start);
2885     BitMap::idx_t limit_idx = _cm->card_bitmap_index_for(limit);
2886     BitMap::idx_t end_idx = _cm->card_bitmap_index_for(end);
2887 
2888     // If ntams is not card aligned then we bump the index for
2889     // limit so that we get the card spanning ntams.
2890     if (!is_card_aligned(limit)) {





2891       limit_idx += 1;
2892     }
2893 
2894     assert(limit_idx <= end_idx, "or else use atomics");
2895 
2896     // Aggregate the "stripe" in the count data associated with hr.
2897     uint hrs_index = hr->hrs_index();
2898     size_t marked_bytes = 0;
2899 
2900     for (int i = 0; (size_t)i < _max_task_num; i += 1) {
2901       size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i);
2902       BitMap* task_card_bm = _cm->count_card_bitmap_for(i);
2903 
2904       // Fetch the marked_bytes in this region for task i and
2905       // add it to the running total for this region.
2906       marked_bytes += marked_bytes_array[hrs_index];
2907 
2908       // Now union the bitmaps[0,max_task_num)[start_idx..limit_idx)
2909       // into the global card bitmap.
2910       BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx);
2911 
2912       while (scan_idx < limit_idx) {
2913         assert(task_card_bm->at(scan_idx) == true, "should be");
2914         _cm_card_bm->set_bit(scan_idx);
2915         assert(_cm_card_bm->at(scan_idx) == true, "should be");
2916 
2917         // BitMap::get_next_one_offset() can handle the case when
2918         // its left_offset parameter is greater than its right_offset
2919         // parameter. If does, however, have an early exit if
2920         // left_offset == right_offset. So let's limit the value
2921         // passed in for left offset here.
2922         BitMap::idx_t next_idx = MIN2(scan_idx + 1, limit_idx);
2923         scan_idx = task_card_bm->get_next_one_offset(next_idx, limit_idx);
2924       }
2925     }
2926 
2927     // Update the marked bytes for this region.
2928     hr->add_to_marked_bytes(marked_bytes);
2929 
2930     // Next heap region
2931     return false;
2932   }
2933 };
2934 
2935 class G1AggregateCountDataTask: public AbstractGangTask {
2936 protected:
2937   G1CollectedHeap* _g1h;
2938   ConcurrentMark* _cm;
2939   BitMap* _cm_card_bm;
2940   size_t _max_task_num;
2941   int _active_workers;
2942 
2943 public:
2944   G1AggregateCountDataTask(G1CollectedHeap* g1h,
2945                            ConcurrentMark* cm,
2946                            BitMap* cm_card_bm,
2947                            size_t max_task_num,
2948                            int n_workers) :
2949     AbstractGangTask("Count Aggregation"),
2950     _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm),
2951     _max_task_num(max_task_num),
2952     _active_workers(n_workers) { }
2953 
2954   void work(uint worker_id) {
2955     AggregateCountDataHRClosure cl(_cm, _cm_card_bm, _max_task_num);
2956 
2957     if (G1CollectedHeap::use_parallel_gc_threads()) {
2958       _g1h->heap_region_par_iterate_chunked(&cl, worker_id,
2959                                             _active_workers,
2960                                             HeapRegion::AggregateCountClaimValue);
2961     } else {
2962       _g1h->heap_region_iterate(&cl);
2963     }
2964   }
2965 };
2966 
2967 
2968 void ConcurrentMark::aggregate_count_data() {
2969   int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
2970                         _g1h->workers()->active_workers() :
2971                         1);
2972 
2973   G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm,
2974                                            _max_task_num, n_workers);
2975 




1173   }
1174 
1175 #if VERIFY_OBJS_PROCESSED
1176   _scan_obj_cl.objs_processed = 0;
1177   ThreadLocalObjQueue::objs_enqueued = 0;
1178 #endif
1179 
1180   // Statistics
1181   double now = os::elapsedTime();
1182   _remark_mark_times.add((mark_work_end - start) * 1000.0);
1183   _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1184   _remark_times.add((now - start) * 1000.0);
1185 
1186   g1p->record_concurrent_mark_remark_end();
1187 }
1188 
1189 // Base class of the closures that finalize and verify the
1190 // liveness counting data.
1191 class CMCountDataClosureBase: public HeapRegionClosure {
1192 protected:
1193   G1CollectedHeap* _g1h;
1194   ConcurrentMark* _cm;
1195   CardTableModRefBS* _ct_bs;
1196 
1197   BitMap* _region_bm;
1198   BitMap* _card_bm;
1199 
1200   // Takes a region that's not empty (i.e., it has at least one

















1201   // live object in it and sets its corresponding bit on the region
1202   // bitmap to 1. If the region is "starts humongous" it will also set
1203   // to 1 the bits on the region bitmap that correspond to its
1204   // associated "continues humongous" regions.
1205   void set_bit_for_region(HeapRegion* hr) {
1206     assert(!hr->continuesHumongous(), "should have filtered those out");
1207 
1208     BitMap::idx_t index = (BitMap::idx_t) hr->hrs_index();
1209     if (!hr->startsHumongous()) {
1210       // Normal (non-humongous) case: just set the bit.
1211       _region_bm->par_at_put(index, true);
1212     } else {
1213       // Starts humongous case: calculate how many regions are part of
1214       // this humongous region and then set the bit range.
1215       BitMap::idx_t end_index = (BitMap::idx_t) hr->last_hc_index();
1216       _region_bm->par_at_put_range(index, end_index, true);
1217     }
1218   }
1219 
1220 public:
1221   CMCountDataClosureBase(G1CollectedHeap* g1h,
1222                          BitMap* region_bm, BitMap* card_bm):
1223     _g1h(g1h), _cm(g1h->concurrent_mark()),
1224     _ct_bs((CardTableModRefBS*) (g1h->barrier_set())),
1225     _region_bm(region_bm), _card_bm(card_bm) { }
1226 };
1227 
1228 // Closure that calculates the # live objects per region. Used
1229 // for verification purposes during the cleanup pause.
1230 class CalcLiveObjectsClosure: public CMCountDataClosureBase {
1231   CMBitMapRO* _bm;
1232   size_t _region_marked_bytes;
1233 
1234 public:
1235   CalcLiveObjectsClosure(CMBitMapRO *bm, G1CollectedHeap* g1h,
1236                          BitMap* region_bm, BitMap* card_bm) :
1237     CMCountDataClosureBase(g1h, region_bm, card_bm),
1238     _bm(bm), _region_marked_bytes(0) { }
1239 
1240   bool doHeapRegion(HeapRegion* hr) {
1241 
1242     if (hr->continuesHumongous()) {
1243       // We will ignore these here and process them when their
1244       // associated "starts humongous" region is processed (see
1245       // set_bit_for_heap_region()). Note that we cannot rely on their
1246       // associated "starts humongous" region to have their bit set to
1247       // 1 since, due to the region chunking in the parallel region
1248       // iteration, a "continues humongous" region might be visited
1249       // before its associated "starts humongous".
1250       return false;
1251     }
1252 
1253     HeapWord* ntams = hr->next_top_at_mark_start();
1254     HeapWord* start = hr->bottom();
1255 
1256     assert(start <= hr->end() && start <= ntams && ntams <= hr->end(),
1257            err_msg("Preconditions not met - "
1258                    "start: "PTR_FORMAT", ntams: "PTR_FORMAT", end: "PTR_FORMAT,
1259                    start, ntams, hr->end()));
1260 
1261     // Find the first marked object at or after "start".
1262     start = _bm->getNextMarkedWordAddress(start, ntams);
1263 
1264     size_t marked_bytes = 0;
1265 
1266     while (start < ntams) {
1267       oop obj = oop(start);
1268       int obj_sz = obj->size();
1269       HeapWord* obj_end = start + obj_sz;
1270 
1271       BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start);
1272       BitMap::idx_t end_idx = _cm->card_bitmap_index_for(obj_end);
1273 
1274       // Note: if we're looking at the last region in heap - obj_end
1275       // could be actually just beyond the end of the heap; end_idx
1276       // will then correspond to a (non-existent) card that is also
1277       // just beyond the heap.
1278       if (_g1h->is_in_g1_reserved(obj_end) && !_ct_bs->is_card_aligned(obj_end)) {
1279         // end of object is not card aligned - increment to cover
1280         // all the cards spanned by the object
1281         end_idx += 1;
1282       }
1283 
1284       // Set the bits in the card BM for the cards spanned by this object.
1285       _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */);
1286 
1287       // Add the size of this object to the number of marked bytes.
1288       marked_bytes += (size_t)obj_sz * HeapWordSize;
1289 
1290       // Find the next marked object after this one.
1291       start = _bm->getNextMarkedWordAddress(obj_end, ntams);
1292     }
1293 
1294     // Mark the allocated-since-marking portion...
1295     HeapWord* top = hr->top();
1296     if (ntams < top) {
1297       BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams);
1298       BitMap::idx_t end_idx = _cm->card_bitmap_index_for(top);
1299 
1300       // Note: if we're looking at the last region in heap - top
1301       // could be actually just beyond the end of the heap; end_idx
1302       // will then correspond to a (non-existent) card that is also
1303       // just beyond the heap.
1304       if (_g1h->is_in_g1_reserved(top) && !_ct_bs->is_card_aligned(top)) {
1305         // end of object is not card aligned - increment to cover
1306         // all the cards spanned by the object
1307         end_idx += 1;
1308       }
1309       _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */);
1310 
1311       // This definitely means the region has live objects.
1312       set_bit_for_region(hr);
1313     }
1314 
1315     // Update the live region bitmap.
1316     if (marked_bytes > 0) {
1317       set_bit_for_region(hr);
1318     }
1319 
1320     // Set the marked bytes for the current region so that
1321     // it can be queried by a calling verificiation routine
1322     _region_marked_bytes = marked_bytes;
1323 
1324     return false;
1325   }
1326 
1327   size_t region_marked_bytes() const { return _region_marked_bytes; }
1328 };
1329 
1330 // Heap region closure used for verifying the counting data
1331 // that was accumulated concurrently and aggregated during
1332 // the remark pause. This closure is applied to the heap
1333 // regions during the STW cleanup pause.
1334 
1335 class VerifyLiveObjectDataHRClosure: public HeapRegionClosure {
1336   G1CollectedHeap* _g1h;
1337   ConcurrentMark* _cm;
1338   CalcLiveObjectsClosure _calc_cl;
1339   BitMap* _region_bm;   // Region BM to be verified
1340   BitMap* _card_bm;     // Card BM to be verified
1341   bool _verbose;        // verbose output?
1342 
1343   BitMap* _exp_region_bm; // Expected Region BM values
1344   BitMap* _exp_card_bm;   // Expected card BM values
1345 
1346   int _failures;
1347 
1348 public:
1349   VerifyLiveObjectDataHRClosure(G1CollectedHeap* g1h,
1350                                 BitMap* region_bm,
1351                                 BitMap* card_bm,
1352                                 BitMap* exp_region_bm,
1353                                 BitMap* exp_card_bm,
1354                                 bool verbose) :
1355     _g1h(g1h), _cm(g1h->concurrent_mark()),
1356     _calc_cl(_cm->nextMarkBitMap(), g1h, exp_region_bm, exp_card_bm),
1357     _region_bm(region_bm), _card_bm(card_bm), _verbose(verbose),
1358     _exp_region_bm(exp_region_bm), _exp_card_bm(exp_card_bm),
1359     _failures(0) { }
1360 
1361   int failures() const { return _failures; }
1362 
1363   bool doHeapRegion(HeapRegion* hr) {
1364     if (hr->continuesHumongous()) {
1365       // We will ignore these here and process them when their
1366       // associated "starts humongous" region is processed (see
1367       // set_bit_for_heap_region()). Note that we cannot rely on their
1368       // associated "starts humongous" region to have their bit set to
1369       // 1 since, due to the region chunking in the parallel region
1370       // iteration, a "continues humongous" region might be visited
1371       // before its associated "starts humongous".
1372       return false;
1373     }
1374 
1375     int failures = 0;
1376 


1483 
1484     // Use the value already set as the number of active threads
1485     // in the call to run_task().
1486     if (G1CollectedHeap::use_parallel_gc_threads()) {
1487       assert( _g1h->workers()->active_workers() > 0,
1488         "Should have been previously set");
1489       _n_workers = _g1h->workers()->active_workers();
1490     } else {
1491       _n_workers = 1;
1492     }
1493 
1494     assert(_expected_card_bm->size() == _actual_card_bm->size(), "sanity");
1495     assert(_expected_region_bm->size() == _actual_region_bm->size(), "sanity");
1496 
1497     _verbose = _cm->verbose_medium();
1498   }
1499 
1500   void work(uint worker_id) {
1501     assert(worker_id < _n_workers, "invariant");
1502 
1503     VerifyLiveObjectDataHRClosure verify_cl(_g1h,
1504                                             _actual_region_bm, _actual_card_bm,
1505                                             _expected_region_bm,
1506                                             _expected_card_bm,
1507                                             _verbose);
1508 
1509     if (G1CollectedHeap::use_parallel_gc_threads()) {
1510       _g1h->heap_region_par_iterate_chunked(&verify_cl,
1511                                             worker_id,
1512                                             _n_workers,
1513                                             HeapRegion::VerifyCountClaimValue);
1514     } else {
1515       _g1h->heap_region_iterate(&verify_cl);
1516     }
1517 
1518     Atomic::add(verify_cl.failures(), &_failures);
1519   }
1520 
1521   int failures() const { return _failures; }
1522 };
1523 
1524 // Closure that finalizes the liveness counting data.
1525 // Used during the cleanup pause.
1526 // Sets the bits corresponding to the interval [NTAMS, top]
1527 // (which contains the implicitly live objects) in the
1528 // card liveness bitmap. Also sets the bit for each region,
1529 // containing live data, in the region liveness bitmap.
1530 
1531 class FinalCountDataUpdateClosure: public CMCountDataClosureBase {
1532  public:
1533   FinalCountDataUpdateClosure(G1CollectedHeap* g1h,
1534                               BitMap* region_bm,
1535                               BitMap* card_bm) :
1536     CMCountDataClosureBase(g1h, region_bm, card_bm) { }
1537 
1538   bool doHeapRegion(HeapRegion* hr) {
1539 
1540     if (hr->continuesHumongous()) {
1541       // We will ignore these here and process them when their
1542       // associated "starts humongous" region is processed (see
1543       // set_bit_for_heap_region()). Note that we cannot rely on their
1544       // associated "starts humongous" region to have their bit set to
1545       // 1 since, due to the region chunking in the parallel region
1546       // iteration, a "continues humongous" region might be visited
1547       // before its associated "starts humongous".
1548       return false;
1549     }
1550 
1551     HeapWord* ntams = hr->next_top_at_mark_start();
1552     HeapWord* top   = hr->top();
1553 
1554     assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions.");
1555 
1556     // Mark the allocated-since-marking portion...
1557     if (ntams < top) {
1558       // This definitely means the region has live objects.
1559       set_bit_for_region(hr);

1560 
1561       // Now set the bits in the card bitmap for [ntams, top)
1562       BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams);
1563       BitMap::idx_t end_idx = _cm->card_bitmap_index_for(top);
1564 
1565       // Note: if we're looking at the last region in heap - top
1566       // could be actually just beyond the end of the heap; end_idx
1567       // will then correspond to a (non-existent) card that is also
1568       // just beyond the heap.
1569       if (_g1h->is_in_g1_reserved(top) && !_ct_bs->is_card_aligned(top)) {
1570         // end of object is not card aligned - increment to cover
1571         // all the cards spanned by the object
1572         end_idx += 1;
1573       }
1574 
1575       assert(end_idx <= _card_bm->size(),
1576              err_msg("oob: end_idx=  "SIZE_FORMAT", bitmap size= "SIZE_FORMAT,
1577                      end_idx, _card_bm->size()));
1578       assert(start_idx < _card_bm->size(),
1579              err_msg("oob: start_idx=  "SIZE_FORMAT", bitmap size= "SIZE_FORMAT,
1580                      start_idx, _card_bm->size()));
1581 
1582       _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */);
1583      }
1584 
1585     // Set the bit for the region if it contains live data
1586     if (hr->next_marked_bytes() > 0) {
1587       set_bit_for_region(hr);
1588     }
1589 
1590     return false;
1591   }
1592 };
1593 
1594 class G1ParFinalCountTask: public AbstractGangTask {
1595 protected:
1596   G1CollectedHeap* _g1h;
1597   ConcurrentMark* _cm;
1598   BitMap* _actual_region_bm;
1599   BitMap* _actual_card_bm;
1600 
1601   uint    _n_workers;
1602 
1603 public:
1604   G1ParFinalCountTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm)
1605     : AbstractGangTask("G1 final counting"),
1606       _g1h(g1h), _cm(_g1h->concurrent_mark()),
1607       _actual_region_bm(region_bm), _actual_card_bm(card_bm),
1608       _n_workers(0) {
1609     // Use the value already set as the number of active threads
1610     // in the call to run_task().
1611     if (G1CollectedHeap::use_parallel_gc_threads()) {
1612       assert( _g1h->workers()->active_workers() > 0,
1613         "Should have been previously set");
1614       _n_workers = _g1h->workers()->active_workers();
1615     } else {
1616       _n_workers = 1;
1617     }
1618   }
1619 
1620   void work(uint worker_id) {
1621     assert(worker_id < _n_workers, "invariant");
1622 
1623     FinalCountDataUpdateClosure final_update_cl(_g1h,
1624                                                 _actual_region_bm,
1625                                                 _actual_card_bm);
1626 
1627     if (G1CollectedHeap::use_parallel_gc_threads()) {
1628       _g1h->heap_region_par_iterate_chunked(&final_update_cl,
1629                                             worker_id,
1630                                             _n_workers,
1631                                             HeapRegion::FinalCountClaimValue);
1632     } else {
1633       _g1h->heap_region_iterate(&final_update_cl);
1634     }
1635   }
1636 };
1637 
1638 class G1ParNoteEndTask;
1639 
1640 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1641   G1CollectedHeap* _g1;
1642   int _worker_num;
1643   size_t _max_live_bytes;


2843 
2844 void ConcurrentMark::clear_marking_state(bool clear_overflow) {
2845   _markStack.setEmpty();
2846   _markStack.clear_overflow();
2847   if (clear_overflow) {
2848     clear_has_overflown();
2849   } else {
2850     assert(has_overflown(), "pre-condition");
2851   }
2852   _finger = _heap_start;
2853 
2854   for (int i = 0; i < (int)_max_task_num; ++i) {
2855     OopTaskQueue* queue = _task_queues->queue(i);
2856     queue->set_empty();
2857   }
2858 }
2859 
2860 // Aggregate the counting data that was constructed concurrently
2861 // with marking.
2862 class AggregateCountDataHRClosure: public HeapRegionClosure {
2863   G1CollectedHeap* _g1h;
2864   ConcurrentMark* _cm;
2865   CardTableModRefBS* _ct_bs;
2866   BitMap* _cm_card_bm;
2867   size_t _max_task_num;
2868 
2869  public:
2870   AggregateCountDataHRClosure(G1CollectedHeap* g1h,
2871                               BitMap* cm_card_bm,
2872                               size_t max_task_num) :
2873     _g1h(g1h), _cm(g1h->concurrent_mark()),
2874     _ct_bs((CardTableModRefBS*) (g1h->barrier_set())),
2875     _cm_card_bm(cm_card_bm), _max_task_num(max_task_num) { }



2876 
2877   bool doHeapRegion(HeapRegion* hr) {
2878     if (hr->continuesHumongous()) {
2879       // We will ignore these here and process them when their
2880       // associated "starts humongous" region is processed.
2881       // Note that we cannot rely on their associated
2882       // "starts humongous" region to have their bit set to 1
2883       // since, due to the region chunking in the parallel region
2884       // iteration, a "continues humongous" region might be visited
2885       // before its associated "starts humongous".
2886       return false;
2887     }
2888 
2889     HeapWord* start = hr->bottom();
2890     HeapWord* limit = hr->next_top_at_mark_start();
2891     HeapWord* end = hr->end();
2892 
2893     assert(start <= limit && limit <= hr->top() && hr->top() <= hr->end(),
2894            err_msg("Preconditions not met - "
2895                    "start: "PTR_FORMAT", limit: "PTR_FORMAT", "
2896                    "top: "PTR_FORMAT", end: "PTR_FORMAT,
2897                    start, limit, hr->top(), hr->end()));
2898 
2899     assert(hr->next_marked_bytes() == 0, "Precondition");
2900 
2901     if (start == limit) {
2902       // NTAMS of this region has not been set so nothing to do.
2903       return false;
2904     }
2905 
2906     // 'start' should be in the heap.
2907     assert(_g1h->is_in_g1_reserved(start) && _ct_bs->is_card_aligned(start), "sanity");
2908     // 'end' *may* be just beyone the end of the heap (if hr is the last region)
2909     assert(!_g1h->is_in_g1_reserved(end) || _ct_bs->is_card_aligned(end), "sanity");
2910 
2911     BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start);
2912     BitMap::idx_t limit_idx = _cm->card_bitmap_index_for(limit);
2913     BitMap::idx_t end_idx = _cm->card_bitmap_index_for(end);
2914 
2915     // If ntams is not card aligned then we bump card bitmap index
2916     // for limit so that we get the all the cards spanned by
2917     // the object ending at ntams.
2918     // Note: if this is the last region in the heap then ntams
2919     // could be actually just beyond the end of the the heap;
2920     // limit_idx will then  correspond to a (non-existent) card
2921     // that is also outside the heap.
2922     if (_g1h->is_in_g1_reserved(limit) && !_ct_bs->is_card_aligned(limit)) {
2923       limit_idx += 1;
2924     }
2925 
2926     assert(limit_idx <= end_idx, "or else use atomics");
2927 
2928     // Aggregate the "stripe" in the count data associated with hr.
2929     uint hrs_index = hr->hrs_index();
2930     size_t marked_bytes = 0;
2931 
2932     for (int i = 0; (size_t)i < _max_task_num; i += 1) {
2933       size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i);
2934       BitMap* task_card_bm = _cm->count_card_bitmap_for(i);
2935 
2936       // Fetch the marked_bytes in this region for task i and
2937       // add it to the running total for this region.
2938       marked_bytes += marked_bytes_array[hrs_index];
2939 
2940       // Now union the bitmaps[0,max_task_num)[start_idx..limit_idx)
2941       // into the global card bitmap.
2942       BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx);
2943 
2944       while (scan_idx < limit_idx) {
2945         assert(task_card_bm->at(scan_idx) == true, "should be");
2946         _cm_card_bm->set_bit(scan_idx);
2947         assert(_cm_card_bm->at(scan_idx) == true, "should be");
2948 
2949         // BitMap::get_next_one_offset() can handle the case when
2950         // its left_offset parameter is greater than its right_offset
2951         // parameter. It does, however, have an early exit if
2952         // left_offset == right_offset. So let's limit the value
2953         // passed in for left offset here.
2954         BitMap::idx_t next_idx = MIN2(scan_idx + 1, limit_idx);
2955         scan_idx = task_card_bm->get_next_one_offset(next_idx, limit_idx);
2956       }
2957     }
2958 
2959     // Update the marked bytes for this region.
2960     hr->add_to_marked_bytes(marked_bytes);
2961 
2962     // Next heap region
2963     return false;
2964   }
2965 };
2966 
2967 class G1AggregateCountDataTask: public AbstractGangTask {
2968 protected:
2969   G1CollectedHeap* _g1h;
2970   ConcurrentMark* _cm;
2971   BitMap* _cm_card_bm;
2972   size_t _max_task_num;
2973   int _active_workers;
2974 
2975 public:
2976   G1AggregateCountDataTask(G1CollectedHeap* g1h,
2977                            ConcurrentMark* cm,
2978                            BitMap* cm_card_bm,
2979                            size_t max_task_num,
2980                            int n_workers) :
2981     AbstractGangTask("Count Aggregation"),
2982     _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm),
2983     _max_task_num(max_task_num),
2984     _active_workers(n_workers) { }
2985 
2986   void work(uint worker_id) {
2987     AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_task_num);
2988 
2989     if (G1CollectedHeap::use_parallel_gc_threads()) {
2990       _g1h->heap_region_par_iterate_chunked(&cl, worker_id,
2991                                             _active_workers,
2992                                             HeapRegion::AggregateCountClaimValue);
2993     } else {
2994       _g1h->heap_region_iterate(&cl);
2995     }
2996   }
2997 };
2998 
2999 
3000 void ConcurrentMark::aggregate_count_data() {
3001   int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
3002                         _g1h->workers()->active_workers() :
3003                         1);
3004 
3005   G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm,
3006                                            _max_task_num, n_workers);
3007