303 // Index of last region in the series + 1.
304 uint last = first + num_regions;
305
306 // We need to initialize the region(s) we just discovered. This is
307 // a bit tricky given that it can happen concurrently with
308 // refinement threads refining cards on these regions and
309 // potentially wanting to refine the BOT as they are scanning
310 // those cards (this can happen shortly after a cleanup; see CR
311 // 6991377). So we have to set up the region(s) carefully and in
312 // a specific order.
313
314 // The word size sum of all the regions we will allocate.
315 size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
316 assert(word_size <= word_size_sum, "sanity");
317
318 // This will be the "starts humongous" region.
319 HeapRegion* first_hr = region_at(first);
320 // The header of the new object will be placed at the bottom of
321 // the first region.
322 HeapWord* new_obj = first_hr->bottom();
323 // This will be the new end of the first region in the series that
324 // should also match the end of the last region in the series.
325 HeapWord* new_end = new_obj + word_size_sum;
326 // This will be the new top of the first region that will reflect
327 // this allocation.
328 HeapWord* new_top = new_obj + word_size;
329
330 // First, we need to zero the header of the space that we will be
331 // allocating. When we update top further down, some refinement
332 // threads might try to scan the region. By zeroing the header we
333 // ensure that any thread that will try to scan the region will
334 // come across the zero klass word and bail out.
335 //
336 // NOTE: It would not have been correct to have used
337 // CollectedHeap::fill_with_object() and make the space look like
338 // an int array. The thread that is doing the allocation will
339 // later update the object header to a potentially different array
340 // type and, for a very short period of time, the klass and length
341 // fields will be inconsistent. This could cause a refinement
342 // thread to calculate the object size incorrectly.
343 Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
344
345 // We will set up the first region as "starts humongous". This
346 // will also update the BOT covering all the regions to reflect
347 // that there is a single object that starts at the bottom of the
348 // first region.
349 first_hr->set_starts_humongous(new_top, new_end);
350 first_hr->set_allocation_context(context);
351 // Then, if there are any, we will set up the "continues
352 // humongous" regions.
353 HeapRegion* hr = NULL;
354 for (uint i = first + 1; i < last; ++i) {
355 hr = region_at(i);
356 hr->set_continues_humongous(first_hr);
357 hr->set_allocation_context(context);
358 }
359 // If we have "continues humongous" regions (hr != NULL), then the
360 // end of the last one should match new_end.
361 assert(hr == NULL || hr->end() == new_end, "sanity");
362
363 // Up to this point no concurrent thread would have been able to
364 // do any scanning on any region in this series. All the top
365 // fields still point to bottom, so the intersection between
366 // [bottom,top] and [card_start,card_end] will be empty. Before we
367 // update the top fields, we'll do a storestore to make sure that
368 // no thread sees the update to top before the zeroing of the
369 // object header and the BOT initialization.
370 OrderAccess::storestore();
371
372 // Now that the BOT and the object header have been initialized,
373 // we can update top of the "starts humongous" region.
374 assert(first_hr->bottom() < new_top && new_top <= first_hr->end(),
375 "new_top should be in this region");
376 first_hr->set_top(new_top);
377 if (_hr_printer.is_active()) {
378 HeapWord* bottom = first_hr->bottom();
379 HeapWord* end = first_hr->orig_end();
380 if ((first + 1) == last) {
381 // the series has a single humongous region
382 _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, new_top);
383 } else {
384 // the series has more than one humongous regions
385 _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, end);
386 }
387 }
388
389 // Now, we will update the top fields of the "continues humongous"
390 // regions. The reason we need to do this is that, otherwise,
391 // these regions would look empty and this will confuse parts of
392 // G1. For example, the code that looks for a consecutive number
393 // of empty regions will consider them empty and try to
394 // re-allocate them. We can extend is_empty() to also include
395 // !is_continues_humongous(), but it is easier to just update the top
396 // fields here. The way we set top for all regions (i.e., top ==
397 // end for all regions but the last one, top == new_top for the
398 // last one) is actually used when we will free up the humongous
399 // region in free_humongous_region().
400 hr = NULL;
401 for (uint i = first + 1; i < last; ++i) {
402 hr = region_at(i);
403 if ((i + 1) == last) {
404 // last continues humongous region
405 assert(hr->bottom() < new_top && new_top <= hr->end(),
406 "new_top should fall on this region");
407 hr->set_top(new_top);
408 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, new_top);
409 } else {
410 // not last one
411 assert(new_top > hr->end(), "new_top should be above this region");
412 hr->set_top(hr->end());
413 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end());
414 }
415 }
416 // If we have continues humongous regions (hr != NULL), then the
417 // end of the last one should match new_end and its top should
418 // match new_top.
419 assert(hr == NULL ||
420 (hr->end() == new_end && hr->top() == new_top), "sanity");
421 check_bitmaps("Humongous Region Allocation", first_hr);
422
423 assert(first_hr->used() == word_size * HeapWordSize, "invariant");
424 increase_used(first_hr->used());
425 _humongous_set.add(first_hr);
426
427 return new_obj;
428 }
429
430 // If could fit into free regions w/o expansion, try.
431 // Otherwise, if can expand, do so.
432 // Otherwise, if using ex regions might help, try with ex given back.
433 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) {
434 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
435
436 verify_region_sets_optional();
437
438 uint first = G1_NO_HRM_INDEX;
439 uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords);
440
441 if (obj_regions == 1) {
442 // Only one region to allocate, try to use a fast path by directly allocating
443 // from the free lists. Do not try to expand here, we will potentially do that
444 // later.
445 HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */);
1122 HeapWord* result = humongous_obj_allocate(word_size, context);
1123 if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) {
1124 collector_state()->set_initiate_conc_mark_if_possible(true);
1125 }
1126 return result;
1127 }
1128
1129 ShouldNotReachHere();
1130 }
1131
1132 class PostMCRemSetClearClosure: public HeapRegionClosure {
1133 G1CollectedHeap* _g1h;
1134 ModRefBarrierSet* _mr_bs;
1135 public:
1136 PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) :
1137 _g1h(g1h), _mr_bs(mr_bs) {}
1138
1139 bool doHeapRegion(HeapRegion* r) {
1140 HeapRegionRemSet* hrrs = r->rem_set();
1141
1142 if (r->is_continues_humongous()) {
1143 // We'll assert that the strong code root list and RSet is empty
1144 assert(hrrs->strong_code_roots_list_length() == 0, "sanity");
1145 assert(hrrs->occupied() == 0, "RSet should be empty");
1146 return false;
1147 }
1148
1149 _g1h->reset_gc_time_stamps(r);
1150 hrrs->clear();
1151 // You might think here that we could clear just the cards
1152 // corresponding to the used region. But no: if we leave a dirty card
1153 // in a region we might allocate into, then it would prevent that card
1154 // from being enqueued, and cause it to be missed.
1155 // Re: the performance cost: we shouldn't be doing full GC anyway!
1156 _mr_bs->clear(MemRegion(r->bottom(), r->end()));
1157
1158 return false;
1159 }
1160 };
1161
1162 void G1CollectedHeap::clear_rsets_post_compaction() {
1163 PostMCRemSetClearClosure rs_clear(this, g1_barrier_set());
1164 heap_region_iterate(&rs_clear);
1165 }
1166
1167 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
1168 G1CollectedHeap* _g1h;
1188
1189 public:
1190 ParRebuildRSTask(G1CollectedHeap* g1) :
1191 AbstractGangTask("ParRebuildRSTask"), _g1(g1), _hrclaimer(g1->workers()->active_workers()) {}
1192
1193 void work(uint worker_id) {
1194 RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id);
1195 _g1->heap_region_par_iterate(&rebuild_rs, worker_id, &_hrclaimer);
1196 }
1197 };
1198
1199 class PostCompactionPrinterClosure: public HeapRegionClosure {
1200 private:
1201 G1HRPrinter* _hr_printer;
1202 public:
1203 bool doHeapRegion(HeapRegion* hr) {
1204 assert(!hr->is_young(), "not expecting to find young regions");
1205 if (hr->is_free()) {
1206 // We only generate output for non-empty regions.
1207 } else if (hr->is_starts_humongous()) {
1208 if (hr->region_num() == 1) {
1209 // single humongous region
1210 _hr_printer->post_compaction(hr, G1HRPrinter::SingleHumongous);
1211 } else {
1212 _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous);
1213 }
1214 } else if (hr->is_continues_humongous()) {
1215 _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous);
1216 } else if (hr->is_archive()) {
1217 _hr_printer->post_compaction(hr, G1HRPrinter::Archive);
1218 } else if (hr->is_old()) {
1219 _hr_printer->post_compaction(hr, G1HRPrinter::Old);
1220 } else {
1221 ShouldNotReachHere();
1222 }
1223 return false;
1224 }
1225
1226 PostCompactionPrinterClosure(G1HRPrinter* hr_printer)
1227 : _hr_printer(hr_printer) { }
1228 };
1229
1230 void G1CollectedHeap::print_hrm_post_compaction() {
1231 PostCompactionPrinterClosure cl(hr_printer());
1232 heap_region_iterate(&cl);
1233 }
2205 (ParallelGCThreads > 1),
2206 // mt discovery
2207 ParallelGCThreads,
2208 // degree of mt discovery
2209 true,
2210 // Reference discovery is atomic
2211 &_is_alive_closure_stw);
2212 // is alive closure
2213 // (for efficiency/performance)
2214 }
2215
2216 CollectorPolicy* G1CollectedHeap::collector_policy() const {
2217 return g1_policy();
2218 }
2219
2220 size_t G1CollectedHeap::capacity() const {
2221 return _hrm.length() * HeapRegion::GrainBytes;
2222 }
2223
2224 void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) {
2225 assert(!hr->is_continues_humongous(), "pre-condition");
2226 hr->reset_gc_time_stamp();
2227 if (hr->is_starts_humongous()) {
2228 uint first_index = hr->hrm_index() + 1;
2229 uint last_index = hr->last_hc_index();
2230 for (uint i = first_index; i < last_index; i += 1) {
2231 HeapRegion* chr = region_at(i);
2232 assert(chr->is_continues_humongous(), "sanity");
2233 chr->reset_gc_time_stamp();
2234 }
2235 }
2236 }
2237
2238 #ifndef PRODUCT
2239
2240 class CheckGCTimeStampsHRClosure : public HeapRegionClosure {
2241 private:
2242 unsigned _gc_time_stamp;
2243 bool _failures;
2244
2245 public:
2246 CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) :
2247 _gc_time_stamp(gc_time_stamp), _failures(false) { }
2248
2249 virtual bool doHeapRegion(HeapRegion* hr) {
2250 unsigned region_gc_time_stamp = hr->get_gc_time_stamp();
2251 if (_gc_time_stamp != region_gc_time_stamp) {
2252 gclog_or_tty->print_cr("Region " HR_FORMAT " has GC time stamp = %d, "
2253 "expected %d", HR_FORMAT_PARAMS(hr),
2254 region_gc_time_stamp, _gc_time_stamp);
2255 _failures = true;
2283 }
2284
2285 // Computes the sum of the storage used by the various regions.
2286 size_t G1CollectedHeap::used() const {
2287 size_t result = _summary_bytes_used + _allocator->used_in_alloc_regions();
2288 if (_archive_allocator != NULL) {
2289 result += _archive_allocator->used();
2290 }
2291 return result;
2292 }
2293
2294 size_t G1CollectedHeap::used_unlocked() const {
2295 return _summary_bytes_used;
2296 }
2297
2298 class SumUsedClosure: public HeapRegionClosure {
2299 size_t _used;
2300 public:
2301 SumUsedClosure() : _used(0) {}
2302 bool doHeapRegion(HeapRegion* r) {
2303 if (!r->is_continues_humongous()) {
2304 _used += r->used();
2305 }
2306 return false;
2307 }
2308 size_t result() { return _used; }
2309 };
2310
2311 size_t G1CollectedHeap::recalculate_used() const {
2312 double recalculate_used_start = os::elapsedTime();
2313
2314 SumUsedClosure blk;
2315 heap_region_iterate(&blk);
2316
2317 g1_policy()->phase_times()->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0);
2318 return blk.result();
2319 }
2320
2321 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {
2322 switch (cause) {
2323 case GCCause::_gc_locker: return GCLockerInvokesConcurrent;
2324 case GCCause::_java_lang_system_gc: return ExplicitGCInvokesConcurrent;
2325 case GCCause::_dcmd_gc_run: return ExplicitGCInvokesConcurrent;
2506 // Schedule a standard evacuation pause. We're setting word_size
2507 // to 0 which means that we are not requesting a post-GC allocation.
2508 VM_G1IncCollectionPause op(gc_count_before,
2509 0, /* word_size */
2510 false, /* should_initiate_conc_mark */
2511 g1_policy()->max_pause_time_ms(),
2512 cause);
2513 VMThread::execute(&op);
2514 } else {
2515 // Schedule a Full GC.
2516 VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);
2517 VMThread::execute(&op);
2518 }
2519 }
2520 } while (retry_gc);
2521 }
2522
2523 bool G1CollectedHeap::is_in(const void* p) const {
2524 if (_hrm.reserved().contains(p)) {
2525 // Given that we know that p is in the reserved space,
2526 // heap_region_containing_raw() should successfully
2527 // return the containing region.
2528 HeapRegion* hr = heap_region_containing_raw(p);
2529 return hr->is_in(p);
2530 } else {
2531 return false;
2532 }
2533 }
2534
2535 #ifdef ASSERT
2536 bool G1CollectedHeap::is_in_exact(const void* p) const {
2537 bool contains = reserved_region().contains(p);
2538 bool available = _hrm.is_available(addr_to_region((HeapWord*)p));
2539 if (contains && available) {
2540 return true;
2541 } else {
2542 return false;
2543 }
2544 }
2545 #endif
2546
2547 bool G1CollectedHeap::obj_in_cs(oop obj) {
2548 HeapRegion* r = _hrm.addr_to_region((HeapWord*) obj);
3045 _vo(vo),
3046 _failures(false) {}
3047
3048 bool failures() {
3049 return _failures;
3050 }
3051
3052 bool doHeapRegion(HeapRegion* r) {
3053 // For archive regions, verify there are no heap pointers to
3054 // non-pinned regions. For all others, verify liveness info.
3055 if (r->is_archive()) {
3056 VerifyArchiveRegionClosure verify_oop_pointers(r);
3057 r->object_iterate(&verify_oop_pointers);
3058 return true;
3059 }
3060 if (!r->is_continues_humongous()) {
3061 bool failures = false;
3062 r->verify(_vo, &failures);
3063 if (failures) {
3064 _failures = true;
3065 } else {
3066 VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo);
3067 r->object_iterate(¬_dead_yet_cl);
3068 if (_vo != VerifyOption_G1UseNextMarking) {
3069 if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
3070 gclog_or_tty->print_cr("[" PTR_FORMAT "," PTR_FORMAT "] "
3071 "max_live_bytes " SIZE_FORMAT " "
3072 "< calculated " SIZE_FORMAT,
3073 p2i(r->bottom()), p2i(r->end()),
3074 r->max_live_bytes(),
3075 not_dead_yet_cl.live_bytes());
3076 _failures = true;
3077 }
3078 } else {
3079 // When vo == UseNextMarking we cannot currently do a sanity
3080 // check on the live bytes as the calculation has not been
3081 // finalized yet.
3082 }
3083 }
3084 }
3085 return false; // stop the region iteration if we hit a failure
5299 assert(free_list != NULL, "pre-condition");
5300
5301 if (G1VerifyBitmaps) {
5302 MemRegion mr(hr->bottom(), hr->end());
5303 concurrent_mark()->clearRangePrevBitmap(mr);
5304 }
5305
5306 // Clear the card counts for this region.
5307 // Note: we only need to do this if the region is not young
5308 // (since we don't refine cards in young regions).
5309 if (!hr->is_young()) {
5310 _cg1r->hot_card_cache()->reset_card_counts(hr);
5311 }
5312 hr->hr_clear(par, true /* clear_space */, locked /* locked */);
5313 free_list->add_ordered(hr);
5314 }
5315
5316 void G1CollectedHeap::free_humongous_region(HeapRegion* hr,
5317 FreeRegionList* free_list,
5318 bool par) {
5319 assert(hr->is_starts_humongous(), "this is only for starts humongous regions");
5320 assert(free_list != NULL, "pre-condition");
5321
5322 size_t hr_capacity = hr->capacity();
5323 // We need to read this before we make the region non-humongous,
5324 // otherwise the information will be gone.
5325 uint last_index = hr->last_hc_index();
5326 hr->clear_humongous();
5327 free_region(hr, free_list, par);
5328
5329 uint i = hr->hrm_index() + 1;
5330 while (i < last_index) {
5331 HeapRegion* curr_hr = region_at(i);
5332 assert(curr_hr->is_continues_humongous(), "invariant");
5333 curr_hr->clear_humongous();
5334 free_region(curr_hr, free_list, par);
5335 i += 1;
5336 }
5337 }
5338
5339 void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed,
5340 const HeapRegionSetCount& humongous_regions_removed) {
5341 if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) {
5342 MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag);
5343 _old_set.bulk_remove(old_regions_removed);
5344 _humongous_set.bulk_remove(humongous_regions_removed);
5345 }
5346
5347 }
5348
5349 void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) {
5350 assert(list != NULL, "list can't be null");
5351 if (!list->is_empty()) {
5352 MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag);
5353 _hrm.insert_list_into_free_list(list);
5354 }
5355 }
5356
5480
5481 void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) {
5482 if (!G1VerifyBitmaps) return;
5483
5484 guarantee(verify_bitmaps(caller, hr), "bitmap verification");
5485 }
5486
5487 class G1VerifyBitmapClosure : public HeapRegionClosure {
5488 private:
5489 const char* _caller;
5490 G1CollectedHeap* _g1h;
5491 bool _failures;
5492
5493 public:
5494 G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) :
5495 _caller(caller), _g1h(g1h), _failures(false) { }
5496
5497 bool failures() { return _failures; }
5498
5499 virtual bool doHeapRegion(HeapRegion* hr) {
5500 if (hr->is_continues_humongous()) return false;
5501
5502 bool result = _g1h->verify_bitmaps(_caller, hr);
5503 if (!result) {
5504 _failures = true;
5505 }
5506 return false;
5507 }
5508 };
5509
5510 void G1CollectedHeap::check_bitmaps(const char* caller) {
5511 if (!G1VerifyBitmaps) return;
5512
5513 G1VerifyBitmapClosure cl(caller, this);
5514 heap_region_iterate(&cl);
5515 guarantee(!cl.failures(), "bitmap verification");
5516 }
5517
5518 class G1CheckCSetFastTableClosure : public HeapRegionClosure {
5519 private:
5520 bool _failures;
5521 public:
5755 //
5756 // It is not required to check whether the object has been found dead by marking
5757 // or not, in fact it would prevent reclamation within a concurrent cycle, as
5758 // all objects allocated during that time are considered live.
5759 // SATB marking is even more conservative than the remembered set.
5760 // So if at this point in the collection there is no remembered set entry,
5761 // nobody has a reference to it.
5762 // At the start of collection we flush all refinement logs, and remembered sets
5763 // are completely up-to-date wrt to references to the humongous object.
5764 //
5765 // Other implementation considerations:
5766 // - never consider object arrays at this time because they would pose
5767 // considerable effort for cleaning up the the remembered sets. This is
5768 // required because stale remembered sets might reference locations that
5769 // are currently allocated into.
5770 uint region_idx = r->hrm_index();
5771 if (!g1h->is_humongous_reclaim_candidate(region_idx) ||
5772 !r->rem_set()->is_empty()) {
5773
5774 if (G1TraceEagerReclaimHumongousObjects) {
5775 gclog_or_tty->print_cr("Live humongous region %u size " SIZE_FORMAT " start " PTR_FORMAT " length %u with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5776 region_idx,
5777 (size_t)obj->size() * HeapWordSize,
5778 p2i(r->bottom()),
5779 r->region_num(),
5780 r->rem_set()->occupied(),
5781 r->rem_set()->strong_code_roots_list_length(),
5782 next_bitmap->isMarked(r->bottom()),
5783 g1h->is_humongous_reclaim_candidate(region_idx),
5784 obj->is_typeArray()
5785 );
5786 }
5787
5788 return false;
5789 }
5790
5791 guarantee(obj->is_typeArray(),
5792 "Only eagerly reclaiming type arrays is supported, but the object "
5793 PTR_FORMAT " is not.", p2i(r->bottom()));
5794
5795 if (G1TraceEagerReclaimHumongousObjects) {
5796 gclog_or_tty->print_cr("Dead humongous region %u size " SIZE_FORMAT " start " PTR_FORMAT " length %u with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5797 region_idx,
5798 (size_t)obj->size() * HeapWordSize,
5799 p2i(r->bottom()),
5800 r->region_num(),
5801 r->rem_set()->occupied(),
5802 r->rem_set()->strong_code_roots_list_length(),
5803 next_bitmap->isMarked(r->bottom()),
5804 g1h->is_humongous_reclaim_candidate(region_idx),
5805 obj->is_typeArray()
5806 );
5807 }
5808 // Need to clear mark bit of the humongous object if already set.
5809 if (next_bitmap->isMarked(r->bottom())) {
5810 next_bitmap->clear(r->bottom());
5811 }
5812 _freed_bytes += r->used();
5813 r->set_containing_set(NULL);
5814 _humongous_regions_removed.increment(1u, r->capacity());
5815 g1h->free_humongous_region(r, _free_region_list, false);
5816
5817 return false;
5818 }
5819
5820 HeapRegionSetCount& humongous_free_count() {
5821 return _humongous_regions_removed;
5822 }
5823
5824 size_t bytes_freed() const {
5825 return _freed_bytes;
5826 }
5827
5828 size_t humongous_reclaimed() const {
5829 return _humongous_regions_removed.length();
5830 }
5831 };
5832
5833 void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
5834 assert_at_safepoint(true);
5835
6030
6031 class RebuildRegionSetsClosure : public HeapRegionClosure {
6032 private:
6033 bool _free_list_only;
6034 HeapRegionSet* _old_set;
6035 HeapRegionManager* _hrm;
6036 size_t _total_used;
6037
6038 public:
6039 RebuildRegionSetsClosure(bool free_list_only,
6040 HeapRegionSet* old_set, HeapRegionManager* hrm) :
6041 _free_list_only(free_list_only),
6042 _old_set(old_set), _hrm(hrm), _total_used(0) {
6043 assert(_hrm->num_free_regions() == 0, "pre-condition");
6044 if (!free_list_only) {
6045 assert(_old_set->is_empty(), "pre-condition");
6046 }
6047 }
6048
6049 bool doHeapRegion(HeapRegion* r) {
6050 if (r->is_continues_humongous()) {
6051 return false;
6052 }
6053
6054 if (r->is_empty()) {
6055 // Add free regions to the free list
6056 r->set_free();
6057 r->set_allocation_context(AllocationContext::system());
6058 _hrm->insert_into_free_list(r);
6059 } else if (!_free_list_only) {
6060 assert(!r->is_young(), "we should not come across young regions");
6061
6062 if (r->is_humongous()) {
6063 // We ignore humongous regions. We left the humongous set unchanged.
6064 } else {
6065 // Objects that were compacted would have ended up on regions
6066 // that were previously old or free. Archive regions (which are
6067 // old) will not have been touched.
6068 assert(r->is_free() || r->is_old(), "invariant");
6069 // We now consider them old, so register as such. Leave
6070 // archive regions set that way, however, while still adding
6071 // them to the old set.
6072 if (!r->is_archive()) {
6073 r->set_old();
6221 // Heap region set verification
6222
6223 class VerifyRegionListsClosure : public HeapRegionClosure {
6224 private:
6225 HeapRegionSet* _old_set;
6226 HeapRegionSet* _humongous_set;
6227 HeapRegionManager* _hrm;
6228
6229 public:
6230 HeapRegionSetCount _old_count;
6231 HeapRegionSetCount _humongous_count;
6232 HeapRegionSetCount _free_count;
6233
6234 VerifyRegionListsClosure(HeapRegionSet* old_set,
6235 HeapRegionSet* humongous_set,
6236 HeapRegionManager* hrm) :
6237 _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm),
6238 _old_count(), _humongous_count(), _free_count(){ }
6239
6240 bool doHeapRegion(HeapRegion* hr) {
6241 if (hr->is_continues_humongous()) {
6242 return false;
6243 }
6244
6245 if (hr->is_young()) {
6246 // TODO
6247 } else if (hr->is_starts_humongous()) {
6248 assert(hr->containing_set() == _humongous_set, "Heap region %u is starts humongous but not in humongous set.", hr->hrm_index());
6249 _humongous_count.increment(1u, hr->capacity());
6250 } else if (hr->is_empty()) {
6251 assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index());
6252 _free_count.increment(1u, hr->capacity());
6253 } else if (hr->is_old()) {
6254 assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index());
6255 _old_count.increment(1u, hr->capacity());
6256 } else {
6257 // There are no other valid region types. Check for one invalid
6258 // one we can identify: pinned without old or humongous set.
6259 assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index());
6260 ShouldNotReachHere();
6261 }
6262 return false;
6263 }
6264
6265 void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) {
6266 guarantee(old_set->length() == _old_count.length(), "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length());
6267 guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), "Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6268 old_set->total_capacity_bytes(), _old_count.capacity());
|
303 // Index of last region in the series + 1.
304 uint last = first + num_regions;
305
306 // We need to initialize the region(s) we just discovered. This is
307 // a bit tricky given that it can happen concurrently with
308 // refinement threads refining cards on these regions and
309 // potentially wanting to refine the BOT as they are scanning
310 // those cards (this can happen shortly after a cleanup; see CR
311 // 6991377). So we have to set up the region(s) carefully and in
312 // a specific order.
313
314 // The word size sum of all the regions we will allocate.
315 size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
316 assert(word_size <= word_size_sum, "sanity");
317
318 // This will be the "starts humongous" region.
319 HeapRegion* first_hr = region_at(first);
320 // The header of the new object will be placed at the bottom of
321 // the first region.
322 HeapWord* new_obj = first_hr->bottom();
323 // This will be the new top of the new object.
324 HeapWord* obj_top = new_obj + word_size;
325
326 // First, we need to zero the header of the space that we will be
327 // allocating. When we update top further down, some refinement
328 // threads might try to scan the region. By zeroing the header we
329 // ensure that any thread that will try to scan the region will
330 // come across the zero klass word and bail out.
331 //
332 // NOTE: It would not have been correct to have used
333 // CollectedHeap::fill_with_object() and make the space look like
334 // an int array. The thread that is doing the allocation will
335 // later update the object header to a potentially different array
336 // type and, for a very short period of time, the klass and length
337 // fields will be inconsistent. This could cause a refinement
338 // thread to calculate the object size incorrectly.
339 Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
340
341 // We will set up the first region as "starts humongous". This
342 // will also update the BOT covering all the regions to reflect
343 // that there is a single object that starts at the bottom of the
344 // first region.
345 first_hr->set_starts_humongous(obj_top);
346 first_hr->set_allocation_context(context);
347 // Then, if there are any, we will set up the "continues
348 // humongous" regions.
349 HeapRegion* hr = NULL;
350 for (uint i = first + 1; i < last; ++i) {
351 hr = region_at(i);
352 hr->set_continues_humongous(first_hr);
353 hr->set_allocation_context(context);
354 }
355
356 // Up to this point no concurrent thread would have been able to
357 // do any scanning on any region in this series. All the top
358 // fields still point to bottom, so the intersection between
359 // [bottom,top] and [card_start,card_end] will be empty. Before we
360 // update the top fields, we'll do a storestore to make sure that
361 // no thread sees the update to top before the zeroing of the
362 // object header and the BOT initialization.
363 OrderAccess::storestore();
364
365 // Now that the BOT and the object header have been initialized,
366 // we can update top of the "starts humongous" region.
367 first_hr->set_top(MIN2(first_hr->end(), obj_top));
368 if (_hr_printer.is_active()) {
369 if ((first + 1) == last) {
370 // the series has a single humongous region
371 _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, obj_top);
372 } else {
373 // the series has more than one humongous regions
374 _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, first_hr->end());
375 }
376 }
377
378 // Now, we will update the top fields of the "continues humongous"
379 // regions. The reason we need to do this is that, otherwise,
380 // these regions would look empty and this will confuse parts of
381 // G1. For example, the code that looks for a consecutive number
382 // of empty regions will consider them empty and try to
383 // re-allocate them. We can extend is_empty() to also include
384 // !is_continues_humongous(), but it is easier to just update the top
385 // fields here. The way we set top for all regions (i.e., top ==
386 // end for all regions but the last one, top == new_top for the
387 // last one) is actually used when we will free up the humongous
388 // region in free_humongous_region().
389 hr = NULL;
390 for (uint i = first + 1; i < last; ++i) {
391 hr = region_at(i);
392 if ((i + 1) == last) {
393 // last continues humongous region
394 assert(hr->bottom() < obj_top && obj_top <= hr->end(),
395 "new_top should fall on this region");
396 hr->set_top(obj_top);
397 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, obj_top);
398 } else {
399 // not last one
400 assert(obj_top > hr->end(), "obj_top should be above this region");
401 hr->set_top(hr->end());
402 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end());
403 }
404 }
405 // If we have continues humongous regions (hr != NULL), its top should
406 // match obj_top.
407 assert(hr == NULL || (hr->top() == obj_top), "sanity");
408 check_bitmaps("Humongous Region Allocation", first_hr);
409
410 increase_used(word_size * HeapWordSize);
411
412 for (uint i = first; i < last; ++i) {
413 _humongous_set.add(region_at(i));
414 }
415
416 return new_obj;
417 }
418
419 // If could fit into free regions w/o expansion, try.
420 // Otherwise, if can expand, do so.
421 // Otherwise, if using ex regions might help, try with ex given back.
422 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) {
423 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
424
425 verify_region_sets_optional();
426
427 uint first = G1_NO_HRM_INDEX;
428 uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords);
429
430 if (obj_regions == 1) {
431 // Only one region to allocate, try to use a fast path by directly allocating
432 // from the free lists. Do not try to expand here, we will potentially do that
433 // later.
434 HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */);
1111 HeapWord* result = humongous_obj_allocate(word_size, context);
1112 if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) {
1113 collector_state()->set_initiate_conc_mark_if_possible(true);
1114 }
1115 return result;
1116 }
1117
1118 ShouldNotReachHere();
1119 }
1120
1121 class PostMCRemSetClearClosure: public HeapRegionClosure {
1122 G1CollectedHeap* _g1h;
1123 ModRefBarrierSet* _mr_bs;
1124 public:
1125 PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) :
1126 _g1h(g1h), _mr_bs(mr_bs) {}
1127
1128 bool doHeapRegion(HeapRegion* r) {
1129 HeapRegionRemSet* hrrs = r->rem_set();
1130
1131 _g1h->reset_gc_time_stamps(r);
1132 hrrs->clear();
1133 // You might think here that we could clear just the cards
1134 // corresponding to the used region. But no: if we leave a dirty card
1135 // in a region we might allocate into, then it would prevent that card
1136 // from being enqueued, and cause it to be missed.
1137 // Re: the performance cost: we shouldn't be doing full GC anyway!
1138 _mr_bs->clear(MemRegion(r->bottom(), r->end()));
1139
1140 return false;
1141 }
1142 };
1143
1144 void G1CollectedHeap::clear_rsets_post_compaction() {
1145 PostMCRemSetClearClosure rs_clear(this, g1_barrier_set());
1146 heap_region_iterate(&rs_clear);
1147 }
1148
1149 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
1150 G1CollectedHeap* _g1h;
1170
1171 public:
1172 ParRebuildRSTask(G1CollectedHeap* g1) :
1173 AbstractGangTask("ParRebuildRSTask"), _g1(g1), _hrclaimer(g1->workers()->active_workers()) {}
1174
1175 void work(uint worker_id) {
1176 RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id);
1177 _g1->heap_region_par_iterate(&rebuild_rs, worker_id, &_hrclaimer);
1178 }
1179 };
1180
1181 class PostCompactionPrinterClosure: public HeapRegionClosure {
1182 private:
1183 G1HRPrinter* _hr_printer;
1184 public:
1185 bool doHeapRegion(HeapRegion* hr) {
1186 assert(!hr->is_young(), "not expecting to find young regions");
1187 if (hr->is_free()) {
1188 // We only generate output for non-empty regions.
1189 } else if (hr->is_starts_humongous()) {
1190 _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous);
1191 } else if (hr->is_continues_humongous()) {
1192 _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous);
1193 } else if (hr->is_archive()) {
1194 _hr_printer->post_compaction(hr, G1HRPrinter::Archive);
1195 } else if (hr->is_old()) {
1196 _hr_printer->post_compaction(hr, G1HRPrinter::Old);
1197 } else {
1198 ShouldNotReachHere();
1199 }
1200 return false;
1201 }
1202
1203 PostCompactionPrinterClosure(G1HRPrinter* hr_printer)
1204 : _hr_printer(hr_printer) { }
1205 };
1206
1207 void G1CollectedHeap::print_hrm_post_compaction() {
1208 PostCompactionPrinterClosure cl(hr_printer());
1209 heap_region_iterate(&cl);
1210 }
2182 (ParallelGCThreads > 1),
2183 // mt discovery
2184 ParallelGCThreads,
2185 // degree of mt discovery
2186 true,
2187 // Reference discovery is atomic
2188 &_is_alive_closure_stw);
2189 // is alive closure
2190 // (for efficiency/performance)
2191 }
2192
2193 CollectorPolicy* G1CollectedHeap::collector_policy() const {
2194 return g1_policy();
2195 }
2196
2197 size_t G1CollectedHeap::capacity() const {
2198 return _hrm.length() * HeapRegion::GrainBytes;
2199 }
2200
2201 void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) {
2202 hr->reset_gc_time_stamp();
2203 }
2204
2205 #ifndef PRODUCT
2206
2207 class CheckGCTimeStampsHRClosure : public HeapRegionClosure {
2208 private:
2209 unsigned _gc_time_stamp;
2210 bool _failures;
2211
2212 public:
2213 CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) :
2214 _gc_time_stamp(gc_time_stamp), _failures(false) { }
2215
2216 virtual bool doHeapRegion(HeapRegion* hr) {
2217 unsigned region_gc_time_stamp = hr->get_gc_time_stamp();
2218 if (_gc_time_stamp != region_gc_time_stamp) {
2219 gclog_or_tty->print_cr("Region " HR_FORMAT " has GC time stamp = %d, "
2220 "expected %d", HR_FORMAT_PARAMS(hr),
2221 region_gc_time_stamp, _gc_time_stamp);
2222 _failures = true;
2250 }
2251
2252 // Computes the sum of the storage used by the various regions.
2253 size_t G1CollectedHeap::used() const {
2254 size_t result = _summary_bytes_used + _allocator->used_in_alloc_regions();
2255 if (_archive_allocator != NULL) {
2256 result += _archive_allocator->used();
2257 }
2258 return result;
2259 }
2260
2261 size_t G1CollectedHeap::used_unlocked() const {
2262 return _summary_bytes_used;
2263 }
2264
2265 class SumUsedClosure: public HeapRegionClosure {
2266 size_t _used;
2267 public:
2268 SumUsedClosure() : _used(0) {}
2269 bool doHeapRegion(HeapRegion* r) {
2270 _used += r->used();
2271 return false;
2272 }
2273 size_t result() { return _used; }
2274 };
2275
2276 size_t G1CollectedHeap::recalculate_used() const {
2277 double recalculate_used_start = os::elapsedTime();
2278
2279 SumUsedClosure blk;
2280 heap_region_iterate(&blk);
2281
2282 g1_policy()->phase_times()->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0);
2283 return blk.result();
2284 }
2285
2286 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {
2287 switch (cause) {
2288 case GCCause::_gc_locker: return GCLockerInvokesConcurrent;
2289 case GCCause::_java_lang_system_gc: return ExplicitGCInvokesConcurrent;
2290 case GCCause::_dcmd_gc_run: return ExplicitGCInvokesConcurrent;
2471 // Schedule a standard evacuation pause. We're setting word_size
2472 // to 0 which means that we are not requesting a post-GC allocation.
2473 VM_G1IncCollectionPause op(gc_count_before,
2474 0, /* word_size */
2475 false, /* should_initiate_conc_mark */
2476 g1_policy()->max_pause_time_ms(),
2477 cause);
2478 VMThread::execute(&op);
2479 } else {
2480 // Schedule a Full GC.
2481 VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);
2482 VMThread::execute(&op);
2483 }
2484 }
2485 } while (retry_gc);
2486 }
2487
2488 bool G1CollectedHeap::is_in(const void* p) const {
2489 if (_hrm.reserved().contains(p)) {
2490 // Given that we know that p is in the reserved space,
2491 // heap_region_containing() should successfully
2492 // return the containing region.
2493 HeapRegion* hr = heap_region_containing(p);
2494 return hr->is_in(p);
2495 } else {
2496 return false;
2497 }
2498 }
2499
2500 #ifdef ASSERT
2501 bool G1CollectedHeap::is_in_exact(const void* p) const {
2502 bool contains = reserved_region().contains(p);
2503 bool available = _hrm.is_available(addr_to_region((HeapWord*)p));
2504 if (contains && available) {
2505 return true;
2506 } else {
2507 return false;
2508 }
2509 }
2510 #endif
2511
2512 bool G1CollectedHeap::obj_in_cs(oop obj) {
2513 HeapRegion* r = _hrm.addr_to_region((HeapWord*) obj);
3010 _vo(vo),
3011 _failures(false) {}
3012
3013 bool failures() {
3014 return _failures;
3015 }
3016
3017 bool doHeapRegion(HeapRegion* r) {
3018 // For archive regions, verify there are no heap pointers to
3019 // non-pinned regions. For all others, verify liveness info.
3020 if (r->is_archive()) {
3021 VerifyArchiveRegionClosure verify_oop_pointers(r);
3022 r->object_iterate(&verify_oop_pointers);
3023 return true;
3024 }
3025 if (!r->is_continues_humongous()) {
3026 bool failures = false;
3027 r->verify(_vo, &failures);
3028 if (failures) {
3029 _failures = true;
3030 } else if (!r->is_starts_humongous()) {
3031 VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo);
3032 r->object_iterate(¬_dead_yet_cl);
3033 if (_vo != VerifyOption_G1UseNextMarking) {
3034 if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
3035 gclog_or_tty->print_cr("[" PTR_FORMAT "," PTR_FORMAT "] "
3036 "max_live_bytes " SIZE_FORMAT " "
3037 "< calculated " SIZE_FORMAT,
3038 p2i(r->bottom()), p2i(r->end()),
3039 r->max_live_bytes(),
3040 not_dead_yet_cl.live_bytes());
3041 _failures = true;
3042 }
3043 } else {
3044 // When vo == UseNextMarking we cannot currently do a sanity
3045 // check on the live bytes as the calculation has not been
3046 // finalized yet.
3047 }
3048 }
3049 }
3050 return false; // stop the region iteration if we hit a failure
5264 assert(free_list != NULL, "pre-condition");
5265
5266 if (G1VerifyBitmaps) {
5267 MemRegion mr(hr->bottom(), hr->end());
5268 concurrent_mark()->clearRangePrevBitmap(mr);
5269 }
5270
5271 // Clear the card counts for this region.
5272 // Note: we only need to do this if the region is not young
5273 // (since we don't refine cards in young regions).
5274 if (!hr->is_young()) {
5275 _cg1r->hot_card_cache()->reset_card_counts(hr);
5276 }
5277 hr->hr_clear(par, true /* clear_space */, locked /* locked */);
5278 free_list->add_ordered(hr);
5279 }
5280
5281 void G1CollectedHeap::free_humongous_region(HeapRegion* hr,
5282 FreeRegionList* free_list,
5283 bool par) {
5284 assert(free_list != NULL, "pre-condition");
5285 hr->clear_humongous();
5286 free_region(hr, free_list, par);
5287 }
5288
5289 void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed,
5290 const HeapRegionSetCount& humongous_regions_removed) {
5291 if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) {
5292 MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag);
5293 _old_set.bulk_remove(old_regions_removed);
5294 _humongous_set.bulk_remove(humongous_regions_removed);
5295 }
5296
5297 }
5298
5299 void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) {
5300 assert(list != NULL, "list can't be null");
5301 if (!list->is_empty()) {
5302 MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag);
5303 _hrm.insert_list_into_free_list(list);
5304 }
5305 }
5306
5430
5431 void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) {
5432 if (!G1VerifyBitmaps) return;
5433
5434 guarantee(verify_bitmaps(caller, hr), "bitmap verification");
5435 }
5436
5437 class G1VerifyBitmapClosure : public HeapRegionClosure {
5438 private:
5439 const char* _caller;
5440 G1CollectedHeap* _g1h;
5441 bool _failures;
5442
5443 public:
5444 G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) :
5445 _caller(caller), _g1h(g1h), _failures(false) { }
5446
5447 bool failures() { return _failures; }
5448
5449 virtual bool doHeapRegion(HeapRegion* hr) {
5450 bool result = _g1h->verify_bitmaps(_caller, hr);
5451 if (!result) {
5452 _failures = true;
5453 }
5454 return false;
5455 }
5456 };
5457
5458 void G1CollectedHeap::check_bitmaps(const char* caller) {
5459 if (!G1VerifyBitmaps) return;
5460
5461 G1VerifyBitmapClosure cl(caller, this);
5462 heap_region_iterate(&cl);
5463 guarantee(!cl.failures(), "bitmap verification");
5464 }
5465
5466 class G1CheckCSetFastTableClosure : public HeapRegionClosure {
5467 private:
5468 bool _failures;
5469 public:
5703 //
5704 // It is not required to check whether the object has been found dead by marking
5705 // or not, in fact it would prevent reclamation within a concurrent cycle, as
5706 // all objects allocated during that time are considered live.
5707 // SATB marking is even more conservative than the remembered set.
5708 // So if at this point in the collection there is no remembered set entry,
5709 // nobody has a reference to it.
5710 // At the start of collection we flush all refinement logs, and remembered sets
5711 // are completely up-to-date wrt to references to the humongous object.
5712 //
5713 // Other implementation considerations:
5714 // - never consider object arrays at this time because they would pose
5715 // considerable effort for cleaning up the the remembered sets. This is
5716 // required because stale remembered sets might reference locations that
5717 // are currently allocated into.
5718 uint region_idx = r->hrm_index();
5719 if (!g1h->is_humongous_reclaim_candidate(region_idx) ||
5720 !r->rem_set()->is_empty()) {
5721
5722 if (G1TraceEagerReclaimHumongousObjects) {
5723 gclog_or_tty->print_cr("Live humongous region %u objectsize " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5724 region_idx,
5725 (size_t)obj->size() * HeapWordSize,
5726 p2i(r->bottom()),
5727 r->rem_set()->occupied(),
5728 r->rem_set()->strong_code_roots_list_length(),
5729 next_bitmap->isMarked(r->bottom()),
5730 g1h->is_humongous_reclaim_candidate(region_idx),
5731 obj->is_typeArray()
5732 );
5733 }
5734
5735 return false;
5736 }
5737
5738 guarantee(obj->is_typeArray(),
5739 "Only eagerly reclaiming type arrays is supported, but the object "
5740 PTR_FORMAT " is not.", p2i(r->bottom()));
5741
5742 if (G1TraceEagerReclaimHumongousObjects) {
5743 gclog_or_tty->print_cr("Dead humongous region %u objectsize " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5744 region_idx,
5745 (size_t)obj->size() * HeapWordSize,
5746 p2i(r->bottom()),
5747 r->rem_set()->occupied(),
5748 r->rem_set()->strong_code_roots_list_length(),
5749 next_bitmap->isMarked(r->bottom()),
5750 g1h->is_humongous_reclaim_candidate(region_idx),
5751 obj->is_typeArray()
5752 );
5753 }
5754 // Need to clear mark bit of the humongous object if already set.
5755 if (next_bitmap->isMarked(r->bottom())) {
5756 next_bitmap->clear(r->bottom());
5757 }
5758 do {
5759 HeapRegion* next = g1h->next_region_by_index(r);
5760 _freed_bytes += r->used();
5761 r->set_containing_set(NULL);
5762 _humongous_regions_removed.increment(1u, r->capacity());
5763 g1h->free_humongous_region(r, _free_region_list, false);
5764 r = next;
5765 } while (r != NULL && r->is_continues_humongous());
5766
5767 return false;
5768 }
5769
5770 HeapRegionSetCount& humongous_free_count() {
5771 return _humongous_regions_removed;
5772 }
5773
5774 size_t bytes_freed() const {
5775 return _freed_bytes;
5776 }
5777
5778 size_t humongous_reclaimed() const {
5779 return _humongous_regions_removed.length();
5780 }
5781 };
5782
5783 void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
5784 assert_at_safepoint(true);
5785
5980
5981 class RebuildRegionSetsClosure : public HeapRegionClosure {
5982 private:
5983 bool _free_list_only;
5984 HeapRegionSet* _old_set;
5985 HeapRegionManager* _hrm;
5986 size_t _total_used;
5987
5988 public:
5989 RebuildRegionSetsClosure(bool free_list_only,
5990 HeapRegionSet* old_set, HeapRegionManager* hrm) :
5991 _free_list_only(free_list_only),
5992 _old_set(old_set), _hrm(hrm), _total_used(0) {
5993 assert(_hrm->num_free_regions() == 0, "pre-condition");
5994 if (!free_list_only) {
5995 assert(_old_set->is_empty(), "pre-condition");
5996 }
5997 }
5998
5999 bool doHeapRegion(HeapRegion* r) {
6000 if (r->is_empty()) {
6001 // Add free regions to the free list
6002 r->set_free();
6003 r->set_allocation_context(AllocationContext::system());
6004 _hrm->insert_into_free_list(r);
6005 } else if (!_free_list_only) {
6006 assert(!r->is_young(), "we should not come across young regions");
6007
6008 if (r->is_humongous()) {
6009 // We ignore humongous regions. We left the humongous set unchanged.
6010 } else {
6011 // Objects that were compacted would have ended up on regions
6012 // that were previously old or free. Archive regions (which are
6013 // old) will not have been touched.
6014 assert(r->is_free() || r->is_old(), "invariant");
6015 // We now consider them old, so register as such. Leave
6016 // archive regions set that way, however, while still adding
6017 // them to the old set.
6018 if (!r->is_archive()) {
6019 r->set_old();
6167 // Heap region set verification
6168
6169 class VerifyRegionListsClosure : public HeapRegionClosure {
6170 private:
6171 HeapRegionSet* _old_set;
6172 HeapRegionSet* _humongous_set;
6173 HeapRegionManager* _hrm;
6174
6175 public:
6176 HeapRegionSetCount _old_count;
6177 HeapRegionSetCount _humongous_count;
6178 HeapRegionSetCount _free_count;
6179
6180 VerifyRegionListsClosure(HeapRegionSet* old_set,
6181 HeapRegionSet* humongous_set,
6182 HeapRegionManager* hrm) :
6183 _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm),
6184 _old_count(), _humongous_count(), _free_count(){ }
6185
6186 bool doHeapRegion(HeapRegion* hr) {
6187 if (hr->is_young()) {
6188 // TODO
6189 } else if (hr->is_humongous()) {
6190 assert(hr->containing_set() == _humongous_set, "Heap region %u is humongous but not in humongous set.", hr->hrm_index());
6191 _humongous_count.increment(1u, hr->capacity());
6192 } else if (hr->is_empty()) {
6193 assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index());
6194 _free_count.increment(1u, hr->capacity());
6195 } else if (hr->is_old()) {
6196 assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index());
6197 _old_count.increment(1u, hr->capacity());
6198 } else {
6199 // There are no other valid region types. Check for one invalid
6200 // one we can identify: pinned without old or humongous set.
6201 assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index());
6202 ShouldNotReachHere();
6203 }
6204 return false;
6205 }
6206
6207 void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) {
6208 guarantee(old_set->length() == _old_count.length(), "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length());
6209 guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), "Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6210 old_set->total_capacity_bytes(), _old_count.capacity());
|