303 // Index of last region in the series + 1.
304 uint last = first + num_regions;
305
306 // We need to initialize the region(s) we just discovered. This is
307 // a bit tricky given that it can happen concurrently with
308 // refinement threads refining cards on these regions and
309 // potentially wanting to refine the BOT as they are scanning
310 // those cards (this can happen shortly after a cleanup; see CR
311 // 6991377). So we have to set up the region(s) carefully and in
312 // a specific order.
313
314 // The word size sum of all the regions we will allocate.
315 size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
316 assert(word_size <= word_size_sum, "sanity");
317
318 // This will be the "starts humongous" region.
319 HeapRegion* first_hr = region_at(first);
320 // The header of the new object will be placed at the bottom of
321 // the first region.
322 HeapWord* new_obj = first_hr->bottom();
323 // This will be the new end of the first region in the series that
324 // should also match the end of the last region in the series.
325 HeapWord* new_end = new_obj + word_size_sum;
326 // This will be the new top of the first region that will reflect
327 // this allocation.
328 HeapWord* new_top = new_obj + word_size;
329
330 // First, we need to zero the header of the space that we will be
331 // allocating. When we update top further down, some refinement
332 // threads might try to scan the region. By zeroing the header we
333 // ensure that any thread that will try to scan the region will
334 // come across the zero klass word and bail out.
335 //
336 // NOTE: It would not have been correct to have used
337 // CollectedHeap::fill_with_object() and make the space look like
338 // an int array. The thread that is doing the allocation will
339 // later update the object header to a potentially different array
340 // type and, for a very short period of time, the klass and length
341 // fields will be inconsistent. This could cause a refinement
342 // thread to calculate the object size incorrectly.
343 Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
344
345 // We will set up the first region as "starts humongous". This
346 // will also update the BOT covering all the regions to reflect
347 // that there is a single object that starts at the bottom of the
348 // first region.
349 first_hr->set_starts_humongous(new_top, new_end);
350 first_hr->set_allocation_context(context);
351 // Then, if there are any, we will set up the "continues
352 // humongous" regions.
353 HeapRegion* hr = NULL;
354 for (uint i = first + 1; i < last; ++i) {
355 hr = region_at(i);
356 hr->set_continues_humongous(first_hr);
357 hr->set_allocation_context(context);
358 }
359 // If we have "continues humongous" regions (hr != NULL), then the
360 // end of the last one should match new_end.
361 assert(hr == NULL || hr->end() == new_end, "sanity");
362
363 // Up to this point no concurrent thread would have been able to
364 // do any scanning on any region in this series. All the top
365 // fields still point to bottom, so the intersection between
366 // [bottom,top] and [card_start,card_end] will be empty. Before we
367 // update the top fields, we'll do a storestore to make sure that
368 // no thread sees the update to top before the zeroing of the
369 // object header and the BOT initialization.
370 OrderAccess::storestore();
371
372 // Now that the BOT and the object header have been initialized,
373 // we can update top of the "starts humongous" region.
374 assert(first_hr->bottom() < new_top && new_top <= first_hr->end(),
375 "new_top should be in this region");
376 first_hr->set_top(new_top);
377 if (_hr_printer.is_active()) {
378 HeapWord* bottom = first_hr->bottom();
379 HeapWord* end = first_hr->orig_end();
380 if ((first + 1) == last) {
381 // the series has a single humongous region
382 _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, new_top);
383 } else {
384 // the series has more than one humongous regions
385 _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, end);
386 }
387 }
388
389 // Now, we will update the top fields of the "continues humongous"
390 // regions. The reason we need to do this is that, otherwise,
391 // these regions would look empty and this will confuse parts of
392 // G1. For example, the code that looks for a consecutive number
393 // of empty regions will consider them empty and try to
394 // re-allocate them. We can extend is_empty() to also include
395 // !is_continues_humongous(), but it is easier to just update the top
396 // fields here. The way we set top for all regions (i.e., top ==
397 // end for all regions but the last one, top == new_top for the
398 // last one) is actually used when we will free up the humongous
399 // region in free_humongous_region().
400 hr = NULL;
401 for (uint i = first + 1; i < last; ++i) {
402 hr = region_at(i);
403 if ((i + 1) == last) {
404 // last continues humongous region
405 assert(hr->bottom() < new_top && new_top <= hr->end(),
406 "new_top should fall on this region");
407 hr->set_top(new_top);
408 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, new_top);
409 } else {
410 // not last one
411 assert(new_top > hr->end(), "new_top should be above this region");
412 hr->set_top(hr->end());
413 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end());
414 }
415 }
416 // If we have continues humongous regions (hr != NULL), then the
417 // end of the last one should match new_end and its top should
418 // match new_top.
419 assert(hr == NULL ||
420 (hr->end() == new_end && hr->top() == new_top), "sanity");
421 check_bitmaps("Humongous Region Allocation", first_hr);
422
423 assert(first_hr->used() == word_size * HeapWordSize, "invariant");
424 increase_used(first_hr->used());
425 _humongous_set.add(first_hr);
426
427 return new_obj;
428 }
429
430 // If could fit into free regions w/o expansion, try.
431 // Otherwise, if can expand, do so.
432 // Otherwise, if using ex regions might help, try with ex given back.
433 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) {
434 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
435
436 verify_region_sets_optional();
437
438 uint first = G1_NO_HRM_INDEX;
439 uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords);
440
441 if (obj_regions == 1) {
442 // Only one region to allocate, try to use a fast path by directly allocating
443 // from the free lists. Do not try to expand here, we will potentially do that
444 // later.
445 HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */);
1122 HeapWord* result = humongous_obj_allocate(word_size, context);
1123 if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) {
1124 collector_state()->set_initiate_conc_mark_if_possible(true);
1125 }
1126 return result;
1127 }
1128
1129 ShouldNotReachHere();
1130 }
1131
1132 class PostMCRemSetClearClosure: public HeapRegionClosure {
1133 G1CollectedHeap* _g1h;
1134 ModRefBarrierSet* _mr_bs;
1135 public:
1136 PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) :
1137 _g1h(g1h), _mr_bs(mr_bs) {}
1138
1139 bool doHeapRegion(HeapRegion* r) {
1140 HeapRegionRemSet* hrrs = r->rem_set();
1141
1142 if (r->is_continues_humongous()) {
1143 // We'll assert that the strong code root list and RSet is empty
1144 assert(hrrs->strong_code_roots_list_length() == 0, "sanity");
1145 assert(hrrs->occupied() == 0, "RSet should be empty");
1146 return false;
1147 }
1148
1149 _g1h->reset_gc_time_stamps(r);
1150 hrrs->clear();
1151 // You might think here that we could clear just the cards
1152 // corresponding to the used region. But no: if we leave a dirty card
1153 // in a region we might allocate into, then it would prevent that card
1154 // from being enqueued, and cause it to be missed.
1155 // Re: the performance cost: we shouldn't be doing full GC anyway!
1156 _mr_bs->clear(MemRegion(r->bottom(), r->end()));
1157
1158 return false;
1159 }
1160 };
1161
1162 void G1CollectedHeap::clear_rsets_post_compaction() {
1163 PostMCRemSetClearClosure rs_clear(this, g1_barrier_set());
1164 heap_region_iterate(&rs_clear);
1165 }
1166
1167 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
1168 G1CollectedHeap* _g1h;
1169 UpdateRSOopClosure _cl;
1188
1189 public:
1190 ParRebuildRSTask(G1CollectedHeap* g1) :
1191 AbstractGangTask("ParRebuildRSTask"), _g1(g1), _hrclaimer(g1->workers()->active_workers()) {}
1192
1193 void work(uint worker_id) {
1194 RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id);
1195 _g1->heap_region_par_iterate(&rebuild_rs, worker_id, &_hrclaimer);
1196 }
1197 };
1198
1199 class PostCompactionPrinterClosure: public HeapRegionClosure {
1200 private:
1201 G1HRPrinter* _hr_printer;
1202 public:
1203 bool doHeapRegion(HeapRegion* hr) {
1204 assert(!hr->is_young(), "not expecting to find young regions");
1205 if (hr->is_free()) {
1206 // We only generate output for non-empty regions.
1207 } else if (hr->is_starts_humongous()) {
1208 if (hr->region_num() == 1) {
1209 // single humongous region
1210 _hr_printer->post_compaction(hr, G1HRPrinter::SingleHumongous);
1211 } else {
1212 _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous);
1213 }
1214 } else if (hr->is_continues_humongous()) {
1215 _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous);
1216 } else if (hr->is_archive()) {
1217 _hr_printer->post_compaction(hr, G1HRPrinter::Archive);
1218 } else if (hr->is_old()) {
1219 _hr_printer->post_compaction(hr, G1HRPrinter::Old);
1220 } else {
1221 ShouldNotReachHere();
1222 }
1223 return false;
1224 }
1225
1226 PostCompactionPrinterClosure(G1HRPrinter* hr_printer)
1227 : _hr_printer(hr_printer) { }
1228 };
1229
1230 void G1CollectedHeap::print_hrm_post_compaction() {
1231 PostCompactionPrinterClosure cl(hr_printer());
1232 heap_region_iterate(&cl);
1233 }
2205 (ParallelGCThreads > 1),
2206 // mt discovery
2207 ParallelGCThreads,
2208 // degree of mt discovery
2209 true,
2210 // Reference discovery is atomic
2211 &_is_alive_closure_stw);
2212 // is alive closure
2213 // (for efficiency/performance)
2214 }
2215
2216 CollectorPolicy* G1CollectedHeap::collector_policy() const {
2217 return g1_policy();
2218 }
2219
2220 size_t G1CollectedHeap::capacity() const {
2221 return _hrm.length() * HeapRegion::GrainBytes;
2222 }
2223
2224 void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) {
2225 assert(!hr->is_continues_humongous(), "pre-condition");
2226 hr->reset_gc_time_stamp();
2227 if (hr->is_starts_humongous()) {
2228 uint first_index = hr->hrm_index() + 1;
2229 uint last_index = hr->last_hc_index();
2230 for (uint i = first_index; i < last_index; i += 1) {
2231 HeapRegion* chr = region_at(i);
2232 assert(chr->is_continues_humongous(), "sanity");
2233 chr->reset_gc_time_stamp();
2234 }
2235 }
2236 }
2237
2238 #ifndef PRODUCT
2239
2240 class CheckGCTimeStampsHRClosure : public HeapRegionClosure {
2241 private:
2242 unsigned _gc_time_stamp;
2243 bool _failures;
2244
2245 public:
2246 CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) :
2247 _gc_time_stamp(gc_time_stamp), _failures(false) { }
2248
2249 virtual bool doHeapRegion(HeapRegion* hr) {
2250 unsigned region_gc_time_stamp = hr->get_gc_time_stamp();
2251 if (_gc_time_stamp != region_gc_time_stamp) {
2252 gclog_or_tty->print_cr("Region " HR_FORMAT " has GC time stamp = %d, "
2253 "expected %d", HR_FORMAT_PARAMS(hr),
2254 region_gc_time_stamp, _gc_time_stamp);
2255 _failures = true;
2283 }
2284
2285 // Computes the sum of the storage used by the various regions.
2286 size_t G1CollectedHeap::used() const {
2287 size_t result = _summary_bytes_used + _allocator->used_in_alloc_regions();
2288 if (_archive_allocator != NULL) {
2289 result += _archive_allocator->used();
2290 }
2291 return result;
2292 }
2293
2294 size_t G1CollectedHeap::used_unlocked() const {
2295 return _summary_bytes_used;
2296 }
2297
2298 class SumUsedClosure: public HeapRegionClosure {
2299 size_t _used;
2300 public:
2301 SumUsedClosure() : _used(0) {}
2302 bool doHeapRegion(HeapRegion* r) {
2303 if (!r->is_continues_humongous()) {
2304 _used += r->used();
2305 }
2306 return false;
2307 }
2308 size_t result() { return _used; }
2309 };
2310
2311 size_t G1CollectedHeap::recalculate_used() const {
2312 double recalculate_used_start = os::elapsedTime();
2313
2314 SumUsedClosure blk;
2315 heap_region_iterate(&blk);
2316
2317 g1_policy()->phase_times()->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0);
2318 return blk.result();
2319 }
2320
2321 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {
2322 switch (cause) {
2323 case GCCause::_gc_locker: return GCLockerInvokesConcurrent;
2324 case GCCause::_java_lang_system_gc: return ExplicitGCInvokesConcurrent;
2325 case GCCause::_dcmd_gc_run: return ExplicitGCInvokesConcurrent;
2506 // Schedule a standard evacuation pause. We're setting word_size
2507 // to 0 which means that we are not requesting a post-GC allocation.
2508 VM_G1IncCollectionPause op(gc_count_before,
2509 0, /* word_size */
2510 false, /* should_initiate_conc_mark */
2511 g1_policy()->max_pause_time_ms(),
2512 cause);
2513 VMThread::execute(&op);
2514 } else {
2515 // Schedule a Full GC.
2516 VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);
2517 VMThread::execute(&op);
2518 }
2519 }
2520 } while (retry_gc);
2521 }
2522
2523 bool G1CollectedHeap::is_in(const void* p) const {
2524 if (_hrm.reserved().contains(p)) {
2525 // Given that we know that p is in the reserved space,
2526 // heap_region_containing_raw() should successfully
2527 // return the containing region.
2528 HeapRegion* hr = heap_region_containing_raw(p);
2529 return hr->is_in(p);
2530 } else {
2531 return false;
2532 }
2533 }
2534
2535 #ifdef ASSERT
2536 bool G1CollectedHeap::is_in_exact(const void* p) const {
2537 bool contains = reserved_region().contains(p);
2538 bool available = _hrm.is_available(addr_to_region((HeapWord*)p));
2539 if (contains && available) {
2540 return true;
2541 } else {
2542 return false;
2543 }
2544 }
2545 #endif
2546
2547 bool G1CollectedHeap::obj_in_cs(oop obj) {
2548 HeapRegion* r = _hrm.addr_to_region((HeapWord*) obj);
3045 _vo(vo),
3046 _failures(false) {}
3047
3048 bool failures() {
3049 return _failures;
3050 }
3051
3052 bool doHeapRegion(HeapRegion* r) {
3053 // For archive regions, verify there are no heap pointers to
3054 // non-pinned regions. For all others, verify liveness info.
3055 if (r->is_archive()) {
3056 VerifyArchiveRegionClosure verify_oop_pointers(r);
3057 r->object_iterate(&verify_oop_pointers);
3058 return true;
3059 }
3060 if (!r->is_continues_humongous()) {
3061 bool failures = false;
3062 r->verify(_vo, &failures);
3063 if (failures) {
3064 _failures = true;
3065 } else {
3066 VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo);
3067 r->object_iterate(¬_dead_yet_cl);
3068 if (_vo != VerifyOption_G1UseNextMarking) {
3069 if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
3070 gclog_or_tty->print_cr("[" PTR_FORMAT "," PTR_FORMAT "] "
3071 "max_live_bytes " SIZE_FORMAT " "
3072 "< calculated " SIZE_FORMAT,
3073 p2i(r->bottom()), p2i(r->end()),
3074 r->max_live_bytes(),
3075 not_dead_yet_cl.live_bytes());
3076 _failures = true;
3077 }
3078 } else {
3079 // When vo == UseNextMarking we cannot currently do a sanity
3080 // check on the live bytes as the calculation has not been
3081 // finalized yet.
3082 }
3083 }
3084 }
3085 return false; // stop the region iteration if we hit a failure
5299 assert(free_list != NULL, "pre-condition");
5300
5301 if (G1VerifyBitmaps) {
5302 MemRegion mr(hr->bottom(), hr->end());
5303 concurrent_mark()->clearRangePrevBitmap(mr);
5304 }
5305
5306 // Clear the card counts for this region.
5307 // Note: we only need to do this if the region is not young
5308 // (since we don't refine cards in young regions).
5309 if (!hr->is_young()) {
5310 _cg1r->hot_card_cache()->reset_card_counts(hr);
5311 }
5312 hr->hr_clear(par, true /* clear_space */, locked /* locked */);
5313 free_list->add_ordered(hr);
5314 }
5315
5316 void G1CollectedHeap::free_humongous_region(HeapRegion* hr,
5317 FreeRegionList* free_list,
5318 bool par) {
5319 assert(hr->is_starts_humongous(), "this is only for starts humongous regions");
5320 assert(free_list != NULL, "pre-condition");
5321
5322 size_t hr_capacity = hr->capacity();
5323 // We need to read this before we make the region non-humongous,
5324 // otherwise the information will be gone.
5325 uint last_index = hr->last_hc_index();
5326 hr->clear_humongous();
5327 free_region(hr, free_list, par);
5328
5329 uint i = hr->hrm_index() + 1;
5330 while (i < last_index) {
5331 HeapRegion* curr_hr = region_at(i);
5332 assert(curr_hr->is_continues_humongous(), "invariant");
5333 curr_hr->clear_humongous();
5334 free_region(curr_hr, free_list, par);
5335 i += 1;
5336 }
5337 }
5338
5339 void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed,
5340 const HeapRegionSetCount& humongous_regions_removed) {
5341 if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) {
5342 MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag);
5343 _old_set.bulk_remove(old_regions_removed);
5344 _humongous_set.bulk_remove(humongous_regions_removed);
5345 }
5346
5347 }
5348
5349 void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) {
5350 assert(list != NULL, "list can't be null");
5351 if (!list->is_empty()) {
5352 MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag);
5353 _hrm.insert_list_into_free_list(list);
5354 }
5355 }
5356
5480
5481 void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) {
5482 if (!G1VerifyBitmaps) return;
5483
5484 guarantee(verify_bitmaps(caller, hr), "bitmap verification");
5485 }
5486
5487 class G1VerifyBitmapClosure : public HeapRegionClosure {
5488 private:
5489 const char* _caller;
5490 G1CollectedHeap* _g1h;
5491 bool _failures;
5492
5493 public:
5494 G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) :
5495 _caller(caller), _g1h(g1h), _failures(false) { }
5496
5497 bool failures() { return _failures; }
5498
5499 virtual bool doHeapRegion(HeapRegion* hr) {
5500 if (hr->is_continues_humongous()) return false;
5501
5502 bool result = _g1h->verify_bitmaps(_caller, hr);
5503 if (!result) {
5504 _failures = true;
5505 }
5506 return false;
5507 }
5508 };
5509
5510 void G1CollectedHeap::check_bitmaps(const char* caller) {
5511 if (!G1VerifyBitmaps) return;
5512
5513 G1VerifyBitmapClosure cl(caller, this);
5514 heap_region_iterate(&cl);
5515 guarantee(!cl.failures(), "bitmap verification");
5516 }
5517
5518 class G1CheckCSetFastTableClosure : public HeapRegionClosure {
5519 private:
5520 bool _failures;
5521 public:
5755 //
5756 // It is not required to check whether the object has been found dead by marking
5757 // or not, in fact it would prevent reclamation within a concurrent cycle, as
5758 // all objects allocated during that time are considered live.
5759 // SATB marking is even more conservative than the remembered set.
5760 // So if at this point in the collection there is no remembered set entry,
5761 // nobody has a reference to it.
5762 // At the start of collection we flush all refinement logs, and remembered sets
5763 // are completely up-to-date wrt to references to the humongous object.
5764 //
5765 // Other implementation considerations:
5766 // - never consider object arrays at this time because they would pose
5767 // considerable effort for cleaning up the the remembered sets. This is
5768 // required because stale remembered sets might reference locations that
5769 // are currently allocated into.
5770 uint region_idx = r->hrm_index();
5771 if (!g1h->is_humongous_reclaim_candidate(region_idx) ||
5772 !r->rem_set()->is_empty()) {
5773
5774 if (G1TraceEagerReclaimHumongousObjects) {
5775 gclog_or_tty->print_cr("Live humongous region %u size " SIZE_FORMAT " start " PTR_FORMAT " length %u with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5776 region_idx,
5777 (size_t)obj->size() * HeapWordSize,
5778 p2i(r->bottom()),
5779 r->region_num(),
5780 r->rem_set()->occupied(),
5781 r->rem_set()->strong_code_roots_list_length(),
5782 next_bitmap->isMarked(r->bottom()),
5783 g1h->is_humongous_reclaim_candidate(region_idx),
5784 obj->is_typeArray()
5785 );
5786 }
5787
5788 return false;
5789 }
5790
5791 guarantee(obj->is_typeArray(),
5792 "Only eagerly reclaiming type arrays is supported, but the object "
5793 PTR_FORMAT " is not.", p2i(r->bottom()));
5794
5795 if (G1TraceEagerReclaimHumongousObjects) {
5796 gclog_or_tty->print_cr("Dead humongous region %u size " SIZE_FORMAT " start " PTR_FORMAT " length %u with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5797 region_idx,
5798 (size_t)obj->size() * HeapWordSize,
5799 p2i(r->bottom()),
5800 r->region_num(),
5801 r->rem_set()->occupied(),
5802 r->rem_set()->strong_code_roots_list_length(),
5803 next_bitmap->isMarked(r->bottom()),
5804 g1h->is_humongous_reclaim_candidate(region_idx),
5805 obj->is_typeArray()
5806 );
5807 }
5808 // Need to clear mark bit of the humongous object if already set.
5809 if (next_bitmap->isMarked(r->bottom())) {
5810 next_bitmap->clear(r->bottom());
5811 }
5812 _freed_bytes += r->used();
5813 r->set_containing_set(NULL);
5814 _humongous_regions_removed.increment(1u, r->capacity());
5815 g1h->free_humongous_region(r, _free_region_list, false);
5816
5817 return false;
5818 }
5819
5820 HeapRegionSetCount& humongous_free_count() {
5821 return _humongous_regions_removed;
5822 }
5823
5824 size_t bytes_freed() const {
5825 return _freed_bytes;
5826 }
5827
5828 size_t humongous_reclaimed() const {
5829 return _humongous_regions_removed.length();
5830 }
5831 };
5832
5833 void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
5834 assert_at_safepoint(true);
5835
6030
6031 class RebuildRegionSetsClosure : public HeapRegionClosure {
6032 private:
6033 bool _free_list_only;
6034 HeapRegionSet* _old_set;
6035 HeapRegionManager* _hrm;
6036 size_t _total_used;
6037
6038 public:
6039 RebuildRegionSetsClosure(bool free_list_only,
6040 HeapRegionSet* old_set, HeapRegionManager* hrm) :
6041 _free_list_only(free_list_only),
6042 _old_set(old_set), _hrm(hrm), _total_used(0) {
6043 assert(_hrm->num_free_regions() == 0, "pre-condition");
6044 if (!free_list_only) {
6045 assert(_old_set->is_empty(), "pre-condition");
6046 }
6047 }
6048
6049 bool doHeapRegion(HeapRegion* r) {
6050 if (r->is_continues_humongous()) {
6051 return false;
6052 }
6053
6054 if (r->is_empty()) {
6055 // Add free regions to the free list
6056 r->set_free();
6057 r->set_allocation_context(AllocationContext::system());
6058 _hrm->insert_into_free_list(r);
6059 } else if (!_free_list_only) {
6060 assert(!r->is_young(), "we should not come across young regions");
6061
6062 if (r->is_humongous()) {
6063 // We ignore humongous regions. We left the humongous set unchanged.
6064 } else {
6065 // Objects that were compacted would have ended up on regions
6066 // that were previously old or free. Archive regions (which are
6067 // old) will not have been touched.
6068 assert(r->is_free() || r->is_old(), "invariant");
6069 // We now consider them old, so register as such. Leave
6070 // archive regions set that way, however, while still adding
6071 // them to the old set.
6072 if (!r->is_archive()) {
6073 r->set_old();
6221 // Heap region set verification
6222
6223 class VerifyRegionListsClosure : public HeapRegionClosure {
6224 private:
6225 HeapRegionSet* _old_set;
6226 HeapRegionSet* _humongous_set;
6227 HeapRegionManager* _hrm;
6228
6229 public:
6230 HeapRegionSetCount _old_count;
6231 HeapRegionSetCount _humongous_count;
6232 HeapRegionSetCount _free_count;
6233
6234 VerifyRegionListsClosure(HeapRegionSet* old_set,
6235 HeapRegionSet* humongous_set,
6236 HeapRegionManager* hrm) :
6237 _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm),
6238 _old_count(), _humongous_count(), _free_count(){ }
6239
6240 bool doHeapRegion(HeapRegion* hr) {
6241 if (hr->is_continues_humongous()) {
6242 return false;
6243 }
6244
6245 if (hr->is_young()) {
6246 // TODO
6247 } else if (hr->is_starts_humongous()) {
6248 assert(hr->containing_set() == _humongous_set, "Heap region %u is starts humongous but not in humongous set.", hr->hrm_index());
6249 _humongous_count.increment(1u, hr->capacity());
6250 } else if (hr->is_empty()) {
6251 assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index());
6252 _free_count.increment(1u, hr->capacity());
6253 } else if (hr->is_old()) {
6254 assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index());
6255 _old_count.increment(1u, hr->capacity());
6256 } else {
6257 // There are no other valid region types. Check for one invalid
6258 // one we can identify: pinned without old or humongous set.
6259 assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index());
6260 ShouldNotReachHere();
6261 }
6262 return false;
6263 }
6264
6265 void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) {
6266 guarantee(old_set->length() == _old_count.length(), "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length());
6267 guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), "Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6268 old_set->total_capacity_bytes(), _old_count.capacity());
|
303 // Index of last region in the series + 1.
304 uint last = first + num_regions;
305
306 // We need to initialize the region(s) we just discovered. This is
307 // a bit tricky given that it can happen concurrently with
308 // refinement threads refining cards on these regions and
309 // potentially wanting to refine the BOT as they are scanning
310 // those cards (this can happen shortly after a cleanup; see CR
311 // 6991377). So we have to set up the region(s) carefully and in
312 // a specific order.
313
314 // The word size sum of all the regions we will allocate.
315 size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
316 assert(word_size <= word_size_sum, "sanity");
317
318 // This will be the "starts humongous" region.
319 HeapRegion* first_hr = region_at(first);
320 // The header of the new object will be placed at the bottom of
321 // the first region.
322 HeapWord* new_obj = first_hr->bottom();
323 // This will be the new top of the new object.
324 HeapWord* obj_top = new_obj + word_size;
325
326 // First, we need to zero the header of the space that we will be
327 // allocating. When we update top further down, some refinement
328 // threads might try to scan the region. By zeroing the header we
329 // ensure that any thread that will try to scan the region will
330 // come across the zero klass word and bail out.
331 //
332 // NOTE: It would not have been correct to have used
333 // CollectedHeap::fill_with_object() and make the space look like
334 // an int array. The thread that is doing the allocation will
335 // later update the object header to a potentially different array
336 // type and, for a very short period of time, the klass and length
337 // fields will be inconsistent. This could cause a refinement
338 // thread to calculate the object size incorrectly.
339 Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
340
341 // We will set up the first region as "starts humongous". This
342 // will also update the BOT covering all the regions to reflect
343 // that there is a single object that starts at the bottom of the
344 // first region.
345 first_hr->set_starts_humongous(obj_top);
346 first_hr->set_allocation_context(context);
347 // Then, if there are any, we will set up the "continues
348 // humongous" regions.
349 HeapRegion* hr = NULL;
350 for (uint i = first + 1; i < last; ++i) {
351 hr = region_at(i);
352 hr->set_continues_humongous(first_hr);
353 hr->set_allocation_context(context);
354 }
355
356 // Up to this point no concurrent thread would have been able to
357 // do any scanning on any region in this series. All the top
358 // fields still point to bottom, so the intersection between
359 // [bottom,top] and [card_start,card_end] will be empty. Before we
360 // update the top fields, we'll do a storestore to make sure that
361 // no thread sees the update to top before the zeroing of the
362 // object header and the BOT initialization.
363 OrderAccess::storestore();
364
365 // Now that the BOT and the object header have been initialized,
366 // we can update top of the "starts humongous" region.
367 first_hr->set_top(MIN2(first_hr->end(), obj_top));
368 if (_hr_printer.is_active()) {
369 if ((first + 1) == last) {
370 // the series has a single humongous region
371 _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, obj_top);
372 } else {
373 // the series has more than one humongous regions
374 _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, first_hr->end());
375 }
376 }
377
378 // Now, we will update the top fields of the "continues humongous"
379 // regions. The reason we need to do this is that, otherwise,
380 // these regions would look empty and this will confuse parts of
381 // G1. For example, the code that looks for a consecutive number
382 // of empty regions will consider them empty and try to
383 // re-allocate them. We can extend is_empty() to also include
384 // !is_continues_humongous(), but it is easier to just update the top
385 // fields here. The way we set top for all regions (i.e., top ==
386 // end for all regions but the last one, top == new_top for the
387 // last one) is actually used when we will free up the humongous
388 // region in free_humongous_region().
389 hr = NULL;
390 for (uint i = first + 1; i < last; ++i) {
391 hr = region_at(i);
392 if ((i + 1) == last) {
393 // last continues humongous region
394 assert(hr->bottom() < obj_top && obj_top <= hr->end(),
395 "new_top should fall on this region");
396 hr->set_top(obj_top);
397 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, obj_top);
398 } else {
399 // not last one
400 assert(obj_top > hr->end(), "obj_top should be above this region");
401 hr->set_top(hr->end());
402 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end());
403 }
404 }
405 // If we have continues humongous regions (hr != NULL), its top should
406 // match obj_top.
407 assert(hr == NULL || (hr->top() == obj_top), "sanity");
408 check_bitmaps("Humongous Region Allocation", first_hr);
409
410 increase_used(word_size * HeapWordSize);
411
412 for (uint i = first; i < last; ++i) {
413 _humongous_set.add(region_at(i));
414 }
415
416 return new_obj;
417 }
418
419 // If could fit into free regions w/o expansion, try.
420 // Otherwise, if can expand, do so.
421 // Otherwise, if using ex regions might help, try with ex given back.
422 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) {
423 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
424
425 verify_region_sets_optional();
426
427 uint first = G1_NO_HRM_INDEX;
428 uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords);
429
430 if (obj_regions == 1) {
431 // Only one region to allocate, try to use a fast path by directly allocating
432 // from the free lists. Do not try to expand here, we will potentially do that
433 // later.
434 HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */);
1111 HeapWord* result = humongous_obj_allocate(word_size, context);
1112 if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) {
1113 collector_state()->set_initiate_conc_mark_if_possible(true);
1114 }
1115 return result;
1116 }
1117
1118 ShouldNotReachHere();
1119 }
1120
1121 class PostMCRemSetClearClosure: public HeapRegionClosure {
1122 G1CollectedHeap* _g1h;
1123 ModRefBarrierSet* _mr_bs;
1124 public:
1125 PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) :
1126 _g1h(g1h), _mr_bs(mr_bs) {}
1127
1128 bool doHeapRegion(HeapRegion* r) {
1129 HeapRegionRemSet* hrrs = r->rem_set();
1130
1131 _g1h->reset_gc_time_stamps(r);
1132
1133 if (r->is_continues_humongous()) {
1134 // We'll assert that the strong code root list and RSet is empty
1135 assert(hrrs->strong_code_roots_list_length() == 0, "sanity");
1136 assert(hrrs->occupied() == 0, "RSet should be empty");
1137 return false;
1138 }
1139
1140 hrrs->clear();
1141 // You might think here that we could clear just the cards
1142 // corresponding to the used region. But no: if we leave a dirty card
1143 // in a region we might allocate into, then it would prevent that card
1144 // from being enqueued, and cause it to be missed.
1145 // Re: the performance cost: we shouldn't be doing full GC anyway!
1146 _mr_bs->clear(MemRegion(r->bottom(), r->end()));
1147
1148 return false;
1149 }
1150 };
1151
1152 void G1CollectedHeap::clear_rsets_post_compaction() {
1153 PostMCRemSetClearClosure rs_clear(this, g1_barrier_set());
1154 heap_region_iterate(&rs_clear);
1155 }
1156
1157 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
1158 G1CollectedHeap* _g1h;
1159 UpdateRSOopClosure _cl;
1178
1179 public:
1180 ParRebuildRSTask(G1CollectedHeap* g1) :
1181 AbstractGangTask("ParRebuildRSTask"), _g1(g1), _hrclaimer(g1->workers()->active_workers()) {}
1182
1183 void work(uint worker_id) {
1184 RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id);
1185 _g1->heap_region_par_iterate(&rebuild_rs, worker_id, &_hrclaimer);
1186 }
1187 };
1188
1189 class PostCompactionPrinterClosure: public HeapRegionClosure {
1190 private:
1191 G1HRPrinter* _hr_printer;
1192 public:
1193 bool doHeapRegion(HeapRegion* hr) {
1194 assert(!hr->is_young(), "not expecting to find young regions");
1195 if (hr->is_free()) {
1196 // We only generate output for non-empty regions.
1197 } else if (hr->is_starts_humongous()) {
1198 _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous);
1199 } else if (hr->is_continues_humongous()) {
1200 _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous);
1201 } else if (hr->is_archive()) {
1202 _hr_printer->post_compaction(hr, G1HRPrinter::Archive);
1203 } else if (hr->is_old()) {
1204 _hr_printer->post_compaction(hr, G1HRPrinter::Old);
1205 } else {
1206 ShouldNotReachHere();
1207 }
1208 return false;
1209 }
1210
1211 PostCompactionPrinterClosure(G1HRPrinter* hr_printer)
1212 : _hr_printer(hr_printer) { }
1213 };
1214
1215 void G1CollectedHeap::print_hrm_post_compaction() {
1216 PostCompactionPrinterClosure cl(hr_printer());
1217 heap_region_iterate(&cl);
1218 }
2190 (ParallelGCThreads > 1),
2191 // mt discovery
2192 ParallelGCThreads,
2193 // degree of mt discovery
2194 true,
2195 // Reference discovery is atomic
2196 &_is_alive_closure_stw);
2197 // is alive closure
2198 // (for efficiency/performance)
2199 }
2200
2201 CollectorPolicy* G1CollectedHeap::collector_policy() const {
2202 return g1_policy();
2203 }
2204
2205 size_t G1CollectedHeap::capacity() const {
2206 return _hrm.length() * HeapRegion::GrainBytes;
2207 }
2208
2209 void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) {
2210 hr->reset_gc_time_stamp();
2211 }
2212
2213 #ifndef PRODUCT
2214
2215 class CheckGCTimeStampsHRClosure : public HeapRegionClosure {
2216 private:
2217 unsigned _gc_time_stamp;
2218 bool _failures;
2219
2220 public:
2221 CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) :
2222 _gc_time_stamp(gc_time_stamp), _failures(false) { }
2223
2224 virtual bool doHeapRegion(HeapRegion* hr) {
2225 unsigned region_gc_time_stamp = hr->get_gc_time_stamp();
2226 if (_gc_time_stamp != region_gc_time_stamp) {
2227 gclog_or_tty->print_cr("Region " HR_FORMAT " has GC time stamp = %d, "
2228 "expected %d", HR_FORMAT_PARAMS(hr),
2229 region_gc_time_stamp, _gc_time_stamp);
2230 _failures = true;
2258 }
2259
2260 // Computes the sum of the storage used by the various regions.
2261 size_t G1CollectedHeap::used() const {
2262 size_t result = _summary_bytes_used + _allocator->used_in_alloc_regions();
2263 if (_archive_allocator != NULL) {
2264 result += _archive_allocator->used();
2265 }
2266 return result;
2267 }
2268
2269 size_t G1CollectedHeap::used_unlocked() const {
2270 return _summary_bytes_used;
2271 }
2272
2273 class SumUsedClosure: public HeapRegionClosure {
2274 size_t _used;
2275 public:
2276 SumUsedClosure() : _used(0) {}
2277 bool doHeapRegion(HeapRegion* r) {
2278 _used += r->used();
2279 return false;
2280 }
2281 size_t result() { return _used; }
2282 };
2283
2284 size_t G1CollectedHeap::recalculate_used() const {
2285 double recalculate_used_start = os::elapsedTime();
2286
2287 SumUsedClosure blk;
2288 heap_region_iterate(&blk);
2289
2290 g1_policy()->phase_times()->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0);
2291 return blk.result();
2292 }
2293
2294 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {
2295 switch (cause) {
2296 case GCCause::_gc_locker: return GCLockerInvokesConcurrent;
2297 case GCCause::_java_lang_system_gc: return ExplicitGCInvokesConcurrent;
2298 case GCCause::_dcmd_gc_run: return ExplicitGCInvokesConcurrent;
2479 // Schedule a standard evacuation pause. We're setting word_size
2480 // to 0 which means that we are not requesting a post-GC allocation.
2481 VM_G1IncCollectionPause op(gc_count_before,
2482 0, /* word_size */
2483 false, /* should_initiate_conc_mark */
2484 g1_policy()->max_pause_time_ms(),
2485 cause);
2486 VMThread::execute(&op);
2487 } else {
2488 // Schedule a Full GC.
2489 VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);
2490 VMThread::execute(&op);
2491 }
2492 }
2493 } while (retry_gc);
2494 }
2495
2496 bool G1CollectedHeap::is_in(const void* p) const {
2497 if (_hrm.reserved().contains(p)) {
2498 // Given that we know that p is in the reserved space,
2499 // heap_region_containing() should successfully
2500 // return the containing region.
2501 HeapRegion* hr = heap_region_containing(p);
2502 return hr->is_in(p);
2503 } else {
2504 return false;
2505 }
2506 }
2507
2508 #ifdef ASSERT
2509 bool G1CollectedHeap::is_in_exact(const void* p) const {
2510 bool contains = reserved_region().contains(p);
2511 bool available = _hrm.is_available(addr_to_region((HeapWord*)p));
2512 if (contains && available) {
2513 return true;
2514 } else {
2515 return false;
2516 }
2517 }
2518 #endif
2519
2520 bool G1CollectedHeap::obj_in_cs(oop obj) {
2521 HeapRegion* r = _hrm.addr_to_region((HeapWord*) obj);
3018 _vo(vo),
3019 _failures(false) {}
3020
3021 bool failures() {
3022 return _failures;
3023 }
3024
3025 bool doHeapRegion(HeapRegion* r) {
3026 // For archive regions, verify there are no heap pointers to
3027 // non-pinned regions. For all others, verify liveness info.
3028 if (r->is_archive()) {
3029 VerifyArchiveRegionClosure verify_oop_pointers(r);
3030 r->object_iterate(&verify_oop_pointers);
3031 return true;
3032 }
3033 if (!r->is_continues_humongous()) {
3034 bool failures = false;
3035 r->verify(_vo, &failures);
3036 if (failures) {
3037 _failures = true;
3038 } else if (!r->is_starts_humongous()) {
3039 VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo);
3040 r->object_iterate(¬_dead_yet_cl);
3041 if (_vo != VerifyOption_G1UseNextMarking) {
3042 if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
3043 gclog_or_tty->print_cr("[" PTR_FORMAT "," PTR_FORMAT "] "
3044 "max_live_bytes " SIZE_FORMAT " "
3045 "< calculated " SIZE_FORMAT,
3046 p2i(r->bottom()), p2i(r->end()),
3047 r->max_live_bytes(),
3048 not_dead_yet_cl.live_bytes());
3049 _failures = true;
3050 }
3051 } else {
3052 // When vo == UseNextMarking we cannot currently do a sanity
3053 // check on the live bytes as the calculation has not been
3054 // finalized yet.
3055 }
3056 }
3057 }
3058 return false; // stop the region iteration if we hit a failure
5272 assert(free_list != NULL, "pre-condition");
5273
5274 if (G1VerifyBitmaps) {
5275 MemRegion mr(hr->bottom(), hr->end());
5276 concurrent_mark()->clearRangePrevBitmap(mr);
5277 }
5278
5279 // Clear the card counts for this region.
5280 // Note: we only need to do this if the region is not young
5281 // (since we don't refine cards in young regions).
5282 if (!hr->is_young()) {
5283 _cg1r->hot_card_cache()->reset_card_counts(hr);
5284 }
5285 hr->hr_clear(par, true /* clear_space */, locked /* locked */);
5286 free_list->add_ordered(hr);
5287 }
5288
5289 void G1CollectedHeap::free_humongous_region(HeapRegion* hr,
5290 FreeRegionList* free_list,
5291 bool par) {
5292 assert(hr->is_humongous(), "this is only for humongous regions");
5293 assert(free_list != NULL, "pre-condition");
5294 hr->clear_humongous();
5295 free_region(hr, free_list, par);
5296 }
5297
5298 void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed,
5299 const HeapRegionSetCount& humongous_regions_removed) {
5300 if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) {
5301 MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag);
5302 _old_set.bulk_remove(old_regions_removed);
5303 _humongous_set.bulk_remove(humongous_regions_removed);
5304 }
5305
5306 }
5307
5308 void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) {
5309 assert(list != NULL, "list can't be null");
5310 if (!list->is_empty()) {
5311 MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag);
5312 _hrm.insert_list_into_free_list(list);
5313 }
5314 }
5315
5439
5440 void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) {
5441 if (!G1VerifyBitmaps) return;
5442
5443 guarantee(verify_bitmaps(caller, hr), "bitmap verification");
5444 }
5445
5446 class G1VerifyBitmapClosure : public HeapRegionClosure {
5447 private:
5448 const char* _caller;
5449 G1CollectedHeap* _g1h;
5450 bool _failures;
5451
5452 public:
5453 G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) :
5454 _caller(caller), _g1h(g1h), _failures(false) { }
5455
5456 bool failures() { return _failures; }
5457
5458 virtual bool doHeapRegion(HeapRegion* hr) {
5459 bool result = _g1h->verify_bitmaps(_caller, hr);
5460 if (!result) {
5461 _failures = true;
5462 }
5463 return false;
5464 }
5465 };
5466
5467 void G1CollectedHeap::check_bitmaps(const char* caller) {
5468 if (!G1VerifyBitmaps) return;
5469
5470 G1VerifyBitmapClosure cl(caller, this);
5471 heap_region_iterate(&cl);
5472 guarantee(!cl.failures(), "bitmap verification");
5473 }
5474
5475 class G1CheckCSetFastTableClosure : public HeapRegionClosure {
5476 private:
5477 bool _failures;
5478 public:
5712 //
5713 // It is not required to check whether the object has been found dead by marking
5714 // or not, in fact it would prevent reclamation within a concurrent cycle, as
5715 // all objects allocated during that time are considered live.
5716 // SATB marking is even more conservative than the remembered set.
5717 // So if at this point in the collection there is no remembered set entry,
5718 // nobody has a reference to it.
5719 // At the start of collection we flush all refinement logs, and remembered sets
5720 // are completely up-to-date wrt to references to the humongous object.
5721 //
5722 // Other implementation considerations:
5723 // - never consider object arrays at this time because they would pose
5724 // considerable effort for cleaning up the the remembered sets. This is
5725 // required because stale remembered sets might reference locations that
5726 // are currently allocated into.
5727 uint region_idx = r->hrm_index();
5728 if (!g1h->is_humongous_reclaim_candidate(region_idx) ||
5729 !r->rem_set()->is_empty()) {
5730
5731 if (G1TraceEagerReclaimHumongousObjects) {
5732 gclog_or_tty->print_cr("Live humongous region %u object size " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5733 region_idx,
5734 (size_t)obj->size() * HeapWordSize,
5735 p2i(r->bottom()),
5736 r->rem_set()->occupied(),
5737 r->rem_set()->strong_code_roots_list_length(),
5738 next_bitmap->isMarked(r->bottom()),
5739 g1h->is_humongous_reclaim_candidate(region_idx),
5740 obj->is_typeArray()
5741 );
5742 }
5743
5744 return false;
5745 }
5746
5747 guarantee(obj->is_typeArray(),
5748 "Only eagerly reclaiming type arrays is supported, but the object "
5749 PTR_FORMAT " is not.", p2i(r->bottom()));
5750
5751 if (G1TraceEagerReclaimHumongousObjects) {
5752 gclog_or_tty->print_cr("Dead humongous region %u object size " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5753 region_idx,
5754 (size_t)obj->size() * HeapWordSize,
5755 p2i(r->bottom()),
5756 r->rem_set()->occupied(),
5757 r->rem_set()->strong_code_roots_list_length(),
5758 next_bitmap->isMarked(r->bottom()),
5759 g1h->is_humongous_reclaim_candidate(region_idx),
5760 obj->is_typeArray()
5761 );
5762 }
5763 // Need to clear mark bit of the humongous object if already set.
5764 if (next_bitmap->isMarked(r->bottom())) {
5765 next_bitmap->clear(r->bottom());
5766 }
5767 do {
5768 HeapRegion* next = g1h->next_region_by_index(r);
5769 _freed_bytes += r->used();
5770 r->set_containing_set(NULL);
5771 _humongous_regions_removed.increment(1u, r->capacity());
5772 g1h->free_humongous_region(r, _free_region_list, false);
5773 r = next;
5774 } while (r != NULL && r->is_continues_humongous());
5775
5776 return false;
5777 }
5778
5779 HeapRegionSetCount& humongous_free_count() {
5780 return _humongous_regions_removed;
5781 }
5782
5783 size_t bytes_freed() const {
5784 return _freed_bytes;
5785 }
5786
5787 size_t humongous_reclaimed() const {
5788 return _humongous_regions_removed.length();
5789 }
5790 };
5791
5792 void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
5793 assert_at_safepoint(true);
5794
5989
5990 class RebuildRegionSetsClosure : public HeapRegionClosure {
5991 private:
5992 bool _free_list_only;
5993 HeapRegionSet* _old_set;
5994 HeapRegionManager* _hrm;
5995 size_t _total_used;
5996
5997 public:
5998 RebuildRegionSetsClosure(bool free_list_only,
5999 HeapRegionSet* old_set, HeapRegionManager* hrm) :
6000 _free_list_only(free_list_only),
6001 _old_set(old_set), _hrm(hrm), _total_used(0) {
6002 assert(_hrm->num_free_regions() == 0, "pre-condition");
6003 if (!free_list_only) {
6004 assert(_old_set->is_empty(), "pre-condition");
6005 }
6006 }
6007
6008 bool doHeapRegion(HeapRegion* r) {
6009 if (r->is_empty()) {
6010 // Add free regions to the free list
6011 r->set_free();
6012 r->set_allocation_context(AllocationContext::system());
6013 _hrm->insert_into_free_list(r);
6014 } else if (!_free_list_only) {
6015 assert(!r->is_young(), "we should not come across young regions");
6016
6017 if (r->is_humongous()) {
6018 // We ignore humongous regions. We left the humongous set unchanged.
6019 } else {
6020 // Objects that were compacted would have ended up on regions
6021 // that were previously old or free. Archive regions (which are
6022 // old) will not have been touched.
6023 assert(r->is_free() || r->is_old(), "invariant");
6024 // We now consider them old, so register as such. Leave
6025 // archive regions set that way, however, while still adding
6026 // them to the old set.
6027 if (!r->is_archive()) {
6028 r->set_old();
6176 // Heap region set verification
6177
6178 class VerifyRegionListsClosure : public HeapRegionClosure {
6179 private:
6180 HeapRegionSet* _old_set;
6181 HeapRegionSet* _humongous_set;
6182 HeapRegionManager* _hrm;
6183
6184 public:
6185 HeapRegionSetCount _old_count;
6186 HeapRegionSetCount _humongous_count;
6187 HeapRegionSetCount _free_count;
6188
6189 VerifyRegionListsClosure(HeapRegionSet* old_set,
6190 HeapRegionSet* humongous_set,
6191 HeapRegionManager* hrm) :
6192 _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm),
6193 _old_count(), _humongous_count(), _free_count(){ }
6194
6195 bool doHeapRegion(HeapRegion* hr) {
6196 if (hr->is_young()) {
6197 // TODO
6198 } else if (hr->is_humongous()) {
6199 assert(hr->containing_set() == _humongous_set, "Heap region %u is humongous but not in humongous set.", hr->hrm_index());
6200 _humongous_count.increment(1u, hr->capacity());
6201 } else if (hr->is_empty()) {
6202 assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index());
6203 _free_count.increment(1u, hr->capacity());
6204 } else if (hr->is_old()) {
6205 assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index());
6206 _old_count.increment(1u, hr->capacity());
6207 } else {
6208 // There are no other valid region types. Check for one invalid
6209 // one we can identify: pinned without old or humongous set.
6210 assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index());
6211 ShouldNotReachHere();
6212 }
6213 return false;
6214 }
6215
6216 void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) {
6217 guarantee(old_set->length() == _old_count.length(), "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length());
6218 guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), "Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6219 old_set->total_capacity_bytes(), _old_count.capacity());
|