303 // Index of last region in the series + 1.
304 uint last = first + num_regions;
305
306 // We need to initialize the region(s) we just discovered. This is
307 // a bit tricky given that it can happen concurrently with
308 // refinement threads refining cards on these regions and
309 // potentially wanting to refine the BOT as they are scanning
310 // those cards (this can happen shortly after a cleanup; see CR
311 // 6991377). So we have to set up the region(s) carefully and in
312 // a specific order.
313
314 // The word size sum of all the regions we will allocate.
315 size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
316 assert(word_size <= word_size_sum, "sanity");
317
318 // This will be the "starts humongous" region.
319 HeapRegion* first_hr = region_at(first);
320 // The header of the new object will be placed at the bottom of
321 // the first region.
322 HeapWord* new_obj = first_hr->bottom();
323 // This will be the new end of the first region in the series that
324 // should also match the end of the last region in the series.
325 HeapWord* new_end = new_obj + word_size_sum;
326 // This will be the new top of the first region that will reflect
327 // this allocation.
328 HeapWord* new_top = new_obj + word_size;
329
330 // First, we need to zero the header of the space that we will be
331 // allocating. When we update top further down, some refinement
332 // threads might try to scan the region. By zeroing the header we
333 // ensure that any thread that will try to scan the region will
334 // come across the zero klass word and bail out.
335 //
336 // NOTE: It would not have been correct to have used
337 // CollectedHeap::fill_with_object() and make the space look like
338 // an int array. The thread that is doing the allocation will
339 // later update the object header to a potentially different array
340 // type and, for a very short period of time, the klass and length
341 // fields will be inconsistent. This could cause a refinement
342 // thread to calculate the object size incorrectly.
343 Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
344
345 // We will set up the first region as "starts humongous". This
346 // will also update the BOT covering all the regions to reflect
347 // that there is a single object that starts at the bottom of the
348 // first region.
349 first_hr->set_starts_humongous(new_top, new_end);
350 first_hr->set_allocation_context(context);
351 // Then, if there are any, we will set up the "continues
352 // humongous" regions.
353 HeapRegion* hr = NULL;
354 for (uint i = first + 1; i < last; ++i) {
355 hr = region_at(i);
356 hr->set_continues_humongous(first_hr);
357 hr->set_allocation_context(context);
358 }
359 // If we have "continues humongous" regions (hr != NULL), then the
360 // end of the last one should match new_end.
361 assert(hr == NULL || hr->end() == new_end, "sanity");
362
363 // Up to this point no concurrent thread would have been able to
364 // do any scanning on any region in this series. All the top
365 // fields still point to bottom, so the intersection between
366 // [bottom,top] and [card_start,card_end] will be empty. Before we
367 // update the top fields, we'll do a storestore to make sure that
368 // no thread sees the update to top before the zeroing of the
369 // object header and the BOT initialization.
370 OrderAccess::storestore();
371
372 // Now that the BOT and the object header have been initialized,
373 // we can update top of the "starts humongous" region.
374 assert(first_hr->bottom() < new_top && new_top <= first_hr->end(),
375 "new_top should be in this region");
376 first_hr->set_top(new_top);
377 if (_hr_printer.is_active()) {
378 HeapWord* bottom = first_hr->bottom();
379 HeapWord* end = first_hr->orig_end();
380 if ((first + 1) == last) {
381 // the series has a single humongous region
382 _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, new_top);
383 } else {
384 // the series has more than one humongous regions
385 _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, end);
386 }
387 }
388
389 // Now, we will update the top fields of the "continues humongous"
390 // regions. The reason we need to do this is that, otherwise,
391 // these regions would look empty and this will confuse parts of
392 // G1. For example, the code that looks for a consecutive number
393 // of empty regions will consider them empty and try to
394 // re-allocate them. We can extend is_empty() to also include
395 // !is_continues_humongous(), but it is easier to just update the top
396 // fields here. The way we set top for all regions (i.e., top ==
397 // end for all regions but the last one, top == new_top for the
398 // last one) is actually used when we will free up the humongous
399 // region in free_humongous_region().
400 hr = NULL;
401 for (uint i = first + 1; i < last; ++i) {
402 hr = region_at(i);
403 if ((i + 1) == last) {
404 // last continues humongous region
405 assert(hr->bottom() < new_top && new_top <= hr->end(),
406 "new_top should fall on this region");
407 hr->set_top(new_top);
408 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, new_top);
409 } else {
410 // not last one
411 assert(new_top > hr->end(), "new_top should be above this region");
412 hr->set_top(hr->end());
413 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end());
414 }
415 }
416 // If we have continues humongous regions (hr != NULL), then the
417 // end of the last one should match new_end and its top should
418 // match new_top.
419 assert(hr == NULL ||
420 (hr->end() == new_end && hr->top() == new_top), "sanity");
421 check_bitmaps("Humongous Region Allocation", first_hr);
422
423 assert(first_hr->used() == word_size * HeapWordSize, "invariant");
424 increase_used(first_hr->used());
425 _humongous_set.add(first_hr);
426
427 return new_obj;
428 }
429
430 // If could fit into free regions w/o expansion, try.
431 // Otherwise, if can expand, do so.
432 // Otherwise, if using ex regions might help, try with ex given back.
433 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) {
434 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
435
436 verify_region_sets_optional();
437
438 uint first = G1_NO_HRM_INDEX;
439 uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords);
440
441 if (obj_regions == 1) {
442 // Only one region to allocate, try to use a fast path by directly allocating
443 // from the free lists. Do not try to expand here, we will potentially do that
444 // later.
445 HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */);
1122 HeapWord* result = humongous_obj_allocate(word_size, context);
1123 if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) {
1124 collector_state()->set_initiate_conc_mark_if_possible(true);
1125 }
1126 return result;
1127 }
1128
1129 ShouldNotReachHere();
1130 }
1131
1132 class PostMCRemSetClearClosure: public HeapRegionClosure {
1133 G1CollectedHeap* _g1h;
1134 ModRefBarrierSet* _mr_bs;
1135 public:
1136 PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) :
1137 _g1h(g1h), _mr_bs(mr_bs) {}
1138
1139 bool doHeapRegion(HeapRegion* r) {
1140 HeapRegionRemSet* hrrs = r->rem_set();
1141
1142 if (r->is_continues_humongous()) {
1143 // We'll assert that the strong code root list and RSet is empty
1144 assert(hrrs->strong_code_roots_list_length() == 0, "sanity");
1145 assert(hrrs->occupied() == 0, "RSet should be empty");
1146 return false;
1147 }
1148
1149 _g1h->reset_gc_time_stamps(r);
1150 hrrs->clear();
1151 // You might think here that we could clear just the cards
1152 // corresponding to the used region. But no: if we leave a dirty card
1153 // in a region we might allocate into, then it would prevent that card
1154 // from being enqueued, and cause it to be missed.
1155 // Re: the performance cost: we shouldn't be doing full GC anyway!
1156 _mr_bs->clear(MemRegion(r->bottom(), r->end()));
1157
1158 return false;
1159 }
1160 };
1161
1162 void G1CollectedHeap::clear_rsets_post_compaction() {
1163 PostMCRemSetClearClosure rs_clear(this, g1_barrier_set());
1164 heap_region_iterate(&rs_clear);
1165 }
1166
1167 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
1168 G1CollectedHeap* _g1h;
1188
1189 public:
1190 ParRebuildRSTask(G1CollectedHeap* g1) :
1191 AbstractGangTask("ParRebuildRSTask"), _g1(g1), _hrclaimer(g1->workers()->active_workers()) {}
1192
1193 void work(uint worker_id) {
1194 RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id);
1195 _g1->heap_region_par_iterate(&rebuild_rs, worker_id, &_hrclaimer);
1196 }
1197 };
1198
1199 class PostCompactionPrinterClosure: public HeapRegionClosure {
1200 private:
1201 G1HRPrinter* _hr_printer;
1202 public:
1203 bool doHeapRegion(HeapRegion* hr) {
1204 assert(!hr->is_young(), "not expecting to find young regions");
1205 if (hr->is_free()) {
1206 // We only generate output for non-empty regions.
1207 } else if (hr->is_starts_humongous()) {
1208 if (hr->region_num() == 1) {
1209 // single humongous region
1210 _hr_printer->post_compaction(hr, G1HRPrinter::SingleHumongous);
1211 } else {
1212 _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous);
1213 }
1214 } else if (hr->is_continues_humongous()) {
1215 _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous);
1216 } else if (hr->is_archive()) {
1217 _hr_printer->post_compaction(hr, G1HRPrinter::Archive);
1218 } else if (hr->is_old()) {
1219 _hr_printer->post_compaction(hr, G1HRPrinter::Old);
1220 } else {
1221 ShouldNotReachHere();
1222 }
1223 return false;
1224 }
1225
1226 PostCompactionPrinterClosure(G1HRPrinter* hr_printer)
1227 : _hr_printer(hr_printer) { }
1228 };
1229
1230 void G1CollectedHeap::print_hrm_post_compaction() {
1231 PostCompactionPrinterClosure cl(hr_printer());
1232 heap_region_iterate(&cl);
1233 }
2205 (ParallelGCThreads > 1),
2206 // mt discovery
2207 ParallelGCThreads,
2208 // degree of mt discovery
2209 true,
2210 // Reference discovery is atomic
2211 &_is_alive_closure_stw);
2212 // is alive closure
2213 // (for efficiency/performance)
2214 }
2215
2216 CollectorPolicy* G1CollectedHeap::collector_policy() const {
2217 return g1_policy();
2218 }
2219
2220 size_t G1CollectedHeap::capacity() const {
2221 return _hrm.length() * HeapRegion::GrainBytes;
2222 }
2223
2224 void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) {
2225 assert(!hr->is_continues_humongous(), "pre-condition");
2226 hr->reset_gc_time_stamp();
2227 if (hr->is_starts_humongous()) {
2228 uint first_index = hr->hrm_index() + 1;
2229 uint last_index = hr->last_hc_index();
2230 for (uint i = first_index; i < last_index; i += 1) {
2231 HeapRegion* chr = region_at(i);
2232 assert(chr->is_continues_humongous(), "sanity");
2233 chr->reset_gc_time_stamp();
2234 }
2235 }
2236 }
2237
2238 #ifndef PRODUCT
2239
2240 class CheckGCTimeStampsHRClosure : public HeapRegionClosure {
2241 private:
2242 unsigned _gc_time_stamp;
2243 bool _failures;
2244
2245 public:
2246 CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) :
2247 _gc_time_stamp(gc_time_stamp), _failures(false) { }
2248
2249 virtual bool doHeapRegion(HeapRegion* hr) {
2250 unsigned region_gc_time_stamp = hr->get_gc_time_stamp();
2251 if (_gc_time_stamp != region_gc_time_stamp) {
2252 gclog_or_tty->print_cr("Region " HR_FORMAT " has GC time stamp = %d, "
2253 "expected %d", HR_FORMAT_PARAMS(hr),
2254 region_gc_time_stamp, _gc_time_stamp);
2255 _failures = true;
2283 }
2284
2285 // Computes the sum of the storage used by the various regions.
2286 size_t G1CollectedHeap::used() const {
2287 size_t result = _summary_bytes_used + _allocator->used_in_alloc_regions();
2288 if (_archive_allocator != NULL) {
2289 result += _archive_allocator->used();
2290 }
2291 return result;
2292 }
2293
2294 size_t G1CollectedHeap::used_unlocked() const {
2295 return _summary_bytes_used;
2296 }
2297
2298 class SumUsedClosure: public HeapRegionClosure {
2299 size_t _used;
2300 public:
2301 SumUsedClosure() : _used(0) {}
2302 bool doHeapRegion(HeapRegion* r) {
2303 if (!r->is_continues_humongous()) {
2304 _used += r->used();
2305 }
2306 return false;
2307 }
2308 size_t result() { return _used; }
2309 };
2310
2311 size_t G1CollectedHeap::recalculate_used() const {
2312 double recalculate_used_start = os::elapsedTime();
2313
2314 SumUsedClosure blk;
2315 heap_region_iterate(&blk);
2316
2317 g1_policy()->phase_times()->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0);
2318 return blk.result();
2319 }
2320
2321 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {
2322 switch (cause) {
2323 case GCCause::_gc_locker: return GCLockerInvokesConcurrent;
2324 case GCCause::_java_lang_system_gc: return ExplicitGCInvokesConcurrent;
2325 case GCCause::_dcmd_gc_run: return ExplicitGCInvokesConcurrent;
2506 // Schedule a standard evacuation pause. We're setting word_size
2507 // to 0 which means that we are not requesting a post-GC allocation.
2508 VM_G1IncCollectionPause op(gc_count_before,
2509 0, /* word_size */
2510 false, /* should_initiate_conc_mark */
2511 g1_policy()->max_pause_time_ms(),
2512 cause);
2513 VMThread::execute(&op);
2514 } else {
2515 // Schedule a Full GC.
2516 VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);
2517 VMThread::execute(&op);
2518 }
2519 }
2520 } while (retry_gc);
2521 }
2522
2523 bool G1CollectedHeap::is_in(const void* p) const {
2524 if (_hrm.reserved().contains(p)) {
2525 // Given that we know that p is in the reserved space,
2526 // heap_region_containing_raw() should successfully
2527 // return the containing region.
2528 HeapRegion* hr = heap_region_containing_raw(p);
2529 return hr->is_in(p);
2530 } else {
2531 return false;
2532 }
2533 }
2534
2535 #ifdef ASSERT
2536 bool G1CollectedHeap::is_in_exact(const void* p) const {
2537 bool contains = reserved_region().contains(p);
2538 bool available = _hrm.is_available(addr_to_region((HeapWord*)p));
2539 if (contains && available) {
2540 return true;
2541 } else {
2542 return false;
2543 }
2544 }
2545 #endif
2546
2547 bool G1CollectedHeap::obj_in_cs(oop obj) {
2548 HeapRegion* r = _hrm.addr_to_region((HeapWord*) obj);
3045 _vo(vo),
3046 _failures(false) {}
3047
3048 bool failures() {
3049 return _failures;
3050 }
3051
3052 bool doHeapRegion(HeapRegion* r) {
3053 // For archive regions, verify there are no heap pointers to
3054 // non-pinned regions. For all others, verify liveness info.
3055 if (r->is_archive()) {
3056 VerifyArchiveRegionClosure verify_oop_pointers(r);
3057 r->object_iterate(&verify_oop_pointers);
3058 return true;
3059 }
3060 if (!r->is_continues_humongous()) {
3061 bool failures = false;
3062 r->verify(_vo, &failures);
3063 if (failures) {
3064 _failures = true;
3065 } else {
3066 VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo);
3067 r->object_iterate(¬_dead_yet_cl);
3068 if (_vo != VerifyOption_G1UseNextMarking) {
3069 if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
3070 gclog_or_tty->print_cr("[" PTR_FORMAT "," PTR_FORMAT "] "
3071 "max_live_bytes " SIZE_FORMAT " "
3072 "< calculated " SIZE_FORMAT,
3073 p2i(r->bottom()), p2i(r->end()),
3074 r->max_live_bytes(),
3075 not_dead_yet_cl.live_bytes());
3076 _failures = true;
3077 }
3078 } else {
3079 // When vo == UseNextMarking we cannot currently do a sanity
3080 // check on the live bytes as the calculation has not been
3081 // finalized yet.
3082 }
3083 }
3084 }
3085 return false; // stop the region iteration if we hit a failure
5300 assert(free_list != NULL, "pre-condition");
5301
5302 if (G1VerifyBitmaps) {
5303 MemRegion mr(hr->bottom(), hr->end());
5304 concurrent_mark()->clearRangePrevBitmap(mr);
5305 }
5306
5307 // Clear the card counts for this region.
5308 // Note: we only need to do this if the region is not young
5309 // (since we don't refine cards in young regions).
5310 if (!hr->is_young()) {
5311 _cg1r->hot_card_cache()->reset_card_counts(hr);
5312 }
5313 hr->hr_clear(par, true /* clear_space */, locked /* locked */);
5314 free_list->add_ordered(hr);
5315 }
5316
5317 void G1CollectedHeap::free_humongous_region(HeapRegion* hr,
5318 FreeRegionList* free_list,
5319 bool par) {
5320 assert(hr->is_starts_humongous(), "this is only for starts humongous regions");
5321 assert(free_list != NULL, "pre-condition");
5322
5323 size_t hr_capacity = hr->capacity();
5324 // We need to read this before we make the region non-humongous,
5325 // otherwise the information will be gone.
5326 uint last_index = hr->last_hc_index();
5327 hr->clear_humongous();
5328 free_region(hr, free_list, par);
5329
5330 uint i = hr->hrm_index() + 1;
5331 while (i < last_index) {
5332 HeapRegion* curr_hr = region_at(i);
5333 assert(curr_hr->is_continues_humongous(), "invariant");
5334 curr_hr->clear_humongous();
5335 free_region(curr_hr, free_list, par);
5336 i += 1;
5337 }
5338 }
5339
5340 void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed,
5341 const HeapRegionSetCount& humongous_regions_removed) {
5342 if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) {
5343 MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag);
5344 _old_set.bulk_remove(old_regions_removed);
5345 _humongous_set.bulk_remove(humongous_regions_removed);
5346 }
5347
5348 }
5349
5350 void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) {
5351 assert(list != NULL, "list can't be null");
5352 if (!list->is_empty()) {
5353 MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag);
5354 _hrm.insert_list_into_free_list(list);
5355 }
5356 }
5357
5481
5482 void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) {
5483 if (!G1VerifyBitmaps) return;
5484
5485 guarantee(verify_bitmaps(caller, hr), "bitmap verification");
5486 }
5487
5488 class G1VerifyBitmapClosure : public HeapRegionClosure {
5489 private:
5490 const char* _caller;
5491 G1CollectedHeap* _g1h;
5492 bool _failures;
5493
5494 public:
5495 G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) :
5496 _caller(caller), _g1h(g1h), _failures(false) { }
5497
5498 bool failures() { return _failures; }
5499
5500 virtual bool doHeapRegion(HeapRegion* hr) {
5501 if (hr->is_continues_humongous()) return false;
5502
5503 bool result = _g1h->verify_bitmaps(_caller, hr);
5504 if (!result) {
5505 _failures = true;
5506 }
5507 return false;
5508 }
5509 };
5510
5511 void G1CollectedHeap::check_bitmaps(const char* caller) {
5512 if (!G1VerifyBitmaps) return;
5513
5514 G1VerifyBitmapClosure cl(caller, this);
5515 heap_region_iterate(&cl);
5516 guarantee(!cl.failures(), "bitmap verification");
5517 }
5518
5519 class G1CheckCSetFastTableClosure : public HeapRegionClosure {
5520 private:
5521 bool _failures;
5522 public:
5756 //
5757 // It is not required to check whether the object has been found dead by marking
5758 // or not, in fact it would prevent reclamation within a concurrent cycle, as
5759 // all objects allocated during that time are considered live.
5760 // SATB marking is even more conservative than the remembered set.
5761 // So if at this point in the collection there is no remembered set entry,
5762 // nobody has a reference to it.
5763 // At the start of collection we flush all refinement logs, and remembered sets
5764 // are completely up-to-date wrt to references to the humongous object.
5765 //
5766 // Other implementation considerations:
5767 // - never consider object arrays at this time because they would pose
5768 // considerable effort for cleaning up the the remembered sets. This is
5769 // required because stale remembered sets might reference locations that
5770 // are currently allocated into.
5771 uint region_idx = r->hrm_index();
5772 if (!g1h->is_humongous_reclaim_candidate(region_idx) ||
5773 !r->rem_set()->is_empty()) {
5774
5775 if (G1TraceEagerReclaimHumongousObjects) {
5776 gclog_or_tty->print_cr("Live humongous region %u size " SIZE_FORMAT " start " PTR_FORMAT " length %u with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5777 region_idx,
5778 (size_t)obj->size() * HeapWordSize,
5779 p2i(r->bottom()),
5780 r->region_num(),
5781 r->rem_set()->occupied(),
5782 r->rem_set()->strong_code_roots_list_length(),
5783 next_bitmap->isMarked(r->bottom()),
5784 g1h->is_humongous_reclaim_candidate(region_idx),
5785 obj->is_typeArray()
5786 );
5787 }
5788
5789 return false;
5790 }
5791
5792 guarantee(obj->is_typeArray(),
5793 "Only eagerly reclaiming type arrays is supported, but the object "
5794 PTR_FORMAT " is not.", p2i(r->bottom()));
5795
5796 if (G1TraceEagerReclaimHumongousObjects) {
5797 gclog_or_tty->print_cr("Dead humongous region %u size " SIZE_FORMAT " start " PTR_FORMAT " length %u with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5798 region_idx,
5799 (size_t)obj->size() * HeapWordSize,
5800 p2i(r->bottom()),
5801 r->region_num(),
5802 r->rem_set()->occupied(),
5803 r->rem_set()->strong_code_roots_list_length(),
5804 next_bitmap->isMarked(r->bottom()),
5805 g1h->is_humongous_reclaim_candidate(region_idx),
5806 obj->is_typeArray()
5807 );
5808 }
5809 // Need to clear mark bit of the humongous object if already set.
5810 if (next_bitmap->isMarked(r->bottom())) {
5811 next_bitmap->clear(r->bottom());
5812 }
5813 _freed_bytes += r->used();
5814 r->set_containing_set(NULL);
5815 _humongous_regions_removed.increment(1u, r->capacity());
5816 g1h->free_humongous_region(r, _free_region_list, false);
5817
5818 return false;
5819 }
5820
5821 HeapRegionSetCount& humongous_free_count() {
5822 return _humongous_regions_removed;
5823 }
5824
5825 size_t bytes_freed() const {
5826 return _freed_bytes;
5827 }
5828
5829 size_t humongous_reclaimed() const {
5830 return _humongous_regions_removed.length();
5831 }
5832 };
5833
5834 void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
5835 assert_at_safepoint(true);
5836
6031
6032 class RebuildRegionSetsClosure : public HeapRegionClosure {
6033 private:
6034 bool _free_list_only;
6035 HeapRegionSet* _old_set;
6036 HeapRegionManager* _hrm;
6037 size_t _total_used;
6038
6039 public:
6040 RebuildRegionSetsClosure(bool free_list_only,
6041 HeapRegionSet* old_set, HeapRegionManager* hrm) :
6042 _free_list_only(free_list_only),
6043 _old_set(old_set), _hrm(hrm), _total_used(0) {
6044 assert(_hrm->num_free_regions() == 0, "pre-condition");
6045 if (!free_list_only) {
6046 assert(_old_set->is_empty(), "pre-condition");
6047 }
6048 }
6049
6050 bool doHeapRegion(HeapRegion* r) {
6051 if (r->is_continues_humongous()) {
6052 return false;
6053 }
6054
6055 if (r->is_empty()) {
6056 // Add free regions to the free list
6057 r->set_free();
6058 r->set_allocation_context(AllocationContext::system());
6059 _hrm->insert_into_free_list(r);
6060 } else if (!_free_list_only) {
6061 assert(!r->is_young(), "we should not come across young regions");
6062
6063 if (r->is_humongous()) {
6064 // We ignore humongous regions. We left the humongous set unchanged.
6065 } else {
6066 // Objects that were compacted would have ended up on regions
6067 // that were previously old or free. Archive regions (which are
6068 // old) will not have been touched.
6069 assert(r->is_free() || r->is_old(), "invariant");
6070 // We now consider them old, so register as such. Leave
6071 // archive regions set that way, however, while still adding
6072 // them to the old set.
6073 if (!r->is_archive()) {
6074 r->set_old();
6222 // Heap region set verification
6223
6224 class VerifyRegionListsClosure : public HeapRegionClosure {
6225 private:
6226 HeapRegionSet* _old_set;
6227 HeapRegionSet* _humongous_set;
6228 HeapRegionManager* _hrm;
6229
6230 public:
6231 HeapRegionSetCount _old_count;
6232 HeapRegionSetCount _humongous_count;
6233 HeapRegionSetCount _free_count;
6234
6235 VerifyRegionListsClosure(HeapRegionSet* old_set,
6236 HeapRegionSet* humongous_set,
6237 HeapRegionManager* hrm) :
6238 _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm),
6239 _old_count(), _humongous_count(), _free_count(){ }
6240
6241 bool doHeapRegion(HeapRegion* hr) {
6242 if (hr->is_continues_humongous()) {
6243 return false;
6244 }
6245
6246 if (hr->is_young()) {
6247 // TODO
6248 } else if (hr->is_starts_humongous()) {
6249 assert(hr->containing_set() == _humongous_set, "Heap region %u is starts humongous but not in humongous set.", hr->hrm_index());
6250 _humongous_count.increment(1u, hr->capacity());
6251 } else if (hr->is_empty()) {
6252 assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index());
6253 _free_count.increment(1u, hr->capacity());
6254 } else if (hr->is_old()) {
6255 assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index());
6256 _old_count.increment(1u, hr->capacity());
6257 } else {
6258 // There are no other valid region types. Check for one invalid
6259 // one we can identify: pinned without old or humongous set.
6260 assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index());
6261 ShouldNotReachHere();
6262 }
6263 return false;
6264 }
6265
6266 void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) {
6267 guarantee(old_set->length() == _old_count.length(), "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length());
6268 guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), "Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6269 old_set->total_capacity_bytes(), _old_count.capacity());
|
303 // Index of last region in the series + 1.
304 uint last = first + num_regions;
305
306 // We need to initialize the region(s) we just discovered. This is
307 // a bit tricky given that it can happen concurrently with
308 // refinement threads refining cards on these regions and
309 // potentially wanting to refine the BOT as they are scanning
310 // those cards (this can happen shortly after a cleanup; see CR
311 // 6991377). So we have to set up the region(s) carefully and in
312 // a specific order.
313
314 // The word size sum of all the regions we will allocate.
315 size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
316 assert(word_size <= word_size_sum, "sanity");
317
318 // This will be the "starts humongous" region.
319 HeapRegion* first_hr = region_at(first);
320 // The header of the new object will be placed at the bottom of
321 // the first region.
322 HeapWord* new_obj = first_hr->bottom();
323 // This will be the new top of the new object.
324 HeapWord* obj_top = new_obj + word_size;
325
326 // First, we need to zero the header of the space that we will be
327 // allocating. When we update top further down, some refinement
328 // threads might try to scan the region. By zeroing the header we
329 // ensure that any thread that will try to scan the region will
330 // come across the zero klass word and bail out.
331 //
332 // NOTE: It would not have been correct to have used
333 // CollectedHeap::fill_with_object() and make the space look like
334 // an int array. The thread that is doing the allocation will
335 // later update the object header to a potentially different array
336 // type and, for a very short period of time, the klass and length
337 // fields will be inconsistent. This could cause a refinement
338 // thread to calculate the object size incorrectly.
339 Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
340
341 // We will set up the first region as "starts humongous". This
342 // will also update the BOT covering all the regions to reflect
343 // that there is a single object that starts at the bottom of the
344 // first region.
345 first_hr->set_starts_humongous(obj_top);
346 first_hr->set_allocation_context(context);
347 // Then, if there are any, we will set up the "continues
348 // humongous" regions.
349 HeapRegion* hr = NULL;
350 for (uint i = first + 1; i < last; ++i) {
351 hr = region_at(i);
352 hr->set_continues_humongous(first_hr);
353 hr->set_allocation_context(context);
354 }
355
356 // Up to this point no concurrent thread would have been able to
357 // do any scanning on any region in this series. All the top
358 // fields still point to bottom, so the intersection between
359 // [bottom,top] and [card_start,card_end] will be empty. Before we
360 // update the top fields, we'll do a storestore to make sure that
361 // no thread sees the update to top before the zeroing of the
362 // object header and the BOT initialization.
363 OrderAccess::storestore();
364
365 // Now that the BOT and the object header have been initialized,
366 // we can update top of the "starts humongous" region.
367 first_hr->set_top(MIN2(first_hr->end(), obj_top));
368 if (_hr_printer.is_active()) {
369 if ((first + 1) == last) {
370 // the series has a single humongous region
371 _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, obj_top);
372 } else {
373 // the series has more than one humongous regions
374 _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, first_hr->end());
375 }
376 }
377
378 // Now, we will update the top fields of the "continues humongous"
379 // regions. The reason we need to do this is that, otherwise,
380 // these regions would look empty and this will confuse parts of
381 // G1. For example, the code that looks for a consecutive number
382 // of empty regions will consider them empty and try to
383 // re-allocate them. We can extend is_empty() to also include
384 // !is_continues_humongous(), but it is easier to just update the top
385 // fields here. The way we set top for all regions (i.e., top ==
386 // end for all regions but the last one, top == new_top for the
387 // last one) is actually used when we will free up the humongous
388 // region in free_humongous_region().
389 hr = NULL;
390 for (uint i = first + 1; i < last; ++i) {
391 hr = region_at(i);
392 if ((i + 1) == last) {
393 // last continues humongous region
394 assert(hr->bottom() < obj_top && obj_top <= hr->end(),
395 "new_top should fall on this region");
396 hr->set_top(obj_top);
397 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, obj_top);
398 } else {
399 // not last one
400 assert(obj_top > hr->end(), "obj_top should be above this region");
401 hr->set_top(hr->end());
402 _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end());
403 }
404 }
405 // If we have continues humongous regions (hr != NULL), its top should
406 // match obj_top.
407 assert(hr == NULL || (hr->top() == obj_top), "sanity");
408 check_bitmaps("Humongous Region Allocation", first_hr);
409
410 increase_used(word_size * HeapWordSize);
411
412 for (uint i = first; i < last; ++i) {
413 _humongous_set.add(region_at(i));
414 }
415
416 return new_obj;
417 }
418
419 // If could fit into free regions w/o expansion, try.
420 // Otherwise, if can expand, do so.
421 // Otherwise, if using ex regions might help, try with ex given back.
422 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) {
423 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
424
425 verify_region_sets_optional();
426
427 uint first = G1_NO_HRM_INDEX;
428 uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords);
429
430 if (obj_regions == 1) {
431 // Only one region to allocate, try to use a fast path by directly allocating
432 // from the free lists. Do not try to expand here, we will potentially do that
433 // later.
434 HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */);
1111 HeapWord* result = humongous_obj_allocate(word_size, context);
1112 if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) {
1113 collector_state()->set_initiate_conc_mark_if_possible(true);
1114 }
1115 return result;
1116 }
1117
1118 ShouldNotReachHere();
1119 }
1120
1121 class PostMCRemSetClearClosure: public HeapRegionClosure {
1122 G1CollectedHeap* _g1h;
1123 ModRefBarrierSet* _mr_bs;
1124 public:
1125 PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) :
1126 _g1h(g1h), _mr_bs(mr_bs) {}
1127
1128 bool doHeapRegion(HeapRegion* r) {
1129 HeapRegionRemSet* hrrs = r->rem_set();
1130
1131 _g1h->reset_gc_time_stamps(r);
1132 hrrs->clear();
1133 // You might think here that we could clear just the cards
1134 // corresponding to the used region. But no: if we leave a dirty card
1135 // in a region we might allocate into, then it would prevent that card
1136 // from being enqueued, and cause it to be missed.
1137 // Re: the performance cost: we shouldn't be doing full GC anyway!
1138 _mr_bs->clear(MemRegion(r->bottom(), r->end()));
1139
1140 return false;
1141 }
1142 };
1143
1144 void G1CollectedHeap::clear_rsets_post_compaction() {
1145 PostMCRemSetClearClosure rs_clear(this, g1_barrier_set());
1146 heap_region_iterate(&rs_clear);
1147 }
1148
1149 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
1150 G1CollectedHeap* _g1h;
1170
1171 public:
1172 ParRebuildRSTask(G1CollectedHeap* g1) :
1173 AbstractGangTask("ParRebuildRSTask"), _g1(g1), _hrclaimer(g1->workers()->active_workers()) {}
1174
1175 void work(uint worker_id) {
1176 RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id);
1177 _g1->heap_region_par_iterate(&rebuild_rs, worker_id, &_hrclaimer);
1178 }
1179 };
1180
1181 class PostCompactionPrinterClosure: public HeapRegionClosure {
1182 private:
1183 G1HRPrinter* _hr_printer;
1184 public:
1185 bool doHeapRegion(HeapRegion* hr) {
1186 assert(!hr->is_young(), "not expecting to find young regions");
1187 if (hr->is_free()) {
1188 // We only generate output for non-empty regions.
1189 } else if (hr->is_starts_humongous()) {
1190 _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous);
1191 } else if (hr->is_continues_humongous()) {
1192 _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous);
1193 } else if (hr->is_archive()) {
1194 _hr_printer->post_compaction(hr, G1HRPrinter::Archive);
1195 } else if (hr->is_old()) {
1196 _hr_printer->post_compaction(hr, G1HRPrinter::Old);
1197 } else {
1198 ShouldNotReachHere();
1199 }
1200 return false;
1201 }
1202
1203 PostCompactionPrinterClosure(G1HRPrinter* hr_printer)
1204 : _hr_printer(hr_printer) { }
1205 };
1206
1207 void G1CollectedHeap::print_hrm_post_compaction() {
1208 PostCompactionPrinterClosure cl(hr_printer());
1209 heap_region_iterate(&cl);
1210 }
2182 (ParallelGCThreads > 1),
2183 // mt discovery
2184 ParallelGCThreads,
2185 // degree of mt discovery
2186 true,
2187 // Reference discovery is atomic
2188 &_is_alive_closure_stw);
2189 // is alive closure
2190 // (for efficiency/performance)
2191 }
2192
2193 CollectorPolicy* G1CollectedHeap::collector_policy() const {
2194 return g1_policy();
2195 }
2196
2197 size_t G1CollectedHeap::capacity() const {
2198 return _hrm.length() * HeapRegion::GrainBytes;
2199 }
2200
2201 void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) {
2202 hr->reset_gc_time_stamp();
2203 }
2204
2205 #ifndef PRODUCT
2206
2207 class CheckGCTimeStampsHRClosure : public HeapRegionClosure {
2208 private:
2209 unsigned _gc_time_stamp;
2210 bool _failures;
2211
2212 public:
2213 CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) :
2214 _gc_time_stamp(gc_time_stamp), _failures(false) { }
2215
2216 virtual bool doHeapRegion(HeapRegion* hr) {
2217 unsigned region_gc_time_stamp = hr->get_gc_time_stamp();
2218 if (_gc_time_stamp != region_gc_time_stamp) {
2219 gclog_or_tty->print_cr("Region " HR_FORMAT " has GC time stamp = %d, "
2220 "expected %d", HR_FORMAT_PARAMS(hr),
2221 region_gc_time_stamp, _gc_time_stamp);
2222 _failures = true;
2250 }
2251
2252 // Computes the sum of the storage used by the various regions.
2253 size_t G1CollectedHeap::used() const {
2254 size_t result = _summary_bytes_used + _allocator->used_in_alloc_regions();
2255 if (_archive_allocator != NULL) {
2256 result += _archive_allocator->used();
2257 }
2258 return result;
2259 }
2260
2261 size_t G1CollectedHeap::used_unlocked() const {
2262 return _summary_bytes_used;
2263 }
2264
2265 class SumUsedClosure: public HeapRegionClosure {
2266 size_t _used;
2267 public:
2268 SumUsedClosure() : _used(0) {}
2269 bool doHeapRegion(HeapRegion* r) {
2270 _used += r->used();
2271 return false;
2272 }
2273 size_t result() { return _used; }
2274 };
2275
2276 size_t G1CollectedHeap::recalculate_used() const {
2277 double recalculate_used_start = os::elapsedTime();
2278
2279 SumUsedClosure blk;
2280 heap_region_iterate(&blk);
2281
2282 g1_policy()->phase_times()->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0);
2283 return blk.result();
2284 }
2285
2286 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {
2287 switch (cause) {
2288 case GCCause::_gc_locker: return GCLockerInvokesConcurrent;
2289 case GCCause::_java_lang_system_gc: return ExplicitGCInvokesConcurrent;
2290 case GCCause::_dcmd_gc_run: return ExplicitGCInvokesConcurrent;
2471 // Schedule a standard evacuation pause. We're setting word_size
2472 // to 0 which means that we are not requesting a post-GC allocation.
2473 VM_G1IncCollectionPause op(gc_count_before,
2474 0, /* word_size */
2475 false, /* should_initiate_conc_mark */
2476 g1_policy()->max_pause_time_ms(),
2477 cause);
2478 VMThread::execute(&op);
2479 } else {
2480 // Schedule a Full GC.
2481 VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);
2482 VMThread::execute(&op);
2483 }
2484 }
2485 } while (retry_gc);
2486 }
2487
2488 bool G1CollectedHeap::is_in(const void* p) const {
2489 if (_hrm.reserved().contains(p)) {
2490 // Given that we know that p is in the reserved space,
2491 // heap_region_containing() should successfully
2492 // return the containing region.
2493 HeapRegion* hr = heap_region_containing(p);
2494 return hr->is_in(p);
2495 } else {
2496 return false;
2497 }
2498 }
2499
2500 #ifdef ASSERT
2501 bool G1CollectedHeap::is_in_exact(const void* p) const {
2502 bool contains = reserved_region().contains(p);
2503 bool available = _hrm.is_available(addr_to_region((HeapWord*)p));
2504 if (contains && available) {
2505 return true;
2506 } else {
2507 return false;
2508 }
2509 }
2510 #endif
2511
2512 bool G1CollectedHeap::obj_in_cs(oop obj) {
2513 HeapRegion* r = _hrm.addr_to_region((HeapWord*) obj);
3010 _vo(vo),
3011 _failures(false) {}
3012
3013 bool failures() {
3014 return _failures;
3015 }
3016
3017 bool doHeapRegion(HeapRegion* r) {
3018 // For archive regions, verify there are no heap pointers to
3019 // non-pinned regions. For all others, verify liveness info.
3020 if (r->is_archive()) {
3021 VerifyArchiveRegionClosure verify_oop_pointers(r);
3022 r->object_iterate(&verify_oop_pointers);
3023 return true;
3024 }
3025 if (!r->is_continues_humongous()) {
3026 bool failures = false;
3027 r->verify(_vo, &failures);
3028 if (failures) {
3029 _failures = true;
3030 } else if (!r->is_starts_humongous()) {
3031 VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo);
3032 r->object_iterate(¬_dead_yet_cl);
3033 if (_vo != VerifyOption_G1UseNextMarking) {
3034 if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
3035 gclog_or_tty->print_cr("[" PTR_FORMAT "," PTR_FORMAT "] "
3036 "max_live_bytes " SIZE_FORMAT " "
3037 "< calculated " SIZE_FORMAT,
3038 p2i(r->bottom()), p2i(r->end()),
3039 r->max_live_bytes(),
3040 not_dead_yet_cl.live_bytes());
3041 _failures = true;
3042 }
3043 } else {
3044 // When vo == UseNextMarking we cannot currently do a sanity
3045 // check on the live bytes as the calculation has not been
3046 // finalized yet.
3047 }
3048 }
3049 }
3050 return false; // stop the region iteration if we hit a failure
5265 assert(free_list != NULL, "pre-condition");
5266
5267 if (G1VerifyBitmaps) {
5268 MemRegion mr(hr->bottom(), hr->end());
5269 concurrent_mark()->clearRangePrevBitmap(mr);
5270 }
5271
5272 // Clear the card counts for this region.
5273 // Note: we only need to do this if the region is not young
5274 // (since we don't refine cards in young regions).
5275 if (!hr->is_young()) {
5276 _cg1r->hot_card_cache()->reset_card_counts(hr);
5277 }
5278 hr->hr_clear(par, true /* clear_space */, locked /* locked */);
5279 free_list->add_ordered(hr);
5280 }
5281
5282 void G1CollectedHeap::free_humongous_region(HeapRegion* hr,
5283 FreeRegionList* free_list,
5284 bool par) {
5285 assert(free_list != NULL, "pre-condition");
5286 hr->clear_humongous();
5287 free_region(hr, free_list, par);
5288 }
5289
5290 void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed,
5291 const HeapRegionSetCount& humongous_regions_removed) {
5292 if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) {
5293 MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag);
5294 _old_set.bulk_remove(old_regions_removed);
5295 _humongous_set.bulk_remove(humongous_regions_removed);
5296 }
5297
5298 }
5299
5300 void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) {
5301 assert(list != NULL, "list can't be null");
5302 if (!list->is_empty()) {
5303 MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag);
5304 _hrm.insert_list_into_free_list(list);
5305 }
5306 }
5307
5431
5432 void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) {
5433 if (!G1VerifyBitmaps) return;
5434
5435 guarantee(verify_bitmaps(caller, hr), "bitmap verification");
5436 }
5437
5438 class G1VerifyBitmapClosure : public HeapRegionClosure {
5439 private:
5440 const char* _caller;
5441 G1CollectedHeap* _g1h;
5442 bool _failures;
5443
5444 public:
5445 G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) :
5446 _caller(caller), _g1h(g1h), _failures(false) { }
5447
5448 bool failures() { return _failures; }
5449
5450 virtual bool doHeapRegion(HeapRegion* hr) {
5451 bool result = _g1h->verify_bitmaps(_caller, hr);
5452 if (!result) {
5453 _failures = true;
5454 }
5455 return false;
5456 }
5457 };
5458
5459 void G1CollectedHeap::check_bitmaps(const char* caller) {
5460 if (!G1VerifyBitmaps) return;
5461
5462 G1VerifyBitmapClosure cl(caller, this);
5463 heap_region_iterate(&cl);
5464 guarantee(!cl.failures(), "bitmap verification");
5465 }
5466
5467 class G1CheckCSetFastTableClosure : public HeapRegionClosure {
5468 private:
5469 bool _failures;
5470 public:
5704 //
5705 // It is not required to check whether the object has been found dead by marking
5706 // or not, in fact it would prevent reclamation within a concurrent cycle, as
5707 // all objects allocated during that time are considered live.
5708 // SATB marking is even more conservative than the remembered set.
5709 // So if at this point in the collection there is no remembered set entry,
5710 // nobody has a reference to it.
5711 // At the start of collection we flush all refinement logs, and remembered sets
5712 // are completely up-to-date wrt to references to the humongous object.
5713 //
5714 // Other implementation considerations:
5715 // - never consider object arrays at this time because they would pose
5716 // considerable effort for cleaning up the the remembered sets. This is
5717 // required because stale remembered sets might reference locations that
5718 // are currently allocated into.
5719 uint region_idx = r->hrm_index();
5720 if (!g1h->is_humongous_reclaim_candidate(region_idx) ||
5721 !r->rem_set()->is_empty()) {
5722
5723 if (G1TraceEagerReclaimHumongousObjects) {
5724 gclog_or_tty->print_cr("Live humongous region %u objectsize " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5725 region_idx,
5726 (size_t)obj->size() * HeapWordSize,
5727 p2i(r->bottom()),
5728 r->rem_set()->occupied(),
5729 r->rem_set()->strong_code_roots_list_length(),
5730 next_bitmap->isMarked(r->bottom()),
5731 g1h->is_humongous_reclaim_candidate(region_idx),
5732 obj->is_typeArray()
5733 );
5734 }
5735
5736 return false;
5737 }
5738
5739 guarantee(obj->is_typeArray(),
5740 "Only eagerly reclaiming type arrays is supported, but the object "
5741 PTR_FORMAT " is not.", p2i(r->bottom()));
5742
5743 if (G1TraceEagerReclaimHumongousObjects) {
5744 gclog_or_tty->print_cr("Dead humongous region %u objectsize " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5745 region_idx,
5746 (size_t)obj->size() * HeapWordSize,
5747 p2i(r->bottom()),
5748 r->rem_set()->occupied(),
5749 r->rem_set()->strong_code_roots_list_length(),
5750 next_bitmap->isMarked(r->bottom()),
5751 g1h->is_humongous_reclaim_candidate(region_idx),
5752 obj->is_typeArray()
5753 );
5754 }
5755 // Need to clear mark bit of the humongous object if already set.
5756 if (next_bitmap->isMarked(r->bottom())) {
5757 next_bitmap->clear(r->bottom());
5758 }
5759 do {
5760 _freed_bytes += r->used();
5761 r->set_containing_set(NULL);
5762 _humongous_regions_removed.increment(1u, r->capacity());
5763 g1h->free_humongous_region(r, _free_region_list, false);
5764 if (++region_idx >= g1h->num_regions()) {
5765 break;
5766 }
5767 r = g1h->region_at(region_idx);
5768 } while (r->is_continues_humongous());
5769
5770 return false;
5771 }
5772
5773 HeapRegionSetCount& humongous_free_count() {
5774 return _humongous_regions_removed;
5775 }
5776
5777 size_t bytes_freed() const {
5778 return _freed_bytes;
5779 }
5780
5781 size_t humongous_reclaimed() const {
5782 return _humongous_regions_removed.length();
5783 }
5784 };
5785
5786 void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
5787 assert_at_safepoint(true);
5788
5983
5984 class RebuildRegionSetsClosure : public HeapRegionClosure {
5985 private:
5986 bool _free_list_only;
5987 HeapRegionSet* _old_set;
5988 HeapRegionManager* _hrm;
5989 size_t _total_used;
5990
5991 public:
5992 RebuildRegionSetsClosure(bool free_list_only,
5993 HeapRegionSet* old_set, HeapRegionManager* hrm) :
5994 _free_list_only(free_list_only),
5995 _old_set(old_set), _hrm(hrm), _total_used(0) {
5996 assert(_hrm->num_free_regions() == 0, "pre-condition");
5997 if (!free_list_only) {
5998 assert(_old_set->is_empty(), "pre-condition");
5999 }
6000 }
6001
6002 bool doHeapRegion(HeapRegion* r) {
6003 if (r->is_empty()) {
6004 // Add free regions to the free list
6005 r->set_free();
6006 r->set_allocation_context(AllocationContext::system());
6007 _hrm->insert_into_free_list(r);
6008 } else if (!_free_list_only) {
6009 assert(!r->is_young(), "we should not come across young regions");
6010
6011 if (r->is_humongous()) {
6012 // We ignore humongous regions. We left the humongous set unchanged.
6013 } else {
6014 // Objects that were compacted would have ended up on regions
6015 // that were previously old or free. Archive regions (which are
6016 // old) will not have been touched.
6017 assert(r->is_free() || r->is_old(), "invariant");
6018 // We now consider them old, so register as such. Leave
6019 // archive regions set that way, however, while still adding
6020 // them to the old set.
6021 if (!r->is_archive()) {
6022 r->set_old();
6170 // Heap region set verification
6171
6172 class VerifyRegionListsClosure : public HeapRegionClosure {
6173 private:
6174 HeapRegionSet* _old_set;
6175 HeapRegionSet* _humongous_set;
6176 HeapRegionManager* _hrm;
6177
6178 public:
6179 HeapRegionSetCount _old_count;
6180 HeapRegionSetCount _humongous_count;
6181 HeapRegionSetCount _free_count;
6182
6183 VerifyRegionListsClosure(HeapRegionSet* old_set,
6184 HeapRegionSet* humongous_set,
6185 HeapRegionManager* hrm) :
6186 _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm),
6187 _old_count(), _humongous_count(), _free_count(){ }
6188
6189 bool doHeapRegion(HeapRegion* hr) {
6190 if (hr->is_young()) {
6191 // TODO
6192 } else if (hr->is_humongous()) {
6193 assert(hr->containing_set() == _humongous_set, "Heap region %u is humongous but not in humongous set.", hr->hrm_index());
6194 _humongous_count.increment(1u, hr->capacity());
6195 } else if (hr->is_empty()) {
6196 assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index());
6197 _free_count.increment(1u, hr->capacity());
6198 } else if (hr->is_old()) {
6199 assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index());
6200 _old_count.increment(1u, hr->capacity());
6201 } else {
6202 // There are no other valid region types. Check for one invalid
6203 // one we can identify: pinned without old or humongous set.
6204 assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index());
6205 ShouldNotReachHere();
6206 }
6207 return false;
6208 }
6209
6210 void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) {
6211 guarantee(old_set->length() == _old_count.length(), "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length());
6212 guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), "Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6213 old_set->total_capacity_bytes(), _old_count.capacity());
|