--- old/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp 2018-04-03 14:10:14.694212642 +0200 +++ new/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp 2018-04-03 14:10:14.442204965 +0200 @@ -1008,6 +1008,8 @@ G1CollectedHeap* _g1h; G1ConcurrentMark* _cm; + G1PrintRegionLivenessInfoClosure _cl; + uint _num_regions_selected_for_rebuild; // The number of regions actually selected for rebuild. void update_remset_before_rebuild(HeapRegion * hr) { @@ -1021,12 +1023,60 @@ _cm->update_top_at_rebuild_start(hr); } + void distribute_marked_bytes(HeapRegion* hr, size_t marked_words) { + uint const region_idx = hr->hrm_index(); + assert(hr->is_starts_humongous(), + "Should not have marked bytes " SIZE_FORMAT " in non-starts humongous region %u (%s)", + marked_words, region_idx, hr->get_type_str()); + uint num_regions_in_humongous = (uint)G1CollectedHeap::humongous_obj_size_in_regions(marked_words); + + for (uint i = region_idx; i < (region_idx + num_regions_in_humongous); i++) { + HeapRegion* const r = _g1h->region_at(i); + size_t const words_to_add = MIN2(HeapRegion::GrainWords, marked_words); + assert(words_to_add > 0, "Out of space to distribute before end of humongous object in region %u (starts %u)", i, region_idx); + + r->add_to_marked_bytes(words_to_add * HeapWordSize); + marked_words -= words_to_add; + } + assert(marked_words == 0, + SIZE_FORMAT " words left after distributing space across %u regions", + marked_words, num_regions_in_humongous); + } + + void update_marked_bytes(HeapRegion* hr) { + uint const region_idx = hr->hrm_index(); + size_t marked_words = _cm->liveness(region_idx); + // The marking attributes the object's size completely to the humongous starts + // region. We need to distribute this value across the entire set of regions a + // humongous object spans. + if (hr->is_humongous()) { + if (marked_words > 0) { + log_trace(gc, marking)("Adding " SIZE_FORMAT " words to humongous start region %u (%s), word size %d (%f)", + marked_words, region_idx, hr->get_type_str(), + oop(hr->bottom())->size(), (double)oop(hr->bottom())->size() / HeapRegion::GrainWords); + distribute_marked_bytes(hr, marked_words); + } else { + assert(marked_words == 0, + "Asked to add " SIZE_FORMAT " words to add to continues humongous region %u (%s)", + marked_words, region_idx, hr->get_type_str()); + } + } else { + log_trace(gc, marking)("Adding " SIZE_FORMAT " words to region %u (%s)", marked_words, region_idx, hr->get_type_str()); + hr->add_to_marked_bytes(marked_words * HeapWordSize); + } + } + public: G1UpdateRemSetTrackingBeforeRebuild(G1CollectedHeap* g1h, G1ConcurrentMark* cm) : - _g1h(g1h), _cm(cm), _num_regions_selected_for_rebuild(0) { } + _g1h(g1h), _cm(cm), _cl("Post-Marking"), _num_regions_selected_for_rebuild(0) { } virtual bool do_heap_region(HeapRegion* r) { update_remset_before_rebuild(r); + update_marked_bytes(r); + if (log_is_enabled(Trace, gc, liveness)) { + _cl.do_heap_region(r); + } + r->note_end_of_marking(); return false; } @@ -1083,6 +1133,8 @@ flush_all_task_caches(); } + // Install newly created mark bitmap as "prev". + swap_mark_bitmaps(); { GCTraceTime(Debug, gc, phases)("Update Remembered Set Tracking Before Rebuild"); G1UpdateRemSetTrackingBeforeRebuild cl(_g1h, this); @@ -1091,7 +1143,7 @@ _g1h->num_regions(), cl.num_selected_for_rebuild()); } - verify_during_pause(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UseNextMarking, "Remark after"); + verify_during_pause(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "Remark after"); assert(!restart_for_overflow(), "sanity"); // Completely reset the marking state since marking completed @@ -1109,7 +1161,7 @@ { GCTraceTime(Debug, gc, phases)("Report Object Count"); - report_object_count(); + report_object_count(mark_finished); } // Statistics @@ -1147,8 +1199,6 @@ const uint humongous_regions_removed() { return _humongous_regions_removed; } bool do_heap_region(HeapRegion *hr) { - hr->note_end_of_marking(); - if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young() && !hr->is_archive()) { _freed_bytes += hr->used(); hr->set_containing_set(NULL); @@ -1243,7 +1293,7 @@ double start = os::elapsedTime(); - verify_during_pause(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UseNextMarking, "Cleanup before"); + verify_during_pause(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "Cleanup before"); { GCTraceTime(Debug, gc, phases)("Update Remembered Set Tracking After Rebuild"); @@ -1256,8 +1306,6 @@ _g1h->heap_region_iterate(&cl); } - // Install newly created mark bitmap as "prev". - swap_mark_bitmaps(); { GCTraceTime(Debug, gc, phases)("Reclaim Empty Regions"); reclaim_empty_regions(); @@ -1625,11 +1673,33 @@ } } -void G1ConcurrentMark::report_object_count() { - G1CMIsAliveClosure is_alive(_g1h); - _gc_tracer_cm->report_object_count_after_gc(&is_alive); +// When sampling object counts, we already swapped the mark bitmaps, so we need to use +// the prev bitmap determining liveness. +class G1ObjectCountIsAliveClosure: public BoolObjectClosure { + G1CollectedHeap* _g1; + public: + G1ObjectCountIsAliveClosure(G1CollectedHeap* g1) : _g1(g1) { } + + bool do_object_b(oop obj) { + HeapWord* addr = (HeapWord*)obj; + return addr != NULL && + (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_dead(obj)); + } +}; + +void G1ConcurrentMark::report_object_count(bool mark_completed) { + // Depending on the completion of the marking liveness needs to be determined + // using either the next or prev bitmap. + if (mark_completed) { + G1ObjectCountIsAliveClosure is_alive(_g1h); + _gc_tracer_cm->report_object_count_after_gc(&is_alive); + } else { + G1CMIsAliveClosure is_alive(_g1h); + _gc_tracer_cm->report_object_count_after_gc(&is_alive); + } } + void G1ConcurrentMark::swap_mark_bitmaps() { G1CMBitMap* temp = _prev_mark_bitmap; _prev_mark_bitmap = _next_mark_bitmap; --- old/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp 2018-04-03 14:10:15.784245847 +0200 +++ new/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp 2018-04-03 14:10:15.528238048 +0200 @@ -367,7 +367,7 @@ void weak_refs_work_parallel_part(BoolObjectClosure* is_alive, bool purged_classes); void weak_refs_work(bool clear_all_soft_refs); - void report_object_count(); + void report_object_count(bool mark_completed); void swap_mark_bitmaps(); --- old/src/hotspot/share/gc/g1/g1RemSet.cpp 2018-04-03 14:10:16.838277955 +0200 +++ new/src/hotspot/share/gc/g1/g1RemSet.cpp 2018-04-03 14:10:16.588270339 +0200 @@ -843,7 +843,7 @@ assert(hr->top() == top_at_mark_start || hr->top() == top_at_rebuild_start, "More than one object in the humongous region?"); humongous_obj->oop_iterate(&_update_cl, mr); - return top_at_mark_start != hr->bottom() ? mr.byte_size() : 0; + return top_at_mark_start != hr->bottom() ? mr.intersection(MemRegion((HeapWord*)humongous_obj, humongous_obj->size())).byte_size() : 0; } else { return 0; } @@ -882,7 +882,7 @@ size_t total_marked_bytes = 0; size_t const chunk_size_in_words = G1RebuildRemSetChunkSize / HeapWordSize; - HeapWord* const top_at_mark_start = hr->next_top_at_mark_start(); + HeapWord* const top_at_mark_start = hr->prev_top_at_mark_start(); HeapWord* cur = hr->bottom(); while (cur < hr->end()) { @@ -899,7 +899,7 @@ } const Ticks start = Ticks::now(); - size_t marked_bytes = rebuild_rem_set_in_region(_cm->next_mark_bitmap(), + size_t marked_bytes = rebuild_rem_set_in_region(_cm->prev_mark_bitmap(), top_at_mark_start, top_at_rebuild_start, hr, @@ -922,7 +922,6 @@ p2i(top_at_rebuild_start)); if (marked_bytes > 0) { - hr->add_to_marked_bytes(marked_bytes); total_marked_bytes += marked_bytes; } cur += chunk_size_in_words; @@ -936,9 +935,8 @@ // Simply filter out those regions. We can not just use region type because there // might have already been new allocations into these regions. DEBUG_ONLY(HeapWord* const top_at_rebuild_start = _cm->top_at_rebuild_start(region_idx);) - assert(!hr->is_old() || - top_at_rebuild_start == NULL || - total_marked_bytes == _cm->liveness(region_idx) * HeapWordSize, + assert(top_at_rebuild_start == NULL || + total_marked_bytes == hr->marked_bytes(), "Marked bytes " SIZE_FORMAT " for region %u (%s) in [bottom, TAMS) do not match liveness during mark " SIZE_FORMAT " " "(" PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT ")", total_marked_bytes, hr->hrm_index(), hr->get_type_str(), _cm->liveness(region_idx) * HeapWordSize,