--- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2014-07-21 15:24:53.620076489 +0200 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2014-07-21 15:24:53.531073890 +0200 @@ -2181,6 +2181,7 @@ } void G1CollectedHeap::clear_humongous_is_live_table() { + guarantee(G1ReclaimDeadHumongousObjectsAtYoungGC, "Should only be called if true"); _humongous_is_live.clear(); } @@ -3692,10 +3693,6 @@ (total_collections() % G1SummarizeRSetStatsPeriod == 0)) { g1_rem_set()->print_periodic_summary_info("Before GC RS summary"); } - - if (G1ReclaimDeadHumongousObjectsAtYoungGC) { - clear_humongous_is_live_table(); - } } void G1CollectedHeap::gc_epilogue(bool full /* Ignored */) { @@ -3775,7 +3772,8 @@ return g1_rem_set()->cardsScanned(); } -bool G1CollectedHeap::humongous_region_is_always_live(HeapRegion* region) { +bool G1CollectedHeap::humongous_region_is_always_live(uint index) { + HeapRegion* region = region_at(index); assert(region->startsHumongous(), "Must start a humongous object"); return oop(region->bottom())->is_objArray() || !region->rem_set()->is_empty(); } @@ -3794,12 +3792,13 @@ } G1CollectedHeap* g1h = G1CollectedHeap::heap(); - bool is_candidate = !g1h->humongous_region_is_always_live(r); + uint region_idx = r->hrs_index(); + bool is_candidate = !g1h->humongous_region_is_always_live(region_idx); + // Is_candidate already filters out humongous regions with some remembered set. + // This will not lead to humongous object that we mistakenly keep alive because + // during young collection the remembered sets will only be added to. if (is_candidate) { - // Do not even try to reclaim a humongous object that we already know will - // not be treated as live later. A young collection will not decrease the - // amount of remembered set entries for that region. - g1h->register_humongous_region_with_in_cset_fast_test(r->hrs_index()); + g1h->register_humongous_region_with_in_cset_fast_test(region_idx); _candidate_humongous++; } _total_humongous++; @@ -3812,11 +3811,20 @@ }; void G1CollectedHeap::register_humongous_regions_with_in_cset_fast_test() { + if (!G1ReclaimDeadHumongousObjectsAtYoungGC) { + g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0, 0); + return; + } + RegisterHumongousWithInCSetFastTestClosure cl; heap_region_iterate(&cl); g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(cl.total_humongous(), cl.candidate_humongous()); _has_humongous_reclaim_candidates = cl.candidate_humongous() > 0; + + if (_has_humongous_reclaim_candidates) { + clear_humongous_is_live_table(); + } } void @@ -4106,9 +4114,7 @@ g1_policy()->finalize_cset(target_pause_time_ms, evacuation_info); - if (G1ReclaimDeadHumongousObjectsAtYoungGC) { - register_humongous_regions_with_in_cset_fast_test(); - } + register_humongous_regions_with_in_cset_fast_test(); _cm->note_start_of_gc(); // We should not verify the per-thread SATB buffers given that @@ -4160,9 +4166,9 @@ true /* verify_fingers */); free_collection_set(g1_policy()->collection_set(), evacuation_info); - if (G1ReclaimDeadHumongousObjectsAtYoungGC && _has_humongous_reclaim_candidates) { - eagerly_reclaim_humongous_regions(); - } + + eagerly_reclaim_humongous_regions(); + g1_policy()->clear_collection_set(); cleanup_surviving_young_words(); @@ -4662,30 +4668,31 @@ oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); assert(_worker_id == _par_scan_state->queue_num(), "sanity"); - bool needs_marking = true; - if (_g1->is_in_cset_or_humongous(obj)) { + G1FastCSetBiasedMappedArray::in_cset_state_t state = _g1->in_cset_state(obj); + + if (state == G1FastCSetBiasedMappedArray::InCSet) { oop forwardee; if (obj->is_forwarded()) { forwardee = obj->forwardee(); } else { forwardee = _par_scan_state->copy_to_survivor_space(obj); } - if (forwardee != NULL) { - oopDesc::encode_store_heap_oop(p, forwardee); - if (do_mark_object != G1MarkNone && forwardee != obj) { - // If the object is self-forwarded we don't need to explicitly - // mark it, the evacuation failure protocol will do so. - mark_forwarded_object(obj, forwardee); - } + assert(forwardee != NULL, "forwardee should not be NULL"); + oopDesc::encode_store_heap_oop(p, forwardee); + if (do_mark_object != G1MarkNone && forwardee != obj) { + // If the object is self-forwarded we don't need to explicitly + // mark it, the evacuation failure protocol will do so. + mark_forwarded_object(obj, forwardee); + } - if (barrier == G1BarrierKlass) { - do_klass_barrier(p, forwardee); - } - needs_marking = false; + if (barrier == G1BarrierKlass) { + do_klass_barrier(p, forwardee); + } + } else { + if (state == G1FastCSetBiasedMappedArray::IsHumongous) { + _g1->set_humongous_is_live(obj); } - } - if (needs_marking) { // The object is not in collection set. If we're a root scanning // closure during an initial mark pause then attempt to mark the object. if (do_mark_object == G1MarkFromRoot) { @@ -5512,14 +5519,17 @@ void do_oop(oop* p) { oop obj = *p; - if (obj == NULL || !_g1->is_in_cset_or_humongous(obj)) { + G1FastCSetBiasedMappedArray::in_cset_state_t cset_state = _g1->in_cset_state(obj); + if (obj == NULL || cset_state == G1FastCSetBiasedMappedArray::InNeither) { return; } - if (_g1->is_in_cset(obj)) { + if (cset_state == G1FastCSetBiasedMappedArray::InCSet) { assert( obj->is_forwarded(), "invariant" ); *p = obj->forwardee(); } else { assert(!obj->is_forwarded(), "invariant" ); + assert(cset_state == G1FastCSetBiasedMappedArray::IsHumongous, + err_msg("Only allowed InCSet state is IsHumongous, but is %d", cset_state)); _g1->set_humongous_is_live(obj); } } @@ -5575,10 +5585,10 @@ assert(!Metaspace::contains((const void*)p), err_msg("Unexpectedly found a pointer from metadata: " PTR_FORMAT, p)); - _copy_non_heap_obj_cl->do_oop(p); - } + _copy_non_heap_obj_cl->do_oop(p); } } + } }; // Serial drain queue closure. Called as the 'complete_gc' @@ -6550,17 +6560,18 @@ // While this cleanup is not strictly necessary to be done (or done instantly), // given that their occurrence is very low, this saves us this additional // complexity. - if (g1h->humongous_is_live(r->hrs_index()) || - g1h->humongous_region_is_always_live(r)) { + uint region_idx = r->hrs_index(); + if (g1h->humongous_is_live(region_idx) || + g1h->humongous_region_is_always_live(region_idx)) { if (G1TraceReclaimDeadHumongousObjectsAtYoungGC) { gclog_or_tty->print_cr("Live humongous %d region %d with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is dead-bitmap %d live-other %d obj array %d", r->isHumongous(), - r->hrs_index(), + region_idx, r->rem_set()->occupied(), r->rem_set()->strong_code_roots_list_length(), g1h->mark_in_progress() && !g1h->g1_policy()->during_initial_mark_pause(), - g1h->humongous_is_live(r->hrs_index()), + g1h->humongous_is_live(region_idx), oop(r->bottom())->is_objArray() ); } @@ -6576,12 +6587,12 @@ gclog_or_tty->print_cr("Reclaim humongous region %d start "PTR_FORMAT" region %d length "UINT32_FORMAT" with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is dead-bitmap %d live-other %d obj array %d", r->isHumongous(), r->bottom(), - r->hrs_index(), + region_idx, r->region_num(), r->rem_set()->occupied(), r->rem_set()->strong_code_roots_list_length(), g1h->mark_in_progress() && !g1h->g1_policy()->during_initial_mark_pause(), - g1h->humongous_is_live(r->hrs_index()), + g1h->humongous_is_live(region_idx), oop(r->bottom())->is_objArray() ); } @@ -6608,8 +6619,11 @@ void G1CollectedHeap::eagerly_reclaim_humongous_regions() { assert_at_safepoint(true); - guarantee(G1ReclaimDeadHumongousObjectsAtYoungGC, "Feature must be enabled"); - guarantee(_has_humongous_reclaim_candidates, "Should not reach here if no candidates for eager reclaim were found."); + + if (!G1ReclaimDeadHumongousObjectsAtYoungGC || !_has_humongous_reclaim_candidates) { + g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms(0.0, 0); + return; + } double start_time = os::elapsedTime(); --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp 2014-07-21 15:24:54.297096258 +0200 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp 2014-07-21 15:24:54.211093747 +0200 @@ -210,8 +210,8 @@ // At the end of GC, we use this information, among other, to determine whether // we can reclaim the humongous object or not. class G1FastCSetBiasedMappedArray : public G1BiasedMappedArray { - private: - enum { + public: + enum in_cset_state_t { InNeither, // neither in collection set nor humongous InCSet, // region is in collection set only IsHumongous // region is a humongous start region @@ -223,11 +223,11 @@ void clear_humongous(uintptr_t index) { set_by_index(index, InNeither); } - void set_in_cset(uintptr_t index) { assert(get_by_index(index) != IsHumongous, "Should not overwrite InCSetOrHumongous value"); set_by_index(index, InCSet); } + void set_in_cset(uintptr_t index) { assert(get_by_index(index) != IsHumongous, "Should not overwrite IsHumongous value"); set_by_index(index, InCSet); } bool is_in_cset_or_humongous(HeapWord* addr) const { return get_by_address(addr) != InNeither; } - bool is_in_cset_and_humongous(HeapWord* addr) const { return get_by_address(addr) == IsHumongous; } bool is_in_cset(HeapWord* addr) const { return get_by_address(addr) == InCSet; } + in_cset_state_t at(HeapWord* addr) const { return (in_cset_state_t)get_by_address(addr); } void clear() { G1BiasedMappedArray::clear(); } }; @@ -746,7 +746,7 @@ // Returns whether the given region (which must be a humongous (start) region) // is to be considered conservatively live regardless of any other conditions. - bool humongous_region_is_always_live(HeapRegion* region); + bool humongous_region_is_always_live(uint index); // Register the given region to be part of the collection set. inline void register_humongous_region_with_in_cset_fast_test(uint index); // Register regions with humongous objects (actually on the start region) in @@ -1352,7 +1352,7 @@ inline bool is_in_cset_or_humongous(const oop obj); - inline bool is_in_cset_and_humongous(const oop obj); + inline G1FastCSetBiasedMappedArray::in_cset_state_t in_cset_state(const oop obj); // Return "TRUE" iff the given object address is in the reserved // region of g1. --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp 2014-07-21 15:24:54.872113049 +0200 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp 2014-07-21 15:24:54.791110684 +0200 @@ -195,8 +195,8 @@ return _in_cset_fast_test.is_in_cset_or_humongous((HeapWord*)obj); } -bool G1CollectedHeap::is_in_cset_and_humongous(const oop obj) { - return _in_cset_fast_test.is_in_cset_and_humongous((HeapWord*)obj); +G1FastCSetBiasedMappedArray::in_cset_state_t G1CollectedHeap::in_cset_state(const oop obj) { + return _in_cset_fast_test.at((HeapWord*)obj); } void G1CollectedHeap::register_humongous_region_with_in_cset_fast_test(uint index) { --- old/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp 2014-07-21 15:24:55.392128233 +0200 +++ new/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp 2014-07-21 15:24:55.311125868 +0200 @@ -67,7 +67,8 @@ if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (_g1->is_in_cset_or_humongous(obj)) { + G1FastCSetBiasedMappedArray::in_cset_state_t state = _g1->in_cset_state(obj); + if (state == G1FastCSetBiasedMappedArray::InCSet) { // We're not going to even bother checking whether the object is // already forwarded or not, as this usually causes an immediate // stall. We'll try to prefetch the object (for write, given that @@ -86,6 +87,9 @@ _par_scan_state->push_on_queue(p); } else { + if (state == G1FastCSetBiasedMappedArray::IsHumongous) { + _g1->set_humongous_is_live(obj); + } _par_scan_state->update_rs(_from, p, _worker_id); } } --- old/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp 2014-07-21 15:24:55.903143155 +0200 +++ new/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp 2014-07-21 15:24:55.822140790 +0200 @@ -176,12 +176,6 @@ #endif // !PRODUCT if (obj_ptr == NULL) { - // The allocation failure may have been caused by attempted allocation of a - // humongous object. Detect this and process appropriately. - if (_g1h->isHumongous(word_sz)) { - _g1h->set_humongous_is_live(old); - return NULL; - } // This will either forward-to-self, or detect that someone else has // installed a forwarding pointer. return _g1h->handle_evacuation_failure_par(this, old); @@ -259,12 +253,6 @@ } HeapWord* G1ParScanThreadState::allocate_slow(GCAllocPurpose purpose, size_t word_sz) { - // We may have reached the slow path because we tried to allocate memory for a - // humongous object. This just indicates that that humongous object is live - // though. - if (_g1h->isHumongous(word_sz)) { - return NULL; - } HeapWord* obj = NULL; size_t gclab_word_size = _g1h->desired_plab_sz(purpose); if (word_sz * 100 < gclab_word_size * ParallelGCBufferWastePct) { --- old/src/share/vm/gc_implementation/g1/g1ParScanThreadState.inline.hpp 2014-07-21 15:24:56.428158486 +0200 +++ new/src/share/vm/gc_implementation/g1/g1ParScanThreadState.inline.hpp 2014-07-21 15:24:56.342155974 +0200 @@ -52,16 +52,20 @@ // set, due to (benign) races in the claim mechanism during RSet scanning more // than one thread might claim the same card. So the same card may be // processed multiple times. So redo this check. - if (_g1h->is_in_cset_or_humongous(obj)) { + G1FastCSetBiasedMappedArray::in_cset_state_t in_cset_state = _g1h->in_cset_state(obj); + if (in_cset_state == G1FastCSetBiasedMappedArray::InCSet) { oop forwardee; if (obj->is_forwarded()) { forwardee = obj->forwardee(); } else { forwardee = copy_to_survivor_space(obj); } - if (forwardee != NULL) { - oopDesc::encode_store_heap_oop(p, forwardee); - } + oopDesc::encode_store_heap_oop(p, forwardee); + } else if (in_cset_state == G1FastCSetBiasedMappedArray::IsHumongous) { + _g1h->set_humongous_is_live(obj); + } else { + assert(in_cset_state == G1FastCSetBiasedMappedArray::InNeither, + err_msg("In_cset_state must be InNeither here, but is %d", in_cset_state)); } assert(obj != NULL, "Must be"); --- old/src/share/vm/gc_implementation/g1/g1_globals.hpp 2014-07-21 15:24:56.942173495 +0200 +++ new/src/share/vm/gc_implementation/g1/g1_globals.hpp 2014-07-21 15:24:56.862171159 +0200 @@ -289,10 +289,10 @@ "The amount of code root chunks that should be kept at most " \ "as percentage of already allocated.") \ \ - experimental(bool, G1ReclaimDeadHumongousObjectsAtYoungGC, true, \ + experimental(bool, G1ReclaimDeadHumongousObjectsAtYoungGC, true, \ "Try to reclaim dead large objects at every young GC.") \ \ - experimental(bool, G1TraceReclaimDeadHumongousObjectsAtYoungGC, false, \ + experimental(bool, G1TraceReclaimDeadHumongousObjectsAtYoungGC, false, \ "Print some information about large object liveness " \ "at every young GC.") \ \ --- /dev/null 2014-07-02 17:16:05.148841867 +0200 +++ new/test/gc/g1/TestEagerReclaimHumongousRegions.java 2014-07-21 15:24:57.372186051 +0200 @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test TestEagerReclaimHumongousRegions + * @bug 8027959 + * @summary Test to make sure that eager reclaim of humongous objects work. We simply try to fill + * up the heap with humongous objects that should be eagerly reclaimable to avoid Full GC. + * @key gc + * @library /testlibrary + */ + +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import java.util.LinkedList; + +import com.oracle.java.testlibrary.OutputAnalyzer; +import com.oracle.java.testlibrary.ProcessTools; +import com.oracle.java.testlibrary.Asserts; + +class ReclaimRegionFast { + public static final int M = 1024*1024; + + public static LinkedList garbageList = new LinkedList(); + + public static void genGarbage() { + for (int i = 0; i < 32*1024; i++) { + garbageList.add(new int[100]); + } + garbageList.clear(); + } + + // A large object referenced by a static. + static int[] filler = new int[10 * M]; + + public static void main(String[] args) { + + int[] large = new int[M]; + + Object ref_from_stack = large; + + for (int i = 0; i < 100; i++) { + // A large object that will be reclaimed eagerly. + large = new int[6*M]; + genGarbage(); + // Make sure that the compiler cannot completely remove + // the allocation of the large object until here. + System.out.println(large); + } + + // Keep the reference to the first object alive. + System.out.println(ref_from_stack); + } +} + +public class TestEagerReclaimHumongousRegions { + public static void main(String[] args) throws Exception { + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( + "-XX:+UseG1GC", + "-Xms128M", + "-Xmx128M", + "-Xmn16M", + "-XX:+PrintGC", + ReclaimRegionFast.class.getName()); + + Pattern p = Pattern.compile("Full GC"); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + + int found = 0; + Matcher m = p.matcher(output.getStdout()); + while (m.find()) { found++; } + System.out.println("Issued " + found + " Full GCs"); + Asserts.assertLT(found, 10, "Found that " + found + " Full GCs were issued. This is larger than the bound. Eager reclaim seems to not work at all"); + + output.shouldHaveExitValue(0); + } +}