--- old/src/share/vm/gc_implementation/g1/concurrentMark.cpp 2013-01-15 10:38:42.826026689 -0800 +++ new/src/share/vm/gc_implementation/g1/concurrentMark.cpp 2013-01-15 10:38:42.582366518 -0800 @@ -1260,6 +1260,7 @@ Universe::verify(/* silent */ false, /* option */ VerifyOption_G1UsePrevMarking); } + g1h->check_bitmaps("Remark Start"); G1CollectorPolicy* g1p = g1h->g1_policy(); g1p->record_concurrent_mark_remark_start(); @@ -1300,6 +1301,7 @@ Universe::verify(/* silent */ false, /* option */ VerifyOption_G1UseNextMarking); } + g1h->check_bitmaps("Remark End"); assert(!restart_for_overflow(), "sanity"); // Completely reset the marking state since marking completed set_non_marking_state(); @@ -1954,6 +1956,7 @@ Universe::verify(/* silent */ false, /* option */ VerifyOption_G1UsePrevMarking); } + g1h->check_bitmaps("Cleanup Start"); G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy(); g1p->record_concurrent_mark_cleanup_start(); @@ -2109,6 +2112,7 @@ Universe::verify(/* silent */ false, /* option */ VerifyOption_G1UsePrevMarking); } + g1h->check_bitmaps("Cleanup End"); g1h->verify_region_sets_optional(); } @@ -3182,6 +3186,11 @@ void ConcurrentMark::abort() { // Clear all marks to force marking thread to do nothing _nextMarkBitMap->clearAll(); + + // Note we cannot clear the previous marking bitmap here + // since VerifyDuringGC verifies the objects marked during + // a full GC against the previous bitmap. + // Clear the liveness counting data clear_all_count_data(); // Empty mark stack --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2013-01-15 10:38:44.108653524 -0800 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2013-01-15 10:38:43.864084548 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -784,6 +784,7 @@ // match new_top. assert(hr == NULL || (hr->end() == new_end && hr->top() == new_top), "sanity"); + check_bitmaps("Humongous Region Allocation", first_hr); assert(first_hr->used() == word_size * HeapWordSize, "invariant"); _summary_bytes_used += first_hr->used(); @@ -1334,6 +1335,7 @@ verify_before_gc(); + check_bitmaps("Full GC Start"); pre_full_gc_dump(); COMPILER2_PRESENT(DerivedPointerTable::clear()); @@ -1345,8 +1347,7 @@ ref_processor_cm()->verify_no_references_recorded(); // Abandon current iterations of concurrent marking and concurrent - // refinement, if any are in progress. We have to do this before - // wait_until_scan_finished() below. + // refinement, if any are in progress. concurrent_mark()->abort(); // Make sure we'll choose a new allocation region afterwards. @@ -1388,6 +1389,14 @@ HandleMark hm; // Discard invalid handles created during gc G1MarkSweep::invoke_at_safepoint(ref_processor_stw(), do_clear_all_soft_refs); } + + // Clear the previous marking bitmap, if needed for bitmap verification. + // Note we cannot do this when we clear the next marking bitmap in + // ConcurrentMark::abort() above since VerifyDuringGC verifies the + // objects marked during a full GC against the previous bitmap. + if (G1VerifyPrevBitmap) { + ((CMBitMap*) concurrent_mark()->prevMarkBitMap())->clearAll(); + } assert(free_regions() == 0, "we should not have added any free regions"); rebuild_region_sets(false /* free_list_only */); @@ -1402,6 +1411,8 @@ verify_after_gc(); + check_bitmaps("Full GC End"); + assert(!ref_processor_stw()->discovery_enabled(), "Postcondition"); ref_processor_stw()->verify_no_references_recorded(); @@ -3775,6 +3786,7 @@ increment_gc_time_stamp(); verify_before_gc(); + check_bitmaps("GC Start"); COMPILER2_PRESENT(DerivedPointerTable::clear()); @@ -4019,6 +4031,7 @@ increment_gc_time_stamp(); verify_after_gc(); + check_bitmaps("GC End"); assert(!ref_processor_stw()->discovery_enabled(), "Postcondition"); ref_processor_stw()->verify_no_references_recorded(); @@ -5741,6 +5754,10 @@ assert(!hr->is_empty(), "the region should not be empty"); assert(free_list != NULL, "pre-condition"); + if (G1VerifyPrevBitmap) { + MemRegion mr(hr->bottom(), hr->end()); + concurrent_mark()->clearRangePrevBitmap(mr); + } *pre_used += hr->used(); hr->hr_clear(par, true /* clear_space */); free_list->add_as_head(hr); @@ -5879,7 +5896,89 @@ void G1CollectedHeap::verify_dirty_young_regions() { verify_dirty_young_list(_young_list->first_region()); } -#endif + +bool G1CollectedHeap::verify_bitmap(const char* bitmap_name, CMBitMapRO* bitmap, + HeapWord* tams, HeapWord* end) { + guarantee(tams <= end, + err_msg("tams: "PTR_FORMAT" end: "PTR_FORMAT, tams, end)); + HeapWord* result = bitmap->getNextMarkedWordAddress(tams, end); + guarantee(result <= end, + err_msg("result: "PTR_FORMAT" end: "PTR_FORMAT, result, end)); + if (result < end) { + gclog_or_tty->cr(); + gclog_or_tty->print_cr("## wrong marked address on %s bitmap: "PTR_FORMAT, + bitmap_name, result); + gclog_or_tty->print_cr("## %s tams: "PTR_FORMAT" end: "PTR_FORMAT, + bitmap_name, tams, end); + return false; + } + return true; +} + +bool G1CollectedHeap::verify_bitmaps(const char* caller, HeapRegion* hr) { + CMBitMapRO* prev_bitmap = concurrent_mark()->prevMarkBitMap(); + CMBitMapRO* next_bitmap = (CMBitMapRO*) concurrent_mark()->nextMarkBitMap(); + + HeapWord* bottom = hr->bottom(); + HeapWord* ptams = hr->prev_top_at_mark_start(); + HeapWord* ntams = hr->next_top_at_mark_start(); + HeapWord* end = hr->end(); + + bool res_p = true; + if (G1VerifyPrevBitmap) { + res_p = verify_bitmap("prev", prev_bitmap, ptams, end); + } + + bool res_n = true; + if (mark_in_progress() || !_cmThread->in_progress()) { + res_n = verify_bitmap("next", next_bitmap, ntams, end); + } + if (!res_p || !res_n) { + gclog_or_tty->print_cr("#### Bitmap verification failed for "HR_FORMAT, + HR_FORMAT_PARAMS(hr)); + gclog_or_tty->print_cr("#### Caller: %s", caller); + return false; + } + return true; +} + +void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) { + if (!G1VerifyBitmaps) return; + + guarantee(verify_bitmaps(caller, hr), "bitmap verification"); +} + +class G1VerifyBitmapClosure : public HeapRegionClosure { +private: + const char* _caller; + G1CollectedHeap* _g1h; + bool _failures; + +public: + G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) : + _caller(caller), _g1h(g1h), _failures(false) { } + + bool failures() { return _failures; } + + virtual bool doHeapRegion(HeapRegion* hr) { + if (hr->continuesHumongous()) return false; + + bool result = _g1h->verify_bitmaps(_caller, hr); + if (!result) { + _failures = true; + } + return false; + } +}; + +void G1CollectedHeap::check_bitmaps(const char* caller) { + if (!G1VerifyBitmaps) return; + + G1VerifyBitmapClosure cl(caller, this); + heap_region_iterate(&cl); + guarantee(!cl.failures(), "bitmap verification"); +} +#endif // PRODUCT void G1CollectedHeap::cleanUpCardTable() { CardTableModRefBS* ct_bs = (CardTableModRefBS*) (barrier_set()); @@ -6261,6 +6360,7 @@ if (new_alloc_region != NULL) { set_region_short_lived_locked(new_alloc_region); _hr_printer.alloc(new_alloc_region, G1HRPrinter::Eden, young_list_full); + check_bitmaps("Mutator Region Allocation", new_alloc_region); return new_alloc_region; } } @@ -6325,8 +6425,10 @@ if (ap == GCAllocForSurvived) { new_alloc_region->set_survivor(); _hr_printer.alloc(new_alloc_region, G1HRPrinter::Survivor); + check_bitmaps("Survivor Region Allocation", new_alloc_region); } else { _hr_printer.alloc(new_alloc_region, G1HRPrinter::Old); + check_bitmaps("Old Region Allocation", new_alloc_region); } bool during_im = g1_policy()->during_initial_mark_pause(); new_alloc_region->note_start_of_copying(during_im); --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp 2013-01-15 10:38:45.419188527 -0800 +++ new/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp 2013-01-15 10:38:45.180815985 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1209,6 +1209,30 @@ void verify_dirty_young_list(HeapRegion* head) PRODUCT_RETURN; void verify_dirty_young_regions() PRODUCT_RETURN; +#ifndef PRODUCT + // Make sure that the given bitmap has no marked objects in the + // range [from,limit). If it does, print an error message and return + // false. Otherwise, just return true. bitmap_name should be "prev" + // or "next". + bool verify_bitmap(const char* bitmap_name, CMBitMapRO* bitmap, + HeapWord* from, HeapWord* limit); + + // Verify that the prev / next bitmap range [tams,end) for the given + // region has no marks. Return true if all is well, false if errors + // are detected. + bool verify_bitmaps(const char* caller, HeapRegion* hr); +#endif // PRODUCT + + // If G1VerifyBitmaps is set, verify that the marking bitmaps for + // the given region do not have any spurious marks. If errors are + // detected, print appropriate error messages and crash. + void check_bitmaps(const char* caller, HeapRegion* hr) PRODUCT_RETURN; + + // If G1VerifyBitmaps is set, verify that the marking bitmaps do not + // have any spurious marks. If errors are detected, print + // appropriate error messages and crash. + void check_bitmaps(const char* caller) PRODUCT_RETURN; + // verify_region_sets() performs verification over the region // lists. It will be compiled in the product code to be used when // necessary (i.e., during heap verification). --- old/src/share/vm/gc_implementation/g1/g1EvacFailure.hpp 2013-01-15 10:38:46.659815482 -0800 +++ new/src/share/vm/gc_implementation/g1/g1EvacFailure.hpp 2013-01-15 10:38:46.417602644 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2013 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -191,6 +191,7 @@ hr->note_self_forwarding_removal_start(during_initial_mark, during_conc_mark); + _g1h->check_bitmaps("Self-Forwarding Ptr Removal", hr); // In the common case (i.e. when there is no evacuation // failure) we make sure that the following is done when --- old/src/share/vm/gc_implementation/g1/g1_globals.hpp 2013-01-15 10:38:47.799609797 -0800 +++ new/src/share/vm/gc_implementation/g1/g1_globals.hpp 2013-01-15 10:38:47.576408110 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -340,8 +340,14 @@ \ develop(bool, G1EvacuationFailureALotDuringMixedGC, true, \ "Force use of evacuation failure handling during mixed " \ - "evacuation pauses") - + "evacuation pauses") \ + \ + develop(bool, G1VerifyBitmaps, false, \ + "Verifies the consistency of the marking bitmaps") \ + \ + develop(bool, G1VerifyPrevBitmap, false, \ + "Verifies the consistency of the prev marking bitmap") + G1_FLAGS(DECLARE_DEVELOPER_FLAG, DECLARE_PD_DEVELOPER_FLAG, DECLARE_PRODUCT_FLAG, DECLARE_PD_PRODUCT_FLAG, DECLARE_DIAGNOSTIC_FLAG, DECLARE_EXPERIMENTAL_FLAG, DECLARE_NOTPRODUCT_FLAG, DECLARE_MANAGEABLE_FLAG, DECLARE_PRODUCT_RW_FLAG) #endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1_GLOBALS_HPP