# HG changeset patch # User david # Date 1453294847 -3600 # Wed Jan 20 14:00:47 2016 +0100 # Node ID f3d67ab3d8b38d168736feffd578cd87a6bf403c # Parent 3a7618a9f2d634e893a128f4332dd51b9159841f [mq]: g1collectedheap diff --git a/src/share/vm/gc/g1/concurrentMark.cpp b/src/share/vm/gc/g1/concurrentMark.cpp --- a/src/share/vm/gc/g1/concurrentMark.cpp +++ b/src/share/vm/gc/g1/concurrentMark.cpp @@ -1062,7 +1062,7 @@ g1h->prepare_for_verify(); Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)"); } - g1h->check_bitmaps("Remark Start"); + g1h->verifier()->check_bitmaps("Remark Start"); G1CollectorPolicy* g1p = g1h->g1_policy(); g1p->record_concurrent_mark_remark_start(); @@ -1111,7 +1111,7 @@ g1h->prepare_for_verify(); Universe::verify(VerifyOption_G1UseNextMarking, "During GC (after)"); } - g1h->check_bitmaps("Remark End"); + g1h->verifier()->check_bitmaps("Remark End"); assert(!restart_for_overflow(), "sanity"); // Completely reset the marking state since marking completed set_non_marking_state(); @@ -1605,14 +1605,14 @@ return; } - g1h->verify_region_sets_optional(); + g1h->verifier()->verify_region_sets_optional(); if (VerifyDuringGC) { HandleMark hm; // handle scope g1h->prepare_for_verify(); Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)"); } - g1h->check_bitmaps("Cleanup Start"); + g1h->verifier()->check_bitmaps("Cleanup Start"); G1CollectorPolicy* g1p = g1h->g1_policy(); g1p->record_concurrent_mark_cleanup_start(); @@ -1702,9 +1702,9 @@ Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (after)"); } - g1h->check_bitmaps("Cleanup End"); - - g1h->verify_region_sets_optional(); + g1h->verifier()->check_bitmaps("Cleanup End"); + + g1h->verifier()->verify_region_sets_optional(); // We need to make this be a "collection" so any collection pause that // races with it goes around and waits for completeCleanup to finish. diff --git a/src/share/vm/gc/g1/g1Allocator.cpp b/src/share/vm/gc/g1/g1Allocator.cpp --- a/src/share/vm/gc/g1/g1Allocator.cpp +++ b/src/share/vm/gc/g1/g1Allocator.cpp @@ -29,8 +29,11 @@ #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1CollectorPolicy.hpp" #include "gc/g1/g1MarkSweep.hpp" +#include "gc/g1/vm_operations_g1.hpp" #include "gc/g1/heapRegion.inline.hpp" #include "gc/g1/heapRegionSet.inline.hpp" +#include "gc/shared/gcLocker.hpp" +#include "runtime/vmThread.hpp" G1DefaultAllocator::G1DefaultAllocator(G1CollectedHeap* heap) : G1Allocator(heap), @@ -158,6 +161,578 @@ } } +inline bool G1Allocator::is_humongous(size_t word_size) { + return G1CollectedHeap::is_humongous(word_size); +} + +HeapWord* G1Allocator::attempt_allocation_slow(size_t word_size, + AllocationContext_t context, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret) { + // Make sure you read the note in attempt_allocation_humongous(). + + assert_heap_not_locked_and_not_at_safepoint(); + assert(!is_humongous(word_size), "attempt_allocation_slow() should not " + "be called for humongous allocation requests"); + + // We should only get here after the first-level allocation attempt + // (attempt_allocation()) failed to allocate. + + // We will loop until a) we manage to successfully perform the + // allocation or b) we successfully schedule a collection which + // fails to perform the allocation. b) is the only case when we'll + // return NULL. + HeapWord* result = NULL; + for (int try_count = 1; /* we'll return */; try_count += 1) { + bool should_try_gc; + uint gc_count_before; + + { + MutexLockerEx x(Heap_lock); + result = attempt_allocation_locked(word_size, context); + if (result != NULL) { + return result; + } + + if (GCLocker::is_active_and_needs_gc()) { + if (_g1h->g1_policy()->can_expand_young_list()) { + // No need for an ergo verbose message here, + // can_expand_young_list() does this when it returns true. + result = attempt_allocation_force(word_size, context); + if (result != NULL) { + return result; + } + } + should_try_gc = false; + } else { + // The GCLocker may not be active but the GCLocker initiated + // GC may not yet have been performed (GCLocker::needs_gc() + // returns true). In this case we do not try this GC and + // wait until the GCLocker initiated GC is performed, and + // then retry the allocation. + if (GCLocker::needs_gc()) { + should_try_gc = false; + } else { + // Read the GC count while still holding the Heap_lock. + gc_count_before = _g1h->total_collections(); + should_try_gc = true; + } + } + } + + if (should_try_gc) { + bool succeeded; + result = _g1h->do_collection_pause(word_size, gc_count_before, &succeeded, + GCCause::_g1_inc_collection_pause); + if (result != NULL) { + assert(succeeded, "only way to get back a non-NULL result"); + return result; + } + + if (succeeded) { + // If we get here we successfully scheduled a collection which + // failed to allocate. No point in trying to allocate + // further. We'll just return NULL. + MutexLockerEx x(Heap_lock); + *gc_count_before_ret = _g1h->total_collections(); + return NULL; + } + } else { + if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) { + MutexLockerEx x(Heap_lock); + *gc_count_before_ret = _g1h->total_collections(); + return NULL; + } + // The GCLocker is either active or the GCLocker initiated + // GC has not yet been performed. Stall until it is and + // then retry the allocation. + GCLocker::stall_until_clear(); + (*gclocker_retry_count_ret) += 1; + } + + // We can reach here if we were unsuccessful in scheduling a + // collection (because another thread beat us to it) or if we were + // stalled due to the GC locker. In either can we should retry the + // allocation attempt in case another thread successfully + // performed a collection and reclaimed enough space. We do the + // first attempt (without holding the Heap_lock) here and the + // follow-on attempt will be at the start of the next loop + // iteration (after taking the Heap_lock). + result = attempt_allocation(word_size, context); + if (result != NULL) { + return result; + } + + // Give a warning if we seem to be looping forever. + if ((QueuedAllocationWarningCount > 0) && + (try_count % QueuedAllocationWarningCount == 0)) { + warning("G1CollectedHeap::attempt_allocation_slow() " + "retries %d times", try_count); + } + } + + ShouldNotReachHere(); + return NULL; +} + +HeapWord* G1Allocator::humongous_obj_allocate_initialize_regions(uint first, + uint num_regions, + size_t word_size, + AllocationContext_t context) { + assert(first != G1_NO_HRM_INDEX, "pre-condition"); + assert(is_humongous(word_size), "word_size should be humongous"); + assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition"); + + // Index of last region in the series. + uint last = first + num_regions - 1; + + // We need to initialize the region(s) we just discovered. This is + // a bit tricky given that it can happen concurrently with + // refinement threads refining cards on these regions and + // potentially wanting to refine the BOT as they are scanning + // those cards (this can happen shortly after a cleanup; see CR + // 6991377). So we have to set up the region(s) carefully and in + // a specific order. + + // The word size sum of all the regions we will allocate. + size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords; + assert(word_size <= word_size_sum, "sanity"); + + // This will be the "starts humongous" region. + HeapRegion* first_hr = _g1h->region_at(first); + // The header of the new object will be placed at the bottom of + // the first region. + HeapWord* new_obj = first_hr->bottom(); + // This will be the new top of the new object. + HeapWord* obj_top = new_obj + word_size; + + // First, we need to zero the header of the space that we will be + // allocating. When we update top further down, some refinement + // threads might try to scan the region. By zeroing the header we + // ensure that any thread that will try to scan the region will + // come across the zero klass word and bail out. + // + // NOTE: It would not have been correct to have used + // CollectedHeap::fill_with_object() and make the space look like + // an int array. The thread that is doing the allocation will + // later update the object header to a potentially different array + // type and, for a very short period of time, the klass and length + // fields will be inconsistent. This could cause a refinement + // thread to calculate the object size incorrectly. + Copy::fill_to_words(new_obj, oopDesc::header_size(), 0); + + // How many words we use for filler objects. + size_t word_fill_size = word_size_sum - word_size; + + // How many words memory we "waste" which cannot hold a filler object. + size_t words_not_fillable = 0; + + if (word_fill_size >= _g1h->min_fill_size()) { + _g1h->fill_with_objects(obj_top, word_fill_size); + } else if (word_fill_size > 0) { + // We have space to fill, but we cannot fit an object there. + words_not_fillable = word_fill_size; + word_fill_size = 0; + } + + // We will set up the first region as "starts humongous". This + // will also update the BOT covering all the regions to reflect + // that there is a single object that starts at the bottom of the + // first region. + first_hr->set_starts_humongous(obj_top, word_fill_size); + first_hr->set_allocation_context(context); + // Then, if there are any, we will set up the "continues + // humongous" regions. + HeapRegion* hr = NULL; + for (uint i = first + 1; i <= last; ++i) { + hr = _g1h->region_at(i); + hr->set_continues_humongous(first_hr); + hr->set_allocation_context(context); + } + + // Up to this point no concurrent thread would have been able to + // do any scanning on any region in this series. All the top + // fields still point to bottom, so the intersection between + // [bottom,top] and [card_start,card_end] will be empty. Before we + // update the top fields, we'll do a storestore to make sure that + // no thread sees the update to top before the zeroing of the + // object header and the BOT initialization. + OrderAccess::storestore(); + + // Now, we will update the top fields of the "continues humongous" + // regions except the last one. + for (uint i = first; i < last; ++i) { + hr = _g1h->region_at(i); + hr->set_top(hr->end()); + } + + hr = _g1h->region_at(last); + // If we cannot fit a filler object, we must set top to the end + // of the humongous object, otherwise we cannot iterate the heap + // and the BOT will not be complete. + hr->set_top(hr->end() - words_not_fillable); + + assert(hr->bottom() < obj_top && obj_top <= hr->end(), + "obj_top should be in last region"); + + _g1h->verifier()->check_bitmaps("Humongous Region Allocation", first_hr); + + assert(words_not_fillable == 0 || + first_hr->bottom() + word_size_sum - words_not_fillable == hr->top(), + "Miscalculation in humongous allocation"); + + _g1h->increase_used((word_size_sum - words_not_fillable) * HeapWordSize); + + for (uint i = first; i <= last; ++i) { + hr = _g1h->region_at(i); + _g1h->_humongous_set.add(hr); + _g1h->hr_printer()->alloc(hr); + } + + return new_obj; +} + +size_t G1Allocator::humongous_obj_size_in_regions(size_t word_size) { + assert(is_humongous(word_size), "Object of size " SIZE_FORMAT " must be humongous here", word_size); + return align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords; +} + +HeapWord* G1Allocator::attempt_allocation_at_safepoint(size_t word_size, + AllocationContext_t context, + bool expect_null_mutator_alloc_region) { + assert_at_safepoint(true /* should_be_vm_thread */); + assert(!has_mutator_alloc_region(context) || !expect_null_mutator_alloc_region, + "the current alloc region was unexpectedly found to be non-NULL"); + + if (!is_humongous(word_size)) { + return attempt_allocation_locked(word_size, context); + } else { + HeapWord* result = humongous_obj_allocate(word_size, context); + if (result != NULL && _g1h->g1_policy()->need_to_start_conc_mark("STW humongous allocation")) { + _g1h->collector_state()->set_initiate_conc_mark_if_possible(true); + } + return result; + } + + ShouldNotReachHere(); +} + +// If could fit into free regions w/o expansion, try. +// Otherwise, if can expand, do so. +// Otherwise, if using ex regions might help, try with ex given back. +HeapWord* G1Allocator::humongous_obj_allocate(size_t word_size, AllocationContext_t context) { + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + + _g1h->verifier()->verify_region_sets_optional(); + + uint first = G1_NO_HRM_INDEX; + uint obj_regions = (uint) humongous_obj_size_in_regions(word_size); + + if (obj_regions == 1) { + // Only one region to allocate, try to use a fast path by directly allocating + // from the free lists. Do not try to expand here, we will potentially do that + // later. + HeapRegion* hr = _g1h->new_region(word_size, true /* is_old */, false /* do_expand */); + if (hr != NULL) { + first = hr->hrm_index(); + } + } else { + // We can't allocate humongous regions spanning more than one region while + // cleanupComplete() is running, since some of the regions we find to be + // empty might not yet be added to the free list. It is not straightforward + // to know in which list they are on so that we can remove them. We only + // need to do this if we need to allocate more than one region to satisfy the + // current humongous allocation request. If we are only allocating one region + // we use the one-region region allocation code (see above), that already + // potentially waits for regions from the secondary free list. + _g1h->wait_while_free_regions_coming(); + _g1h->append_secondary_free_list_if_not_empty_with_lock(); + + // Policy: Try only empty regions (i.e. already committed first). Maybe we + // are lucky enough to find some. + first = _g1h->_hrm.find_contiguous_only_empty(obj_regions); + if (first != G1_NO_HRM_INDEX) { + _g1h->_hrm.allocate_free_regions_starting_at(first, obj_regions); + } + } + + if (first == G1_NO_HRM_INDEX) { + // Policy: We could not find enough regions for the humongous object in the + // free list. Look through the heap to find a mix of free and uncommitted regions. + // If so, try expansion. + first = _g1h->_hrm.find_contiguous_empty_or_unavailable(obj_regions); + if (first != G1_NO_HRM_INDEX) { + // We found something. Make sure these regions are committed, i.e. expand + // the heap. Alternatively we could do a defragmentation GC. + log_debug(gc, ergo, heap)("Attempt heap expansion (humongous allocation request failed). Allocation request: " SIZE_FORMAT "B", + word_size * HeapWordSize); + + + _g1h->_hrm.expand_at(first, obj_regions); + _g1h->g1_policy()->record_new_heap_size(_g1h->num_regions()); + +#ifdef ASSERT + for (uint i = first; i < first + obj_regions; ++i) { + HeapRegion* hr = _g1h->region_at(i); + assert(hr->is_free(), "sanity"); + assert(hr->is_empty(), "sanity"); + assert(_g1h->is_on_master_free_list(hr), "sanity"); + } +#endif + _g1h->_hrm.allocate_free_regions_starting_at(first, obj_regions); + } else { + // Policy: Potentially trigger a defragmentation GC. + } + } + + HeapWord* result = NULL; + if (first != G1_NO_HRM_INDEX) { + result = humongous_obj_allocate_initialize_regions(first, obj_regions, + word_size, context); + assert(result != NULL, "it should always return a valid result"); + + // A successful humongous object allocation changes the used space + // information of the old generation so we need to recalculate the + // sizes and update the jstat counters here. + _g1h->g1mm()->update_sizes(); + } + + _g1h->verifier()->verify_region_sets_optional(); + + return result; +} + + +HeapWord* G1Allocator::attempt_allocation_humongous(size_t word_size, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret) { + // The structure of this method has a lot of similarities to + // attempt_allocation_slow(). The reason these two were not merged + // into a single one is that such a method would require several "if + // allocation is not humongous do this, otherwise do that" + // conditional paths which would obscure its flow. In fact, an early + // version of this code did use a unified method which was harder to + // follow and, as a result, it had subtle bugs that were hard to + // track down. So keeping these two methods separate allows each to + // be more readable. It will be good to keep these two in sync as + // much as possible. + + assert_heap_not_locked_and_not_at_safepoint(); + assert(is_humongous(word_size), "attempt_allocation_humongous() " + "should only be called for humongous allocations"); + + // Humongous objects can exhaust the heap quickly, so we should check if we + // need to start a marking cycle at each humongous object allocation. We do + // the check before we do the actual allocation. The reason for doing it + // before the allocation is that we avoid having to keep track of the newly + // allocated memory while we do a GC. + if (_g1h->g1_policy()->need_to_start_conc_mark("concurrent humongous allocation", + word_size)) { + _g1h->collect(GCCause::_g1_humongous_allocation); + } + + // We will loop until a) we manage to successfully perform the + // allocation or b) we successfully schedule a collection which + // fails to perform the allocation. b) is the only case when we'll + // return NULL. + HeapWord* result = NULL; + for (int try_count = 1; /* we'll return */; try_count += 1) { + bool should_try_gc; + uint gc_count_before; + + { + MutexLockerEx x(Heap_lock); + + // Given that humongous objects are not allocated in young + // regions, we'll first try to do the allocation without doing a + // collection hoping that there's enough space in the heap. + result = humongous_obj_allocate(word_size, AllocationContext::current()); + if (result != NULL) { + size_t size_in_regions = humongous_obj_size_in_regions(word_size); + _g1h->g1_policy()->add_bytes_allocated_in_old_since_last_gc(size_in_regions * HeapRegion::GrainBytes); + return result; + } + + if (GCLocker::is_active_and_needs_gc()) { + should_try_gc = false; + } else { + // The GCLocker may not be active but the GCLocker initiated + // GC may not yet have been performed (GCLocker::needs_gc() + // returns true). In this case we do not try this GC and + // wait until the GCLocker initiated GC is performed, and + // then retry the allocation. + if (GCLocker::needs_gc()) { + should_try_gc = false; + } else { + // Read the GC count while still holding the Heap_lock. + gc_count_before = _g1h->total_collections(); + should_try_gc = true; + } + } + } + + if (should_try_gc) { + // If we failed to allocate the humongous object, we should try to + // do a collection pause (if we're allowed) in case it reclaims + // enough space for the allocation to succeed after the pause. + + bool succeeded; + result = _g1h->do_collection_pause(word_size, gc_count_before, &succeeded, + GCCause::_g1_humongous_allocation); + if (result != NULL) { + assert(succeeded, "only way to get back a non-NULL result"); + return result; + } + + if (succeeded) { + // If we get here we successfully scheduled a collection which + // failed to allocate. No point in trying to allocate + // further. We'll just return NULL. + MutexLockerEx x(Heap_lock); + *gc_count_before_ret = _g1h->total_collections(); + return NULL; + } + } else { + if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) { + MutexLockerEx x(Heap_lock); + *gc_count_before_ret = _g1h->total_collections(); + return NULL; + } + // The GCLocker is either active or the GCLocker initiated + // GC has not yet been performed. Stall until it is and + // then retry the allocation. + GCLocker::stall_until_clear(); + (*gclocker_retry_count_ret) += 1; + } + + // We can reach here if we were unsuccessful in scheduling a + // collection (because another thread beat us to it) or if we were + // stalled due to the GC locker. In either can we should retry the + // allocation attempt in case another thread successfully + // performed a collection and reclaimed enough space. Give a + // warning if we seem to be looping forever. + + if ((QueuedAllocationWarningCount > 0) && + (try_count % QueuedAllocationWarningCount == 0)) { + warning("G1CollectedHeap::attempt_allocation_humongous() " + "retries %d times", try_count); + } + } + + ShouldNotReachHere(); + return NULL; +} + + +HeapWord* G1Allocator::mem_allocate(size_t word_size, + bool* gc_overhead_limit_was_exceeded) { + assert_heap_not_locked_and_not_at_safepoint(); + + // Loop until the allocation is satisfied, or unsatisfied after GC. + for (uint try_count = 1, gclocker_retry_count = 0; /* we'll return */; try_count += 1) { + uint gc_count_before; + + HeapWord* result = NULL; + if (!is_humongous(word_size)) { + result = attempt_allocation(word_size, &gc_count_before, &gclocker_retry_count); + } else { + result = attempt_allocation_humongous(word_size, &gc_count_before, &gclocker_retry_count); + } + if (result != NULL) { + return result; + } + + // Create the garbage collection operation... + VM_G1CollectForAllocation op(gc_count_before, word_size); + op.set_allocation_context(AllocationContext::current()); + + // ...and get the VM thread to execute it. + VMThread::execute(&op); + + if (op.prologue_succeeded() && op.pause_succeeded()) { + // If the operation was successful we'll return the result even + // if it is NULL. If the allocation attempt failed immediately + // after a Full GC, it's unlikely we'll be able to allocate now. + HeapWord* result = op.result(); + if (result != NULL && !is_humongous(word_size)) { + // Allocations that take place on VM operations do not do any + // card dirtying and we have to do it here. We only have to do + // this for non-humongous allocations, though. + _g1h->dirty_young_block(result, word_size); + } + return result; + } else { + if (gclocker_retry_count > GCLockerRetryAllocationCount) { + return NULL; + } + assert(op.result() == NULL, + "the result should be NULL if the VM op did not succeed"); + } + + // Give a warning if we seem to be looping forever. + if ((QueuedAllocationWarningCount > 0) && + (try_count % QueuedAllocationWarningCount == 0)) { + warning("G1CollectedHeap::mem_allocate retries %d times", try_count); + } + } + + ShouldNotReachHere(); + return NULL; +} + +#ifndef PRODUCT +void G1Allocator::allocate_dummy_regions(size_t word_size) { + // And as a result the region we'll allocate will be humongous. + guarantee(is_humongous(word_size), "sanity"); + + for (uintx i = 0; i < G1DummyRegionsPerGC; ++i) { + // Let's use the existing mechanism for the allocation + HeapWord* dummy_obj = humongous_obj_allocate(word_size, + AllocationContext::system()); + if (dummy_obj != NULL) { + MemRegion mr(dummy_obj, word_size); + CollectedHeap::fill_with_object(mr); + } else { + // If we can't allocate once, we probably cannot allocate + // again. Let's get out of the loop. + break; + } + } +} +#endif // !PRODUCT + +inline HeapWord* G1Allocator::attempt_allocation(size_t word_size, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret) { + assert_heap_not_locked_and_not_at_safepoint(); + assert(!is_humongous(word_size), "attempt_allocation() should not " + "be called for humongous allocation requests"); + + AllocationContext_t context = AllocationContext::current(); + HeapWord* result = attempt_allocation(word_size, context); + + if (result == NULL) { + result = attempt_allocation_slow(word_size, + context, + gc_count_before_ret, + gclocker_retry_count_ret); + } + assert_heap_not_locked(); + if (result != NULL) { + _g1h->dirty_young_block(result, word_size); + } + return result; +} + +HeapWord* G1Allocator::allocate_new_tlab(size_t word_size) { + assert_heap_not_locked_and_not_at_safepoint(); + assert(!is_humongous(word_size), "we do not allow humongous TLABs"); + + uint dummy_gc_count_before; + uint dummy_gclocker_retry_count = 0; + return attempt_allocation(word_size, &dummy_gc_count_before, &dummy_gclocker_retry_count); +} + HeapWord* G1Allocator::par_allocate_during_gc(InCSetState dest, size_t word_size, AllocationContext_t context) { @@ -189,7 +764,7 @@ size_t desired_word_size, size_t* actual_word_size, AllocationContext_t context) { - assert(!_g1h->is_humongous(desired_word_size), + assert(!is_humongous(desired_word_size), "we should not be seeing humongous-size allocations in this path"); HeapWord* result = survivor_gc_alloc_region(context)->attempt_allocation(min_word_size, @@ -216,7 +791,7 @@ size_t desired_word_size, size_t* actual_word_size, AllocationContext_t context) { - assert(!_g1h->is_humongous(desired_word_size), + assert(!is_humongous(desired_word_size), "we should not be seeing humongous-size allocations in this path"); HeapWord* result = old_gc_alloc_region(context)->attempt_allocation(min_word_size, diff --git a/src/share/vm/gc/g1/g1Allocator.hpp b/src/share/vm/gc/g1/g1Allocator.hpp --- a/src/share/vm/gc/g1/g1Allocator.hpp +++ b/src/share/vm/gc/g1/g1Allocator.hpp @@ -41,6 +41,8 @@ protected: G1CollectedHeap* _g1h; + static inline bool is_humongous(size_t word_size); + virtual MutatorAllocRegion* mutator_alloc_region(AllocationContext_t context) = 0; virtual bool survivor_is_full(AllocationContext_t context) const = 0; @@ -63,6 +65,53 @@ size_t desired_word_size, size_t* actual_word_size, AllocationContext_t context); + + // Second-level mutator allocation attempt: take the Heap_lock and + // retry the allocation attempt, potentially scheduling a GC + // pause. This should only be used for non-humongous allocations. + HeapWord* attempt_allocation_slow(size_t word_size, + AllocationContext_t context, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret); + + // Takes the Heap_lock and attempts a humongous allocation. It can + // potentially schedule a GC pause. + HeapWord* attempt_allocation_humongous(size_t word_size, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret); + + // Initialize a contiguous set of free regions of length num_regions + // and starting at index first so that they appear as a single + // humongous region. + HeapWord* humongous_obj_allocate_initialize_regions(uint first, + uint num_regions, + size_t word_size, + AllocationContext_t context); + + // Attempt to allocate a humongous object of the given size. Return + // NULL if unsuccessful. + HeapWord* humongous_obj_allocate(size_t word_size, AllocationContext_t context); + + // Returns the number of regions the humongous object of the given word size + // requires. + static size_t humongous_obj_size_in_regions(size_t word_size); + + // The following three methods take a gc_count_before_ret + // parameter which is used to return the GC count if the method + // returns NULL. Given that we are required to read the GC count + // while holding the Heap_lock, and these paths will take the + // Heap_lock at some point, it's easier to get them to read the GC + // count while holding the Heap_lock before they return NULL instead + // of the caller (namely: mem_allocate()) having to also take the + // Heap_lock just to read the GC count. + + // First-level mutator allocation attempt: try to allocate out of + // the mutator alloc region without taking the Heap_lock. This + // should only be used for non-humongous allocations. + inline HeapWord* attempt_allocation(size_t word_size, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret); + public: G1Allocator(G1CollectedHeap* heap) : _g1h(heap) { } virtual ~G1Allocator() { } @@ -73,6 +122,57 @@ // Do we currently have an active mutator region to allocate into? bool has_mutator_alloc_region(AllocationContext_t context) { return mutator_alloc_region(context)->get() != NULL; } #endif + + // Allocation attempt that should be called during safepoints (e.g., + // at the end of a successful GC). expect_null_mutator_alloc_region + // specifies whether the mutator alloc region is expected to be NULL + // or not. + HeapWord* attempt_allocation_at_safepoint(size_t word_size, + AllocationContext_t context, + bool expect_null_mutator_alloc_region); + + // This is a non-product method that is helpful for testing. It is + // called at the end of a GC and artificially expands the heap by + // allocating a number of dead regions. This way we can induce very + // frequent marking cycles and stress the cleanup / concurrent + // cleanup code more (as all the regions that will be allocated by + // this method will be found dead by the marking cycle). + void allocate_dummy_regions(size_t word_size) PRODUCT_RETURN; + + // The following two methods, allocate_new_tlab() and + // mem_allocate(), are the two main entry points from the runtime + // into the G1's allocation routines. They have the following + // assumptions: + // + // * They should both be called outside safepoints. + // + // * They should both be called without holding the Heap_lock. + // + // * All allocation requests for new TLABs should go to + // allocate_new_tlab(). + // + // * All non-TLAB allocation requests should go to mem_allocate(). + // + // * If either call cannot satisfy the allocation request using the + // current allocating region, they will try to get a new one. If + // this fails, they will attempt to do an evacuation pause and + // retry the allocation. + // + // * If all allocation attempts fail, even after trying to schedule + // an evacuation pause, allocate_new_tlab() will return NULL, + // whereas mem_allocate() will attempt a heap expansion and/or + // schedule a Full GC. + // + // * We do not allow humongous-sized TLABs. So, allocate_new_tlab + // should never be called with word_size being humongous. All + // humongous allocation requests should go to mem_allocate() which + // will satisfy them with a special path. + + HeapWord* allocate_new_tlab(size_t word_size); + + virtual HeapWord* mem_allocate(size_t word_size, + bool* gc_overhead_limit_was_exceeded); + virtual void init_mutator_alloc_region() = 0; virtual void release_mutator_alloc_region() = 0; diff --git a/src/share/vm/gc/g1/g1CollectedHeap.cpp b/src/share/vm/gc/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc/g1/g1CollectedHeap.cpp +++ b/src/share/vm/gc/g1/g1CollectedHeap.cpp @@ -38,6 +38,7 @@ #include "gc/g1/g1CollectorState.hpp" #include "gc/g1/g1EvacStats.inline.hpp" #include "gc/g1/g1GCPhaseTimes.hpp" +#include "gc/g1/g1HeapVerifier.hpp" #include "gc/g1/g1MarkSweep.hpp" #include "gc/g1/g1OopClosures.inline.hpp" #include "gc/g1/g1ParScanThreadState.inline.hpp" @@ -298,388 +299,13 @@ return res; } -HeapWord* -G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first, - uint num_regions, - size_t word_size, - AllocationContext_t context) { - assert(first != G1_NO_HRM_INDEX, "pre-condition"); - assert(is_humongous(word_size), "word_size should be humongous"); - assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition"); - - // Index of last region in the series. - uint last = first + num_regions - 1; - - // We need to initialize the region(s) we just discovered. This is - // a bit tricky given that it can happen concurrently with - // refinement threads refining cards on these regions and - // potentially wanting to refine the BOT as they are scanning - // those cards (this can happen shortly after a cleanup; see CR - // 6991377). So we have to set up the region(s) carefully and in - // a specific order. - - // The word size sum of all the regions we will allocate. - size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords; - assert(word_size <= word_size_sum, "sanity"); - - // This will be the "starts humongous" region. - HeapRegion* first_hr = region_at(first); - // The header of the new object will be placed at the bottom of - // the first region. - HeapWord* new_obj = first_hr->bottom(); - // This will be the new top of the new object. - HeapWord* obj_top = new_obj + word_size; - - // First, we need to zero the header of the space that we will be - // allocating. When we update top further down, some refinement - // threads might try to scan the region. By zeroing the header we - // ensure that any thread that will try to scan the region will - // come across the zero klass word and bail out. - // - // NOTE: It would not have been correct to have used - // CollectedHeap::fill_with_object() and make the space look like - // an int array. The thread that is doing the allocation will - // later update the object header to a potentially different array - // type and, for a very short period of time, the klass and length - // fields will be inconsistent. This could cause a refinement - // thread to calculate the object size incorrectly. - Copy::fill_to_words(new_obj, oopDesc::header_size(), 0); - - // How many words we use for filler objects. - size_t word_fill_size = word_size_sum - word_size; - - // How many words memory we "waste" which cannot hold a filler object. - size_t words_not_fillable = 0; - - if (word_fill_size >= min_fill_size()) { - fill_with_objects(obj_top, word_fill_size); - } else if (word_fill_size > 0) { - // We have space to fill, but we cannot fit an object there. - words_not_fillable = word_fill_size; - word_fill_size = 0; - } - - // We will set up the first region as "starts humongous". This - // will also update the BOT covering all the regions to reflect - // that there is a single object that starts at the bottom of the - // first region. - first_hr->set_starts_humongous(obj_top, word_fill_size); - first_hr->set_allocation_context(context); - // Then, if there are any, we will set up the "continues - // humongous" regions. - HeapRegion* hr = NULL; - for (uint i = first + 1; i <= last; ++i) { - hr = region_at(i); - hr->set_continues_humongous(first_hr); - hr->set_allocation_context(context); - } - - // Up to this point no concurrent thread would have been able to - // do any scanning on any region in this series. All the top - // fields still point to bottom, so the intersection between - // [bottom,top] and [card_start,card_end] will be empty. Before we - // update the top fields, we'll do a storestore to make sure that - // no thread sees the update to top before the zeroing of the - // object header and the BOT initialization. - OrderAccess::storestore(); - - // Now, we will update the top fields of the "continues humongous" - // regions except the last one. - for (uint i = first; i < last; ++i) { - hr = region_at(i); - hr->set_top(hr->end()); - } - - hr = region_at(last); - // If we cannot fit a filler object, we must set top to the end - // of the humongous object, otherwise we cannot iterate the heap - // and the BOT will not be complete. - hr->set_top(hr->end() - words_not_fillable); - - assert(hr->bottom() < obj_top && obj_top <= hr->end(), - "obj_top should be in last region"); - - check_bitmaps("Humongous Region Allocation", first_hr); - - assert(words_not_fillable == 0 || - first_hr->bottom() + word_size_sum - words_not_fillable == hr->top(), - "Miscalculation in humongous allocation"); - - increase_used((word_size_sum - words_not_fillable) * HeapWordSize); - - for (uint i = first; i <= last; ++i) { - hr = region_at(i); - _humongous_set.add(hr); - _hr_printer.alloc(hr); - } - - return new_obj; +HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) { + return _allocator->allocate_new_tlab(word_size); } -size_t G1CollectedHeap::humongous_obj_size_in_regions(size_t word_size) { - assert(is_humongous(word_size), "Object of size " SIZE_FORMAT " must be humongous here", word_size); - return align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords; -} - -// If could fit into free regions w/o expansion, try. -// Otherwise, if can expand, do so. -// Otherwise, if using ex regions might help, try with ex given back. -HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) { - assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); - - verify_region_sets_optional(); - - uint first = G1_NO_HRM_INDEX; - uint obj_regions = (uint) humongous_obj_size_in_regions(word_size); - - if (obj_regions == 1) { - // Only one region to allocate, try to use a fast path by directly allocating - // from the free lists. Do not try to expand here, we will potentially do that - // later. - HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */); - if (hr != NULL) { - first = hr->hrm_index(); - } - } else { - // We can't allocate humongous regions spanning more than one region while - // cleanupComplete() is running, since some of the regions we find to be - // empty might not yet be added to the free list. It is not straightforward - // to know in which list they are on so that we can remove them. We only - // need to do this if we need to allocate more than one region to satisfy the - // current humongous allocation request. If we are only allocating one region - // we use the one-region region allocation code (see above), that already - // potentially waits for regions from the secondary free list. - wait_while_free_regions_coming(); - append_secondary_free_list_if_not_empty_with_lock(); - - // Policy: Try only empty regions (i.e. already committed first). Maybe we - // are lucky enough to find some. - first = _hrm.find_contiguous_only_empty(obj_regions); - if (first != G1_NO_HRM_INDEX) { - _hrm.allocate_free_regions_starting_at(first, obj_regions); - } - } - - if (first == G1_NO_HRM_INDEX) { - // Policy: We could not find enough regions for the humongous object in the - // free list. Look through the heap to find a mix of free and uncommitted regions. - // If so, try expansion. - first = _hrm.find_contiguous_empty_or_unavailable(obj_regions); - if (first != G1_NO_HRM_INDEX) { - // We found something. Make sure these regions are committed, i.e. expand - // the heap. Alternatively we could do a defragmentation GC. - log_debug(gc, ergo, heap)("Attempt heap expansion (humongous allocation request failed). Allocation request: " SIZE_FORMAT "B", - word_size * HeapWordSize); - - - _hrm.expand_at(first, obj_regions); - g1_policy()->record_new_heap_size(num_regions()); - -#ifdef ASSERT - for (uint i = first; i < first + obj_regions; ++i) { - HeapRegion* hr = region_at(i); - assert(hr->is_free(), "sanity"); - assert(hr->is_empty(), "sanity"); - assert(is_on_master_free_list(hr), "sanity"); - } -#endif - _hrm.allocate_free_regions_starting_at(first, obj_regions); - } else { - // Policy: Potentially trigger a defragmentation GC. - } - } - - HeapWord* result = NULL; - if (first != G1_NO_HRM_INDEX) { - result = humongous_obj_allocate_initialize_regions(first, obj_regions, - word_size, context); - assert(result != NULL, "it should always return a valid result"); - - // A successful humongous object allocation changes the used space - // information of the old generation so we need to recalculate the - // sizes and update the jstat counters here. - g1mm()->update_sizes(); - } - - verify_region_sets_optional(); - - return result; -} - -HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) { - assert_heap_not_locked_and_not_at_safepoint(); - assert(!is_humongous(word_size), "we do not allow humongous TLABs"); - - uint dummy_gc_count_before; - uint dummy_gclocker_retry_count = 0; - return attempt_allocation(word_size, &dummy_gc_count_before, &dummy_gclocker_retry_count); -} - -HeapWord* -G1CollectedHeap::mem_allocate(size_t word_size, - bool* gc_overhead_limit_was_exceeded) { - assert_heap_not_locked_and_not_at_safepoint(); - - // Loop until the allocation is satisfied, or unsatisfied after GC. - for (uint try_count = 1, gclocker_retry_count = 0; /* we'll return */; try_count += 1) { - uint gc_count_before; - - HeapWord* result = NULL; - if (!is_humongous(word_size)) { - result = attempt_allocation(word_size, &gc_count_before, &gclocker_retry_count); - } else { - result = attempt_allocation_humongous(word_size, &gc_count_before, &gclocker_retry_count); - } - if (result != NULL) { - return result; - } - - // Create the garbage collection operation... - VM_G1CollectForAllocation op(gc_count_before, word_size); - op.set_allocation_context(AllocationContext::current()); - - // ...and get the VM thread to execute it. - VMThread::execute(&op); - - if (op.prologue_succeeded() && op.pause_succeeded()) { - // If the operation was successful we'll return the result even - // if it is NULL. If the allocation attempt failed immediately - // after a Full GC, it's unlikely we'll be able to allocate now. - HeapWord* result = op.result(); - if (result != NULL && !is_humongous(word_size)) { - // Allocations that take place on VM operations do not do any - // card dirtying and we have to do it here. We only have to do - // this for non-humongous allocations, though. - dirty_young_block(result, word_size); - } - return result; - } else { - if (gclocker_retry_count > GCLockerRetryAllocationCount) { - return NULL; - } - assert(op.result() == NULL, - "the result should be NULL if the VM op did not succeed"); - } - - // Give a warning if we seem to be looping forever. - if ((QueuedAllocationWarningCount > 0) && - (try_count % QueuedAllocationWarningCount == 0)) { - warning("G1CollectedHeap::mem_allocate retries %d times", try_count); - } - } - - ShouldNotReachHere(); - return NULL; -} - -HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size, - AllocationContext_t context, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret) { - // Make sure you read the note in attempt_allocation_humongous(). - - assert_heap_not_locked_and_not_at_safepoint(); - assert(!is_humongous(word_size), "attempt_allocation_slow() should not " - "be called for humongous allocation requests"); - - // We should only get here after the first-level allocation attempt - // (attempt_allocation()) failed to allocate. - - // We will loop until a) we manage to successfully perform the - // allocation or b) we successfully schedule a collection which - // fails to perform the allocation. b) is the only case when we'll - // return NULL. - HeapWord* result = NULL; - for (int try_count = 1; /* we'll return */; try_count += 1) { - bool should_try_gc; - uint gc_count_before; - - { - MutexLockerEx x(Heap_lock); - result = _allocator->attempt_allocation_locked(word_size, context); - if (result != NULL) { - return result; - } - - if (GCLocker::is_active_and_needs_gc()) { - if (g1_policy()->can_expand_young_list()) { - // No need for an ergo verbose message here, - // can_expand_young_list() does this when it returns true. - result = _allocator->attempt_allocation_force(word_size, context); - if (result != NULL) { - return result; - } - } - should_try_gc = false; - } else { - // The GCLocker may not be active but the GCLocker initiated - // GC may not yet have been performed (GCLocker::needs_gc() - // returns true). In this case we do not try this GC and - // wait until the GCLocker initiated GC is performed, and - // then retry the allocation. - if (GCLocker::needs_gc()) { - should_try_gc = false; - } else { - // Read the GC count while still holding the Heap_lock. - gc_count_before = total_collections(); - should_try_gc = true; - } - } - } - - if (should_try_gc) { - bool succeeded; - result = do_collection_pause(word_size, gc_count_before, &succeeded, - GCCause::_g1_inc_collection_pause); - if (result != NULL) { - assert(succeeded, "only way to get back a non-NULL result"); - return result; - } - - if (succeeded) { - // If we get here we successfully scheduled a collection which - // failed to allocate. No point in trying to allocate - // further. We'll just return NULL. - MutexLockerEx x(Heap_lock); - *gc_count_before_ret = total_collections(); - return NULL; - } - } else { - if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) { - MutexLockerEx x(Heap_lock); - *gc_count_before_ret = total_collections(); - return NULL; - } - // The GCLocker is either active or the GCLocker initiated - // GC has not yet been performed. Stall until it is and - // then retry the allocation. - GCLocker::stall_until_clear(); - (*gclocker_retry_count_ret) += 1; - } - - // We can reach here if we were unsuccessful in scheduling a - // collection (because another thread beat us to it) or if we were - // stalled due to the GC locker. In either can we should retry the - // allocation attempt in case another thread successfully - // performed a collection and reclaimed enough space. We do the - // first attempt (without holding the Heap_lock) here and the - // follow-on attempt will be at the start of the next loop - // iteration (after taking the Heap_lock). - result = _allocator->attempt_allocation(word_size, context); - if (result != NULL) { - return result; - } - - // Give a warning if we seem to be looping forever. - if ((QueuedAllocationWarningCount > 0) && - (try_count % QueuedAllocationWarningCount == 0)) { - warning("G1CollectedHeap::attempt_allocation_slow() " - "retries %d times", try_count); - } - } - - ShouldNotReachHere(); - return NULL; +HeapWord* G1CollectedHeap::mem_allocate(size_t word_size, + bool* gc_overhead_limit_was_exceeded) { + return _allocator->mem_allocate(word_size, gc_overhead_limit_was_exceeded); } void G1CollectedHeap::begin_archive_alloc_range() { @@ -883,29 +509,6 @@ } } -inline HeapWord* G1CollectedHeap::attempt_allocation(size_t word_size, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret) { - assert_heap_not_locked_and_not_at_safepoint(); - assert(!is_humongous(word_size), "attempt_allocation() should not " - "be called for humongous allocation requests"); - - AllocationContext_t context = AllocationContext::current(); - HeapWord* result = _allocator->attempt_allocation(word_size, context); - - if (result == NULL) { - result = attempt_allocation_slow(word_size, - context, - gc_count_before_ret, - gclocker_retry_count_ret); - } - assert_heap_not_locked(); - if (result != NULL) { - dirty_young_block(result, word_size); - } - return result; -} - void G1CollectedHeap::dealloc_archive_regions(MemRegion* ranges, size_t count) { assert(!is_init_completed(), "Expect to be called at JVM init time"); assert(ranges != NULL, "MemRegion array NULL"); @@ -978,146 +581,6 @@ decrease_used(size_used); } -HeapWord* G1CollectedHeap::attempt_allocation_humongous(size_t word_size, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret) { - // The structure of this method has a lot of similarities to - // attempt_allocation_slow(). The reason these two were not merged - // into a single one is that such a method would require several "if - // allocation is not humongous do this, otherwise do that" - // conditional paths which would obscure its flow. In fact, an early - // version of this code did use a unified method which was harder to - // follow and, as a result, it had subtle bugs that were hard to - // track down. So keeping these two methods separate allows each to - // be more readable. It will be good to keep these two in sync as - // much as possible. - - assert_heap_not_locked_and_not_at_safepoint(); - assert(is_humongous(word_size), "attempt_allocation_humongous() " - "should only be called for humongous allocations"); - - // Humongous objects can exhaust the heap quickly, so we should check if we - // need to start a marking cycle at each humongous object allocation. We do - // the check before we do the actual allocation. The reason for doing it - // before the allocation is that we avoid having to keep track of the newly - // allocated memory while we do a GC. - if (g1_policy()->need_to_start_conc_mark("concurrent humongous allocation", - word_size)) { - collect(GCCause::_g1_humongous_allocation); - } - - // We will loop until a) we manage to successfully perform the - // allocation or b) we successfully schedule a collection which - // fails to perform the allocation. b) is the only case when we'll - // return NULL. - HeapWord* result = NULL; - for (int try_count = 1; /* we'll return */; try_count += 1) { - bool should_try_gc; - uint gc_count_before; - - { - MutexLockerEx x(Heap_lock); - - // Given that humongous objects are not allocated in young - // regions, we'll first try to do the allocation without doing a - // collection hoping that there's enough space in the heap. - result = humongous_obj_allocate(word_size, AllocationContext::current()); - if (result != NULL) { - size_t size_in_regions = humongous_obj_size_in_regions(word_size); - g1_policy()->add_bytes_allocated_in_old_since_last_gc(size_in_regions * HeapRegion::GrainBytes); - return result; - } - - if (GCLocker::is_active_and_needs_gc()) { - should_try_gc = false; - } else { - // The GCLocker may not be active but the GCLocker initiated - // GC may not yet have been performed (GCLocker::needs_gc() - // returns true). In this case we do not try this GC and - // wait until the GCLocker initiated GC is performed, and - // then retry the allocation. - if (GCLocker::needs_gc()) { - should_try_gc = false; - } else { - // Read the GC count while still holding the Heap_lock. - gc_count_before = total_collections(); - should_try_gc = true; - } - } - } - - if (should_try_gc) { - // If we failed to allocate the humongous object, we should try to - // do a collection pause (if we're allowed) in case it reclaims - // enough space for the allocation to succeed after the pause. - - bool succeeded; - result = do_collection_pause(word_size, gc_count_before, &succeeded, - GCCause::_g1_humongous_allocation); - if (result != NULL) { - assert(succeeded, "only way to get back a non-NULL result"); - return result; - } - - if (succeeded) { - // If we get here we successfully scheduled a collection which - // failed to allocate. No point in trying to allocate - // further. We'll just return NULL. - MutexLockerEx x(Heap_lock); - *gc_count_before_ret = total_collections(); - return NULL; - } - } else { - if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) { - MutexLockerEx x(Heap_lock); - *gc_count_before_ret = total_collections(); - return NULL; - } - // The GCLocker is either active or the GCLocker initiated - // GC has not yet been performed. Stall until it is and - // then retry the allocation. - GCLocker::stall_until_clear(); - (*gclocker_retry_count_ret) += 1; - } - - // We can reach here if we were unsuccessful in scheduling a - // collection (because another thread beat us to it) or if we were - // stalled due to the GC locker. In either can we should retry the - // allocation attempt in case another thread successfully - // performed a collection and reclaimed enough space. Give a - // warning if we seem to be looping forever. - - if ((QueuedAllocationWarningCount > 0) && - (try_count % QueuedAllocationWarningCount == 0)) { - warning("G1CollectedHeap::attempt_allocation_humongous() " - "retries %d times", try_count); - } - } - - ShouldNotReachHere(); - return NULL; -} - -HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size, - AllocationContext_t context, - bool expect_null_mutator_alloc_region) { - assert_at_safepoint(true /* should_be_vm_thread */); - assert(!_allocator->has_mutator_alloc_region(context) || !expect_null_mutator_alloc_region, - "the current alloc region was unexpectedly found to be non-NULL"); - - if (!is_humongous(word_size)) { - return _allocator->attempt_allocation_locked(word_size, context); - } else { - HeapWord* result = humongous_obj_allocate(word_size, context); - if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) { - collector_state()->set_initiate_conc_mark_if_possible(true); - } - return result; - } - - ShouldNotReachHere(); -} - class PostMCRemSetClearClosure: public HeapRegionClosure { G1CollectedHeap* _g1h; ModRefBarrierSet* _mr_bs; @@ -1230,7 +693,7 @@ size_t metadata_prev_used = MetaspaceAux::used_bytes(); - verify_region_sets_optional(); + _verifier->verify_region_sets_optional(); const bool do_clear_all_soft_refs = clear_all_soft_refs || collector_policy()->should_clear_all_soft_refs(); @@ -1271,9 +734,9 @@ assert(used() == recalculate_used(), "Should be equal"); - verify_before_gc(); - - check_bitmaps("Full GC Start"); + _verifier->verify_before_gc(); + + _verifier->check_bitmaps("Full GC Start"); pre_full_gc_dump(gc_timer); #if defined(COMPILER2) || INCLUDE_JVMCI @@ -1408,9 +871,9 @@ increment_old_marking_cycles_completed(false /* concurrent */); _hrm.verify_optional(); - verify_region_sets_optional(); - - verify_after_gc(); + _verifier->verify_region_sets_optional(); + + _verifier->verify_after_gc(); // Clear the previous marking bitmap, if needed for bitmap verification. // Note we cannot do this when we clear the next marking bitmap in @@ -1422,7 +885,7 @@ if (G1VerifyBitmaps) { ((CMBitMap*) concurrent_mark()->prevMarkBitMap())->clearAll(); } - check_bitmaps("Full GC End"); + _verifier->check_bitmaps("Full GC End"); // Start a new incremental collection set for the next pause assert(g1_policy()->collection_set() == NULL, "must be"); @@ -1550,9 +1013,9 @@ *gc_succeeded = true; // Let's attempt the allocation first. HeapWord* result = - attempt_allocation_at_safepoint(word_size, - context, - expect_null_mutator_alloc_region); + _allocator->attempt_allocation_at_safepoint(word_size, + context, + expect_null_mutator_alloc_region); if (result != NULL) { assert(*gc_succeeded, "sanity"); return result; @@ -1639,7 +1102,7 @@ HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size, AllocationContext_t context) { assert_at_safepoint(true /* should_be_vm_thread */); - verify_region_sets_optional(); + _verifier->verify_region_sets_optional(); size_t expand_bytes = MAX2(word_size * HeapWordSize, MinHeapDeltaBytes); log_debug(gc, ergo, heap)("Attempt heap expansion (allocation request failed). Allocation request: " SIZE_FORMAT "B", @@ -1648,10 +1111,10 @@ if (expand(expand_bytes)) { _hrm.verify_optional(); - verify_region_sets_optional(); - return attempt_allocation_at_safepoint(word_size, - context, - false /* expect_null_mutator_alloc_region */); + _verifier->verify_region_sets_optional(); + return _allocator->attempt_allocation_at_safepoint(word_size, + context, + false /* expect_null_mutator_alloc_region */); } return NULL; } @@ -1717,7 +1180,7 @@ } void G1CollectedHeap::shrink(size_t shrink_bytes) { - verify_region_sets_optional(); + _verifier->verify_region_sets_optional(); // We should only reach here at the end of a Full GC which means we // should not not be holding to any GC alloc regions. The method @@ -1732,7 +1195,7 @@ rebuild_region_sets(true /* free_list_only */); _hrm.verify_optional(); - verify_region_sets_optional(); + _verifier->verify_region_sets_optional(); } // Public methods. @@ -1779,6 +1242,7 @@ /* are_ConcurrentGC_threads */false); _workers->initialize_workers(); + _verifier = new G1HeapVerifier(this); _allocator = G1Allocator::create_allocator(this); _humongous_object_threshold_in_words = humongous_threshold_for(HeapRegion::GrainWords); @@ -2228,33 +1692,6 @@ } } -#ifndef PRODUCT -void G1CollectedHeap::allocate_dummy_regions() { - // Let's fill up most of the region - size_t word_size = HeapRegion::GrainWords - 1024; - // And as a result the region we'll allocate will be humongous. - guarantee(is_humongous(word_size), "sanity"); - - // _filler_array_max_size is set to humongous object threshold - // but temporarily change it to use CollectedHeap::fill_with_object(). - SizeTFlagSetting fs(_filler_array_max_size, word_size); - - for (uintx i = 0; i < G1DummyRegionsPerGC; ++i) { - // Let's use the existing mechanism for the allocation - HeapWord* dummy_obj = humongous_obj_allocate(word_size, - AllocationContext::system()); - if (dummy_obj != NULL) { - MemRegion mr(dummy_obj, word_size); - CollectedHeap::fill_with_object(mr); - } else { - // If we can't allocate once, we probably cannot allocate - // again. Let's get out of the loop. - break; - } - } -} -#endif // !PRODUCT - void G1CollectedHeap::increment_old_marking_cycles_started() { assert(_old_marking_cycles_started == _old_marking_cycles_completed || _old_marking_cycles_started == _old_marking_cycles_completed + 1, @@ -2667,452 +2104,7 @@ } void G1CollectedHeap::prepare_for_verify() { - if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { - ensure_parsability(false); - } - g1_rem_set()->prepare_for_verify(); -} - -bool G1CollectedHeap::allocated_since_marking(oop obj, HeapRegion* hr, - VerifyOption vo) { - switch (vo) { - case VerifyOption_G1UsePrevMarking: - return hr->obj_allocated_since_prev_marking(obj); - case VerifyOption_G1UseNextMarking: - return hr->obj_allocated_since_next_marking(obj); - case VerifyOption_G1UseMarkWord: - return false; - default: - ShouldNotReachHere(); - } - return false; // keep some compilers happy -} - -HeapWord* G1CollectedHeap::top_at_mark_start(HeapRegion* hr, VerifyOption vo) { - switch (vo) { - case VerifyOption_G1UsePrevMarking: return hr->prev_top_at_mark_start(); - case VerifyOption_G1UseNextMarking: return hr->next_top_at_mark_start(); - case VerifyOption_G1UseMarkWord: return NULL; - default: ShouldNotReachHere(); - } - return NULL; // keep some compilers happy -} - -bool G1CollectedHeap::is_marked(oop obj, VerifyOption vo) { - switch (vo) { - case VerifyOption_G1UsePrevMarking: return isMarkedPrev(obj); - case VerifyOption_G1UseNextMarking: return isMarkedNext(obj); - case VerifyOption_G1UseMarkWord: return obj->is_gc_marked(); - default: ShouldNotReachHere(); - } - return false; // keep some compilers happy -} - -const char* G1CollectedHeap::top_at_mark_start_str(VerifyOption vo) { - switch (vo) { - case VerifyOption_G1UsePrevMarking: return "PTAMS"; - case VerifyOption_G1UseNextMarking: return "NTAMS"; - case VerifyOption_G1UseMarkWord: return "NONE"; - default: ShouldNotReachHere(); - } - return NULL; // keep some compilers happy -} - -class VerifyRootsClosure: public OopClosure { -private: - G1CollectedHeap* _g1h; - VerifyOption _vo; - bool _failures; -public: - // _vo == UsePrevMarking -> use "prev" marking information, - // _vo == UseNextMarking -> use "next" marking information, - // _vo == UseMarkWord -> use mark word from object header. - VerifyRootsClosure(VerifyOption vo) : - _g1h(G1CollectedHeap::heap()), - _vo(vo), - _failures(false) { } - - bool failures() { return _failures; } - - template void do_oop_nv(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (_g1h->is_obj_dead_cond(obj, _vo)) { - LogHandle(gc, verify) log; - log.info("Root location " PTR_FORMAT " points to dead obj " PTR_FORMAT, p2i(p), p2i(obj)); - if (_vo == VerifyOption_G1UseMarkWord) { - log.info(" Mark word: " PTR_FORMAT, p2i(obj->mark())); - } - ResourceMark rm; - obj->print_on(log.info_stream()); - _failures = true; - } - } - } - - void do_oop(oop* p) { do_oop_nv(p); } - void do_oop(narrowOop* p) { do_oop_nv(p); } -}; - -class G1VerifyCodeRootOopClosure: public OopClosure { - G1CollectedHeap* _g1h; - OopClosure* _root_cl; - nmethod* _nm; - VerifyOption _vo; - bool _failures; - - template void do_oop_work(T* p) { - // First verify that this root is live - _root_cl->do_oop(p); - - if (!G1VerifyHeapRegionCodeRoots) { - // We're not verifying the code roots attached to heap region. - return; - } - - // Don't check the code roots during marking verification in a full GC - if (_vo == VerifyOption_G1UseMarkWord) { - return; - } - - // Now verify that the current nmethod (which contains p) is - // in the code root list of the heap region containing the - // object referenced by p. - - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - - // Now fetch the region containing the object - HeapRegion* hr = _g1h->heap_region_containing(obj); - HeapRegionRemSet* hrrs = hr->rem_set(); - // Verify that the strong code root list for this region - // contains the nmethod - if (!hrrs->strong_code_roots_list_contains(_nm)) { - log_info(gc, verify)("Code root location " PTR_FORMAT " " - "from nmethod " PTR_FORMAT " not in strong " - "code roots for region [" PTR_FORMAT "," PTR_FORMAT ")", - p2i(p), p2i(_nm), p2i(hr->bottom()), p2i(hr->end())); - _failures = true; - } - } - } - -public: - G1VerifyCodeRootOopClosure(G1CollectedHeap* g1h, OopClosure* root_cl, VerifyOption vo): - _g1h(g1h), _root_cl(root_cl), _vo(vo), _nm(NULL), _failures(false) {} - - void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { do_oop_work(p); } - - void set_nmethod(nmethod* nm) { _nm = nm; } - bool failures() { return _failures; } -}; - -class G1VerifyCodeRootBlobClosure: public CodeBlobClosure { - G1VerifyCodeRootOopClosure* _oop_cl; - -public: - G1VerifyCodeRootBlobClosure(G1VerifyCodeRootOopClosure* oop_cl): - _oop_cl(oop_cl) {} - - void do_code_blob(CodeBlob* cb) { - nmethod* nm = cb->as_nmethod_or_null(); - if (nm != NULL) { - _oop_cl->set_nmethod(nm); - nm->oops_do(_oop_cl); - } - } -}; - -class YoungRefCounterClosure : public OopClosure { - G1CollectedHeap* _g1h; - int _count; - public: - YoungRefCounterClosure(G1CollectedHeap* g1h) : _g1h(g1h), _count(0) {} - void do_oop(oop* p) { if (_g1h->is_in_young(*p)) { _count++; } } - void do_oop(narrowOop* p) { ShouldNotReachHere(); } - - int count() { return _count; } - void reset_count() { _count = 0; }; -}; - -class VerifyKlassClosure: public KlassClosure { - YoungRefCounterClosure _young_ref_counter_closure; - OopClosure *_oop_closure; - public: - VerifyKlassClosure(G1CollectedHeap* g1h, OopClosure* cl) : _young_ref_counter_closure(g1h), _oop_closure(cl) {} - void do_klass(Klass* k) { - k->oops_do(_oop_closure); - - _young_ref_counter_closure.reset_count(); - k->oops_do(&_young_ref_counter_closure); - if (_young_ref_counter_closure.count() > 0) { - guarantee(k->has_modified_oops(), "Klass " PTR_FORMAT ", has young refs but is not dirty.", p2i(k)); - } - } -}; - -class VerifyLivenessOopClosure: public OopClosure { - G1CollectedHeap* _g1h; - VerifyOption _vo; -public: - VerifyLivenessOopClosure(G1CollectedHeap* g1h, VerifyOption vo): - _g1h(g1h), _vo(vo) - { } - void do_oop(narrowOop *p) { do_oop_work(p); } - void do_oop( oop *p) { do_oop_work(p); } - - template void do_oop_work(T *p) { - oop obj = oopDesc::load_decode_heap_oop(p); - guarantee(obj == NULL || !_g1h->is_obj_dead_cond(obj, _vo), - "Dead object referenced by a not dead object"); - } -}; - -class VerifyObjsInRegionClosure: public ObjectClosure { -private: - G1CollectedHeap* _g1h; - size_t _live_bytes; - HeapRegion *_hr; - VerifyOption _vo; -public: - // _vo == UsePrevMarking -> use "prev" marking information, - // _vo == UseNextMarking -> use "next" marking information, - // _vo == UseMarkWord -> use mark word from object header. - VerifyObjsInRegionClosure(HeapRegion *hr, VerifyOption vo) - : _live_bytes(0), _hr(hr), _vo(vo) { - _g1h = G1CollectedHeap::heap(); - } - void do_object(oop o) { - VerifyLivenessOopClosure isLive(_g1h, _vo); - assert(o != NULL, "Huh?"); - if (!_g1h->is_obj_dead_cond(o, _vo)) { - // If the object is alive according to the mark word, - // then verify that the marking information agrees. - // Note we can't verify the contra-positive of the - // above: if the object is dead (according to the mark - // word), it may not be marked, or may have been marked - // but has since became dead, or may have been allocated - // since the last marking. - if (_vo == VerifyOption_G1UseMarkWord) { - guarantee(!_g1h->is_obj_dead(o), "mark word and concurrent mark mismatch"); - } - - o->oop_iterate_no_header(&isLive); - if (!_hr->obj_allocated_since_prev_marking(o)) { - size_t obj_size = o->size(); // Make sure we don't overflow - _live_bytes += (obj_size * HeapWordSize); - } - } - } - size_t live_bytes() { return _live_bytes; } -}; - -class VerifyArchiveOopClosure: public OopClosure { -public: - VerifyArchiveOopClosure(HeapRegion *hr) { } - void do_oop(narrowOop *p) { do_oop_work(p); } - void do_oop( oop *p) { do_oop_work(p); } - - template void do_oop_work(T *p) { - oop obj = oopDesc::load_decode_heap_oop(p); - guarantee(obj == NULL || G1MarkSweep::in_archive_range(obj), - "Archive object at " PTR_FORMAT " references a non-archive object at " PTR_FORMAT, - p2i(p), p2i(obj)); - } -}; - -class VerifyArchiveRegionClosure: public ObjectClosure { -public: - VerifyArchiveRegionClosure(HeapRegion *hr) { } - // Verify that all object pointers are to archive regions. - void do_object(oop o) { - VerifyArchiveOopClosure checkOop(NULL); - assert(o != NULL, "Should not be here for NULL oops"); - o->oop_iterate_no_header(&checkOop); - } -}; - -class VerifyRegionClosure: public HeapRegionClosure { -private: - bool _par; - VerifyOption _vo; - bool _failures; -public: - // _vo == UsePrevMarking -> use "prev" marking information, - // _vo == UseNextMarking -> use "next" marking information, - // _vo == UseMarkWord -> use mark word from object header. - VerifyRegionClosure(bool par, VerifyOption vo) - : _par(par), - _vo(vo), - _failures(false) {} - - bool failures() { - return _failures; - } - - bool doHeapRegion(HeapRegion* r) { - // For archive regions, verify there are no heap pointers to - // non-pinned regions. For all others, verify liveness info. - if (r->is_archive()) { - VerifyArchiveRegionClosure verify_oop_pointers(r); - r->object_iterate(&verify_oop_pointers); - return true; - } - if (!r->is_continues_humongous()) { - bool failures = false; - r->verify(_vo, &failures); - if (failures) { - _failures = true; - } else if (!r->is_starts_humongous()) { - VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo); - r->object_iterate(¬_dead_yet_cl); - if (_vo != VerifyOption_G1UseNextMarking) { - if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) { - log_info(gc, verify)("[" PTR_FORMAT "," PTR_FORMAT "] max_live_bytes " SIZE_FORMAT " < calculated " SIZE_FORMAT, - p2i(r->bottom()), p2i(r->end()), r->max_live_bytes(), not_dead_yet_cl.live_bytes()); - _failures = true; - } - } else { - // When vo == UseNextMarking we cannot currently do a sanity - // check on the live bytes as the calculation has not been - // finalized yet. - } - } - } - return false; // stop the region iteration if we hit a failure - } -}; - -// This is the task used for parallel verification of the heap regions - -class G1ParVerifyTask: public AbstractGangTask { -private: - G1CollectedHeap* _g1h; - VerifyOption _vo; - bool _failures; - HeapRegionClaimer _hrclaimer; - -public: - // _vo == UsePrevMarking -> use "prev" marking information, - // _vo == UseNextMarking -> use "next" marking information, - // _vo == UseMarkWord -> use mark word from object header. - G1ParVerifyTask(G1CollectedHeap* g1h, VerifyOption vo) : - AbstractGangTask("Parallel verify task"), - _g1h(g1h), - _vo(vo), - _failures(false), - _hrclaimer(g1h->workers()->active_workers()) {} - - bool failures() { - return _failures; - } - - void work(uint worker_id) { - HandleMark hm; - VerifyRegionClosure blk(true, _vo); - _g1h->heap_region_par_iterate(&blk, worker_id, &_hrclaimer); - if (blk.failures()) { - _failures = true; - } - } -}; - -void G1CollectedHeap::verify(VerifyOption vo) { - if (!SafepointSynchronize::is_at_safepoint()) { - log_info(gc, verify)("Skipping verification. Not at safepoint."); - } - - assert(Thread::current()->is_VM_thread(), - "Expected to be executed serially by the VM thread at this point"); - - log_debug(gc, verify)("Roots"); - VerifyRootsClosure rootsCl(vo); - VerifyKlassClosure klassCl(this, &rootsCl); - CLDToKlassAndOopClosure cldCl(&klassCl, &rootsCl, false); - - // We apply the relevant closures to all the oops in the - // system dictionary, class loader data graph, the string table - // and the nmethods in the code cache. - G1VerifyCodeRootOopClosure codeRootsCl(this, &rootsCl, vo); - G1VerifyCodeRootBlobClosure blobsCl(&codeRootsCl); - - { - G1RootProcessor root_processor(this, 1); - root_processor.process_all_roots(&rootsCl, - &cldCl, - &blobsCl); - } - - bool failures = rootsCl.failures() || codeRootsCl.failures(); - - if (vo != VerifyOption_G1UseMarkWord) { - // If we're verifying during a full GC then the region sets - // will have been torn down at the start of the GC. Therefore - // verifying the region sets will fail. So we only verify - // the region sets when not in a full GC. - log_debug(gc, verify)("HeapRegionSets"); - verify_region_sets(); - } - - log_debug(gc, verify)("HeapRegions"); - if (GCParallelVerificationEnabled && ParallelGCThreads > 1) { - - G1ParVerifyTask task(this, vo); - workers()->run_task(&task); - if (task.failures()) { - failures = true; - } - - } else { - VerifyRegionClosure blk(false, vo); - heap_region_iterate(&blk); - if (blk.failures()) { - failures = true; - } - } - - if (G1StringDedup::is_enabled()) { - log_debug(gc, verify)("StrDedup"); - G1StringDedup::verify(); - } - - if (failures) { - log_info(gc, verify)("Heap after failed verification:"); - // It helps to have the per-region information in the output to - // help us track down what went wrong. This is why we call - // print_extended_on() instead of print_on(). - LogHandle(gc, verify) log; - ResourceMark rm; - print_extended_on(log.info_stream()); - } - guarantee(!failures, "there should not have been any failures"); -} - -double G1CollectedHeap::verify(bool guard, const char* msg) { - double verify_time_ms = 0.0; - - if (guard && total_collections() >= VerifyGCStartAt) { - double verify_start = os::elapsedTime(); - HandleMark hm; // Discard invalid handles created during verification - prepare_for_verify(); - Universe::verify(VerifyOption_G1UsePrevMarking, msg); - verify_time_ms = (os::elapsedTime() - verify_start) * 1000; - } - - return verify_time_ms; -} - -void G1CollectedHeap::verify_before_gc() { - double verify_time_ms = verify(VerifyBeforeGC, "Before GC"); - g1_policy()->phase_times()->record_verify_before_time_ms(verify_time_ms); -} - -void G1CollectedHeap::verify_after_gc() { - double verify_time_ms = verify(VerifyAfterGC, "After GC"); - g1_policy()->phase_times()->record_verify_after_time_ms(verify_time_ms); + _verifier->prepare_for_verify(); } class PrintRegionClosure: public HeapRegionClosure { @@ -3657,8 +2649,8 @@ print_heap_before_gc(); trace_heap_before_gc(_gc_tracer_stw); - verify_region_sets_optional(); - verify_dirty_young_regions(); + _verifier->verify_region_sets_optional(); + _verifier->verify_dirty_young_regions(); // This call will decide whether this pause is an initial-mark // pause. If it is, during_initial_mark_pause() will return true @@ -3741,9 +2733,9 @@ heap_region_iterate(&v_cl); } - verify_before_gc(); - - check_bitmaps("GC Start"); + _verifier->verify_before_gc(); + + _verifier->check_bitmaps("GC Start"); #if defined(COMPILER2) || INCLUDE_JVMCI DerivedPointerTable::clear(); @@ -3801,7 +2793,7 @@ register_humongous_regions_with_cset(); - assert(check_cset_fast_test(), "Inconsistency in the InCSetState table."); + assert(_verifier->check_cset_fast_test(), "Inconsistency in the InCSetState table."); _cm->note_start_of_gc(); // We call this after finalize_cset() to @@ -3886,7 +2878,17 @@ // the current thread has completed its logging output. } - allocate_dummy_regions(); +#ifdef ASSERT + { + // Let's fill up most of the region + size_t word_size = HeapRegion::GrainWords - 1024; + + // _filler_array_max_size is set to humongous object threshold + // but temporarily change it to use CollectedHeap::fill_with_object(). + SizeTFlagSetting fs(_filler_array_max_size, word_size); + _allocator->allocate_dummy_regions(word_size); + } +#endif _allocator->init_mutator_alloc_region(); @@ -3951,8 +2953,8 @@ heap_region_iterate(&v_cl); } - verify_after_gc(); - check_bitmaps("GC End"); + _verifier->verify_after_gc(); + _verifier->check_bitmaps("GC End"); assert(!ref_processor_stw()->discovery_enabled(), "Postcondition"); ref_processor_stw()->verify_no_references_recorded(); @@ -3976,7 +2978,7 @@ // logging output either. _hrm.verify_optional(); - verify_region_sets_optional(); + _verifier->verify_region_sets_optional(); TASKQUEUE_STATS_ONLY(print_taskqueue_stats()); TASKQUEUE_STATS_ONLY(reset_taskqueue_stats()); @@ -5245,197 +4247,6 @@ } }; -#ifndef PRODUCT -class G1VerifyCardTableCleanup: public HeapRegionClosure { - G1CollectedHeap* _g1h; - G1SATBCardTableModRefBS* _ct_bs; -public: - G1VerifyCardTableCleanup(G1CollectedHeap* g1h, G1SATBCardTableModRefBS* ct_bs) - : _g1h(g1h), _ct_bs(ct_bs) { } - virtual bool doHeapRegion(HeapRegion* r) { - if (r->is_survivor()) { - _g1h->verify_dirty_region(r); - } else { - _g1h->verify_not_dirty_region(r); - } - return false; - } -}; - -void G1CollectedHeap::verify_not_dirty_region(HeapRegion* hr) { - // All of the region should be clean. - G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); - MemRegion mr(hr->bottom(), hr->end()); - ct_bs->verify_not_dirty_region(mr); -} - -void G1CollectedHeap::verify_dirty_region(HeapRegion* hr) { - // We cannot guarantee that [bottom(),end()] is dirty. Threads - // dirty allocated blocks as they allocate them. The thread that - // retires each region and replaces it with a new one will do a - // maximal allocation to fill in [pre_dummy_top(),end()] but will - // not dirty that area (one less thing to have to do while holding - // a lock). So we can only verify that [bottom(),pre_dummy_top()] - // is dirty. - G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); - MemRegion mr(hr->bottom(), hr->pre_dummy_top()); - if (hr->is_young()) { - ct_bs->verify_g1_young_region(mr); - } else { - ct_bs->verify_dirty_region(mr); - } -} - -void G1CollectedHeap::verify_dirty_young_list(HeapRegion* head) { - G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); - for (HeapRegion* hr = head; hr != NULL; hr = hr->get_next_young_region()) { - verify_dirty_region(hr); - } -} - -void G1CollectedHeap::verify_dirty_young_regions() { - verify_dirty_young_list(_young_list->first_region()); -} - -bool G1CollectedHeap::verify_no_bits_over_tams(const char* bitmap_name, CMBitMapRO* bitmap, - HeapWord* tams, HeapWord* end) { - guarantee(tams <= end, - "tams: " PTR_FORMAT " end: " PTR_FORMAT, p2i(tams), p2i(end)); - HeapWord* result = bitmap->getNextMarkedWordAddress(tams, end); - if (result < end) { - log_info(gc, verify)("## wrong marked address on %s bitmap: " PTR_FORMAT, bitmap_name, p2i(result)); - log_info(gc, verify)("## %s tams: " PTR_FORMAT " end: " PTR_FORMAT, bitmap_name, p2i(tams), p2i(end)); - return false; - } - return true; -} - -bool G1CollectedHeap::verify_bitmaps(const char* caller, HeapRegion* hr) { - CMBitMapRO* prev_bitmap = concurrent_mark()->prevMarkBitMap(); - CMBitMapRO* next_bitmap = (CMBitMapRO*) concurrent_mark()->nextMarkBitMap(); - - HeapWord* bottom = hr->bottom(); - HeapWord* ptams = hr->prev_top_at_mark_start(); - HeapWord* ntams = hr->next_top_at_mark_start(); - HeapWord* end = hr->end(); - - bool res_p = verify_no_bits_over_tams("prev", prev_bitmap, ptams, end); - - bool res_n = true; - // We reset mark_in_progress() before we reset _cmThread->in_progress() and in this window - // we do the clearing of the next bitmap concurrently. Thus, we can not verify the bitmap - // if we happen to be in that state. - if (collector_state()->mark_in_progress() || !_cmThread->in_progress()) { - res_n = verify_no_bits_over_tams("next", next_bitmap, ntams, end); - } - if (!res_p || !res_n) { - log_info(gc, verify)("#### Bitmap verification failed for " HR_FORMAT, HR_FORMAT_PARAMS(hr)); - log_info(gc, verify)("#### Caller: %s", caller); - return false; - } - return true; -} - -void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) { - if (!G1VerifyBitmaps) return; - - guarantee(verify_bitmaps(caller, hr), "bitmap verification"); -} - -class G1VerifyBitmapClosure : public HeapRegionClosure { -private: - const char* _caller; - G1CollectedHeap* _g1h; - bool _failures; - -public: - G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) : - _caller(caller), _g1h(g1h), _failures(false) { } - - bool failures() { return _failures; } - - virtual bool doHeapRegion(HeapRegion* hr) { - bool result = _g1h->verify_bitmaps(_caller, hr); - if (!result) { - _failures = true; - } - return false; - } -}; - -void G1CollectedHeap::check_bitmaps(const char* caller) { - if (!G1VerifyBitmaps) return; - - G1VerifyBitmapClosure cl(caller, this); - heap_region_iterate(&cl); - guarantee(!cl.failures(), "bitmap verification"); -} - -class G1CheckCSetFastTableClosure : public HeapRegionClosure { - private: - bool _failures; - public: - G1CheckCSetFastTableClosure() : HeapRegionClosure(), _failures(false) { } - - virtual bool doHeapRegion(HeapRegion* hr) { - uint i = hr->hrm_index(); - InCSetState cset_state = (InCSetState) G1CollectedHeap::heap()->_in_cset_fast_test.get_by_index(i); - if (hr->is_humongous()) { - if (hr->in_collection_set()) { - log_info(gc, verify)("## humongous region %u in CSet", i); - _failures = true; - return true; - } - if (cset_state.is_in_cset()) { - log_info(gc, verify)("## inconsistent cset state " CSETSTATE_FORMAT " for humongous region %u", cset_state.value(), i); - _failures = true; - return true; - } - if (hr->is_continues_humongous() && cset_state.is_humongous()) { - log_info(gc, verify)("## inconsistent cset state " CSETSTATE_FORMAT " for continues humongous region %u", cset_state.value(), i); - _failures = true; - return true; - } - } else { - if (cset_state.is_humongous()) { - log_info(gc, verify)("## inconsistent cset state " CSETSTATE_FORMAT " for non-humongous region %u", cset_state.value(), i); - _failures = true; - return true; - } - if (hr->in_collection_set() != cset_state.is_in_cset()) { - log_info(gc, verify)("## in CSet %d / cset state " CSETSTATE_FORMAT " inconsistency for region %u", - hr->in_collection_set(), cset_state.value(), i); - _failures = true; - return true; - } - if (cset_state.is_in_cset()) { - if (hr->is_young() != (cset_state.is_young())) { - log_info(gc, verify)("## is_young %d / cset state " CSETSTATE_FORMAT " inconsistency for region %u", - hr->is_young(), cset_state.value(), i); - _failures = true; - return true; - } - if (hr->is_old() != (cset_state.is_old())) { - log_info(gc, verify)("## is_old %d / cset state " CSETSTATE_FORMAT " inconsistency for region %u", - hr->is_old(), cset_state.value(), i); - _failures = true; - return true; - } - } - } - return false; - } - - bool failures() const { return _failures; } -}; - -bool G1CollectedHeap::check_cset_fast_test() { - G1CheckCSetFastTableClosure cl; - _hrm.iterate(&cl); - return !cl.failures(); -} -#endif // PRODUCT - class G1ParScrubRemSetTask: public AbstractGangTask { protected: G1RemSet* _g1rs; @@ -5473,10 +4284,7 @@ workers()->run_task(&cleanup_task); #ifndef PRODUCT - if (G1VerifyCTCleanup || VerifyAfterGC) { - G1VerifyCardTableCleanup cleanup_verifier(this, ct_bs); - heap_region_iterate(&cleanup_verifier); - } + _verifier->verify_card_table_cleanup(); #endif } @@ -5998,7 +4806,7 @@ if (new_alloc_region != NULL) { set_region_short_lived_locked(new_alloc_region); _hr_printer.alloc(new_alloc_region, young_list_full); - check_bitmaps("Mutator Region Allocation", new_alloc_region); + _verifier->check_bitmaps("Mutator Region Allocation", new_alloc_region); return new_alloc_region; } } @@ -6038,10 +4846,10 @@ new_alloc_region->record_timestamp(); if (is_survivor) { new_alloc_region->set_survivor(); - check_bitmaps("Survivor Region Allocation", new_alloc_region); + _verifier->check_bitmaps("Survivor Region Allocation", new_alloc_region); } else { new_alloc_region->set_old(); - check_bitmaps("Old Region Allocation", new_alloc_region); + _verifier->check_bitmaps("Old Region Allocation", new_alloc_region); } _hr_printer.alloc(new_alloc_region); bool during_im = collector_state()->during_initial_mark_pause(); @@ -6081,93 +4889,6 @@ return NULL; } -// Heap region set verification - -class VerifyRegionListsClosure : public HeapRegionClosure { -private: - HeapRegionSet* _old_set; - HeapRegionSet* _humongous_set; - HeapRegionManager* _hrm; - -public: - uint _old_count; - uint _humongous_count; - uint _free_count; - - VerifyRegionListsClosure(HeapRegionSet* old_set, - HeapRegionSet* humongous_set, - HeapRegionManager* hrm) : - _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm), - _old_count(), _humongous_count(), _free_count(){ } - - bool doHeapRegion(HeapRegion* hr) { - if (hr->is_young()) { - // TODO - } else if (hr->is_humongous()) { - assert(hr->containing_set() == _humongous_set, "Heap region %u is humongous but not in humongous set.", hr->hrm_index()); - _humongous_count++; - } else if (hr->is_empty()) { - assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index()); - _free_count++; - } else if (hr->is_old()) { - assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index()); - _old_count++; - } else { - // There are no other valid region types. Check for one invalid - // one we can identify: pinned without old or humongous set. - assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index()); - ShouldNotReachHere(); - } - return false; - } - - void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) { - guarantee(old_set->length() == _old_count, "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count); - guarantee(humongous_set->length() == _humongous_count, "Hum set count mismatch. Expected %u, actual %u.", humongous_set->length(), _humongous_count); - guarantee(free_list->num_free_regions() == _free_count, "Free list count mismatch. Expected %u, actual %u.", free_list->num_free_regions(), _free_count); - } -}; - -void G1CollectedHeap::verify_region_sets() { - assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); - - // First, check the explicit lists. - _hrm.verify(); - { - // Given that a concurrent operation might be adding regions to - // the secondary free list we have to take the lock before - // verifying it. - MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); - _secondary_free_list.verify_list(); - } - - // If a concurrent region freeing operation is in progress it will - // be difficult to correctly attributed any free regions we come - // across to the correct free list given that they might belong to - // one of several (free_list, secondary_free_list, any local lists, - // etc.). So, if that's the case we will skip the rest of the - // verification operation. Alternatively, waiting for the concurrent - // operation to complete will have a non-trivial effect on the GC's - // operation (no concurrent operation will last longer than the - // interval between two calls to verification) and it might hide - // any issues that we would like to catch during testing. - if (free_regions_coming()) { - return; - } - - // Make sure we append the secondary_free_list on the free_list so - // that all free regions we will come across can be safely - // attributed to the free_list. - append_secondary_free_list_if_not_empty_with_lock(); - - // Finally, make sure that the region accounting in the lists is - // consistent with what we see in the heap. - - VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_hrm); - heap_region_iterate(&cl); - cl.verify_counts(&_old_set, &_humongous_set, &_hrm); -} - // Optimized nmethod scanning class RegisterNMethodOopClosure: public OopClosure { diff --git a/src/share/vm/gc/g1/g1CollectedHeap.hpp b/src/share/vm/gc/g1/g1CollectedHeap.hpp --- a/src/share/vm/gc/g1/g1CollectedHeap.hpp +++ b/src/share/vm/gc/g1/g1CollectedHeap.hpp @@ -30,6 +30,7 @@ #include "gc/g1/g1AllocationContext.hpp" #include "gc/g1/g1BiasedArray.hpp" #include "gc/g1/g1CollectorState.hpp" +#include "gc/g1/g1HeapVerifier.hpp" #include "gc/g1/g1HRPrinter.hpp" #include "gc/g1/g1InCSetState.hpp" #include "gc/g1/g1MonitoringSupport.hpp" @@ -118,6 +119,8 @@ friend class VMStructs; friend class MutatorAllocRegion; friend class G1GCAllocRegion; + friend class G1Allocator; + friend class G1HeapVerifier; // Closures used in implementation. friend class G1ParScanThreadState; @@ -178,9 +181,12 @@ // The sequence of all heap regions in the heap. HeapRegionManager _hrm; - // Manages all allocations with regions except humongous object allocations. + // Manages all allocations with regions. G1Allocator* _allocator; + // Manages all heap verification. + G1HeapVerifier* _verifier; + // Outside of GC pauses, the number of bytes used in all regions other // than the current allocation region(s). size_t _summary_bytes_used; @@ -265,14 +271,6 @@ bool _heap_summary_sent; - // This is a non-product method that is helpful for testing. It is - // called at the end of a GC and artificially expands the heap by - // allocating a number of dead regions. This way we can induce very - // frequent marking cycles and stress the cleanup / concurrent - // cleanup code more (as all the regions that will be allocated by - // this method will be found dead by the marking cycle). - void allocate_dummy_regions() PRODUCT_RETURN; - // Clear RSets after a compaction. It also resets the GC time stamps. void clear_rsets_post_compaction(); @@ -286,10 +284,6 @@ size_t size, size_t translation_factor); - double verify(bool guard, const char* msg); - void verify_before_gc(); - void verify_after_gc(); - void log_gc_footer(jlong pause_time_counter); void trace_heap(GCWhen::Type when, const GCTracer* tracer); @@ -378,18 +372,6 @@ // humongous object, set is_old to true. If not, to false. HeapRegion* new_region(size_t word_size, bool is_old, bool do_expand); - // Initialize a contiguous set of free regions of length num_regions - // and starting at index first so that they appear as a single - // humongous region. - HeapWord* humongous_obj_allocate_initialize_regions(uint first, - uint num_regions, - size_t word_size, - AllocationContext_t context); - - // Attempt to allocate a humongous object of the given size. Return - // NULL if unsuccessful. - HeapWord* humongous_obj_allocate(size_t word_size, AllocationContext_t context); - // The following two methods, allocate_new_tlab() and // mem_allocate(), are the two main entry points from the runtime // into the G1's allocation routines. They have the following @@ -424,44 +406,6 @@ virtual HeapWord* mem_allocate(size_t word_size, bool* gc_overhead_limit_was_exceeded); - // The following three methods take a gc_count_before_ret - // parameter which is used to return the GC count if the method - // returns NULL. Given that we are required to read the GC count - // while holding the Heap_lock, and these paths will take the - // Heap_lock at some point, it's easier to get them to read the GC - // count while holding the Heap_lock before they return NULL instead - // of the caller (namely: mem_allocate()) having to also take the - // Heap_lock just to read the GC count. - - // First-level mutator allocation attempt: try to allocate out of - // the mutator alloc region without taking the Heap_lock. This - // should only be used for non-humongous allocations. - inline HeapWord* attempt_allocation(size_t word_size, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret); - - // Second-level mutator allocation attempt: take the Heap_lock and - // retry the allocation attempt, potentially scheduling a GC - // pause. This should only be used for non-humongous allocations. - HeapWord* attempt_allocation_slow(size_t word_size, - AllocationContext_t context, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret); - - // Takes the Heap_lock and attempts a humongous allocation. It can - // potentially schedule a GC pause. - HeapWord* attempt_allocation_humongous(size_t word_size, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret); - - // Allocation attempt that should be called during safepoints (e.g., - // at the end of a successful GC). expect_null_mutator_alloc_region - // specifies whether the mutator alloc region is expected to be NULL - // or not. - HeapWord* attempt_allocation_at_safepoint(size_t word_size, - AllocationContext_t context, - bool expect_null_mutator_alloc_region); - // These methods are the "callbacks" from the G1AllocRegion class. // For mutator alloc regions. @@ -527,6 +471,10 @@ return _allocator; } + G1HeapVerifier* verifier() { + return _verifier; + } + G1MonitoringSupport* g1mm() { assert(_g1mm != NULL, "should have been initialized"); return _g1mm; @@ -1056,54 +1004,6 @@ // The number of regions that are not completely free. uint num_used_regions() const { return num_regions() - num_free_regions(); } - void verify_not_dirty_region(HeapRegion* hr) PRODUCT_RETURN; - void verify_dirty_region(HeapRegion* hr) PRODUCT_RETURN; - void verify_dirty_young_list(HeapRegion* head) PRODUCT_RETURN; - void verify_dirty_young_regions() PRODUCT_RETURN; - -#ifndef PRODUCT - // Make sure that the given bitmap has no marked objects in the - // range [from,limit). If it does, print an error message and return - // false. Otherwise, just return true. bitmap_name should be "prev" - // or "next". - bool verify_no_bits_over_tams(const char* bitmap_name, CMBitMapRO* bitmap, - HeapWord* from, HeapWord* limit); - - // Verify that the prev / next bitmap range [tams,end) for the given - // region has no marks. Return true if all is well, false if errors - // are detected. - bool verify_bitmaps(const char* caller, HeapRegion* hr); -#endif // PRODUCT - - // If G1VerifyBitmaps is set, verify that the marking bitmaps for - // the given region do not have any spurious marks. If errors are - // detected, print appropriate error messages and crash. - void check_bitmaps(const char* caller, HeapRegion* hr) PRODUCT_RETURN; - - // If G1VerifyBitmaps is set, verify that the marking bitmaps do not - // have any spurious marks. If errors are detected, print - // appropriate error messages and crash. - void check_bitmaps(const char* caller) PRODUCT_RETURN; - - // Do sanity check on the contents of the in-cset fast test table. - bool check_cset_fast_test() PRODUCT_RETURN_( return true; ); - - // verify_region_sets() performs verification over the region - // lists. It will be compiled in the product code to be used when - // necessary (i.e., during heap verification). - void verify_region_sets(); - - // verify_region_sets_optional() is planted in the code for - // list verification in non-product builds (and it can be enabled in - // product builds by defining HEAP_REGION_SET_FORCE_VERIFY to be 1). -#if HEAP_REGION_SET_FORCE_VERIFY - void verify_region_sets_optional() { - verify_region_sets(); - } -#else // HEAP_REGION_SET_FORCE_VERIFY - void verify_region_sets_optional() { } -#endif // HEAP_REGION_SET_FORCE_VERIFY - #ifdef ASSERT bool is_on_master_free_list(HeapRegion* hr) { return _hrm.is_free(hr); @@ -1350,10 +1250,6 @@ return (region_size / 2); } - // Returns the number of regions the humongous object of the given word size - // requires. - static size_t humongous_obj_size_in_regions(size_t word_size); - // Print the maximum heap capacity. virtual size_t max_capacity() const; @@ -1425,11 +1321,6 @@ inline bool is_obj_ill(const oop obj) const; - bool allocated_since_marking(oop obj, HeapRegion* hr, VerifyOption vo); - HeapWord* top_at_mark_start(HeapRegion* hr, VerifyOption vo); - bool is_marked(oop obj, VerifyOption vo); - const char* top_at_mark_start_str(VerifyOption vo); - ConcurrentMark* concurrent_mark() const { return _cm; } // Refinement @@ -1490,7 +1381,9 @@ // Currently there is only one place where this is called with // vo == UseMarkWord, which is to verify the marking during a // full GC. - void verify(VerifyOption vo); + void verify(VerifyOption vo) { + _verifier->verify(vo); + } // The methods below are here for convenience and dispatch the // appropriate method depending on value of the given VerifyOption diff --git a/src/share/vm/gc/g1/g1EvacFailure.cpp b/src/share/vm/gc/g1/g1EvacFailure.cpp --- a/src/share/vm/gc/g1/g1EvacFailure.cpp +++ b/src/share/vm/gc/g1/g1EvacFailure.cpp @@ -223,7 +223,7 @@ if (hr->evacuation_failed()) { hr->note_self_forwarding_removal_start(during_initial_mark, during_conc_mark); - _g1h->check_bitmaps("Self-Forwarding Ptr Removal", hr); + _g1h->verifier()->check_bitmaps("Self-Forwarding Ptr Removal", hr); // In the common case (i.e. when there is no evacuation // failure) we make sure that the following is done when diff --git a/src/share/vm/gc/g1/g1HeapVerifier.cpp b/src/share/vm/gc/g1/g1HeapVerifier.cpp new file mode 100644 --- /dev/null +++ b/src/share/vm/gc/g1/g1HeapVerifier.cpp @@ -0,0 +1,731 @@ +/* + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "logging/log.hpp" +#include "gc/g1/concurrentMarkThread.hpp" +#include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1HeapVerifier.hpp" +#include "gc/g1/g1MarkSweep.hpp" +#include "gc/g1/g1RemSet.hpp" +#include "gc/g1/g1RootProcessor.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/heapRegion.inline.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "gc/g1/g1StringDedup.hpp" +#include "gc/g1/youngList.hpp" +#include "memory/resourceArea.hpp" +#include "oops/oop.inline.hpp" + +class VerifyRootsClosure: public OopClosure { +private: + G1CollectedHeap* _g1h; + VerifyOption _vo; + bool _failures; +public: + // _vo == UsePrevMarking -> use "prev" marking information, + // _vo == UseNextMarking -> use "next" marking information, + // _vo == UseMarkWord -> use mark word from object header. + VerifyRootsClosure(VerifyOption vo) : + _g1h(G1CollectedHeap::heap()), + _vo(vo), + _failures(false) { } + + bool failures() { return _failures; } + + template void do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if (_g1h->is_obj_dead_cond(obj, _vo)) { + LogHandle(gc, verify) log; + log.info("Root location " PTR_FORMAT " points to dead obj " PTR_FORMAT, p2i(p), p2i(obj)); + if (_vo == VerifyOption_G1UseMarkWord) { + log.info(" Mark word: " PTR_FORMAT, p2i(obj->mark())); + } + ResourceMark rm; + obj->print_on(log.info_stream()); + _failures = true; + } + } + } + + void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_nv(p); } +}; + +class G1VerifyCodeRootOopClosure: public OopClosure { + G1CollectedHeap* _g1h; + OopClosure* _root_cl; + nmethod* _nm; + VerifyOption _vo; + bool _failures; + + template void do_oop_work(T* p) { + // First verify that this root is live + _root_cl->do_oop(p); + + if (!G1VerifyHeapRegionCodeRoots) { + // We're not verifying the code roots attached to heap region. + return; + } + + // Don't check the code roots during marking verification in a full GC + if (_vo == VerifyOption_G1UseMarkWord) { + return; + } + + // Now verify that the current nmethod (which contains p) is + // in the code root list of the heap region containing the + // object referenced by p. + + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + + // Now fetch the region containing the object + HeapRegion* hr = _g1h->heap_region_containing(obj); + HeapRegionRemSet* hrrs = hr->rem_set(); + // Verify that the strong code root list for this region + // contains the nmethod + if (!hrrs->strong_code_roots_list_contains(_nm)) { + log_info(gc, verify)("Code root location " PTR_FORMAT " " + "from nmethod " PTR_FORMAT " not in strong " + "code roots for region [" PTR_FORMAT "," PTR_FORMAT ")", + p2i(p), p2i(_nm), p2i(hr->bottom()), p2i(hr->end())); + _failures = true; + } + } + } + +public: + G1VerifyCodeRootOopClosure(G1CollectedHeap* g1h, OopClosure* root_cl, VerifyOption vo): + _g1h(g1h), _root_cl(root_cl), _vo(vo), _nm(NULL), _failures(false) {} + + void do_oop(oop* p) { do_oop_work(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + + void set_nmethod(nmethod* nm) { _nm = nm; } + bool failures() { return _failures; } +}; + +class G1VerifyCodeRootBlobClosure: public CodeBlobClosure { + G1VerifyCodeRootOopClosure* _oop_cl; + +public: + G1VerifyCodeRootBlobClosure(G1VerifyCodeRootOopClosure* oop_cl): + _oop_cl(oop_cl) {} + + void do_code_blob(CodeBlob* cb) { + nmethod* nm = cb->as_nmethod_or_null(); + if (nm != NULL) { + _oop_cl->set_nmethod(nm); + nm->oops_do(_oop_cl); + } + } +}; + +class YoungRefCounterClosure : public OopClosure { + G1CollectedHeap* _g1h; + int _count; + public: + YoungRefCounterClosure(G1CollectedHeap* g1h) : _g1h(g1h), _count(0) {} + void do_oop(oop* p) { if (_g1h->is_in_young(*p)) { _count++; } } + void do_oop(narrowOop* p) { ShouldNotReachHere(); } + + int count() { return _count; } + void reset_count() { _count = 0; }; +}; + +class VerifyKlassClosure: public KlassClosure { + YoungRefCounterClosure _young_ref_counter_closure; + OopClosure *_oop_closure; + public: + VerifyKlassClosure(G1CollectedHeap* g1h, OopClosure* cl) : _young_ref_counter_closure(g1h), _oop_closure(cl) {} + void do_klass(Klass* k) { + k->oops_do(_oop_closure); + + _young_ref_counter_closure.reset_count(); + k->oops_do(&_young_ref_counter_closure); + if (_young_ref_counter_closure.count() > 0) { + guarantee(k->has_modified_oops(), "Klass " PTR_FORMAT ", has young refs but is not dirty.", p2i(k)); + } + } +}; + +class VerifyLivenessOopClosure: public OopClosure { + G1CollectedHeap* _g1h; + VerifyOption _vo; +public: + VerifyLivenessOopClosure(G1CollectedHeap* g1h, VerifyOption vo): + _g1h(g1h), _vo(vo) + { } + void do_oop(narrowOop *p) { do_oop_work(p); } + void do_oop( oop *p) { do_oop_work(p); } + + template void do_oop_work(T *p) { + oop obj = oopDesc::load_decode_heap_oop(p); + guarantee(obj == NULL || !_g1h->is_obj_dead_cond(obj, _vo), + "Dead object referenced by a not dead object"); + } +}; + +class VerifyObjsInRegionClosure: public ObjectClosure { +private: + G1CollectedHeap* _g1h; + size_t _live_bytes; + HeapRegion *_hr; + VerifyOption _vo; +public: + // _vo == UsePrevMarking -> use "prev" marking information, + // _vo == UseNextMarking -> use "next" marking information, + // _vo == UseMarkWord -> use mark word from object header. + VerifyObjsInRegionClosure(HeapRegion *hr, VerifyOption vo) + : _live_bytes(0), _hr(hr), _vo(vo) { + _g1h = G1CollectedHeap::heap(); + } + void do_object(oop o) { + VerifyLivenessOopClosure isLive(_g1h, _vo); + assert(o != NULL, "Huh?"); + if (!_g1h->is_obj_dead_cond(o, _vo)) { + // If the object is alive according to the mark word, + // then verify that the marking information agrees. + // Note we can't verify the contra-positive of the + // above: if the object is dead (according to the mark + // word), it may not be marked, or may have been marked + // but has since became dead, or may have been allocated + // since the last marking. + if (_vo == VerifyOption_G1UseMarkWord) { + guarantee(!_g1h->is_obj_dead(o), "mark word and concurrent mark mismatch"); + } + + o->oop_iterate_no_header(&isLive); + if (!_hr->obj_allocated_since_prev_marking(o)) { + size_t obj_size = o->size(); // Make sure we don't overflow + _live_bytes += (obj_size * HeapWordSize); + } + } + } + size_t live_bytes() { return _live_bytes; } +}; + +class VerifyArchiveOopClosure: public OopClosure { +public: + VerifyArchiveOopClosure(HeapRegion *hr) { } + void do_oop(narrowOop *p) { do_oop_work(p); } + void do_oop( oop *p) { do_oop_work(p); } + + template void do_oop_work(T *p) { + oop obj = oopDesc::load_decode_heap_oop(p); + guarantee(obj == NULL || G1MarkSweep::in_archive_range(obj), + "Archive object at " PTR_FORMAT " references a non-archive object at " PTR_FORMAT, + p2i(p), p2i(obj)); + } +}; + +class VerifyArchiveRegionClosure: public ObjectClosure { +public: + VerifyArchiveRegionClosure(HeapRegion *hr) { } + // Verify that all object pointers are to archive regions. + void do_object(oop o) { + VerifyArchiveOopClosure checkOop(NULL); + assert(o != NULL, "Should not be here for NULL oops"); + o->oop_iterate_no_header(&checkOop); + } +}; + +class VerifyRegionClosure: public HeapRegionClosure { +private: + bool _par; + VerifyOption _vo; + bool _failures; +public: + // _vo == UsePrevMarking -> use "prev" marking information, + // _vo == UseNextMarking -> use "next" marking information, + // _vo == UseMarkWord -> use mark word from object header. + VerifyRegionClosure(bool par, VerifyOption vo) + : _par(par), + _vo(vo), + _failures(false) {} + + bool failures() { + return _failures; + } + + bool doHeapRegion(HeapRegion* r) { + // For archive regions, verify there are no heap pointers to + // non-pinned regions. For all others, verify liveness info. + if (r->is_archive()) { + VerifyArchiveRegionClosure verify_oop_pointers(r); + r->object_iterate(&verify_oop_pointers); + return true; + } + if (!r->is_continues_humongous()) { + bool failures = false; + r->verify(_vo, &failures); + if (failures) { + _failures = true; + } else if (!r->is_starts_humongous()) { + VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo); + r->object_iterate(¬_dead_yet_cl); + if (_vo != VerifyOption_G1UseNextMarking) { + if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) { + log_info(gc, verify)("[" PTR_FORMAT "," PTR_FORMAT "] max_live_bytes " SIZE_FORMAT " < calculated " SIZE_FORMAT, + p2i(r->bottom()), p2i(r->end()), r->max_live_bytes(), not_dead_yet_cl.live_bytes()); + _failures = true; + } + } else { + // When vo == UseNextMarking we cannot currently do a sanity + // check on the live bytes as the calculation has not been + // finalized yet. + } + } + } + return false; // stop the region iteration if we hit a failure + } +}; + +// This is the task used for parallel verification of the heap regions + +class G1ParVerifyTask: public AbstractGangTask { +private: + G1CollectedHeap* _g1h; + VerifyOption _vo; + bool _failures; + HeapRegionClaimer _hrclaimer; + +public: + // _vo == UsePrevMarking -> use "prev" marking information, + // _vo == UseNextMarking -> use "next" marking information, + // _vo == UseMarkWord -> use mark word from object header. + G1ParVerifyTask(G1CollectedHeap* g1h, VerifyOption vo) : + AbstractGangTask("Parallel verify task"), + _g1h(g1h), + _vo(vo), + _failures(false), + _hrclaimer(g1h->workers()->active_workers()) {} + + bool failures() { + return _failures; + } + + void work(uint worker_id) { + HandleMark hm; + VerifyRegionClosure blk(true, _vo); + _g1h->heap_region_par_iterate(&blk, worker_id, &_hrclaimer); + if (blk.failures()) { + _failures = true; + } + } +}; + + +void G1HeapVerifier::verify(VerifyOption vo) { + if (!SafepointSynchronize::is_at_safepoint()) { + log_info(gc, verify)("Skipping verification. Not at safepoint."); + } + + assert(Thread::current()->is_VM_thread(), + "Expected to be executed serially by the VM thread at this point"); + + log_debug(gc, verify)("Roots"); + VerifyRootsClosure rootsCl(vo); + VerifyKlassClosure klassCl(_g1h, &rootsCl); + CLDToKlassAndOopClosure cldCl(&klassCl, &rootsCl, false); + + // We apply the relevant closures to all the oops in the + // system dictionary, class loader data graph, the string table + // and the nmethods in the code cache. + G1VerifyCodeRootOopClosure codeRootsCl(_g1h, &rootsCl, vo); + G1VerifyCodeRootBlobClosure blobsCl(&codeRootsCl); + + { + G1RootProcessor root_processor(_g1h, 1); + root_processor.process_all_roots(&rootsCl, + &cldCl, + &blobsCl); + } + + bool failures = rootsCl.failures() || codeRootsCl.failures(); + + if (vo != VerifyOption_G1UseMarkWord) { + // If we're verifying during a full GC then the region sets + // will have been torn down at the start of the GC. Therefore + // verifying the region sets will fail. So we only verify + // the region sets when not in a full GC. + log_debug(gc, verify)("HeapRegionSets"); + verify_region_sets(); + } + + log_debug(gc, verify)("HeapRegions"); + if (GCParallelVerificationEnabled && ParallelGCThreads > 1) { + + G1ParVerifyTask task(_g1h, vo); + _g1h->workers()->run_task(&task); + if (task.failures()) { + failures = true; + } + + } else { + VerifyRegionClosure blk(false, vo); + _g1h->heap_region_iterate(&blk); + if (blk.failures()) { + failures = true; + } + } + + if (G1StringDedup::is_enabled()) { + log_debug(gc, verify)("StrDedup"); + G1StringDedup::verify(); + } + + if (failures) { + log_info(gc, verify)("Heap after failed verification:"); + // It helps to have the per-region information in the output to + // help us track down what went wrong. This is why we call + // print_extended_on() instead of print_on(). + LogHandle(gc, verify) log; + ResourceMark rm; + _g1h->print_extended_on(log.info_stream()); + } + guarantee(!failures, "there should not have been any failures"); +} + +// Heap region set verification + +class VerifyRegionListsClosure : public HeapRegionClosure { +private: + HeapRegionSet* _old_set; + HeapRegionSet* _humongous_set; + HeapRegionManager* _hrm; + +public: + uint _old_count; + uint _humongous_count; + uint _free_count; + + VerifyRegionListsClosure(HeapRegionSet* old_set, + HeapRegionSet* humongous_set, + HeapRegionManager* hrm) : + _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm), + _old_count(), _humongous_count(), _free_count(){ } + + bool doHeapRegion(HeapRegion* hr) { + if (hr->is_young()) { + // TODO + } else if (hr->is_humongous()) { + assert(hr->containing_set() == _humongous_set, "Heap region %u is humongous but not in humongous set.", hr->hrm_index()); + _humongous_count++; + } else if (hr->is_empty()) { + assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index()); + _free_count++; + } else if (hr->is_old()) { + assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index()); + _old_count++; + } else { + // There are no other valid region types. Check for one invalid + // one we can identify: pinned without old or humongous set. + assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index()); + ShouldNotReachHere(); + } + return false; + } + + void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) { + guarantee(old_set->length() == _old_count, "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count); + guarantee(humongous_set->length() == _humongous_count, "Hum set count mismatch. Expected %u, actual %u.", humongous_set->length(), _humongous_count); + guarantee(free_list->num_free_regions() == _free_count, "Free list count mismatch. Expected %u, actual %u.", free_list->num_free_regions(), _free_count); + } +}; + +void G1HeapVerifier::verify_region_sets() { + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + + // First, check the explicit lists. + _g1h->_hrm.verify(); + { + // Given that a concurrent operation might be adding regions to + // the secondary free list we have to take the lock before + // verifying it. + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + _g1h->_secondary_free_list.verify_list(); + } + + // If a concurrent region freeing operation is in progress it will + // be difficult to correctly attributed any free regions we come + // across to the correct free list given that they might belong to + // one of several (free_list, secondary_free_list, any local lists, + // etc.). So, if that's the case we will skip the rest of the + // verification operation. Alternatively, waiting for the concurrent + // operation to complete will have a non-trivial effect on the GC's + // operation (no concurrent operation will last longer than the + // interval between two calls to verification) and it might hide + // any issues that we would like to catch during testing. + if (_g1h->free_regions_coming()) { + return; + } + + // Make sure we append the secondary_free_list on the free_list so + // that all free regions we will come across can be safely + // attributed to the free_list. + _g1h->append_secondary_free_list_if_not_empty_with_lock(); + + // Finally, make sure that the region accounting in the lists is + // consistent with what we see in the heap. + + VerifyRegionListsClosure cl(&_g1h->_old_set, &_g1h->_humongous_set, &_g1h->_hrm); + _g1h->heap_region_iterate(&cl); + cl.verify_counts(&_g1h->_old_set, &_g1h->_humongous_set, &_g1h->_hrm); +} + +void G1HeapVerifier::prepare_for_verify() { + if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { + _g1h->ensure_parsability(false); + } + _g1h->g1_rem_set()->prepare_for_verify(); +} + +double G1HeapVerifier::verify(bool guard, const char* msg) { + double verify_time_ms = 0.0; + + if (guard && _g1h->total_collections() >= VerifyGCStartAt) { + double verify_start = os::elapsedTime(); + HandleMark hm; // Discard invalid handles created during verification + prepare_for_verify(); + Universe::verify(VerifyOption_G1UsePrevMarking, msg); + verify_time_ms = (os::elapsedTime() - verify_start) * 1000; + } + + return verify_time_ms; +} + +void G1HeapVerifier::verify_before_gc() { + double verify_time_ms = verify(VerifyBeforeGC, "Before GC"); + _g1h->g1_policy()->phase_times()->record_verify_before_time_ms(verify_time_ms); +} + +void G1HeapVerifier::verify_after_gc() { + double verify_time_ms = verify(VerifyAfterGC, "After GC"); + _g1h->g1_policy()->phase_times()->record_verify_after_time_ms(verify_time_ms); +} + + +#ifndef PRODUCT +class G1VerifyCardTableCleanup: public HeapRegionClosure { + G1HeapVerifier* _verifier; + G1SATBCardTableModRefBS* _ct_bs; +public: + G1VerifyCardTableCleanup(G1HeapVerifier* verifier, G1SATBCardTableModRefBS* ct_bs) + : _verifier(verifier), _ct_bs(ct_bs) { } + virtual bool doHeapRegion(HeapRegion* r) { + if (r->is_survivor()) { + _verifier->verify_dirty_region(r); + } else { + _verifier->verify_not_dirty_region(r); + } + return false; + } +}; + +void G1HeapVerifier::verify_card_table_cleanup() { + if (G1VerifyCTCleanup || VerifyAfterGC) { + G1VerifyCardTableCleanup cleanup_verifier(this, _g1h->g1_barrier_set()); + _g1h->heap_region_iterate(&cleanup_verifier); + } +} + +void G1HeapVerifier::verify_not_dirty_region(HeapRegion* hr) { + // All of the region should be clean. + G1SATBCardTableModRefBS* ct_bs = _g1h->g1_barrier_set(); + MemRegion mr(hr->bottom(), hr->end()); + ct_bs->verify_not_dirty_region(mr); +} + +void G1HeapVerifier::verify_dirty_region(HeapRegion* hr) { + // We cannot guarantee that [bottom(),end()] is dirty. Threads + // dirty allocated blocks as they allocate them. The thread that + // retires each region and replaces it with a new one will do a + // maximal allocation to fill in [pre_dummy_top(),end()] but will + // not dirty that area (one less thing to have to do while holding + // a lock). So we can only verify that [bottom(),pre_dummy_top()] + // is dirty. + G1SATBCardTableModRefBS* ct_bs = _g1h->g1_barrier_set(); + MemRegion mr(hr->bottom(), hr->pre_dummy_top()); + if (hr->is_young()) { + ct_bs->verify_g1_young_region(mr); + } else { + ct_bs->verify_dirty_region(mr); + } +} + +void G1HeapVerifier::verify_dirty_young_list(HeapRegion* head) { + G1SATBCardTableModRefBS* ct_bs = _g1h->g1_barrier_set(); + for (HeapRegion* hr = head; hr != NULL; hr = hr->get_next_young_region()) { + verify_dirty_region(hr); + } +} + +void G1HeapVerifier::verify_dirty_young_regions() { + verify_dirty_young_list(_g1h->young_list()->first_region()); +} + +bool G1HeapVerifier::verify_no_bits_over_tams(const char* bitmap_name, CMBitMapRO* bitmap, + HeapWord* tams, HeapWord* end) { + guarantee(tams <= end, + "tams: " PTR_FORMAT " end: " PTR_FORMAT, p2i(tams), p2i(end)); + HeapWord* result = bitmap->getNextMarkedWordAddress(tams, end); + if (result < end) { + log_info(gc, verify)("## wrong marked address on %s bitmap: " PTR_FORMAT, bitmap_name, p2i(result)); + log_info(gc, verify)("## %s tams: " PTR_FORMAT " end: " PTR_FORMAT, bitmap_name, p2i(tams), p2i(end)); + return false; + } + return true; +} + +bool G1HeapVerifier::verify_bitmaps(const char* caller, HeapRegion* hr) { + CMBitMapRO* prev_bitmap = _g1h->concurrent_mark()->prevMarkBitMap(); + CMBitMapRO* next_bitmap = (CMBitMapRO*) _g1h->concurrent_mark()->nextMarkBitMap(); + + HeapWord* bottom = hr->bottom(); + HeapWord* ptams = hr->prev_top_at_mark_start(); + HeapWord* ntams = hr->next_top_at_mark_start(); + HeapWord* end = hr->end(); + + bool res_p = verify_no_bits_over_tams("prev", prev_bitmap, ptams, end); + + bool res_n = true; + // We reset mark_in_progress() before we reset _cmThread->in_progress() and in this window + // we do the clearing of the next bitmap concurrently. Thus, we can not verify the bitmap + // if we happen to be in that state. + if (_g1h->collector_state()->mark_in_progress() || !_g1h->_cmThread->in_progress()) { + res_n = verify_no_bits_over_tams("next", next_bitmap, ntams, end); + } + if (!res_p || !res_n) { + log_info(gc, verify)("#### Bitmap verification failed for " HR_FORMAT, HR_FORMAT_PARAMS(hr)); + log_info(gc, verify)("#### Caller: %s", caller); + return false; + } + return true; +} + +void G1HeapVerifier::check_bitmaps(const char* caller, HeapRegion* hr) { + if (!G1VerifyBitmaps) return; + + guarantee(verify_bitmaps(caller, hr), "bitmap verification"); +} + +class G1VerifyBitmapClosure : public HeapRegionClosure { +private: + const char* _caller; + G1HeapVerifier* _verifier; + bool _failures; + +public: + G1VerifyBitmapClosure(const char* caller, G1HeapVerifier* verifier) : + _caller(caller), _verifier(verifier), _failures(false) { } + + bool failures() { return _failures; } + + virtual bool doHeapRegion(HeapRegion* hr) { + bool result = _verifier->verify_bitmaps(_caller, hr); + if (!result) { + _failures = true; + } + return false; + } +}; + +void G1HeapVerifier::check_bitmaps(const char* caller) { + if (!G1VerifyBitmaps) return; + + G1VerifyBitmapClosure cl(caller, this); + _g1h->heap_region_iterate(&cl); + guarantee(!cl.failures(), "bitmap verification"); +} + +class G1CheckCSetFastTableClosure : public HeapRegionClosure { + private: + bool _failures; + public: + G1CheckCSetFastTableClosure() : HeapRegionClosure(), _failures(false) { } + + virtual bool doHeapRegion(HeapRegion* hr) { + uint i = hr->hrm_index(); + InCSetState cset_state = (InCSetState) G1CollectedHeap::heap()->_in_cset_fast_test.get_by_index(i); + if (hr->is_humongous()) { + if (hr->in_collection_set()) { + log_info(gc, verify)("## humongous region %u in CSet", i); + _failures = true; + return true; + } + if (cset_state.is_in_cset()) { + log_info(gc, verify)("## inconsistent cset state " CSETSTATE_FORMAT " for humongous region %u", cset_state.value(), i); + _failures = true; + return true; + } + if (hr->is_continues_humongous() && cset_state.is_humongous()) { + log_info(gc, verify)("## inconsistent cset state " CSETSTATE_FORMAT " for continues humongous region %u", cset_state.value(), i); + _failures = true; + return true; + } + } else { + if (cset_state.is_humongous()) { + log_info(gc, verify)("## inconsistent cset state " CSETSTATE_FORMAT " for non-humongous region %u", cset_state.value(), i); + _failures = true; + return true; + } + if (hr->in_collection_set() != cset_state.is_in_cset()) { + log_info(gc, verify)("## in CSet %d / cset state " CSETSTATE_FORMAT " inconsistency for region %u", + hr->in_collection_set(), cset_state.value(), i); + _failures = true; + return true; + } + if (cset_state.is_in_cset()) { + if (hr->is_young() != (cset_state.is_young())) { + log_info(gc, verify)("## is_young %d / cset state " CSETSTATE_FORMAT " inconsistency for region %u", + hr->is_young(), cset_state.value(), i); + _failures = true; + return true; + } + if (hr->is_old() != (cset_state.is_old())) { + log_info(gc, verify)("## is_old %d / cset state " CSETSTATE_FORMAT " inconsistency for region %u", + hr->is_old(), cset_state.value(), i); + _failures = true; + return true; + } + } + } + return false; + } + + bool failures() const { return _failures; } +}; + +bool G1HeapVerifier::check_cset_fast_test() { + G1CheckCSetFastTableClosure cl; + _g1h->_hrm.iterate(&cl); + return !cl.failures(); +} +#endif // PRODUCT diff --git a/src/share/vm/gc/g1/g1HeapVerifier.hpp b/src/share/vm/gc/g1/g1HeapVerifier.hpp new file mode 100644 --- /dev/null +++ b/src/share/vm/gc/g1/g1HeapVerifier.hpp @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1HEAPVERIFIER_HPP +#define SHARE_VM_GC_G1_G1HEAPVERIFIER_HPP + +#include "gc/g1/heapRegionSet.hpp" +#include "memory/allocation.hpp" +#include "memory/universe.hpp" + +class G1CollectedHeap; + +class G1HeapVerifier : public CHeapObj { +private: + G1CollectedHeap* _g1h; + + // verify_region_sets() performs verification over the region + // lists. It will be compiled in the product code to be used when + // necessary (i.e., during heap verification). + void verify_region_sets(); + +public: + + G1HeapVerifier(G1CollectedHeap* heap) : _g1h(heap) { } + + // Perform verification. + + // vo == UsePrevMarking -> use "prev" marking information, + // vo == UseNextMarking -> use "next" marking information + // vo == UseMarkWord -> use the mark word in the object header + // + // NOTE: Only the "prev" marking information is guaranteed to be + // consistent most of the time, so most calls to this should use + // vo == UsePrevMarking. + // Currently, there is only one case where this is called with + // vo == UseNextMarking, which is to verify the "next" marking + // information at the end of remark. + // Currently there is only one place where this is called with + // vo == UseMarkWord, which is to verify the marking during a + // full GC. + void verify(VerifyOption vo); + + // verify_region_sets_optional() is planted in the code for + // list verification in non-product builds (and it can be enabled in + // product builds by defining HEAP_REGION_SET_FORCE_VERIFY to be 1). +#if HEAP_REGION_SET_FORCE_VERIFY + void verify_region_sets_optional() { + verify_region_sets(); + } +#else // HEAP_REGION_SET_FORCE_VERIFY + void verify_region_sets_optional() { } +#endif // HEAP_REGION_SET_FORCE_VERIFY + + void prepare_for_verify(); + double verify(bool guard, const char* msg); + void verify_before_gc(); + void verify_after_gc(); + +#ifndef PRODUCT + // Make sure that the given bitmap has no marked objects in the + // range [from,limit). If it does, print an error message and return + // false. Otherwise, just return true. bitmap_name should be "prev" + // or "next". + bool verify_no_bits_over_tams(const char* bitmap_name, CMBitMapRO* bitmap, + HeapWord* from, HeapWord* limit); + + // Verify that the prev / next bitmap range [tams,end) for the given + // region has no marks. Return true if all is well, false if errors + // are detected. + bool verify_bitmaps(const char* caller, HeapRegion* hr); +#endif // PRODUCT + + // If G1VerifyBitmaps is set, verify that the marking bitmaps for + // the given region do not have any spurious marks. If errors are + // detected, print appropriate error messages and crash. + void check_bitmaps(const char* caller, HeapRegion* hr) PRODUCT_RETURN; + + // If G1VerifyBitmaps is set, verify that the marking bitmaps do not + // have any spurious marks. If errors are detected, print + // appropriate error messages and crash. + void check_bitmaps(const char* caller) PRODUCT_RETURN; + + // Do sanity check on the contents of the in-cset fast test table. + bool check_cset_fast_test() PRODUCT_RETURN_( return true; ); + + void verify_card_table_cleanup() PRODUCT_RETURN; + + void verify_not_dirty_region(HeapRegion* hr) PRODUCT_RETURN; + void verify_dirty_region(HeapRegion* hr) PRODUCT_RETURN; + void verify_dirty_young_list(HeapRegion* head) PRODUCT_RETURN; + void verify_dirty_young_regions() PRODUCT_RETURN; +}; + +#endif // SHARE_VM_GC_G1_G1HEAPVERIFIER_HPP diff --git a/src/share/vm/gc/g1/vm_operations_g1.cpp b/src/share/vm/gc/g1/vm_operations_g1.cpp --- a/src/share/vm/gc/g1/vm_operations_g1.cpp +++ b/src/share/vm/gc/g1/vm_operations_g1.cpp @@ -24,10 +24,12 @@ #include "precompiled.hpp" #include "gc/g1/concurrentMarkThread.inline.hpp" +#include "gc/g1/g1Allocator.hpp" +#include "gc/g1/g1Allocator.inline.hpp" #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/vm_operations_g1.hpp" #include "gc/shared/gcId.hpp" -#include "gc/g1/vm_operations_g1.hpp" #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/isGCActiveMark.hpp" @@ -94,9 +96,9 @@ if (_word_size > 0) { // An allocation has been requested. So, try to do that first. - _result = g1h->attempt_allocation_at_safepoint(_word_size, - allocation_context(), - false /* expect_null_cur_alloc_region */); + _result = g1h->allocator()->attempt_allocation_at_safepoint(_word_size, + allocation_context(), + false /* expect_null_cur_alloc_region */); if (_result != NULL) { // If we can successfully allocate before we actually do the // pause then we will consider this pause successful. @@ -148,9 +150,9 @@ g1h->do_collection_pause_at_safepoint(_target_pause_time_ms); if (_pause_succeeded && _word_size > 0) { // An allocation had been requested. - _result = g1h->attempt_allocation_at_safepoint(_word_size, - allocation_context(), - true /* expect_null_cur_alloc_region */); + _result = g1h->allocator()->attempt_allocation_at_safepoint(_word_size, + allocation_context(), + true /* expect_null_cur_alloc_region */); } else { assert(_result == NULL, "invariant"); if (!_pause_succeeded) {