# HG changeset patch # User david # Date 1446121543 -3600 # Thu Oct 29 13:25:43 2015 +0100 # Node ID 2ee482db7f929de8ae5aeb3a5a72d37967b4864d # Parent 978ced4575b1e6508a1dc34add93d7332bba6fe6 imported patch humongous diff --git a/src/share/vm/gc/g1/concurrentMark.cpp b/src/share/vm/gc/g1/concurrentMark.cpp --- a/src/share/vm/gc/g1/concurrentMark.cpp +++ b/src/share/vm/gc/g1/concurrentMark.cpp @@ -823,12 +823,7 @@ // This closure can be called concurrently to the mutator, so we must make sure // that the result of the getNextMarkedWordAddress() call is compared to the // value passed to it as limit to detect any found bits. - // We can use the region's orig_end() for the limit and the comparison value - // as it always contains the "real" end of the region that never changes and - // has no side effects. - // Due to the latter, there can also be no problem with the compiler generating - // reloads of the orig_end() call. - HeapWord* end = r->orig_end(); + HeapWord* end = r->end(); return _bitmap->getNextMarkedWordAddress(r->bottom(), end) != end; } }; @@ -842,9 +837,7 @@ class NoteStartOfMarkHRClosure: public HeapRegionClosure { public: bool doHeapRegion(HeapRegion* r) { - if (!r->is_continues_humongous()) { - r->note_start_of_marking(); - } + r->note_start_of_marking(); return false; } }; @@ -1332,22 +1325,10 @@ // Takes a region that's not empty (i.e., it has at least one // live object in it and sets its corresponding bit on the region - // bitmap to 1. If the region is "starts humongous" it will also set - // to 1 the bits on the region bitmap that correspond to its - // associated "continues humongous" regions. + // bitmap to 1. void set_bit_for_region(HeapRegion* hr) { - assert(!hr->is_continues_humongous(), "should have filtered those out"); - BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index(); - if (!hr->is_starts_humongous()) { - // Normal (non-humongous) case: just set the bit. - _region_bm->par_at_put(index, true); - } else { - // Starts humongous case: calculate how many regions are part of - // this humongous region and then set the bit range. - BitMap::idx_t end_index = (BitMap::idx_t) hr->last_hc_index(); - _region_bm->par_at_put_range(index, end_index, true); - } + _region_bm->par_at_put(index, true); } public: @@ -1371,18 +1352,6 @@ _bm(bm), _region_marked_bytes(0) { } bool doHeapRegion(HeapRegion* hr) { - - if (hr->is_continues_humongous()) { - // We will ignore these here and process them when their - // associated "starts humongous" region is processed (see - // set_bit_for_heap_region()). Note that we cannot rely on their - // associated "starts humongous" region to have their bit set to - // 1 since, due to the region chunking in the parallel region - // iteration, a "continues humongous" region might be visited - // before its associated "starts humongous". - return false; - } - HeapWord* ntams = hr->next_top_at_mark_start(); HeapWord* start = hr->bottom(); @@ -1420,6 +1389,9 @@ // Add the size of this object to the number of marked bytes. marked_bytes += (size_t)obj_sz * HeapWordSize; + if (obj_end > hr->end()) { + break; + } // Find the next marked object after this one. start = _bm->getNextMarkedWordAddress(obj_end, ntams); } @@ -1494,17 +1466,6 @@ int failures() const { return _failures; } bool doHeapRegion(HeapRegion* hr) { - if (hr->is_continues_humongous()) { - // We will ignore these here and process them when their - // associated "starts humongous" region is processed (see - // set_bit_for_heap_region()). Note that we cannot rely on their - // associated "starts humongous" region to have their bit set to - // 1 since, due to the region chunking in the parallel region - // iteration, a "continues humongous" region might be visited - // before its associated "starts humongous". - return false; - } - int failures = 0; // Call the CalcLiveObjectsClosure to walk the marking bitmap for @@ -1652,18 +1613,6 @@ CMCountDataClosureBase(g1h, region_bm, card_bm) { } bool doHeapRegion(HeapRegion* hr) { - - if (hr->is_continues_humongous()) { - // We will ignore these here and process them when their - // associated "starts humongous" region is processed (see - // set_bit_for_heap_region()). Note that we cannot rely on their - // associated "starts humongous" region to have their bit set to - // 1 since, due to the region chunking in the parallel region - // iteration, a "continues humongous" region might be visited - // before its associated "starts humongous". - return false; - } - HeapWord* ntams = hr->next_top_at_mark_start(); HeapWord* top = hr->top(); @@ -1760,7 +1709,7 @@ const HeapRegionSetCount& humongous_regions_removed() { return _humongous_regions_removed; } bool doHeapRegion(HeapRegion *hr) { - if (hr->is_continues_humongous() || hr->is_archive()) { + if (hr->is_archive()) { return false; } // We use a claim value of zero here because all regions @@ -1772,7 +1721,6 @@ _freed_bytes += hr->used(); hr->set_containing_set(NULL); if (hr->is_humongous()) { - assert(hr->is_starts_humongous(), "we should only see starts humongous"); _humongous_regions_removed.increment(1u, hr->capacity()); _g1->free_humongous_region(hr, _local_cleanup_list, true); } else { @@ -2771,17 +2719,6 @@ _cm_card_bm(cm_card_bm), _max_worker_id(max_worker_id) { } bool doHeapRegion(HeapRegion* hr) { - if (hr->is_continues_humongous()) { - // We will ignore these here and process them when their - // associated "starts humongous" region is processed. - // Note that we cannot rely on their associated - // "starts humongous" region to have their bit set to 1 - // since, due to the region chunking in the parallel region - // iteration, a "continues humongous" region might be visited - // before its associated "starts humongous". - return false; - } - HeapWord* start = hr->bottom(); HeapWord* limit = hr->next_top_at_mark_start(); HeapWord* end = hr->end(); @@ -3103,8 +3040,6 @@ void CMTask::setup_for_region(HeapRegion* hr) { assert(hr != NULL, "claim_region() should have filtered out NULL regions"); - assert(!hr->is_continues_humongous(), - "claim_region() should have filtered out continues humongous regions"); if (_cm->verbose_low()) { gclog_or_tty->print_cr("[%u] setting up for region " PTR_FORMAT, diff --git a/src/share/vm/gc/g1/concurrentMark.hpp b/src/share/vm/gc/g1/concurrentMark.hpp --- a/src/share/vm/gc/g1/concurrentMark.hpp +++ b/src/share/vm/gc/g1/concurrentMark.hpp @@ -820,14 +820,18 @@ // Counts the given memory region in the task/worker counting // data structures for the given worker id. - inline void count_region(MemRegion mr, HeapRegion* hr, uint worker_id); + inline void count_region(oop obj, + HeapRegion* hr, + uint worker_id, + size_t word_size); // Counts the given object in the given task/worker counting // data structures. inline void count_object(oop obj, HeapRegion* hr, size_t* marked_bytes_array, - BitMap* task_card_bm); + BitMap* task_card_bm, + size_t word_size); // Attempts to mark the given object and, if successful, counts // the object in the given task/worker counting structures. diff --git a/src/share/vm/gc/g1/concurrentMark.inline.hpp b/src/share/vm/gc/g1/concurrentMark.inline.hpp --- a/src/share/vm/gc/g1/concurrentMark.inline.hpp +++ b/src/share/vm/gc/g1/concurrentMark.inline.hpp @@ -89,9 +89,7 @@ size_t region_size_bytes = mr.byte_size(); uint index = hr->hrm_index(); - assert(!hr->is_continues_humongous(), "should not be HC region"); assert(hr == g1h->heap_region_containing(start), "sanity"); - assert(hr == g1h->heap_region_containing(mr.last()), "sanity"); assert(marked_bytes_array != NULL, "pre-condition"); assert(task_card_bm != NULL, "pre-condition"); @@ -118,21 +116,36 @@ // Counts the given memory region in the task/worker counting // data structures for the given worker id. -inline void ConcurrentMark::count_region(MemRegion mr, +inline void ConcurrentMark::count_region(oop obj, HeapRegion* hr, - uint worker_id) { + uint worker_id, + size_t word_size) { size_t* marked_bytes_array = count_marked_bytes_array_for(worker_id); BitMap* task_card_bm = count_card_bitmap_for(worker_id); - count_region(mr, hr, marked_bytes_array, task_card_bm); + count_object(obj, hr, marked_bytes_array, task_card_bm, word_size); } // Counts the given object in the given task/worker counting data structures. inline void ConcurrentMark::count_object(oop obj, HeapRegion* hr, size_t* marked_bytes_array, - BitMap* task_card_bm) { - MemRegion mr((HeapWord*)obj, obj->size()); - count_region(mr, hr, marked_bytes_array, task_card_bm); + BitMap* task_card_bm, + size_t word_size) { + assert(!hr->is_continues_humongous(), "Cannot enter count_object with continues humongous"); + if (!hr->is_starts_humongous()) { + MemRegion mr((HeapWord*)obj, word_size); + count_region(mr, hr, marked_bytes_array, task_card_bm); + } else { + uint index = hr->hrm_index(); + do { + MemRegion mr(hr->bottom(), hr->top()); + count_region(mr, hr, marked_bytes_array, task_card_bm); + if (++index >= _g1h->num_regions()) { + break; + } + hr = _g1h->region_at(index); + } while (hr->is_continues_humongous()); + } } // Attempts to mark the given object and, if successful, counts @@ -141,10 +154,10 @@ HeapRegion* hr, size_t* marked_bytes_array, BitMap* task_card_bm) { - HeapWord* addr = (HeapWord*)obj; - if (_nextMarkBitMap->parMark(addr)) { + //HeapWord* addr = (HeapWord*)obj; + if (_nextMarkBitMap->parMark((HeapWord*)obj)) { // Update the task specific count data for the object. - count_object(obj, hr, marked_bytes_array, task_card_bm); + count_object(obj, hr, marked_bytes_array, task_card_bm, obj->size()); return true; } return false; @@ -157,10 +170,10 @@ size_t word_size, HeapRegion* hr, uint worker_id) { - HeapWord* addr = (HeapWord*)obj; - if (_nextMarkBitMap->parMark(addr)) { - MemRegion mr(addr, word_size); - count_region(mr, hr, worker_id); +// HeapWord* addr = (HeapWord*)obj; + if (_nextMarkBitMap->parMark((HeapWord*)obj)) { + //MemRegion mr(addr, word_size); + count_region(obj, hr, worker_id, word_size); return true; } return false; @@ -420,16 +433,6 @@ // header it's impossible to get back a HC region. assert(!hr->is_continues_humongous(), "sanity"); - // We cannot assert that word_size == obj->size() given that obj - // might not be in a consistent state (another thread might be in - // the process of copying it). So the best thing we can do is to - // assert that word_size is under an upper bound which is its - // containing region's capacity. - assert(word_size * HeapWordSize <= hr->capacity(), - "size: " SIZE_FORMAT " capacity: " SIZE_FORMAT " " HR_FORMAT, - word_size * HeapWordSize, hr->capacity(), - HR_FORMAT_PARAMS(hr)); - if (addr < hr->next_top_at_mark_start()) { if (!_nextMarkBitMap->isMarked(addr)) { par_mark_and_count(obj, word_size, hr, worker_id); diff --git a/src/share/vm/gc/g1/g1BlockOffsetTable.cpp b/src/share/vm/gc/g1/g1BlockOffsetTable.cpp --- a/src/share/vm/gc/g1/g1BlockOffsetTable.cpp +++ b/src/share/vm/gc/g1/g1BlockOffsetTable.cpp @@ -500,12 +500,10 @@ } void -G1BlockOffsetArrayContigSpace::set_for_starts_humongous(HeapWord* new_top) { - assert(new_top <= _end, "_end should have already been updated"); - +G1BlockOffsetArrayContigSpace::set_for_starts_humongous(HeapWord* obj_top) { // The first BOT entry should have offset 0. reset_bot(); - alloc_block(_bottom, new_top); + alloc_block(_bottom, obj_top); } #ifndef PRODUCT diff --git a/src/share/vm/gc/g1/g1BlockOffsetTable.hpp b/src/share/vm/gc/g1/g1BlockOffsetTable.hpp --- a/src/share/vm/gc/g1/g1BlockOffsetTable.hpp +++ b/src/share/vm/gc/g1/g1BlockOffsetTable.hpp @@ -361,17 +361,18 @@ // implementation, that's true because NULL is represented as 0, and thus // never exceeds the "_next_offset_threshold". void alloc_block(HeapWord* blk_start, HeapWord* blk_end) { - if (blk_end > _next_offset_threshold) + if (blk_end > _next_offset_threshold) { alloc_block_work1(blk_start, blk_end); + } } void alloc_block(HeapWord* blk, size_t size) { - alloc_block(blk, blk+size); + alloc_block(blk, blk+size); } HeapWord* block_start_unsafe(const void* addr); HeapWord* block_start_unsafe_const(const void* addr) const; - void set_for_starts_humongous(HeapWord* new_top); + void set_for_starts_humongous(HeapWord* obj_top); virtual void print_on(outputStream* out) PRODUCT_RETURN; }; diff --git a/src/share/vm/gc/g1/g1BlockOffsetTable.inline.hpp b/src/share/vm/gc/g1/g1BlockOffsetTable.inline.hpp --- a/src/share/vm/gc/g1/g1BlockOffsetTable.inline.hpp +++ b/src/share/vm/gc/g1/g1BlockOffsetTable.inline.hpp @@ -123,7 +123,6 @@ // to go back by. size_t n_cards_back = BlockOffsetArray::entry_to_cards_back(offset); q -= (N_words * n_cards_back); - assert(q >= gsp()->bottom(), "Went below bottom!"); index -= n_cards_back; offset = _array->offset_array(index); } diff --git a/src/share/vm/gc/g1/g1CollectedHeap.cpp b/src/share/vm/gc/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc/g1/g1CollectedHeap.cpp +++ b/src/share/vm/gc/g1/g1CollectedHeap.cpp @@ -320,12 +320,8 @@ // The header of the new object will be placed at the bottom of // the first region. HeapWord* new_obj = first_hr->bottom(); - // This will be the new end of the first region in the series that - // should also match the end of the last region in the series. - HeapWord* new_end = new_obj + word_size_sum; - // This will be the new top of the first region that will reflect - // this allocation. - HeapWord* new_top = new_obj + word_size; + // This will be the new top of the new object. + HeapWord* obj_top = new_obj + word_size; // First, we need to zero the header of the space that we will be // allocating. When we update top further down, some refinement @@ -346,7 +342,7 @@ // will also update the BOT covering all the regions to reflect // that there is a single object that starts at the bottom of the // first region. - first_hr->set_starts_humongous(new_top, new_end); + first_hr->set_starts_humongous(obj_top); first_hr->set_allocation_context(context); // Then, if there are any, we will set up the "continues // humongous" regions. @@ -356,9 +352,6 @@ hr->set_continues_humongous(first_hr); hr->set_allocation_context(context); } - // If we have "continues humongous" regions (hr != NULL), then the - // end of the last one should match new_end. - assert(hr == NULL || hr->end() == new_end, "sanity"); // Up to this point no concurrent thread would have been able to // do any scanning on any region in this series. All the top @@ -371,18 +364,14 @@ // Now that the BOT and the object header have been initialized, // we can update top of the "starts humongous" region. - assert(first_hr->bottom() < new_top && new_top <= first_hr->end(), - "new_top should be in this region"); - first_hr->set_top(new_top); + first_hr->set_top(MIN2(first_hr->end(), obj_top)); if (_hr_printer.is_active()) { - HeapWord* bottom = first_hr->bottom(); - HeapWord* end = first_hr->orig_end(); if ((first + 1) == last) { // the series has a single humongous region - _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, new_top); + _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, obj_top); } else { // the series has more than one humongous regions - _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, end); + _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, first_hr->end()); } } @@ -402,27 +391,27 @@ hr = region_at(i); if ((i + 1) == last) { // last continues humongous region - assert(hr->bottom() < new_top && new_top <= hr->end(), + assert(hr->bottom() < obj_top && obj_top <= hr->end(), "new_top should fall on this region"); - hr->set_top(new_top); - _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, new_top); + hr->set_top(obj_top); + _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, obj_top); } else { // not last one - assert(new_top > hr->end(), "new_top should be above this region"); + assert(obj_top > hr->end(), "obj_top should be above this region"); hr->set_top(hr->end()); _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end()); } } - // If we have continues humongous regions (hr != NULL), then the - // end of the last one should match new_end and its top should - // match new_top. - assert(hr == NULL || - (hr->end() == new_end && hr->top() == new_top), "sanity"); + // If we have continues humongous regions (hr != NULL), its top should + // match obj_top. + assert(hr == NULL || (hr->top() == obj_top), "sanity"); check_bitmaps("Humongous Region Allocation", first_hr); - assert(first_hr->used() == word_size * HeapWordSize, "invariant"); - increase_used(first_hr->used()); - _humongous_set.add(first_hr); + increase_used(word_size * HeapWordSize); + + for (uint i = first; i < last; ++i) { + _humongous_set.add(region_at(i)); + } return new_obj; } @@ -1139,13 +1128,6 @@ bool doHeapRegion(HeapRegion* r) { HeapRegionRemSet* hrrs = r->rem_set(); - if (r->is_continues_humongous()) { - // We'll assert that the strong code root list and RSet is empty - assert(hrrs->strong_code_roots_list_length() == 0, "sanity"); - assert(hrrs->occupied() == 0, "RSet should be empty"); - return false; - } - _g1h->reset_gc_time_stamps(r); hrrs->clear(); // You might think here that we could clear just the cards @@ -1205,12 +1187,7 @@ if (hr->is_free()) { // We only generate output for non-empty regions. } else if (hr->is_starts_humongous()) { - if (hr->region_num() == 1) { - // single humongous region - _hr_printer->post_compaction(hr, G1HRPrinter::SingleHumongous); - } else { - _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous); - } + _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous); } else if (hr->is_continues_humongous()) { _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous); } else if (hr->is_archive()) { @@ -2222,17 +2199,7 @@ } void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) { - assert(!hr->is_continues_humongous(), "pre-condition"); hr->reset_gc_time_stamp(); - if (hr->is_starts_humongous()) { - uint first_index = hr->hrm_index() + 1; - uint last_index = hr->last_hc_index(); - for (uint i = first_index; i < last_index; i += 1) { - HeapRegion* chr = region_at(i); - assert(chr->is_continues_humongous(), "sanity"); - chr->reset_gc_time_stamp(); - } - } } #ifndef PRODUCT @@ -2300,9 +2267,7 @@ public: SumUsedClosure() : _used(0) {} bool doHeapRegion(HeapRegion* r) { - if (!r->is_continues_humongous()) { - _used += r->used(); - } + _used += r->used(); return false; } size_t result() { return _used; } @@ -3062,7 +3027,7 @@ r->verify(_vo, &failures); if (failures) { _failures = true; - } else { + } else if (!r->is_starts_humongous()) { VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo); r->object_iterate(¬_dead_yet_cl); if (_vo != VerifyOption_G1UseNextMarking) { @@ -5317,24 +5282,9 @@ void G1CollectedHeap::free_humongous_region(HeapRegion* hr, FreeRegionList* free_list, bool par) { - assert(hr->is_starts_humongous(), "this is only for starts humongous regions"); assert(free_list != NULL, "pre-condition"); - - size_t hr_capacity = hr->capacity(); - // We need to read this before we make the region non-humongous, - // otherwise the information will be gone. - uint last_index = hr->last_hc_index(); hr->clear_humongous(); free_region(hr, free_list, par); - - uint i = hr->hrm_index() + 1; - while (i < last_index) { - HeapRegion* curr_hr = region_at(i); - assert(curr_hr->is_continues_humongous(), "invariant"); - curr_hr->clear_humongous(); - free_region(curr_hr, free_list, par); - i += 1; - } } void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed, @@ -5498,8 +5448,6 @@ bool failures() { return _failures; } virtual bool doHeapRegion(HeapRegion* hr) { - if (hr->is_continues_humongous()) return false; - bool result = _g1h->verify_bitmaps(_caller, hr); if (!result) { _failures = true; @@ -5773,11 +5721,10 @@ !r->rem_set()->is_empty()) { if (G1TraceEagerReclaimHumongousObjects) { - gclog_or_tty->print_cr("Live humongous region %u size " SIZE_FORMAT " start " PTR_FORMAT " length %u with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d", + gclog_or_tty->print_cr("Live humongous region %u objectsize " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d", region_idx, (size_t)obj->size() * HeapWordSize, p2i(r->bottom()), - r->region_num(), r->rem_set()->occupied(), r->rem_set()->strong_code_roots_list_length(), next_bitmap->isMarked(r->bottom()), @@ -5794,11 +5741,10 @@ PTR_FORMAT " is not.", p2i(r->bottom())); if (G1TraceEagerReclaimHumongousObjects) { - gclog_or_tty->print_cr("Dead humongous region %u size " SIZE_FORMAT " start " PTR_FORMAT " length %u with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d", + gclog_or_tty->print_cr("Dead humongous region %u objectsize " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d", region_idx, (size_t)obj->size() * HeapWordSize, p2i(r->bottom()), - r->region_num(), r->rem_set()->occupied(), r->rem_set()->strong_code_roots_list_length(), next_bitmap->isMarked(r->bottom()), @@ -5810,10 +5756,16 @@ if (next_bitmap->isMarked(r->bottom())) { next_bitmap->clear(r->bottom()); } - _freed_bytes += r->used(); - r->set_containing_set(NULL); - _humongous_regions_removed.increment(1u, r->capacity()); - g1h->free_humongous_region(r, _free_region_list, false); + do { + _freed_bytes += r->used(); + r->set_containing_set(NULL); + _humongous_regions_removed.increment(1u, r->capacity()); + g1h->free_humongous_region(r, _free_region_list, false); + if (++region_idx >= g1h->num_regions()) { + break; + } + r = g1h->region_at(region_idx); + } while (r->is_continues_humongous()); return false; } @@ -6048,10 +6000,6 @@ } bool doHeapRegion(HeapRegion* r) { - if (r->is_continues_humongous()) { - return false; - } - if (r->is_empty()) { // Add free regions to the free list r->set_free(); @@ -6239,14 +6187,10 @@ _old_count(), _humongous_count(), _free_count(){ } bool doHeapRegion(HeapRegion* hr) { - if (hr->is_continues_humongous()) { - return false; - } - if (hr->is_young()) { // TODO - } else if (hr->is_starts_humongous()) { - assert(hr->containing_set() == _humongous_set, "Heap region %u is starts humongous but not in humongous set.", hr->hrm_index()); + } else if (hr->is_humongous()) { + assert(hr->containing_set() == _humongous_set, "Heap region %u is humongous but not in humongous set.", hr->hrm_index()); _humongous_count.increment(1u, hr->capacity()); } else if (hr->is_empty()) { assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index()); diff --git a/src/share/vm/gc/g1/g1CollectedHeap.inline.hpp b/src/share/vm/gc/g1/g1CollectedHeap.inline.hpp --- a/src/share/vm/gc/g1/g1CollectedHeap.inline.hpp +++ b/src/share/vm/gc/g1/g1CollectedHeap.inline.hpp @@ -87,11 +87,7 @@ template inline HeapRegion* G1CollectedHeap::heap_region_containing(const T addr) const { - HeapRegion* hr = heap_region_containing_raw(addr); - if (hr->is_continues_humongous()) { - return hr->humongous_start_region(); - } - return hr; + return heap_region_containing_raw(addr); } inline void G1CollectedHeap::reset_gc_time_stamp() { diff --git a/src/share/vm/gc/g1/g1MarkSweep.cpp b/src/share/vm/gc/g1/g1MarkSweep.cpp --- a/src/share/vm/gc/g1/g1MarkSweep.cpp +++ b/src/share/vm/gc/g1/g1MarkSweep.cpp @@ -279,8 +279,8 @@ } else { assert(hr->is_empty(), "Should have been cleared in phase 2."); } - hr->reset_during_compaction(); } + hr->reset_during_compaction(); } else if (!hr->is_pinned()) { hr->compact(); } @@ -334,9 +334,6 @@ HeapWord* end = hr->end(); FreeRegionList dummy_free_list("Dummy Free List for G1MarkSweep"); - assert(hr->is_starts_humongous(), - "Only the start of a humongous region should be freed."); - hr->set_containing_set(NULL); _humongous_regions_removed.increment(1u, hr->capacity()); @@ -373,8 +370,9 @@ bool G1PrepareCompactClosure::doHeapRegion(HeapRegion* hr) { if (hr->is_humongous()) { + oop obj; if (hr->is_starts_humongous()) { - oop obj = oop(hr->bottom()); + obj = oop(hr->bottom()); if (obj->is_gc_marked()) { obj->forward_to(obj); } else { @@ -382,6 +380,10 @@ } } else { assert(hr->is_continues_humongous(), "Invalid humongous."); + obj = oop(hr->humongous_start_region()->bottom()); + if (!obj->is_gc_marked()) { + free_humongous_region(hr); + } } } else if (!hr->is_pinned()) { prepare_for_compaction(hr, hr->end()); diff --git a/src/share/vm/gc/g1/g1RemSet.inline.hpp b/src/share/vm/gc/g1/g1RemSet.inline.hpp --- a/src/share/vm/gc/g1/g1RemSet.inline.hpp +++ b/src/share/vm/gc/g1/g1RemSet.inline.hpp @@ -60,8 +60,6 @@ assert(_g1->is_in_reserved(obj), "must be in heap"); #endif // ASSERT - assert(from == NULL || from->is_in_reserved(p), "p is not in from"); - HeapRegion* to = _g1->heap_region_containing(obj); if (from != to) { assert(to->rem_set() != NULL, "Need per-region 'into' remsets."); diff --git a/src/share/vm/gc/g1/heapRegion.cpp b/src/share/vm/gc/g1/heapRegion.cpp --- a/src/share/vm/gc/g1/heapRegion.cpp +++ b/src/share/vm/gc/g1/heapRegion.cpp @@ -67,7 +67,7 @@ // not considered dead, either because it is marked (in the mark bitmap) // or it was allocated after marking finished, then we add it. Otherwise // we can safely ignore the object. - if (!g1h->is_obj_dead(oop(cur), _hr)) { + if (!g1h->is_obj_dead(oop(cur))) { oop_size = oop(cur)->oop_iterate_size(_rs_scan, mr); } else { oop_size = _hr->block_size(cur); @@ -81,7 +81,7 @@ HeapWord* next_obj = cur + oop_size; while (next_obj < top) { // Keep filtering the remembered set. - if (!g1h->is_obj_dead(cur_oop, _hr)) { + if (!g1h->is_obj_dead(cur_oop)) { // Bottom lies entirely below top, so we can call the // non-memRegion version of oop_iterate below. cur_oop->oop_iterate(_rs_scan); @@ -93,7 +93,7 @@ } // Last object. Need to do dead-obj filtering here too. - if (!g1h->is_obj_dead(oop(cur), _hr)) { + if (!g1h->is_obj_dead(oop(cur))) { oop(cur)->oop_iterate(_rs_scan, mr); } } @@ -162,8 +162,6 @@ void HeapRegion::hr_clear(bool par, bool clear_space, bool locked) { assert(_humongous_start_region == NULL, "we should have already filtered out humongous regions"); - assert(_end == orig_end(), - "we should have already filtered out humongous regions"); assert(!in_collection_set(), "Should not clear heap region %u in the collection set", hrm_index()); @@ -213,24 +211,18 @@ _gc_efficiency = (double) reclaimable_bytes() / region_elapsed_time_ms; } -void HeapRegion::set_starts_humongous(HeapWord* new_top, HeapWord* new_end) { +void HeapRegion::set_starts_humongous(HeapWord* obj_top) { assert(!is_humongous(), "sanity / pre-condition"); - assert(end() == orig_end(), - "Should be normal before the humongous object allocation"); assert(top() == bottom(), "should be empty"); - assert(bottom() <= new_top && new_top <= new_end, "pre-condition"); _type.set_starts_humongous(); _humongous_start_region = this; - set_end(new_end); - _offsets.set_for_starts_humongous(new_top); + _offsets.set_for_starts_humongous(obj_top); } void HeapRegion::set_continues_humongous(HeapRegion* first_hr) { assert(!is_humongous(), "sanity / pre-condition"); - assert(end() == orig_end(), - "Should be normal before the humongous object allocation"); assert(top() == bottom(), "should be empty"); assert(first_hr->is_starts_humongous(), "pre-condition"); @@ -241,18 +233,6 @@ void HeapRegion::clear_humongous() { assert(is_humongous(), "pre-condition"); - if (is_starts_humongous()) { - assert(top() <= end(), "pre-condition"); - set_end(orig_end()); - if (top() > end()) { - // at least one "continues humongous" region after it - set_top(end()); - } - } else { - // continues humongous - assert(end() == orig_end(), "sanity"); - } - assert(capacity() == HeapRegion::GrainBytes, "pre-condition"); _humongous_start_region = NULL; } @@ -290,11 +270,6 @@ hr_clear(false /*par*/, false /*clear_space*/); set_top(bottom()); record_timestamp(); - - assert(mr.end() == orig_end(), - "Given region end address " PTR_FORMAT " should match exactly " - "bottom plus one region size, i.e. " PTR_FORMAT, - p2i(mr.end()), p2i(orig_end())); } CompactibleSpace* HeapRegion::next_compaction_space() const { @@ -720,7 +695,8 @@ HeapRegion* to = _g1h->heap_region_containing(obj); if (from != NULL && to != NULL && from != to && - !to->is_pinned()) { + !to->is_pinned() && + !from->is_continues_humongous()) { jbyte cv_obj = *_bs->byte_for_const(_containing_obj); jbyte cv_field = *_bs->byte_for_const(p); const jbyte dirty = CardTableModRefBS::dirty_card_val(); @@ -832,7 +808,7 @@ _offsets.verify(); } - if (p != top()) { + if (!is_region_humongous && p != top()) { gclog_or_tty->print_cr("end of last object " PTR_FORMAT " " "does not match top " PTR_FORMAT, p2i(p), p2i(top())); *failures = true; @@ -840,7 +816,6 @@ } HeapWord* the_end = end(); - assert(p == top(), "it should still hold"); // Do some extra BOT consistency checking for addresses in the // range [top, end). BOT look-ups in this range should yield // top. No point in doing that if top == end (there's nothing there). diff --git a/src/share/vm/gc/g1/heapRegion.hpp b/src/share/vm/gc/g1/heapRegion.hpp --- a/src/share/vm/gc/g1/heapRegion.hpp +++ b/src/share/vm/gc/g1/heapRegion.hpp @@ -389,8 +389,6 @@ size_t garbage_bytes() { size_t used_at_mark_start_bytes = (prev_top_at_mark_start() - bottom()) * HeapWordSize; - assert(used_at_mark_start_bytes >= marked_bytes(), - "Can't mark more than we have."); return used_at_mark_start_bytes - marked_bytes(); } @@ -409,7 +407,6 @@ void add_to_marked_bytes(size_t incr_bytes) { _next_marked_bytes = _next_marked_bytes + incr_bytes; - assert(_next_marked_bytes <= used(), "invariant" ); } void zero_marked_bytes() { @@ -445,57 +442,13 @@ return _humongous_start_region; } - // Return the number of distinct regions that are covered by this region: - // 1 if the region is not humongous, >= 1 if the region is humongous. - uint region_num() const { - if (!is_humongous()) { - return 1U; - } else { - assert(is_starts_humongous(), "doesn't make sense on HC regions"); - assert(capacity() % HeapRegion::GrainBytes == 0, "sanity"); - return (uint) (capacity() >> HeapRegion::LogOfHRGrainBytes); - } - } - - // Return the index + 1 of the last HC regions that's associated - // with this HS region. - uint last_hc_index() const { - assert(is_starts_humongous(), "don't call this otherwise"); - return hrm_index() + region_num(); - } - - // Same as Space::is_in_reserved, but will use the original size of the region. - // The original size is different only for start humongous regions. They get - // their _end set up to be the end of the last continues region of the - // corresponding humongous object. - bool is_in_reserved_raw(const void* p) const { - return _bottom <= p && p < orig_end(); - } - // Makes the current region be a "starts humongous" region, i.e., // the first region in a series of one or more contiguous regions - // that will contain a single "humongous" object. The two parameters - // are as follows: + // that will contain a single "humongous" object. // - // new_top : The new value of the top field of this region which - // points to the end of the humongous object that's being - // allocated. If there is more than one region in the series, top - // will lie beyond this region's original end field and on the last - // region in the series. - // - // new_end : The new value of the end field of this region which - // points to the end of the last region in the series. If there is - // one region in the series (namely: this one) end will be the same - // as the original end of this region. - // - // Updating top and end as described above makes this region look as - // if it spans the entire space taken up by all the regions in the - // series and an single allocation moved its top to new_top. This - // ensures that the space (capacity / allocated) taken up by all - // humongous regions can be calculated by just looking at the - // "starts humongous" regions and by ignoring the "continues - // humongous" regions. - void set_starts_humongous(HeapWord* new_top, HeapWord* new_end); + // obj_top : points to the end of the humongous object that's being + // allocated. + void set_starts_humongous(HeapWord* obj_top); // Makes the current region be a "continues humongous' // region. first_hr is the "start humongous" region of the series @@ -566,9 +519,6 @@ void set_next_dirty_cards_region(HeapRegion* hr) { _next_dirty_cards_region = hr; } bool is_on_dirty_cards_region_list() const { return get_next_dirty_cards_region() != NULL; } - // For the start region of a humongous sequence, it's original end(). - HeapWord* orig_end() const { return _bottom + GrainWords; } - // Reset HR stuff to default values. void hr_clear(bool par, bool clear_space, bool locked = false); void par_clear(); @@ -614,9 +564,6 @@ bool is_marked() { return _prev_top_at_mark_start != bottom(); } void reset_during_compaction() { - assert(is_starts_humongous(), - "should only be called for starts humongous regions"); - zero_marked_bytes(); init_top_at_mark_start(); } diff --git a/src/share/vm/gc/g1/heapRegion.inline.hpp b/src/share/vm/gc/g1/heapRegion.inline.hpp --- a/src/share/vm/gc/g1/heapRegion.inline.hpp +++ b/src/share/vm/gc/g1/heapRegion.inline.hpp @@ -115,6 +115,11 @@ inline bool HeapRegion::block_is_obj(const HeapWord* p) const { G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + if (!this->is_in(p)) { + HeapRegion* hr = g1h->heap_region_containing_raw(p); + return hr->block_is_obj(p); + } if (ClassUnloadingWithConcurrentMark) { return !g1h->is_obj_dead(oop(p), this); } @@ -176,10 +181,6 @@ _prev_top_at_mark_start = _next_top_at_mark_start; _prev_marked_bytes = _next_marked_bytes; _next_marked_bytes = 0; - - assert(_prev_marked_bytes <= - (size_t) pointer_delta(prev_top_at_mark_start(), bottom()) * - HeapWordSize, "invariant"); } inline void HeapRegion::note_start_of_copying(bool during_initial_mark) { diff --git a/src/share/vm/gc/g1/heapRegionManager.cpp b/src/share/vm/gc/g1/heapRegionManager.cpp --- a/src/share/vm/gc/g1/heapRegionManager.cpp +++ b/src/share/vm/gc/g1/heapRegionManager.cpp @@ -343,63 +343,18 @@ continue; } HeapRegion* r = _regions.get_by_index(index); - // We'll ignore "continues humongous" regions (we'll process them - // when we come across their corresponding "start humongous" - // region) and regions already claimed. + // We'll ignore regions already claimed. // However, if the iteration is specified as concurrent, the values for // is_starts_humongous and is_continues_humongous can not be trusted, // and we should just blindly iterate over regions regardless of their // humongous status. - if (hrclaimer->is_region_claimed(index) || (!concurrent && r->is_continues_humongous())) { + if (hrclaimer->is_region_claimed(index)) { continue; } // OK, try to claim it if (!hrclaimer->claim_region(index)) { continue; } - // Success! - // As mentioned above, special treatment of humongous regions can only be - // done if we are iterating non-concurrently. - if (!concurrent && r->is_starts_humongous()) { - // If the region is "starts humongous" we'll iterate over its - // "continues humongous" first; in fact we'll do them - // first. The order is important. In one case, calling the - // closure on the "starts humongous" region might de-allocate - // and clear all its "continues humongous" regions and, as a - // result, we might end up processing them twice. So, we'll do - // them first (note: most closures will ignore them anyway) and - // then we'll do the "starts humongous" region. - for (uint ch_index = index + 1; ch_index < index + r->region_num(); ch_index++) { - HeapRegion* chr = _regions.get_by_index(ch_index); - - assert(chr->is_continues_humongous(), "Must be humongous region"); - assert(chr->humongous_start_region() == r, - "Must work on humongous continuation of the original start region " - PTR_FORMAT ", but is " PTR_FORMAT, p2i(r), p2i(chr)); - assert(!hrclaimer->is_region_claimed(ch_index), - "Must not have been claimed yet because claiming of humongous continuation first claims the start region"); - - // Claim the region so no other worker tries to process the region. When a worker processes a - // starts_humongous region it may also process the associated continues_humongous regions. - // The continues_humongous regions can be changed to free regions. Unless this worker claims - // all of these regions, other workers might try claim and process these newly free regions. - bool claim_result = hrclaimer->claim_region(ch_index); - guarantee(claim_result, "We should always be able to claim the continuesHumongous part of the humongous object"); - - bool res2 = blk->doHeapRegion(chr); - if (res2) { - return; - } - - // Right now, this holds (i.e., no closure that actually - // does something with "continues humongous" regions - // clears them). We might have to weaken it in the future, - // but let's leave these two asserts here for extra safety. - assert(chr->is_continues_humongous(), "should still be the case"); - assert(chr->humongous_start_region() == r, "sanity"); - } - } - bool res = blk->doHeapRegion(r); if (res) { return; @@ -508,11 +463,7 @@ // this method may be called, we have only completed allocation of the regions, // but not put into a region set. prev_committed = true; - if (hr->is_starts_humongous()) { - prev_end = hr->orig_end(); - } else { - prev_end = hr->end(); - } + prev_end = hr->end(); } for (uint i = _allocated_heapregions_length; i < max_length(); i++) { guarantee(_regions.get_by_index(i) == NULL, "invariant i: %u", i); diff --git a/src/share/vm/gc/g1/heapRegionRemSet.cpp b/src/share/vm/gc/g1/heapRegionRemSet.cpp --- a/src/share/vm/gc/g1/heapRegionRemSet.cpp +++ b/src/share/vm/gc/g1/heapRegionRemSet.cpp @@ -105,7 +105,7 @@ // now reused for the corresponding start humongous region, we need to // make sure that we detect this. Thus, we call is_in_reserved_raw() // instead of just is_in_reserved() here. - if (loc_hr->is_in_reserved_raw(from)) { + if (loc_hr->is_in_reserved(from)) { size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom()); CardIdx_t from_card = (CardIdx_t) hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize);