--- old/src/share/vm/gc/shenandoah/shenandoahHeap.cpp 2017-10-09 11:48:33.695332613 +0200 +++ new/src/share/vm/gc/shenandoah/shenandoahHeap.cpp 2017-10-09 11:48:33.571332308 +0200 @@ -82,11 +82,10 @@ ShenandoahHeapRegionSet* _regions; const size_t _bitmap_size; const size_t _page_size; - char* _bitmap0_base; - char* _bitmap1_base; + char* _bitmap_base; public: ShenandoahPretouchTask(ShenandoahHeapRegionSet* regions, - char* bitmap0_base, char* bitmap1_base, size_t bitmap_size, + char* bitmap_base, size_t bitmap_size, size_t page_size) : AbstractGangTask("Shenandoah PreTouch", Universe::is_fully_initialized() ? GCId::current_raw() : @@ -94,8 +93,7 @@ // no GC cycle that this task can be // associated with. GCId::undefined()), - _bitmap0_base(bitmap0_base), - _bitmap1_base(bitmap1_base), + _bitmap_base(bitmap_base), _regions(regions), _bitmap_size(bitmap_size), _page_size(page_size) { @@ -114,12 +112,8 @@ assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size); log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT, - r->region_number(), p2i(_bitmap0_base + start), p2i(_bitmap0_base + end)); - os::pretouch_memory(_bitmap0_base + start, _bitmap0_base + end, _page_size); - - log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT, - r->region_number(), p2i(_bitmap1_base + start), p2i(_bitmap1_base + end)); - os::pretouch_memory(_bitmap1_base + start, _bitmap1_base + end, _page_size); + r->region_number(), p2i(_bitmap_base + start), p2i(_bitmap_base + end)); + os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size); r = _regions->claim_next(); } @@ -171,13 +165,10 @@ _collection_set = new ShenandoahCollectionSet(this, (HeapWord*)pgc_rs.base()); - _next_top_at_mark_starts_base = NEW_C_HEAP_ARRAY(HeapWord*, _num_regions, mtGC); - _next_top_at_mark_starts = _next_top_at_mark_starts_base - + _top_at_mark_starts_base = NEW_C_HEAP_ARRAY(HeapWord*, _num_regions, mtGC); + _top_at_mark_starts = _top_at_mark_starts_base - ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::region_size_bytes_shift()); - _complete_top_at_mark_starts_base = NEW_C_HEAP_ARRAY(HeapWord*, _num_regions, mtGC); - _complete_top_at_mark_starts = _complete_top_at_mark_starts_base - - ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::region_size_bytes_shift()); { ShenandoahHeapLocker locker(lock()); @@ -188,8 +179,7 @@ i, i < num_committed_regions); - _complete_top_at_mark_starts_base[i] = r->bottom(); - _next_top_at_mark_starts_base[i] = r->bottom(); + _top_at_mark_starts_base[i] = r->bottom(); // Add to ordered regions first. // We use the active size of ordered regions as the number of active regions in heap, @@ -244,13 +234,9 @@ size_t bitmap_page_size = UseLargePages && (bitmap_bytes_per_region >= (size_t)os::large_page_size()) ? (size_t)os::large_page_size() : (size_t)os::vm_page_size(); - ReservedSpace bitmap0(_bitmap_size, bitmap_page_size); - MemTracker::record_virtual_memory_type(bitmap0.base(), mtGC); - _bitmap0_region = MemRegion((HeapWord*) bitmap0.base(), bitmap0.size() / HeapWordSize); - - ReservedSpace bitmap1(_bitmap_size, bitmap_page_size); - MemTracker::record_virtual_memory_type(bitmap1.base(), mtGC); - _bitmap1_region = MemRegion((HeapWord*) bitmap1.base(), bitmap1.size() / HeapWordSize); + ReservedSpace bitmap(_bitmap_size, bitmap_page_size); + MemTracker::record_virtual_memory_type(bitmap.base(), mtGC); + _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize); { ShenandoahHeapLocker locker(lock()); @@ -283,15 +269,11 @@ log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages", _ordered_regions->count(), page_size); - ShenandoahPretouchTask cl(_ordered_regions, bitmap0.base(), bitmap1.base(), _bitmap_size, page_size); + ShenandoahPretouchTask cl(_ordered_regions, bitmap.base(), _bitmap_size, page_size); _workers->run_task(&cl); } - _mark_bit_map0.initialize(_heap_region, _bitmap0_region); - _complete_mark_bit_map = &_mark_bit_map0; - - _mark_bit_map1.initialize(_heap_region, _bitmap1_region); - _next_mark_bit_map = &_mark_bit_map1; + _mark_bit_map.initialize(_heap_region, _bitmap_region); if (UseShenandoahMatrix) { _connection_matrix = new ShenandoahConnectionMatrix(_num_regions); @@ -337,16 +319,14 @@ _used_start_gc(0), _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)), _ref_processor(NULL), - _next_top_at_mark_starts(NULL), - _next_top_at_mark_starts_base(NULL), - _complete_top_at_mark_starts(NULL), - _complete_top_at_mark_starts_base(NULL), - _mark_bit_map0(), - _mark_bit_map1(), + _top_at_mark_starts(NULL), + _top_at_mark_starts_base(NULL), + _mark_bit_map(), _connection_matrix(NULL), _cancelled_concgc(0), _need_update_refs(false), - _need_reset_bitmaps(false), + _need_reset_bitmap(false), + _bitmap_valid(true), _verifier(NULL), _heap_lock(0), _used_at_last_gc(0), @@ -386,12 +366,12 @@ } } -class ShenandoahResetNextBitmapTask : public AbstractGangTask { +class ShenandoahResetBitmapTask : public AbstractGangTask { private: ShenandoahHeapRegionSet* _regions; public: - ShenandoahResetNextBitmapTask(ShenandoahHeapRegionSet* regions) : + ShenandoahResetBitmapTask(ShenandoahHeapRegionSet* regions) : AbstractGangTask("Parallel Reset Bitmap Task"), _regions(regions) { _regions->clear_current_index(); @@ -403,75 +383,37 @@ while (region != NULL) { if (region->is_committed()) { HeapWord* bottom = region->bottom(); - HeapWord* top = heap->next_top_at_mark_start(region->bottom()); + HeapWord* top = heap->top_at_mark_start(region->bottom()); if (top > bottom) { - heap->next_mark_bit_map()->clear_range_large(MemRegion(bottom, top)); + heap->mark_bit_map()->clear_range_large(MemRegion(bottom, top)); } - assert(heap->is_next_bitmap_clear_range(bottom, region->end()), "must be clear"); + assert(heap->is_bitmap_clear_range(bottom, region->end()), "must be clear"); + heap->set_top_at_mark_start(region->bottom(), region->bottom()); } region = _regions->claim_next(); } } }; -void ShenandoahHeap::reset_next_mark_bitmap(WorkGang* workers) { +void ShenandoahHeap::reset_mark_bitmap(WorkGang* workers) { assert_gc_workers(workers->active_workers()); - ShenandoahResetNextBitmapTask task = ShenandoahResetNextBitmapTask(_ordered_regions); + ShenandoahResetBitmapTask task = ShenandoahResetBitmapTask(_ordered_regions); workers->run_task(&task); } -class ShenandoahResetCompleteBitmapTask : public AbstractGangTask { -private: - ShenandoahHeapRegionSet* _regions; - -public: - ShenandoahResetCompleteBitmapTask(ShenandoahHeapRegionSet* regions) : - AbstractGangTask("Parallel Reset Bitmap Task"), - _regions(regions) { - _regions->clear_current_index(); - } - - void work(uint worker_id) { - ShenandoahHeapRegion* region = _regions->claim_next(); - ShenandoahHeap* heap = ShenandoahHeap::heap(); - while (region != NULL) { - if (region->is_committed()) { - HeapWord* bottom = region->bottom(); - HeapWord* top = heap->complete_top_at_mark_start(region->bottom()); - if (top > bottom) { - heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top)); - } - assert(heap->is_complete_bitmap_clear_range(bottom, region->end()), "must be clear"); - } - region = _regions->claim_next(); - } - } -}; - -void ShenandoahHeap::reset_complete_mark_bitmap(WorkGang* workers) { - assert_gc_workers(workers->active_workers()); - - ShenandoahResetCompleteBitmapTask task = ShenandoahResetCompleteBitmapTask(_ordered_regions); - workers->run_task(&task); -} - -bool ShenandoahHeap::is_next_bitmap_clear() { +bool ShenandoahHeap::is_bitmap_clear() { for (size_t idx = 0; idx < _num_regions; idx++) { ShenandoahHeapRegion* r = _ordered_regions->get(idx); - if (r->is_committed() && !is_next_bitmap_clear_range(r->bottom(), r->end())) { + if (r->is_committed() && !is_bitmap_clear_range(r->bottom(), r->end())) { return false; } } return true; } -bool ShenandoahHeap::is_next_bitmap_clear_range(HeapWord* start, HeapWord* end) { - return _next_mark_bit_map->getNextMarkedWordAddress(start, end) == end; -} - -bool ShenandoahHeap::is_complete_bitmap_clear_range(HeapWord* start, HeapWord* end) { - return _complete_mark_bit_map->getNextMarkedWordAddress(start, end) == end; +bool ShenandoahHeap::is_bitmap_clear_range(HeapWord* start, HeapWord* end) { + return _mark_bit_map.getNextMarkedWordAddress(start, end) == end; } void ShenandoahHeap::print_on(outputStream* st) const { @@ -787,8 +729,8 @@ if (! oopDesc::is_null(o)) { oop obj = oopDesc::decode_heap_oop_not_null(o); if (_heap->in_collection_set(obj)) { - assert(_heap->is_marked_complete(obj), "only evacuate marked objects %d %d", - _heap->is_marked_complete(obj), _heap->is_marked_complete(ShenandoahBarrierSet::resolve_oop_static_not_null(obj))); + assert(_heap->is_marked(obj), "only evacuate marked objects %d %d", + _heap->is_marked(obj), _heap->is_marked(ShenandoahBarrierSet::resolve_oop_static_not_null(obj))); oop resolved = ShenandoahBarrierSet::resolve_oop_static_not_null(obj); if (oopDesc::unsafe_equals(resolved, obj)) { bool evac; @@ -851,7 +793,7 @@ _heap(heap), _thread(Thread::current()) {} void do_object(oop p) { - assert(_heap->is_marked_complete(p), "expect only marked objects"); + assert(_heap->is_marked(p), "expect only marked objects"); if (oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p))) { bool evac; _heap->evacuate_object(p, _thread, evac); @@ -1454,7 +1396,7 @@ bool heap_region_do(ShenandoahHeapRegion* r) { r->clear_live_data(); - sh->set_next_top_at_mark_start(r->bottom(), r->top()); + sh->set_top_at_mark_start(r->bottom(), r->top()); return false; } }; @@ -1496,30 +1438,12 @@ } } -void ShenandoahHeap::swap_mark_bitmaps() { - // Swap bitmaps. - MarkBitMap* tmp1 = _complete_mark_bit_map; - _complete_mark_bit_map = _next_mark_bit_map; - _next_mark_bit_map = tmp1; - - // Swap top-at-mark-start pointers - HeapWord** tmp2 = _complete_top_at_mark_starts; - _complete_top_at_mark_starts = _next_top_at_mark_starts; - _next_top_at_mark_starts = tmp2; - - HeapWord** tmp3 = _complete_top_at_mark_starts_base; - _complete_top_at_mark_starts_base = _next_top_at_mark_starts_base; - _next_top_at_mark_starts_base = tmp3; -} - - void ShenandoahHeap::stop_concurrent_marking() { assert(concurrent_mark_in_progress(), "How else could we get here?"); if (! cancelled_concgc()) { // If we needed to update refs, and concurrent marking has been cancelled, // we need to finish updating references. set_need_update_refs(false); - swap_mark_bitmaps(); } set_concurrent_mark_in_progress(false); @@ -1601,7 +1525,7 @@ } #endif assert(!oopDesc::is_null(obj), "null"); - return _heap->is_marked_next(obj); + return _heap->is_marked(obj); } ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() : @@ -1612,7 +1536,7 @@ assert(_heap != NULL, "sanity"); assert(!oopDesc::is_null(obj), "null"); assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only query to-space"); - return _heap->is_marked_next(obj); + return _heap->is_marked(obj); } BoolObjectClosure* ShenandoahHeap::is_alive_closure() { @@ -1799,12 +1723,8 @@ return _monitoring_support; } -MarkBitMap* ShenandoahHeap::complete_mark_bit_map() { - return _complete_mark_bit_map; -} - -MarkBitMap* ShenandoahHeap::next_mark_bit_map() { - return _next_mark_bit_map; +MarkBitMap* ShenandoahHeap::mark_bit_map() { + return &_mark_bit_map; } void ShenandoahHeap::add_free_region(ShenandoahHeapRegion* r) { @@ -1838,24 +1758,14 @@ _bytes_allocated_since_cm = bytes; } -void ShenandoahHeap::set_next_top_at_mark_start(HeapWord* region_base, HeapWord* addr) { - uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_bytes_shift(); - _next_top_at_mark_starts[index] = addr; -} - -HeapWord* ShenandoahHeap::next_top_at_mark_start(HeapWord* region_base) { - uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_bytes_shift(); - return _next_top_at_mark_starts[index]; -} - -void ShenandoahHeap::set_complete_top_at_mark_start(HeapWord* region_base, HeapWord* addr) { +void ShenandoahHeap::set_top_at_mark_start(HeapWord* region_base, HeapWord* addr) { uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_bytes_shift(); - _complete_top_at_mark_starts[index] = addr; + _top_at_mark_starts[index] = addr; } -HeapWord* ShenandoahHeap::complete_top_at_mark_start(HeapWord* region_base) { +HeapWord* ShenandoahHeap::top_at_mark_start(HeapWord* region_base) { uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_bytes_shift(); - return _complete_top_at_mark_starts[index]; + return _top_at_mark_starts[index]; } void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) { @@ -1985,9 +1895,9 @@ while (r != NULL) { if (_heap->in_collection_set(r)) { HeapWord* bottom = r->bottom(); - HeapWord* top = _heap->complete_top_at_mark_start(r->bottom()); + HeapWord* top = _heap->top_at_mark_start(r->bottom()); if (top > bottom) { - _heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top)); + _heap->mark_bit_map()->clear_range_large(MemRegion(bottom, top)); } } else { if (r->is_active()) { @@ -2189,10 +2099,7 @@ bool ShenandoahHeap::commit_bitmaps(ShenandoahHeapRegion* r) { size_t len = _bitmap_words_per_region * HeapWordSize; size_t off = r->region_number() * _bitmap_words_per_region; - if (!os::commit_memory((char*)(_bitmap0_region.start() + off), len, false)) { - return false; - } - if (!os::commit_memory((char*)(_bitmap1_region.start() + off), len, false)) { + if (!os::commit_memory((char*)(_bitmap_region.start() + off), len, false)) { return false; } return true; @@ -2201,10 +2108,7 @@ bool ShenandoahHeap::uncommit_bitmaps(ShenandoahHeapRegion* r) { size_t len = _bitmap_words_per_region * HeapWordSize; size_t off = r->region_number() * _bitmap_words_per_region; - if (!os::uncommit_memory((char*)(_bitmap0_region.start() + off), len)) { - return false; - } - if (!os::uncommit_memory((char*)(_bitmap1_region.start() + off), len)) { + if (!os::uncommit_memory((char*)(_bitmap_region.start() + off), len)) { return false; } return true;