< prev index next >
src/share/vm/gc/shenandoah/shenandoahHeap.cpp
Print this page
rev 14453 : Remove secondary marking bitmap.
*** 80,103 ****
class ShenandoahPretouchTask : public AbstractGangTask {
private:
ShenandoahHeapRegionSet* _regions;
const size_t _bitmap_size;
const size_t _page_size;
! char* _bitmap0_base;
! char* _bitmap1_base;
public:
ShenandoahPretouchTask(ShenandoahHeapRegionSet* regions,
! char* bitmap0_base, char* bitmap1_base, size_t bitmap_size,
size_t page_size) :
AbstractGangTask("Shenandoah PreTouch",
Universe::is_fully_initialized() ? GCId::current_raw() :
// During VM initialization there is
// no GC cycle that this task can be
// associated with.
GCId::undefined()),
! _bitmap0_base(bitmap0_base),
! _bitmap1_base(bitmap1_base),
_regions(regions),
_bitmap_size(bitmap_size),
_page_size(page_size) {
_regions->clear_current_index();
};
--- 80,101 ----
class ShenandoahPretouchTask : public AbstractGangTask {
private:
ShenandoahHeapRegionSet* _regions;
const size_t _bitmap_size;
const size_t _page_size;
! char* _bitmap_base;
public:
ShenandoahPretouchTask(ShenandoahHeapRegionSet* regions,
! char* bitmap_base, size_t bitmap_size,
size_t page_size) :
AbstractGangTask("Shenandoah PreTouch",
Universe::is_fully_initialized() ? GCId::current_raw() :
// During VM initialization there is
// no GC cycle that this task can be
// associated with.
GCId::undefined()),
! _bitmap_base(bitmap_base),
_regions(regions),
_bitmap_size(bitmap_size),
_page_size(page_size) {
_regions->clear_current_index();
};
*** 112,127 ****
size_t start = r->region_number() * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
size_t end = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
! r->region_number(), p2i(_bitmap0_base + start), p2i(_bitmap0_base + end));
! os::pretouch_memory(_bitmap0_base + start, _bitmap0_base + end, _page_size);
!
! log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
! r->region_number(), p2i(_bitmap1_base + start), p2i(_bitmap1_base + end));
! os::pretouch_memory(_bitmap1_base + start, _bitmap1_base + end, _page_size);
r = _regions->claim_next();
}
}
};
--- 110,121 ----
size_t start = r->region_number() * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
size_t end = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
! r->region_number(), p2i(_bitmap_base + start), p2i(_bitmap_base + end));
! os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
r = _regions->claim_next();
}
}
};
*** 169,197 ****
_ordered_regions = new ShenandoahHeapRegionSet(_num_regions);
_free_regions = new ShenandoahFreeSet(_ordered_regions, _num_regions);
_collection_set = new ShenandoahCollectionSet(this, (HeapWord*)pgc_rs.base());
! _next_top_at_mark_starts_base = NEW_C_HEAP_ARRAY(HeapWord*, _num_regions, mtGC);
! _next_top_at_mark_starts = _next_top_at_mark_starts_base -
((uintx) pgc_rs.base() >> ShenandoahHeapRegion::region_size_bytes_shift());
- _complete_top_at_mark_starts_base = NEW_C_HEAP_ARRAY(HeapWord*, _num_regions, mtGC);
- _complete_top_at_mark_starts = _complete_top_at_mark_starts_base -
- ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::region_size_bytes_shift());
{
ShenandoahHeapLocker locker(lock());
for (size_t i = 0; i < _num_regions; i++) {
ShenandoahHeapRegion* r = new ShenandoahHeapRegion(this,
(HeapWord*) pgc_rs.base() + reg_size_words * i,
reg_size_words,
i,
i < num_committed_regions);
! _complete_top_at_mark_starts_base[i] = r->bottom();
! _next_top_at_mark_starts_base[i] = r->bottom();
// Add to ordered regions first.
// We use the active size of ordered regions as the number of active regions in heap,
// free set and collection set use the number to assert the correctness of incoming regions.
_ordered_regions->add_region(r);
--- 163,187 ----
_ordered_regions = new ShenandoahHeapRegionSet(_num_regions);
_free_regions = new ShenandoahFreeSet(_ordered_regions, _num_regions);
_collection_set = new ShenandoahCollectionSet(this, (HeapWord*)pgc_rs.base());
! _top_at_mark_starts_base = NEW_C_HEAP_ARRAY(HeapWord*, _num_regions, mtGC);
! _top_at_mark_starts = _top_at_mark_starts_base -
((uintx) pgc_rs.base() >> ShenandoahHeapRegion::region_size_bytes_shift());
{
ShenandoahHeapLocker locker(lock());
for (size_t i = 0; i < _num_regions; i++) {
ShenandoahHeapRegion* r = new ShenandoahHeapRegion(this,
(HeapWord*) pgc_rs.base() + reg_size_words * i,
reg_size_words,
i,
i < num_committed_regions);
! _top_at_mark_starts_base[i] = r->bottom();
// Add to ordered regions first.
// We use the active size of ordered regions as the number of active regions in heap,
// free set and collection set use the number to assert the correctness of incoming regions.
_ordered_regions->add_region(r);
*** 241,257 ****
"Bitmap words per region Should be power of two: " SIZE_FORMAT, _bitmap_words_per_region);
size_t bitmap_page_size = UseLargePages && (bitmap_bytes_per_region >= (size_t)os::large_page_size()) ?
(size_t)os::large_page_size() : (size_t)os::vm_page_size();
! ReservedSpace bitmap0(_bitmap_size, bitmap_page_size);
! MemTracker::record_virtual_memory_type(bitmap0.base(), mtGC);
! _bitmap0_region = MemRegion((HeapWord*) bitmap0.base(), bitmap0.size() / HeapWordSize);
!
! ReservedSpace bitmap1(_bitmap_size, bitmap_page_size);
! MemTracker::record_virtual_memory_type(bitmap1.base(), mtGC);
! _bitmap1_region = MemRegion((HeapWord*) bitmap1.base(), bitmap1.size() / HeapWordSize);
{
ShenandoahHeapLocker locker(lock());
for (size_t i = 0; i < _num_regions; i++) {
ShenandoahHeapRegion* r = _ordered_regions->get(i);
--- 231,243 ----
"Bitmap words per region Should be power of two: " SIZE_FORMAT, _bitmap_words_per_region);
size_t bitmap_page_size = UseLargePages && (bitmap_bytes_per_region >= (size_t)os::large_page_size()) ?
(size_t)os::large_page_size() : (size_t)os::vm_page_size();
! ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
! MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
! _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
{
ShenandoahHeapLocker locker(lock());
for (size_t i = 0; i < _num_regions; i++) {
ShenandoahHeapRegion* r = _ordered_regions->get(i);
*** 280,298 ****
// before initialize() below zeroes it with initializing thread. For any given region,
// we touch the region and the corresponding bitmaps from the same thread.
log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages",
_ordered_regions->count(), page_size);
! ShenandoahPretouchTask cl(_ordered_regions, bitmap0.base(), bitmap1.base(), _bitmap_size, page_size);
_workers->run_task(&cl);
}
! _mark_bit_map0.initialize(_heap_region, _bitmap0_region);
! _complete_mark_bit_map = &_mark_bit_map0;
! _mark_bit_map1.initialize(_heap_region, _bitmap1_region);
! _next_mark_bit_map = &_mark_bit_map1;
if (UseShenandoahMatrix) {
_connection_matrix = new ShenandoahConnectionMatrix(_num_regions);
} else {
_connection_matrix = NULL;
--- 266,286 ----
// before initialize() below zeroes it with initializing thread. For any given region,
// we touch the region and the corresponding bitmaps from the same thread.
log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages",
_ordered_regions->count(), page_size);
! ShenandoahPretouchTask cl(_ordered_regions, bitmap.base(), _bitmap_size, page_size);
_workers->run_task(&cl);
}
! _mark_bit_map.initialize(_heap_region, _bitmap_region);
! // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
! ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
! MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
! _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
! _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
if (UseShenandoahMatrix) {
_connection_matrix = new ShenandoahConnectionMatrix(_num_regions);
} else {
_connection_matrix = NULL;
*** 335,354 ****
_bytes_allocated_during_cm(0),
_allocated_last_gc(0),
_used_start_gc(0),
_max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
_ref_processor(NULL),
! _next_top_at_mark_starts(NULL),
! _next_top_at_mark_starts_base(NULL),
! _complete_top_at_mark_starts(NULL),
! _complete_top_at_mark_starts_base(NULL),
! _mark_bit_map0(),
! _mark_bit_map1(),
_connection_matrix(NULL),
_cancelled_concgc(0),
_need_update_refs(false),
! _need_reset_bitmaps(false),
_verifier(NULL),
_heap_lock(0),
_used_at_last_gc(0),
_alloc_seq_at_last_gc_start(0),
_alloc_seq_at_last_gc_end(0),
--- 323,341 ----
_bytes_allocated_during_cm(0),
_allocated_last_gc(0),
_used_start_gc(0),
_max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
_ref_processor(NULL),
! _top_at_mark_starts(NULL),
! _top_at_mark_starts_base(NULL),
! _mark_bit_map(),
! _aux_bit_map(),
_connection_matrix(NULL),
_cancelled_concgc(0),
_need_update_refs(false),
! _need_reset_bitmap(false),
! _bitmap_valid(true),
_verifier(NULL),
_heap_lock(0),
_used_at_last_gc(0),
_alloc_seq_at_last_gc_start(0),
_alloc_seq_at_last_gc_end(0),
*** 384,399 ****
false, false);
_safepoint_workers->initialize_workers();
}
}
! class ShenandoahResetNextBitmapTask : public AbstractGangTask {
private:
ShenandoahHeapRegionSet* _regions;
public:
! ShenandoahResetNextBitmapTask(ShenandoahHeapRegionSet* regions) :
AbstractGangTask("Parallel Reset Bitmap Task"),
_regions(regions) {
_regions->clear_current_index();
}
--- 371,386 ----
false, false);
_safepoint_workers->initialize_workers();
}
}
! class ShenandoahResetBitmapTask : public AbstractGangTask {
private:
ShenandoahHeapRegionSet* _regions;
public:
! ShenandoahResetBitmapTask(ShenandoahHeapRegionSet* regions) :
AbstractGangTask("Parallel Reset Bitmap Task"),
_regions(regions) {
_regions->clear_current_index();
}
*** 401,479 ****
ShenandoahHeapRegion* region = _regions->claim_next();
ShenandoahHeap* heap = ShenandoahHeap::heap();
while (region != NULL) {
if (region->is_committed()) {
HeapWord* bottom = region->bottom();
! HeapWord* top = heap->next_top_at_mark_start(region->bottom());
if (top > bottom) {
! heap->next_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
}
! assert(heap->is_next_bitmap_clear_range(bottom, region->end()), "must be clear");
}
region = _regions->claim_next();
}
}
};
! void ShenandoahHeap::reset_next_mark_bitmap(WorkGang* workers) {
assert_gc_workers(workers->active_workers());
! ShenandoahResetNextBitmapTask task = ShenandoahResetNextBitmapTask(_ordered_regions);
workers->run_task(&task);
}
! class ShenandoahResetCompleteBitmapTask : public AbstractGangTask {
! private:
! ShenandoahHeapRegionSet* _regions;
!
! public:
! ShenandoahResetCompleteBitmapTask(ShenandoahHeapRegionSet* regions) :
! AbstractGangTask("Parallel Reset Bitmap Task"),
! _regions(regions) {
! _regions->clear_current_index();
! }
!
! void work(uint worker_id) {
! ShenandoahHeapRegion* region = _regions->claim_next();
! ShenandoahHeap* heap = ShenandoahHeap::heap();
! while (region != NULL) {
! if (region->is_committed()) {
! HeapWord* bottom = region->bottom();
! HeapWord* top = heap->complete_top_at_mark_start(region->bottom());
! if (top > bottom) {
! heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
! }
! assert(heap->is_complete_bitmap_clear_range(bottom, region->end()), "must be clear");
! }
! region = _regions->claim_next();
! }
! }
! };
!
! void ShenandoahHeap::reset_complete_mark_bitmap(WorkGang* workers) {
! assert_gc_workers(workers->active_workers());
!
! ShenandoahResetCompleteBitmapTask task = ShenandoahResetCompleteBitmapTask(_ordered_regions);
! workers->run_task(&task);
! }
!
! bool ShenandoahHeap::is_next_bitmap_clear() {
for (size_t idx = 0; idx < _num_regions; idx++) {
ShenandoahHeapRegion* r = _ordered_regions->get(idx);
! if (r->is_committed() && !is_next_bitmap_clear_range(r->bottom(), r->end())) {
return false;
}
}
return true;
}
! bool ShenandoahHeap::is_next_bitmap_clear_range(HeapWord* start, HeapWord* end) {
! return _next_mark_bit_map->getNextMarkedWordAddress(start, end) == end;
! }
!
! bool ShenandoahHeap::is_complete_bitmap_clear_range(HeapWord* start, HeapWord* end) {
! return _complete_mark_bit_map->getNextMarkedWordAddress(start, end) == end;
}
void ShenandoahHeap::print_on(outputStream* st) const {
st->print_cr("Shenandoah Heap");
st->print_cr(" " SIZE_FORMAT "K total, " SIZE_FORMAT "K committed, " SIZE_FORMAT "K used",
--- 388,428 ----
ShenandoahHeapRegion* region = _regions->claim_next();
ShenandoahHeap* heap = ShenandoahHeap::heap();
while (region != NULL) {
if (region->is_committed()) {
HeapWord* bottom = region->bottom();
! HeapWord* top = heap->top_at_mark_start(region->bottom());
if (top > bottom) {
! heap->mark_bit_map()->clear_range_large(MemRegion(bottom, top));
}
! assert(heap->is_bitmap_clear_range(bottom, region->end()), "must be clear");
! heap->set_top_at_mark_start(region->bottom(), region->bottom());
}
region = _regions->claim_next();
}
}
};
! void ShenandoahHeap::reset_mark_bitmap(WorkGang* workers) {
assert_gc_workers(workers->active_workers());
! ShenandoahResetBitmapTask task = ShenandoahResetBitmapTask(_ordered_regions);
workers->run_task(&task);
}
! bool ShenandoahHeap::is_bitmap_clear() {
for (size_t idx = 0; idx < _num_regions; idx++) {
ShenandoahHeapRegion* r = _ordered_regions->get(idx);
! if (r->is_committed() && !is_bitmap_clear_range(r->bottom(), r->end())) {
return false;
}
}
return true;
}
! bool ShenandoahHeap::is_bitmap_clear_range(HeapWord* start, HeapWord* end) {
! return _mark_bit_map.getNextMarkedWordAddress(start, end) == end;
}
void ShenandoahHeap::print_on(outputStream* st) const {
st->print_cr("Shenandoah Heap");
st->print_cr(" " SIZE_FORMAT "K total, " SIZE_FORMAT "K committed, " SIZE_FORMAT "K used",
*** 785,796 ****
T o = oopDesc::load_heap_oop(p);
if (! oopDesc::is_null(o)) {
oop obj = oopDesc::decode_heap_oop_not_null(o);
if (_heap->in_collection_set(obj)) {
! assert(_heap->is_marked_complete(obj), "only evacuate marked objects %d %d",
! _heap->is_marked_complete(obj), _heap->is_marked_complete(ShenandoahBarrierSet::resolve_oop_static_not_null(obj)));
oop resolved = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
if (oopDesc::unsafe_equals(resolved, obj)) {
bool evac;
resolved = _heap->evacuate_object(obj, _thread, evac);
}
--- 734,745 ----
T o = oopDesc::load_heap_oop(p);
if (! oopDesc::is_null(o)) {
oop obj = oopDesc::decode_heap_oop_not_null(o);
if (_heap->in_collection_set(obj)) {
! assert(_heap->is_marked(obj), "only evacuate marked objects %d %d",
! _heap->is_marked(obj), _heap->is_marked(ShenandoahBarrierSet::resolve_oop_static_not_null(obj)));
oop resolved = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
if (oopDesc::unsafe_equals(resolved, obj)) {
bool evac;
resolved = _heap->evacuate_object(obj, _thread, evac);
}
*** 849,859 ****
public:
ShenandoahParallelEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
_heap(heap), _thread(Thread::current()) {}
void do_object(oop p) {
! assert(_heap->is_marked_complete(p), "expect only marked objects");
if (oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p))) {
bool evac;
_heap->evacuate_object(p, _thread, evac);
}
}
--- 798,808 ----
public:
ShenandoahParallelEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
_heap(heap), _thread(Thread::current()) {}
void do_object(oop p) {
! assert(_heap->is_marked(p), "expect only marked objects");
if (oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p))) {
bool evac;
_heap->evacuate_object(p, _thread, evac);
}
}
*** 979,989 ****
if (!cancelled_concgc()) {
// Allocations might have happened before we STWed here, record peak:
shenandoahPolicy()->record_peak_occupancy();
! ensure_parsability(true);
if (ShenandoahVerify) {
verifier()->verify_after_concmark();
}
--- 928,938 ----
if (!cancelled_concgc()) {
// Allocations might have happened before we STWed here, record peak:
shenandoahPolicy()->record_peak_occupancy();
! make_tlabs_parsable(true);
if (ShenandoahVerify) {
verifier()->verify_after_concmark();
}
*** 1035,1045 ****
assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name());
thread->gclab().make_parsable(_retire);
}
};
! void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
if (UseTLAB) {
CollectedHeap::ensure_parsability(retire_tlabs);
ShenandoahRetireTLABClosure cl(retire_tlabs);
Threads::java_threads_do(&cl);
gc_threads_do(&cl);
--- 984,994 ----
assert(thread->gclab().is_initialized(), "GCLAB should be initialized for %s", thread->name());
thread->gclab().make_parsable(_retire);
}
};
! void ShenandoahHeap::make_tlabs_parsable(bool retire_tlabs) {
if (UseTLAB) {
CollectedHeap::ensure_parsability(retire_tlabs);
ShenandoahRetireTLABClosure cl(retire_tlabs);
Threads::java_threads_do(&cl);
gc_threads_do(&cl);
*** 1301,1311 ****
return 0;
}
void ShenandoahHeap::prepare_for_verify() {
if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
! ensure_parsability(false);
}
}
void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
workers()->print_worker_threads_on(st);
--- 1250,1260 ----
return 0;
}
void ShenandoahHeap::prepare_for_verify() {
if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
! make_tlabs_parsable(false);
}
}
void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
workers()->print_worker_threads_on(st);
*** 1362,1434 ****
ShenandoahHeap::heap()->marked_object_iterate(r, _cl);
return false;
}
};
! void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
! ShenandoahIterateObjectClosureRegionClosure blk(cl);
! heap_region_iterate(&blk, false, true);
! }
!
! class ShenandoahSafeObjectIterateAdjustPtrsClosure : public MetadataAwareOopClosure {
private:
! ShenandoahHeap* _heap;
- public:
- ShenandoahSafeObjectIterateAdjustPtrsClosure() : _heap(ShenandoahHeap::heap()) {}
-
- private:
template <class T>
! inline void do_oop_work(T* p) {
T o = oopDesc::load_heap_oop(p);
if (!oopDesc::is_null(o)) {
oop obj = oopDesc::decode_heap_oop_not_null(o);
! oopDesc::encode_store_heap_oop(p, BrooksPointer::forwardee(obj));
! }
}
- public:
- void do_oop(oop* p) {
- do_oop_work(p);
}
- void do_oop(narrowOop* p) {
- do_oop_work(p);
}
- };
- class ShenandoahSafeObjectIterateAndUpdate : public ObjectClosure {
- private:
- ObjectClosure* _cl;
public:
! ShenandoahSafeObjectIterateAndUpdate(ObjectClosure *cl) : _cl(cl) {}
! virtual void do_object(oop obj) {
! assert (oopDesc::unsafe_equals(obj, BrooksPointer::forwardee(obj)),
! "avoid double-counting: only non-forwarded objects here");
!
! // Fix up the ptrs.
! ShenandoahSafeObjectIterateAdjustPtrsClosure adjust_ptrs;
! obj->oop_iterate(&adjust_ptrs);
- // Can reply the object now:
- _cl->do_object(obj);
- }
- };
! void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) {
assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
! // Safe iteration does objects only with correct references.
! // This is why we skip collection set regions that have stale copies of objects,
! // and fix up the pointers in the returned objects.
!
! ShenandoahSafeObjectIterateAndUpdate safe_cl(cl);
! ShenandoahIterateObjectClosureRegionClosure blk(&safe_cl);
! heap_region_iterate(&blk,
! /* skip_cset_regions = */ true,
! /* skip_humongous_continuations = */ true);
! _need_update_refs = false; // already updated the references
}
// Apply blk->heap_region_do() on all committed regions in address order,
// terminating the iteration early if heap_region_do() returns true.
void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_cset_regions, bool skip_humongous_continuation) const {
--- 1311,1406 ----
ShenandoahHeap::heap()->marked_object_iterate(r, _cl);
return false;
}
};
! class ObjectIterateScanRootClosure : public ExtendedOopClosure {
private:
! MarkBitMap* _bitmap;
! Stack<oop,mtGC>* _oop_stack;
template <class T>
! void do_oop_work(T* p) {
T o = oopDesc::load_heap_oop(p);
if (!oopDesc::is_null(o)) {
oop obj = oopDesc::decode_heap_oop_not_null(o);
! obj = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
! assert(oopDesc::is_oop(obj), "must be a valid oop");
! if (!_bitmap->isMarked((HeapWord*) obj)) {
! _bitmap->mark((HeapWord*) obj);
! _oop_stack->push(obj);
}
}
}
public:
! ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
! _bitmap(bitmap), _oop_stack(oop_stack) {}
! void do_oop(oop* p) { do_oop_work(p); }
! void do_oop(narrowOop* p) { do_oop_work(p); }
! };
! /*
! * This is public API, used in preparation of object_iterate().
! * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
! * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
! * control, we call SH::make_tlabs_parsable().
! */
! void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
! // No-op.
! }
! /*
! * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
! *
! * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
! * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
! * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
! * scanning therefore depends on having a valid marking bitmap to support it. However, we only
! * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
! * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
! * wiped the bitmap in preparation for next marking).
! *
! * For all those reasons, we implement object iteration as a single marking traversal, reporting
! * objects as we mark+traverse through the heap, starting from GC roots. This is ok. JVMTI
! * IterateThroughHeap is allowed to report dead objects, but is not required to do so.
! */
! void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
+ if (!os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
+ log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
+ return;
+ }
+
+ Stack<oop,mtGC> oop_stack;
+
+ // First, we process all GC roots. This populates the work stack with initial objects.
+ ShenandoahRootProcessor rp(this, 1, ShenandoahPhaseTimings::_num_phases);
+ ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
+ CLDToOopClosure clds(&oops, false);
+ CodeBlobToOopClosure blobs(&oops, false);
+ rp.process_all_roots(&oops, &oops, &clds, &blobs, 0);
+
+ // Work through the oop stack to traverse heap.
+ while (! oop_stack.is_empty()) {
+ oop obj = oop_stack.pop();
+ assert(oopDesc::is_oop(obj), "must be a valid oop");
+ cl->do_object(obj);
+ obj->oop_iterate(&oops);
+ }
+
+ assert(oop_stack.is_empty(), "should be empty");
! if (!os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
! log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
! }
! }
! void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) {
! assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
! object_iterate(cl);
}
// Apply blk->heap_region_do() on all committed regions in address order,
// terminating the iteration early if heap_region_do() returns true.
void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_cset_regions, bool skip_humongous_continuation) const {
*** 1452,1462 ****
public:
ShenandoahClearLivenessClosure(ShenandoahHeap* heap) : sh(heap) {}
bool heap_region_do(ShenandoahHeapRegion* r) {
r->clear_live_data();
! sh->set_next_top_at_mark_start(r->bottom(), r->top());
return false;
}
};
void ShenandoahHeap::start_concurrent_marking() {
--- 1424,1434 ----
public:
ShenandoahClearLivenessClosure(ShenandoahHeap* heap) : sh(heap) {}
bool heap_region_do(ShenandoahHeapRegion* r) {
r->clear_live_data();
! sh->set_top_at_mark_start(r->bottom(), r->top());
return false;
}
};
void ShenandoahHeap::start_concurrent_marking() {
*** 1471,1481 ****
set_concurrent_mark_in_progress(true);
// We need to reset all TLABs because we'd lose marks on all objects allocated in them.
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
! ensure_parsability(true);
}
_shenandoah_policy->record_bytes_allocated(_bytes_allocated_since_cm);
_used_start_gc = used();
--- 1443,1453 ----
set_concurrent_mark_in_progress(true);
// We need to reset all TLABs because we'd lose marks on all objects allocated in them.
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
! make_tlabs_parsable(true);
}
_shenandoah_policy->record_bytes_allocated(_bytes_allocated_since_cm);
_used_start_gc = used();
*** 1494,1527 ****
ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
resize_all_tlabs();
}
}
- void ShenandoahHeap::swap_mark_bitmaps() {
- // Swap bitmaps.
- MarkBitMap* tmp1 = _complete_mark_bit_map;
- _complete_mark_bit_map = _next_mark_bit_map;
- _next_mark_bit_map = tmp1;
-
- // Swap top-at-mark-start pointers
- HeapWord** tmp2 = _complete_top_at_mark_starts;
- _complete_top_at_mark_starts = _next_top_at_mark_starts;
- _next_top_at_mark_starts = tmp2;
-
- HeapWord** tmp3 = _complete_top_at_mark_starts_base;
- _complete_top_at_mark_starts_base = _next_top_at_mark_starts_base;
- _next_top_at_mark_starts_base = tmp3;
- }
-
-
void ShenandoahHeap::stop_concurrent_marking() {
assert(concurrent_mark_in_progress(), "How else could we get here?");
if (! cancelled_concgc()) {
// If we needed to update refs, and concurrent marking has been cancelled,
// we need to finish updating references.
set_need_update_refs(false);
- swap_mark_bitmaps();
}
set_concurrent_mark_in_progress(false);
LogTarget(Trace, gc, region) lt;
if (lt.is_enabled()) {
--- 1466,1481 ----
*** 1605,1626 ****
if (_heap->concurrent_mark_in_progress()) {
assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only query to-space");
}
#endif
assert(!oopDesc::is_null(obj), "null");
! return _heap->is_marked_next(obj);
}
ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() :
_heap(ShenandoahHeap::heap_no_check()) {
}
bool ShenandoahIsAliveClosure::do_object_b(oop obj) {
assert(_heap != NULL, "sanity");
assert(!oopDesc::is_null(obj), "null");
assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only query to-space");
! return _heap->is_marked_next(obj);
}
BoolObjectClosure* ShenandoahHeap::is_alive_closure() {
return need_update_refs() ?
(BoolObjectClosure*) &_forwarded_is_alive :
--- 1559,1580 ----
if (_heap->concurrent_mark_in_progress()) {
assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only query to-space");
}
#endif
assert(!oopDesc::is_null(obj), "null");
! return _heap->is_marked(obj);
}
ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() :
_heap(ShenandoahHeap::heap_no_check()) {
}
bool ShenandoahIsAliveClosure::do_object_b(oop obj) {
assert(_heap != NULL, "sanity");
assert(!oopDesc::is_null(obj), "null");
assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only query to-space");
! return _heap->is_marked(obj);
}
BoolObjectClosure* ShenandoahHeap::is_alive_closure() {
return need_update_refs() ?
(BoolObjectClosure*) &_forwarded_is_alive :
*** 1803,1818 ****
ShenandoahMonitoringSupport* ShenandoahHeap::monitoring_support() {
return _monitoring_support;
}
! MarkBitMap* ShenandoahHeap::complete_mark_bit_map() {
! return _complete_mark_bit_map;
! }
!
! MarkBitMap* ShenandoahHeap::next_mark_bit_map() {
! return _next_mark_bit_map;
}
void ShenandoahHeap::add_free_region(ShenandoahHeapRegion* r) {
_free_regions->add_region(r);
}
--- 1757,1768 ----
ShenandoahMonitoringSupport* ShenandoahHeap::monitoring_support() {
return _monitoring_support;
}
! MarkBitMap* ShenandoahHeap::mark_bit_map() {
! return &_mark_bit_map;
}
void ShenandoahHeap::add_free_region(ShenandoahHeapRegion* r) {
_free_regions->add_region(r);
}
*** 1842,1869 ****
void ShenandoahHeap::set_bytes_allocated_since_cm(size_t bytes) {
_bytes_allocated_since_cm = bytes;
}
! void ShenandoahHeap::set_next_top_at_mark_start(HeapWord* region_base, HeapWord* addr) {
uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_bytes_shift();
! _next_top_at_mark_starts[index] = addr;
}
! HeapWord* ShenandoahHeap::next_top_at_mark_start(HeapWord* region_base) {
uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_bytes_shift();
! return _next_top_at_mark_starts[index];
! }
!
! void ShenandoahHeap::set_complete_top_at_mark_start(HeapWord* region_base, HeapWord* addr) {
! uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_bytes_shift();
! _complete_top_at_mark_starts[index] = addr;
! }
!
! HeapWord* ShenandoahHeap::complete_top_at_mark_start(HeapWord* region_base) {
! uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_bytes_shift();
! return _complete_top_at_mark_starts[index];
}
void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
_full_gc_in_progress = in_progress;
}
--- 1792,1809 ----
void ShenandoahHeap::set_bytes_allocated_since_cm(size_t bytes) {
_bytes_allocated_since_cm = bytes;
}
! void ShenandoahHeap::set_top_at_mark_start(HeapWord* region_base, HeapWord* addr) {
uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_bytes_shift();
! _top_at_mark_starts[index] = addr;
}
! HeapWord* ShenandoahHeap::top_at_mark_start(HeapWord* region_base) {
uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_bytes_shift();
! return _top_at_mark_starts[index];
}
void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
_full_gc_in_progress = in_progress;
}
*** 1985,1997 ****
SuspendibleThreadSetJoiner stsj(_concurrent && ShenandoahSuspendibleWorkers);
ShenandoahHeapRegion* r = _regions->claim_next();
while (r != NULL) {
if (_heap->in_collection_set(r)) {
HeapWord* bottom = r->bottom();
! HeapWord* top = _heap->complete_top_at_mark_start(r->bottom());
if (top > bottom) {
! _heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
}
} else {
if (r->is_active()) {
_heap->marked_object_oop_safe_iterate(r, &cl);
}
--- 1925,1937 ----
SuspendibleThreadSetJoiner stsj(_concurrent && ShenandoahSuspendibleWorkers);
ShenandoahHeapRegion* r = _regions->claim_next();
while (r != NULL) {
if (_heap->in_collection_set(r)) {
HeapWord* bottom = r->bottom();
! HeapWord* top = _heap->top_at_mark_start(r->bottom());
if (top > bottom) {
! _heap->mark_bit_map()->clear_range_large(MemRegion(bottom, top));
}
} else {
if (r->is_active()) {
_heap->marked_object_oop_safe_iterate(r, &cl);
}
*** 2028,2038 ****
verifier()->verify_before_updaterefs();
}
set_evacuation_in_progress_at_safepoint(false);
set_update_refs_in_progress(true);
! ensure_parsability(true);
if (UseShenandoahMatrix) {
connection_matrix()->clear_all();
}
for (uint i = 0; i < num_regions(); i++) {
ShenandoahHeapRegion* r = _ordered_regions->get(i);
--- 1968,1978 ----
verifier()->verify_before_updaterefs();
}
set_evacuation_in_progress_at_safepoint(false);
set_update_refs_in_progress(true);
! make_tlabs_parsable(true);
if (UseShenandoahMatrix) {
connection_matrix()->clear_all();
}
for (uint i = 0; i < num_regions(); i++) {
ShenandoahHeapRegion* r = _ordered_regions->get(i);
*** 2193,2217 ****
}
bool ShenandoahHeap::commit_bitmaps(ShenandoahHeapRegion* r) {
size_t len = _bitmap_words_per_region * HeapWordSize;
size_t off = r->region_number() * _bitmap_words_per_region;
! if (!os::commit_memory((char*)(_bitmap0_region.start() + off), len, false)) {
! return false;
! }
! if (!os::commit_memory((char*)(_bitmap1_region.start() + off), len, false)) {
return false;
}
return true;
}
bool ShenandoahHeap::uncommit_bitmaps(ShenandoahHeapRegion* r) {
size_t len = _bitmap_words_per_region * HeapWordSize;
size_t off = r->region_number() * _bitmap_words_per_region;
! if (!os::uncommit_memory((char*)(_bitmap0_region.start() + off), len)) {
! return false;
! }
! if (!os::uncommit_memory((char*)(_bitmap1_region.start() + off), len)) {
return false;
}
return true;
}
--- 2133,2151 ----
}
bool ShenandoahHeap::commit_bitmaps(ShenandoahHeapRegion* r) {
size_t len = _bitmap_words_per_region * HeapWordSize;
size_t off = r->region_number() * _bitmap_words_per_region;
! if (!os::commit_memory((char*)(_bitmap_region.start() + off), len, false)) {
return false;
}
return true;
}
bool ShenandoahHeap::uncommit_bitmaps(ShenandoahHeapRegion* r) {
size_t len = _bitmap_words_per_region * HeapWordSize;
size_t off = r->region_number() * _bitmap_words_per_region;
! if (!os::uncommit_memory((char*)(_bitmap_region.start() + off), len)) {
return false;
}
return true;
}
< prev index next >