< prev index next >
src/share/vm/gc/shenandoah/shenandoahHeap.cpp
Print this page
*** 228,248 ****
SATB_Q_FL_lock,
20 /*G1SATBProcessCompletedThreshold */,
Shared_SATB_Q_lock);
// Reserve space for prev and next bitmap.
! size_t bitmap_size = CMBitMap::compute_size(heap_rs.size());
! MemRegion heap_region = MemRegion((HeapWord*) heap_rs.base(), heap_rs.size() / HeapWordSize);
size_t page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
! ReservedSpace bitmap0(bitmap_size, page_size);
os::commit_memory_or_exit(bitmap0.base(), bitmap0.size(), false, "couldn't allocate mark bitmap");
MemTracker::record_virtual_memory_type(bitmap0.base(), mtGC);
MemRegion bitmap_region0 = MemRegion((HeapWord*) bitmap0.base(), bitmap0.size() / HeapWordSize);
! ReservedSpace bitmap1(bitmap_size, page_size);
os::commit_memory_or_exit(bitmap1.base(), bitmap1.size(), false, "couldn't allocate mark bitmap");
MemTracker::record_virtual_memory_type(bitmap1.base(), mtGC);
MemRegion bitmap_region1 = MemRegion((HeapWord*) bitmap1.base(), bitmap1.size() / HeapWordSize);
if (ShenandoahAlwaysPreTouch) {
--- 228,248 ----
SATB_Q_FL_lock,
20 /*G1SATBProcessCompletedThreshold */,
Shared_SATB_Q_lock);
// Reserve space for prev and next bitmap.
! _bitmap_size = CMBitMap::compute_size(heap_rs.size());
! _heap_region = MemRegion((HeapWord*) heap_rs.base(), heap_rs.size() / HeapWordSize);
size_t page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
! ReservedSpace bitmap0(_bitmap_size, page_size);
os::commit_memory_or_exit(bitmap0.base(), bitmap0.size(), false, "couldn't allocate mark bitmap");
MemTracker::record_virtual_memory_type(bitmap0.base(), mtGC);
MemRegion bitmap_region0 = MemRegion((HeapWord*) bitmap0.base(), bitmap0.size() / HeapWordSize);
! ReservedSpace bitmap1(_bitmap_size, page_size);
os::commit_memory_or_exit(bitmap1.base(), bitmap1.size(), false, "couldn't allocate mark bitmap");
MemTracker::record_virtual_memory_type(bitmap1.base(), mtGC);
MemRegion bitmap_region1 = MemRegion((HeapWord*) bitmap1.base(), bitmap1.size() / HeapWordSize);
if (ShenandoahAlwaysPreTouch) {
*** 252,269 ****
// before initialize() below zeroes it with initializing thread. For any given region,
// we touch the region and the corresponding bitmaps from the same thread.
log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages",
_ordered_regions->count(), page_size);
! ShenandoahPretouchTask cl(_ordered_regions, bitmap0.base(), bitmap1.base(), bitmap_size, page_size);
_workers->run_task(&cl);
}
! _mark_bit_map0.initialize(heap_region, bitmap_region0);
_complete_mark_bit_map = &_mark_bit_map0;
! _mark_bit_map1.initialize(heap_region, bitmap_region1);
_next_mark_bit_map = &_mark_bit_map1;
_connection_matrix = new ShenandoahConnectionMatrix(_max_regions);
_partial_gc = new ShenandoahPartialGC(this, _max_regions);
--- 252,269 ----
// before initialize() below zeroes it with initializing thread. For any given region,
// we touch the region and the corresponding bitmaps from the same thread.
log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages",
_ordered_regions->count(), page_size);
! ShenandoahPretouchTask cl(_ordered_regions, bitmap0.base(), bitmap1.base(), _bitmap_size, page_size);
_workers->run_task(&cl);
}
! _mark_bit_map0.initialize(_heap_region, bitmap_region0);
_complete_mark_bit_map = &_mark_bit_map0;
! _mark_bit_map1.initialize(_heap_region, bitmap_region1);
_next_mark_bit_map = &_mark_bit_map1;
_connection_matrix = new ShenandoahConnectionMatrix(_max_regions);
_partial_gc = new ShenandoahPartialGC(this, _max_regions);
*** 1139,1151 ****
if (UseShenandoahMatrix) {
if (PrintShenandoahMatrix) {
outputStream* log = Log(gc)::info_stream();
connection_matrix()->print_on(log);
}
- if (VerifyShenandoahMatrix) {
- verify_matrix();
}
}
#ifdef ASSERT
if (ShenandoahVerify) {
verify_heap_after_marking();
--- 1139,1152 ----
if (UseShenandoahMatrix) {
if (PrintShenandoahMatrix) {
outputStream* log = Log(gc)::info_stream();
connection_matrix()->print_on(log);
}
}
+
+ if (ShenandoahVerify || (UseShenandoahMatrix && VerifyShenandoahMatrix)) {
+ verify_heap_reachable_at_safepoint();
}
#ifdef ASSERT
if (ShenandoahVerify) {
verify_heap_after_marking();
*** 1902,1988 ****
HeapWord** tmp3 = _complete_top_at_mark_starts_base;
_complete_top_at_mark_starts_base = _next_top_at_mark_starts_base;
_next_top_at_mark_starts_base = tmp3;
}
! class ShenandoahVerifyMatrixOopClosure : public ExtendedOopClosure {
private:
oop _obj;
!
template <class T>
! inline void do_oop_nv(T* p) {
T o = oopDesc::load_heap_oop(p);
! if (! oopDesc::is_null(o)) {
oop obj = oopDesc::decode_heap_oop_not_null(o);
! ShenandoahHeap* heap = ShenandoahHeap::heap();
! guarantee(heap->is_marked_complete(obj), "must be marked");
! uint from_idx = heap->heap_region_index_containing(p);
! uint to_idx = heap->heap_region_index_containing(obj);
! if (!heap->connection_matrix()->is_connected(from_idx, to_idx)) {
tty->print_cr("from-obj: ");
_obj->print_on(tty);
tty->print_cr("to-obj:");
obj->print_on(tty);
! tty->print_cr("from-obj allocated after mark: %s", BOOL_TO_STR(heap->allocated_after_complete_mark_start((HeapWord*) _obj)));
! tty->print_cr("to-obj allocated after mark: %s", BOOL_TO_STR(heap->allocated_after_complete_mark_start((HeapWord*) obj)));
! tty->print_cr("from-obj marked: %s", BOOL_TO_STR(heap->is_marked_complete(_obj)));
! tty->print_cr("to-obj marked: %s", BOOL_TO_STR(heap->is_marked_complete(obj)));
tty->print_cr("from-idx: %u, to-idx: %u", from_idx, to_idx);
oop fwd_from = BrooksPointer::forwardee(_obj);
oop fwd_to = BrooksPointer::forwardee(obj);
tty->print_cr("from-obj forwardee: " PTR_FORMAT, p2i(fwd_from));
tty->print_cr("to-obj forwardee: " PTR_FORMAT, p2i(fwd_to));
! tty->print_cr("forward(from-obj) marked: %s", BOOL_TO_STR(heap->is_marked_complete(fwd_from)));
! tty->print_cr("forward(to-obj) marked: %s", BOOL_TO_STR(heap->is_marked_complete(fwd_to)));
! uint fwd_from_idx = heap->heap_region_index_containing(fwd_from);
! uint fwd_to_idx = heap->heap_region_index_containing(fwd_to);
tty->print_cr("forward(from-idx): %u, forward(to-idx): %u", fwd_from_idx, fwd_to_idx);
! tty->print_cr("forward(from) connected with forward(to)? %s", BOOL_TO_STR(heap->connection_matrix()->is_connected(fwd_from_idx, fwd_to_idx)));
! tty->print_cr("sizeof(bool): %lu", sizeof(bool));
}
guarantee(oopDesc::unsafe_equals(ShenandoahBarrierSet::resolve_oop_static_not_null(obj), obj), "polizeilich verboten");
! guarantee(heap->connection_matrix()->is_connected(from_idx, to_idx), "must be connected");
! }
}
! public:
! ShenandoahVerifyMatrixOopClosure(oop obj) : _obj(obj) {}
!
! void do_oop(oop* o) {
! do_oop_nv(o);
}
-
- void do_oop(narrowOop* o) {
- do_oop_nv(o);
}
- };
-
- class ShenandoahVerifyMatrixObjectClosure : public ObjectClosure {
- public:
- void do_object(oop obj) {
- guarantee(ShenandoahHeap::heap()->is_marked_complete(obj), "must be marked");
- ShenandoahVerifyMatrixOopClosure cl(obj);
- obj->oop_iterate(&cl);
}
};
! class ShenandoahVerifyMatrixRegionClosure : public ShenandoahHeapRegionClosure {
! bool doHeapRegion(ShenandoahHeapRegion* r) {
! ShenandoahVerifyMatrixObjectClosure cl;
! ShenandoahHeap::heap()->marked_object_iterate(r, &cl);
! return false;
! }
! };
- void ShenandoahHeap::verify_matrix() {
OrderAccess::fence();
ensure_parsability(false);
! ShenandoahVerifyMatrixRegionClosure cl;
! heap_region_iterate(&cl, true, true);
}
void ShenandoahHeap::stop_concurrent_marking() {
assert(concurrent_mark_in_progress(), "How else could we get here?");
if (! cancelled_concgc()) {
--- 1903,2028 ----
HeapWord** tmp3 = _complete_top_at_mark_starts_base;
_complete_top_at_mark_starts_base = _next_top_at_mark_starts_base;
_next_top_at_mark_starts_base = tmp3;
}
! class VerifyReachableHeapClosure : public ExtendedOopClosure {
private:
+ SCMObjToScanQueue* _queue;
+ ShenandoahHeap* _heap;
+ CMBitMap* _map;
+ bool _check_matrix;
oop _obj;
! public:
! VerifyReachableHeapClosure(SCMObjToScanQueue* queue, CMBitMap* map, bool check_matrix) :
! _queue(queue), _heap(ShenandoahHeap::heap()), _map(map), _check_matrix(check_matrix) {};
template <class T>
! void do_oop_work(T* p) {
T o = oopDesc::load_heap_oop(p);
! if (!oopDesc::is_null(o)) {
oop obj = oopDesc::decode_heap_oop_not_null(o);
! guarantee(check_obj_alignment(obj), "sanity");
!
! guarantee(!oopDesc::is_null(obj), "sanity");
! guarantee(_heap->is_in(obj), "sanity");
! oop forw = BrooksPointer::forwardee(obj);
! guarantee(!oopDesc::is_null(forw), "sanity");
! guarantee(_heap->is_in(forw), "sanity");
!
! guarantee(oopDesc::unsafe_equals(obj, forw), "should not be forwarded");
!
! if (_check_matrix) {
! uint from_idx = _heap->heap_region_index_containing(p);
! uint to_idx = _heap->heap_region_index_containing(obj);
! if (!_heap->connection_matrix()->is_connected(from_idx, to_idx)) {
tty->print_cr("from-obj: ");
_obj->print_on(tty);
tty->print_cr("to-obj:");
obj->print_on(tty);
! tty->print_cr("from-obj allocated after mark: %s", BOOL_TO_STR(_heap->allocated_after_complete_mark_start((HeapWord*) _obj)));
! tty->print_cr("to-obj allocated after mark: %s", BOOL_TO_STR(_heap->allocated_after_complete_mark_start((HeapWord*) obj)));
! tty->print_cr("from-obj marked: %s", BOOL_TO_STR(_heap->is_marked_complete(_obj)));
! tty->print_cr("to-obj marked: %s", BOOL_TO_STR(_heap->is_marked_complete(obj)));
tty->print_cr("from-idx: %u, to-idx: %u", from_idx, to_idx);
oop fwd_from = BrooksPointer::forwardee(_obj);
oop fwd_to = BrooksPointer::forwardee(obj);
tty->print_cr("from-obj forwardee: " PTR_FORMAT, p2i(fwd_from));
tty->print_cr("to-obj forwardee: " PTR_FORMAT, p2i(fwd_to));
! tty->print_cr("forward(from-obj) marked: %s", BOOL_TO_STR(_heap->is_marked_complete(fwd_from)));
! tty->print_cr("forward(to-obj) marked: %s", BOOL_TO_STR(_heap->is_marked_complete(fwd_to)));
! uint fwd_from_idx = _heap->heap_region_index_containing(fwd_from);
! uint fwd_to_idx = _heap->heap_region_index_containing(fwd_to);
tty->print_cr("forward(from-idx): %u, forward(to-idx): %u", fwd_from_idx, fwd_to_idx);
! tty->print_cr("forward(from) connected with forward(to)? %s", BOOL_TO_STR(_heap->connection_matrix()->is_connected(fwd_from_idx, fwd_to_idx)));
}
guarantee(oopDesc::unsafe_equals(ShenandoahBarrierSet::resolve_oop_static_not_null(obj), obj), "polizeilich verboten");
! guarantee(_heap->connection_matrix()->is_connected(from_idx, to_idx), "must be connected");
}
! if (_map->parMark((HeapWord*) obj)) {
! _queue->push(SCMTask(obj));
}
}
}
+ void do_oop(oop* p) { do_oop_work(p); }
+ void do_oop(narrowOop* p) { do_oop_work(p); }
+ void set_obj(oop o) { _obj = o; }
};
! void ShenandoahHeap::verify_heap_reachable_at_safepoint() {
! guarantee(SafepointSynchronize::is_at_safepoint(), "only when nothing else happens");
OrderAccess::fence();
ensure_parsability(false);
!
! // Allocate temporary bitmap for storing marking wavefront:
! ReservedSpace bm(_bitmap_size, os::vm_page_size());
! os::commit_memory_or_exit(bm.base(), bm.size(), false, "couldn't allocate verification bitmap");
! MemTracker::record_virtual_memory_type(bm.base(), mtGC);
! MemRegion verify_bitmap_region = MemRegion((HeapWord*) bm.base(), bm.size() / HeapWordSize);
!
! CMBitMap* _verification_bit_map = new CMBitMap();
! _verification_bit_map->initialize(_heap_region, verify_bitmap_region);
! MemRegion mr = MemRegion(_verification_bit_map->startWord(), _verification_bit_map->endWord());
! _verification_bit_map->clear_range_large(mr);
!
! // Initialize a single queue
! SCMObjToScanQueue* q = new SCMObjToScanQueue();
! q->initialize();
!
! // Scan root set
! ClassLoaderDataGraph::clear_claimed_marks();
! ShenandoahRootProcessor rp(this, 1);
!
! {
! VerifyReachableHeapClosure cl(q, _verification_bit_map, false);
! CLDToOopClosure cld_cl(&cl);
! CodeBlobToOopClosure code_cl(&cl, ! CodeBlobToOopClosure::FixRelocations);
! rp.process_all_roots(&cl, &cl, &cld_cl, &code_cl, 0);
! }
!
! // Finish the scan
! {
! VerifyReachableHeapClosure cl(q, _verification_bit_map, UseShenandoahMatrix && VerifyShenandoahMatrix);
! SCMTask task;
! while ((q->pop_buffer(task) ||
! q->pop_local(task) ||
! q->pop_overflow(task))) {
! oop obj = task.obj();
! assert(!oopDesc::is_null(obj), "must not be null");
! cl.set_obj(obj);
! obj->oop_iterate(&cl);
! }
! }
!
! // Clean up!
! os::uncommit_memory(bm.base(), bm.size());
! MemTracker::record_free(bm.base());
! delete(q);
}
void ShenandoahHeap::stop_concurrent_marking() {
assert(concurrent_mark_in_progress(), "How else could we get here?");
if (! cancelled_concgc()) {
< prev index next >