< prev index next >
src/share/vm/gc_implementation/shenandoah/shenandoahMarkCompact.cpp
Print this page
rev 10658 : [backport] Single marking bitmap
rev 10659 : [backport] Make sure bitmap is marked incomplete before bitmap resets
rev 10660 : [backport] Refactor bitmap cleaning
rev 10664 : [backport] Fix indenting in ShMarkCompact::do_it
rev 10665 : [backport] Coarsen Full GC use of heap lock
rev 10668 : [backport] Cleanup buffered queue handling
rev 10678 : [backport] Rename concurrentMark -> concurrent_mark
rev 10690 : [backport] Cleanup header files and forward declarations
rev 10695 : [backport] Inline/rename confusingly named methods in ShConcurrentMark
rev 10715 : [backport] Cleanup up superfluous newlines
rev 10737 : [backport] Support VerifyBeforeGC and VerifyAfterGC VM options
rev 10743 : [backport] Clean up ShHeap::heap_region_iterate uses
rev 10749 : [backport] Adjust metaspace size during each GC
rev 10755 : [backport] Make heuristics tell if we can process references or unload classes
rev 10764 : [backport] Rename BrooksPointer to ShenandoahBrooksPointer
rev 10772 : [backport] Update copyrights
rev 10801 : [backport] Rename vm_operations_shenandoah.* to shenandoahVMOperations.*
*** 1,7 ****
/*
! * Copyright (c) 2014, 2015, Red Hat, Inc. and/or its affiliates.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
--- 1,7 ----
/*
! * Copyright (c) 2014, 2018, Red Hat, Inc. All rights reserved.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
*** 20,69 ****
* questions.
*
*/
#include "precompiled.hpp"
#include "code/codeCache.hpp"
#include "gc_implementation/shenandoah/shenandoahGCTraceTime.hpp"
#include "gc_implementation/shared/gcTimer.hpp"
! #include "gc_implementation/shenandoah/brooksPointer.hpp"
#include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahCollectionSet.hpp"
#include "gc_implementation/shenandoah/shenandoahFreeSet.hpp"
#include "gc_implementation/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc_implementation/shenandoah/shenandoahMarkCompact.hpp"
- #include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
#include "gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp"
#include "gc_implementation/shenandoah/shenandoahHeap.hpp"
#include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp"
#include "gc_implementation/shenandoah/shenandoahUtils.hpp"
#include "gc_implementation/shenandoah/shenandoahVerifier.hpp"
#include "gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp"
! #include "gc_implementation/shenandoah/vm_operations_shenandoah.hpp"
#include "oops/oop.inline.hpp"
- #include "runtime/biasedLocking.hpp"
#include "runtime/thread.hpp"
#include "utilities/copy.hpp"
#include "utilities/growableArray.hpp"
- #include "utilities/taskqueue.hpp"
#include "utilities/workgroup.hpp"
void ShenandoahMarkCompact::initialize(GCTimer* gc_timer) {
_gc_timer = gc_timer;
}
void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
- {
if (ShenandoahVerify) {
heap->verifier()->verify_before_fullgc();
}
heap->set_full_gc_in_progress(true);
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
--- 20,75 ----
* questions.
*
*/
#include "precompiled.hpp"
+
#include "code/codeCache.hpp"
#include "gc_implementation/shenandoah/shenandoahGCTraceTime.hpp"
#include "gc_implementation/shared/gcTimer.hpp"
! #include "gc_implementation/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc_implementation/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc_implementation/shenandoah/shenandoahConcurrentMark.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahCollectionSet.hpp"
#include "gc_implementation/shenandoah/shenandoahFreeSet.hpp"
#include "gc_implementation/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc_implementation/shenandoah/shenandoahMarkCompact.hpp"
#include "gc_implementation/shenandoah/shenandoahHeapRegionSet.hpp"
#include "gc_implementation/shenandoah/shenandoahHeap.hpp"
#include "gc_implementation/shenandoah/shenandoahHeap.inline.hpp"
+ #include "gc_implementation/shenandoah/shenandoahHeuristics.hpp"
#include "gc_implementation/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahRootProcessor.hpp"
+ #include "gc_implementation/shenandoah/shenandoahTaskqueue.hpp"
+ #include "gc_implementation/shenandoah/shenandoahTaskqueue.inline.hpp"
#include "gc_implementation/shenandoah/shenandoahUtils.hpp"
#include "gc_implementation/shenandoah/shenandoahVerifier.hpp"
+ #include "gc_implementation/shenandoah/shenandoahVMOperations.hpp"
+ #include "gc_implementation/shenandoah/shenandoahWorkGroup.hpp"
#include "gc_implementation/shenandoah/shenandoahWorkerPolicy.hpp"
! #include "memory/metaspace.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/thread.hpp"
#include "utilities/copy.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/workgroup.hpp"
void ShenandoahMarkCompact::initialize(GCTimer* gc_timer) {
_gc_timer = gc_timer;
}
void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
if (ShenandoahVerify) {
heap->verifier()->verify_before_fullgc();
}
+ if (VerifyBeforeGC) {
+ Universe::verify();
+ }
+
heap->set_full_gc_in_progress(true);
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
*** 79,89 ****
// 0. Remember if we have forwarded objects
bool has_forwarded_objects = heap->has_forwarded_objects();
// a. Cancel concurrent mark, if in progress
if (heap->is_concurrent_mark_in_progress()) {
! heap->concurrentMark()->cancel();
heap->stop_concurrent_marking();
}
assert(!heap->is_concurrent_mark_in_progress(), "sanity");
// b1. Cancel evacuation, if in progress
--- 85,95 ----
// 0. Remember if we have forwarded objects
bool has_forwarded_objects = heap->has_forwarded_objects();
// a. Cancel concurrent mark, if in progress
if (heap->is_concurrent_mark_in_progress()) {
! heap->concurrent_mark()->cancel();
heap->stop_concurrent_marking();
}
assert(!heap->is_concurrent_mark_in_progress(), "sanity");
// b1. Cancel evacuation, if in progress
*** 97,108 ****
heap->set_update_refs_in_progress(false);
}
assert(!heap->is_update_refs_in_progress(), "sanity");
// c. Reset the bitmaps for new marking
! heap->reset_next_mark_bitmap();
! assert(heap->next_marking_context()->is_bitmap_clear(), "sanity");
// d. Abandon reference discovery and clear all discovered references.
ReferenceProcessor *rp = heap->ref_processor();
rp->disable_discovery();
rp->abandon_partial_discovery();
--- 103,115 ----
heap->set_update_refs_in_progress(false);
}
assert(!heap->is_update_refs_in_progress(), "sanity");
// c. Reset the bitmaps for new marking
! heap->reset_mark_bitmap();
! assert(heap->marking_context()->is_bitmap_clear(), "sanity");
! assert(!heap->marking_context()->is_complete(), "sanity");
// d. Abandon reference discovery and clear all discovered references.
ReferenceProcessor *rp = heap->ref_processor();
rp->disable_discovery();
rp->abandon_partial_discovery();
*** 110,120 ****
// e. Set back forwarded objects bit back, in case some steps above dropped it.
heap->set_has_forwarded_objects(has_forwarded_objects);
}
- {
heap->make_parsable(true);
CodeCache::gc_prologue();
OrderAccess::fence();
--- 117,126 ----
*** 127,169 ****
heap->set_has_forwarded_objects(false);
heap->set_full_gc_move_in_progress(true);
// Setup workers for the rest
- {
OrderAccess::fence();
// Initialize worker slices
ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
for (uint i = 0; i < heap->max_workers(); i++) {
worker_slices[i] = new ShenandoahHeapRegionSet();
}
phase2_calculate_target_addresses(worker_slices);
OrderAccess::fence();
phase3_update_references();
phase4_compact_objects(worker_slices);
// Free worker slices
for (uint i = 0; i < heap->max_workers(); i++) {
delete worker_slices[i];
}
FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices, mtGC);
CodeCache::gc_epilogue();
JvmtiExport::gc_epilogue();
- }
heap->set_full_gc_move_in_progress(false);
heap->set_full_gc_in_progress(false);
if (ShenandoahVerify) {
heap->verifier()->verify_after_fullgc();
}
}
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
heap->post_full_gc_dump(_gc_timer);
--- 133,185 ----
heap->set_has_forwarded_objects(false);
heap->set_full_gc_move_in_progress(true);
// Setup workers for the rest
OrderAccess::fence();
// Initialize worker slices
ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
for (uint i = 0; i < heap->max_workers(); i++) {
worker_slices[i] = new ShenandoahHeapRegionSet();
}
+ {
+ // The rest of code performs region moves, where region status is undefined
+ // until all phases run together.
+ ShenandoahHeapLocker lock(heap->lock());
+
phase2_calculate_target_addresses(worker_slices);
OrderAccess::fence();
phase3_update_references();
phase4_compact_objects(worker_slices);
+ }
+
+ // Resize metaspace
+ MetaspaceGC::compute_new_size();
// Free worker slices
for (uint i = 0; i < heap->max_workers(); i++) {
delete worker_slices[i];
}
FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices, mtGC);
CodeCache::gc_epilogue();
JvmtiExport::gc_epilogue();
heap->set_full_gc_move_in_progress(false);
heap->set_full_gc_in_progress(false);
if (ShenandoahVerify) {
heap->verifier()->verify_after_fullgc();
}
+
+ if (VerifyAfterGC) {
+ Universe::verify();
}
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
heap->post_full_gc_dump(_gc_timer);
*** 171,227 ****
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
heap->resize_all_tlabs();
}
- }
}
class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
private:
ShenandoahMarkingContext* const _ctx;
public:
! ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->next_marking_context()) {}
! bool heap_region_do(ShenandoahHeapRegion *r) {
! _ctx->set_top_at_mark_start(r->region_number(), r->top());
r->clear_live_data();
r->set_concurrent_iteration_safe_limit(r->top());
- return false;
}
};
void ShenandoahMarkCompact::phase1_mark_heap() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
GCTraceTime time("Phase 1: Mark live objects", ShenandoahLogDebug, _gc_timer, heap->tracer()->gc_id());
ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
- {
- ShenandoahHeapLocker lock(heap->lock());
ShenandoahPrepareForMarkClosure cl;
! heap->heap_region_iterate(&cl, false, false);
! }
! ShenandoahConcurrentMark* cm = heap->concurrentMark();
! // Do not trust heuristics, because this can be our last resort collection.
! // Only ignore processing references and class unloading if explicitly disabled.
! heap->set_process_references(ShenandoahRefProcFrequency != 0);
! heap->set_unload_classes(ClassUnloading);
ReferenceProcessor* rp = heap->ref_processor();
// enable ("weak") refs discovery
rp->enable_discovery(true /*verify_no_refs*/, true);
rp->setup_policy(true); // forcefully purge all soft references
rp->set_active_mt_degree(heap->workers()->active_workers());
cm->update_roots(ShenandoahPhaseTimings::full_gc_roots);
cm->mark_roots(ShenandoahPhaseTimings::full_gc_roots);
! cm->shared_finish_mark_from_roots(/* full_gc = */ true);
! heap->swap_mark_contexts();
}
class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
--- 187,236 ----
if (UseTLAB) {
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
heap->resize_all_tlabs();
}
}
class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
private:
ShenandoahMarkingContext* const _ctx;
public:
! ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
! void heap_region_do(ShenandoahHeapRegion *r) {
! _ctx->capture_top_at_mark_start(r);
r->clear_live_data();
r->set_concurrent_iteration_safe_limit(r->top());
}
};
void ShenandoahMarkCompact::phase1_mark_heap() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
GCTraceTime time("Phase 1: Mark live objects", ShenandoahLogDebug, _gc_timer, heap->tracer()->gc_id());
ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
ShenandoahPrepareForMarkClosure cl;
! heap->heap_region_iterate(&cl);
! ShenandoahConcurrentMark* cm = heap->concurrent_mark();
! heap->set_process_references(heap->heuristics()->can_process_references());
! heap->set_unload_classes(heap->heuristics()->can_unload_classes());
ReferenceProcessor* rp = heap->ref_processor();
// enable ("weak") refs discovery
rp->enable_discovery(true /*verify_no_refs*/, true);
rp->setup_policy(true); // forcefully purge all soft references
rp->set_active_mt_degree(heap->workers()->active_workers());
cm->update_roots(ShenandoahPhaseTimings::full_gc_roots);
cm->mark_roots(ShenandoahPhaseTimings::full_gc_roots);
! cm->finish_mark_from_roots(/* full_gc = */ true);
! heap->mark_complete_marking_context();
}
class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
*** 260,270 ****
void do_object(oop p) {
assert(_from_region != NULL, "must set before work");
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
! size_t obj_size = p->size() + BrooksPointer::word_size();
if (_compact_point + obj_size > _to_region->end()) {
finish_region();
// Object doesn't fit. Pick next empty region and start compacting there.
ShenandoahHeapRegion* new_to_region;
--- 269,279 ----
void do_object(oop p) {
assert(_from_region != NULL, "must set before work");
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
! size_t obj_size = p->size() + ShenandoahBrooksPointer::word_size();
if (_compact_point + obj_size > _to_region->end()) {
finish_region();
// Object doesn't fit. Pick next empty region and start compacting there.
ShenandoahHeapRegion* new_to_region;
*** 283,293 ****
}
// Object fits into current region, record new location:
assert(_compact_point + obj_size <= _to_region->end(), "must fit");
shenandoah_assert_not_forwarded(NULL, p);
! BrooksPointer::set_raw(p, _compact_point + BrooksPointer::word_size());
_compact_point += obj_size;
}
};
class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
--- 292,302 ----
}
// Object fits into current region, record new location:
assert(_compact_point + obj_size <= _to_region->end(), "must fit");
shenandoah_assert_not_forwarded(NULL, p);
! ShenandoahBrooksPointer::set_raw(p, _compact_point + ShenandoahBrooksPointer::word_size());
_compact_point += obj_size;
}
};
class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
*** 380,398 ****
continue;
}
if (r->is_humongous_start() && r->is_move_allowed()) {
// From-region candidate: movable humongous region
! oop old_obj = oop(r->bottom() + BrooksPointer::word_size());
! size_t words_size = old_obj->size() + BrooksPointer::word_size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t start = to_end - num_regions;
if (start >= to_begin && start != r->region_number()) {
// Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
! BrooksPointer::set_raw(old_obj, heap->get_region(start)->bottom() + BrooksPointer::word_size());
to_end = start;
continue;
}
}
--- 389,407 ----
continue;
}
if (r->is_humongous_start() && r->is_move_allowed()) {
// From-region candidate: movable humongous region
! oop old_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
! size_t words_size = old_obj->size() + ShenandoahBrooksPointer::word_size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t start = to_end - num_regions;
if (start >= to_begin && start != r->region_number()) {
// Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
! ShenandoahBrooksPointer::set_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahBrooksPointer::word_size());
to_end = start;
continue;
}
}
*** 406,416 ****
private:
ShenandoahHeap* const _heap;
public:
ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
! bool heap_region_do(ShenandoahHeapRegion* r) {
if (r->is_trash()) {
r->recycle();
}
if (r->is_cset()) {
r->make_regular_bypass();
--- 415,425 ----
private:
ShenandoahHeap* const _heap;
public:
ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
! void heap_region_do(ShenandoahHeapRegion* r) {
if (r->is_trash()) {
r->recycle();
}
if (r->is_cset()) {
r->make_regular_bypass();
*** 421,431 ****
assert (r->is_committed(), err_msg("only committed regions in heap now, see region " SIZE_FORMAT, r->region_number()));
// Record current region occupancy: this communicates empty regions are free
// to the rest of Full GC code.
r->set_new_top(r->top());
- return false;
}
};
class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure {
private:
--- 430,439 ----
*** 435,447 ****
public:
ShenandoahTrashImmediateGarbageClosure() :
_heap(ShenandoahHeap::heap()),
_ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
! bool heap_region_do(ShenandoahHeapRegion* r) {
if (r->is_humongous_start()) {
! oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
if (!_ctx->is_marked(humongous_obj)) {
assert(!r->has_live(),
err_msg("Region " SIZE_FORMAT " is not marked, should not have live", r->region_number()));
_heap->trash_humongous_region_at(r);
} else {
--- 443,455 ----
public:
ShenandoahTrashImmediateGarbageClosure() :
_heap(ShenandoahHeap::heap()),
_ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
! void heap_region_do(ShenandoahHeapRegion* r) {
if (r->is_humongous_start()) {
! oop humongous_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
if (!_ctx->is_marked(humongous_obj)) {
assert(!r->has_live(),
err_msg("Region " SIZE_FORMAT " is not marked, should not have live", r->region_number()));
_heap->trash_humongous_region_at(r);
} else {
*** 452,486 ****
// If we hit continuation, the non-live humongous starts should have been trashed already
assert(r->humongous_start_region()->has_live(),
err_msg("Region " SIZE_FORMAT " should have live", r->region_number()));
} else if (r->is_regular()) {
if (!r->has_live()) {
! assert(_ctx->is_bitmap_clear_range(r->bottom(), r->end()),
! err_msg("Region " SIZE_FORMAT " should not have marks in bitmap", r->region_number()));
! r->make_trash();
}
}
- return false;
}
};
void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
GCTraceTime time("Phase 2: Compute new object addresses", ShenandoahLogDebug, _gc_timer, heap->tracer()->gc_id());
ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
{
- ShenandoahHeapLocker lock(heap->lock());
-
// Trash the immediately collectible regions before computing addresses
ShenandoahTrashImmediateGarbageClosure tigcl;
! heap->heap_region_iterate(&tigcl, false, false);
// Make sure regions are in good state: committed, active, clean.
// This is needed because we are potentially sliding the data through them.
ShenandoahEnsureHeapActiveClosure ecl;
! heap->heap_region_iterate(&ecl, false, false);
}
// Compute the new addresses for regular objects
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
--- 460,489 ----
// If we hit continuation, the non-live humongous starts should have been trashed already
assert(r->humongous_start_region()->has_live(),
err_msg("Region " SIZE_FORMAT " should have live", r->region_number()));
} else if (r->is_regular()) {
if (!r->has_live()) {
! r->make_trash_immediate();
}
}
}
};
void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
GCTraceTime time("Phase 2: Compute new object addresses", ShenandoahLogDebug, _gc_timer, heap->tracer()->gc_id());
ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
{
// Trash the immediately collectible regions before computing addresses
ShenandoahTrashImmediateGarbageClosure tigcl;
! heap->heap_region_iterate(&tigcl);
// Make sure regions are in good state: committed, active, clean.
// This is needed because we are potentially sliding the data through them.
ShenandoahEnsureHeapActiveClosure ecl;
! heap->heap_region_iterate(&ecl);
}
// Compute the new addresses for regular objects
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
*** 504,514 ****
inline void do_oop_work(T* p) {
T o = oopDesc::load_heap_oop(p);
if (! oopDesc::is_null(o)) {
oop obj = oopDesc::decode_heap_oop_not_null(o);
assert(_ctx->is_marked(obj), "must be marked");
! oop forw = oop(BrooksPointer::get_raw(obj));
oopDesc::encode_store_heap_oop(p, forw);
}
}
public:
--- 507,517 ----
inline void do_oop_work(T* p) {
T o = oopDesc::load_heap_oop(p);
if (! oopDesc::is_null(o)) {
oop obj = oopDesc::decode_heap_oop_not_null(o);
assert(_ctx->is_marked(obj), "must be marked");
! oop forw = oop(ShenandoahBrooksPointer::get_raw(obj));
oopDesc::encode_store_heap_oop(p, forw);
}
}
public:
*** 606,622 ****
ShenandoahCompactObjectsClosure() : _heap(ShenandoahHeap::heap()) {}
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
size_t size = (size_t)p->size();
! HeapWord* compact_to = BrooksPointer::get_raw(p);
HeapWord* compact_from = (HeapWord*) p;
if (compact_from != compact_to) {
Copy::aligned_conjoint_words(compact_from, compact_to, size);
}
oop new_obj = oop(compact_to);
! BrooksPointer::initialize(new_obj);
}
};
class ShenandoahCompactObjectsTask : public AbstractGangTask {
private:
--- 609,625 ----
ShenandoahCompactObjectsClosure() : _heap(ShenandoahHeap::heap()) {}
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
size_t size = (size_t)p->size();
! HeapWord* compact_to = ShenandoahBrooksPointer::get_raw(p);
HeapWord* compact_from = (HeapWord*) p;
if (compact_from != compact_to) {
Copy::aligned_conjoint_words(compact_from, compact_to, size);
}
oop new_obj = oop(compact_to);
! ShenandoahBrooksPointer::initialize(new_obj);
}
};
class ShenandoahCompactObjectsTask : public AbstractGangTask {
private:
*** 654,673 ****
public:
ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0) {
_heap->free_set()->clear();
}
! bool heap_region_do(ShenandoahHeapRegion* r) {
assert (!r->is_cset(), "cset regions should have been demoted already");
// Need to reset the complete-top-at-mark-start pointer here because
// the complete marking bitmap is no longer valid. This ensures
// size-based iteration in marked_object_iterate().
// NOTE: See blurb at ShenandoahMCResetCompleteBitmapTask on why we need to skip
// pinned regions.
if (!r->is_pinned()) {
! _heap->complete_marking_context()->set_top_at_mark_start(r->region_number(), r->bottom());
}
size_t live = r->used();
// Make empty regions that have been allocated into regular
--- 657,676 ----
public:
ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0) {
_heap->free_set()->clear();
}
! void heap_region_do(ShenandoahHeapRegion* r) {
assert (!r->is_cset(), "cset regions should have been demoted already");
// Need to reset the complete-top-at-mark-start pointer here because
// the complete marking bitmap is no longer valid. This ensures
// size-based iteration in marked_object_iterate().
// NOTE: See blurb at ShenandoahMCResetCompleteBitmapTask on why we need to skip
// pinned regions.
if (!r->is_pinned()) {
! _heap->complete_marking_context()->reset_top_at_mark_start(r);
}
size_t live = r->used();
// Make empty regions that have been allocated into regular
*** 687,697 ****
}
r->set_live_data(live);
r->reset_alloc_metadata_to_shared();
_live += live;
- return false;
}
size_t get_live() {
return _live;
}
--- 690,699 ----
*** 707,723 ****
ShenandoahHeap* heap = ShenandoahHeap::heap();
for (size_t c = heap->num_regions() - 1; c > 0; c--) {
ShenandoahHeapRegion* r = heap->get_region(c);
if (r->is_humongous_start()) {
! oop old_obj = oop(r->bottom() + BrooksPointer::word_size());
! size_t words_size = old_obj->size() + BrooksPointer::word_size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t old_start = r->region_number();
size_t old_end = old_start + num_regions - 1;
! size_t new_start = heap->heap_region_index_containing(BrooksPointer::get_raw(old_obj));
size_t new_end = new_start + num_regions - 1;
if (old_start == new_start) {
// No need to move the object, it stays at the same slot
continue;
--- 709,725 ----
ShenandoahHeap* heap = ShenandoahHeap::heap();
for (size_t c = heap->num_regions() - 1; c > 0; c--) {
ShenandoahHeapRegion* r = heap->get_region(c);
if (r->is_humongous_start()) {
! oop old_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
! size_t words_size = old_obj->size() + ShenandoahBrooksPointer::word_size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t old_start = r->region_number();
size_t old_end = old_start + num_regions - 1;
! size_t new_start = heap->heap_region_index_containing(ShenandoahBrooksPointer::get_raw(old_obj));
size_t new_end = new_start + num_regions - 1;
if (old_start == new_start) {
// No need to move the object, it stays at the same slot
continue;
*** 727,742 ****
Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
heap->get_region(new_start)->bottom(),
ShenandoahHeapRegion::region_size_words()*num_regions);
! oop new_obj = oop(heap->get_region(new_start)->bottom() + BrooksPointer::word_size());
! BrooksPointer::initialize(new_obj);
{
- ShenandoahHeapLocker lock(heap->lock());
-
for (size_t c = old_start; c <= old_end; c++) {
ShenandoahHeapRegion* r = heap->get_region(c);
r->make_regular_bypass();
r->set_top(r->bottom());
}
--- 729,742 ----
Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
heap->get_region(new_start)->bottom(),
ShenandoahHeapRegion::region_size_words()*num_regions);
! oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahBrooksPointer::word_size());
! ShenandoahBrooksPointer::initialize(new_obj);
{
for (size_t c = old_start; c <= old_end; c++) {
ShenandoahHeapRegion* r = heap->get_region(c);
r->make_regular_bypass();
r->set_top(r->bottom());
}
*** 762,772 ****
}
}
}
}
-
// This is slightly different to ShHeap::reset_next_mark_bitmap:
// we need to remain able to walk pinned regions.
// Since pinned region do not move and don't get compacted, we will get holes with
// unreachable objects in them (which may have pointers to unloaded Klasses and thus
// cannot be iterated over using oop->size(). The only way to safely iterate over those is using
--- 762,771 ----
*** 784,800 ****
void work(uint worker_id) {
ShenandoahHeapRegion* region = _regions.next();
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
while (region != NULL) {
! if (heap->is_bitmap_slice_committed(region) && !region->is_pinned()) {
! HeapWord* bottom = region->bottom();
! HeapWord* top = ctx->top_at_mark_start(region->region_number());
! if (top > bottom && region->has_live()) {
! ctx->clear_bitmap(bottom, top);
! }
! assert(ctx->is_bitmap_clear_range(bottom, region->end()), "must be clear");
}
region = _regions.next();
}
}
};
--- 783,794 ----
void work(uint worker_id) {
ShenandoahHeapRegion* region = _regions.next();
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
while (region != NULL) {
! if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
! ctx->clear_bitmap(region);
}
region = _regions.next();
}
}
};
*** 827,848 ****
// Bring regions in proper states after the collection, and set heap properties.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
- ShenandoahHeapLocker lock(heap->lock());
ShenandoahPostCompactClosure post_compact;
heap->heap_region_iterate(&post_compact);
heap->set_used(post_compact.get_live());
heap->collection_set()->clear();
heap->free_set()->rebuild();
}
heap->clear_cancelled_gc();
-
- // Also clear the next bitmap in preparation for next marking.
- {
- ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_next);
- heap->reset_next_mark_bitmap();
- }
}
--- 821,835 ----
< prev index next >