< prev index next >
src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp
Print this page
rev 55608 : Rename ShenandoahBrooksPointer to ShenandoahForwarding
rev 55609 : Eliminate extra forwarding pointer per object
*** 23,33 ****
#include "precompiled.hpp"
#include "code/codeCache.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
- #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
#include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahMarkCompact.hpp"
--- 23,32 ----
*** 226,244 ****
heap->mark_complete_marking_context();
}
class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
int _empty_regions_pos;
ShenandoahHeapRegion* _to_region;
ShenandoahHeapRegion* _from_region;
HeapWord* _compact_point;
public:
! ShenandoahPrepareForCompactionObjectClosure(GrowableArray<ShenandoahHeapRegion*>& empty_regions, ShenandoahHeapRegion* to_region) :
_heap(ShenandoahHeap::heap()),
_empty_regions(empty_regions),
_empty_regions_pos(0),
_to_region(to_region),
_from_region(NULL),
--- 225,245 ----
heap->mark_complete_marking_context();
}
class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
private:
+ ShenandoahMarkCompact* const _mark_compact;
ShenandoahHeap* const _heap;
GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
int _empty_regions_pos;
ShenandoahHeapRegion* _to_region;
ShenandoahHeapRegion* _from_region;
HeapWord* _compact_point;
public:
! ShenandoahPrepareForCompactionObjectClosure(ShenandoahMarkCompact* mc, GrowableArray<ShenandoahHeapRegion*>& empty_regions, ShenandoahHeapRegion* to_region) :
! _mark_compact(mc),
_heap(ShenandoahHeap::heap()),
_empty_regions(empty_regions),
_empty_regions_pos(0),
_to_region(to_region),
_from_region(NULL),
*** 264,274 ****
void do_object(oop p) {
assert(_from_region != NULL, "must set before work");
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
! size_t obj_size = p->size() + ShenandoahForwarding::word_size();
if (_compact_point + obj_size > _to_region->end()) {
finish_region();
// Object doesn't fit. Pick next empty region and start compacting there.
ShenandoahHeapRegion* new_to_region;
--- 265,275 ----
void do_object(oop p) {
assert(_from_region != NULL, "must set before work");
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
! size_t obj_size = p->size();
if (_compact_point + obj_size > _to_region->end()) {
finish_region();
// Object doesn't fit. Pick next empty region and start compacting there.
ShenandoahHeapRegion* new_to_region;
*** 287,303 ****
}
// Object fits into current region, record new location:
assert(_compact_point + obj_size <= _to_region->end(), "must fit");
shenandoah_assert_not_forwarded(NULL, p);
! ShenandoahForwarding::set_forwardee_raw(p, _compact_point + ShenandoahForwarding::word_size());
_compact_point += obj_size;
}
};
class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
private:
ShenandoahHeap* const _heap;
ShenandoahHeapRegionSet** const _worker_slices;
ShenandoahRegionIterator _heap_regions;
ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* slice) {
--- 288,306 ----
}
// Object fits into current region, record new location:
assert(_compact_point + obj_size <= _to_region->end(), "must fit");
shenandoah_assert_not_forwarded(NULL, p);
! _mark_compact->preserve_mark(p);
! p->forward_to(oop(_compact_point));
_compact_point += obj_size;
}
};
class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
private:
+ ShenandoahMarkCompact* const _mark_compact;
ShenandoahHeap* const _heap;
ShenandoahHeapRegionSet** const _worker_slices;
ShenandoahRegionIterator _heap_regions;
ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* slice) {
*** 316,327 ****
return from_region;
}
public:
! ShenandoahPrepareForCompactionTask(ShenandoahHeapRegionSet** worker_slices) :
AbstractGangTask("Shenandoah Prepare For Compaction Task"),
_heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
}
void work(uint worker_id) {
ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
--- 319,331 ----
return from_region;
}
public:
! ShenandoahPrepareForCompactionTask(ShenandoahMarkCompact* mc, ShenandoahHeapRegionSet** worker_slices) :
AbstractGangTask("Shenandoah Prepare For Compaction Task"),
+ _mark_compact(mc),
_heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
}
void work(uint worker_id) {
ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
*** 333,343 ****
// Sliding compaction. Walk all regions in the slice, and compact them.
// Remember empty regions and reuse them as needed.
ResourceMark rm;
GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
! ShenandoahPrepareForCompactionObjectClosure cl(empty_regions, from_region);
while (from_region != NULL) {
cl.set_from_region(from_region);
if (from_region->has_live()) {
_heap->marked_object_iterate(from_region, &cl);
}
--- 337,347 ----
// Sliding compaction. Walk all regions in the slice, and compact them.
// Remember empty regions and reuse them as needed.
ResourceMark rm;
GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
! ShenandoahPrepareForCompactionObjectClosure cl(_mark_compact, empty_regions, from_region);
while (from_region != NULL) {
cl.set_from_region(from_region);
if (from_region->has_live()) {
_heap->marked_object_iterate(from_region, &cl);
}
*** 373,401 ****
// detected, then sliding restarts towards that non-movable region.
size_t to_begin = heap->num_regions();
size_t to_end = heap->num_regions();
! for (size_t c = heap->num_regions() - 1; c > 0; c--) {
! ShenandoahHeapRegion *r = heap->get_region(c);
if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
// To-region candidate: record this, and continue scan
to_begin = r->region_number();
continue;
}
if (r->is_humongous_start() && r->is_move_allowed()) {
// From-region candidate: movable humongous region
! oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
! size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t start = to_end - num_regions;
if (start >= to_begin && start != r->region_number()) {
// Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
! ShenandoahForwarding::set_forwardee_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahForwarding::word_size());
to_end = start;
continue;
}
}
--- 377,405 ----
// detected, then sliding restarts towards that non-movable region.
size_t to_begin = heap->num_regions();
size_t to_end = heap->num_regions();
! for (size_t c = heap->num_regions(); c > 0; c--) {
! ShenandoahHeapRegion *r = heap->get_region(c - 1);
if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
// To-region candidate: record this, and continue scan
to_begin = r->region_number();
continue;
}
if (r->is_humongous_start() && r->is_move_allowed()) {
// From-region candidate: movable humongous region
! oop old_obj = oop(r->bottom());
! size_t words_size = old_obj->size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t start = to_end - num_regions;
if (start >= to_begin && start != r->region_number()) {
// Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
! old_obj->forward_to(oop(heap->get_region(start)->bottom()));
to_end = start;
continue;
}
}
*** 439,449 ****
_heap(ShenandoahHeap::heap()),
_ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
void heap_region_do(ShenandoahHeapRegion* r) {
if (r->is_humongous_start()) {
! oop humongous_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
if (!_ctx->is_marked(humongous_obj)) {
assert(!r->has_live(),
"Region " SIZE_FORMAT " is not marked, should not have live", r->region_number());
_heap->trash_humongous_region_at(r);
} else {
--- 443,453 ----
_heap(ShenandoahHeap::heap()),
_ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
void heap_region_do(ShenandoahHeapRegion* r) {
if (r->is_humongous_start()) {
! oop humongous_obj = oop(r->bottom());
if (!_ctx->is_marked(humongous_obj)) {
assert(!r->has_live(),
"Region " SIZE_FORMAT " is not marked, should not have live", r->region_number());
_heap->trash_humongous_region_at(r);
} else {
*** 480,490 ****
}
// Compute the new addresses for regular objects
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
! ShenandoahPrepareForCompactionTask prepare_task(worker_slices);
heap->workers()->run_task(&prepare_task);
}
// Compute the new addresses for humongous objects
{
--- 484,494 ----
}
// Compute the new addresses for regular objects
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
! ShenandoahPrepareForCompactionTask prepare_task(this, worker_slices);
heap->workers()->run_task(&prepare_task);
}
// Compute the new addresses for humongous objects
{
*** 502,515 ****
inline void do_oop_work(T* p) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
assert(_ctx->is_marked(obj), "must be marked");
! oop forw = oop(ShenandoahForwarding::get_forwardee_raw(obj));
RawAccess<IS_NOT_NULL>::oop_store(p, forw);
}
}
public:
ShenandoahAdjustPointersClosure() :
_heap(ShenandoahHeap::heap()),
_ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
--- 506,521 ----
inline void do_oop_work(T* p) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
assert(_ctx->is_marked(obj), "must be marked");
! if (obj->is_forwarded()) {
! oop forw = obj->forwardee();
RawAccess<IS_NOT_NULL>::oop_store(p, forw);
}
}
+ }
public:
ShenandoahAdjustPointersClosure() :
_heap(ShenandoahHeap::heap()),
_ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
*** 527,537 ****
ShenandoahAdjustPointersObjectClosure() :
_heap(ShenandoahHeap::heap()) {
}
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
- HeapWord* forw = ShenandoahForwarding::get_forwardee_raw(p);
p->oop_iterate(&_cl);
}
};
class ShenandoahAdjustPointersTask : public AbstractGangTask {
--- 533,542 ----
*** 598,607 ****
--- 603,614 ----
#endif
}
ShenandoahAdjustPointersTask adjust_pointers_task;
workers->run_task(&adjust_pointers_task);
+
+ adjust_marks();
}
class ShenandoahCompactObjectsClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
*** 612,628 ****
_heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
size_t size = (size_t)p->size();
! HeapWord* compact_to = ShenandoahForwarding::get_forwardee_raw(p);
HeapWord* compact_from = (HeapWord*) p;
! if (compact_from != compact_to) {
Copy::aligned_conjoint_words(compact_from, compact_to, size);
- }
oop new_obj = oop(compact_to);
! ShenandoahForwarding::initialize(new_obj);
}
};
class ShenandoahCompactObjectsTask : public AbstractGangTask {
private:
--- 619,635 ----
_heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
size_t size = (size_t)p->size();
! if (p->is_forwarded()) {
HeapWord* compact_from = (HeapWord*) p;
! HeapWord* compact_to = (HeapWord*) p->forwardee();
Copy::aligned_conjoint_words(compact_from, compact_to, size);
oop new_obj = oop(compact_to);
! new_obj->init_mark();
! }
}
};
class ShenandoahCompactObjectsTask : public AbstractGangTask {
private:
*** 709,743 ****
// humongous regions are already compacted, and do not require further moves, which alleviates
// sliding costs. We may consider doing this in parallel in future.
ShenandoahHeap* heap = ShenandoahHeap::heap();
! for (size_t c = heap->num_regions() - 1; c > 0; c--) {
! ShenandoahHeapRegion* r = heap->get_region(c);
if (r->is_humongous_start()) {
! oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
! size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t old_start = r->region_number();
size_t old_end = old_start + num_regions - 1;
! size_t new_start = heap->heap_region_index_containing(ShenandoahForwarding::get_forwardee_raw(old_obj));
size_t new_end = new_start + num_regions - 1;
!
! if (old_start == new_start) {
! // No need to move the object, it stays at the same slot
! continue;
! }
!
assert (r->is_move_allowed(), "should be movable");
Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
heap->get_region(new_start)->bottom(),
ShenandoahHeapRegion::region_size_words()*num_regions);
! oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahForwarding::word_size());
! ShenandoahForwarding::initialize(new_obj);
{
for (size_t c = old_start; c <= old_end; c++) {
ShenandoahHeapRegion* r = heap->get_region(c);
r->make_regular_bypass();
--- 716,749 ----
// humongous regions are already compacted, and do not require further moves, which alleviates
// sliding costs. We may consider doing this in parallel in future.
ShenandoahHeap* heap = ShenandoahHeap::heap();
! for (size_t c = heap->num_regions(); c > 0; c--) {
! ShenandoahHeapRegion* r = heap->get_region(c - 1);
if (r->is_humongous_start()) {
! oop old_obj = oop(r->bottom());
! if (!old_obj->is_forwarded()) {
! // No need to move the object, it stays at the same slot
! continue;
! }
! size_t words_size = old_obj->size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t old_start = r->region_number();
size_t old_end = old_start + num_regions - 1;
! size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
size_t new_end = new_start + num_regions - 1;
! assert(old_start != new_start, "must be real move");
assert (r->is_move_allowed(), "should be movable");
Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
heap->get_region(new_start)->bottom(),
ShenandoahHeapRegion::region_size_words()*num_regions);
! oop new_obj = oop(heap->get_region(new_start)->bottom());
! new_obj->init_mark();
{
for (size_t c = old_start; c <= old_end; c++) {
ShenandoahHeapRegion* r = heap->get_region(c);
r->make_regular_bypass();
*** 813,822 ****
--- 819,830 ----
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
compact_humongous_objects();
}
+ restore_marks();
+
// Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
// and must ensure the bitmap is in sync.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_complete);
ShenandoahMCResetCompleteBitmapTask task;
*** 835,839 ****
--- 843,882 ----
heap->free_set()->rebuild();
}
heap->clear_cancelled_gc();
}
+
+ void ShenandoahMarkCompact::preserve_mark(oop obj) {
+ markOop mark = obj->mark_raw();
+ if (mark->must_be_preserved(obj)) {
+ MutexLocker ml(&_preserved_mark_lock, Mutex::_no_safepoint_check_flag);
+ assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
+ "inconsistent preserved oop stacks: oop_stack size: " SIZE_FORMAT ", mark_stack size: " SIZE_FORMAT, _preserved_oop_stack.size(), _preserved_mark_stack.size());
+ _preserved_mark_stack.push(mark);
+ _preserved_oop_stack.push(obj);
+ }
+ }
+
+ void ShenandoahMarkCompact::restore_marks() {
+ MutexLocker ml(&_preserved_mark_lock, Mutex::_no_safepoint_check_flag);
+ assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
+ "inconsistent preserved oop stacks: oop_stack size: " SIZE_FORMAT ", mark_stack size: " SIZE_FORMAT, _preserved_oop_stack.size(), _preserved_mark_stack.size());
+ while (!_preserved_oop_stack.is_empty()) {
+ oop obj = _preserved_oop_stack.pop();
+ markOop mark = _preserved_mark_stack.pop();
+ obj->set_mark_raw(mark);
+ }
+ }
+
+ void ShenandoahMarkCompact::adjust_marks() {
+ MutexLocker ml(&_preserved_mark_lock, Mutex::_no_safepoint_check_flag);
+ assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
+ "inconsistent preserved oop stacks: oop_stack size: " SIZE_FORMAT ", mark_stack size: " SIZE_FORMAT, _preserved_oop_stack.size(), _preserved_mark_stack.size());
+ StackIterator<oop, mtGC> iter(_preserved_oop_stack);
+ while (!iter.is_empty()) {
+ oop* p = iter.next_addr();
+ oop obj = *p;
+ oop fwd = obj->forwardee();
+ *p = fwd;
+ }
+ }
< prev index next >