< prev index next >
src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp
Print this page
rev 55608 : Rename ShenandoahBrooksPointer to ShenandoahForwarding
rev 55609 : Eliminate extra forwarding pointer per object
@@ -23,11 +23,10 @@
#include "precompiled.hpp"
#include "code/codeCache.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
-#include "gc/shenandoah/shenandoahForwarding.inline.hpp"
#include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahMarkCompact.hpp"
@@ -226,19 +225,21 @@
heap->mark_complete_marking_context();
}
class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
private:
+ ShenandoahMarkCompact* const _mark_compact;
ShenandoahHeap* const _heap;
GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
int _empty_regions_pos;
ShenandoahHeapRegion* _to_region;
ShenandoahHeapRegion* _from_region;
HeapWord* _compact_point;
public:
- ShenandoahPrepareForCompactionObjectClosure(GrowableArray<ShenandoahHeapRegion*>& empty_regions, ShenandoahHeapRegion* to_region) :
+ ShenandoahPrepareForCompactionObjectClosure(ShenandoahMarkCompact* mc, GrowableArray<ShenandoahHeapRegion*>& empty_regions, ShenandoahHeapRegion* to_region) :
+ _mark_compact(mc),
_heap(ShenandoahHeap::heap()),
_empty_regions(empty_regions),
_empty_regions_pos(0),
_to_region(to_region),
_from_region(NULL),
@@ -264,11 +265,11 @@
void do_object(oop p) {
assert(_from_region != NULL, "must set before work");
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
- size_t obj_size = p->size() + ShenandoahForwarding::word_size();
+ size_t obj_size = p->size();
if (_compact_point + obj_size > _to_region->end()) {
finish_region();
// Object doesn't fit. Pick next empty region and start compacting there.
ShenandoahHeapRegion* new_to_region;
@@ -287,17 +288,19 @@
}
// Object fits into current region, record new location:
assert(_compact_point + obj_size <= _to_region->end(), "must fit");
shenandoah_assert_not_forwarded(NULL, p);
- ShenandoahForwarding::set_forwardee_raw(p, _compact_point + ShenandoahForwarding::word_size());
+ _mark_compact->preserve_mark(p);
+ p->forward_to(oop(_compact_point));
_compact_point += obj_size;
}
};
class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
private:
+ ShenandoahMarkCompact* const _mark_compact;
ShenandoahHeap* const _heap;
ShenandoahHeapRegionSet** const _worker_slices;
ShenandoahRegionIterator _heap_regions;
ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* slice) {
@@ -316,12 +319,13 @@
return from_region;
}
public:
- ShenandoahPrepareForCompactionTask(ShenandoahHeapRegionSet** worker_slices) :
+ ShenandoahPrepareForCompactionTask(ShenandoahMarkCompact* mc, ShenandoahHeapRegionSet** worker_slices) :
AbstractGangTask("Shenandoah Prepare For Compaction Task"),
+ _mark_compact(mc),
_heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
}
void work(uint worker_id) {
ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
@@ -333,11 +337,11 @@
// Sliding compaction. Walk all regions in the slice, and compact them.
// Remember empty regions and reuse them as needed.
ResourceMark rm;
GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
- ShenandoahPrepareForCompactionObjectClosure cl(empty_regions, from_region);
+ ShenandoahPrepareForCompactionObjectClosure cl(_mark_compact, empty_regions, from_region);
while (from_region != NULL) {
cl.set_from_region(from_region);
if (from_region->has_live()) {
_heap->marked_object_iterate(from_region, &cl);
}
@@ -373,29 +377,29 @@
// detected, then sliding restarts towards that non-movable region.
size_t to_begin = heap->num_regions();
size_t to_end = heap->num_regions();
- for (size_t c = heap->num_regions() - 1; c > 0; c--) {
- ShenandoahHeapRegion *r = heap->get_region(c);
+ for (size_t c = heap->num_regions(); c > 0; c--) {
+ ShenandoahHeapRegion *r = heap->get_region(c - 1);
if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
// To-region candidate: record this, and continue scan
to_begin = r->region_number();
continue;
}
if (r->is_humongous_start() && r->is_move_allowed()) {
// From-region candidate: movable humongous region
- oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
- size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
+ oop old_obj = oop(r->bottom());
+ size_t words_size = old_obj->size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t start = to_end - num_regions;
if (start >= to_begin && start != r->region_number()) {
// Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
- ShenandoahForwarding::set_forwardee_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahForwarding::word_size());
+ old_obj->forward_to(oop(heap->get_region(start)->bottom()));
to_end = start;
continue;
}
}
@@ -439,11 +443,11 @@
_heap(ShenandoahHeap::heap()),
_ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
void heap_region_do(ShenandoahHeapRegion* r) {
if (r->is_humongous_start()) {
- oop humongous_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
+ oop humongous_obj = oop(r->bottom());
if (!_ctx->is_marked(humongous_obj)) {
assert(!r->has_live(),
"Region " SIZE_FORMAT " is not marked, should not have live", r->region_number());
_heap->trash_humongous_region_at(r);
} else {
@@ -480,11 +484,11 @@
}
// Compute the new addresses for regular objects
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
- ShenandoahPrepareForCompactionTask prepare_task(worker_slices);
+ ShenandoahPrepareForCompactionTask prepare_task(this, worker_slices);
heap->workers()->run_task(&prepare_task);
}
// Compute the new addresses for humongous objects
{
@@ -502,14 +506,16 @@
inline void do_oop_work(T* p) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
assert(_ctx->is_marked(obj), "must be marked");
- oop forw = oop(ShenandoahForwarding::get_forwardee_raw(obj));
+ if (obj->is_forwarded()) {
+ oop forw = obj->forwardee();
RawAccess<IS_NOT_NULL>::oop_store(p, forw);
}
}
+ }
public:
ShenandoahAdjustPointersClosure() :
_heap(ShenandoahHeap::heap()),
_ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
@@ -527,11 +533,10 @@
ShenandoahAdjustPointersObjectClosure() :
_heap(ShenandoahHeap::heap()) {
}
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
- HeapWord* forw = ShenandoahForwarding::get_forwardee_raw(p);
p->oop_iterate(&_cl);
}
};
class ShenandoahAdjustPointersTask : public AbstractGangTask {
@@ -598,10 +603,12 @@
#endif
}
ShenandoahAdjustPointersTask adjust_pointers_task;
workers->run_task(&adjust_pointers_task);
+
+ adjust_marks();
}
class ShenandoahCompactObjectsClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
@@ -612,17 +619,17 @@
_heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
size_t size = (size_t)p->size();
- HeapWord* compact_to = ShenandoahForwarding::get_forwardee_raw(p);
+ if (p->is_forwarded()) {
HeapWord* compact_from = (HeapWord*) p;
- if (compact_from != compact_to) {
+ HeapWord* compact_to = (HeapWord*) p->forwardee();
Copy::aligned_conjoint_words(compact_from, compact_to, size);
- }
oop new_obj = oop(compact_to);
- ShenandoahForwarding::initialize(new_obj);
+ new_obj->init_mark();
+ }
}
};
class ShenandoahCompactObjectsTask : public AbstractGangTask {
private:
@@ -709,35 +716,34 @@
// humongous regions are already compacted, and do not require further moves, which alleviates
// sliding costs. We may consider doing this in parallel in future.
ShenandoahHeap* heap = ShenandoahHeap::heap();
- for (size_t c = heap->num_regions() - 1; c > 0; c--) {
- ShenandoahHeapRegion* r = heap->get_region(c);
+ for (size_t c = heap->num_regions(); c > 0; c--) {
+ ShenandoahHeapRegion* r = heap->get_region(c - 1);
if (r->is_humongous_start()) {
- oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
- size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
+ oop old_obj = oop(r->bottom());
+ if (!old_obj->is_forwarded()) {
+ // No need to move the object, it stays at the same slot
+ continue;
+ }
+ size_t words_size = old_obj->size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t old_start = r->region_number();
size_t old_end = old_start + num_regions - 1;
- size_t new_start = heap->heap_region_index_containing(ShenandoahForwarding::get_forwardee_raw(old_obj));
+ size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
size_t new_end = new_start + num_regions - 1;
-
- if (old_start == new_start) {
- // No need to move the object, it stays at the same slot
- continue;
- }
-
+ assert(old_start != new_start, "must be real move");
assert (r->is_move_allowed(), "should be movable");
Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
heap->get_region(new_start)->bottom(),
ShenandoahHeapRegion::region_size_words()*num_regions);
- oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahForwarding::word_size());
- ShenandoahForwarding::initialize(new_obj);
+ oop new_obj = oop(heap->get_region(new_start)->bottom());
+ new_obj->init_mark();
{
for (size_t c = old_start; c <= old_end; c++) {
ShenandoahHeapRegion* r = heap->get_region(c);
r->make_regular_bypass();
@@ -813,10 +819,12 @@
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
compact_humongous_objects();
}
+ restore_marks();
+
// Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
// and must ensure the bitmap is in sync.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_complete);
ShenandoahMCResetCompleteBitmapTask task;
@@ -835,5 +843,40 @@
heap->free_set()->rebuild();
}
heap->clear_cancelled_gc();
}
+
+void ShenandoahMarkCompact::preserve_mark(oop obj) {
+ markOop mark = obj->mark_raw();
+ if (mark->must_be_preserved(obj)) {
+ MutexLocker ml(&_preserved_mark_lock, Mutex::_no_safepoint_check_flag);
+ assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
+ "inconsistent preserved oop stacks: oop_stack size: " SIZE_FORMAT ", mark_stack size: " SIZE_FORMAT, _preserved_oop_stack.size(), _preserved_mark_stack.size());
+ _preserved_mark_stack.push(mark);
+ _preserved_oop_stack.push(obj);
+ }
+}
+
+void ShenandoahMarkCompact::restore_marks() {
+ MutexLocker ml(&_preserved_mark_lock, Mutex::_no_safepoint_check_flag);
+ assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
+ "inconsistent preserved oop stacks: oop_stack size: " SIZE_FORMAT ", mark_stack size: " SIZE_FORMAT, _preserved_oop_stack.size(), _preserved_mark_stack.size());
+ while (!_preserved_oop_stack.is_empty()) {
+ oop obj = _preserved_oop_stack.pop();
+ markOop mark = _preserved_mark_stack.pop();
+ obj->set_mark_raw(mark);
+ }
+}
+
+void ShenandoahMarkCompact::adjust_marks() {
+ MutexLocker ml(&_preserved_mark_lock, Mutex::_no_safepoint_check_flag);
+ assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
+ "inconsistent preserved oop stacks: oop_stack size: " SIZE_FORMAT ", mark_stack size: " SIZE_FORMAT, _preserved_oop_stack.size(), _preserved_mark_stack.size());
+ StackIterator<oop, mtGC> iter(_preserved_oop_stack);
+ while (!iter.is_empty()) {
+ oop* p = iter.next_addr();
+ oop obj = *p;
+ oop fwd = obj->forwardee();
+ *p = fwd;
+ }
+}
< prev index next >