< prev index next >
src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
Print this page
rev 55608 : Rename ShenandoahBrooksPointer to ShenandoahForwarding
rev 55609 : Eliminate extra forwarding pointer per object
*** 232,285 ****
return ShenandoahBarrierSet::resolve_forwarded(p);
}
assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
! size_t size_no_fwdptr = (size_t) p->size();
! size_t size_with_fwdptr = size_no_fwdptr + ShenandoahForwarding::word_size();
assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
bool alloc_from_gclab = true;
! HeapWord* filler = NULL;
#ifdef ASSERT
if (ShenandoahOOMDuringEvacALot &&
(os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
! filler = NULL;
} else {
#endif
if (UseTLAB) {
! filler = allocate_from_gclab(thread, size_with_fwdptr);
}
! if (filler == NULL) {
! ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size_with_fwdptr);
! filler = allocate_memory(req);
alloc_from_gclab = false;
}
#ifdef ASSERT
}
#endif
! if (filler == NULL) {
! control_thread()->handle_alloc_failure_evac(size_with_fwdptr);
_oom_evac_handler.handle_out_of_memory_during_evacuation();
return ShenandoahBarrierSet::resolve_forwarded(p);
}
! // Copy the object and initialize its forwarding ptr:
! HeapWord* copy = filler + ShenandoahForwarding::word_size();
! oop copy_val = oop(copy);
!
! Copy::aligned_disjoint_words((HeapWord*) p, copy, size_no_fwdptr);
! ShenandoahForwarding::initialize(oop(copy));
// Try to install the new forwarding pointer.
oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
-
if (oopDesc::equals_raw(result, p)) {
// Successfully evacuated. Our copy is now the public one!
shenandoah_assert_correct(NULL, copy_val);
return copy_val;
} else {
--- 232,280 ----
return ShenandoahBarrierSet::resolve_forwarded(p);
}
assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
! size_t size = p->size();
assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
bool alloc_from_gclab = true;
! HeapWord* copy = NULL;
#ifdef ASSERT
if (ShenandoahOOMDuringEvacALot &&
(os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
! copy = NULL;
} else {
#endif
if (UseTLAB) {
! copy = allocate_from_gclab(thread, size);
}
! if (copy == NULL) {
! ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size);
! copy = allocate_memory(req);
alloc_from_gclab = false;
}
#ifdef ASSERT
}
#endif
! if (copy == NULL) {
! control_thread()->handle_alloc_failure_evac(size);
_oom_evac_handler.handle_out_of_memory_during_evacuation();
return ShenandoahBarrierSet::resolve_forwarded(p);
}
! // Copy the object:
! Copy::aligned_disjoint_words((HeapWord*) p, copy, size);
// Try to install the new forwarding pointer.
+ oop copy_val = oop(copy);
oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
if (oopDesc::equals_raw(result, p)) {
// Successfully evacuated. Our copy is now the public one!
shenandoah_assert_correct(NULL, copy_val);
return copy_val;
} else {
*** 293,307 ****
// object will overwrite this stale copy, or the filler object on LAB retirement will
// do this. For non-GCLAB allocations, we have no way to retract the allocation, and
// have to explicitly overwrite the copy with the filler object. With that overwrite,
// we have to keep the fwdptr initialized and pointing to our (stale) copy.
if (alloc_from_gclab) {
! ShenandoahThreadLocalData::gclab(thread)->undo_allocation(filler, size_with_fwdptr);
} else {
! fill_with_object(copy, size_no_fwdptr);
! }
shenandoah_assert_correct(NULL, copy_val);
shenandoah_assert_correct(NULL, result);
return result;
}
}
--- 288,302 ----
// object will overwrite this stale copy, or the filler object on LAB retirement will
// do this. For non-GCLAB allocations, we have no way to retract the allocation, and
// have to explicitly overwrite the copy with the filler object. With that overwrite,
// we have to keep the fwdptr initialized and pointing to our (stale) copy.
if (alloc_from_gclab) {
! ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
} else {
! fill_with_object(copy, size);
shenandoah_assert_correct(NULL, copy_val);
+ }
shenandoah_assert_correct(NULL, result);
return result;
}
}
*** 368,390 ****
marked_object_iterate(region, cl, region->top());
}
template<class T>
inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) {
- assert(ShenandoahForwarding::word_offset() < 0, "skip_delta calculation below assumes the forwarding ptr is before obj");
assert(! region->is_humongous_continuation(), "no humongous continuation regions here");
ShenandoahMarkingContext* const ctx = complete_marking_context();
assert(ctx->is_complete(), "sanity");
MarkBitMap* mark_bit_map = ctx->mark_bit_map();
HeapWord* tams = ctx->top_at_mark_start(region);
! size_t skip_bitmap_delta = ShenandoahForwarding::word_size() + 1;
! size_t skip_objsize_delta = ShenandoahForwarding::word_size() /* + actual obj.size() below */;
! HeapWord* start = region->bottom() + ShenandoahForwarding::word_size();
! HeapWord* end = MIN2(tams + ShenandoahForwarding::word_size(), region->end());
// Step 1. Scan below the TAMS based on bitmap data.
HeapWord* limit_bitmap = MIN2(limit, tams);
// Try to scan the initial candidate. If the candidate is above the TAMS, it would
--- 363,383 ----
marked_object_iterate(region, cl, region->top());
}
template<class T>
inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) {
assert(! region->is_humongous_continuation(), "no humongous continuation regions here");
ShenandoahMarkingContext* const ctx = complete_marking_context();
assert(ctx->is_complete(), "sanity");
MarkBitMap* mark_bit_map = ctx->mark_bit_map();
HeapWord* tams = ctx->top_at_mark_start(region);
! size_t skip_bitmap_delta = 1;
! HeapWord* start = region->bottom();
! HeapWord* end = MIN2(tams, region->end());
// Step 1. Scan below the TAMS based on bitmap data.
HeapWord* limit_bitmap = MIN2(limit, tams);
// Try to scan the initial candidate. If the candidate is above the TAMS, it would
*** 410,420 ****
int avail;
do {
avail = 0;
for (int c = 0; (c < dist) && (cb < limit_bitmap); c++) {
! Prefetch::read(cb, ShenandoahForwarding::byte_offset());
slots[avail++] = cb;
cb += skip_bitmap_delta;
if (cb < limit_bitmap) {
cb = mark_bit_map->get_next_marked_addr(cb, limit_bitmap);
}
--- 403,413 ----
int avail;
do {
avail = 0;
for (int c = 0; (c < dist) && (cb < limit_bitmap); c++) {
! Prefetch::read(cb, oopDesc::mark_offset_in_bytes());
slots[avail++] = cb;
cb += skip_bitmap_delta;
if (cb < limit_bitmap) {
cb = mark_bit_map->get_next_marked_addr(cb, limit_bitmap);
}
*** 445,464 ****
}
// Step 2. Accurate size-based traversal, happens past the TAMS.
// This restarts the scan at TAMS, which makes sure we traverse all objects,
// regardless of what happened at Step 1.
! HeapWord* cs = tams + ShenandoahForwarding::word_size();
while (cs < limit) {
! assert (cs > tams, "only objects past TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams));
assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit));
oop obj = oop(cs);
assert(oopDesc::is_oop(obj), "sanity");
assert(ctx->is_marked(obj), "object expected to be marked");
int size = obj->size();
cl->do_object(obj);
! cs += size + skip_objsize_delta;
}
}
template <class T>
class ShenandoahObjectToOopClosure : public ObjectClosure {
--- 438,457 ----
}
// Step 2. Accurate size-based traversal, happens past the TAMS.
// This restarts the scan at TAMS, which makes sure we traverse all objects,
// regardless of what happened at Step 1.
! HeapWord* cs = tams;
while (cs < limit) {
! assert (cs >= tams, "only objects past TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams));
assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit));
oop obj = oop(cs);
assert(oopDesc::is_oop(obj), "sanity");
assert(ctx->is_marked(obj), "object expected to be marked");
int size = obj->size();
cl->do_object(obj);
! cs += size;
}
}
template <class T>
class ShenandoahObjectToOopClosure : public ObjectClosure {
< prev index next >