# HG changeset patch # User mlarsson # Date 1411633930 -7200 # Thu Sep 25 10:32:10 2014 +0200 # Node ID 5c8fe5b6d877fd5ed1fea6cbbed96bc8f727cea3 # Parent 4e985654db5bbe7a7b195fe489b42038e541e055 [mq]: demacro diff --git a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp --- a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp @@ -2095,17 +2095,13 @@ } // Support for compaction - void CompactibleFreeListSpace::prepare_for_compaction(CompactPoint* cp) { - SCAN_AND_FORWARD(cp,end,block_is_obj,block_size); + scan_and_forward(this, cp); // Prepare_for_compaction() uses the space between live objects // so that later phase can skip dead space quickly. So verification // of the free lists doesn't work after. } -#define obj_size(q) adjustObjectSize(oop(q)->size()) -#define adjust_obj_size(s) adjustObjectSize(s) - void CompactibleFreeListSpace::adjust_pointers() { // In other versions of adjust_pointers(), a bail out // based on the amount of live data in the generation @@ -2113,12 +2109,12 @@ // Cannot test used() == 0 here because the free lists have already // been mangled by the compaction. - SCAN_AND_ADJUST_POINTERS(adjust_obj_size); + scan_and_adjust_pointers(this); // See note about verification in prepare_for_compaction(). } void CompactibleFreeListSpace::compact() { - SCAN_AND_COMPACT(obj_size); + scan_and_compact(this); } // Fragmentation metric = 1 - [sum of (fbs**2) / (sum of fbs)**2] diff --git a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp --- a/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp @@ -405,6 +405,27 @@ size_t block_size_nopar(const HeapWord* p) const; bool block_is_obj_nopar(const HeapWord* p) const; + // Functions for scan_and_{forward,adjust_pointers,compact} support. + inline HeapWord* scan_limit() const { + return end(); + } + + inline bool scanned_block_is_obj(const HeapWord* addr) const { + return CompactibleFreeListSpace::block_is_obj(addr); // Avoid virtual call + } + + inline size_t scanned_block_size(const HeapWord* addr) const { + return CompactibleFreeListSpace::block_size(addr); // Avoid virtual call + } + + inline size_t adjust_obj_size(size_t size) const { + return adjustObjectSize(size); + } + + inline size_t obj_size(const HeapWord* addr) const { + return adjustObjectSize(oop(addr)->size()); + } + // Iteration support for promotion void save_marks(); bool no_allocs_since_save_marks(); diff --git a/src/share/vm/gc_implementation/g1/heapRegion.cpp b/src/share/vm/gc_implementation/g1/heapRegion.cpp --- a/src/share/vm/gc_implementation/g1/heapRegion.cpp +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp @@ -983,6 +983,10 @@ verify(VerifyOption_G1UsePrevMarking, /* failures */ &dummy); } +void HeapRegion::prepare_for_compaction(CompactPoint* cp) { + scan_and_forward(this, cp); +} + // G1OffsetTableContigSpace code; copied from space.cpp. Hope this can go // away eventually. @@ -1066,12 +1070,6 @@ } } -#define block_is_always_obj(q) true -void G1OffsetTableContigSpace::prepare_for_compaction(CompactPoint* cp) { - SCAN_AND_FORWARD(cp, top, block_is_always_obj, block_size); -} -#undef block_is_always_obj - G1OffsetTableContigSpace:: G1OffsetTableContigSpace(G1BlockOffsetSharedArray* sharedOffsetArray, MemRegion mr) : diff --git a/src/share/vm/gc_implementation/g1/heapRegion.hpp b/src/share/vm/gc_implementation/g1/heapRegion.hpp --- a/src/share/vm/gc_implementation/g1/heapRegion.hpp +++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp @@ -186,8 +186,6 @@ HeapWord* block_start(const void* p); HeapWord* block_start_const(const void* p) const; - void prepare_for_compaction(CompactPoint* cp); - // Add offset table update. virtual HeapWord* allocate(size_t word_size); HeapWord* par_allocate(size_t word_size); @@ -353,6 +351,21 @@ // and the amount of unallocated words if called on top() size_t block_size(const HeapWord* p) const; + // Functions for scan_and_forward support. + void prepare_for_compaction(CompactPoint* cp); + + inline HeapWord* scan_limit() const { + return top(); + } + + inline bool scanned_block_is_obj(const HeapWord* addr) const { + return true; // Always true, since scan_limit is top + } + + inline size_t scanned_block_size(const HeapWord* addr) const { + return HeapRegion::block_size(addr); // Avoid virtual call + } + inline HeapWord* par_allocate_no_bot_updates(size_t word_size); inline HeapWord* allocate_no_bot_updates(size_t word_size); diff --git a/src/share/vm/gc_implementation/shared/markSweep.inline.hpp b/src/share/vm/gc_implementation/shared/markSweep.inline.hpp --- a/src/share/vm/gc_implementation/shared/markSweep.inline.hpp +++ b/src/share/vm/gc_implementation/shared/markSweep.inline.hpp @@ -27,6 +27,7 @@ #include "gc_implementation/shared/markSweep.hpp" #include "gc_interface/collectedHeap.hpp" +#include "oops/markOop.inline.hpp" #include "utilities/stack.inline.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS diff --git a/src/share/vm/memory/space.cpp b/src/share/vm/memory/space.cpp --- a/src/share/vm/memory/space.cpp +++ b/src/share/vm/memory/space.cpp @@ -438,17 +438,8 @@ } } -#define block_is_always_obj(q) true -#define obj_size(q) oop(q)->size() -#define adjust_obj_size(s) s - -void CompactibleSpace::prepare_for_compaction(CompactPoint* cp) { - SCAN_AND_FORWARD(cp, end, block_is_obj, block_size); -} - -// Faster object search. void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) { - SCAN_AND_FORWARD(cp, top, block_is_always_obj, obj_size); + scan_and_forward(this, cp); } void Space::adjust_pointers() { @@ -492,11 +483,11 @@ return; // Nothing to do. } - SCAN_AND_ADJUST_POINTERS(adjust_obj_size); + scan_and_adjust_pointers(this); } void CompactibleSpace::compact() { - SCAN_AND_COMPACT(obj_size); + scan_and_compact(this); } void Space::print_short() const { print_short_on(tty); } diff --git a/src/share/vm/memory/space.hpp b/src/share/vm/memory/space.hpp --- a/src/share/vm/memory/space.hpp +++ b/src/share/vm/memory/space.hpp @@ -46,6 +46,7 @@ // - Space -- an abstract base class describing a heap area // - CompactibleSpace -- a space supporting compaction // - CompactibleFreeListSpace -- (used for CMS generation) +// - G1OffsetTableContigSpace -- G1 version of OffsetTableContigSpace // - ContiguousSpace -- a compactible space in which all free space // is contiguous // - EdenSpace -- contiguous space used as nursery @@ -391,7 +392,7 @@ // "cp->compaction_space" up-to-date. Offset tables may be updated in // this phase as if the final copy had occurred; if so, "cp->threshold" // indicates when the next such action should be taken. - virtual void prepare_for_compaction(CompactPoint* cp); + virtual void prepare_for_compaction(CompactPoint* cp) = 0; // MarkSweep support phase3 virtual void adjust_pointers(); // MarkSweep support phase4 @@ -426,6 +427,21 @@ // Return a size with adjustments as required of the space. virtual size_t adjust_object_size_v(size_t size) const { return size; } + // Functions for scan_and_{forward,adjust_pointers,compact} support. + inline bool scanned_block_is_obj(const HeapWord* addr) const { + // Perform virtual call. This is currently not a problem since this + // function is only used in an assert (from scan_and_adjust_pointers). + return block_is_obj(addr); + } + + inline size_t adjust_obj_size(size_t size) const { + return size; + } + + inline size_t obj_size(const HeapWord* addr) const { + return oop(addr)->size(); + } + protected: // Used during compaction. HeapWord* _first_dead; @@ -450,6 +466,25 @@ // words remaining after this operation. bool insert_deadspace(size_t& allowed_deadspace_words, HeapWord* q, size_t word_len); + + // Below are template functions for scan_and_* algorithms (avoiding virtual calls). + // The space argument should be a subclass of CompactibleSpace, implementing + // scan_limit(), scanned_block_is_obj(), and scanned_block_size(), + // and possibly also overriding obj_size(), and adjust_obj_size(). + // These functions should avoid virtual calls whenever possible. + + // Frequently calls adjust_obj_size(). (Asserts on scanned_block_is_obj().) + template + static inline void scan_and_adjust_pointers(SpaceType* space); + + // Frequently calls obj_size(). + template + static inline void scan_and_compact(SpaceType* space); + + // Frequently calls scanned_block_is_obj() and scanned_block_size(). + // Requires the scan_limit() function. + template + static inline void scan_and_forward(SpaceType* space, CompactPoint* cp); }; class GenSpaceMangler; @@ -624,6 +659,18 @@ // space. void allocate_temporary_filler(int factor); + // Functions for scan_and_{forward,adjust_pointers,compact} support. + inline HeapWord* scan_limit() const { + return top(); + } + + inline bool scanned_block_is_obj(const HeapWord* addr) const { + return true; // Always true, since scan_limit is top + } + + inline size_t scanned_block_size(const HeapWord* addr) const { + return oop(addr)->size(); + } }; diff --git a/src/share/vm/memory/space.inline.hpp b/src/share/vm/memory/space.inline.hpp --- a/src/share/vm/memory/space.inline.hpp +++ b/src/share/vm/memory/space.inline.hpp @@ -25,6 +25,9 @@ #ifndef SHARE_VM_MEMORY_SPACE_INLINE_HPP #define SHARE_VM_MEMORY_SPACE_INLINE_HPP +#include "gc_implementation/shared/liveRange.hpp" +#include "gc_implementation/shared/markSweep.inline.hpp" +#include "gc_implementation/shared/spaceDecorator.hpp" #include "gc_interface/collectedHeap.hpp" #include "memory/space.hpp" #include "memory/universe.hpp" @@ -35,272 +38,6 @@ return block_start_const(p); } -#define SCAN_AND_FORWARD(cp,scan_limit,block_is_obj,block_size) { \ - /* Compute the new addresses for the live objects and store it in the mark \ - * Used by universe::mark_sweep_phase2() \ - */ \ - HeapWord* compact_top; /* This is where we are currently compacting to. */ \ - \ - /* We're sure to be here before any objects are compacted into this \ - * space, so this is a good time to initialize this: \ - */ \ - set_compaction_top(bottom()); \ - \ - if (cp->space == NULL) { \ - assert(cp->gen != NULL, "need a generation"); \ - assert(cp->threshold == NULL, "just checking"); \ - assert(cp->gen->first_compaction_space() == this, "just checking"); \ - cp->space = cp->gen->first_compaction_space(); \ - compact_top = cp->space->bottom(); \ - cp->space->set_compaction_top(compact_top); \ - cp->threshold = cp->space->initialize_threshold(); \ - } else { \ - compact_top = cp->space->compaction_top(); \ - } \ - \ - /* We allow some amount of garbage towards the bottom of the space, so \ - * we don't start compacting before there is a significant gain to be made.\ - * Occasionally, we want to ensure a full compaction, which is determined \ - * by the MarkSweepAlwaysCompactCount parameter. \ - */ \ - uint invocations = MarkSweep::total_invocations(); \ - bool skip_dead = ((invocations % MarkSweepAlwaysCompactCount) != 0); \ - \ - size_t allowed_deadspace = 0; \ - if (skip_dead) { \ - const size_t ratio = allowed_dead_ratio(); \ - allowed_deadspace = (capacity() * ratio / 100) / HeapWordSize; \ - } \ - \ - HeapWord* q = bottom(); \ - HeapWord* t = scan_limit(); \ - \ - HeapWord* end_of_live= q; /* One byte beyond the last byte of the last \ - live object. */ \ - HeapWord* first_dead = end();/* The first dead object. */ \ - LiveRange* liveRange = NULL; /* The current live range, recorded in the \ - first header of preceding free area. */ \ - _first_dead = first_dead; \ - \ - const intx interval = PrefetchScanIntervalInBytes; \ - \ - while (q < t) { \ - assert(!block_is_obj(q) || \ - oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() || \ - oop(q)->mark()->has_bias_pattern(), \ - "these are the only valid states during a mark sweep"); \ - if (block_is_obj(q) && oop(q)->is_gc_marked()) { \ - /* prefetch beyond q */ \ - Prefetch::write(q, interval); \ - size_t size = block_size(q); \ - compact_top = cp->space->forward(oop(q), size, cp, compact_top); \ - q += size; \ - end_of_live = q; \ - } else { \ - /* run over all the contiguous dead objects */ \ - HeapWord* end = q; \ - do { \ - /* prefetch beyond end */ \ - Prefetch::write(end, interval); \ - end += block_size(end); \ - } while (end < t && (!block_is_obj(end) || !oop(end)->is_gc_marked()));\ - \ - /* see if we might want to pretend this object is alive so that \ - * we don't have to compact quite as often. \ - */ \ - if (allowed_deadspace > 0 && q == compact_top) { \ - size_t sz = pointer_delta(end, q); \ - if (insert_deadspace(allowed_deadspace, q, sz)) { \ - compact_top = cp->space->forward(oop(q), sz, cp, compact_top); \ - q = end; \ - end_of_live = end; \ - continue; \ - } \ - } \ - \ - /* otherwise, it really is a free region. */ \ - \ - /* for the previous LiveRange, record the end of the live objects. */ \ - if (liveRange) { \ - liveRange->set_end(q); \ - } \ - \ - /* record the current LiveRange object. \ - * liveRange->start() is overlaid on the mark word. \ - */ \ - liveRange = (LiveRange*)q; \ - liveRange->set_start(end); \ - liveRange->set_end(end); \ - \ - /* see if this is the first dead region. */ \ - if (q < first_dead) { \ - first_dead = q; \ - } \ - \ - /* move on to the next object */ \ - q = end; \ - } \ - } \ - \ - assert(q == t, "just checking"); \ - if (liveRange != NULL) { \ - liveRange->set_end(q); \ - } \ - _end_of_live = end_of_live; \ - if (end_of_live < first_dead) { \ - first_dead = end_of_live; \ - } \ - _first_dead = first_dead; \ - \ - /* save the compaction_top of the compaction space. */ \ - cp->space->set_compaction_top(compact_top); \ -} - -#define SCAN_AND_ADJUST_POINTERS(adjust_obj_size) { \ - /* adjust all the interior pointers to point at the new locations of objects \ - * Used by MarkSweep::mark_sweep_phase3() */ \ - \ - HeapWord* q = bottom(); \ - HeapWord* t = _end_of_live; /* Established by "prepare_for_compaction". */ \ - \ - assert(_first_dead <= _end_of_live, "Stands to reason, no?"); \ - \ - if (q < t && _first_dead > q && \ - !oop(q)->is_gc_marked()) { \ - /* we have a chunk of the space which hasn't moved and we've \ - * reinitialized the mark word during the previous pass, so we can't \ - * use is_gc_marked for the traversal. */ \ - HeapWord* end = _first_dead; \ - \ - while (q < end) { \ - /* I originally tried to conjoin "block_start(q) == q" to the \ - * assertion below, but that doesn't work, because you can't \ - * accurately traverse previous objects to get to the current one \ - * after their pointers have been \ - * updated, until the actual compaction is done. dld, 4/00 */ \ - assert(block_is_obj(q), \ - "should be at block boundaries, and should be looking at objs"); \ - \ - /* point all the oops to the new location */ \ - size_t size = oop(q)->adjust_pointers(); \ - size = adjust_obj_size(size); \ - \ - q += size; \ - } \ - \ - if (_first_dead == t) { \ - q = t; \ - } else { \ - /* $$$ This is funky. Using this to read the previously written \ - * LiveRange. See also use below. */ \ - q = (HeapWord*)oop(_first_dead)->mark()->decode_pointer(); \ - } \ - } \ - \ - const intx interval = PrefetchScanIntervalInBytes; \ - \ - debug_only(HeapWord* prev_q = NULL); \ - while (q < t) { \ - /* prefetch beyond q */ \ - Prefetch::write(q, interval); \ - if (oop(q)->is_gc_marked()) { \ - /* q is alive */ \ - /* point all the oops to the new location */ \ - size_t size = oop(q)->adjust_pointers(); \ - size = adjust_obj_size(size); \ - debug_only(prev_q = q); \ - q += size; \ - } else { \ - /* q is not a live object, so its mark should point at the next \ - * live object */ \ - debug_only(prev_q = q); \ - q = (HeapWord*) oop(q)->mark()->decode_pointer(); \ - assert(q > prev_q, "we should be moving forward through memory"); \ - } \ - } \ - \ - assert(q == t, "just checking"); \ -} - -#define SCAN_AND_COMPACT(obj_size) { \ - /* Copy all live objects to their new location \ - * Used by MarkSweep::mark_sweep_phase4() */ \ - \ - HeapWord* q = bottom(); \ - HeapWord* const t = _end_of_live; \ - debug_only(HeapWord* prev_q = NULL); \ - \ - if (q < t && _first_dead > q && \ - !oop(q)->is_gc_marked()) { \ - debug_only( \ - /* we have a chunk of the space which hasn't moved and we've reinitialized \ - * the mark word during the previous pass, so we can't use is_gc_marked for \ - * the traversal. */ \ - HeapWord* const end = _first_dead; \ - \ - while (q < end) { \ - size_t size = obj_size(q); \ - assert(!oop(q)->is_gc_marked(), \ - "should be unmarked (special dense prefix handling)"); \ - debug_only(prev_q = q); \ - q += size; \ - } \ - ) /* debug_only */ \ - \ - if (_first_dead == t) { \ - q = t; \ - } else { \ - /* $$$ Funky */ \ - q = (HeapWord*) oop(_first_dead)->mark()->decode_pointer(); \ - } \ - } \ - \ - const intx scan_interval = PrefetchScanIntervalInBytes; \ - const intx copy_interval = PrefetchCopyIntervalInBytes; \ - while (q < t) { \ - if (!oop(q)->is_gc_marked()) { \ - /* mark is pointer to next marked oop */ \ - debug_only(prev_q = q); \ - q = (HeapWord*) oop(q)->mark()->decode_pointer(); \ - assert(q > prev_q, "we should be moving forward through memory"); \ - } else { \ - /* prefetch beyond q */ \ - Prefetch::read(q, scan_interval); \ - \ - /* size and destination */ \ - size_t size = obj_size(q); \ - HeapWord* compaction_top = (HeapWord*)oop(q)->forwardee(); \ - \ - /* prefetch beyond compaction_top */ \ - Prefetch::write(compaction_top, copy_interval); \ - \ - /* copy object and reinit its mark */ \ - assert(q != compaction_top, "everything in this pass should be moving"); \ - Copy::aligned_conjoint_words(q, compaction_top, size); \ - oop(compaction_top)->init_mark(); \ - assert(oop(compaction_top)->klass() != NULL, "should have a class"); \ - \ - debug_only(prev_q = q); \ - q += size; \ - } \ - } \ - \ - /* Let's remember if we were empty before we did the compaction. */ \ - bool was_empty = used_region().is_empty(); \ - /* Reset space after compaction is complete */ \ - reset_after_compaction(); \ - /* We do this clear, below, since it has overloaded meanings for some */ \ - /* space subtypes. For example, OffsetTableContigSpace's that were */ \ - /* compacted into will have had their offset table thresholds updated */ \ - /* continuously, but those that weren't need to have their thresholds */ \ - /* re-initialized. Also mangles unused area for debugging. */ \ - if (used_region().is_empty()) { \ - if (!was_empty) clear(SpaceDecorator::Mangle); \ - } else { \ - if (ZapUnusedHeapArea) mangle_unused_area(); \ - } \ -} - inline HeapWord* OffsetTableContigSpace::allocate(size_t size) { HeapWord* res = ContiguousSpace::allocate(size); if (res != NULL) { @@ -334,4 +71,263 @@ return _offsets.block_start(p); } +template +inline void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp) { + // Compute the new addresses for the live objects and store it in the mark + // Used by universe::mark_sweep_phase2() + HeapWord* compact_top; // This is where we are currently compacting to. + + // We're sure to be here before any objects are compacted into this + // space, so this is a good time to initialize this: + space->set_compaction_top(space->bottom()); + + if (cp->space == NULL) { + assert(cp->gen != NULL, "need a generation"); + assert(cp->threshold == NULL, "just checking"); + assert(cp->gen->first_compaction_space() == space, "just checking"); + cp->space = cp->gen->first_compaction_space(); + compact_top = cp->space->bottom(); + cp->space->set_compaction_top(compact_top); + cp->threshold = cp->space->initialize_threshold(); + } else { + compact_top = cp->space->compaction_top(); + } + + // We allow some amount of garbage towards the bottom of the space, so + // we don't start compacting before there is a significant gain to be made.\ + // Occasionally, we want to ensure a full compaction, which is determined + // by the MarkSweepAlwaysCompactCount parameter. + uint invocations = MarkSweep::total_invocations(); + bool skip_dead = ((invocations % MarkSweepAlwaysCompactCount) != 0); + + size_t allowed_deadspace = 0; + if (skip_dead) { + const size_t ratio = space->allowed_dead_ratio(); + allowed_deadspace = (space->capacity() * ratio / 100) / HeapWordSize; + } + + HeapWord* q = space->bottom(); + HeapWord* t = space->scan_limit(); + + HeapWord* end_of_live= q; // One byte beyond the last byte of the last + // live object. + HeapWord* first_dead = space->end(); // The first dead object. + LiveRange* liveRange = NULL; // The current live range, recorded in the + // first header of preceding free area. + space->_first_dead = first_dead; + + const intx interval = PrefetchScanIntervalInBytes; + + while (q < t) { + assert(!space->scanned_block_is_obj(q) || + oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() || + oop(q)->mark()->has_bias_pattern(), + "these are the only valid states during a mark sweep"); + if (space->scanned_block_is_obj(q) && oop(q)->is_gc_marked()) { + // prefetch beyond q + Prefetch::write(q, interval); + size_t size = space->scanned_block_size(q); + compact_top = cp->space->forward(oop(q), size, cp, compact_top); + q += size; + end_of_live = q; + } else { + // run over all the contiguous dead objects + HeapWord* end = q; + do { + // prefetch beyond end + Prefetch::write(end, interval); + end += space->scanned_block_size(end); + } while (end < t && (!space->scanned_block_is_obj(end) || !oop(end)->is_gc_marked())); + + // see if we might want to pretend this object is alive so that + // we don't have to compact quite as often. + if (allowed_deadspace > 0 && q == compact_top) { + size_t sz = pointer_delta(end, q); + if (space->insert_deadspace(allowed_deadspace, q, sz)) { + compact_top = cp->space->forward(oop(q), sz, cp, compact_top); + q = end; + end_of_live = end; + continue; + } + } + + // otherwise, it really is a free region. + + // for the previous LiveRange, record the end of the live objects. + if (liveRange) { + liveRange->set_end(q); + } + + // record the current LiveRange object. + // liveRange->start() is overlaid on the mark word. + liveRange = (LiveRange*)q; + liveRange->set_start(end); + liveRange->set_end(end); + + // see if this is the first dead region. + if (q < first_dead) { + first_dead = q; + } + + // move on to the next object + q = end; + } + } + + assert(q == t, "just checking"); + if (liveRange != NULL) { + liveRange->set_end(q); + } + space->_end_of_live = end_of_live; + if (end_of_live < first_dead) { + first_dead = end_of_live; + } + space->_first_dead = first_dead; + + // save the compaction_top of the compaction space. + cp->space->set_compaction_top(compact_top); +} + +template +inline void CompactibleSpace::scan_and_adjust_pointers(SpaceType* space) { + // adjust all the interior pointers to point at the new locations of objects + // Used by MarkSweep::mark_sweep_phase3() + + HeapWord* q = space->bottom(); + HeapWord* t = space->_end_of_live; // Established by "prepare_for_compaction". + + assert(space->_first_dead <= space->_end_of_live, "Stands to reason, no?"); + + if (q < t && space->_first_dead > q && !oop(q)->is_gc_marked()) { + // we have a chunk of the space which hasn't moved and we've + // reinitialized the mark word during the previous pass, so we can't + // use is_gc_marked for the traversal. + HeapWord* end = space->_first_dead; + + while (q < end) { + // I originally tried to conjoin "block_start(q) == q" to the + // assertion below, but that doesn't work, because you can't + // accurately traverse previous objects to get to the current one + // after their pointers have been + // updated, until the actual compaction is done. dld, 4/00 + assert(space->scanned_block_is_obj(q), "should be at block boundaries, and should be looking at objs"); + + // point all the oops to the new location + size_t size = oop(q)->adjust_pointers(); + size = space->adjust_obj_size(size); + + q += size; + } + + if (space->_first_dead == t) { + q = t; + } else { + // $$$ This is funky. Using this to read the previously written + // LiveRange. See also use below. + q = (HeapWord*)oop(space->_first_dead)->mark()->decode_pointer(); + } + } + + const intx interval = PrefetchScanIntervalInBytes; + + debug_only(HeapWord* prev_q = NULL); + while (q < t) { + // prefetch beyond q + Prefetch::write(q, interval); + if (oop(q)->is_gc_marked()) { + // q is alive + // point all the oops to the new location + size_t size = oop(q)->adjust_pointers(); + size = space->adjust_obj_size(size); + debug_only(prev_q = q); + q += size; + } else { + // q is not a live object, so its mark should point at the next + // live object + debug_only(prev_q = q); + q = (HeapWord*) oop(q)->mark()->decode_pointer(); + assert(q > prev_q, "we should be moving forward through memory"); + } + } + + assert(q == t, "just checking"); +} + +template +inline void CompactibleSpace::scan_and_compact(SpaceType* space) { + // Copy all live objects to their new location + // Used by MarkSweep::mark_sweep_phase4() + + HeapWord* q = space->bottom(); + HeapWord* const t = space->_end_of_live; + debug_only(HeapWord* prev_q = NULL); + + if (q < t && space->_first_dead > q && !oop(q)->is_gc_marked()) { + #ifdef ASSERT // Debug only + // we have a chunk of the space which hasn't moved and we've reinitialized + // the mark word during the previous pass, so we can't use is_gc_marked for + // the traversal. + HeapWord* const end = space->_first_dead; + + while (q < end) { + size_t size = space->obj_size(q); + assert(!oop(q)->is_gc_marked(), "should be unmarked (special dense prefix handling)"); + prev_q = q; + q += size; + } + #endif + + if (space->_first_dead == t) { + q = t; + } else { + // $$$ Funky + q = (HeapWord*) oop(space->_first_dead)->mark()->decode_pointer(); + } + } + + const intx scan_interval = PrefetchScanIntervalInBytes; + const intx copy_interval = PrefetchCopyIntervalInBytes; + while (q < t) { + if (!oop(q)->is_gc_marked()) { + // mark is pointer to next marked oop + debug_only(prev_q = q); + q = (HeapWord*) oop(q)->mark()->decode_pointer(); + assert(q > prev_q, "we should be moving forward through memory"); + } else { + // prefetch beyond q + Prefetch::read(q, scan_interval); + + // size and destination + size_t size = space->obj_size(q); + HeapWord* compaction_top = (HeapWord*)oop(q)->forwardee(); + + // prefetch beyond compaction_top + Prefetch::write(compaction_top, copy_interval); + + // copy object and reinit its mark + assert(q != compaction_top, "everything in this pass should be moving"); + Copy::aligned_conjoint_words(q, compaction_top, size); + oop(compaction_top)->init_mark(); + assert(oop(compaction_top)->klass() != NULL, "should have a class"); + + debug_only(prev_q = q); + q += size; + } + } + + // Let's remember if we were empty before we did the compaction. + bool was_empty = space->used_region().is_empty(); + // Reset space after compaction is complete + space->reset_after_compaction(); + // We do this clear, below, since it has overloaded meanings for some + // space subtypes. For example, OffsetTableContigSpace's that were + // compacted into will have had their offset table thresholds updated + // continuously, but those that weren't need to have their thresholds + // re-initialized. Also mangles unused area for debugging. + if (space->used_region().is_empty()) { + if (!was_empty) space->clear(SpaceDecorator::Mangle); + } else { + if (ZapUnusedHeapArea) space->mangle_unused_area(); + } +} #endif // SHARE_VM_MEMORY_SPACE_INLINE_HPP