--- old/src/hotspot/share/gc/parallel/psParallelCompact.hpp 2019-10-28 14:10:42.284590654 +0100 +++ new/src/hotspot/share/gc/parallel/psParallelCompact.hpp 2019-10-28 14:10:41.848588492 +0100 @@ -239,6 +239,9 @@ // The first region containing data destined for this region. size_t source_region() const { return _source_region; } + // Reuse _source_region to store the corresponding shadow region index + size_t shadow_region() const { return _source_region; } + // The object (if any) starting in this region and ending in a different // region that could not be updated during the main (parallel) compaction // phase. This is different from _partial_obj_addr, which is an object that @@ -307,6 +310,7 @@ // These are not atomic. void set_destination(HeapWord* addr) { _destination = addr; } void set_source_region(size_t region) { _source_region = region; } + void set_shadow_region(size_t region) { _source_region = region; } void set_deferred_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; } void set_partial_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; } void set_partial_obj_size(size_t words) { @@ -326,6 +330,26 @@ inline void decrement_destination_count(); inline bool claim(); + // Possible values of _shadow_state, and transition is as follows + // Normal Path: + // UNUSED -> try_push() -> FINISHED + // Steal Path: + // UNUSED -> try_steal() -> SHADOW -> mark_filled() -> FILLED -> try_copy() -> FINISHED + static const int UNUSED; // Original state + static const int SHADOW; // Stolen by an idle thread, and a shadow region is created for it + static const int FILLED; // Its shadow region has been filled and ready to be copied back + static const int FINISH; // Work has been done + + // Preempt the region to avoid double processes + inline bool try_push(); + inline bool try_steal(); + // Mark the region as filled and ready to be copied back + inline bool mark_filled(); + // Preempt the region to copy the shadow region content back + inline bool try_copy(); + + int shadow_state() { return _shadow_state; } + private: // The type used to represent object sizes within a region. typedef uint region_sz_t; @@ -346,6 +370,7 @@ region_sz_t _partial_obj_size; region_sz_t volatile _dc_and_los; bool volatile _blocks_filled; + int volatile _shadow_state; #ifdef ASSERT size_t _blocks_filled_count; // Number of block table fills. @@ -596,6 +621,22 @@ return old == los; } +inline bool ParallelCompactData::RegionData::try_push() { + return Atomic::cmpxchg(FINISH, &_shadow_state, UNUSED) == UNUSED; +} + +inline bool ParallelCompactData::RegionData::try_steal() { + return Atomic::cmpxchg(SHADOW, &_shadow_state, UNUSED) == UNUSED; +} + +inline bool ParallelCompactData::RegionData::mark_filled() { + return Atomic::cmpxchg(FILLED, &_shadow_state, SHADOW) == SHADOW; +} + +inline bool ParallelCompactData::RegionData::try_copy() { + return Atomic::cmpxchg(FINISH, &_shadow_state, FILLED) == FILLED; +} + inline ParallelCompactData::RegionData* ParallelCompactData::region(size_t region_idx) const { @@ -1179,11 +1220,20 @@ size_t beg_region, HeapWord* end_addr); - // Fill a region, copying objects from one or more source regions. - static void fill_region(ParCompactionManager* cm, size_t region_idx); - static void fill_and_update_region(ParCompactionManager* cm, size_t region) { - fill_region(cm, region); + static void fill_region(ParCompactionManager* cm, MoveAndUpdateClosure& closure, size_t region); + static void fill_and_update_region(ParCompactionManager* cm, size_t region); + + static bool steal_shadow_region(ParCompactionManager* cm, size_t& region_idx); + static void fill_shadow_region(ParCompactionManager* cm, size_t region_idx); + static void fill_and_update_shadow_region(ParCompactionManager* cm, size_t region) { + fill_shadow_region(cm, region); } + // Copy the content of a shadow region back to its corresponding heap region + static void copy_back(HeapWord* shadow_addr, HeapWord* region_addr); + // Initialize the steal record of a GC thread + static void initialize_steal_record(uint which); + // Reuse the empty heap regions as shadow regions, like to-space regions + static void enqueue_shadow_region(); // Fill in the block table for the specified region. static void fill_blocks(size_t region_idx); @@ -1230,10 +1280,10 @@ }; class MoveAndUpdateClosure: public ParMarkBitMapClosure { + static inline size_t calculate_words_remaining(size_t region); public: inline MoveAndUpdateClosure(ParMarkBitMap* bitmap, ParCompactionManager* cm, - ObjectStartArray* start_array, - HeapWord* destination, size_t words); + size_t region); // Accessors. HeapWord* destination() const { return _destination; } @@ -1242,7 +1292,7 @@ // destination, update the interior oops and the start array and return either // full (if the closure is full) or incomplete. If the object will not fit, // return would_overflow. - virtual IterationStatus do_addr(HeapWord* addr, size_t size); + IterationStatus do_addr(HeapWord* addr, size_t size); // Copy enough words to fill this closure, starting at source(). Interior // oops and the start array are not updated. Return full. @@ -1253,25 +1303,39 @@ // array are not updated. void copy_partial_obj(); - protected: + virtual void complete_region(ParCompactionManager* cm, HeapWord* dest_addr, + PSParallelCompact::RegionData* region_ptr); + + virtual void allocate_block(); + + virtual void set_deferred_obj_addr_for(PSParallelCompact::RegionData* region_ptr); + +protected: // Update variables to indicate that word_count words were processed. inline void update_state(size_t word_count); protected: - ObjectStartArray* const _start_array; HeapWord* _destination; // Next addr to be written. + ObjectStartArray* const _start_array; }; +inline size_t MoveAndUpdateClosure::calculate_words_remaining(size_t region) { + HeapWord* dest_addr = PSParallelCompact::summary_data().region_to_addr(region); + PSParallelCompact::SpaceId dest_space_id = PSParallelCompact::space_id(dest_addr); + HeapWord* new_top = PSParallelCompact::new_top(dest_space_id); + assert(dest_addr < new_top, "sanity"); + + return MIN2(pointer_delta(new_top, dest_addr), ParallelCompactData::RegionSize); +} + inline MoveAndUpdateClosure::MoveAndUpdateClosure(ParMarkBitMap* bitmap, ParCompactionManager* cm, - ObjectStartArray* start_array, - HeapWord* destination, - size_t words) : - ParMarkBitMapClosure(bitmap, cm, words), _start_array(start_array) -{ - _destination = destination; -} + size_t region_idx) : + ParMarkBitMapClosure(bitmap, cm, calculate_words_remaining(region_idx)), + _destination(PSParallelCompact::summary_data().region_to_addr(region_idx)), + _start_array(PSParallelCompact::start_array(PSParallelCompact::space_id(_destination))) { } + inline void MoveAndUpdateClosure::update_state(size_t words) { @@ -1280,6 +1344,43 @@ _destination += words; } +class ShadowClosure: public MoveAndUpdateClosure { + inline size_t calculate_shadow_offset(size_t region_idx, size_t shadow_idx); +public: + inline ShadowClosure(ParMarkBitMap* bitmap, ParCompactionManager* cm, + size_t region, size_t shadow); + + virtual void complete_region(ParCompactionManager* cm, HeapWord* dest_addr, + PSParallelCompact::RegionData* region_ptr); + + virtual void allocate_block(); + + virtual void set_deferred_obj_addr_for(PSParallelCompact::RegionData* region_ptr); + +private: + size_t _shadow; + size_t _offset; +}; + +inline size_t ShadowClosure::calculate_shadow_offset(size_t region_idx, size_t shadow_idx) { + ParallelCompactData& sd = PSParallelCompact::summary_data(); + HeapWord* dest_addr = sd.region_to_addr(region_idx); + HeapWord* shadow_addr = sd.region_to_addr(shadow_idx); + return pointer_delta(shadow_addr, dest_addr); +} + +inline +ShadowClosure::ShadowClosure(ParMarkBitMap *bitmap, + ParCompactionManager *cm, + size_t region, + size_t shadow) : + MoveAndUpdateClosure(bitmap, cm, region), + _shadow(shadow), + _offset(calculate_shadow_offset(region, shadow)) { + ParallelCompactData& sd = PSParallelCompact::summary_data(); + _destination = sd.region_to_addr(shadow); +} + class UpdateOnlyClosure: public ParMarkBitMapClosure { private: const PSParallelCompact::SpaceId _space_id;