--- old/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.hpp Fri May 31 16:04:11 2013 +++ new/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.hpp Fri May 31 16:04:11 2013 @@ -223,6 +223,17 @@ // Mask for the bits in a pointer to get the address of the start of a region. static const size_t RegionAddrMask; + static const size_t Log2BlockSize; + static const size_t BlockSize; + static const size_t BlockSizeBytes; + + static const size_t BlockSizeOffsetMask; + static const size_t BlockAddrOffsetMask; + static const size_t BlockAddrMask; + + static const size_t BlocksPerRegion; + static const size_t Log2BlocksPerRegion; + class RegionData { public: @@ -275,6 +286,12 @@ inline uint destination_count() const; inline uint destination_count_raw() const; + // Whether the block table for this region has been filled. + inline bool blocks_filled() const; + + // Number of times the block table was filled. + DEBUG_ONLY(inline size_t blocks_filled_count() const;) + // The location of the java heap data that corresponds to this region. inline HeapWord* data_location() const; @@ -299,6 +316,7 @@ void set_partial_obj_size(size_t words) { _partial_obj_size = (region_sz_t) words; } + inline void set_blocks_filled(); inline void set_destination_count(uint count); inline void set_live_obj_size(size_t words); @@ -331,7 +349,11 @@ HeapWord* _partial_obj_addr; region_sz_t _partial_obj_size; region_sz_t volatile _dc_and_los; + bool _blocks_filled; + #ifdef ASSERT + size_t _blocks_filled_count; // Number of block table fills. + // These enable optimizations that are only partially implemented. Use // debug builds to prevent the code fragments from breaking. HeapWord* _data_location; @@ -340,11 +362,26 @@ #ifdef ASSERT public: - uint _pushed; // 0 until region is pushed onto a worker's stack + uint _pushed; // 0 until region is pushed onto a stack private: #endif }; + // "Blocks" allow shorter sections of the bitmap to be searched. Each Block + // holds an offset, which is the amount of live data in the Region to the left + // of the first live object that starts in the Block. + class BlockData + { + public: + typedef unsigned short int blk_ofs_t; + + blk_ofs_t offset() const { return _offset; } + void set_offset(size_t val) { _offset = (blk_ofs_t)val; } + + private: + blk_ofs_t _offset; + }; + public: ParallelCompactData(); bool initialize(MemRegion covered_region); @@ -355,8 +392,9 @@ inline RegionData* region(size_t region_idx) const; inline size_t region(const RegionData* const region_ptr) const; - // Returns true if the given address is contained within the region - bool region_contains(size_t region_index, HeapWord* addr); + size_t block_count() const { return _block_count; } + inline BlockData* block(size_t block_idx) const; + inline size_t block(const BlockData* block_ptr) const; void add_obj(HeapWord* addr, size_t len); void add_obj(oop p, size_t len) { add_obj((HeapWord*)p, len); } @@ -396,11 +434,24 @@ inline HeapWord* region_align_up(HeapWord* addr) const; inline bool is_region_aligned(HeapWord* addr) const; + // Analogous to region_offset() for blocks. + size_t block_offset(const HeapWord* addr) const; + size_t addr_to_block_idx(const HeapWord* addr) const; + size_t addr_to_block_idx(const oop obj) const { + return addr_to_block_idx((HeapWord*) obj); + } + inline BlockData* addr_to_block_ptr(const HeapWord* addr) const; + inline HeapWord* block_to_addr(size_t block) const; + inline size_t region_to_block_idx(size_t region) const; + + inline HeapWord* block_align_down(HeapWord* addr) const; + inline HeapWord* block_align_up(HeapWord* addr) const; + inline bool is_block_aligned(HeapWord* addr) const; + // Return the address one past the end of the partial object. HeapWord* partial_obj_end(size_t region_idx) const; - // Return the new location of the object p after the - // the compaction. + // Return the location of the object after compaction. HeapWord* calc_new_pointer(HeapWord* addr); HeapWord* calc_new_pointer(oop p) { @@ -416,6 +467,7 @@ #endif // #ifdef ASSERT private: + bool initialize_block_data(); bool initialize_region_data(size_t region_size); PSVirtualSpace* create_vspace(size_t count, size_t element_size); @@ -428,6 +480,10 @@ PSVirtualSpace* _region_vspace; RegionData* _region_data; size_t _region_count; + + PSVirtualSpace* _block_vspace; + BlockData* _block_data; + size_t _block_count; }; inline uint @@ -442,7 +498,29 @@ return destination_count_raw() >> dc_shift; } +inline bool +ParallelCompactData::RegionData::blocks_filled() const +{ + return _blocks_filled; +} + +#ifdef ASSERT +inline size_t +ParallelCompactData::RegionData::blocks_filled_count() const +{ + return _blocks_filled_count; +} +#endif // #ifdef ASSERT + inline void +ParallelCompactData::RegionData::set_blocks_filled() +{ + _blocks_filled = true; + // Debug builds count the number of times the table was filled. + DEBUG_ONLY(Atomic::inc_ptr(&_blocks_filled_count)); +} + +inline void ParallelCompactData::RegionData::set_destination_count(uint count) { assert(count <= (dc_completed >> dc_shift), "count too large"); @@ -536,6 +614,12 @@ return pointer_delta(region_ptr, _region_data, sizeof(RegionData)); } +inline ParallelCompactData::BlockData* +ParallelCompactData::block(size_t n) const { + assert(n < block_count(), "bad arg"); + return _block_data + n; +} + inline size_t ParallelCompactData::region_offset(const HeapWord* addr) const { @@ -602,6 +686,63 @@ return region_offset(addr) == 0; } +inline size_t +ParallelCompactData::block_offset(const HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr <= _region_end, "bad addr"); + return (size_t(addr) & BlockAddrOffsetMask) >> LogHeapWordSize; +} + +inline size_t +ParallelCompactData::addr_to_block_idx(const HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr <= _region_end, "bad addr"); + return pointer_delta(addr, _region_start) >> Log2BlockSize; +} + +inline ParallelCompactData::BlockData* +ParallelCompactData::addr_to_block_ptr(const HeapWord* addr) const +{ + return block(addr_to_block_idx(addr)); +} + +inline HeapWord* +ParallelCompactData::block_to_addr(size_t block) const +{ + assert(block < _block_count, "block out of range"); + return _region_start + (block << Log2BlockSize); +} + +inline size_t +ParallelCompactData::region_to_block_idx(size_t region) const +{ + return region << Log2BlocksPerRegion; +} + +inline HeapWord* +ParallelCompactData::block_align_down(HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr < _region_end + RegionSize, "bad addr"); + return (HeapWord*)(size_t(addr) & BlockAddrMask); +} + +inline HeapWord* +ParallelCompactData::block_align_up(HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr <= _region_end, "bad addr"); + return block_align_down(addr + BlockSizeOffsetMask); +} + +inline bool +ParallelCompactData::is_block_aligned(HeapWord* addr) const +{ + return block_offset(addr) == 0; +} + // Abstract closure for use with ParMarkBitMap::iterate(), which will invoke the // do_addr() method. // @@ -779,6 +920,7 @@ // Convenient access to type names. typedef ParMarkBitMap::idx_t idx_t; typedef ParallelCompactData::RegionData RegionData; + typedef ParallelCompactData::BlockData BlockData; typedef enum { perm_space_id, old_space_id, eden_space_id, @@ -987,6 +1129,8 @@ // Serial code executed in preparation for the compaction phase. static void compact_prologue(); + DEBUG_ONLY(static void write_block_fill_histogram(outputStream* const out);) + // Move objects to new locations. static void compact_perm(ParCompactionManager* cm); static void compact(); @@ -1158,6 +1302,9 @@ fill_region(cm, region); } + // Fill in the block table for the specified region. + static void fill_blocks(size_t region_idx); + // Update the deferred objects in the space. static void update_deferred_objects(ParCompactionManager* cm, SpaceId id);