src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.hpp
Print this page
rev 4535 : 6725714: par compact - add a table to speed up bitmap searches
Reviewed-by: jmasa, tschatzl
@@ -221,10 +221,21 @@
// Mask for the bits in a pointer to get an offset within a region.
static const size_t RegionAddrOffsetMask;
// Mask for the bits in a pointer to get the address of the start of a region.
static const size_t RegionAddrMask;
+ static const size_t Log2BlockSize;
+ static const size_t BlockSize;
+ static const size_t BlockSizeBytes;
+
+ static const size_t BlockSizeOffsetMask;
+ static const size_t BlockAddrOffsetMask;
+ static const size_t BlockAddrMask;
+
+ static const size_t BlocksPerRegion;
+ static const size_t Log2BlocksPerRegion;
+
class RegionData
{
public:
// Destination address of the region.
HeapWord* destination() const { return _destination; }
@@ -273,10 +284,16 @@
// been filled, the destination_count should be set to the completed value
// (dc_completed).
inline uint destination_count() const;
inline uint destination_count_raw() const;
+ // Whether the block table for this region has been filled.
+ inline bool blocks_filled() const;
+
+ // Number of times the block table was filled.
+ DEBUG_ONLY(inline size_t blocks_filled_count() const;)
+
// The location of the java heap data that corresponds to this region.
inline HeapWord* data_location() const;
// The highest address referenced by objects in this region.
inline HeapWord* highest_ref() const;
@@ -297,10 +314,11 @@
void set_deferred_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; }
void set_partial_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; }
void set_partial_obj_size(size_t words) {
_partial_obj_size = (region_sz_t) words;
}
+ inline void set_blocks_filled();
inline void set_destination_count(uint count);
inline void set_live_obj_size(size_t words);
inline void set_data_location(HeapWord* addr);
inline void set_completed();
@@ -329,24 +347,43 @@
HeapWord* _destination;
size_t _source_region;
HeapWord* _partial_obj_addr;
region_sz_t _partial_obj_size;
region_sz_t volatile _dc_and_los;
+ bool _blocks_filled;
+
#ifdef ASSERT
+ size_t _blocks_filled_count; // Number of block table fills.
+
// These enable optimizations that are only partially implemented. Use
// debug builds to prevent the code fragments from breaking.
HeapWord* _data_location;
HeapWord* _highest_ref;
#endif // #ifdef ASSERT
#ifdef ASSERT
public:
- uint _pushed; // 0 until region is pushed onto a worker's stack
+ uint _pushed; // 0 until region is pushed onto a stack
private:
#endif
};
+ // "Blocks" allow shorter sections of the bitmap to be searched. Each Block
+ // holds an offset, which is the amount of live data in the Region to the left
+ // of the first live object that starts in the Block.
+ class BlockData
+ {
+ public:
+ typedef unsigned short int blk_ofs_t;
+
+ blk_ofs_t offset() const { return _offset; }
+ void set_offset(size_t val) { _offset = (blk_ofs_t)val; }
+
+ private:
+ blk_ofs_t _offset;
+ };
+
public:
ParallelCompactData();
bool initialize(MemRegion covered_region);
size_t region_count() const { return _region_count; }
@@ -353,12 +390,13 @@
// Convert region indices to/from RegionData pointers.
inline RegionData* region(size_t region_idx) const;
inline size_t region(const RegionData* const region_ptr) const;
- // Returns true if the given address is contained within the region
- bool region_contains(size_t region_index, HeapWord* addr);
+ size_t block_count() const { return _block_count; }
+ inline BlockData* block(size_t block_idx) const;
+ inline size_t block(const BlockData* block_ptr) const;
void add_obj(HeapWord* addr, size_t len);
void add_obj(oop p, size_t len) { add_obj((HeapWord*)p, len); }
// Fill in the regions covering [beg, end) so that no data moves; i.e., the
@@ -394,15 +432,28 @@
inline HeapWord* region_align_down(HeapWord* addr) const;
inline HeapWord* region_align_up(HeapWord* addr) const;
inline bool is_region_aligned(HeapWord* addr) const;
+ // Analogous to region_offset() for blocks.
+ size_t block_offset(const HeapWord* addr) const;
+ size_t addr_to_block_idx(const HeapWord* addr) const;
+ size_t addr_to_block_idx(const oop obj) const {
+ return addr_to_block_idx((HeapWord*) obj);
+ }
+ inline BlockData* addr_to_block_ptr(const HeapWord* addr) const;
+ inline HeapWord* block_to_addr(size_t block) const;
+ inline size_t region_to_block_idx(size_t region) const;
+
+ inline HeapWord* block_align_down(HeapWord* addr) const;
+ inline HeapWord* block_align_up(HeapWord* addr) const;
+ inline bool is_block_aligned(HeapWord* addr) const;
+
// Return the address one past the end of the partial object.
HeapWord* partial_obj_end(size_t region_idx) const;
- // Return the new location of the object p after the
- // the compaction.
+ // Return the location of the object after compaction.
HeapWord* calc_new_pointer(HeapWord* addr);
HeapWord* calc_new_pointer(oop p) {
return calc_new_pointer((HeapWord*) p);
}
@@ -414,10 +465,11 @@
void verify_clear(const PSVirtualSpace* vspace);
void verify_clear();
#endif // #ifdef ASSERT
private:
+ bool initialize_block_data();
bool initialize_region_data(size_t region_size);
PSVirtualSpace* create_vspace(size_t count, size_t element_size);
private:
HeapWord* _region_start;
@@ -426,10 +478,14 @@
#endif // #ifdef ASSERT
PSVirtualSpace* _region_vspace;
RegionData* _region_data;
size_t _region_count;
+
+ PSVirtualSpace* _block_vspace;
+ BlockData* _block_data;
+ size_t _block_count;
};
inline uint
ParallelCompactData::RegionData::destination_count_raw() const
{
@@ -440,11 +496,33 @@
ParallelCompactData::RegionData::destination_count() const
{
return destination_count_raw() >> dc_shift;
}
+inline bool
+ParallelCompactData::RegionData::blocks_filled() const
+{
+ return _blocks_filled;
+}
+
+#ifdef ASSERT
+inline size_t
+ParallelCompactData::RegionData::blocks_filled_count() const
+{
+ return _blocks_filled_count;
+}
+#endif // #ifdef ASSERT
+
inline void
+ParallelCompactData::RegionData::set_blocks_filled()
+{
+ _blocks_filled = true;
+ // Debug builds count the number of times the table was filled.
+ DEBUG_ONLY(Atomic::inc_ptr(&_blocks_filled_count));
+}
+
+inline void
ParallelCompactData::RegionData::set_destination_count(uint count)
{
assert(count <= (dc_completed >> dc_shift), "count too large");
const region_sz_t live_sz = (region_sz_t) live_obj_size();
_dc_and_los = (count << dc_shift) | live_sz;
@@ -534,10 +612,16 @@
assert(region_ptr >= _region_data, "bad arg");
assert(region_ptr <= _region_data + region_count(), "bad arg");
return pointer_delta(region_ptr, _region_data, sizeof(RegionData));
}
+inline ParallelCompactData::BlockData*
+ParallelCompactData::block(size_t n) const {
+ assert(n < block_count(), "bad arg");
+ return _block_data + n;
+}
+
inline size_t
ParallelCompactData::region_offset(const HeapWord* addr) const
{
assert(addr >= _region_start, "bad addr");
assert(addr <= _region_end, "bad addr");
@@ -600,10 +684,67 @@
ParallelCompactData::is_region_aligned(HeapWord* addr) const
{
return region_offset(addr) == 0;
}
+inline size_t
+ParallelCompactData::block_offset(const HeapWord* addr) const
+{
+ assert(addr >= _region_start, "bad addr");
+ assert(addr <= _region_end, "bad addr");
+ return (size_t(addr) & BlockAddrOffsetMask) >> LogHeapWordSize;
+}
+
+inline size_t
+ParallelCompactData::addr_to_block_idx(const HeapWord* addr) const
+{
+ assert(addr >= _region_start, "bad addr");
+ assert(addr <= _region_end, "bad addr");
+ return pointer_delta(addr, _region_start) >> Log2BlockSize;
+}
+
+inline ParallelCompactData::BlockData*
+ParallelCompactData::addr_to_block_ptr(const HeapWord* addr) const
+{
+ return block(addr_to_block_idx(addr));
+}
+
+inline HeapWord*
+ParallelCompactData::block_to_addr(size_t block) const
+{
+ assert(block < _block_count, "block out of range");
+ return _region_start + (block << Log2BlockSize);
+}
+
+inline size_t
+ParallelCompactData::region_to_block_idx(size_t region) const
+{
+ return region << Log2BlocksPerRegion;
+}
+
+inline HeapWord*
+ParallelCompactData::block_align_down(HeapWord* addr) const
+{
+ assert(addr >= _region_start, "bad addr");
+ assert(addr < _region_end + RegionSize, "bad addr");
+ return (HeapWord*)(size_t(addr) & BlockAddrMask);
+}
+
+inline HeapWord*
+ParallelCompactData::block_align_up(HeapWord* addr) const
+{
+ assert(addr >= _region_start, "bad addr");
+ assert(addr <= _region_end, "bad addr");
+ return block_align_down(addr + BlockSizeOffsetMask);
+}
+
+inline bool
+ParallelCompactData::is_block_aligned(HeapWord* addr) const
+{
+ return block_offset(addr) == 0;
+}
+
// Abstract closure for use with ParMarkBitMap::iterate(), which will invoke the
// do_addr() method.
//
// The closure is initialized with the number of heap words to process
// (words_remaining()), and becomes 'full' when it reaches 0. The do_addr()
@@ -777,10 +918,11 @@
class PSParallelCompact : AllStatic {
public:
// Convenient access to type names.
typedef ParMarkBitMap::idx_t idx_t;
typedef ParallelCompactData::RegionData RegionData;
+ typedef ParallelCompactData::BlockData BlockData;
typedef enum {
perm_space_id, old_space_id, eden_space_id,
from_space_id, to_space_id, last_space_id
} SpaceId;
@@ -985,10 +1127,12 @@
static void adjust_roots();
// Serial code executed in preparation for the compaction phase.
static void compact_prologue();
+ DEBUG_ONLY(static void write_block_fill_histogram(outputStream* const out);)
+
// Move objects to new locations.
static void compact_perm(ParCompactionManager* cm);
static void compact();
// Add available regions to the stack and draining tasks to the task queue.
@@ -1156,10 +1300,13 @@
static void fill_region(ParCompactionManager* cm, size_t region_idx);
static void fill_and_update_region(ParCompactionManager* cm, size_t region) {
fill_region(cm, region);
}
+ // Fill in the block table for the specified region.
+ static void fill_blocks(size_t region_idx);
+
// Update the deferred objects in the space.
static void update_deferred_objects(ParCompactionManager* cm, SpaceId id);
static ParMarkBitMap* mark_bitmap() { return &_mark_bitmap; }
static ParallelCompactData& summary_data() { return _summary_data; }