206 HeapWord* _dense_prefix;
207 ObjectStartArray* _start_array;
208 SplitInfo _split_info;
209 };
210
211 class ParallelCompactData
212 {
213 public:
214 // Sizes are in HeapWords, unless indicated otherwise.
215 static const size_t Log2RegionSize;
216 static const size_t RegionSize;
217 static const size_t RegionSizeBytes;
218
219 // Mask for the bits in a size_t to get an offset within a region.
220 static const size_t RegionSizeOffsetMask;
221 // Mask for the bits in a pointer to get an offset within a region.
222 static const size_t RegionAddrOffsetMask;
223 // Mask for the bits in a pointer to get the address of the start of a region.
224 static const size_t RegionAddrMask;
225
226 class RegionData
227 {
228 public:
229 // Destination address of the region.
230 HeapWord* destination() const { return _destination; }
231
232 // The first region containing data destined for this region.
233 size_t source_region() const { return _source_region; }
234
235 // The object (if any) starting in this region and ending in a different
236 // region that could not be updated during the main (parallel) compaction
237 // phase. This is different from _partial_obj_addr, which is an object that
238 // extends onto a source region. However, the two uses do not overlap in
239 // time, so the same field is used to save space.
240 HeapWord* deferred_obj_addr() const { return _partial_obj_addr; }
241
242 // The starting address of the partial object extending onto the region.
243 HeapWord* partial_obj_addr() const { return _partial_obj_addr; }
244
245 // Size of the partial object extending onto the region (words).
258 // this region will be copied. At the end of the summary phase, the valid
259 // values of destination_count are
260 //
261 // 0 - data from the region will be compacted completely into itself, or the
262 // region is empty. The region can be claimed and then filled.
263 // 1 - data from the region will be compacted into 1 other region; some
264 // data from the region may also be compacted into the region itself.
265 // 2 - data from the region will be copied to 2 other regions.
266 //
267 // During compaction as regions are emptied, the destination_count is
268 // decremented (atomically) and when it reaches 0, it can be claimed and
269 // then filled.
270 //
271 // A region is claimed for processing by atomically changing the
272 // destination_count to the claimed value (dc_claimed). After a region has
273 // been filled, the destination_count should be set to the completed value
274 // (dc_completed).
275 inline uint destination_count() const;
276 inline uint destination_count_raw() const;
277
278 // The location of the java heap data that corresponds to this region.
279 inline HeapWord* data_location() const;
280
281 // The highest address referenced by objects in this region.
282 inline HeapWord* highest_ref() const;
283
284 // Whether this region is available to be claimed, has been claimed, or has
285 // been completed.
286 //
287 // Minor subtlety: claimed() returns true if the region is marked
288 // completed(), which is desirable since a region must be claimed before it
289 // can be completed.
290 bool available() const { return _dc_and_los < dc_one; }
291 bool claimed() const { return _dc_and_los >= dc_claimed; }
292 bool completed() const { return _dc_and_los >= dc_completed; }
293
294 // These are not atomic.
295 void set_destination(HeapWord* addr) { _destination = addr; }
296 void set_source_region(size_t region) { _source_region = region; }
297 void set_deferred_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; }
298 void set_partial_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; }
299 void set_partial_obj_size(size_t words) {
300 _partial_obj_size = (region_sz_t) words;
301 }
302
303 inline void set_destination_count(uint count);
304 inline void set_live_obj_size(size_t words);
305 inline void set_data_location(HeapWord* addr);
306 inline void set_completed();
307 inline bool claim_unsafe();
308
309 // These are atomic.
310 inline void add_live_obj(size_t words);
311 inline void set_highest_ref(HeapWord* addr);
312 inline void decrement_destination_count();
313 inline bool claim();
314
315 private:
316 // The type used to represent object sizes within a region.
317 typedef uint region_sz_t;
318
319 // Constants for manipulating the _dc_and_los field, which holds both the
320 // destination count and live obj size. The live obj size lives at the
321 // least significant end so no masking is necessary when adding.
322 static const region_sz_t dc_shift; // Shift amount.
323 static const region_sz_t dc_mask; // Mask for destination count.
324 static const region_sz_t dc_one; // 1, shifted appropriately.
325 static const region_sz_t dc_claimed; // Region has been claimed.
326 static const region_sz_t dc_completed; // Region has been completed.
327 static const region_sz_t los_mask; // Mask for live obj size.
328
329 HeapWord* _destination;
330 size_t _source_region;
331 HeapWord* _partial_obj_addr;
332 region_sz_t _partial_obj_size;
333 region_sz_t volatile _dc_and_los;
334 #ifdef ASSERT
335 // These enable optimizations that are only partially implemented. Use
336 // debug builds to prevent the code fragments from breaking.
337 HeapWord* _data_location;
338 HeapWord* _highest_ref;
339 #endif // #ifdef ASSERT
340
341 #ifdef ASSERT
342 public:
343 uint _pushed; // 0 until region is pushed onto a worker's stack
344 private:
345 #endif
346 };
347
348 public:
349 ParallelCompactData();
350 bool initialize(MemRegion covered_region);
351
352 size_t region_count() const { return _region_count; }
353
354 // Convert region indices to/from RegionData pointers.
355 inline RegionData* region(size_t region_idx) const;
356 inline size_t region(const RegionData* const region_ptr) const;
357
358 // Returns true if the given address is contained within the region
359 bool region_contains(size_t region_index, HeapWord* addr);
360
361 void add_obj(HeapWord* addr, size_t len);
362 void add_obj(oop p, size_t len) { add_obj((HeapWord*)p, len); }
363
364 // Fill in the regions covering [beg, end) so that no data moves; i.e., the
365 // destination of region n is simply the start of region n. The argument beg
366 // must be region-aligned; end need not be.
367 void summarize_dense_prefix(HeapWord* beg, HeapWord* end);
368
369 HeapWord* summarize_split_space(size_t src_region, SplitInfo& split_info,
370 HeapWord* destination, HeapWord* target_end,
371 HeapWord** target_next);
372 bool summarize(SplitInfo& split_info,
373 HeapWord* source_beg, HeapWord* source_end,
374 HeapWord** source_next,
375 HeapWord* target_beg, HeapWord* target_end,
376 HeapWord** target_next);
377
378 void clear();
379 void clear_range(size_t beg_region, size_t end_region);
380 void clear_range(HeapWord* beg, HeapWord* end) {
381 clear_range(addr_to_region_idx(beg), addr_to_region_idx(end));
382 }
383
384 // Return the number of words between addr and the start of the region
385 // containing addr.
386 inline size_t region_offset(const HeapWord* addr) const;
387
388 // Convert addresses to/from a region index or region pointer.
389 inline size_t addr_to_region_idx(const HeapWord* addr) const;
390 inline RegionData* addr_to_region_ptr(const HeapWord* addr) const;
391 inline HeapWord* region_to_addr(size_t region) const;
392 inline HeapWord* region_to_addr(size_t region, size_t offset) const;
393 inline HeapWord* region_to_addr(const RegionData* region) const;
394
395 inline HeapWord* region_align_down(HeapWord* addr) const;
396 inline HeapWord* region_align_up(HeapWord* addr) const;
397 inline bool is_region_aligned(HeapWord* addr) const;
398
399 // Return the address one past the end of the partial object.
400 HeapWord* partial_obj_end(size_t region_idx) const;
401
402 // Return the new location of the object p after the
403 // the compaction.
404 HeapWord* calc_new_pointer(HeapWord* addr);
405
406 HeapWord* calc_new_pointer(oop p) {
407 return calc_new_pointer((HeapWord*) p);
408 }
409
410 // Return the updated address for the given klass
411 klassOop calc_new_klass(klassOop);
412
413 #ifdef ASSERT
414 void verify_clear(const PSVirtualSpace* vspace);
415 void verify_clear();
416 #endif // #ifdef ASSERT
417
418 private:
419 bool initialize_region_data(size_t region_size);
420 PSVirtualSpace* create_vspace(size_t count, size_t element_size);
421
422 private:
423 HeapWord* _region_start;
424 #ifdef ASSERT
425 HeapWord* _region_end;
426 #endif // #ifdef ASSERT
427
428 PSVirtualSpace* _region_vspace;
429 RegionData* _region_data;
430 size_t _region_count;
431 };
432
433 inline uint
434 ParallelCompactData::RegionData::destination_count_raw() const
435 {
436 return _dc_and_los & dc_mask;
437 }
438
439 inline uint
440 ParallelCompactData::RegionData::destination_count() const
441 {
442 return destination_count_raw() >> dc_shift;
443 }
444
445 inline void
446 ParallelCompactData::RegionData::set_destination_count(uint count)
447 {
448 assert(count <= (dc_completed >> dc_shift), "count too large");
449 const region_sz_t live_sz = (region_sz_t) live_obj_size();
450 _dc_and_los = (count << dc_shift) | live_sz;
451 }
452
453 inline void ParallelCompactData::RegionData::set_live_obj_size(size_t words)
454 {
455 assert(words <= los_mask, "would overflow");
456 _dc_and_los = destination_count_raw() | (region_sz_t)words;
457 }
458
459 inline void ParallelCompactData::RegionData::decrement_destination_count()
460 {
461 assert(_dc_and_los < dc_claimed, "already claimed");
462 assert(_dc_and_los >= dc_one, "count would go negative");
463 Atomic::add((int)dc_mask, (volatile int*)&_dc_and_los);
464 }
465
519 const int old = Atomic::cmpxchg(dc_claimed | los,
520 (volatile int*) &_dc_and_los, los);
521 return old == los;
522 }
523
524 inline ParallelCompactData::RegionData*
525 ParallelCompactData::region(size_t region_idx) const
526 {
527 assert(region_idx <= region_count(), "bad arg");
528 return _region_data + region_idx;
529 }
530
531 inline size_t
532 ParallelCompactData::region(const RegionData* const region_ptr) const
533 {
534 assert(region_ptr >= _region_data, "bad arg");
535 assert(region_ptr <= _region_data + region_count(), "bad arg");
536 return pointer_delta(region_ptr, _region_data, sizeof(RegionData));
537 }
538
539 inline size_t
540 ParallelCompactData::region_offset(const HeapWord* addr) const
541 {
542 assert(addr >= _region_start, "bad addr");
543 assert(addr <= _region_end, "bad addr");
544 return (size_t(addr) & RegionAddrOffsetMask) >> LogHeapWordSize;
545 }
546
547 inline size_t
548 ParallelCompactData::addr_to_region_idx(const HeapWord* addr) const
549 {
550 assert(addr >= _region_start, "bad addr");
551 assert(addr <= _region_end, "bad addr");
552 return pointer_delta(addr, _region_start) >> Log2RegionSize;
553 }
554
555 inline ParallelCompactData::RegionData*
556 ParallelCompactData::addr_to_region_ptr(const HeapWord* addr) const
557 {
558 return region(addr_to_region_idx(addr));
585 {
586 assert(addr >= _region_start, "bad addr");
587 assert(addr < _region_end + RegionSize, "bad addr");
588 return (HeapWord*)(size_t(addr) & RegionAddrMask);
589 }
590
591 inline HeapWord*
592 ParallelCompactData::region_align_up(HeapWord* addr) const
593 {
594 assert(addr >= _region_start, "bad addr");
595 assert(addr <= _region_end, "bad addr");
596 return region_align_down(addr + RegionSizeOffsetMask);
597 }
598
599 inline bool
600 ParallelCompactData::is_region_aligned(HeapWord* addr) const
601 {
602 return region_offset(addr) == 0;
603 }
604
605 // Abstract closure for use with ParMarkBitMap::iterate(), which will invoke the
606 // do_addr() method.
607 //
608 // The closure is initialized with the number of heap words to process
609 // (words_remaining()), and becomes 'full' when it reaches 0. The do_addr()
610 // methods in subclasses should update the total as words are processed. Since
611 // only one subclass actually uses this mechanism to terminate iteration, the
612 // default initial value is > 0. The implementation is here and not in the
613 // single subclass that uses it to avoid making is_full() virtual, and thus
614 // adding a virtual call per live object.
615
616 class ParMarkBitMapClosure: public StackObj {
617 public:
618 typedef ParMarkBitMap::idx_t idx_t;
619 typedef ParMarkBitMap::IterationStatus IterationStatus;
620
621 public:
622 inline ParMarkBitMapClosure(ParMarkBitMap* mbm, ParCompactionManager* cm,
623 size_t words = max_uintx);
624
762 // has been updated. KKK likely resides in a region to the left of the region
763 // containing AAA. These AAA's have there references updated at the end in a
764 // clean up phase. See the method PSParallelCompact::update_deferred_objects().
765 // An alternate strategy is being investigated for this deferral of updating.
766 //
767 // Compaction is done on a region basis. A region that is ready to be filled is
768 // put on a ready list and GC threads take region off the list and fill them. A
769 // region is ready to be filled if it empty of live objects. Such a region may
770 // have been initially empty (only contained dead objects) or may have had all
771 // its live objects copied out already. A region that compacts into itself is
772 // also ready for filling. The ready list is initially filled with empty
773 // regions and regions compacting into themselves. There is always at least 1
774 // region that can be put on the ready list. The regions are atomically added
775 // and removed from the ready list.
776
777 class PSParallelCompact : AllStatic {
778 public:
779 // Convenient access to type names.
780 typedef ParMarkBitMap::idx_t idx_t;
781 typedef ParallelCompactData::RegionData RegionData;
782
783 typedef enum {
784 perm_space_id, old_space_id, eden_space_id,
785 from_space_id, to_space_id, last_space_id
786 } SpaceId;
787
788 public:
789 // Inline closure decls
790 //
791 class IsAliveClosure: public BoolObjectClosure {
792 public:
793 virtual void do_object(oop p);
794 virtual bool do_object_b(oop p);
795 };
796
797 class KeepAliveClosure: public OopClosure {
798 private:
799 ParCompactionManager* _compaction_manager;
800 protected:
801 template <class T> inline void do_oop_work(T* p);
970 static void summarize_new_objects(SpaceId id, HeapWord* start);
971
972 // Add live objects to a survivor space since it's rare that both survivors
973 // are non-empty.
974 static void provoke_split_fill_survivor(SpaceId id);
975
976 // Add live objects and/or choose the dense prefix to provoke splitting.
977 static void provoke_split(bool & maximum_compaction);
978 #endif
979
980 static void summarize_spaces_quick();
981 static void summarize_space(SpaceId id, bool maximum_compaction);
982 static void summary_phase(ParCompactionManager* cm, bool maximum_compaction);
983
984 // Adjust addresses in roots. Does not adjust addresses in heap.
985 static void adjust_roots();
986
987 // Serial code executed in preparation for the compaction phase.
988 static void compact_prologue();
989
990 // Move objects to new locations.
991 static void compact_perm(ParCompactionManager* cm);
992 static void compact();
993
994 // Add available regions to the stack and draining tasks to the task queue.
995 static void enqueue_region_draining_tasks(GCTaskQueue* q,
996 uint parallel_gc_threads);
997
998 // Add dense prefix update tasks to the task queue.
999 static void enqueue_dense_prefix_tasks(GCTaskQueue* q,
1000 uint parallel_gc_threads);
1001
1002 // Add region stealing tasks to the task queue.
1003 static void enqueue_region_stealing_tasks(
1004 GCTaskQueue* q,
1005 ParallelTaskTerminator* terminator_ptr,
1006 uint parallel_gc_threads);
1007
1008 // If objects are left in eden after a collection, try to move the boundary
1009 // and absorb them into the old gen. Returns true if eden was emptied.
1141 // (out parameter) accordingly.
1142 static size_t next_src_region(MoveAndUpdateClosure& closure,
1143 SpaceId& src_space_id,
1144 HeapWord*& src_space_top,
1145 HeapWord* end_addr);
1146
1147 // Decrement the destination count for each non-empty source region in the
1148 // range [beg_region, region(region_align_up(end_addr))). If the destination
1149 // count for a region goes to 0 and it needs to be filled, enqueue it.
1150 static void decrement_destination_counts(ParCompactionManager* cm,
1151 SpaceId src_space_id,
1152 size_t beg_region,
1153 HeapWord* end_addr);
1154
1155 // Fill a region, copying objects from one or more source regions.
1156 static void fill_region(ParCompactionManager* cm, size_t region_idx);
1157 static void fill_and_update_region(ParCompactionManager* cm, size_t region) {
1158 fill_region(cm, region);
1159 }
1160
1161 // Update the deferred objects in the space.
1162 static void update_deferred_objects(ParCompactionManager* cm, SpaceId id);
1163
1164 static ParMarkBitMap* mark_bitmap() { return &_mark_bitmap; }
1165 static ParallelCompactData& summary_data() { return _summary_data; }
1166
1167 static inline void adjust_pointer(oop* p) { adjust_pointer(p, false); }
1168 static inline void adjust_pointer(narrowOop* p) { adjust_pointer(p, false); }
1169
1170 // Reference Processing
1171 static ReferenceProcessor* const ref_processor() { return _ref_processor; }
1172
1173 static STWGCTimer* gc_timer() { return &_gc_timer; }
1174
1175 // Return the SpaceId for the given address.
1176 static SpaceId space_id(HeapWord* addr);
1177
1178 // Time since last full gc (in milliseconds).
1179 static jlong millis_since_last_gc();
1180
|
206 HeapWord* _dense_prefix;
207 ObjectStartArray* _start_array;
208 SplitInfo _split_info;
209 };
210
211 class ParallelCompactData
212 {
213 public:
214 // Sizes are in HeapWords, unless indicated otherwise.
215 static const size_t Log2RegionSize;
216 static const size_t RegionSize;
217 static const size_t RegionSizeBytes;
218
219 // Mask for the bits in a size_t to get an offset within a region.
220 static const size_t RegionSizeOffsetMask;
221 // Mask for the bits in a pointer to get an offset within a region.
222 static const size_t RegionAddrOffsetMask;
223 // Mask for the bits in a pointer to get the address of the start of a region.
224 static const size_t RegionAddrMask;
225
226 static const size_t Log2BlockSize;
227 static const size_t BlockSize;
228 static const size_t BlockSizeBytes;
229
230 static const size_t BlockSizeOffsetMask;
231 static const size_t BlockAddrOffsetMask;
232 static const size_t BlockAddrMask;
233
234 static const size_t BlocksPerRegion;
235 static const size_t Log2BlocksPerRegion;
236
237 class RegionData
238 {
239 public:
240 // Destination address of the region.
241 HeapWord* destination() const { return _destination; }
242
243 // The first region containing data destined for this region.
244 size_t source_region() const { return _source_region; }
245
246 // The object (if any) starting in this region and ending in a different
247 // region that could not be updated during the main (parallel) compaction
248 // phase. This is different from _partial_obj_addr, which is an object that
249 // extends onto a source region. However, the two uses do not overlap in
250 // time, so the same field is used to save space.
251 HeapWord* deferred_obj_addr() const { return _partial_obj_addr; }
252
253 // The starting address of the partial object extending onto the region.
254 HeapWord* partial_obj_addr() const { return _partial_obj_addr; }
255
256 // Size of the partial object extending onto the region (words).
269 // this region will be copied. At the end of the summary phase, the valid
270 // values of destination_count are
271 //
272 // 0 - data from the region will be compacted completely into itself, or the
273 // region is empty. The region can be claimed and then filled.
274 // 1 - data from the region will be compacted into 1 other region; some
275 // data from the region may also be compacted into the region itself.
276 // 2 - data from the region will be copied to 2 other regions.
277 //
278 // During compaction as regions are emptied, the destination_count is
279 // decremented (atomically) and when it reaches 0, it can be claimed and
280 // then filled.
281 //
282 // A region is claimed for processing by atomically changing the
283 // destination_count to the claimed value (dc_claimed). After a region has
284 // been filled, the destination_count should be set to the completed value
285 // (dc_completed).
286 inline uint destination_count() const;
287 inline uint destination_count_raw() const;
288
289 // Whether the block table for this region has been filled.
290 inline bool blocks_filled() const;
291
292 // Number of times the block table was filled.
293 DEBUG_ONLY(inline size_t blocks_filled_count() const;)
294
295 // The location of the java heap data that corresponds to this region.
296 inline HeapWord* data_location() const;
297
298 // The highest address referenced by objects in this region.
299 inline HeapWord* highest_ref() const;
300
301 // Whether this region is available to be claimed, has been claimed, or has
302 // been completed.
303 //
304 // Minor subtlety: claimed() returns true if the region is marked
305 // completed(), which is desirable since a region must be claimed before it
306 // can be completed.
307 bool available() const { return _dc_and_los < dc_one; }
308 bool claimed() const { return _dc_and_los >= dc_claimed; }
309 bool completed() const { return _dc_and_los >= dc_completed; }
310
311 // These are not atomic.
312 void set_destination(HeapWord* addr) { _destination = addr; }
313 void set_source_region(size_t region) { _source_region = region; }
314 void set_deferred_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; }
315 void set_partial_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; }
316 void set_partial_obj_size(size_t words) {
317 _partial_obj_size = (region_sz_t) words;
318 }
319 inline void set_blocks_filled();
320
321 inline void set_destination_count(uint count);
322 inline void set_live_obj_size(size_t words);
323 inline void set_data_location(HeapWord* addr);
324 inline void set_completed();
325 inline bool claim_unsafe();
326
327 // These are atomic.
328 inline void add_live_obj(size_t words);
329 inline void set_highest_ref(HeapWord* addr);
330 inline void decrement_destination_count();
331 inline bool claim();
332
333 private:
334 // The type used to represent object sizes within a region.
335 typedef uint region_sz_t;
336
337 // Constants for manipulating the _dc_and_los field, which holds both the
338 // destination count and live obj size. The live obj size lives at the
339 // least significant end so no masking is necessary when adding.
340 static const region_sz_t dc_shift; // Shift amount.
341 static const region_sz_t dc_mask; // Mask for destination count.
342 static const region_sz_t dc_one; // 1, shifted appropriately.
343 static const region_sz_t dc_claimed; // Region has been claimed.
344 static const region_sz_t dc_completed; // Region has been completed.
345 static const region_sz_t los_mask; // Mask for live obj size.
346
347 HeapWord* _destination;
348 size_t _source_region;
349 HeapWord* _partial_obj_addr;
350 region_sz_t _partial_obj_size;
351 region_sz_t volatile _dc_and_los;
352 bool _blocks_filled;
353
354 #ifdef ASSERT
355 size_t _blocks_filled_count; // Number of block table fills.
356
357 // These enable optimizations that are only partially implemented. Use
358 // debug builds to prevent the code fragments from breaking.
359 HeapWord* _data_location;
360 HeapWord* _highest_ref;
361 #endif // #ifdef ASSERT
362
363 #ifdef ASSERT
364 public:
365 uint _pushed; // 0 until region is pushed onto a stack
366 private:
367 #endif
368 };
369
370 // "Blocks" allow shorter sections of the bitmap to be searched. Each Block
371 // holds an offset, which is the amount of live data in the Region to the left
372 // of the first live object that starts in the Block.
373 class BlockData
374 {
375 public:
376 typedef unsigned short int blk_ofs_t;
377
378 blk_ofs_t offset() const { return _offset; }
379 void set_offset(size_t val) { _offset = (blk_ofs_t)val; }
380
381 private:
382 blk_ofs_t _offset;
383 };
384
385 public:
386 ParallelCompactData();
387 bool initialize(MemRegion covered_region);
388
389 size_t region_count() const { return _region_count; }
390
391 // Convert region indices to/from RegionData pointers.
392 inline RegionData* region(size_t region_idx) const;
393 inline size_t region(const RegionData* const region_ptr) const;
394
395 size_t block_count() const { return _block_count; }
396 inline BlockData* block(size_t block_idx) const;
397 inline size_t block(const BlockData* block_ptr) const;
398
399 void add_obj(HeapWord* addr, size_t len);
400 void add_obj(oop p, size_t len) { add_obj((HeapWord*)p, len); }
401
402 // Fill in the regions covering [beg, end) so that no data moves; i.e., the
403 // destination of region n is simply the start of region n. The argument beg
404 // must be region-aligned; end need not be.
405 void summarize_dense_prefix(HeapWord* beg, HeapWord* end);
406
407 HeapWord* summarize_split_space(size_t src_region, SplitInfo& split_info,
408 HeapWord* destination, HeapWord* target_end,
409 HeapWord** target_next);
410 bool summarize(SplitInfo& split_info,
411 HeapWord* source_beg, HeapWord* source_end,
412 HeapWord** source_next,
413 HeapWord* target_beg, HeapWord* target_end,
414 HeapWord** target_next);
415
416 void clear();
417 void clear_range(size_t beg_region, size_t end_region);
418 void clear_range(HeapWord* beg, HeapWord* end) {
419 clear_range(addr_to_region_idx(beg), addr_to_region_idx(end));
420 }
421
422 // Return the number of words between addr and the start of the region
423 // containing addr.
424 inline size_t region_offset(const HeapWord* addr) const;
425
426 // Convert addresses to/from a region index or region pointer.
427 inline size_t addr_to_region_idx(const HeapWord* addr) const;
428 inline RegionData* addr_to_region_ptr(const HeapWord* addr) const;
429 inline HeapWord* region_to_addr(size_t region) const;
430 inline HeapWord* region_to_addr(size_t region, size_t offset) const;
431 inline HeapWord* region_to_addr(const RegionData* region) const;
432
433 inline HeapWord* region_align_down(HeapWord* addr) const;
434 inline HeapWord* region_align_up(HeapWord* addr) const;
435 inline bool is_region_aligned(HeapWord* addr) const;
436
437 // Analogous to region_offset() for blocks.
438 size_t block_offset(const HeapWord* addr) const;
439 size_t addr_to_block_idx(const HeapWord* addr) const;
440 size_t addr_to_block_idx(const oop obj) const {
441 return addr_to_block_idx((HeapWord*) obj);
442 }
443 inline BlockData* addr_to_block_ptr(const HeapWord* addr) const;
444 inline HeapWord* block_to_addr(size_t block) const;
445 inline size_t region_to_block_idx(size_t region) const;
446
447 inline HeapWord* block_align_down(HeapWord* addr) const;
448 inline HeapWord* block_align_up(HeapWord* addr) const;
449 inline bool is_block_aligned(HeapWord* addr) const;
450
451 // Return the address one past the end of the partial object.
452 HeapWord* partial_obj_end(size_t region_idx) const;
453
454 // Return the location of the object after compaction.
455 HeapWord* calc_new_pointer(HeapWord* addr);
456
457 HeapWord* calc_new_pointer(oop p) {
458 return calc_new_pointer((HeapWord*) p);
459 }
460
461 // Return the updated address for the given klass
462 klassOop calc_new_klass(klassOop);
463
464 #ifdef ASSERT
465 void verify_clear(const PSVirtualSpace* vspace);
466 void verify_clear();
467 #endif // #ifdef ASSERT
468
469 private:
470 bool initialize_block_data();
471 bool initialize_region_data(size_t region_size);
472 PSVirtualSpace* create_vspace(size_t count, size_t element_size);
473
474 private:
475 HeapWord* _region_start;
476 #ifdef ASSERT
477 HeapWord* _region_end;
478 #endif // #ifdef ASSERT
479
480 PSVirtualSpace* _region_vspace;
481 RegionData* _region_data;
482 size_t _region_count;
483
484 PSVirtualSpace* _block_vspace;
485 BlockData* _block_data;
486 size_t _block_count;
487 };
488
489 inline uint
490 ParallelCompactData::RegionData::destination_count_raw() const
491 {
492 return _dc_and_los & dc_mask;
493 }
494
495 inline uint
496 ParallelCompactData::RegionData::destination_count() const
497 {
498 return destination_count_raw() >> dc_shift;
499 }
500
501 inline bool
502 ParallelCompactData::RegionData::blocks_filled() const
503 {
504 return _blocks_filled;
505 }
506
507 #ifdef ASSERT
508 inline size_t
509 ParallelCompactData::RegionData::blocks_filled_count() const
510 {
511 return _blocks_filled_count;
512 }
513 #endif // #ifdef ASSERT
514
515 inline void
516 ParallelCompactData::RegionData::set_blocks_filled()
517 {
518 _blocks_filled = true;
519 // Debug builds count the number of times the table was filled.
520 DEBUG_ONLY(Atomic::inc_ptr(&_blocks_filled_count));
521 }
522
523 inline void
524 ParallelCompactData::RegionData::set_destination_count(uint count)
525 {
526 assert(count <= (dc_completed >> dc_shift), "count too large");
527 const region_sz_t live_sz = (region_sz_t) live_obj_size();
528 _dc_and_los = (count << dc_shift) | live_sz;
529 }
530
531 inline void ParallelCompactData::RegionData::set_live_obj_size(size_t words)
532 {
533 assert(words <= los_mask, "would overflow");
534 _dc_and_los = destination_count_raw() | (region_sz_t)words;
535 }
536
537 inline void ParallelCompactData::RegionData::decrement_destination_count()
538 {
539 assert(_dc_and_los < dc_claimed, "already claimed");
540 assert(_dc_and_los >= dc_one, "count would go negative");
541 Atomic::add((int)dc_mask, (volatile int*)&_dc_and_los);
542 }
543
597 const int old = Atomic::cmpxchg(dc_claimed | los,
598 (volatile int*) &_dc_and_los, los);
599 return old == los;
600 }
601
602 inline ParallelCompactData::RegionData*
603 ParallelCompactData::region(size_t region_idx) const
604 {
605 assert(region_idx <= region_count(), "bad arg");
606 return _region_data + region_idx;
607 }
608
609 inline size_t
610 ParallelCompactData::region(const RegionData* const region_ptr) const
611 {
612 assert(region_ptr >= _region_data, "bad arg");
613 assert(region_ptr <= _region_data + region_count(), "bad arg");
614 return pointer_delta(region_ptr, _region_data, sizeof(RegionData));
615 }
616
617 inline ParallelCompactData::BlockData*
618 ParallelCompactData::block(size_t n) const {
619 assert(n < block_count(), "bad arg");
620 return _block_data + n;
621 }
622
623 inline size_t
624 ParallelCompactData::region_offset(const HeapWord* addr) const
625 {
626 assert(addr >= _region_start, "bad addr");
627 assert(addr <= _region_end, "bad addr");
628 return (size_t(addr) & RegionAddrOffsetMask) >> LogHeapWordSize;
629 }
630
631 inline size_t
632 ParallelCompactData::addr_to_region_idx(const HeapWord* addr) const
633 {
634 assert(addr >= _region_start, "bad addr");
635 assert(addr <= _region_end, "bad addr");
636 return pointer_delta(addr, _region_start) >> Log2RegionSize;
637 }
638
639 inline ParallelCompactData::RegionData*
640 ParallelCompactData::addr_to_region_ptr(const HeapWord* addr) const
641 {
642 return region(addr_to_region_idx(addr));
669 {
670 assert(addr >= _region_start, "bad addr");
671 assert(addr < _region_end + RegionSize, "bad addr");
672 return (HeapWord*)(size_t(addr) & RegionAddrMask);
673 }
674
675 inline HeapWord*
676 ParallelCompactData::region_align_up(HeapWord* addr) const
677 {
678 assert(addr >= _region_start, "bad addr");
679 assert(addr <= _region_end, "bad addr");
680 return region_align_down(addr + RegionSizeOffsetMask);
681 }
682
683 inline bool
684 ParallelCompactData::is_region_aligned(HeapWord* addr) const
685 {
686 return region_offset(addr) == 0;
687 }
688
689 inline size_t
690 ParallelCompactData::block_offset(const HeapWord* addr) const
691 {
692 assert(addr >= _region_start, "bad addr");
693 assert(addr <= _region_end, "bad addr");
694 return (size_t(addr) & BlockAddrOffsetMask) >> LogHeapWordSize;
695 }
696
697 inline size_t
698 ParallelCompactData::addr_to_block_idx(const HeapWord* addr) const
699 {
700 assert(addr >= _region_start, "bad addr");
701 assert(addr <= _region_end, "bad addr");
702 return pointer_delta(addr, _region_start) >> Log2BlockSize;
703 }
704
705 inline ParallelCompactData::BlockData*
706 ParallelCompactData::addr_to_block_ptr(const HeapWord* addr) const
707 {
708 return block(addr_to_block_idx(addr));
709 }
710
711 inline HeapWord*
712 ParallelCompactData::block_to_addr(size_t block) const
713 {
714 assert(block < _block_count, "block out of range");
715 return _region_start + (block << Log2BlockSize);
716 }
717
718 inline size_t
719 ParallelCompactData::region_to_block_idx(size_t region) const
720 {
721 return region << Log2BlocksPerRegion;
722 }
723
724 inline HeapWord*
725 ParallelCompactData::block_align_down(HeapWord* addr) const
726 {
727 assert(addr >= _region_start, "bad addr");
728 assert(addr < _region_end + RegionSize, "bad addr");
729 return (HeapWord*)(size_t(addr) & BlockAddrMask);
730 }
731
732 inline HeapWord*
733 ParallelCompactData::block_align_up(HeapWord* addr) const
734 {
735 assert(addr >= _region_start, "bad addr");
736 assert(addr <= _region_end, "bad addr");
737 return block_align_down(addr + BlockSizeOffsetMask);
738 }
739
740 inline bool
741 ParallelCompactData::is_block_aligned(HeapWord* addr) const
742 {
743 return block_offset(addr) == 0;
744 }
745
746 // Abstract closure for use with ParMarkBitMap::iterate(), which will invoke the
747 // do_addr() method.
748 //
749 // The closure is initialized with the number of heap words to process
750 // (words_remaining()), and becomes 'full' when it reaches 0. The do_addr()
751 // methods in subclasses should update the total as words are processed. Since
752 // only one subclass actually uses this mechanism to terminate iteration, the
753 // default initial value is > 0. The implementation is here and not in the
754 // single subclass that uses it to avoid making is_full() virtual, and thus
755 // adding a virtual call per live object.
756
757 class ParMarkBitMapClosure: public StackObj {
758 public:
759 typedef ParMarkBitMap::idx_t idx_t;
760 typedef ParMarkBitMap::IterationStatus IterationStatus;
761
762 public:
763 inline ParMarkBitMapClosure(ParMarkBitMap* mbm, ParCompactionManager* cm,
764 size_t words = max_uintx);
765
903 // has been updated. KKK likely resides in a region to the left of the region
904 // containing AAA. These AAA's have there references updated at the end in a
905 // clean up phase. See the method PSParallelCompact::update_deferred_objects().
906 // An alternate strategy is being investigated for this deferral of updating.
907 //
908 // Compaction is done on a region basis. A region that is ready to be filled is
909 // put on a ready list and GC threads take region off the list and fill them. A
910 // region is ready to be filled if it empty of live objects. Such a region may
911 // have been initially empty (only contained dead objects) or may have had all
912 // its live objects copied out already. A region that compacts into itself is
913 // also ready for filling. The ready list is initially filled with empty
914 // regions and regions compacting into themselves. There is always at least 1
915 // region that can be put on the ready list. The regions are atomically added
916 // and removed from the ready list.
917
918 class PSParallelCompact : AllStatic {
919 public:
920 // Convenient access to type names.
921 typedef ParMarkBitMap::idx_t idx_t;
922 typedef ParallelCompactData::RegionData RegionData;
923 typedef ParallelCompactData::BlockData BlockData;
924
925 typedef enum {
926 perm_space_id, old_space_id, eden_space_id,
927 from_space_id, to_space_id, last_space_id
928 } SpaceId;
929
930 public:
931 // Inline closure decls
932 //
933 class IsAliveClosure: public BoolObjectClosure {
934 public:
935 virtual void do_object(oop p);
936 virtual bool do_object_b(oop p);
937 };
938
939 class KeepAliveClosure: public OopClosure {
940 private:
941 ParCompactionManager* _compaction_manager;
942 protected:
943 template <class T> inline void do_oop_work(T* p);
1112 static void summarize_new_objects(SpaceId id, HeapWord* start);
1113
1114 // Add live objects to a survivor space since it's rare that both survivors
1115 // are non-empty.
1116 static void provoke_split_fill_survivor(SpaceId id);
1117
1118 // Add live objects and/or choose the dense prefix to provoke splitting.
1119 static void provoke_split(bool & maximum_compaction);
1120 #endif
1121
1122 static void summarize_spaces_quick();
1123 static void summarize_space(SpaceId id, bool maximum_compaction);
1124 static void summary_phase(ParCompactionManager* cm, bool maximum_compaction);
1125
1126 // Adjust addresses in roots. Does not adjust addresses in heap.
1127 static void adjust_roots();
1128
1129 // Serial code executed in preparation for the compaction phase.
1130 static void compact_prologue();
1131
1132 DEBUG_ONLY(static void write_block_fill_histogram(outputStream* const out);)
1133
1134 // Move objects to new locations.
1135 static void compact_perm(ParCompactionManager* cm);
1136 static void compact();
1137
1138 // Add available regions to the stack and draining tasks to the task queue.
1139 static void enqueue_region_draining_tasks(GCTaskQueue* q,
1140 uint parallel_gc_threads);
1141
1142 // Add dense prefix update tasks to the task queue.
1143 static void enqueue_dense_prefix_tasks(GCTaskQueue* q,
1144 uint parallel_gc_threads);
1145
1146 // Add region stealing tasks to the task queue.
1147 static void enqueue_region_stealing_tasks(
1148 GCTaskQueue* q,
1149 ParallelTaskTerminator* terminator_ptr,
1150 uint parallel_gc_threads);
1151
1152 // If objects are left in eden after a collection, try to move the boundary
1153 // and absorb them into the old gen. Returns true if eden was emptied.
1285 // (out parameter) accordingly.
1286 static size_t next_src_region(MoveAndUpdateClosure& closure,
1287 SpaceId& src_space_id,
1288 HeapWord*& src_space_top,
1289 HeapWord* end_addr);
1290
1291 // Decrement the destination count for each non-empty source region in the
1292 // range [beg_region, region(region_align_up(end_addr))). If the destination
1293 // count for a region goes to 0 and it needs to be filled, enqueue it.
1294 static void decrement_destination_counts(ParCompactionManager* cm,
1295 SpaceId src_space_id,
1296 size_t beg_region,
1297 HeapWord* end_addr);
1298
1299 // Fill a region, copying objects from one or more source regions.
1300 static void fill_region(ParCompactionManager* cm, size_t region_idx);
1301 static void fill_and_update_region(ParCompactionManager* cm, size_t region) {
1302 fill_region(cm, region);
1303 }
1304
1305 // Fill in the block table for the specified region.
1306 static void fill_blocks(size_t region_idx);
1307
1308 // Update the deferred objects in the space.
1309 static void update_deferred_objects(ParCompactionManager* cm, SpaceId id);
1310
1311 static ParMarkBitMap* mark_bitmap() { return &_mark_bitmap; }
1312 static ParallelCompactData& summary_data() { return _summary_data; }
1313
1314 static inline void adjust_pointer(oop* p) { adjust_pointer(p, false); }
1315 static inline void adjust_pointer(narrowOop* p) { adjust_pointer(p, false); }
1316
1317 // Reference Processing
1318 static ReferenceProcessor* const ref_processor() { return _ref_processor; }
1319
1320 static STWGCTimer* gc_timer() { return &_gc_timer; }
1321
1322 // Return the SpaceId for the given address.
1323 static SpaceId space_id(HeapWord* addr);
1324
1325 // Time since last full gc (in milliseconds).
1326 static jlong millis_since_last_gc();
1327
|