327 // These are atomic.
328 inline void add_live_obj(size_t words);
329 inline void set_highest_ref(HeapWord* addr);
330 inline void decrement_destination_count();
331 inline bool claim();
332
333 // Possible values of _shadow_state, and transition is as follows
334 // Normal Path:
335 // UNUSED -> try_push() -> FINISHED
336 // Steal Path:
337 // UNUSED -> try_steal() -> SHADOW -> mark_filled() -> FILLED -> try_copy() -> FINISHED
338 static const int UNUSED; // Original state
339 static const int SHADOW; // Stolen by an idle thread, and a shadow region is created for it
340 static const int FILLED; // Its shadow region has been filled and ready to be copied back
341 static const int FINISH; // Work has been done
342
343 // Preempt the region to avoid double processes
344 inline bool try_push();
345 inline bool try_steal();
346 // Mark the region as filled and ready to be copied back
347 inline bool mark_filled();
348 // Preempt the region to copy the shadow region content back
349 inline bool try_copy();
350
351 int shadow_state() { return _shadow_state; }
352
353 private:
354 // The type used to represent object sizes within a region.
355 typedef uint region_sz_t;
356
357 // Constants for manipulating the _dc_and_los field, which holds both the
358 // destination count and live obj size. The live obj size lives at the
359 // least significant end so no masking is necessary when adding.
360 static const region_sz_t dc_shift; // Shift amount.
361 static const region_sz_t dc_mask; // Mask for destination count.
362 static const region_sz_t dc_one; // 1, shifted appropriately.
363 static const region_sz_t dc_claimed; // Region has been claimed.
364 static const region_sz_t dc_completed; // Region has been completed.
365 static const region_sz_t los_mask; // Mask for live obj size.
366
367 HeapWord* _destination;
368 size_t _source_region;
369 HeapWord* _partial_obj_addr;
612 tmp = Atomic::cmpxchg(addr, &_highest_ref, tmp);
613 }
614 #endif // #ifdef ASSERT
615 }
616
617 inline bool ParallelCompactData::RegionData::claim()
618 {
619 const region_sz_t los = static_cast<region_sz_t>(live_obj_size());
620 const region_sz_t old = Atomic::cmpxchg(dc_claimed | los, &_dc_and_los, los);
621 return old == los;
622 }
623
624 inline bool ParallelCompactData::RegionData::try_push() {
625 return Atomic::cmpxchg(FINISH, &_shadow_state, UNUSED) == UNUSED;
626 }
627
628 inline bool ParallelCompactData::RegionData::try_steal() {
629 return Atomic::cmpxchg(SHADOW, &_shadow_state, UNUSED) == UNUSED;
630 }
631
632 inline bool ParallelCompactData::RegionData::mark_filled() {
633 return Atomic::cmpxchg(FILLED, &_shadow_state, SHADOW) == SHADOW;
634 }
635
636 inline bool ParallelCompactData::RegionData::try_copy() {
637 return Atomic::cmpxchg(FINISH, &_shadow_state, FILLED) == FILLED;
638 }
639
640 inline ParallelCompactData::RegionData*
641 ParallelCompactData::region(size_t region_idx) const
642 {
643 assert(region_idx <= region_count(), "bad arg");
644 return _region_data + region_idx;
645 }
646
647 inline size_t
648 ParallelCompactData::region(const RegionData* const region_ptr) const
649 {
650 assert(region_ptr >= _region_data, "bad arg");
651 assert(region_ptr <= _region_data + region_count(), "bad arg");
652 return pointer_delta(region_ptr, _region_data, sizeof(RegionData));
653 }
654
655 inline ParallelCompactData::BlockData*
656 ParallelCompactData::block(size_t n) const {
657 assert(n < block_count(), "bad arg");
658 return _block_data + n;
659 }
1270 SpaceId src_space_id,
1271 HeapWord* src_beg, HeapWord* src_end);
1272 #endif // #ifndef PRODUCT
1273
1274 #ifdef ASSERT
1275 // Sanity check the new location of a word in the heap.
1276 static inline void check_new_location(HeapWord* old_addr, HeapWord* new_addr);
1277 // Verify that all the regions have been emptied.
1278 static void verify_complete(SpaceId space_id);
1279 #endif // #ifdef ASSERT
1280 };
1281
1282 class MoveAndUpdateClosure: public ParMarkBitMapClosure {
1283 static inline size_t calculate_words_remaining(size_t region);
1284 public:
1285 inline MoveAndUpdateClosure(ParMarkBitMap* bitmap, ParCompactionManager* cm,
1286 size_t region);
1287
1288 // Accessors.
1289 HeapWord* destination() const { return _destination; }
1290
1291 // If the object will fit (size <= words_remaining()), copy it to the current
1292 // destination, update the interior oops and the start array and return either
1293 // full (if the closure is full) or incomplete. If the object will not fit,
1294 // return would_overflow.
1295 IterationStatus do_addr(HeapWord* addr, size_t size);
1296
1297 // Copy enough words to fill this closure, starting at source(). Interior
1298 // oops and the start array are not updated. Return full.
1299 IterationStatus copy_until_full();
1300
1301 // Copy enough words to fill this closure or to the end of an object,
1302 // whichever is smaller, starting at source(). Interior oops and the start
1303 // array are not updated.
1304 void copy_partial_obj();
1305
1306 virtual void complete_region(ParCompactionManager* cm, HeapWord* dest_addr,
1307 PSParallelCompact::RegionData* region_ptr);
1308
1309 virtual void allocate_block();
1310
1311 virtual void set_deferred_obj_addr_for(PSParallelCompact::RegionData* region_ptr);
1312
1313 protected:
1314 // Update variables to indicate that word_count words were processed.
1315 inline void update_state(size_t word_count);
1316
1317 protected:
1318 HeapWord* _destination; // Next addr to be written.
1319 ObjectStartArray* const _start_array;
1320 };
1321
1322 inline size_t MoveAndUpdateClosure::calculate_words_remaining(size_t region) {
1323 HeapWord* dest_addr = PSParallelCompact::summary_data().region_to_addr(region);
1324 PSParallelCompact::SpaceId dest_space_id = PSParallelCompact::space_id(dest_addr);
1325 HeapWord* new_top = PSParallelCompact::new_top(dest_space_id);
1326 assert(dest_addr < new_top, "sanity");
1327
1328 return MIN2(pointer_delta(new_top, dest_addr), ParallelCompactData::RegionSize);
1329 }
1330
1331 inline
1332 MoveAndUpdateClosure::MoveAndUpdateClosure(ParMarkBitMap* bitmap,
1333 ParCompactionManager* cm,
1334 size_t region_idx) :
1335 ParMarkBitMapClosure(bitmap, cm, calculate_words_remaining(region_idx)),
1336 _destination(PSParallelCompact::summary_data().region_to_addr(region_idx)),
1337 _start_array(PSParallelCompact::start_array(PSParallelCompact::space_id(_destination))) { }
1338
1339
1340 inline void MoveAndUpdateClosure::update_state(size_t words)
1341 {
1342 decrement_words_remaining(words);
1343 _source += words;
1344 _destination += words;
1345 }
1346
1347 class ShadowClosure: public MoveAndUpdateClosure {
1348 inline size_t calculate_shadow_offset(size_t region_idx, size_t shadow_idx);
1349 public:
1350 inline ShadowClosure(ParMarkBitMap* bitmap, ParCompactionManager* cm,
1351 size_t region, size_t shadow);
1352
1353 virtual void complete_region(ParCompactionManager* cm, HeapWord* dest_addr,
1354 PSParallelCompact::RegionData* region_ptr);
1355
1356 virtual void allocate_block();
1357
1358 virtual void set_deferred_obj_addr_for(PSParallelCompact::RegionData* region_ptr);
1359
1360 private:
1361 size_t _shadow;
1362 size_t _offset;
1363 };
1364
1365 inline size_t ShadowClosure::calculate_shadow_offset(size_t region_idx, size_t shadow_idx) {
1366 ParallelCompactData& sd = PSParallelCompact::summary_data();
1367 HeapWord* dest_addr = sd.region_to_addr(region_idx);
1368 HeapWord* shadow_addr = sd.region_to_addr(shadow_idx);
1369 return pointer_delta(shadow_addr, dest_addr);
1370 }
1371
1372 inline
1373 ShadowClosure::ShadowClosure(ParMarkBitMap *bitmap,
1374 ParCompactionManager *cm,
1375 size_t region,
1376 size_t shadow) :
1377 MoveAndUpdateClosure(bitmap, cm, region),
1378 _shadow(shadow),
1379 _offset(calculate_shadow_offset(region, shadow)) {
1380 ParallelCompactData& sd = PSParallelCompact::summary_data();
1381 _destination = sd.region_to_addr(shadow);
1382 }
1383
1384 class UpdateOnlyClosure: public ParMarkBitMapClosure {
1385 private:
1386 const PSParallelCompact::SpaceId _space_id;
1387 ObjectStartArray* const _start_array;
1388
1389 public:
1390 UpdateOnlyClosure(ParMarkBitMap* mbm,
1391 ParCompactionManager* cm,
1392 PSParallelCompact::SpaceId space_id);
1393
1394 // Update the object.
1395 virtual IterationStatus do_addr(HeapWord* addr, size_t words);
1396
1397 inline void do_addr(HeapWord* addr);
1398 };
1399
1400 class FillClosure: public ParMarkBitMapClosure {
1401 public:
|
327 // These are atomic.
328 inline void add_live_obj(size_t words);
329 inline void set_highest_ref(HeapWord* addr);
330 inline void decrement_destination_count();
331 inline bool claim();
332
333 // Possible values of _shadow_state, and transition is as follows
334 // Normal Path:
335 // UNUSED -> try_push() -> FINISHED
336 // Steal Path:
337 // UNUSED -> try_steal() -> SHADOW -> mark_filled() -> FILLED -> try_copy() -> FINISHED
338 static const int UNUSED; // Original state
339 static const int SHADOW; // Stolen by an idle thread, and a shadow region is created for it
340 static const int FILLED; // Its shadow region has been filled and ready to be copied back
341 static const int FINISH; // Work has been done
342
343 // Preempt the region to avoid double processes
344 inline bool try_push();
345 inline bool try_steal();
346 // Mark the region as filled and ready to be copied back
347 inline void mark_filled();
348 // Preempt the region to copy the shadow region content back
349 inline bool try_copy();
350 // Special case: see the comment in PSParallelCompact::fill_shadow_region.
351 // Return to the normal path here
352 inline void mark_normal();
353
354
355 int shadow_state() { return _shadow_state; }
356
357 private:
358 // The type used to represent object sizes within a region.
359 typedef uint region_sz_t;
360
361 // Constants for manipulating the _dc_and_los field, which holds both the
362 // destination count and live obj size. The live obj size lives at the
363 // least significant end so no masking is necessary when adding.
364 static const region_sz_t dc_shift; // Shift amount.
365 static const region_sz_t dc_mask; // Mask for destination count.
366 static const region_sz_t dc_one; // 1, shifted appropriately.
367 static const region_sz_t dc_claimed; // Region has been claimed.
368 static const region_sz_t dc_completed; // Region has been completed.
369 static const region_sz_t los_mask; // Mask for live obj size.
370
371 HeapWord* _destination;
372 size_t _source_region;
373 HeapWord* _partial_obj_addr;
616 tmp = Atomic::cmpxchg(addr, &_highest_ref, tmp);
617 }
618 #endif // #ifdef ASSERT
619 }
620
621 inline bool ParallelCompactData::RegionData::claim()
622 {
623 const region_sz_t los = static_cast<region_sz_t>(live_obj_size());
624 const region_sz_t old = Atomic::cmpxchg(dc_claimed | los, &_dc_and_los, los);
625 return old == los;
626 }
627
628 inline bool ParallelCompactData::RegionData::try_push() {
629 return Atomic::cmpxchg(FINISH, &_shadow_state, UNUSED) == UNUSED;
630 }
631
632 inline bool ParallelCompactData::RegionData::try_steal() {
633 return Atomic::cmpxchg(SHADOW, &_shadow_state, UNUSED) == UNUSED;
634 }
635
636 inline void ParallelCompactData::RegionData::mark_filled() {
637 int old = Atomic::cmpxchg(FILLED, &_shadow_state, SHADOW);
638 assert(old == SHADOW, "Fail to mark the region as filled");
639 }
640
641 inline bool ParallelCompactData::RegionData::try_copy() {
642 return Atomic::cmpxchg(FINISH, &_shadow_state, FILLED) == FILLED;
643 }
644
645 void ParallelCompactData::RegionData::mark_normal() {
646 _shadow_state = FINISH;
647 }
648
649 inline ParallelCompactData::RegionData*
650 ParallelCompactData::region(size_t region_idx) const
651 {
652 assert(region_idx <= region_count(), "bad arg");
653 return _region_data + region_idx;
654 }
655
656 inline size_t
657 ParallelCompactData::region(const RegionData* const region_ptr) const
658 {
659 assert(region_ptr >= _region_data, "bad arg");
660 assert(region_ptr <= _region_data + region_count(), "bad arg");
661 return pointer_delta(region_ptr, _region_data, sizeof(RegionData));
662 }
663
664 inline ParallelCompactData::BlockData*
665 ParallelCompactData::block(size_t n) const {
666 assert(n < block_count(), "bad arg");
667 return _block_data + n;
668 }
1279 SpaceId src_space_id,
1280 HeapWord* src_beg, HeapWord* src_end);
1281 #endif // #ifndef PRODUCT
1282
1283 #ifdef ASSERT
1284 // Sanity check the new location of a word in the heap.
1285 static inline void check_new_location(HeapWord* old_addr, HeapWord* new_addr);
1286 // Verify that all the regions have been emptied.
1287 static void verify_complete(SpaceId space_id);
1288 #endif // #ifdef ASSERT
1289 };
1290
1291 class MoveAndUpdateClosure: public ParMarkBitMapClosure {
1292 static inline size_t calculate_words_remaining(size_t region);
1293 public:
1294 inline MoveAndUpdateClosure(ParMarkBitMap* bitmap, ParCompactionManager* cm,
1295 size_t region);
1296
1297 // Accessors.
1298 HeapWord* destination() const { return _destination; }
1299 HeapWord* copy_destination() const { return _destination + _offset; }
1300
1301 // If the object will fit (size <= words_remaining()), copy it to the current
1302 // destination, update the interior oops and the start array and return either
1303 // full (if the closure is full) or incomplete. If the object will not fit,
1304 // return would_overflow.
1305 IterationStatus do_addr(HeapWord* addr, size_t size);
1306
1307 // Copy enough words to fill this closure, starting at source(). Interior
1308 // oops and the start array are not updated. Return full.
1309 IterationStatus copy_until_full();
1310
1311 // Copy enough words to fill this closure or to the end of an object,
1312 // whichever is smaller, starting at source(). Interior oops and the start
1313 // array are not updated.
1314 void copy_partial_obj();
1315
1316 virtual void complete_region(ParCompactionManager* cm, HeapWord* dest_addr,
1317 PSParallelCompact::RegionData* region_ptr);
1318
1319 protected:
1320 // Update variables to indicate that word_count words were processed.
1321 inline void update_state(size_t word_count);
1322
1323 protected:
1324 HeapWord* _destination; // Next addr to be written.
1325 ObjectStartArray* const _start_array;
1326 size_t _offset;
1327 };
1328
1329 inline size_t MoveAndUpdateClosure::calculate_words_remaining(size_t region) {
1330 HeapWord* dest_addr = PSParallelCompact::summary_data().region_to_addr(region);
1331 PSParallelCompact::SpaceId dest_space_id = PSParallelCompact::space_id(dest_addr);
1332 HeapWord* new_top = PSParallelCompact::new_top(dest_space_id);
1333 assert(dest_addr < new_top, "sanity");
1334
1335 return MIN2(pointer_delta(new_top, dest_addr), ParallelCompactData::RegionSize);
1336 }
1337
1338 inline
1339 MoveAndUpdateClosure::MoveAndUpdateClosure(ParMarkBitMap* bitmap,
1340 ParCompactionManager* cm,
1341 size_t region_idx) :
1342 ParMarkBitMapClosure(bitmap, cm, calculate_words_remaining(region_idx)),
1343 _destination(PSParallelCompact::summary_data().region_to_addr(region_idx)),
1344 _start_array(PSParallelCompact::start_array(PSParallelCompact::space_id(_destination))),
1345 _offset(0) { }
1346
1347
1348 inline void MoveAndUpdateClosure::update_state(size_t words)
1349 {
1350 decrement_words_remaining(words);
1351 _source += words;
1352 _destination += words;
1353 }
1354
1355 class ShadowClosure: public MoveAndUpdateClosure {
1356 inline size_t calculate_shadow_offset(size_t region_idx, size_t shadow_idx);
1357 public:
1358 inline ShadowClosure(ParMarkBitMap* bitmap, ParCompactionManager* cm,
1359 size_t region, size_t shadow);
1360
1361 virtual void complete_region(ParCompactionManager* cm, HeapWord* dest_addr,
1362 PSParallelCompact::RegionData* region_ptr);
1363
1364 private:
1365 size_t _shadow;
1366 };
1367
1368 inline size_t ShadowClosure::calculate_shadow_offset(size_t region_idx, size_t shadow_idx) {
1369 ParallelCompactData& sd = PSParallelCompact::summary_data();
1370 HeapWord* dest_addr = sd.region_to_addr(region_idx);
1371 HeapWord* shadow_addr = sd.region_to_addr(shadow_idx);
1372 return pointer_delta(shadow_addr, dest_addr);
1373 }
1374
1375 inline
1376 ShadowClosure::ShadowClosure(ParMarkBitMap *bitmap,
1377 ParCompactionManager *cm,
1378 size_t region,
1379 size_t shadow) :
1380 MoveAndUpdateClosure(bitmap, cm, region),
1381 _shadow(shadow) {
1382 _offset = calculate_shadow_offset(region, shadow);
1383 }
1384
1385 class UpdateOnlyClosure: public ParMarkBitMapClosure {
1386 private:
1387 const PSParallelCompact::SpaceId _space_id;
1388 ObjectStartArray* const _start_array;
1389
1390 public:
1391 UpdateOnlyClosure(ParMarkBitMap* mbm,
1392 ParCompactionManager* cm,
1393 PSParallelCompact::SpaceId space_id);
1394
1395 // Update the object.
1396 virtual IterationStatus do_addr(HeapWord* addr, size_t words);
1397
1398 inline void do_addr(HeapWord* addr);
1399 };
1400
1401 class FillClosure: public ParMarkBitMapClosure {
1402 public:
|