428 st->cr();
429 }
430 #endif // #ifndef PRODUCT
431
432 CMSCollector::CollectorState CMSCollector::_collectorState =
433 CMSCollector::Idling;
434 bool CMSCollector::_foregroundGCIsActive = false;
435 bool CMSCollector::_foregroundGCShouldWait = false;
436
437 CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
438 CardTableRS* ct,
439 ConcurrentMarkSweepPolicy* cp):
440 _cmsGen(cmsGen),
441 _ct(ct),
442 _ref_processor(NULL), // will be set later
443 _conc_workers(NULL), // may be set later
444 _abort_preclean(false),
445 _start_sampling(false),
446 _between_prologue_and_epilogue(false),
447 _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
448 _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize),
449 -1 /* lock-free */, "No_lock" /* dummy */),
450 _modUnionClosurePar(&_modUnionTable),
451 // Adjust my span to cover old (cms) gen
452 _span(cmsGen->reserved()),
453 // Construct the is_alive_closure with _span & markBitMap
454 _is_alive_closure(_span, &_markBitMap),
455 _restart_addr(NULL),
456 _overflow_list(NULL),
457 _stats(cmsGen),
458 _eden_chunk_lock(new Mutex(Mutex::leaf + 1, "CMS_eden_chunk_lock", true,
459 //verify that this lock should be acquired with safepoint check.
460 Monitor::_safepoint_check_sometimes)),
461 _eden_chunk_array(NULL), // may be set in ctor body
462 _eden_chunk_capacity(0), // -- ditto --
463 _eden_chunk_index(0), // -- ditto --
464 _survivor_plab_array(NULL), // -- ditto --
465 _survivor_chunk_array(NULL), // -- ditto --
466 _survivor_chunk_capacity(0), // -- ditto --
467 _survivor_chunk_index(0), // -- ditto --
468 _ser_pmc_preclean_ovflw(0),
880 // is in flux from being free to being allocated (and in
881 // transition while being copied into) and subsequently
882 // becoming a bona-fide object when the copy/promotion is complete.
883 assert(SafepointSynchronize::is_at_safepoint(),
884 "expect promotion only at safepoints");
885
886 if (_collectorState < Sweeping) {
887 // Mark the appropriate cards in the modUnionTable, so that
888 // this object gets scanned before the sweep. If this is
889 // not done, CMS generation references in the object might
890 // not get marked.
891 // For the case of arrays, which are otherwise precisely
892 // marked, we need to dirty the entire array, not just its head.
893 if (is_obj_array) {
894 // The [par_]mark_range() method expects mr.end() below to
895 // be aligned to the granularity of a bit's representation
896 // in the heap. In the case of the MUT below, that's a
897 // card size.
898 MemRegion mr(start,
899 (HeapWord*)round_to((intptr_t)(start + obj_size),
900 CardTableModRefBS::card_size /* bytes */));
901 if (par) {
902 _modUnionTable.par_mark_range(mr);
903 } else {
904 _modUnionTable.mark_range(mr);
905 }
906 } else { // not an obj array; we can just mark the head
907 if (par) {
908 _modUnionTable.par_mark(start);
909 } else {
910 _modUnionTable.mark(start);
911 }
912 }
913 }
914 }
915 }
916
917 oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) {
918 assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");
919 // allocate, copy and if necessary update promoinfo --
920 // delegate to underlying space.
3204 assert(cur != NULL, "Counted wrong?");
3205 work_q->push(cur);
3206 }
3207 return num > 0;
3208 }
3209
3210 void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
3211 SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
3212 int n_tasks = pst->n_tasks();
3213 // We allow that there may be no tasks to do here because
3214 // we are restarting after a stack overflow.
3215 assert(pst->valid() || n_tasks == 0, "Uninitialized use?");
3216 uint nth_task = 0;
3217
3218 HeapWord* aligned_start = sp->bottom();
3219 if (sp->used_region().contains(_restart_addr)) {
3220 // Align down to a card boundary for the start of 0th task
3221 // for this space.
3222 aligned_start =
3223 (HeapWord*)align_size_down((uintptr_t)_restart_addr,
3224 CardTableModRefBS::card_size);
3225 }
3226
3227 size_t chunk_size = sp->marking_task_size();
3228 while (!pst->is_task_claimed(/* reference */ nth_task)) {
3229 // Having claimed the nth task in this space,
3230 // compute the chunk that it corresponds to:
3231 MemRegion span = MemRegion(aligned_start + nth_task*chunk_size,
3232 aligned_start + (nth_task+1)*chunk_size);
3233 // Try and bump the global finger via a CAS;
3234 // note that we need to do the global finger bump
3235 // _before_ taking the intersection below, because
3236 // the task corresponding to that region will be
3237 // deemed done even if the used_region() expands
3238 // because of allocation -- as it almost certainly will
3239 // during start-up while the threads yield in the
3240 // closure below.
3241 HeapWord* finger = span.end();
3242 bump_global_finger(finger); // atomically
3243 // There are null tasks here corresponding to chunks
3244 // beyond the "top" address of the space.
4007 HeapWord *lastAddr, *nextAddr;
4008
4009 for (cumNumDirtyCards = numDirtyCards = 0,
4010 nextAddr = lastAddr = startAddr;
4011 nextAddr < endAddr;
4012 nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
4013
4014 ResourceMark rm;
4015 HandleMark hm;
4016
4017 MemRegion dirtyRegion;
4018 {
4019 // See comments in "Precleaning notes" above on why we
4020 // do this locking. XXX Could the locking overheads be
4021 // too high when dirty cards are sparse? [I don't think so.]
4022 stopTimer();
4023 CMSTokenSync x(true); // is cms thread
4024 startTimer();
4025 sample_eden();
4026 // Get and clear dirty region from card table
4027 dirtyRegion = _ct->ct_bs()->dirty_card_range_after_reset(
4028 MemRegion(nextAddr, endAddr),
4029 true,
4030 CardTableModRefBS::precleaned_card_val());
4031
4032 assert(dirtyRegion.start() >= nextAddr,
4033 "returned region inconsistent?");
4034 }
4035 lastAddr = dirtyRegion.end();
4036 numDirtyCards =
4037 dirtyRegion.word_size()/CardTableModRefBS::card_size_in_words;
4038
4039 if (!dirtyRegion.is_empty()) {
4040 stopTimer();
4041 CMSTokenSyncWithLocks ts(true, old_gen->freelistLock(), bitMapLock());
4042 startTimer();
4043 sample_eden();
4044 verify_work_stacks_empty();
4045 verify_overflow_empty();
4046 HeapWord* stop_point =
4047 old_gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
4048 if (stop_point != NULL) {
4049 assert((_collectorState == AbortablePreclean && should_abort_preclean()),
4050 "Should only be AbortablePreclean.");
4051 _ct->ct_bs()->invalidate(MemRegion(stop_point, dirtyRegion.end()));
4052 if (should_abort_preclean()) {
4053 break; // out of preclean loop
4054 } else {
4055 // Compute the next address at which preclean should pick up.
4056 lastAddr = next_card_start_after_block(stop_point);
4057 }
4058 }
4059 } else {
4060 break;
4061 }
4062 }
4063 verify_work_stacks_empty();
4064 verify_overflow_empty();
4065 return cumNumDirtyCards;
4066 }
4067
4068 class PrecleanKlassClosure : public KlassClosure {
4069 KlassToOopClosure _cm_klass_closure;
4070 public:
4071 PrecleanKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {}
4558 // work-partitioning form than the current "sequential tasks"
4559 // paradigm, the use of that persistent state will have to be
4560 // revisited and modified appropriately. See also related
4561 // bug 4756801 work on which should examine this code to make
4562 // sure that the changes there do not run counter to the
4563 // assumptions made here and necessary for correctness and
4564 // efficiency. Note also that this code might yield inefficient
4565 // behavior in the case of very large objects that span one or
4566 // more work chunks. Such objects would potentially be scanned
4567 // several times redundantly. Work on 4756801 should try and
4568 // address that performance anomaly if at all possible. XXX
4569 MemRegion full_span = _collector->_span;
4570 CMSBitMap* bm = &(_collector->_markBitMap); // shared
4571 MarkFromDirtyCardsClosure
4572 greyRescanClosure(_collector, full_span, // entire span of interest
4573 sp, bm, work_q, cl);
4574
4575 SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
4576 assert(pst->valid(), "Uninitialized use?");
4577 uint nth_task = 0;
4578 const int alignment = CardTableModRefBS::card_size * BitsPerWord;
4579 MemRegion span = sp->used_region();
4580 HeapWord* start_addr = span.start();
4581 HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(),
4582 alignment);
4583 const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units
4584 assert((HeapWord*)round_to((intptr_t)start_addr, alignment) ==
4585 start_addr, "Check alignment");
4586 assert((size_t)round_to((intptr_t)chunk_size, alignment) ==
4587 chunk_size, "Check alignment");
4588
4589 while (!pst->is_task_claimed(/* reference */ nth_task)) {
4590 // Having claimed the nth_task, compute corresponding mem-region,
4591 // which is a-fortiori aligned correctly (i.e. at a MUT boundary).
4592 // The alignment restriction ensures that we do not need any
4593 // synchronization with other gang-workers while setting or
4594 // clearing bits in thus chunk of the MUT.
4595 MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size,
4596 start_addr + (nth_task+1)*chunk_size);
4597 // The last chunk's end might be way beyond end of the
4598 // used region. In that case pull back appropriately.
4599 if (this_span.end() > end_addr) {
4600 this_span.set_end(end_addr);
4601 assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)");
4602 }
4603 // Iterate over the dirty cards covering this chunk, marking them
4604 // precleaned, and setting the corresponding bits in the mod union
4605 // table. Since we have been careful to partition at Card and MUT-word
4606 // boundaries no synchronization is needed between parallel threads.
4607 _collector->_ct->ct_bs()->dirty_card_iterate(this_span,
4608 &modUnionClosure);
4609
4610 // Having transferred these marks into the modUnionTable,
4611 // rescan the marked objects on the dirty cards in the modUnionTable.
4612 // Even if this is at a synchronous collection, the initial marking
4613 // may have been done during an asynchronous collection so there
4614 // may be dirty bits in the mod-union table.
4615 _collector->_modUnionTable.dirty_range_iterate_clear(
4616 this_span, &greyRescanClosure);
4617 _collector->_modUnionTable.verifyNoOneBitsInRange(
4618 this_span.start(),
4619 this_span.end());
4620 }
4621 pst->all_tasks_completed(); // declare that i am done
4622 }
4623
4624 // . see if we can share work_queues with ParNew? XXX
4625 void
4626 CMSParRemarkTask::do_work_steal(int i, ParMarkRefsIntoAndScanClosure* cl,
4627 int* seed) {
4898 void CMSCollector::do_remark_non_parallel() {
4899 ResourceMark rm;
4900 HandleMark hm;
4901 GenCollectedHeap* gch = GenCollectedHeap::heap();
4902 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
4903
4904 MarkRefsIntoAndScanClosure
4905 mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */,
4906 &_markStack, this,
4907 false /* should_yield */, false /* not precleaning */);
4908 MarkFromDirtyCardsClosure
4909 markFromDirtyCardsClosure(this, _span,
4910 NULL, // space is set further below
4911 &_markBitMap, &_markStack, &mrias_cl);
4912 {
4913 GCTraceTime(Trace, gc, phases) t("Grey Object Rescan", _gc_timer_cm);
4914 // Iterate over the dirty cards, setting the corresponding bits in the
4915 // mod union table.
4916 {
4917 ModUnionClosure modUnionClosure(&_modUnionTable);
4918 _ct->ct_bs()->dirty_card_iterate(
4919 _cmsGen->used_region(),
4920 &modUnionClosure);
4921 }
4922 // Having transferred these marks into the modUnionTable, we just need
4923 // to rescan the marked objects on the dirty cards in the modUnionTable.
4924 // The initial marking may have been done during an asynchronous
4925 // collection so there may be dirty bits in the mod-union table.
4926 const int alignment =
4927 CardTableModRefBS::card_size * BitsPerWord;
4928 {
4929 // ... First handle dirty cards in CMS gen
4930 markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
4931 MemRegion ur = _cmsGen->used_region();
4932 HeapWord* lb = ur.start();
4933 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
4934 MemRegion cms_span(lb, ub);
4935 _modUnionTable.dirty_range_iterate_clear(cms_span,
4936 &markFromDirtyCardsClosure);
4937 verify_work_stacks_empty();
4938 log_trace(gc)(" (re-scanned " SIZE_FORMAT " dirty cards in cms gen) ", markFromDirtyCardsClosure.num_dirty_cards());
4939 }
4940 }
4941 if (VerifyDuringGC &&
4942 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4943 HandleMark hm; // Discard invalid handles created during verification
4944 Universe::verify();
4945 }
4946 {
4947 GCTraceTime(Trace, gc, phases) t("Root Rescan", _gc_timer_cm);
5612 size_t size = pointer_delta(nextOneAddr + 1, addr);
5613 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
5614 "alignment problem");
5615 assert(size >= 3, "Necessary for Printezis marks to work");
5616 return size;
5617 }
5618 return 0;
5619 }
5620
5621 HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
5622 size_t sz = 0;
5623 oop p = (oop)addr;
5624 if (p->klass_or_null_acquire() != NULL) {
5625 sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
5626 } else {
5627 sz = block_size_using_printezis_bits(addr);
5628 }
5629 assert(sz > 0, "size must be nonzero");
5630 HeapWord* next_block = addr + sz;
5631 HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block,
5632 CardTableModRefBS::card_size);
5633 assert(round_down((uintptr_t)addr, CardTableModRefBS::card_size) <
5634 round_down((uintptr_t)next_card, CardTableModRefBS::card_size),
5635 "must be different cards");
5636 return next_card;
5637 }
5638
5639
5640 // CMS Bit Map Wrapper /////////////////////////////////////////
5641
5642 // Construct a CMS bit map infrastructure, but don't create the
5643 // bit vector itself. That is done by a separate call CMSBitMap::allocate()
5644 // further below.
5645 CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
5646 _bm(),
5647 _shifter(shifter),
5648 _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true,
5649 Monitor::_safepoint_check_sometimes) : NULL)
5650 {
5651 _bmStartWord = 0;
5652 _bmWordSize = 0;
5653 }
5654
6274 _span(span),
6275 _bitMap(bitMap),
6276 _mut(&collector->_modUnionTable),
6277 _markStack(markStack),
6278 _yield(should_yield),
6279 _skipBits(0)
6280 {
6281 assert(_markStack->isEmpty(), "stack should be empty");
6282 _finger = _bitMap->startWord();
6283 _threshold = _finger;
6284 assert(_collector->_restart_addr == NULL, "Sanity check");
6285 assert(_span.contains(_finger), "Out of bounds _finger?");
6286 DEBUG_ONLY(_verifying = verifying;)
6287 }
6288
6289 void MarkFromRootsClosure::reset(HeapWord* addr) {
6290 assert(_markStack->isEmpty(), "would cause duplicates on stack");
6291 assert(_span.contains(addr), "Out of bounds _finger?");
6292 _finger = addr;
6293 _threshold = (HeapWord*)round_to(
6294 (intptr_t)_finger, CardTableModRefBS::card_size);
6295 }
6296
6297 // Should revisit to see if this should be restructured for
6298 // greater efficiency.
6299 bool MarkFromRootsClosure::do_bit(size_t offset) {
6300 if (_skipBits > 0) {
6301 _skipBits--;
6302 return true;
6303 }
6304 // convert offset into a HeapWord*
6305 HeapWord* addr = _bitMap->startWord() + offset;
6306 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
6307 "address out of range");
6308 assert(_bitMap->isMarked(addr), "tautology");
6309 if (_bitMap->isMarked(addr+1)) {
6310 // this is an allocated but not yet initialized object
6311 assert(_skipBits == 0, "tautology");
6312 _skipBits = 2; // skip next two marked bits ("Printezis-marks")
6313 oop p = oop(addr);
6314 if (p->klass_or_null_acquire() == NULL) {
6315 DEBUG_ONLY(if (!_verifying) {)
6316 // We re-dirty the cards on which this object lies and increase
6317 // the _threshold so that we'll come back to scan this object
6318 // during the preclean or remark phase. (CMSCleanOnEnter)
6319 if (CMSCleanOnEnter) {
6320 size_t sz = _collector->block_size_using_printezis_bits(addr);
6321 HeapWord* end_card_addr = (HeapWord*)round_to(
6322 (intptr_t)(addr+sz), CardTableModRefBS::card_size);
6323 MemRegion redirty_range = MemRegion(addr, end_card_addr);
6324 assert(!redirty_range.is_empty(), "Arithmetical tautology");
6325 // Bump _threshold to end_card_addr; note that
6326 // _threshold cannot possibly exceed end_card_addr, anyhow.
6327 // This prevents future clearing of the card as the scan proceeds
6328 // to the right.
6329 assert(_threshold <= end_card_addr,
6330 "Because we are just scanning into this object");
6331 if (_threshold < end_card_addr) {
6332 _threshold = end_card_addr;
6333 }
6334 if (p->klass_or_null_acquire() != NULL) {
6335 // Redirty the range of cards...
6336 _mut->mark_range(redirty_range);
6337 } // ...else the setting of klass will dirty the card anyway.
6338 }
6339 DEBUG_ONLY(})
6340 return true;
6341 }
6342 }
6390 // accumulated in the card table and the mod union table --
6391 // these mutation records are redundant until we have
6392 // actually traced into the corresponding card.
6393 // Here, we check whether advancing the finger would make
6394 // us cross into a new card, and if so clear corresponding
6395 // cards in the MUT (preclean them in the card-table in the
6396 // future).
6397
6398 DEBUG_ONLY(if (!_verifying) {)
6399 // The clean-on-enter optimization is disabled by default,
6400 // until we fix 6178663.
6401 if (CMSCleanOnEnter && (_finger > _threshold)) {
6402 // [_threshold, _finger) represents the interval
6403 // of cards to be cleared in MUT (or precleaned in card table).
6404 // The set of cards to be cleared is all those that overlap
6405 // with the interval [_threshold, _finger); note that
6406 // _threshold is always kept card-aligned but _finger isn't
6407 // always card-aligned.
6408 HeapWord* old_threshold = _threshold;
6409 assert(old_threshold == (HeapWord*)round_to(
6410 (intptr_t)old_threshold, CardTableModRefBS::card_size),
6411 "_threshold should always be card-aligned");
6412 _threshold = (HeapWord*)round_to(
6413 (intptr_t)_finger, CardTableModRefBS::card_size);
6414 MemRegion mr(old_threshold, _threshold);
6415 assert(!mr.is_empty(), "Control point invariant");
6416 assert(_span.contains(mr), "Should clear within span");
6417 _mut->clear_range(mr);
6418 }
6419 DEBUG_ONLY(})
6420 // Note: the finger doesn't advance while we drain
6421 // the stack below.
6422 PushOrMarkClosure pushOrMarkClosure(_collector,
6423 _span, _bitMap, _markStack,
6424 _finger, this);
6425 bool res = _markStack->push(obj);
6426 assert(res, "Empty non-zero size stack should have space for single push");
6427 while (!_markStack->isEmpty()) {
6428 oop new_oop = _markStack->pop();
6429 // Skip verifying header mark word below because we are
6430 // running concurrent with mutators.
6431 assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
6432 // now scan this oop's oops
6433 new_oop->oop_iterate(&pushOrMarkClosure);
6504 // this time it's possible that a lot of mutations have
6505 // accumulated in the card table and the mod union table --
6506 // these mutation records are redundant until we have
6507 // actually traced into the corresponding card.
6508 // Here, we check whether advancing the finger would make
6509 // us cross into a new card, and if so clear corresponding
6510 // cards in the MUT (preclean them in the card-table in the
6511 // future).
6512
6513 // The clean-on-enter optimization is disabled by default,
6514 // until we fix 6178663.
6515 if (CMSCleanOnEnter && (_finger > _threshold)) {
6516 // [_threshold, _finger) represents the interval
6517 // of cards to be cleared in MUT (or precleaned in card table).
6518 // The set of cards to be cleared is all those that overlap
6519 // with the interval [_threshold, _finger); note that
6520 // _threshold is always kept card-aligned but _finger isn't
6521 // always card-aligned.
6522 HeapWord* old_threshold = _threshold;
6523 assert(old_threshold == (HeapWord*)round_to(
6524 (intptr_t)old_threshold, CardTableModRefBS::card_size),
6525 "_threshold should always be card-aligned");
6526 _threshold = (HeapWord*)round_to(
6527 (intptr_t)_finger, CardTableModRefBS::card_size);
6528 MemRegion mr(old_threshold, _threshold);
6529 assert(!mr.is_empty(), "Control point invariant");
6530 assert(_span.contains(mr), "Should clear within span"); // _whole_span ??
6531 _mut->clear_range(mr);
6532 }
6533
6534 // Note: the local finger doesn't advance while we drain
6535 // the stack below, but the global finger sure can and will.
6536 HeapWord* volatile* gfa = _task->global_finger_addr();
6537 ParPushOrMarkClosure pushOrMarkClosure(_collector,
6538 _span, _bit_map,
6539 _work_queue,
6540 _overflow_stack,
6541 _finger,
6542 gfa, this);
6543 bool res = _work_queue->push(obj); // overflow could occur here
6544 assert(res, "Will hold once we use workqueues");
6545 while (true) {
6546 oop new_oop;
6547 if (!_work_queue->pop_local(new_oop)) {
6875 if (CMSMarkStackOverflowALot &&
6876 _collector->simulate_overflow()) {
6877 // simulate a stack overflow
6878 simulate_overflow = true;
6879 }
6880 )
6881 if (simulate_overflow || !_mark_stack->push(obj)) {
6882 if (_concurrent_precleaning) {
6883 // During precleaning we can just dirty the appropriate card(s)
6884 // in the mod union table, thus ensuring that the object remains
6885 // in the grey set and continue. In the case of object arrays
6886 // we need to dirty all of the cards that the object spans,
6887 // since the rescan of object arrays will be limited to the
6888 // dirty cards.
6889 // Note that no one can be interfering with us in this action
6890 // of dirtying the mod union table, so no locking or atomics
6891 // are required.
6892 if (obj->is_objArray()) {
6893 size_t sz = obj->size();
6894 HeapWord* end_card_addr = (HeapWord*)round_to(
6895 (intptr_t)(addr+sz), CardTableModRefBS::card_size);
6896 MemRegion redirty_range = MemRegion(addr, end_card_addr);
6897 assert(!redirty_range.is_empty(), "Arithmetical tautology");
6898 _mod_union_table->mark_range(redirty_range);
6899 } else {
6900 _mod_union_table->mark(addr);
6901 }
6902 _collector->_ser_pmc_preclean_ovflw++;
6903 } else {
6904 // During the remark phase, we need to remember this oop
6905 // in the overflow list.
6906 _collector->push_on_overflow_list(obj);
6907 _collector->_ser_pmc_remark_ovflw++;
6908 }
6909 }
6910 }
6911 }
6912
6913 ParPushAndMarkClosure::ParPushAndMarkClosure(CMSCollector* collector,
6914 MemRegion span,
6915 ReferenceProcessor* rp,
6988 for (unsigned i = 0; i < CMSYieldSleepCount &&
6989 ConcurrentMarkSweepThread::should_yield() &&
6990 !CMSCollector::foregroundGCIsActive(); ++i) {
6991 os::sleep(Thread::current(), 1, false);
6992 }
6993
6994 ConcurrentMarkSweepThread::synchronize(true);
6995 bml->lock();
6996
6997 _collector->startTimer();
6998 }
6999
7000 bool CMSPrecleanRefsYieldClosure::should_return() {
7001 if (ConcurrentMarkSweepThread::should_yield()) {
7002 do_yield_work();
7003 }
7004 return _collector->foregroundGCIsActive();
7005 }
7006
7007 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
7008 assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
7009 "mr should be aligned to start at a card boundary");
7010 // We'd like to assert:
7011 // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
7012 // "mr should be a range of cards");
7013 // However, that would be too strong in one case -- the last
7014 // partition ends at _unallocated_block which, in general, can be
7015 // an arbitrary boundary, not necessarily card aligned.
7016 _num_dirty_cards += mr.word_size()/CardTableModRefBS::card_size_in_words;
7017 _space->object_iterate_mem(mr, &_scan_cl);
7018 }
7019
7020 SweepClosure::SweepClosure(CMSCollector* collector,
7021 ConcurrentMarkSweepGeneration* g,
7022 CMSBitMap* bitMap, bool should_yield) :
7023 _collector(collector),
7024 _g(g),
7025 _sp(g->cmsSpace()),
7026 _limit(_sp->sweep_limit()),
7027 _freelistLock(_sp->freelistLock()),
7028 _bitMap(bitMap),
7029 _yield(should_yield),
7030 _inFreeRange(false), // No free range at beginning of sweep
7031 _freeRangeInFreeLists(false), // No free range at beginning of sweep
7032 _lastFreeRangeCoalesced(false),
7033 _freeFinger(g->used_region().start())
7034 {
7035 NOT_PRODUCT(
7036 _numObjectsFreed = 0;
7605 bool simulate_overflow = false;
7606 NOT_PRODUCT(
7607 if (CMSMarkStackOverflowALot &&
7608 _collector->simulate_overflow()) {
7609 // simulate a stack overflow
7610 simulate_overflow = true;
7611 }
7612 )
7613 if (simulate_overflow || !_mark_stack->push(obj)) {
7614 if (_concurrent_precleaning) {
7615 // We dirty the overflown object and let the remark
7616 // phase deal with it.
7617 assert(_collector->overflow_list_is_empty(), "Error");
7618 // In the case of object arrays, we need to dirty all of
7619 // the cards that the object spans. No locking or atomics
7620 // are needed since no one else can be mutating the mod union
7621 // table.
7622 if (obj->is_objArray()) {
7623 size_t sz = obj->size();
7624 HeapWord* end_card_addr =
7625 (HeapWord*)round_to((intptr_t)(addr+sz), CardTableModRefBS::card_size);
7626 MemRegion redirty_range = MemRegion(addr, end_card_addr);
7627 assert(!redirty_range.is_empty(), "Arithmetical tautology");
7628 _collector->_modUnionTable.mark_range(redirty_range);
7629 } else {
7630 _collector->_modUnionTable.mark(addr);
7631 }
7632 _collector->_ser_kac_preclean_ovflw++;
7633 } else {
7634 _collector->push_on_overflow_list(obj);
7635 _collector->_ser_kac_ovflw++;
7636 }
7637 }
7638 }
7639 }
7640
7641 void CMSKeepAliveClosure::do_oop(oop* p) { CMSKeepAliveClosure::do_oop_work(p); }
7642 void CMSKeepAliveClosure::do_oop(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); }
7643
7644 // CMSParKeepAliveClosure: a parallel version of the above.
7645 // The work queues are private to each closure (thread),
|
428 st->cr();
429 }
430 #endif // #ifndef PRODUCT
431
432 CMSCollector::CollectorState CMSCollector::_collectorState =
433 CMSCollector::Idling;
434 bool CMSCollector::_foregroundGCIsActive = false;
435 bool CMSCollector::_foregroundGCShouldWait = false;
436
437 CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
438 CardTableRS* ct,
439 ConcurrentMarkSweepPolicy* cp):
440 _cmsGen(cmsGen),
441 _ct(ct),
442 _ref_processor(NULL), // will be set later
443 _conc_workers(NULL), // may be set later
444 _abort_preclean(false),
445 _start_sampling(false),
446 _between_prologue_and_epilogue(false),
447 _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
448 _modUnionTable((CardTable::card_shift - LogHeapWordSize),
449 -1 /* lock-free */, "No_lock" /* dummy */),
450 _modUnionClosurePar(&_modUnionTable),
451 // Adjust my span to cover old (cms) gen
452 _span(cmsGen->reserved()),
453 // Construct the is_alive_closure with _span & markBitMap
454 _is_alive_closure(_span, &_markBitMap),
455 _restart_addr(NULL),
456 _overflow_list(NULL),
457 _stats(cmsGen),
458 _eden_chunk_lock(new Mutex(Mutex::leaf + 1, "CMS_eden_chunk_lock", true,
459 //verify that this lock should be acquired with safepoint check.
460 Monitor::_safepoint_check_sometimes)),
461 _eden_chunk_array(NULL), // may be set in ctor body
462 _eden_chunk_capacity(0), // -- ditto --
463 _eden_chunk_index(0), // -- ditto --
464 _survivor_plab_array(NULL), // -- ditto --
465 _survivor_chunk_array(NULL), // -- ditto --
466 _survivor_chunk_capacity(0), // -- ditto --
467 _survivor_chunk_index(0), // -- ditto --
468 _ser_pmc_preclean_ovflw(0),
880 // is in flux from being free to being allocated (and in
881 // transition while being copied into) and subsequently
882 // becoming a bona-fide object when the copy/promotion is complete.
883 assert(SafepointSynchronize::is_at_safepoint(),
884 "expect promotion only at safepoints");
885
886 if (_collectorState < Sweeping) {
887 // Mark the appropriate cards in the modUnionTable, so that
888 // this object gets scanned before the sweep. If this is
889 // not done, CMS generation references in the object might
890 // not get marked.
891 // For the case of arrays, which are otherwise precisely
892 // marked, we need to dirty the entire array, not just its head.
893 if (is_obj_array) {
894 // The [par_]mark_range() method expects mr.end() below to
895 // be aligned to the granularity of a bit's representation
896 // in the heap. In the case of the MUT below, that's a
897 // card size.
898 MemRegion mr(start,
899 (HeapWord*)round_to((intptr_t)(start + obj_size),
900 CardTable::card_size /* bytes */));
901 if (par) {
902 _modUnionTable.par_mark_range(mr);
903 } else {
904 _modUnionTable.mark_range(mr);
905 }
906 } else { // not an obj array; we can just mark the head
907 if (par) {
908 _modUnionTable.par_mark(start);
909 } else {
910 _modUnionTable.mark(start);
911 }
912 }
913 }
914 }
915 }
916
917 oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) {
918 assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");
919 // allocate, copy and if necessary update promoinfo --
920 // delegate to underlying space.
3204 assert(cur != NULL, "Counted wrong?");
3205 work_q->push(cur);
3206 }
3207 return num > 0;
3208 }
3209
3210 void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
3211 SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
3212 int n_tasks = pst->n_tasks();
3213 // We allow that there may be no tasks to do here because
3214 // we are restarting after a stack overflow.
3215 assert(pst->valid() || n_tasks == 0, "Uninitialized use?");
3216 uint nth_task = 0;
3217
3218 HeapWord* aligned_start = sp->bottom();
3219 if (sp->used_region().contains(_restart_addr)) {
3220 // Align down to a card boundary for the start of 0th task
3221 // for this space.
3222 aligned_start =
3223 (HeapWord*)align_size_down((uintptr_t)_restart_addr,
3224 CardTable::card_size);
3225 }
3226
3227 size_t chunk_size = sp->marking_task_size();
3228 while (!pst->is_task_claimed(/* reference */ nth_task)) {
3229 // Having claimed the nth task in this space,
3230 // compute the chunk that it corresponds to:
3231 MemRegion span = MemRegion(aligned_start + nth_task*chunk_size,
3232 aligned_start + (nth_task+1)*chunk_size);
3233 // Try and bump the global finger via a CAS;
3234 // note that we need to do the global finger bump
3235 // _before_ taking the intersection below, because
3236 // the task corresponding to that region will be
3237 // deemed done even if the used_region() expands
3238 // because of allocation -- as it almost certainly will
3239 // during start-up while the threads yield in the
3240 // closure below.
3241 HeapWord* finger = span.end();
3242 bump_global_finger(finger); // atomically
3243 // There are null tasks here corresponding to chunks
3244 // beyond the "top" address of the space.
4007 HeapWord *lastAddr, *nextAddr;
4008
4009 for (cumNumDirtyCards = numDirtyCards = 0,
4010 nextAddr = lastAddr = startAddr;
4011 nextAddr < endAddr;
4012 nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
4013
4014 ResourceMark rm;
4015 HandleMark hm;
4016
4017 MemRegion dirtyRegion;
4018 {
4019 // See comments in "Precleaning notes" above on why we
4020 // do this locking. XXX Could the locking overheads be
4021 // too high when dirty cards are sparse? [I don't think so.]
4022 stopTimer();
4023 CMSTokenSync x(true); // is cms thread
4024 startTimer();
4025 sample_eden();
4026 // Get and clear dirty region from card table
4027 dirtyRegion = _ct->dirty_card_range_after_reset(MemRegion(nextAddr, endAddr),
4028 true,
4029 CardTable::precleaned_card_val());
4030
4031 assert(dirtyRegion.start() >= nextAddr,
4032 "returned region inconsistent?");
4033 }
4034 lastAddr = dirtyRegion.end();
4035 numDirtyCards =
4036 dirtyRegion.word_size()/CardTable::card_size_in_words;
4037
4038 if (!dirtyRegion.is_empty()) {
4039 stopTimer();
4040 CMSTokenSyncWithLocks ts(true, old_gen->freelistLock(), bitMapLock());
4041 startTimer();
4042 sample_eden();
4043 verify_work_stacks_empty();
4044 verify_overflow_empty();
4045 HeapWord* stop_point =
4046 old_gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
4047 if (stop_point != NULL) {
4048 assert((_collectorState == AbortablePreclean && should_abort_preclean()),
4049 "Should only be AbortablePreclean.");
4050 _ct->invalidate(MemRegion(stop_point, dirtyRegion.end()));
4051 if (should_abort_preclean()) {
4052 break; // out of preclean loop
4053 } else {
4054 // Compute the next address at which preclean should pick up.
4055 lastAddr = next_card_start_after_block(stop_point);
4056 }
4057 }
4058 } else {
4059 break;
4060 }
4061 }
4062 verify_work_stacks_empty();
4063 verify_overflow_empty();
4064 return cumNumDirtyCards;
4065 }
4066
4067 class PrecleanKlassClosure : public KlassClosure {
4068 KlassToOopClosure _cm_klass_closure;
4069 public:
4070 PrecleanKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {}
4557 // work-partitioning form than the current "sequential tasks"
4558 // paradigm, the use of that persistent state will have to be
4559 // revisited and modified appropriately. See also related
4560 // bug 4756801 work on which should examine this code to make
4561 // sure that the changes there do not run counter to the
4562 // assumptions made here and necessary for correctness and
4563 // efficiency. Note also that this code might yield inefficient
4564 // behavior in the case of very large objects that span one or
4565 // more work chunks. Such objects would potentially be scanned
4566 // several times redundantly. Work on 4756801 should try and
4567 // address that performance anomaly if at all possible. XXX
4568 MemRegion full_span = _collector->_span;
4569 CMSBitMap* bm = &(_collector->_markBitMap); // shared
4570 MarkFromDirtyCardsClosure
4571 greyRescanClosure(_collector, full_span, // entire span of interest
4572 sp, bm, work_q, cl);
4573
4574 SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
4575 assert(pst->valid(), "Uninitialized use?");
4576 uint nth_task = 0;
4577 const int alignment = CardTable::card_size * BitsPerWord;
4578 MemRegion span = sp->used_region();
4579 HeapWord* start_addr = span.start();
4580 HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(),
4581 alignment);
4582 const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units
4583 assert((HeapWord*)round_to((intptr_t)start_addr, alignment) ==
4584 start_addr, "Check alignment");
4585 assert((size_t)round_to((intptr_t)chunk_size, alignment) ==
4586 chunk_size, "Check alignment");
4587
4588 while (!pst->is_task_claimed(/* reference */ nth_task)) {
4589 // Having claimed the nth_task, compute corresponding mem-region,
4590 // which is a-fortiori aligned correctly (i.e. at a MUT boundary).
4591 // The alignment restriction ensures that we do not need any
4592 // synchronization with other gang-workers while setting or
4593 // clearing bits in thus chunk of the MUT.
4594 MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size,
4595 start_addr + (nth_task+1)*chunk_size);
4596 // The last chunk's end might be way beyond end of the
4597 // used region. In that case pull back appropriately.
4598 if (this_span.end() > end_addr) {
4599 this_span.set_end(end_addr);
4600 assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)");
4601 }
4602 // Iterate over the dirty cards covering this chunk, marking them
4603 // precleaned, and setting the corresponding bits in the mod union
4604 // table. Since we have been careful to partition at Card and MUT-word
4605 // boundaries no synchronization is needed between parallel threads.
4606 _collector->_ct->dirty_card_iterate(this_span,
4607 &modUnionClosure);
4608
4609 // Having transferred these marks into the modUnionTable,
4610 // rescan the marked objects on the dirty cards in the modUnionTable.
4611 // Even if this is at a synchronous collection, the initial marking
4612 // may have been done during an asynchronous collection so there
4613 // may be dirty bits in the mod-union table.
4614 _collector->_modUnionTable.dirty_range_iterate_clear(
4615 this_span, &greyRescanClosure);
4616 _collector->_modUnionTable.verifyNoOneBitsInRange(
4617 this_span.start(),
4618 this_span.end());
4619 }
4620 pst->all_tasks_completed(); // declare that i am done
4621 }
4622
4623 // . see if we can share work_queues with ParNew? XXX
4624 void
4625 CMSParRemarkTask::do_work_steal(int i, ParMarkRefsIntoAndScanClosure* cl,
4626 int* seed) {
4897 void CMSCollector::do_remark_non_parallel() {
4898 ResourceMark rm;
4899 HandleMark hm;
4900 GenCollectedHeap* gch = GenCollectedHeap::heap();
4901 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
4902
4903 MarkRefsIntoAndScanClosure
4904 mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */,
4905 &_markStack, this,
4906 false /* should_yield */, false /* not precleaning */);
4907 MarkFromDirtyCardsClosure
4908 markFromDirtyCardsClosure(this, _span,
4909 NULL, // space is set further below
4910 &_markBitMap, &_markStack, &mrias_cl);
4911 {
4912 GCTraceTime(Trace, gc, phases) t("Grey Object Rescan", _gc_timer_cm);
4913 // Iterate over the dirty cards, setting the corresponding bits in the
4914 // mod union table.
4915 {
4916 ModUnionClosure modUnionClosure(&_modUnionTable);
4917 _ct->dirty_card_iterate(_cmsGen->used_region(),
4918 &modUnionClosure);
4919 }
4920 // Having transferred these marks into the modUnionTable, we just need
4921 // to rescan the marked objects on the dirty cards in the modUnionTable.
4922 // The initial marking may have been done during an asynchronous
4923 // collection so there may be dirty bits in the mod-union table.
4924 const int alignment = CardTable::card_size * BitsPerWord;
4925 {
4926 // ... First handle dirty cards in CMS gen
4927 markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
4928 MemRegion ur = _cmsGen->used_region();
4929 HeapWord* lb = ur.start();
4930 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
4931 MemRegion cms_span(lb, ub);
4932 _modUnionTable.dirty_range_iterate_clear(cms_span,
4933 &markFromDirtyCardsClosure);
4934 verify_work_stacks_empty();
4935 log_trace(gc)(" (re-scanned " SIZE_FORMAT " dirty cards in cms gen) ", markFromDirtyCardsClosure.num_dirty_cards());
4936 }
4937 }
4938 if (VerifyDuringGC &&
4939 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4940 HandleMark hm; // Discard invalid handles created during verification
4941 Universe::verify();
4942 }
4943 {
4944 GCTraceTime(Trace, gc, phases) t("Root Rescan", _gc_timer_cm);
5609 size_t size = pointer_delta(nextOneAddr + 1, addr);
5610 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
5611 "alignment problem");
5612 assert(size >= 3, "Necessary for Printezis marks to work");
5613 return size;
5614 }
5615 return 0;
5616 }
5617
5618 HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
5619 size_t sz = 0;
5620 oop p = (oop)addr;
5621 if (p->klass_or_null_acquire() != NULL) {
5622 sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
5623 } else {
5624 sz = block_size_using_printezis_bits(addr);
5625 }
5626 assert(sz > 0, "size must be nonzero");
5627 HeapWord* next_block = addr + sz;
5628 HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block,
5629 CardTable::card_size);
5630 assert(round_down((uintptr_t)addr, CardTable::card_size) <
5631 round_down((uintptr_t)next_card, CardTable::card_size),
5632 "must be different cards");
5633 return next_card;
5634 }
5635
5636
5637 // CMS Bit Map Wrapper /////////////////////////////////////////
5638
5639 // Construct a CMS bit map infrastructure, but don't create the
5640 // bit vector itself. That is done by a separate call CMSBitMap::allocate()
5641 // further below.
5642 CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
5643 _bm(),
5644 _shifter(shifter),
5645 _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true,
5646 Monitor::_safepoint_check_sometimes) : NULL)
5647 {
5648 _bmStartWord = 0;
5649 _bmWordSize = 0;
5650 }
5651
6271 _span(span),
6272 _bitMap(bitMap),
6273 _mut(&collector->_modUnionTable),
6274 _markStack(markStack),
6275 _yield(should_yield),
6276 _skipBits(0)
6277 {
6278 assert(_markStack->isEmpty(), "stack should be empty");
6279 _finger = _bitMap->startWord();
6280 _threshold = _finger;
6281 assert(_collector->_restart_addr == NULL, "Sanity check");
6282 assert(_span.contains(_finger), "Out of bounds _finger?");
6283 DEBUG_ONLY(_verifying = verifying;)
6284 }
6285
6286 void MarkFromRootsClosure::reset(HeapWord* addr) {
6287 assert(_markStack->isEmpty(), "would cause duplicates on stack");
6288 assert(_span.contains(addr), "Out of bounds _finger?");
6289 _finger = addr;
6290 _threshold = (HeapWord*)round_to(
6291 (intptr_t)_finger, CardTable::card_size);
6292 }
6293
6294 // Should revisit to see if this should be restructured for
6295 // greater efficiency.
6296 bool MarkFromRootsClosure::do_bit(size_t offset) {
6297 if (_skipBits > 0) {
6298 _skipBits--;
6299 return true;
6300 }
6301 // convert offset into a HeapWord*
6302 HeapWord* addr = _bitMap->startWord() + offset;
6303 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
6304 "address out of range");
6305 assert(_bitMap->isMarked(addr), "tautology");
6306 if (_bitMap->isMarked(addr+1)) {
6307 // this is an allocated but not yet initialized object
6308 assert(_skipBits == 0, "tautology");
6309 _skipBits = 2; // skip next two marked bits ("Printezis-marks")
6310 oop p = oop(addr);
6311 if (p->klass_or_null_acquire() == NULL) {
6312 DEBUG_ONLY(if (!_verifying) {)
6313 // We re-dirty the cards on which this object lies and increase
6314 // the _threshold so that we'll come back to scan this object
6315 // during the preclean or remark phase. (CMSCleanOnEnter)
6316 if (CMSCleanOnEnter) {
6317 size_t sz = _collector->block_size_using_printezis_bits(addr);
6318 HeapWord* end_card_addr = (HeapWord*)round_to(
6319 (intptr_t)(addr+sz), CardTable::card_size);
6320 MemRegion redirty_range = MemRegion(addr, end_card_addr);
6321 assert(!redirty_range.is_empty(), "Arithmetical tautology");
6322 // Bump _threshold to end_card_addr; note that
6323 // _threshold cannot possibly exceed end_card_addr, anyhow.
6324 // This prevents future clearing of the card as the scan proceeds
6325 // to the right.
6326 assert(_threshold <= end_card_addr,
6327 "Because we are just scanning into this object");
6328 if (_threshold < end_card_addr) {
6329 _threshold = end_card_addr;
6330 }
6331 if (p->klass_or_null_acquire() != NULL) {
6332 // Redirty the range of cards...
6333 _mut->mark_range(redirty_range);
6334 } // ...else the setting of klass will dirty the card anyway.
6335 }
6336 DEBUG_ONLY(})
6337 return true;
6338 }
6339 }
6387 // accumulated in the card table and the mod union table --
6388 // these mutation records are redundant until we have
6389 // actually traced into the corresponding card.
6390 // Here, we check whether advancing the finger would make
6391 // us cross into a new card, and if so clear corresponding
6392 // cards in the MUT (preclean them in the card-table in the
6393 // future).
6394
6395 DEBUG_ONLY(if (!_verifying) {)
6396 // The clean-on-enter optimization is disabled by default,
6397 // until we fix 6178663.
6398 if (CMSCleanOnEnter && (_finger > _threshold)) {
6399 // [_threshold, _finger) represents the interval
6400 // of cards to be cleared in MUT (or precleaned in card table).
6401 // The set of cards to be cleared is all those that overlap
6402 // with the interval [_threshold, _finger); note that
6403 // _threshold is always kept card-aligned but _finger isn't
6404 // always card-aligned.
6405 HeapWord* old_threshold = _threshold;
6406 assert(old_threshold == (HeapWord*)round_to(
6407 (intptr_t)old_threshold, CardTable::card_size),
6408 "_threshold should always be card-aligned");
6409 _threshold = (HeapWord*)round_to(
6410 (intptr_t)_finger, CardTable::card_size);
6411 MemRegion mr(old_threshold, _threshold);
6412 assert(!mr.is_empty(), "Control point invariant");
6413 assert(_span.contains(mr), "Should clear within span");
6414 _mut->clear_range(mr);
6415 }
6416 DEBUG_ONLY(})
6417 // Note: the finger doesn't advance while we drain
6418 // the stack below.
6419 PushOrMarkClosure pushOrMarkClosure(_collector,
6420 _span, _bitMap, _markStack,
6421 _finger, this);
6422 bool res = _markStack->push(obj);
6423 assert(res, "Empty non-zero size stack should have space for single push");
6424 while (!_markStack->isEmpty()) {
6425 oop new_oop = _markStack->pop();
6426 // Skip verifying header mark word below because we are
6427 // running concurrent with mutators.
6428 assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
6429 // now scan this oop's oops
6430 new_oop->oop_iterate(&pushOrMarkClosure);
6501 // this time it's possible that a lot of mutations have
6502 // accumulated in the card table and the mod union table --
6503 // these mutation records are redundant until we have
6504 // actually traced into the corresponding card.
6505 // Here, we check whether advancing the finger would make
6506 // us cross into a new card, and if so clear corresponding
6507 // cards in the MUT (preclean them in the card-table in the
6508 // future).
6509
6510 // The clean-on-enter optimization is disabled by default,
6511 // until we fix 6178663.
6512 if (CMSCleanOnEnter && (_finger > _threshold)) {
6513 // [_threshold, _finger) represents the interval
6514 // of cards to be cleared in MUT (or precleaned in card table).
6515 // The set of cards to be cleared is all those that overlap
6516 // with the interval [_threshold, _finger); note that
6517 // _threshold is always kept card-aligned but _finger isn't
6518 // always card-aligned.
6519 HeapWord* old_threshold = _threshold;
6520 assert(old_threshold == (HeapWord*)round_to(
6521 (intptr_t)old_threshold, CardTable::card_size),
6522 "_threshold should always be card-aligned");
6523 _threshold = (HeapWord*)round_to(
6524 (intptr_t)_finger, CardTable::card_size);
6525 MemRegion mr(old_threshold, _threshold);
6526 assert(!mr.is_empty(), "Control point invariant");
6527 assert(_span.contains(mr), "Should clear within span"); // _whole_span ??
6528 _mut->clear_range(mr);
6529 }
6530
6531 // Note: the local finger doesn't advance while we drain
6532 // the stack below, but the global finger sure can and will.
6533 HeapWord* volatile* gfa = _task->global_finger_addr();
6534 ParPushOrMarkClosure pushOrMarkClosure(_collector,
6535 _span, _bit_map,
6536 _work_queue,
6537 _overflow_stack,
6538 _finger,
6539 gfa, this);
6540 bool res = _work_queue->push(obj); // overflow could occur here
6541 assert(res, "Will hold once we use workqueues");
6542 while (true) {
6543 oop new_oop;
6544 if (!_work_queue->pop_local(new_oop)) {
6872 if (CMSMarkStackOverflowALot &&
6873 _collector->simulate_overflow()) {
6874 // simulate a stack overflow
6875 simulate_overflow = true;
6876 }
6877 )
6878 if (simulate_overflow || !_mark_stack->push(obj)) {
6879 if (_concurrent_precleaning) {
6880 // During precleaning we can just dirty the appropriate card(s)
6881 // in the mod union table, thus ensuring that the object remains
6882 // in the grey set and continue. In the case of object arrays
6883 // we need to dirty all of the cards that the object spans,
6884 // since the rescan of object arrays will be limited to the
6885 // dirty cards.
6886 // Note that no one can be interfering with us in this action
6887 // of dirtying the mod union table, so no locking or atomics
6888 // are required.
6889 if (obj->is_objArray()) {
6890 size_t sz = obj->size();
6891 HeapWord* end_card_addr = (HeapWord*)round_to(
6892 (intptr_t)(addr+sz), CardTable::card_size);
6893 MemRegion redirty_range = MemRegion(addr, end_card_addr);
6894 assert(!redirty_range.is_empty(), "Arithmetical tautology");
6895 _mod_union_table->mark_range(redirty_range);
6896 } else {
6897 _mod_union_table->mark(addr);
6898 }
6899 _collector->_ser_pmc_preclean_ovflw++;
6900 } else {
6901 // During the remark phase, we need to remember this oop
6902 // in the overflow list.
6903 _collector->push_on_overflow_list(obj);
6904 _collector->_ser_pmc_remark_ovflw++;
6905 }
6906 }
6907 }
6908 }
6909
6910 ParPushAndMarkClosure::ParPushAndMarkClosure(CMSCollector* collector,
6911 MemRegion span,
6912 ReferenceProcessor* rp,
6985 for (unsigned i = 0; i < CMSYieldSleepCount &&
6986 ConcurrentMarkSweepThread::should_yield() &&
6987 !CMSCollector::foregroundGCIsActive(); ++i) {
6988 os::sleep(Thread::current(), 1, false);
6989 }
6990
6991 ConcurrentMarkSweepThread::synchronize(true);
6992 bml->lock();
6993
6994 _collector->startTimer();
6995 }
6996
6997 bool CMSPrecleanRefsYieldClosure::should_return() {
6998 if (ConcurrentMarkSweepThread::should_yield()) {
6999 do_yield_work();
7000 }
7001 return _collector->foregroundGCIsActive();
7002 }
7003
7004 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
7005 assert(((size_t)mr.start())%CardTable::card_size_in_words == 0,
7006 "mr should be aligned to start at a card boundary");
7007 // We'd like to assert:
7008 // assert(mr.word_size()%CardTable::card_size_in_words == 0,
7009 // "mr should be a range of cards");
7010 // However, that would be too strong in one case -- the last
7011 // partition ends at _unallocated_block which, in general, can be
7012 // an arbitrary boundary, not necessarily card aligned.
7013 _num_dirty_cards += mr.word_size()/CardTable::card_size_in_words;
7014 _space->object_iterate_mem(mr, &_scan_cl);
7015 }
7016
7017 SweepClosure::SweepClosure(CMSCollector* collector,
7018 ConcurrentMarkSweepGeneration* g,
7019 CMSBitMap* bitMap, bool should_yield) :
7020 _collector(collector),
7021 _g(g),
7022 _sp(g->cmsSpace()),
7023 _limit(_sp->sweep_limit()),
7024 _freelistLock(_sp->freelistLock()),
7025 _bitMap(bitMap),
7026 _yield(should_yield),
7027 _inFreeRange(false), // No free range at beginning of sweep
7028 _freeRangeInFreeLists(false), // No free range at beginning of sweep
7029 _lastFreeRangeCoalesced(false),
7030 _freeFinger(g->used_region().start())
7031 {
7032 NOT_PRODUCT(
7033 _numObjectsFreed = 0;
7602 bool simulate_overflow = false;
7603 NOT_PRODUCT(
7604 if (CMSMarkStackOverflowALot &&
7605 _collector->simulate_overflow()) {
7606 // simulate a stack overflow
7607 simulate_overflow = true;
7608 }
7609 )
7610 if (simulate_overflow || !_mark_stack->push(obj)) {
7611 if (_concurrent_precleaning) {
7612 // We dirty the overflown object and let the remark
7613 // phase deal with it.
7614 assert(_collector->overflow_list_is_empty(), "Error");
7615 // In the case of object arrays, we need to dirty all of
7616 // the cards that the object spans. No locking or atomics
7617 // are needed since no one else can be mutating the mod union
7618 // table.
7619 if (obj->is_objArray()) {
7620 size_t sz = obj->size();
7621 HeapWord* end_card_addr =
7622 (HeapWord*)round_to((intptr_t)(addr+sz), CardTable::card_size);
7623 MemRegion redirty_range = MemRegion(addr, end_card_addr);
7624 assert(!redirty_range.is_empty(), "Arithmetical tautology");
7625 _collector->_modUnionTable.mark_range(redirty_range);
7626 } else {
7627 _collector->_modUnionTable.mark(addr);
7628 }
7629 _collector->_ser_kac_preclean_ovflw++;
7630 } else {
7631 _collector->push_on_overflow_list(obj);
7632 _collector->_ser_kac_ovflw++;
7633 }
7634 }
7635 }
7636 }
7637
7638 void CMSKeepAliveClosure::do_oop(oop* p) { CMSKeepAliveClosure::do_oop_work(p); }
7639 void CMSKeepAliveClosure::do_oop(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); }
7640
7641 // CMSParKeepAliveClosure: a parallel version of the above.
7642 // The work queues are private to each closure (thread),
|