< prev index next >

src/share/vm/gc/cms/concurrentMarkSweepGeneration.cpp

Print this page
rev 12906 : [mq]: gc_interface


 428   st->cr();
 429 }
 430 #endif // #ifndef PRODUCT
 431 
 432 CMSCollector::CollectorState CMSCollector::_collectorState =
 433                              CMSCollector::Idling;
 434 bool CMSCollector::_foregroundGCIsActive = false;
 435 bool CMSCollector::_foregroundGCShouldWait = false;
 436 
 437 CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
 438                            CardTableRS*                   ct,
 439                            ConcurrentMarkSweepPolicy*     cp):
 440   _cmsGen(cmsGen),
 441   _ct(ct),
 442   _ref_processor(NULL),    // will be set later
 443   _conc_workers(NULL),     // may be set later
 444   _abort_preclean(false),
 445   _start_sampling(false),
 446   _between_prologue_and_epilogue(false),
 447   _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
 448   _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize),
 449                  -1 /* lock-free */, "No_lock" /* dummy */),
 450   _modUnionClosurePar(&_modUnionTable),
 451   // Adjust my span to cover old (cms) gen
 452   _span(cmsGen->reserved()),
 453   // Construct the is_alive_closure with _span & markBitMap
 454   _is_alive_closure(_span, &_markBitMap),
 455   _restart_addr(NULL),
 456   _overflow_list(NULL),
 457   _stats(cmsGen),
 458   _eden_chunk_lock(new Mutex(Mutex::leaf + 1, "CMS_eden_chunk_lock", true,
 459                              //verify that this lock should be acquired with safepoint check.
 460                              Monitor::_safepoint_check_sometimes)),
 461   _eden_chunk_array(NULL),     // may be set in ctor body
 462   _eden_chunk_capacity(0),     // -- ditto --
 463   _eden_chunk_index(0),        // -- ditto --
 464   _survivor_plab_array(NULL),  // -- ditto --
 465   _survivor_chunk_array(NULL), // -- ditto --
 466   _survivor_chunk_capacity(0), // -- ditto --
 467   _survivor_chunk_index(0),    // -- ditto --
 468   _ser_pmc_preclean_ovflw(0),


 880     // is in flux from being free to being allocated (and in
 881     // transition while being copied into) and subsequently
 882     // becoming a bona-fide object when the copy/promotion is complete.
 883     assert(SafepointSynchronize::is_at_safepoint(),
 884            "expect promotion only at safepoints");
 885 
 886     if (_collectorState < Sweeping) {
 887       // Mark the appropriate cards in the modUnionTable, so that
 888       // this object gets scanned before the sweep. If this is
 889       // not done, CMS generation references in the object might
 890       // not get marked.
 891       // For the case of arrays, which are otherwise precisely
 892       // marked, we need to dirty the entire array, not just its head.
 893       if (is_obj_array) {
 894         // The [par_]mark_range() method expects mr.end() below to
 895         // be aligned to the granularity of a bit's representation
 896         // in the heap. In the case of the MUT below, that's a
 897         // card size.
 898         MemRegion mr(start,
 899                      (HeapWord*)round_to((intptr_t)(start + obj_size),
 900                         CardTableModRefBS::card_size /* bytes */));
 901         if (par) {
 902           _modUnionTable.par_mark_range(mr);
 903         } else {
 904           _modUnionTable.mark_range(mr);
 905         }
 906       } else {  // not an obj array; we can just mark the head
 907         if (par) {
 908           _modUnionTable.par_mark(start);
 909         } else {
 910           _modUnionTable.mark(start);
 911         }
 912       }
 913     }
 914   }
 915 }
 916 
 917 oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) {
 918   assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");
 919   // allocate, copy and if necessary update promoinfo --
 920   // delegate to underlying space.


3204     assert(cur != NULL, "Counted wrong?");
3205     work_q->push(cur);
3206   }
3207   return num > 0;
3208 }
3209 
3210 void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
3211   SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
3212   int n_tasks = pst->n_tasks();
3213   // We allow that there may be no tasks to do here because
3214   // we are restarting after a stack overflow.
3215   assert(pst->valid() || n_tasks == 0, "Uninitialized use?");
3216   uint nth_task = 0;
3217 
3218   HeapWord* aligned_start = sp->bottom();
3219   if (sp->used_region().contains(_restart_addr)) {
3220     // Align down to a card boundary for the start of 0th task
3221     // for this space.
3222     aligned_start =
3223       (HeapWord*)align_size_down((uintptr_t)_restart_addr,
3224                                  CardTableModRefBS::card_size);
3225   }
3226 
3227   size_t chunk_size = sp->marking_task_size();
3228   while (!pst->is_task_claimed(/* reference */ nth_task)) {
3229     // Having claimed the nth task in this space,
3230     // compute the chunk that it corresponds to:
3231     MemRegion span = MemRegion(aligned_start + nth_task*chunk_size,
3232                                aligned_start + (nth_task+1)*chunk_size);
3233     // Try and bump the global finger via a CAS;
3234     // note that we need to do the global finger bump
3235     // _before_ taking the intersection below, because
3236     // the task corresponding to that region will be
3237     // deemed done even if the used_region() expands
3238     // because of allocation -- as it almost certainly will
3239     // during start-up while the threads yield in the
3240     // closure below.
3241     HeapWord* finger = span.end();
3242     bump_global_finger(finger);   // atomically
3243     // There are null tasks here corresponding to chunks
3244     // beyond the "top" address of the space.


4007   HeapWord *lastAddr, *nextAddr;
4008 
4009   for (cumNumDirtyCards = numDirtyCards = 0,
4010        nextAddr = lastAddr = startAddr;
4011        nextAddr < endAddr;
4012        nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
4013 
4014     ResourceMark rm;
4015     HandleMark   hm;
4016 
4017     MemRegion dirtyRegion;
4018     {
4019       // See comments in "Precleaning notes" above on why we
4020       // do this locking. XXX Could the locking overheads be
4021       // too high when dirty cards are sparse? [I don't think so.]
4022       stopTimer();
4023       CMSTokenSync x(true); // is cms thread
4024       startTimer();
4025       sample_eden();
4026       // Get and clear dirty region from card table
4027       dirtyRegion = _ct->ct_bs()->dirty_card_range_after_reset(
4028                                     MemRegion(nextAddr, endAddr),
4029                                     true,
4030                                     CardTableModRefBS::precleaned_card_val());
4031 
4032       assert(dirtyRegion.start() >= nextAddr,
4033              "returned region inconsistent?");
4034     }
4035     lastAddr = dirtyRegion.end();
4036     numDirtyCards =
4037       dirtyRegion.word_size()/CardTableModRefBS::card_size_in_words;
4038 
4039     if (!dirtyRegion.is_empty()) {
4040       stopTimer();
4041       CMSTokenSyncWithLocks ts(true, old_gen->freelistLock(), bitMapLock());
4042       startTimer();
4043       sample_eden();
4044       verify_work_stacks_empty();
4045       verify_overflow_empty();
4046       HeapWord* stop_point =
4047         old_gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
4048       if (stop_point != NULL) {
4049         assert((_collectorState == AbortablePreclean && should_abort_preclean()),
4050                "Should only be AbortablePreclean.");
4051         _ct->ct_bs()->invalidate(MemRegion(stop_point, dirtyRegion.end()));
4052         if (should_abort_preclean()) {
4053           break; // out of preclean loop
4054         } else {
4055           // Compute the next address at which preclean should pick up.
4056           lastAddr = next_card_start_after_block(stop_point);
4057         }
4058       }
4059     } else {
4060       break;
4061     }
4062   }
4063   verify_work_stacks_empty();
4064   verify_overflow_empty();
4065   return cumNumDirtyCards;
4066 }
4067 
4068 class PrecleanKlassClosure : public KlassClosure {
4069   KlassToOopClosure _cm_klass_closure;
4070  public:
4071   PrecleanKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {}


4558   // work-partitioning form than the current "sequential tasks"
4559   // paradigm, the use of that persistent state will have to be
4560   // revisited and modified appropriately. See also related
4561   // bug 4756801 work on which should examine this code to make
4562   // sure that the changes there do not run counter to the
4563   // assumptions made here and necessary for correctness and
4564   // efficiency. Note also that this code might yield inefficient
4565   // behavior in the case of very large objects that span one or
4566   // more work chunks. Such objects would potentially be scanned
4567   // several times redundantly. Work on 4756801 should try and
4568   // address that performance anomaly if at all possible. XXX
4569   MemRegion  full_span  = _collector->_span;
4570   CMSBitMap* bm    = &(_collector->_markBitMap);     // shared
4571   MarkFromDirtyCardsClosure
4572     greyRescanClosure(_collector, full_span, // entire span of interest
4573                       sp, bm, work_q, cl);
4574 
4575   SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
4576   assert(pst->valid(), "Uninitialized use?");
4577   uint nth_task = 0;
4578   const int alignment = CardTableModRefBS::card_size * BitsPerWord;
4579   MemRegion span = sp->used_region();
4580   HeapWord* start_addr = span.start();
4581   HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(),
4582                                            alignment);
4583   const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units
4584   assert((HeapWord*)round_to((intptr_t)start_addr, alignment) ==
4585          start_addr, "Check alignment");
4586   assert((size_t)round_to((intptr_t)chunk_size, alignment) ==
4587          chunk_size, "Check alignment");
4588 
4589   while (!pst->is_task_claimed(/* reference */ nth_task)) {
4590     // Having claimed the nth_task, compute corresponding mem-region,
4591     // which is a-fortiori aligned correctly (i.e. at a MUT boundary).
4592     // The alignment restriction ensures that we do not need any
4593     // synchronization with other gang-workers while setting or
4594     // clearing bits in thus chunk of the MUT.
4595     MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size,
4596                                     start_addr + (nth_task+1)*chunk_size);
4597     // The last chunk's end might be way beyond end of the
4598     // used region. In that case pull back appropriately.
4599     if (this_span.end() > end_addr) {
4600       this_span.set_end(end_addr);
4601       assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)");
4602     }
4603     // Iterate over the dirty cards covering this chunk, marking them
4604     // precleaned, and setting the corresponding bits in the mod union
4605     // table. Since we have been careful to partition at Card and MUT-word
4606     // boundaries no synchronization is needed between parallel threads.
4607     _collector->_ct->ct_bs()->dirty_card_iterate(this_span,
4608                                                  &modUnionClosure);
4609 
4610     // Having transferred these marks into the modUnionTable,
4611     // rescan the marked objects on the dirty cards in the modUnionTable.
4612     // Even if this is at a synchronous collection, the initial marking
4613     // may have been done during an asynchronous collection so there
4614     // may be dirty bits in the mod-union table.
4615     _collector->_modUnionTable.dirty_range_iterate_clear(
4616                   this_span, &greyRescanClosure);
4617     _collector->_modUnionTable.verifyNoOneBitsInRange(
4618                                  this_span.start(),
4619                                  this_span.end());
4620   }
4621   pst->all_tasks_completed();  // declare that i am done
4622 }
4623 
4624 // . see if we can share work_queues with ParNew? XXX
4625 void
4626 CMSParRemarkTask::do_work_steal(int i, ParMarkRefsIntoAndScanClosure* cl,
4627                                 int* seed) {


4898 void CMSCollector::do_remark_non_parallel() {
4899   ResourceMark rm;
4900   HandleMark   hm;
4901   GenCollectedHeap* gch = GenCollectedHeap::heap();
4902   ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
4903 
4904   MarkRefsIntoAndScanClosure
4905     mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */,
4906              &_markStack, this,
4907              false /* should_yield */, false /* not precleaning */);
4908   MarkFromDirtyCardsClosure
4909     markFromDirtyCardsClosure(this, _span,
4910                               NULL,  // space is set further below
4911                               &_markBitMap, &_markStack, &mrias_cl);
4912   {
4913     GCTraceTime(Trace, gc, phases) t("Grey Object Rescan", _gc_timer_cm);
4914     // Iterate over the dirty cards, setting the corresponding bits in the
4915     // mod union table.
4916     {
4917       ModUnionClosure modUnionClosure(&_modUnionTable);
4918       _ct->ct_bs()->dirty_card_iterate(
4919                       _cmsGen->used_region(),
4920                       &modUnionClosure);
4921     }
4922     // Having transferred these marks into the modUnionTable, we just need
4923     // to rescan the marked objects on the dirty cards in the modUnionTable.
4924     // The initial marking may have been done during an asynchronous
4925     // collection so there may be dirty bits in the mod-union table.
4926     const int alignment =
4927       CardTableModRefBS::card_size * BitsPerWord;
4928     {
4929       // ... First handle dirty cards in CMS gen
4930       markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
4931       MemRegion ur = _cmsGen->used_region();
4932       HeapWord* lb = ur.start();
4933       HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
4934       MemRegion cms_span(lb, ub);
4935       _modUnionTable.dirty_range_iterate_clear(cms_span,
4936                                                &markFromDirtyCardsClosure);
4937       verify_work_stacks_empty();
4938       log_trace(gc)(" (re-scanned " SIZE_FORMAT " dirty cards in cms gen) ", markFromDirtyCardsClosure.num_dirty_cards());
4939     }
4940   }
4941   if (VerifyDuringGC &&
4942       GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4943     HandleMark hm;  // Discard invalid handles created during verification
4944     Universe::verify();
4945   }
4946   {
4947     GCTraceTime(Trace, gc, phases) t("Root Rescan", _gc_timer_cm);


5612     size_t size = pointer_delta(nextOneAddr + 1, addr);
5613     assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
5614            "alignment problem");
5615     assert(size >= 3, "Necessary for Printezis marks to work");
5616     return size;
5617   }
5618   return 0;
5619 }
5620 
5621 HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
5622   size_t sz = 0;
5623   oop p = (oop)addr;
5624   if (p->klass_or_null_acquire() != NULL) {
5625     sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
5626   } else {
5627     sz = block_size_using_printezis_bits(addr);
5628   }
5629   assert(sz > 0, "size must be nonzero");
5630   HeapWord* next_block = addr + sz;
5631   HeapWord* next_card  = (HeapWord*)round_to((uintptr_t)next_block,
5632                                              CardTableModRefBS::card_size);
5633   assert(round_down((uintptr_t)addr,      CardTableModRefBS::card_size) <
5634          round_down((uintptr_t)next_card, CardTableModRefBS::card_size),
5635          "must be different cards");
5636   return next_card;
5637 }
5638 
5639 
5640 // CMS Bit Map Wrapper /////////////////////////////////////////
5641 
5642 // Construct a CMS bit map infrastructure, but don't create the
5643 // bit vector itself. That is done by a separate call CMSBitMap::allocate()
5644 // further below.
5645 CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
5646   _bm(),
5647   _shifter(shifter),
5648   _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true,
5649                                     Monitor::_safepoint_check_sometimes) : NULL)
5650 {
5651   _bmStartWord = 0;
5652   _bmWordSize  = 0;
5653 }
5654 


6274   _span(span),
6275   _bitMap(bitMap),
6276   _mut(&collector->_modUnionTable),
6277   _markStack(markStack),
6278   _yield(should_yield),
6279   _skipBits(0)
6280 {
6281   assert(_markStack->isEmpty(), "stack should be empty");
6282   _finger = _bitMap->startWord();
6283   _threshold = _finger;
6284   assert(_collector->_restart_addr == NULL, "Sanity check");
6285   assert(_span.contains(_finger), "Out of bounds _finger?");
6286   DEBUG_ONLY(_verifying = verifying;)
6287 }
6288 
6289 void MarkFromRootsClosure::reset(HeapWord* addr) {
6290   assert(_markStack->isEmpty(), "would cause duplicates on stack");
6291   assert(_span.contains(addr), "Out of bounds _finger?");
6292   _finger = addr;
6293   _threshold = (HeapWord*)round_to(
6294                  (intptr_t)_finger, CardTableModRefBS::card_size);
6295 }
6296 
6297 // Should revisit to see if this should be restructured for
6298 // greater efficiency.
6299 bool MarkFromRootsClosure::do_bit(size_t offset) {
6300   if (_skipBits > 0) {
6301     _skipBits--;
6302     return true;
6303   }
6304   // convert offset into a HeapWord*
6305   HeapWord* addr = _bitMap->startWord() + offset;
6306   assert(_bitMap->endWord() && addr < _bitMap->endWord(),
6307          "address out of range");
6308   assert(_bitMap->isMarked(addr), "tautology");
6309   if (_bitMap->isMarked(addr+1)) {
6310     // this is an allocated but not yet initialized object
6311     assert(_skipBits == 0, "tautology");
6312     _skipBits = 2;  // skip next two marked bits ("Printezis-marks")
6313     oop p = oop(addr);
6314     if (p->klass_or_null_acquire() == NULL) {
6315       DEBUG_ONLY(if (!_verifying) {)
6316         // We re-dirty the cards on which this object lies and increase
6317         // the _threshold so that we'll come back to scan this object
6318         // during the preclean or remark phase. (CMSCleanOnEnter)
6319         if (CMSCleanOnEnter) {
6320           size_t sz = _collector->block_size_using_printezis_bits(addr);
6321           HeapWord* end_card_addr   = (HeapWord*)round_to(
6322                                          (intptr_t)(addr+sz), CardTableModRefBS::card_size);
6323           MemRegion redirty_range = MemRegion(addr, end_card_addr);
6324           assert(!redirty_range.is_empty(), "Arithmetical tautology");
6325           // Bump _threshold to end_card_addr; note that
6326           // _threshold cannot possibly exceed end_card_addr, anyhow.
6327           // This prevents future clearing of the card as the scan proceeds
6328           // to the right.
6329           assert(_threshold <= end_card_addr,
6330                  "Because we are just scanning into this object");
6331           if (_threshold < end_card_addr) {
6332             _threshold = end_card_addr;
6333           }
6334           if (p->klass_or_null_acquire() != NULL) {
6335             // Redirty the range of cards...
6336             _mut->mark_range(redirty_range);
6337           } // ...else the setting of klass will dirty the card anyway.
6338         }
6339       DEBUG_ONLY(})
6340       return true;
6341     }
6342   }


6390   // accumulated in the card table and the mod union table --
6391   // these mutation records are redundant until we have
6392   // actually traced into the corresponding card.
6393   // Here, we check whether advancing the finger would make
6394   // us cross into a new card, and if so clear corresponding
6395   // cards in the MUT (preclean them in the card-table in the
6396   // future).
6397 
6398   DEBUG_ONLY(if (!_verifying) {)
6399     // The clean-on-enter optimization is disabled by default,
6400     // until we fix 6178663.
6401     if (CMSCleanOnEnter && (_finger > _threshold)) {
6402       // [_threshold, _finger) represents the interval
6403       // of cards to be cleared  in MUT (or precleaned in card table).
6404       // The set of cards to be cleared is all those that overlap
6405       // with the interval [_threshold, _finger); note that
6406       // _threshold is always kept card-aligned but _finger isn't
6407       // always card-aligned.
6408       HeapWord* old_threshold = _threshold;
6409       assert(old_threshold == (HeapWord*)round_to(
6410               (intptr_t)old_threshold, CardTableModRefBS::card_size),
6411              "_threshold should always be card-aligned");
6412       _threshold = (HeapWord*)round_to(
6413                      (intptr_t)_finger, CardTableModRefBS::card_size);
6414       MemRegion mr(old_threshold, _threshold);
6415       assert(!mr.is_empty(), "Control point invariant");
6416       assert(_span.contains(mr), "Should clear within span");
6417       _mut->clear_range(mr);
6418     }
6419   DEBUG_ONLY(})
6420   // Note: the finger doesn't advance while we drain
6421   // the stack below.
6422   PushOrMarkClosure pushOrMarkClosure(_collector,
6423                                       _span, _bitMap, _markStack,
6424                                       _finger, this);
6425   bool res = _markStack->push(obj);
6426   assert(res, "Empty non-zero size stack should have space for single push");
6427   while (!_markStack->isEmpty()) {
6428     oop new_oop = _markStack->pop();
6429     // Skip verifying header mark word below because we are
6430     // running concurrent with mutators.
6431     assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
6432     // now scan this oop's oops
6433     new_oop->oop_iterate(&pushOrMarkClosure);


6504   // this time it's possible that a lot of mutations have
6505   // accumulated in the card table and the mod union table --
6506   // these mutation records are redundant until we have
6507   // actually traced into the corresponding card.
6508   // Here, we check whether advancing the finger would make
6509   // us cross into a new card, and if so clear corresponding
6510   // cards in the MUT (preclean them in the card-table in the
6511   // future).
6512 
6513   // The clean-on-enter optimization is disabled by default,
6514   // until we fix 6178663.
6515   if (CMSCleanOnEnter && (_finger > _threshold)) {
6516     // [_threshold, _finger) represents the interval
6517     // of cards to be cleared  in MUT (or precleaned in card table).
6518     // The set of cards to be cleared is all those that overlap
6519     // with the interval [_threshold, _finger); note that
6520     // _threshold is always kept card-aligned but _finger isn't
6521     // always card-aligned.
6522     HeapWord* old_threshold = _threshold;
6523     assert(old_threshold == (HeapWord*)round_to(
6524             (intptr_t)old_threshold, CardTableModRefBS::card_size),
6525            "_threshold should always be card-aligned");
6526     _threshold = (HeapWord*)round_to(
6527                    (intptr_t)_finger, CardTableModRefBS::card_size);
6528     MemRegion mr(old_threshold, _threshold);
6529     assert(!mr.is_empty(), "Control point invariant");
6530     assert(_span.contains(mr), "Should clear within span"); // _whole_span ??
6531     _mut->clear_range(mr);
6532   }
6533 
6534   // Note: the local finger doesn't advance while we drain
6535   // the stack below, but the global finger sure can and will.
6536   HeapWord* volatile* gfa = _task->global_finger_addr();
6537   ParPushOrMarkClosure pushOrMarkClosure(_collector,
6538                                          _span, _bit_map,
6539                                          _work_queue,
6540                                          _overflow_stack,
6541                                          _finger,
6542                                          gfa, this);
6543   bool res = _work_queue->push(obj);   // overflow could occur here
6544   assert(res, "Will hold once we use workqueues");
6545   while (true) {
6546     oop new_oop;
6547     if (!_work_queue->pop_local(new_oop)) {


6875       if (CMSMarkStackOverflowALot &&
6876           _collector->simulate_overflow()) {
6877         // simulate a stack overflow
6878         simulate_overflow = true;
6879       }
6880     )
6881     if (simulate_overflow || !_mark_stack->push(obj)) {
6882       if (_concurrent_precleaning) {
6883          // During precleaning we can just dirty the appropriate card(s)
6884          // in the mod union table, thus ensuring that the object remains
6885          // in the grey set  and continue. In the case of object arrays
6886          // we need to dirty all of the cards that the object spans,
6887          // since the rescan of object arrays will be limited to the
6888          // dirty cards.
6889          // Note that no one can be interfering with us in this action
6890          // of dirtying the mod union table, so no locking or atomics
6891          // are required.
6892          if (obj->is_objArray()) {
6893            size_t sz = obj->size();
6894            HeapWord* end_card_addr = (HeapWord*)round_to(
6895                                         (intptr_t)(addr+sz), CardTableModRefBS::card_size);
6896            MemRegion redirty_range = MemRegion(addr, end_card_addr);
6897            assert(!redirty_range.is_empty(), "Arithmetical tautology");
6898            _mod_union_table->mark_range(redirty_range);
6899          } else {
6900            _mod_union_table->mark(addr);
6901          }
6902          _collector->_ser_pmc_preclean_ovflw++;
6903       } else {
6904          // During the remark phase, we need to remember this oop
6905          // in the overflow list.
6906          _collector->push_on_overflow_list(obj);
6907          _collector->_ser_pmc_remark_ovflw++;
6908       }
6909     }
6910   }
6911 }
6912 
6913 ParPushAndMarkClosure::ParPushAndMarkClosure(CMSCollector* collector,
6914                                              MemRegion span,
6915                                              ReferenceProcessor* rp,


6988   for (unsigned i = 0; i < CMSYieldSleepCount &&
6989                        ConcurrentMarkSweepThread::should_yield() &&
6990                        !CMSCollector::foregroundGCIsActive(); ++i) {
6991     os::sleep(Thread::current(), 1, false);
6992   }
6993 
6994   ConcurrentMarkSweepThread::synchronize(true);
6995   bml->lock();
6996 
6997   _collector->startTimer();
6998 }
6999 
7000 bool CMSPrecleanRefsYieldClosure::should_return() {
7001   if (ConcurrentMarkSweepThread::should_yield()) {
7002     do_yield_work();
7003   }
7004   return _collector->foregroundGCIsActive();
7005 }
7006 
7007 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
7008   assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
7009          "mr should be aligned to start at a card boundary");
7010   // We'd like to assert:
7011   // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
7012   //        "mr should be a range of cards");
7013   // However, that would be too strong in one case -- the last
7014   // partition ends at _unallocated_block which, in general, can be
7015   // an arbitrary boundary, not necessarily card aligned.
7016   _num_dirty_cards += mr.word_size()/CardTableModRefBS::card_size_in_words;
7017   _space->object_iterate_mem(mr, &_scan_cl);
7018 }
7019 
7020 SweepClosure::SweepClosure(CMSCollector* collector,
7021                            ConcurrentMarkSweepGeneration* g,
7022                            CMSBitMap* bitMap, bool should_yield) :
7023   _collector(collector),
7024   _g(g),
7025   _sp(g->cmsSpace()),
7026   _limit(_sp->sweep_limit()),
7027   _freelistLock(_sp->freelistLock()),
7028   _bitMap(bitMap),
7029   _yield(should_yield),
7030   _inFreeRange(false),           // No free range at beginning of sweep
7031   _freeRangeInFreeLists(false),  // No free range at beginning of sweep
7032   _lastFreeRangeCoalesced(false),
7033   _freeFinger(g->used_region().start())
7034 {
7035   NOT_PRODUCT(
7036     _numObjectsFreed = 0;


7605     bool simulate_overflow = false;
7606     NOT_PRODUCT(
7607       if (CMSMarkStackOverflowALot &&
7608           _collector->simulate_overflow()) {
7609         // simulate a stack overflow
7610         simulate_overflow = true;
7611       }
7612     )
7613     if (simulate_overflow || !_mark_stack->push(obj)) {
7614       if (_concurrent_precleaning) {
7615         // We dirty the overflown object and let the remark
7616         // phase deal with it.
7617         assert(_collector->overflow_list_is_empty(), "Error");
7618         // In the case of object arrays, we need to dirty all of
7619         // the cards that the object spans. No locking or atomics
7620         // are needed since no one else can be mutating the mod union
7621         // table.
7622         if (obj->is_objArray()) {
7623           size_t sz = obj->size();
7624           HeapWord* end_card_addr =
7625             (HeapWord*)round_to((intptr_t)(addr+sz), CardTableModRefBS::card_size);
7626           MemRegion redirty_range = MemRegion(addr, end_card_addr);
7627           assert(!redirty_range.is_empty(), "Arithmetical tautology");
7628           _collector->_modUnionTable.mark_range(redirty_range);
7629         } else {
7630           _collector->_modUnionTable.mark(addr);
7631         }
7632         _collector->_ser_kac_preclean_ovflw++;
7633       } else {
7634         _collector->push_on_overflow_list(obj);
7635         _collector->_ser_kac_ovflw++;
7636       }
7637     }
7638   }
7639 }
7640 
7641 void CMSKeepAliveClosure::do_oop(oop* p)       { CMSKeepAliveClosure::do_oop_work(p); }
7642 void CMSKeepAliveClosure::do_oop(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); }
7643 
7644 // CMSParKeepAliveClosure: a parallel version of the above.
7645 // The work queues are private to each closure (thread),




 428   st->cr();
 429 }
 430 #endif // #ifndef PRODUCT
 431 
 432 CMSCollector::CollectorState CMSCollector::_collectorState =
 433                              CMSCollector::Idling;
 434 bool CMSCollector::_foregroundGCIsActive = false;
 435 bool CMSCollector::_foregroundGCShouldWait = false;
 436 
 437 CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
 438                            CardTableRS*                   ct,
 439                            ConcurrentMarkSweepPolicy*     cp):
 440   _cmsGen(cmsGen),
 441   _ct(ct),
 442   _ref_processor(NULL),    // will be set later
 443   _conc_workers(NULL),     // may be set later
 444   _abort_preclean(false),
 445   _start_sampling(false),
 446   _between_prologue_and_epilogue(false),
 447   _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
 448   _modUnionTable((CardTable::card_shift - LogHeapWordSize),
 449                  -1 /* lock-free */, "No_lock" /* dummy */),
 450   _modUnionClosurePar(&_modUnionTable),
 451   // Adjust my span to cover old (cms) gen
 452   _span(cmsGen->reserved()),
 453   // Construct the is_alive_closure with _span & markBitMap
 454   _is_alive_closure(_span, &_markBitMap),
 455   _restart_addr(NULL),
 456   _overflow_list(NULL),
 457   _stats(cmsGen),
 458   _eden_chunk_lock(new Mutex(Mutex::leaf + 1, "CMS_eden_chunk_lock", true,
 459                              //verify that this lock should be acquired with safepoint check.
 460                              Monitor::_safepoint_check_sometimes)),
 461   _eden_chunk_array(NULL),     // may be set in ctor body
 462   _eden_chunk_capacity(0),     // -- ditto --
 463   _eden_chunk_index(0),        // -- ditto --
 464   _survivor_plab_array(NULL),  // -- ditto --
 465   _survivor_chunk_array(NULL), // -- ditto --
 466   _survivor_chunk_capacity(0), // -- ditto --
 467   _survivor_chunk_index(0),    // -- ditto --
 468   _ser_pmc_preclean_ovflw(0),


 880     // is in flux from being free to being allocated (and in
 881     // transition while being copied into) and subsequently
 882     // becoming a bona-fide object when the copy/promotion is complete.
 883     assert(SafepointSynchronize::is_at_safepoint(),
 884            "expect promotion only at safepoints");
 885 
 886     if (_collectorState < Sweeping) {
 887       // Mark the appropriate cards in the modUnionTable, so that
 888       // this object gets scanned before the sweep. If this is
 889       // not done, CMS generation references in the object might
 890       // not get marked.
 891       // For the case of arrays, which are otherwise precisely
 892       // marked, we need to dirty the entire array, not just its head.
 893       if (is_obj_array) {
 894         // The [par_]mark_range() method expects mr.end() below to
 895         // be aligned to the granularity of a bit's representation
 896         // in the heap. In the case of the MUT below, that's a
 897         // card size.
 898         MemRegion mr(start,
 899                      (HeapWord*)round_to((intptr_t)(start + obj_size),
 900                         CardTable::card_size /* bytes */));
 901         if (par) {
 902           _modUnionTable.par_mark_range(mr);
 903         } else {
 904           _modUnionTable.mark_range(mr);
 905         }
 906       } else {  // not an obj array; we can just mark the head
 907         if (par) {
 908           _modUnionTable.par_mark(start);
 909         } else {
 910           _modUnionTable.mark(start);
 911         }
 912       }
 913     }
 914   }
 915 }
 916 
 917 oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) {
 918   assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");
 919   // allocate, copy and if necessary update promoinfo --
 920   // delegate to underlying space.


3204     assert(cur != NULL, "Counted wrong?");
3205     work_q->push(cur);
3206   }
3207   return num > 0;
3208 }
3209 
3210 void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
3211   SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
3212   int n_tasks = pst->n_tasks();
3213   // We allow that there may be no tasks to do here because
3214   // we are restarting after a stack overflow.
3215   assert(pst->valid() || n_tasks == 0, "Uninitialized use?");
3216   uint nth_task = 0;
3217 
3218   HeapWord* aligned_start = sp->bottom();
3219   if (sp->used_region().contains(_restart_addr)) {
3220     // Align down to a card boundary for the start of 0th task
3221     // for this space.
3222     aligned_start =
3223       (HeapWord*)align_size_down((uintptr_t)_restart_addr,
3224                                  CardTable::card_size);
3225   }
3226 
3227   size_t chunk_size = sp->marking_task_size();
3228   while (!pst->is_task_claimed(/* reference */ nth_task)) {
3229     // Having claimed the nth task in this space,
3230     // compute the chunk that it corresponds to:
3231     MemRegion span = MemRegion(aligned_start + nth_task*chunk_size,
3232                                aligned_start + (nth_task+1)*chunk_size);
3233     // Try and bump the global finger via a CAS;
3234     // note that we need to do the global finger bump
3235     // _before_ taking the intersection below, because
3236     // the task corresponding to that region will be
3237     // deemed done even if the used_region() expands
3238     // because of allocation -- as it almost certainly will
3239     // during start-up while the threads yield in the
3240     // closure below.
3241     HeapWord* finger = span.end();
3242     bump_global_finger(finger);   // atomically
3243     // There are null tasks here corresponding to chunks
3244     // beyond the "top" address of the space.


4007   HeapWord *lastAddr, *nextAddr;
4008 
4009   for (cumNumDirtyCards = numDirtyCards = 0,
4010        nextAddr = lastAddr = startAddr;
4011        nextAddr < endAddr;
4012        nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
4013 
4014     ResourceMark rm;
4015     HandleMark   hm;
4016 
4017     MemRegion dirtyRegion;
4018     {
4019       // See comments in "Precleaning notes" above on why we
4020       // do this locking. XXX Could the locking overheads be
4021       // too high when dirty cards are sparse? [I don't think so.]
4022       stopTimer();
4023       CMSTokenSync x(true); // is cms thread
4024       startTimer();
4025       sample_eden();
4026       // Get and clear dirty region from card table
4027       dirtyRegion = _ct->dirty_card_range_after_reset(MemRegion(nextAddr, endAddr),

4028                                                       true,
4029                                                       CardTable::precleaned_card_val());
4030 
4031       assert(dirtyRegion.start() >= nextAddr,
4032              "returned region inconsistent?");
4033     }
4034     lastAddr = dirtyRegion.end();
4035     numDirtyCards =
4036       dirtyRegion.word_size()/CardTable::card_size_in_words;
4037 
4038     if (!dirtyRegion.is_empty()) {
4039       stopTimer();
4040       CMSTokenSyncWithLocks ts(true, old_gen->freelistLock(), bitMapLock());
4041       startTimer();
4042       sample_eden();
4043       verify_work_stacks_empty();
4044       verify_overflow_empty();
4045       HeapWord* stop_point =
4046         old_gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
4047       if (stop_point != NULL) {
4048         assert((_collectorState == AbortablePreclean && should_abort_preclean()),
4049                "Should only be AbortablePreclean.");
4050         _ct->invalidate(MemRegion(stop_point, dirtyRegion.end()));
4051         if (should_abort_preclean()) {
4052           break; // out of preclean loop
4053         } else {
4054           // Compute the next address at which preclean should pick up.
4055           lastAddr = next_card_start_after_block(stop_point);
4056         }
4057       }
4058     } else {
4059       break;
4060     }
4061   }
4062   verify_work_stacks_empty();
4063   verify_overflow_empty();
4064   return cumNumDirtyCards;
4065 }
4066 
4067 class PrecleanKlassClosure : public KlassClosure {
4068   KlassToOopClosure _cm_klass_closure;
4069  public:
4070   PrecleanKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {}


4557   // work-partitioning form than the current "sequential tasks"
4558   // paradigm, the use of that persistent state will have to be
4559   // revisited and modified appropriately. See also related
4560   // bug 4756801 work on which should examine this code to make
4561   // sure that the changes there do not run counter to the
4562   // assumptions made here and necessary for correctness and
4563   // efficiency. Note also that this code might yield inefficient
4564   // behavior in the case of very large objects that span one or
4565   // more work chunks. Such objects would potentially be scanned
4566   // several times redundantly. Work on 4756801 should try and
4567   // address that performance anomaly if at all possible. XXX
4568   MemRegion  full_span  = _collector->_span;
4569   CMSBitMap* bm    = &(_collector->_markBitMap);     // shared
4570   MarkFromDirtyCardsClosure
4571     greyRescanClosure(_collector, full_span, // entire span of interest
4572                       sp, bm, work_q, cl);
4573 
4574   SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
4575   assert(pst->valid(), "Uninitialized use?");
4576   uint nth_task = 0;
4577   const int alignment = CardTable::card_size * BitsPerWord;
4578   MemRegion span = sp->used_region();
4579   HeapWord* start_addr = span.start();
4580   HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(),
4581                                            alignment);
4582   const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units
4583   assert((HeapWord*)round_to((intptr_t)start_addr, alignment) ==
4584          start_addr, "Check alignment");
4585   assert((size_t)round_to((intptr_t)chunk_size, alignment) ==
4586          chunk_size, "Check alignment");
4587 
4588   while (!pst->is_task_claimed(/* reference */ nth_task)) {
4589     // Having claimed the nth_task, compute corresponding mem-region,
4590     // which is a-fortiori aligned correctly (i.e. at a MUT boundary).
4591     // The alignment restriction ensures that we do not need any
4592     // synchronization with other gang-workers while setting or
4593     // clearing bits in thus chunk of the MUT.
4594     MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size,
4595                                     start_addr + (nth_task+1)*chunk_size);
4596     // The last chunk's end might be way beyond end of the
4597     // used region. In that case pull back appropriately.
4598     if (this_span.end() > end_addr) {
4599       this_span.set_end(end_addr);
4600       assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)");
4601     }
4602     // Iterate over the dirty cards covering this chunk, marking them
4603     // precleaned, and setting the corresponding bits in the mod union
4604     // table. Since we have been careful to partition at Card and MUT-word
4605     // boundaries no synchronization is needed between parallel threads.
4606     _collector->_ct->dirty_card_iterate(this_span,
4607                                                  &modUnionClosure);
4608 
4609     // Having transferred these marks into the modUnionTable,
4610     // rescan the marked objects on the dirty cards in the modUnionTable.
4611     // Even if this is at a synchronous collection, the initial marking
4612     // may have been done during an asynchronous collection so there
4613     // may be dirty bits in the mod-union table.
4614     _collector->_modUnionTable.dirty_range_iterate_clear(
4615                   this_span, &greyRescanClosure);
4616     _collector->_modUnionTable.verifyNoOneBitsInRange(
4617                                  this_span.start(),
4618                                  this_span.end());
4619   }
4620   pst->all_tasks_completed();  // declare that i am done
4621 }
4622 
4623 // . see if we can share work_queues with ParNew? XXX
4624 void
4625 CMSParRemarkTask::do_work_steal(int i, ParMarkRefsIntoAndScanClosure* cl,
4626                                 int* seed) {


4897 void CMSCollector::do_remark_non_parallel() {
4898   ResourceMark rm;
4899   HandleMark   hm;
4900   GenCollectedHeap* gch = GenCollectedHeap::heap();
4901   ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
4902 
4903   MarkRefsIntoAndScanClosure
4904     mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */,
4905              &_markStack, this,
4906              false /* should_yield */, false /* not precleaning */);
4907   MarkFromDirtyCardsClosure
4908     markFromDirtyCardsClosure(this, _span,
4909                               NULL,  // space is set further below
4910                               &_markBitMap, &_markStack, &mrias_cl);
4911   {
4912     GCTraceTime(Trace, gc, phases) t("Grey Object Rescan", _gc_timer_cm);
4913     // Iterate over the dirty cards, setting the corresponding bits in the
4914     // mod union table.
4915     {
4916       ModUnionClosure modUnionClosure(&_modUnionTable);
4917       _ct->dirty_card_iterate(_cmsGen->used_region(),

4918                               &modUnionClosure);
4919     }
4920     // Having transferred these marks into the modUnionTable, we just need
4921     // to rescan the marked objects on the dirty cards in the modUnionTable.
4922     // The initial marking may have been done during an asynchronous
4923     // collection so there may be dirty bits in the mod-union table.
4924     const int alignment = CardTable::card_size * BitsPerWord;

4925     {
4926       // ... First handle dirty cards in CMS gen
4927       markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
4928       MemRegion ur = _cmsGen->used_region();
4929       HeapWord* lb = ur.start();
4930       HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
4931       MemRegion cms_span(lb, ub);
4932       _modUnionTable.dirty_range_iterate_clear(cms_span,
4933                                                &markFromDirtyCardsClosure);
4934       verify_work_stacks_empty();
4935       log_trace(gc)(" (re-scanned " SIZE_FORMAT " dirty cards in cms gen) ", markFromDirtyCardsClosure.num_dirty_cards());
4936     }
4937   }
4938   if (VerifyDuringGC &&
4939       GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4940     HandleMark hm;  // Discard invalid handles created during verification
4941     Universe::verify();
4942   }
4943   {
4944     GCTraceTime(Trace, gc, phases) t("Root Rescan", _gc_timer_cm);


5609     size_t size = pointer_delta(nextOneAddr + 1, addr);
5610     assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
5611            "alignment problem");
5612     assert(size >= 3, "Necessary for Printezis marks to work");
5613     return size;
5614   }
5615   return 0;
5616 }
5617 
5618 HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
5619   size_t sz = 0;
5620   oop p = (oop)addr;
5621   if (p->klass_or_null_acquire() != NULL) {
5622     sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
5623   } else {
5624     sz = block_size_using_printezis_bits(addr);
5625   }
5626   assert(sz > 0, "size must be nonzero");
5627   HeapWord* next_block = addr + sz;
5628   HeapWord* next_card  = (HeapWord*)round_to((uintptr_t)next_block,
5629                                              CardTable::card_size);
5630   assert(round_down((uintptr_t)addr,      CardTable::card_size) <
5631          round_down((uintptr_t)next_card, CardTable::card_size),
5632          "must be different cards");
5633   return next_card;
5634 }
5635 
5636 
5637 // CMS Bit Map Wrapper /////////////////////////////////////////
5638 
5639 // Construct a CMS bit map infrastructure, but don't create the
5640 // bit vector itself. That is done by a separate call CMSBitMap::allocate()
5641 // further below.
5642 CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
5643   _bm(),
5644   _shifter(shifter),
5645   _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true,
5646                                     Monitor::_safepoint_check_sometimes) : NULL)
5647 {
5648   _bmStartWord = 0;
5649   _bmWordSize  = 0;
5650 }
5651 


6271   _span(span),
6272   _bitMap(bitMap),
6273   _mut(&collector->_modUnionTable),
6274   _markStack(markStack),
6275   _yield(should_yield),
6276   _skipBits(0)
6277 {
6278   assert(_markStack->isEmpty(), "stack should be empty");
6279   _finger = _bitMap->startWord();
6280   _threshold = _finger;
6281   assert(_collector->_restart_addr == NULL, "Sanity check");
6282   assert(_span.contains(_finger), "Out of bounds _finger?");
6283   DEBUG_ONLY(_verifying = verifying;)
6284 }
6285 
6286 void MarkFromRootsClosure::reset(HeapWord* addr) {
6287   assert(_markStack->isEmpty(), "would cause duplicates on stack");
6288   assert(_span.contains(addr), "Out of bounds _finger?");
6289   _finger = addr;
6290   _threshold = (HeapWord*)round_to(
6291                  (intptr_t)_finger, CardTable::card_size);
6292 }
6293 
6294 // Should revisit to see if this should be restructured for
6295 // greater efficiency.
6296 bool MarkFromRootsClosure::do_bit(size_t offset) {
6297   if (_skipBits > 0) {
6298     _skipBits--;
6299     return true;
6300   }
6301   // convert offset into a HeapWord*
6302   HeapWord* addr = _bitMap->startWord() + offset;
6303   assert(_bitMap->endWord() && addr < _bitMap->endWord(),
6304          "address out of range");
6305   assert(_bitMap->isMarked(addr), "tautology");
6306   if (_bitMap->isMarked(addr+1)) {
6307     // this is an allocated but not yet initialized object
6308     assert(_skipBits == 0, "tautology");
6309     _skipBits = 2;  // skip next two marked bits ("Printezis-marks")
6310     oop p = oop(addr);
6311     if (p->klass_or_null_acquire() == NULL) {
6312       DEBUG_ONLY(if (!_verifying) {)
6313         // We re-dirty the cards on which this object lies and increase
6314         // the _threshold so that we'll come back to scan this object
6315         // during the preclean or remark phase. (CMSCleanOnEnter)
6316         if (CMSCleanOnEnter) {
6317           size_t sz = _collector->block_size_using_printezis_bits(addr);
6318           HeapWord* end_card_addr   = (HeapWord*)round_to(
6319                                          (intptr_t)(addr+sz), CardTable::card_size);
6320           MemRegion redirty_range = MemRegion(addr, end_card_addr);
6321           assert(!redirty_range.is_empty(), "Arithmetical tautology");
6322           // Bump _threshold to end_card_addr; note that
6323           // _threshold cannot possibly exceed end_card_addr, anyhow.
6324           // This prevents future clearing of the card as the scan proceeds
6325           // to the right.
6326           assert(_threshold <= end_card_addr,
6327                  "Because we are just scanning into this object");
6328           if (_threshold < end_card_addr) {
6329             _threshold = end_card_addr;
6330           }
6331           if (p->klass_or_null_acquire() != NULL) {
6332             // Redirty the range of cards...
6333             _mut->mark_range(redirty_range);
6334           } // ...else the setting of klass will dirty the card anyway.
6335         }
6336       DEBUG_ONLY(})
6337       return true;
6338     }
6339   }


6387   // accumulated in the card table and the mod union table --
6388   // these mutation records are redundant until we have
6389   // actually traced into the corresponding card.
6390   // Here, we check whether advancing the finger would make
6391   // us cross into a new card, and if so clear corresponding
6392   // cards in the MUT (preclean them in the card-table in the
6393   // future).
6394 
6395   DEBUG_ONLY(if (!_verifying) {)
6396     // The clean-on-enter optimization is disabled by default,
6397     // until we fix 6178663.
6398     if (CMSCleanOnEnter && (_finger > _threshold)) {
6399       // [_threshold, _finger) represents the interval
6400       // of cards to be cleared  in MUT (or precleaned in card table).
6401       // The set of cards to be cleared is all those that overlap
6402       // with the interval [_threshold, _finger); note that
6403       // _threshold is always kept card-aligned but _finger isn't
6404       // always card-aligned.
6405       HeapWord* old_threshold = _threshold;
6406       assert(old_threshold == (HeapWord*)round_to(
6407               (intptr_t)old_threshold, CardTable::card_size),
6408              "_threshold should always be card-aligned");
6409       _threshold = (HeapWord*)round_to(
6410                      (intptr_t)_finger, CardTable::card_size);
6411       MemRegion mr(old_threshold, _threshold);
6412       assert(!mr.is_empty(), "Control point invariant");
6413       assert(_span.contains(mr), "Should clear within span");
6414       _mut->clear_range(mr);
6415     }
6416   DEBUG_ONLY(})
6417   // Note: the finger doesn't advance while we drain
6418   // the stack below.
6419   PushOrMarkClosure pushOrMarkClosure(_collector,
6420                                       _span, _bitMap, _markStack,
6421                                       _finger, this);
6422   bool res = _markStack->push(obj);
6423   assert(res, "Empty non-zero size stack should have space for single push");
6424   while (!_markStack->isEmpty()) {
6425     oop new_oop = _markStack->pop();
6426     // Skip verifying header mark word below because we are
6427     // running concurrent with mutators.
6428     assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
6429     // now scan this oop's oops
6430     new_oop->oop_iterate(&pushOrMarkClosure);


6501   // this time it's possible that a lot of mutations have
6502   // accumulated in the card table and the mod union table --
6503   // these mutation records are redundant until we have
6504   // actually traced into the corresponding card.
6505   // Here, we check whether advancing the finger would make
6506   // us cross into a new card, and if so clear corresponding
6507   // cards in the MUT (preclean them in the card-table in the
6508   // future).
6509 
6510   // The clean-on-enter optimization is disabled by default,
6511   // until we fix 6178663.
6512   if (CMSCleanOnEnter && (_finger > _threshold)) {
6513     // [_threshold, _finger) represents the interval
6514     // of cards to be cleared  in MUT (or precleaned in card table).
6515     // The set of cards to be cleared is all those that overlap
6516     // with the interval [_threshold, _finger); note that
6517     // _threshold is always kept card-aligned but _finger isn't
6518     // always card-aligned.
6519     HeapWord* old_threshold = _threshold;
6520     assert(old_threshold == (HeapWord*)round_to(
6521             (intptr_t)old_threshold, CardTable::card_size),
6522            "_threshold should always be card-aligned");
6523     _threshold = (HeapWord*)round_to(
6524                    (intptr_t)_finger, CardTable::card_size);
6525     MemRegion mr(old_threshold, _threshold);
6526     assert(!mr.is_empty(), "Control point invariant");
6527     assert(_span.contains(mr), "Should clear within span"); // _whole_span ??
6528     _mut->clear_range(mr);
6529   }
6530 
6531   // Note: the local finger doesn't advance while we drain
6532   // the stack below, but the global finger sure can and will.
6533   HeapWord* volatile* gfa = _task->global_finger_addr();
6534   ParPushOrMarkClosure pushOrMarkClosure(_collector,
6535                                          _span, _bit_map,
6536                                          _work_queue,
6537                                          _overflow_stack,
6538                                          _finger,
6539                                          gfa, this);
6540   bool res = _work_queue->push(obj);   // overflow could occur here
6541   assert(res, "Will hold once we use workqueues");
6542   while (true) {
6543     oop new_oop;
6544     if (!_work_queue->pop_local(new_oop)) {


6872       if (CMSMarkStackOverflowALot &&
6873           _collector->simulate_overflow()) {
6874         // simulate a stack overflow
6875         simulate_overflow = true;
6876       }
6877     )
6878     if (simulate_overflow || !_mark_stack->push(obj)) {
6879       if (_concurrent_precleaning) {
6880          // During precleaning we can just dirty the appropriate card(s)
6881          // in the mod union table, thus ensuring that the object remains
6882          // in the grey set  and continue. In the case of object arrays
6883          // we need to dirty all of the cards that the object spans,
6884          // since the rescan of object arrays will be limited to the
6885          // dirty cards.
6886          // Note that no one can be interfering with us in this action
6887          // of dirtying the mod union table, so no locking or atomics
6888          // are required.
6889          if (obj->is_objArray()) {
6890            size_t sz = obj->size();
6891            HeapWord* end_card_addr = (HeapWord*)round_to(
6892                                         (intptr_t)(addr+sz), CardTable::card_size);
6893            MemRegion redirty_range = MemRegion(addr, end_card_addr);
6894            assert(!redirty_range.is_empty(), "Arithmetical tautology");
6895            _mod_union_table->mark_range(redirty_range);
6896          } else {
6897            _mod_union_table->mark(addr);
6898          }
6899          _collector->_ser_pmc_preclean_ovflw++;
6900       } else {
6901          // During the remark phase, we need to remember this oop
6902          // in the overflow list.
6903          _collector->push_on_overflow_list(obj);
6904          _collector->_ser_pmc_remark_ovflw++;
6905       }
6906     }
6907   }
6908 }
6909 
6910 ParPushAndMarkClosure::ParPushAndMarkClosure(CMSCollector* collector,
6911                                              MemRegion span,
6912                                              ReferenceProcessor* rp,


6985   for (unsigned i = 0; i < CMSYieldSleepCount &&
6986                        ConcurrentMarkSweepThread::should_yield() &&
6987                        !CMSCollector::foregroundGCIsActive(); ++i) {
6988     os::sleep(Thread::current(), 1, false);
6989   }
6990 
6991   ConcurrentMarkSweepThread::synchronize(true);
6992   bml->lock();
6993 
6994   _collector->startTimer();
6995 }
6996 
6997 bool CMSPrecleanRefsYieldClosure::should_return() {
6998   if (ConcurrentMarkSweepThread::should_yield()) {
6999     do_yield_work();
7000   }
7001   return _collector->foregroundGCIsActive();
7002 }
7003 
7004 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
7005   assert(((size_t)mr.start())%CardTable::card_size_in_words == 0,
7006          "mr should be aligned to start at a card boundary");
7007   // We'd like to assert:
7008   // assert(mr.word_size()%CardTable::card_size_in_words == 0,
7009   //        "mr should be a range of cards");
7010   // However, that would be too strong in one case -- the last
7011   // partition ends at _unallocated_block which, in general, can be
7012   // an arbitrary boundary, not necessarily card aligned.
7013   _num_dirty_cards += mr.word_size()/CardTable::card_size_in_words;
7014   _space->object_iterate_mem(mr, &_scan_cl);
7015 }
7016 
7017 SweepClosure::SweepClosure(CMSCollector* collector,
7018                            ConcurrentMarkSweepGeneration* g,
7019                            CMSBitMap* bitMap, bool should_yield) :
7020   _collector(collector),
7021   _g(g),
7022   _sp(g->cmsSpace()),
7023   _limit(_sp->sweep_limit()),
7024   _freelistLock(_sp->freelistLock()),
7025   _bitMap(bitMap),
7026   _yield(should_yield),
7027   _inFreeRange(false),           // No free range at beginning of sweep
7028   _freeRangeInFreeLists(false),  // No free range at beginning of sweep
7029   _lastFreeRangeCoalesced(false),
7030   _freeFinger(g->used_region().start())
7031 {
7032   NOT_PRODUCT(
7033     _numObjectsFreed = 0;


7602     bool simulate_overflow = false;
7603     NOT_PRODUCT(
7604       if (CMSMarkStackOverflowALot &&
7605           _collector->simulate_overflow()) {
7606         // simulate a stack overflow
7607         simulate_overflow = true;
7608       }
7609     )
7610     if (simulate_overflow || !_mark_stack->push(obj)) {
7611       if (_concurrent_precleaning) {
7612         // We dirty the overflown object and let the remark
7613         // phase deal with it.
7614         assert(_collector->overflow_list_is_empty(), "Error");
7615         // In the case of object arrays, we need to dirty all of
7616         // the cards that the object spans. No locking or atomics
7617         // are needed since no one else can be mutating the mod union
7618         // table.
7619         if (obj->is_objArray()) {
7620           size_t sz = obj->size();
7621           HeapWord* end_card_addr =
7622             (HeapWord*)round_to((intptr_t)(addr+sz), CardTable::card_size);
7623           MemRegion redirty_range = MemRegion(addr, end_card_addr);
7624           assert(!redirty_range.is_empty(), "Arithmetical tautology");
7625           _collector->_modUnionTable.mark_range(redirty_range);
7626         } else {
7627           _collector->_modUnionTable.mark(addr);
7628         }
7629         _collector->_ser_kac_preclean_ovflw++;
7630       } else {
7631         _collector->push_on_overflow_list(obj);
7632         _collector->_ser_kac_ovflw++;
7633       }
7634     }
7635   }
7636 }
7637 
7638 void CMSKeepAliveClosure::do_oop(oop* p)       { CMSKeepAliveClosure::do_oop_work(p); }
7639 void CMSKeepAliveClosure::do_oop(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); }
7640 
7641 // CMSParKeepAliveClosure: a parallel version of the above.
7642 // The work queues are private to each closure (thread),


< prev index next >