558 _permGen(permGen),
559 _ct(ct),
560 _ref_processor(NULL), // will be set later
561 _conc_workers(NULL), // may be set later
562 _abort_preclean(false),
563 _start_sampling(false),
564 _between_prologue_and_epilogue(false),
565 _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
566 _perm_gen_verify_bit_map(0, -1 /* no mutex */, "No_lock"),
567 _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize),
568 -1 /* lock-free */, "No_lock" /* dummy */),
569 _modUnionClosure(&_modUnionTable),
570 _modUnionClosurePar(&_modUnionTable),
571 // Adjust my span to cover old (cms) gen and perm gen
572 _span(cmsGen->reserved()._union(permGen->reserved())),
573 // Construct the is_alive_closure with _span & markBitMap
574 _is_alive_closure(_span, &_markBitMap),
575 _restart_addr(NULL),
576 _overflow_list(NULL),
577 _stats(cmsGen),
578 _eden_chunk_array(NULL), // may be set in ctor body
579 _eden_chunk_capacity(0), // -- ditto --
580 _eden_chunk_index(0), // -- ditto --
581 _survivor_plab_array(NULL), // -- ditto --
582 _survivor_chunk_array(NULL), // -- ditto --
583 _survivor_chunk_capacity(0), // -- ditto --
584 _survivor_chunk_index(0), // -- ditto --
585 _ser_pmc_preclean_ovflw(0),
586 _ser_kac_preclean_ovflw(0),
587 _ser_pmc_remark_ovflw(0),
588 _par_pmc_remark_ovflw(0),
589 _ser_kac_ovflw(0),
590 _par_kac_ovflw(0),
591 #ifndef PRODUCT
592 _num_par_pushes(0),
593 #endif
594 _collection_count_start(0),
595 _verifying(false),
596 _icms_start_limit(NULL),
597 _icms_stop_limit(NULL),
737 assert(CGC_lock != NULL, "Where's the CGC_lock?");
738
739 // Support for parallelizing young gen rescan
740 GenCollectedHeap* gch = GenCollectedHeap::heap();
741 _young_gen = gch->prev_gen(_cmsGen);
742 if (gch->supports_inline_contig_alloc()) {
743 _top_addr = gch->top_addr();
744 _end_addr = gch->end_addr();
745 assert(_young_gen != NULL, "no _young_gen");
746 _eden_chunk_index = 0;
747 _eden_chunk_capacity = (_young_gen->max_capacity()+CMSSamplingGrain)/CMSSamplingGrain;
748 _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity, mtGC);
749 if (_eden_chunk_array == NULL) {
750 _eden_chunk_capacity = 0;
751 warning("GC/CMS: _eden_chunk_array allocation failure");
752 }
753 }
754 assert(_eden_chunk_array != NULL || _eden_chunk_capacity == 0, "Error");
755
756 // Support for parallelizing survivor space rescan
757 if (CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) {
758 const size_t max_plab_samples =
759 ((DefNewGeneration*)_young_gen)->max_survivor_size()/MinTLABSize;
760
761 _survivor_plab_array = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads, mtGC);
762 _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, 2*max_plab_samples, mtGC);
763 _cursor = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads, mtGC);
764 if (_survivor_plab_array == NULL || _survivor_chunk_array == NULL
765 || _cursor == NULL) {
766 warning("Failed to allocate survivor plab/chunk array");
767 if (_survivor_plab_array != NULL) {
768 FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array, mtGC);
769 _survivor_plab_array = NULL;
770 }
771 if (_survivor_chunk_array != NULL) {
772 FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array, mtGC);
773 _survivor_chunk_array = NULL;
774 }
775 if (_cursor != NULL) {
776 FREE_C_HEAP_ARRAY(size_t, _cursor, mtGC);
777 _cursor = NULL;
2119 // restarted from scratch; start the cycle.
2120 _collectorState = InitialMarking;
2121 }
2122 // If first_state was not Idling, then a background GC
2123 // was in progress and has now finished. No need to do it
2124 // again. Leave the state as Idling.
2125 break;
2126 case Precleaning:
2127 // In the foreground case don't do the precleaning since
2128 // it is not done concurrently and there is extra work
2129 // required.
2130 _collectorState = FinalMarking;
2131 }
2132 collect_in_foreground(clear_all_soft_refs, GenCollectedHeap::heap()->gc_cause());
2133
2134 // For a mark-sweep, compute_new_size() will be called
2135 // in the heap's do_collection() method.
2136 }
2137
2138
2139 void CMSCollector::getFreelistLocks() const {
2140 // Get locks for all free lists in all generations that this
2141 // collector is responsible for
2142 _cmsGen->freelistLock()->lock_without_safepoint_check();
2143 _permGen->freelistLock()->lock_without_safepoint_check();
2144 }
2145
2146 void CMSCollector::releaseFreelistLocks() const {
2147 // Release locks for all free lists in all generations that this
2148 // collector is responsible for
2149 _cmsGen->freelistLock()->unlock();
2150 _permGen->freelistLock()->unlock();
2151 }
2152
2153 bool CMSCollector::haveFreelistLocks() const {
2154 // Check locks for all free lists in all generations that this
2155 // collector is responsible for
2156 assert_lock_strong(_cmsGen->freelistLock());
2157 assert_lock_strong(_permGen->freelistLock());
2158 PRODUCT_ONLY(ShouldNotReachHere());
3509 _collector->stopTimer();
3510 _wallclock.stop();
3511 if (PrintGCDetails) {
3512 gclog_or_tty->date_stamp(PrintGCDateStamps);
3513 gclog_or_tty->stamp(PrintGCTimeStamps);
3514 gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]",
3515 _collector->cmsGen()->short_name(),
3516 _phase, _collector->timerValue(), _wallclock.seconds());
3517 if (_print_cr) {
3518 gclog_or_tty->print_cr("");
3519 }
3520 if (PrintCMSStatistics != 0) {
3521 gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase,
3522 _collector->yields());
3523 }
3524 }
3525 }
3526
3527 // CMS work
3528
3529 // Checkpoint the roots into this generation from outside
3530 // this generation. [Note this initial checkpoint need only
3531 // be approximate -- we'll do a catch up phase subsequently.]
3532 void CMSCollector::checkpointRootsInitial(bool asynch) {
3533 assert(_collectorState == InitialMarking, "Wrong collector state");
3534 check_correct_thread_executing();
3535 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
3536
3537 save_heap_summary();
3538 report_heap_summary(GCWhen::BeforeGC);
3539
3540 ReferenceProcessor* rp = ref_processor();
3541 SpecializationStats::clear();
3542 assert(_restart_addr == NULL, "Control point invariant");
3543 if (asynch) {
3544 // acquire locks for subsequent manipulations
3545 MutexLockerEx x(bitMapLock(),
3546 Mutex::_no_safepoint_check_flag);
3547 checkpointRootsInitialWork(asynch);
3548 // enable ("weak") refs discovery
3598
3599 FalseClosure falseClosure;
3600 // In the case of a synchronous collection, we will elide the
3601 // remark step, so it's important to catch all the nmethod oops
3602 // in this step.
3603 // The final 'true' flag to gen_process_strong_roots will ensure this.
3604 // If 'async' is true, we can relax the nmethod tracing.
3605 MarkRefsIntoClosure notOlder(_span, &_markBitMap);
3606 GenCollectedHeap* gch = GenCollectedHeap::heap();
3607
3608 verify_work_stacks_empty();
3609 verify_overflow_empty();
3610
3611 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
3612 // Update the saved marks which may affect the root scans.
3613 gch->save_marks();
3614
3615 // weak reference processing has not started yet.
3616 ref_processor()->set_enqueuing_is_done(false);
3617
3618 {
3619 // This is not needed. DEBUG_ONLY(RememberKlassesChecker imx(true);)
3620 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
3621 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
3622 gch->gen_process_strong_roots(_cmsGen->level(),
3623 true, // younger gens are roots
3624 true, // activate StrongRootsScope
3625 true, // collecting perm gen
3626 SharedHeap::ScanningOption(roots_scanning_options()),
3627 ¬Older,
3628 true, // walk all of code cache if (so & SO_CodeCache)
3629 NULL);
3630 }
3631
3632 // Clear mod-union table; it will be dirtied in the prologue of
3633 // CMS generation per each younger generation collection.
3634
3635 assert(_modUnionTable.isAllClear(),
3636 "Was cleared in most recent final checkpoint phase"
3637 " or no bits are set in the gc_prologue before the start of the next "
3638 "subsequent marking phase.");
3639
3640 // Save the end of the used_region of the constituent generations
3641 // to be used to limit the extent of sweep in each generation.
3642 save_sweep_limits();
3643 if (UseAdaptiveSizePolicy) {
3644 size_policy()->checkpoint_roots_initial_end(gch->gc_cause());
3645 }
3646 verify_overflow_empty();
3647 }
3648
3649 bool CMSCollector::markFromRoots(bool asynch) {
3650 // we might be tempted to assert that:
3651 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
4393 _restart_addr = NULL;
4394 return false; // indicating failure to complete marking
4395 }
4396 // Deal with stack overflow:
4397 // we restart marking from _restart_addr
4398 HeapWord* ra = _restart_addr;
4399 markFromRootsClosure.reset(ra);
4400 _restart_addr = NULL;
4401 _markBitMap.iterate(&markFromRootsClosure, ra, _span.end());
4402 }
4403 return true;
4404 }
4405
4406 void CMSCollector::preclean() {
4407 check_correct_thread_executing();
4408 assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
4409 verify_work_stacks_empty();
4410 verify_overflow_empty();
4411 _abort_preclean = false;
4412 if (CMSPrecleaningEnabled) {
4413 _eden_chunk_index = 0;
4414 size_t used = get_eden_used();
4415 size_t capacity = get_eden_capacity();
4416 // Don't start sampling unless we will get sufficiently
4417 // many samples.
4418 if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
4419 * CMSScheduleRemarkEdenPenetration)) {
4420 _start_sampling = true;
4421 } else {
4422 _start_sampling = false;
4423 }
4424 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
4425 CMSPhaseAccounting pa(this, "preclean", !PrintGCDetails);
4426 preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
4427 }
4428 CMSTokenSync x(true); // is cms thread
4429 if (CMSPrecleaningEnabled) {
4430 sample_eden();
4431 _collectorState = AbortablePreclean;
4432 } else {
4433 _collectorState = FinalMarking;
4502 CMSTokenSync x(true); // is cms thread
4503 if (_collectorState != Idling) {
4504 assert(_collectorState == AbortablePreclean,
4505 "Spontaneous state transition?");
4506 _collectorState = FinalMarking;
4507 } // Else, a foreground collection completed this CMS cycle.
4508 return;
4509 }
4510
4511 // Respond to an Eden sampling opportunity
4512 void CMSCollector::sample_eden() {
4513 // Make sure a young gc cannot sneak in between our
4514 // reading and recording of a sample.
4515 assert(Thread::current()->is_ConcurrentGC_thread(),
4516 "Only the cms thread may collect Eden samples");
4517 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
4518 "Should collect samples while holding CMS token");
4519 if (!_start_sampling) {
4520 return;
4521 }
4522 if (_eden_chunk_array) {
4523 if (_eden_chunk_index < _eden_chunk_capacity) {
4524 _eden_chunk_array[_eden_chunk_index] = *_top_addr; // take sample
4525 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
4526 "Unexpected state of Eden");
4527 // We'd like to check that what we just sampled is an oop-start address;
4528 // however, we cannot do that here since the object may not yet have been
4529 // initialized. So we'll instead do the check when we _use_ this sample
4530 // later.
4531 if (_eden_chunk_index == 0 ||
4532 (pointer_delta(_eden_chunk_array[_eden_chunk_index],
4533 _eden_chunk_array[_eden_chunk_index-1])
4534 >= CMSSamplingGrain)) {
4535 _eden_chunk_index++; // commit sample
4536 }
4537 }
4538 }
4539 if ((_collectorState == AbortablePreclean) && !_abort_preclean) {
4540 size_t used = get_eden_used();
4541 size_t capacity = get_eden_capacity();
4542 assert(used <= capacity, "Unexpected state of Eden");
4979 assert_lock_strong(bitMapLock());
4980
4981 DEBUG_ONLY(RememberKlassesChecker fmx(should_unload_classes());)
4982 if (!init_mark_was_synchronous) {
4983 // We might assume that we need not fill TLAB's when
4984 // CMSScavengeBeforeRemark is set, because we may have just done
4985 // a scavenge which would have filled all TLAB's -- and besides
4986 // Eden would be empty. This however may not always be the case --
4987 // for instance although we asked for a scavenge, it may not have
4988 // happened because of a JNI critical section. We probably need
4989 // a policy for deciding whether we can in that case wait until
4990 // the critical section releases and then do the remark following
4991 // the scavenge, and skip it here. In the absence of that policy,
4992 // or of an indication of whether the scavenge did indeed occur,
4993 // we cannot rely on TLAB's having been filled and must do
4994 // so here just in case a scavenge did not happen.
4995 gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
4996 // Update the saved marks which may affect the root scans.
4997 gch->save_marks();
4998
4999 {
5000 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
5001
5002 // Note on the role of the mod union table:
5003 // Since the marker in "markFromRoots" marks concurrently with
5004 // mutators, it is possible for some reachable objects not to have been
5005 // scanned. For instance, an only reference to an object A was
5006 // placed in object B after the marker scanned B. Unless B is rescanned,
5007 // A would be collected. Such updates to references in marked objects
5008 // are detected via the mod union table which is the set of all cards
5009 // dirtied since the first checkpoint in this GC cycle and prior to
5010 // the most recent young generation GC, minus those cleaned up by the
5011 // concurrent precleaning.
5012 if (CMSParallelRemarkEnabled && CollectedHeap::use_parallel_gc_threads()) {
5013 GCTraceTime t("Rescan (parallel) ", PrintGCDetails, false, _gc_timer_cm);
5014 do_remark_parallel();
5015 } else {
5016 GCTraceTime t("Rescan (non-parallel) ", PrintGCDetails, false,
5017 _gc_timer_cm);
5018 do_remark_non_parallel();
5086 // Check that all the klasses have been checked
5087 assert(_revisitStack.isEmpty(), "Not all klasses revisited");
5088
5089 if ((VerifyAfterGC || VerifyDuringGC) &&
5090 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
5091 verify_after_remark();
5092 }
5093
5094 _gc_tracer_cm->report_object_count_after_gc(&_is_alive_closure);
5095
5096 // Change under the freelistLocks.
5097 _collectorState = Sweeping;
5098 // Call isAllClear() under bitMapLock
5099 assert(_modUnionTable.isAllClear(), "Should be clear by end of the"
5100 " final marking");
5101 if (UseAdaptiveSizePolicy) {
5102 size_policy()->checkpoint_roots_final_end(gch->gc_cause());
5103 }
5104 }
5105
5106 // Parallel remark task
5107 class CMSParRemarkTask: public AbstractGangTask {
5108 CMSCollector* _collector;
5109 int _n_workers;
5110 CompactibleFreeListSpace* _cms_space;
5111 CompactibleFreeListSpace* _perm_space;
5112
5113 // The per-thread work queues, available here for stealing.
5114 OopTaskQueueSet* _task_queues;
5115 ParallelTaskTerminator _term;
5116
5117 public:
5118 // A value of 0 passed to n_workers will cause the number of
5119 // workers to be taken from the active workers in the work gang.
5120 CMSParRemarkTask(CMSCollector* collector,
5121 CompactibleFreeListSpace* cms_space,
5122 CompactibleFreeListSpace* perm_space,
5123 int n_workers, FlexibleWorkGang* workers,
5124 OopTaskQueueSet* task_queues):
5125 AbstractGangTask("Rescan roots and grey objects in parallel"),
5126 _collector(collector),
5127 _cms_space(cms_space), _perm_space(perm_space),
5128 _n_workers(n_workers),
5129 _task_queues(task_queues),
5130 _term(n_workers, task_queues) { }
5131
5132 OopTaskQueueSet* task_queues() { return _task_queues; }
5133
5134 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
5135
5136 ParallelTaskTerminator* terminator() { return &_term; }
5137 int n_workers() { return _n_workers; }
5138
5139 void work(uint worker_id);
5140
5141 private:
5142 // Work method in support of parallel rescan ... of young gen spaces
5143 void do_young_space_rescan(int i, Par_MarkRefsIntoAndScanClosure* cl,
5144 ContiguousSpace* space,
5145 HeapWord** chunk_array, size_t chunk_top);
5146
5147 // ... of dirty cards in old space
5148 void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i,
5149 Par_MarkRefsIntoAndScanClosure* cl);
5150
5151 // ... work stealing for the above
5152 void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed);
5153 };
5154
5155 // work_queue(i) is passed to the closure
5156 // Par_MarkRefsIntoAndScanClosure. The "i" parameter
5157 // also is passed to do_dirty_card_rescan_tasks() and to
5158 // do_work_steal() to select the i-th task_queue.
5159
5160 void CMSParRemarkTask::work(uint worker_id) {
5161 elapsedTimer _timer;
5162 ResourceMark rm;
5163 HandleMark hm;
5164
5165 // ---------- rescan from roots --------------
5166 _timer.start();
5167 GenCollectedHeap* gch = GenCollectedHeap::heap();
5168 Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
5169 _collector->_span, _collector->ref_processor(),
5170 &(_collector->_markBitMap),
5171 work_queue(worker_id), &(_collector->_revisitStack));
5172
5173 // Rescan young gen roots first since these are likely
5174 // coarsely partitioned and may, on that account, constitute
5175 // the critical path; thus, it's best to start off that
5176 // work first.
5177 // ---------- young gen roots --------------
5178 {
5179 DefNewGeneration* dng = _collector->_young_gen->as_DefNewGeneration();
5180 EdenSpace* eden_space = dng->eden();
5181 ContiguousSpace* from_space = dng->from();
5182 ContiguousSpace* to_space = dng->to();
5183
5184 HeapWord** eca = _collector->_eden_chunk_array;
5185 size_t ect = _collector->_eden_chunk_index;
5186 HeapWord** sca = _collector->_survivor_chunk_array;
5187 size_t sct = _collector->_survivor_chunk_index;
5188
5189 assert(ect <= _collector->_eden_chunk_capacity, "out of bounds");
5190 assert(sct <= _collector->_survivor_chunk_capacity, "out of bounds");
5191
5192 do_young_space_rescan(worker_id, &par_mrias_cl, to_space, NULL, 0);
5193 do_young_space_rescan(worker_id, &par_mrias_cl, from_space, sca, sct);
5194 do_young_space_rescan(worker_id, &par_mrias_cl, eden_space, eca, ect);
5195
5196 _timer.stop();
5197 if (PrintCMSStatistics != 0) {
5198 gclog_or_tty->print_cr(
5199 "Finished young gen rescan work in %dth thread: %3.3f sec",
5200 worker_id, _timer.seconds());
5201 }
5202 }
5203
5204 // ---------- remaining roots --------------
5205 _timer.reset();
5206 _timer.start();
5207 gch->gen_process_strong_roots(_collector->_cmsGen->level(),
5208 false, // yg was scanned above
5209 false, // this is parallel code
5210 true, // collecting perm gen
5211 SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
5212 &par_mrias_cl,
5213 true, // walk all of code cache if (so & SO_CodeCache)
5214 NULL);
5215 assert(_collector->should_unload_classes()
5236 gclog_or_tty->print_cr(
5237 "Finished dirty card rescan work in %dth thread: %3.3f sec",
5238 worker_id, _timer.seconds());
5239 }
5240
5241 // ---------- steal work from other threads ...
5242 // ---------- ... and drain overflow list.
5243 _timer.reset();
5244 _timer.start();
5245 do_work_steal(worker_id, &par_mrias_cl, _collector->hash_seed(worker_id));
5246 _timer.stop();
5247 if (PrintCMSStatistics != 0) {
5248 gclog_or_tty->print_cr(
5249 "Finished work stealing in %dth thread: %3.3f sec",
5250 worker_id, _timer.seconds());
5251 }
5252 }
5253
5254 // Note that parameter "i" is not used.
5255 void
5256 CMSParRemarkTask::do_young_space_rescan(int i,
5257 Par_MarkRefsIntoAndScanClosure* cl, ContiguousSpace* space,
5258 HeapWord** chunk_array, size_t chunk_top) {
5259 // Until all tasks completed:
5260 // . claim an unclaimed task
5261 // . compute region boundaries corresponding to task claimed
5262 // using chunk_array
5263 // . par_oop_iterate(cl) over that region
5264
5265 ResourceMark rm;
5266 HandleMark hm;
5267
5268 SequentialSubTasksDone* pst = space->par_seq_tasks();
5269 assert(pst->valid(), "Uninitialized use?");
5270
5271 uint nth_task = 0;
5272 uint n_tasks = pst->n_tasks();
5273
5274 HeapWord *start, *end;
5275 while (!pst->is_task_claimed(/* reference */ nth_task)) {
5276 // We claimed task # nth_task; compute its boundaries.
5277 if (chunk_top == 0) { // no samples were taken
5433 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5434 NOT_PRODUCT(num_steals++;)
5435 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5436 assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5437 // Do scanning work
5438 obj_to_scan->oop_iterate(cl);
5439 // Loop around, finish this work, and try to steal some more
5440 } else if (terminator()->offer_termination()) {
5441 break; // nirvana from the infinite cycle
5442 }
5443 }
5444 NOT_PRODUCT(
5445 if (PrintCMSStatistics != 0) {
5446 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
5447 }
5448 )
5449 assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
5450 "Else our work is not yet done");
5451 }
5452
5453 // Return a thread-local PLAB recording array, as appropriate.
5454 void* CMSCollector::get_data_recorder(int thr_num) {
5455 if (_survivor_plab_array != NULL &&
5456 (CMSPLABRecordAlways ||
5457 (_collectorState > Marking && _collectorState < FinalMarking))) {
5458 assert(thr_num < (int)ParallelGCThreads, "thr_num is out of bounds");
5459 ChunkArray* ca = &_survivor_plab_array[thr_num];
5460 ca->reset(); // clear it so that fresh data is recorded
5461 return (void*) ca;
5462 } else {
5463 return NULL;
5464 }
5465 }
5466
5467 // Reset all the thread-local PLAB recording arrays
5468 void CMSCollector::reset_survivor_plab_arrays() {
5469 for (uint i = 0; i < ParallelGCThreads; i++) {
5470 _survivor_plab_array[i].reset();
5471 }
5472 }
5473
5474 // Merge the per-thread plab arrays into the global survivor chunk
5475 // array which will provide the partitioning of the survivor space
5476 // for CMS rescan.
5477 void CMSCollector::merge_survivor_plab_arrays(ContiguousSpace* surv,
5478 int no_of_gc_threads) {
5479 assert(_survivor_plab_array != NULL, "Error");
5480 assert(_survivor_chunk_array != NULL, "Error");
5481 assert(_collectorState == FinalMarking, "Error");
5482 for (int j = 0; j < no_of_gc_threads; j++) {
5483 _cursor[j] = 0;
5484 }
5485 HeapWord* top = surv->top();
5486 size_t i;
5487 for (i = 0; i < _survivor_chunk_capacity; i++) { // all sca entries
5488 HeapWord* min_val = top; // Higher than any PLAB address
5489 uint min_tid = 0; // position of min_val this round
5490 for (int j = 0; j < no_of_gc_threads; j++) {
5491 ChunkArray* cur_sca = &_survivor_plab_array[j];
5492 if (_cursor[j] == cur_sca->end()) {
5493 continue;
5494 }
5495 assert(_cursor[j] < cur_sca->end(), "ctl pt invariant");
5496 HeapWord* cur_val = cur_sca->nth(_cursor[j]);
5497 assert(surv->used_region().contains(cur_val), "Out of bounds value");
5498 if (cur_val < min_val) {
5499 min_tid = j;
5500 min_val = cur_val;
5501 } else {
5524 for (int j = 0; j < no_of_gc_threads; j++) {
5525 assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
5526 total += _cursor[j];
5527 }
5528 assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
5529 // Check that the merged array is in sorted order
5530 if (total > 0) {
5531 for (size_t i = 0; i < total - 1; i++) {
5532 if (PrintCMSStatistics > 0) {
5533 gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
5534 i, _survivor_chunk_array[i]);
5535 }
5536 assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
5537 "Not sorted");
5538 }
5539 }
5540 #endif // ASSERT
5541 }
5542
5543 // Set up the space's par_seq_tasks structure for work claiming
5544 // for parallel rescan of young gen.
5545 // See ParRescanTask where this is currently used.
5546 void
5547 CMSCollector::
5548 initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
5549 assert(n_threads > 0, "Unexpected n_threads argument");
5550 DefNewGeneration* dng = (DefNewGeneration*)_young_gen;
5551
5552 // Eden space
5553 {
5554 SequentialSubTasksDone* pst = dng->eden()->par_seq_tasks();
5555 assert(!pst->valid(), "Clobbering existing data?");
5556 // Each valid entry in [0, _eden_chunk_index) represents a task.
5557 size_t n_tasks = _eden_chunk_index + 1;
5558 assert(n_tasks == 1 || _eden_chunk_array != NULL, "Error");
5559 // Sets the condition for completion of the subtask (how many threads
5560 // need to finish in order to be done).
5561 pst->set_n_threads(n_threads);
5562 pst->set_n_tasks((int)n_tasks);
5563 }
5564
6674 _span(span),
6675 _bitMap(bitMap)
6676 {
6677 assert(_ref_processor == NULL, "deliberately left NULL");
6678 assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
6679 }
6680
6681 void MarkRefsIntoClosure::do_oop(oop obj) {
6682 // if p points into _span, then mark corresponding bit in _markBitMap
6683 assert(obj->is_oop(), "expected an oop");
6684 HeapWord* addr = (HeapWord*)obj;
6685 if (_span.contains(addr)) {
6686 // this should be made more efficient
6687 _bitMap->mark(addr);
6688 }
6689 }
6690
6691 void MarkRefsIntoClosure::do_oop(oop* p) { MarkRefsIntoClosure::do_oop_work(p); }
6692 void MarkRefsIntoClosure::do_oop(narrowOop* p) { MarkRefsIntoClosure::do_oop_work(p); }
6693
6694 // A variant of the above, used for CMS marking verification.
6695 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
6696 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
6697 _span(span),
6698 _verification_bm(verification_bm),
6699 _cms_bm(cms_bm)
6700 {
6701 assert(_ref_processor == NULL, "deliberately left NULL");
6702 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
6703 }
6704
6705 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
6706 // if p points into _span, then mark corresponding bit in _markBitMap
6707 assert(obj->is_oop(), "expected an oop");
6708 HeapWord* addr = (HeapWord*)obj;
6709 if (_span.contains(addr)) {
6710 _verification_bm->mark(addr);
6711 if (!_cms_bm->isMarked(addr)) {
6712 oop(addr)->print();
6713 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", addr);
9339 if (UsePerfData) {
9340 _space_counters->update_capacity();
9341 _gen_counters->update_all();
9342 }
9343
9344 if (Verbose && PrintGCDetails) {
9345 size_t new_mem_size = _virtual_space.committed_size();
9346 size_t old_mem_size = new_mem_size + bytes;
9347 gclog_or_tty->print_cr("Shrinking %s from %ldK by %ldK to %ldK",
9348 name(), old_mem_size/K, bytes/K, new_mem_size/K);
9349 }
9350 }
9351
9352 assert(_cmsSpace->unallocated_block() <= _cmsSpace->end(),
9353 "Inconsistency at end of space");
9354 assert(chunk_at_end->end() == _cmsSpace->end(),
9355 "Shrinking is inconsistent");
9356 return;
9357 }
9358 }
9359
9360 // Transfer some number of overflown objects to usual marking
9361 // stack. Return true if some objects were transferred.
9362 bool MarkRefsIntoAndScanClosure::take_from_overflow_list() {
9363 size_t num = MIN2((size_t)(_mark_stack->capacity() - _mark_stack->length())/4,
9364 (size_t)ParGCDesiredObjsFromOverflowList);
9365
9366 bool res = _collector->take_from_overflow_list(num, _mark_stack);
9367 assert(_collector->overflow_list_is_empty() || res,
9368 "If list is not empty, we should have taken something");
9369 assert(!res || !_mark_stack->isEmpty(),
9370 "If we took something, it should now be on our stack");
9371 return res;
9372 }
9373
9374 size_t MarkDeadObjectsClosure::do_blk(HeapWord* addr) {
9375 size_t res = _sp->block_size_no_stall(addr, _collector);
9376 if (_sp->block_is_obj(addr)) {
9377 if (_live_bit_map->isMarked(addr)) {
9378 // It can't have been dead in a previous cycle
9379 guarantee(!_dead_bit_map->isMarked(addr), "No resurrection!");
|
558 _permGen(permGen),
559 _ct(ct),
560 _ref_processor(NULL), // will be set later
561 _conc_workers(NULL), // may be set later
562 _abort_preclean(false),
563 _start_sampling(false),
564 _between_prologue_and_epilogue(false),
565 _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
566 _perm_gen_verify_bit_map(0, -1 /* no mutex */, "No_lock"),
567 _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize),
568 -1 /* lock-free */, "No_lock" /* dummy */),
569 _modUnionClosure(&_modUnionTable),
570 _modUnionClosurePar(&_modUnionTable),
571 // Adjust my span to cover old (cms) gen and perm gen
572 _span(cmsGen->reserved()._union(permGen->reserved())),
573 // Construct the is_alive_closure with _span & markBitMap
574 _is_alive_closure(_span, &_markBitMap),
575 _restart_addr(NULL),
576 _overflow_list(NULL),
577 _stats(cmsGen),
578 _eden_chunk_lock(new Mutex(Mutex::leaf + 1, "CMS_eden_chunk_lock", true)),
579 _eden_chunk_array(NULL), // may be set in ctor body
580 _eden_chunk_capacity(0), // -- ditto --
581 _eden_chunk_index(0), // -- ditto --
582 _survivor_plab_array(NULL), // -- ditto --
583 _survivor_chunk_array(NULL), // -- ditto --
584 _survivor_chunk_capacity(0), // -- ditto --
585 _survivor_chunk_index(0), // -- ditto --
586 _ser_pmc_preclean_ovflw(0),
587 _ser_kac_preclean_ovflw(0),
588 _ser_pmc_remark_ovflw(0),
589 _par_pmc_remark_ovflw(0),
590 _ser_kac_ovflw(0),
591 _par_kac_ovflw(0),
592 #ifndef PRODUCT
593 _num_par_pushes(0),
594 #endif
595 _collection_count_start(0),
596 _verifying(false),
597 _icms_start_limit(NULL),
598 _icms_stop_limit(NULL),
738 assert(CGC_lock != NULL, "Where's the CGC_lock?");
739
740 // Support for parallelizing young gen rescan
741 GenCollectedHeap* gch = GenCollectedHeap::heap();
742 _young_gen = gch->prev_gen(_cmsGen);
743 if (gch->supports_inline_contig_alloc()) {
744 _top_addr = gch->top_addr();
745 _end_addr = gch->end_addr();
746 assert(_young_gen != NULL, "no _young_gen");
747 _eden_chunk_index = 0;
748 _eden_chunk_capacity = (_young_gen->max_capacity()+CMSSamplingGrain)/CMSSamplingGrain;
749 _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity, mtGC);
750 if (_eden_chunk_array == NULL) {
751 _eden_chunk_capacity = 0;
752 warning("GC/CMS: _eden_chunk_array allocation failure");
753 }
754 }
755 assert(_eden_chunk_array != NULL || _eden_chunk_capacity == 0, "Error");
756
757 // Support for parallelizing survivor space rescan
758 if ((CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) || CMSParallelInitialMarkEnabled) {
759 const size_t max_plab_samples =
760 ((DefNewGeneration*)_young_gen)->max_survivor_size()/MinTLABSize;
761
762 _survivor_plab_array = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads, mtGC);
763 _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, 2*max_plab_samples, mtGC);
764 _cursor = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads, mtGC);
765 if (_survivor_plab_array == NULL || _survivor_chunk_array == NULL
766 || _cursor == NULL) {
767 warning("Failed to allocate survivor plab/chunk array");
768 if (_survivor_plab_array != NULL) {
769 FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array, mtGC);
770 _survivor_plab_array = NULL;
771 }
772 if (_survivor_chunk_array != NULL) {
773 FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array, mtGC);
774 _survivor_chunk_array = NULL;
775 }
776 if (_cursor != NULL) {
777 FREE_C_HEAP_ARRAY(size_t, _cursor, mtGC);
778 _cursor = NULL;
2120 // restarted from scratch; start the cycle.
2121 _collectorState = InitialMarking;
2122 }
2123 // If first_state was not Idling, then a background GC
2124 // was in progress and has now finished. No need to do it
2125 // again. Leave the state as Idling.
2126 break;
2127 case Precleaning:
2128 // In the foreground case don't do the precleaning since
2129 // it is not done concurrently and there is extra work
2130 // required.
2131 _collectorState = FinalMarking;
2132 }
2133 collect_in_foreground(clear_all_soft_refs, GenCollectedHeap::heap()->gc_cause());
2134
2135 // For a mark-sweep, compute_new_size() will be called
2136 // in the heap's do_collection() method.
2137 }
2138
2139
2140 void CMSCollector::print_eden_and_survivor_chunk_arrays() {
2141 DefNewGeneration* dng = _young_gen->as_DefNewGeneration();
2142 EdenSpace* eden_space = dng->eden();
2143 ContiguousSpace* from_space = dng->from();
2144 ContiguousSpace* to_space = dng->to();
2145 // Eden
2146 if (_eden_chunk_array != NULL) {
2147 gclog_or_tty->print_cr("eden " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
2148 eden_space->bottom(), eden_space->top(),
2149 eden_space->end(), eden_space->capacity());
2150 gclog_or_tty->print_cr("_eden_chunk_index=" SIZE_FORMAT ", "
2151 "_eden_chunk_capacity=" SIZE_FORMAT,
2152 _eden_chunk_index, _eden_chunk_capacity);
2153 for (size_t i = 0; i < _eden_chunk_index; i++) {
2154 gclog_or_tty->print_cr("_eden_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT,
2155 i, _eden_chunk_array[i]);
2156 }
2157 }
2158 // Survivor
2159 if (_survivor_chunk_array != NULL) {
2160 gclog_or_tty->print_cr("survivor " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
2161 from_space->bottom(), from_space->top(),
2162 from_space->end(), from_space->capacity());
2163 gclog_or_tty->print_cr("_survivor_chunk_index=" SIZE_FORMAT ", "
2164 "_survivor_chunk_capacity=" SIZE_FORMAT,
2165 _survivor_chunk_index, _survivor_chunk_capacity);
2166 for (size_t i = 0; i < _survivor_chunk_index; i++) {
2167 gclog_or_tty->print_cr("_survivor_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT,
2168 i, _survivor_chunk_array[i]);
2169 }
2170 }
2171 }
2172
2173 void CMSCollector::getFreelistLocks() const {
2174 // Get locks for all free lists in all generations that this
2175 // collector is responsible for
2176 _cmsGen->freelistLock()->lock_without_safepoint_check();
2177 _permGen->freelistLock()->lock_without_safepoint_check();
2178 }
2179
2180 void CMSCollector::releaseFreelistLocks() const {
2181 // Release locks for all free lists in all generations that this
2182 // collector is responsible for
2183 _cmsGen->freelistLock()->unlock();
2184 _permGen->freelistLock()->unlock();
2185 }
2186
2187 bool CMSCollector::haveFreelistLocks() const {
2188 // Check locks for all free lists in all generations that this
2189 // collector is responsible for
2190 assert_lock_strong(_cmsGen->freelistLock());
2191 assert_lock_strong(_permGen->freelistLock());
2192 PRODUCT_ONLY(ShouldNotReachHere());
3543 _collector->stopTimer();
3544 _wallclock.stop();
3545 if (PrintGCDetails) {
3546 gclog_or_tty->date_stamp(PrintGCDateStamps);
3547 gclog_or_tty->stamp(PrintGCTimeStamps);
3548 gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]",
3549 _collector->cmsGen()->short_name(),
3550 _phase, _collector->timerValue(), _wallclock.seconds());
3551 if (_print_cr) {
3552 gclog_or_tty->print_cr("");
3553 }
3554 if (PrintCMSStatistics != 0) {
3555 gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase,
3556 _collector->yields());
3557 }
3558 }
3559 }
3560
3561 // CMS work
3562
3563 // The common parts of CMSParInitialMarkTask and CMSParRemarkTask.
3564 class CMSParMarkTask : public AbstractGangTask {
3565 protected:
3566 CMSCollector* _collector;
3567 int _n_workers;
3568 CMSParMarkTask(const char* name, CMSCollector* collector, int n_workers) :
3569 AbstractGangTask(name),
3570 _collector(collector),
3571 _n_workers(n_workers) {}
3572 // Work method in support of parallel rescan ... of young gen spaces
3573 void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl,
3574 ContiguousSpace* space,
3575 HeapWord** chunk_array, size_t chunk_top);
3576 void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl);
3577 };
3578
3579 // Parallel initial mark task
3580 class CMSParInitialMarkTask: public CMSParMarkTask {
3581 public:
3582 CMSParInitialMarkTask(CMSCollector* collector, int n_workers) :
3583 CMSParMarkTask("Scan roots and young gen for initial mark in parallel",
3584 collector, n_workers) {}
3585 void work(uint worker_id);
3586 };
3587
3588 // Checkpoint the roots into this generation from outside
3589 // this generation. [Note this initial checkpoint need only
3590 // be approximate -- we'll do a catch up phase subsequently.]
3591 void CMSCollector::checkpointRootsInitial(bool asynch) {
3592 assert(_collectorState == InitialMarking, "Wrong collector state");
3593 check_correct_thread_executing();
3594 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
3595
3596 save_heap_summary();
3597 report_heap_summary(GCWhen::BeforeGC);
3598
3599 ReferenceProcessor* rp = ref_processor();
3600 SpecializationStats::clear();
3601 assert(_restart_addr == NULL, "Control point invariant");
3602 if (asynch) {
3603 // acquire locks for subsequent manipulations
3604 MutexLockerEx x(bitMapLock(),
3605 Mutex::_no_safepoint_check_flag);
3606 checkpointRootsInitialWork(asynch);
3607 // enable ("weak") refs discovery
3657
3658 FalseClosure falseClosure;
3659 // In the case of a synchronous collection, we will elide the
3660 // remark step, so it's important to catch all the nmethod oops
3661 // in this step.
3662 // The final 'true' flag to gen_process_strong_roots will ensure this.
3663 // If 'async' is true, we can relax the nmethod tracing.
3664 MarkRefsIntoClosure notOlder(_span, &_markBitMap);
3665 GenCollectedHeap* gch = GenCollectedHeap::heap();
3666
3667 verify_work_stacks_empty();
3668 verify_overflow_empty();
3669
3670 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
3671 // Update the saved marks which may affect the root scans.
3672 gch->save_marks();
3673
3674 // weak reference processing has not started yet.
3675 ref_processor()->set_enqueuing_is_done(false);
3676
3677 if (CMSPrintEdenSurvivorChunks) {
3678 print_eden_and_survivor_chunk_arrays();
3679 }
3680
3681 {
3682 // This is not needed. DEBUG_ONLY(RememberKlassesChecker imx(true);)
3683 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
3684 if (CMSParallelInitialMarkEnabled && CollectedHeap::use_parallel_gc_threads()) {
3685 // The parallel version.
3686 FlexibleWorkGang* workers = gch->workers();
3687 assert(workers != NULL, "Need parallel worker threads.");
3688 int n_workers = workers->active_workers();
3689 CMSParInitialMarkTask tsk(this, n_workers);
3690 gch->set_par_threads(n_workers);
3691 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
3692 if (n_workers > 1) {
3693 GenCollectedHeap::StrongRootsScope srs(gch);
3694 workers->run_task(&tsk);
3695 } else {
3696 GenCollectedHeap::StrongRootsScope srs(gch);
3697 tsk.work(0);
3698 }
3699 gch->set_par_threads(0);
3700 } else {
3701 // The serial version.
3702 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
3703 gch->gen_process_strong_roots(_cmsGen->level(),
3704 true, // younger gens are roots
3705 true, // activate StrongRootsScope
3706 true, // collecting perm gen
3707 SharedHeap::ScanningOption(roots_scanning_options()),
3708 ¬Older,
3709 true, // walk all of code cache if (so & SO_CodeCache)
3710 NULL);
3711 }
3712 }
3713 // Clear mod-union table; it will be dirtied in the prologue of
3714 // CMS generation per each younger generation collection.
3715
3716 assert(_modUnionTable.isAllClear(),
3717 "Was cleared in most recent final checkpoint phase"
3718 " or no bits are set in the gc_prologue before the start of the next "
3719 "subsequent marking phase.");
3720
3721 // Save the end of the used_region of the constituent generations
3722 // to be used to limit the extent of sweep in each generation.
3723 save_sweep_limits();
3724 if (UseAdaptiveSizePolicy) {
3725 size_policy()->checkpoint_roots_initial_end(gch->gc_cause());
3726 }
3727 verify_overflow_empty();
3728 }
3729
3730 bool CMSCollector::markFromRoots(bool asynch) {
3731 // we might be tempted to assert that:
3732 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
4474 _restart_addr = NULL;
4475 return false; // indicating failure to complete marking
4476 }
4477 // Deal with stack overflow:
4478 // we restart marking from _restart_addr
4479 HeapWord* ra = _restart_addr;
4480 markFromRootsClosure.reset(ra);
4481 _restart_addr = NULL;
4482 _markBitMap.iterate(&markFromRootsClosure, ra, _span.end());
4483 }
4484 return true;
4485 }
4486
4487 void CMSCollector::preclean() {
4488 check_correct_thread_executing();
4489 assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
4490 verify_work_stacks_empty();
4491 verify_overflow_empty();
4492 _abort_preclean = false;
4493 if (CMSPrecleaningEnabled) {
4494 if (!CMSEdenChunksRecordAlways) {
4495 _eden_chunk_index = 0;
4496 }
4497 size_t used = get_eden_used();
4498 size_t capacity = get_eden_capacity();
4499 // Don't start sampling unless we will get sufficiently
4500 // many samples.
4501 if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
4502 * CMSScheduleRemarkEdenPenetration)) {
4503 _start_sampling = true;
4504 } else {
4505 _start_sampling = false;
4506 }
4507 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
4508 CMSPhaseAccounting pa(this, "preclean", !PrintGCDetails);
4509 preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
4510 }
4511 CMSTokenSync x(true); // is cms thread
4512 if (CMSPrecleaningEnabled) {
4513 sample_eden();
4514 _collectorState = AbortablePreclean;
4515 } else {
4516 _collectorState = FinalMarking;
4585 CMSTokenSync x(true); // is cms thread
4586 if (_collectorState != Idling) {
4587 assert(_collectorState == AbortablePreclean,
4588 "Spontaneous state transition?");
4589 _collectorState = FinalMarking;
4590 } // Else, a foreground collection completed this CMS cycle.
4591 return;
4592 }
4593
4594 // Respond to an Eden sampling opportunity
4595 void CMSCollector::sample_eden() {
4596 // Make sure a young gc cannot sneak in between our
4597 // reading and recording of a sample.
4598 assert(Thread::current()->is_ConcurrentGC_thread(),
4599 "Only the cms thread may collect Eden samples");
4600 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
4601 "Should collect samples while holding CMS token");
4602 if (!_start_sampling) {
4603 return;
4604 }
4605 // When CMSEdenChunksRecordAlways is true, the eden chunk array
4606 // is populated by the young generation.
4607 if (_eden_chunk_array != NULL && !CMSEdenChunksRecordAlways) {
4608 if (_eden_chunk_index < _eden_chunk_capacity) {
4609 _eden_chunk_array[_eden_chunk_index] = *_top_addr; // take sample
4610 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
4611 "Unexpected state of Eden");
4612 // We'd like to check that what we just sampled is an oop-start address;
4613 // however, we cannot do that here since the object may not yet have been
4614 // initialized. So we'll instead do the check when we _use_ this sample
4615 // later.
4616 if (_eden_chunk_index == 0 ||
4617 (pointer_delta(_eden_chunk_array[_eden_chunk_index],
4618 _eden_chunk_array[_eden_chunk_index-1])
4619 >= CMSSamplingGrain)) {
4620 _eden_chunk_index++; // commit sample
4621 }
4622 }
4623 }
4624 if ((_collectorState == AbortablePreclean) && !_abort_preclean) {
4625 size_t used = get_eden_used();
4626 size_t capacity = get_eden_capacity();
4627 assert(used <= capacity, "Unexpected state of Eden");
5064 assert_lock_strong(bitMapLock());
5065
5066 DEBUG_ONLY(RememberKlassesChecker fmx(should_unload_classes());)
5067 if (!init_mark_was_synchronous) {
5068 // We might assume that we need not fill TLAB's when
5069 // CMSScavengeBeforeRemark is set, because we may have just done
5070 // a scavenge which would have filled all TLAB's -- and besides
5071 // Eden would be empty. This however may not always be the case --
5072 // for instance although we asked for a scavenge, it may not have
5073 // happened because of a JNI critical section. We probably need
5074 // a policy for deciding whether we can in that case wait until
5075 // the critical section releases and then do the remark following
5076 // the scavenge, and skip it here. In the absence of that policy,
5077 // or of an indication of whether the scavenge did indeed occur,
5078 // we cannot rely on TLAB's having been filled and must do
5079 // so here just in case a scavenge did not happen.
5080 gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
5081 // Update the saved marks which may affect the root scans.
5082 gch->save_marks();
5083
5084 if (CMSPrintEdenSurvivorChunks) {
5085 print_eden_and_survivor_chunk_arrays();
5086 }
5087
5088 {
5089 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
5090
5091 // Note on the role of the mod union table:
5092 // Since the marker in "markFromRoots" marks concurrently with
5093 // mutators, it is possible for some reachable objects not to have been
5094 // scanned. For instance, an only reference to an object A was
5095 // placed in object B after the marker scanned B. Unless B is rescanned,
5096 // A would be collected. Such updates to references in marked objects
5097 // are detected via the mod union table which is the set of all cards
5098 // dirtied since the first checkpoint in this GC cycle and prior to
5099 // the most recent young generation GC, minus those cleaned up by the
5100 // concurrent precleaning.
5101 if (CMSParallelRemarkEnabled && CollectedHeap::use_parallel_gc_threads()) {
5102 GCTraceTime t("Rescan (parallel) ", PrintGCDetails, false, _gc_timer_cm);
5103 do_remark_parallel();
5104 } else {
5105 GCTraceTime t("Rescan (non-parallel) ", PrintGCDetails, false,
5106 _gc_timer_cm);
5107 do_remark_non_parallel();
5175 // Check that all the klasses have been checked
5176 assert(_revisitStack.isEmpty(), "Not all klasses revisited");
5177
5178 if ((VerifyAfterGC || VerifyDuringGC) &&
5179 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
5180 verify_after_remark();
5181 }
5182
5183 _gc_tracer_cm->report_object_count_after_gc(&_is_alive_closure);
5184
5185 // Change under the freelistLocks.
5186 _collectorState = Sweeping;
5187 // Call isAllClear() under bitMapLock
5188 assert(_modUnionTable.isAllClear(), "Should be clear by end of the"
5189 " final marking");
5190 if (UseAdaptiveSizePolicy) {
5191 size_policy()->checkpoint_roots_final_end(gch->gc_cause());
5192 }
5193 }
5194
5195 void CMSParInitialMarkTask::work(uint worker_id) {
5196 elapsedTimer _timer;
5197 ResourceMark rm;
5198 HandleMark hm;
5199
5200 // ---------- scan from roots --------------
5201 _timer.start();
5202 GenCollectedHeap* gch = GenCollectedHeap::heap();
5203 Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap));
5204
5205 // ---------- young gen roots --------------
5206 {
5207 work_on_young_gen_roots(worker_id, &par_mri_cl);
5208 _timer.stop();
5209 if (PrintCMSStatistics != 0) {
5210 gclog_or_tty->print_cr(
5211 "Finished young gen initial mark scan work in %dth thread: %3.3f sec",
5212 worker_id, _timer.seconds());
5213 }
5214 }
5215
5216 // ---------- remaining roots --------------
5217 _timer.reset();
5218 _timer.start();
5219 gch->gen_process_strong_roots(_collector->_cmsGen->level(),
5220 false, // yg was scanned above
5221 false, // this is parallel code
5222 true, // collecting perm gen
5223 SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
5224 &par_mri_cl,
5225 true, // walk all of code cache if (so & SO_CodeCache)
5226 NULL);
5227 assert(_collector->should_unload_classes()
5228 || (_collector->CMSCollector::roots_scanning_options() & SharedHeap::SO_CodeCache),
5229 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
5230 _timer.stop();
5231 if (PrintCMSStatistics != 0) {
5232 gclog_or_tty->print_cr(
5233 "Finished remaining root initial mark scan work in %dth thread: %3.3f sec",
5234 worker_id, _timer.seconds());
5235 }
5236 }
5237
5238 // Parallel remark task
5239 class CMSParRemarkTask: public CMSParMarkTask {
5240 CompactibleFreeListSpace* _cms_space;
5241 CompactibleFreeListSpace* _perm_space;
5242
5243 // The per-thread work queues, available here for stealing.
5244 OopTaskQueueSet* _task_queues;
5245 ParallelTaskTerminator _term;
5246
5247 public:
5248 // A value of 0 passed to n_workers will cause the number of
5249 // workers to be taken from the active workers in the work gang.
5250 CMSParRemarkTask(CMSCollector* collector,
5251 CompactibleFreeListSpace* cms_space,
5252 CompactibleFreeListSpace* perm_space,
5253 int n_workers, FlexibleWorkGang* workers,
5254 OopTaskQueueSet* task_queues):
5255 CMSParMarkTask("Rescan roots and grey objects in parallel",
5256 collector, n_workers),
5257 _cms_space(cms_space), _perm_space(perm_space),
5258 _task_queues(task_queues),
5259 _term(n_workers, task_queues) { }
5260
5261 OopTaskQueueSet* task_queues() { return _task_queues; }
5262
5263 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
5264
5265 ParallelTaskTerminator* terminator() { return &_term; }
5266 int n_workers() { return _n_workers; }
5267
5268 void work(uint worker_id);
5269
5270 private:
5271 // ... of dirty cards in old space
5272 void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i,
5273 Par_MarkRefsIntoAndScanClosure* cl);
5274
5275 // ... work stealing for the above
5276 void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed);
5277 };
5278
5279 void CMSParMarkTask::work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl) {
5280 DefNewGeneration* dng = _collector->_young_gen->as_DefNewGeneration();
5281 EdenSpace* eden_space = dng->eden();
5282 ContiguousSpace* from_space = dng->from();
5283 ContiguousSpace* to_space = dng->to();
5284
5285 HeapWord** eca = _collector->_eden_chunk_array;
5286 size_t ect = _collector->_eden_chunk_index;
5287 HeapWord** sca = _collector->_survivor_chunk_array;
5288 size_t sct = _collector->_survivor_chunk_index;
5289
5290 assert(ect <= _collector->_eden_chunk_capacity, "out of bounds");
5291 assert(sct <= _collector->_survivor_chunk_capacity, "out of bounds");
5292
5293 do_young_space_rescan(worker_id, cl, to_space, NULL, 0);
5294 do_young_space_rescan(worker_id, cl, from_space, sca, sct);
5295 do_young_space_rescan(worker_id, cl, eden_space, eca, ect);
5296 }
5297
5298 // work_queue(i) is passed to the closure
5299 // Par_MarkRefsIntoAndScanClosure. The "i" parameter
5300 // also is passed to do_dirty_card_rescan_tasks() and to
5301 // do_work_steal() to select the i-th task_queue.
5302
5303 void CMSParRemarkTask::work(uint worker_id) {
5304 elapsedTimer _timer;
5305 ResourceMark rm;
5306 HandleMark hm;
5307
5308 // ---------- rescan from roots --------------
5309 _timer.start();
5310 GenCollectedHeap* gch = GenCollectedHeap::heap();
5311 Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
5312 _collector->_span, _collector->ref_processor(),
5313 &(_collector->_markBitMap),
5314 work_queue(worker_id), &(_collector->_revisitStack));
5315
5316 // Rescan young gen roots first since these are likely
5317 // coarsely partitioned and may, on that account, constitute
5318 // the critical path; thus, it's best to start off that
5319 // work first.
5320 // ---------- young gen roots --------------
5321 {
5322 work_on_young_gen_roots(worker_id, &par_mrias_cl);
5323 _timer.stop();
5324 if (PrintCMSStatistics != 0) {
5325 gclog_or_tty->print_cr(
5326 "Finished young gen rescan work in %dth thread: %3.3f sec",
5327 worker_id, _timer.seconds());
5328 }
5329 }
5330
5331 // ---------- remaining roots --------------
5332 _timer.reset();
5333 _timer.start();
5334 gch->gen_process_strong_roots(_collector->_cmsGen->level(),
5335 false, // yg was scanned above
5336 false, // this is parallel code
5337 true, // collecting perm gen
5338 SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
5339 &par_mrias_cl,
5340 true, // walk all of code cache if (so & SO_CodeCache)
5341 NULL);
5342 assert(_collector->should_unload_classes()
5363 gclog_or_tty->print_cr(
5364 "Finished dirty card rescan work in %dth thread: %3.3f sec",
5365 worker_id, _timer.seconds());
5366 }
5367
5368 // ---------- steal work from other threads ...
5369 // ---------- ... and drain overflow list.
5370 _timer.reset();
5371 _timer.start();
5372 do_work_steal(worker_id, &par_mrias_cl, _collector->hash_seed(worker_id));
5373 _timer.stop();
5374 if (PrintCMSStatistics != 0) {
5375 gclog_or_tty->print_cr(
5376 "Finished work stealing in %dth thread: %3.3f sec",
5377 worker_id, _timer.seconds());
5378 }
5379 }
5380
5381 // Note that parameter "i" is not used.
5382 void
5383 CMSParMarkTask::do_young_space_rescan(uint worker_id,
5384 OopsInGenClosure* cl, ContiguousSpace* space,
5385 HeapWord** chunk_array, size_t chunk_top) {
5386 // Until all tasks completed:
5387 // . claim an unclaimed task
5388 // . compute region boundaries corresponding to task claimed
5389 // using chunk_array
5390 // . par_oop_iterate(cl) over that region
5391
5392 ResourceMark rm;
5393 HandleMark hm;
5394
5395 SequentialSubTasksDone* pst = space->par_seq_tasks();
5396 assert(pst->valid(), "Uninitialized use?");
5397
5398 uint nth_task = 0;
5399 uint n_tasks = pst->n_tasks();
5400
5401 HeapWord *start, *end;
5402 while (!pst->is_task_claimed(/* reference */ nth_task)) {
5403 // We claimed task # nth_task; compute its boundaries.
5404 if (chunk_top == 0) { // no samples were taken
5560 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5561 NOT_PRODUCT(num_steals++;)
5562 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5563 assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5564 // Do scanning work
5565 obj_to_scan->oop_iterate(cl);
5566 // Loop around, finish this work, and try to steal some more
5567 } else if (terminator()->offer_termination()) {
5568 break; // nirvana from the infinite cycle
5569 }
5570 }
5571 NOT_PRODUCT(
5572 if (PrintCMSStatistics != 0) {
5573 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
5574 }
5575 )
5576 assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
5577 "Else our work is not yet done");
5578 }
5579
5580 // Record object boundaries in _eden_chunk_array by sampling the eden
5581 // top in the slow-path eden object allocation code path and record
5582 // the boundaries, if CMSEdenChunksRecordAlways is true. If
5583 // CMSEdenChunksRecordAlways is false, we use the other asynchronous
5584 // sampling in sample_eden() that activates during the part of the
5585 // preclean phase.
5586 void CMSCollector::sample_eden_chunk() {
5587 if (CMSEdenChunksRecordAlways && _eden_chunk_array != NULL) {
5588 if (_eden_chunk_lock->try_lock()) {
5589 // Record a sample. This is the critical section. The contents
5590 // of the _eden_chunk_array have to be non-decreasing in the
5591 // address order.
5592 _eden_chunk_array[_eden_chunk_index] = *_top_addr;
5593 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
5594 "Unexpected state of Eden");
5595 if (_eden_chunk_index == 0 ||
5596 ((_eden_chunk_array[_eden_chunk_index] > _eden_chunk_array[_eden_chunk_index-1]) &&
5597 (pointer_delta(_eden_chunk_array[_eden_chunk_index],
5598 _eden_chunk_array[_eden_chunk_index-1]) >= CMSSamplingGrain))) {
5599 _eden_chunk_index++; // commit sample
5600 }
5601 _eden_chunk_lock->unlock();
5602 }
5603 }
5604 }
5605
5606 // Return a thread-local PLAB recording array, as appropriate.
5607 void* CMSCollector::get_data_recorder(int thr_num) {
5608 if (_survivor_plab_array != NULL &&
5609 (CMSPLABRecordAlways ||
5610 (_collectorState > Marking && _collectorState < FinalMarking))) {
5611 assert(thr_num < (int)ParallelGCThreads, "thr_num is out of bounds");
5612 ChunkArray* ca = &_survivor_plab_array[thr_num];
5613 ca->reset(); // clear it so that fresh data is recorded
5614 return (void*) ca;
5615 } else {
5616 return NULL;
5617 }
5618 }
5619
5620 // Reset all the thread-local PLAB recording arrays
5621 void CMSCollector::reset_survivor_plab_arrays() {
5622 for (uint i = 0; i < ParallelGCThreads; i++) {
5623 _survivor_plab_array[i].reset();
5624 }
5625 }
5626
5627 // Merge the per-thread plab arrays into the global survivor chunk
5628 // array which will provide the partitioning of the survivor space
5629 // for CMS initial scan and rescan.
5630 void CMSCollector::merge_survivor_plab_arrays(ContiguousSpace* surv,
5631 int no_of_gc_threads) {
5632 assert(_survivor_plab_array != NULL, "Error");
5633 assert(_survivor_chunk_array != NULL, "Error");
5634 assert(_collectorState == FinalMarking ||
5635 (CMSParallelInitialMarkEnabled && _collectorState == InitialMarking), "Error");
5636 for (int j = 0; j < no_of_gc_threads; j++) {
5637 _cursor[j] = 0;
5638 }
5639 HeapWord* top = surv->top();
5640 size_t i;
5641 for (i = 0; i < _survivor_chunk_capacity; i++) { // all sca entries
5642 HeapWord* min_val = top; // Higher than any PLAB address
5643 uint min_tid = 0; // position of min_val this round
5644 for (int j = 0; j < no_of_gc_threads; j++) {
5645 ChunkArray* cur_sca = &_survivor_plab_array[j];
5646 if (_cursor[j] == cur_sca->end()) {
5647 continue;
5648 }
5649 assert(_cursor[j] < cur_sca->end(), "ctl pt invariant");
5650 HeapWord* cur_val = cur_sca->nth(_cursor[j]);
5651 assert(surv->used_region().contains(cur_val), "Out of bounds value");
5652 if (cur_val < min_val) {
5653 min_tid = j;
5654 min_val = cur_val;
5655 } else {
5678 for (int j = 0; j < no_of_gc_threads; j++) {
5679 assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
5680 total += _cursor[j];
5681 }
5682 assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
5683 // Check that the merged array is in sorted order
5684 if (total > 0) {
5685 for (size_t i = 0; i < total - 1; i++) {
5686 if (PrintCMSStatistics > 0) {
5687 gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
5688 i, _survivor_chunk_array[i]);
5689 }
5690 assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
5691 "Not sorted");
5692 }
5693 }
5694 #endif // ASSERT
5695 }
5696
5697 // Set up the space's par_seq_tasks structure for work claiming
5698 // for parallel initial scan and rescan of young gen.
5699 // See ParRescanTask where this is currently used.
5700 void
5701 CMSCollector::
5702 initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
5703 assert(n_threads > 0, "Unexpected n_threads argument");
5704 DefNewGeneration* dng = (DefNewGeneration*)_young_gen;
5705
5706 // Eden space
5707 {
5708 SequentialSubTasksDone* pst = dng->eden()->par_seq_tasks();
5709 assert(!pst->valid(), "Clobbering existing data?");
5710 // Each valid entry in [0, _eden_chunk_index) represents a task.
5711 size_t n_tasks = _eden_chunk_index + 1;
5712 assert(n_tasks == 1 || _eden_chunk_array != NULL, "Error");
5713 // Sets the condition for completion of the subtask (how many threads
5714 // need to finish in order to be done).
5715 pst->set_n_threads(n_threads);
5716 pst->set_n_tasks((int)n_tasks);
5717 }
5718
6828 _span(span),
6829 _bitMap(bitMap)
6830 {
6831 assert(_ref_processor == NULL, "deliberately left NULL");
6832 assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
6833 }
6834
6835 void MarkRefsIntoClosure::do_oop(oop obj) {
6836 // if p points into _span, then mark corresponding bit in _markBitMap
6837 assert(obj->is_oop(), "expected an oop");
6838 HeapWord* addr = (HeapWord*)obj;
6839 if (_span.contains(addr)) {
6840 // this should be made more efficient
6841 _bitMap->mark(addr);
6842 }
6843 }
6844
6845 void MarkRefsIntoClosure::do_oop(oop* p) { MarkRefsIntoClosure::do_oop_work(p); }
6846 void MarkRefsIntoClosure::do_oop(narrowOop* p) { MarkRefsIntoClosure::do_oop_work(p); }
6847
6848 Par_MarkRefsIntoClosure::Par_MarkRefsIntoClosure(
6849 MemRegion span, CMSBitMap* bitMap):
6850 _span(span),
6851 _bitMap(bitMap)
6852 {
6853 assert(_ref_processor == NULL, "deliberately left NULL");
6854 assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
6855 }
6856
6857 void Par_MarkRefsIntoClosure::do_oop(oop obj) {
6858 // if p points into _span, then mark corresponding bit in _markBitMap
6859 assert(obj->is_oop(), "expected an oop");
6860 HeapWord* addr = (HeapWord*)obj;
6861 if (_span.contains(addr)) {
6862 // this should be made more efficient
6863 _bitMap->par_mark(addr);
6864 }
6865 }
6866
6867 void Par_MarkRefsIntoClosure::do_oop(oop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); }
6868 void Par_MarkRefsIntoClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); }
6869
6870 // A variant of the above, used for CMS marking verification.
6871 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
6872 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
6873 _span(span),
6874 _verification_bm(verification_bm),
6875 _cms_bm(cms_bm)
6876 {
6877 assert(_ref_processor == NULL, "deliberately left NULL");
6878 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
6879 }
6880
6881 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
6882 // if p points into _span, then mark corresponding bit in _markBitMap
6883 assert(obj->is_oop(), "expected an oop");
6884 HeapWord* addr = (HeapWord*)obj;
6885 if (_span.contains(addr)) {
6886 _verification_bm->mark(addr);
6887 if (!_cms_bm->isMarked(addr)) {
6888 oop(addr)->print();
6889 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", addr);
9515 if (UsePerfData) {
9516 _space_counters->update_capacity();
9517 _gen_counters->update_all();
9518 }
9519
9520 if (Verbose && PrintGCDetails) {
9521 size_t new_mem_size = _virtual_space.committed_size();
9522 size_t old_mem_size = new_mem_size + bytes;
9523 gclog_or_tty->print_cr("Shrinking %s from %ldK by %ldK to %ldK",
9524 name(), old_mem_size/K, bytes/K, new_mem_size/K);
9525 }
9526 }
9527
9528 assert(_cmsSpace->unallocated_block() <= _cmsSpace->end(),
9529 "Inconsistency at end of space");
9530 assert(chunk_at_end->end() == _cmsSpace->end(),
9531 "Shrinking is inconsistent");
9532 return;
9533 }
9534 }
9535 // Transfer some number of overflown objects to usual marking
9536 // stack. Return true if some objects were transferred.
9537 bool MarkRefsIntoAndScanClosure::take_from_overflow_list() {
9538 size_t num = MIN2((size_t)(_mark_stack->capacity() - _mark_stack->length())/4,
9539 (size_t)ParGCDesiredObjsFromOverflowList);
9540
9541 bool res = _collector->take_from_overflow_list(num, _mark_stack);
9542 assert(_collector->overflow_list_is_empty() || res,
9543 "If list is not empty, we should have taken something");
9544 assert(!res || !_mark_stack->isEmpty(),
9545 "If we took something, it should now be on our stack");
9546 return res;
9547 }
9548
9549 size_t MarkDeadObjectsClosure::do_blk(HeapWord* addr) {
9550 size_t res = _sp->block_size_no_stall(addr, _collector);
9551 if (_sp->block_is_obj(addr)) {
9552 if (_live_bit_map->isMarked(addr)) {
9553 // It can't have been dead in a previous cycle
9554 guarantee(!_dead_bit_map->isMarked(addr), "No resurrection!");
|