171 //////////////////////////////////////////////////////////////////
172 // Concurrent Mark-Sweep Generation /////////////////////////////
173 //////////////////////////////////////////////////////////////////
174
175 NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;)
176
177 // This struct contains per-thread things necessary to support parallel
178 // young-gen collection.
179 class CMSParGCThreadState: public CHeapObj<mtGC> {
180 public:
181 CFLS_LAB lab;
182 PromotionInfo promo;
183
184 // Constructor.
185 CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) {
186 promo.setSpace(cfls);
187 }
188 };
189
190 ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration(
191 ReservedSpace rs, size_t initial_byte_size, int level,
192 CardTableRS* ct, bool use_adaptive_freelists,
193 FreeBlockDictionary<FreeChunk>::DictionaryChoice dictionaryChoice) :
194 CardGeneration(rs, initial_byte_size, level, ct),
195 _dilatation_factor(((double)MinChunkSize)/((double)(CollectedHeap::min_fill_size()))),
196 _did_compact(false)
197 {
198 HeapWord* bottom = (HeapWord*) _virtual_space.low();
199 HeapWord* end = (HeapWord*) _virtual_space.high();
200
201 _direct_allocated_words = 0;
202 NOT_PRODUCT(
203 _numObjectsPromoted = 0;
204 _numWordsPromoted = 0;
205 _numObjectsAllocated = 0;
206 _numWordsAllocated = 0;
207 )
208
209 _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end),
210 use_adaptive_freelists,
211 dictionaryChoice);
212 NOT_PRODUCT(debug_cms_space = _cmsSpace;)
213 _cmsSpace->_gen = this;
214
668 _space_counters->update_used(used);
669 _space_counters->update_capacity();
670 _gen_counters->update_all();
671 }
672 }
673
674 void ConcurrentMarkSweepGeneration::print() const {
675 Generation::print();
676 cmsSpace()->print();
677 }
678
679 #ifndef PRODUCT
680 void ConcurrentMarkSweepGeneration::print_statistics() {
681 cmsSpace()->printFLCensus(0);
682 }
683 #endif
684
685 void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) {
686 GenCollectedHeap* gch = GenCollectedHeap::heap();
687 if (PrintGCDetails) {
688 if (Verbose) {
689 gclog_or_tty->print("[%d %s-%s: "SIZE_FORMAT"("SIZE_FORMAT")]",
690 level(), short_name(), s, used(), capacity());
691 } else {
692 gclog_or_tty->print("[%d %s-%s: "SIZE_FORMAT"K("SIZE_FORMAT"K)]",
693 level(), short_name(), s, used() / K, capacity() / K);
694 }
695 }
696 if (Verbose) {
697 gclog_or_tty->print(" "SIZE_FORMAT"("SIZE_FORMAT")",
698 gch->used(), gch->capacity());
699 } else {
700 gclog_or_tty->print(" "SIZE_FORMAT"K("SIZE_FORMAT"K)",
701 gch->used() / K, gch->capacity() / K);
702 }
703 }
704
705 size_t
706 ConcurrentMarkSweepGeneration::contiguous_available() const {
707 // dld proposes an improvement in precision here. If the committed
708 // part of the space ends in a free block we should add that to
709 // uncommitted size in the calculation below. Will make this
710 // change later, staying with the approximation below for the
711 // time being. -- ysr.
712 return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
713 }
782 // to the limit.
783 if (incremental_collection_failed()) {
784 clear_incremental_collection_failed();
785 grow_to_reserved();
786 return;
787 }
788
789 double free_percentage = ((double) free()) / capacity();
790 double desired_free_percentage = (double) MinHeapFreeRatio / 100;
791 double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
792
793 // compute expansion delta needed for reaching desired free percentage
794 if (free_percentage < desired_free_percentage) {
795 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
796 assert(desired_capacity >= capacity(), "invalid expansion size");
797 size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
798 if (PrintGCDetails && Verbose) {
799 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
800 gclog_or_tty->print_cr("\nFrom compute_new_size: ");
801 gclog_or_tty->print_cr(" Free fraction %f", free_percentage);
802 gclog_or_tty->print_cr(" Desired free fraction %f",
803 desired_free_percentage);
804 gclog_or_tty->print_cr(" Maximum free fraction %f",
805 maximum_free_percentage);
806 gclog_or_tty->print_cr(" Capacity "SIZE_FORMAT, capacity()/1000);
807 gclog_or_tty->print_cr(" Desired capacity "SIZE_FORMAT,
808 desired_capacity/1000);
809 int prev_level = level() - 1;
810 if (prev_level >= 0) {
811 size_t prev_size = 0;
812 GenCollectedHeap* gch = GenCollectedHeap::heap();
813 Generation* prev_gen = gch->young_gen();
814 prev_size = prev_gen->capacity();
815 gclog_or_tty->print_cr(" Younger gen size "SIZE_FORMAT,
816 prev_size/1000);
817 }
818 gclog_or_tty->print_cr(" unsafe_max_alloc_nogc "SIZE_FORMAT,
819 unsafe_max_alloc_nogc()/1000);
820 gclog_or_tty->print_cr(" contiguous available "SIZE_FORMAT,
821 contiguous_available()/1000);
822 gclog_or_tty->print_cr(" Expand by "SIZE_FORMAT" (bytes)",
823 expand_bytes);
824 }
825 // safe if expansion fails
826 expand_for_gc_cause(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
827 if (PrintGCDetails && Verbose) {
828 gclog_or_tty->print_cr(" Expanded free fraction %f",
829 ((double) free()) / capacity());
830 }
831 } else {
832 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
833 assert(desired_capacity <= capacity(), "invalid expansion size");
834 size_t shrink_bytes = capacity() - desired_capacity;
835 // Don't shrink unless the delta is greater than the minimum shrink we want
836 if (shrink_bytes >= MinHeapDeltaBytes) {
837 shrink_free_list_by(shrink_bytes);
838 }
839 }
840 }
841
842 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
843 return cmsSpace()->freelistLock();
1636 // collection, clear the _modUnionTable.
1637 assert(_collectorState != Idling || _modUnionTable.isAllClear(),
1638 "_modUnionTable should be clear if the baton was not passed");
1639 _modUnionTable.clear_all();
1640 assert(_collectorState != Idling || _ct->klass_rem_set()->mod_union_is_clear(),
1641 "mod union for klasses should be clear if the baton was passed");
1642 _ct->klass_rem_set()->clear_mod_union();
1643
1644 // We must adjust the allocation statistics being maintained
1645 // in the free list space. We do so by reading and clearing
1646 // the sweep timer and updating the block flux rate estimates below.
1647 assert(!_intra_sweep_timer.is_active(), "_intra_sweep_timer should be inactive");
1648 if (_inter_sweep_timer.is_active()) {
1649 _inter_sweep_timer.stop();
1650 // Note that we do not use this sample to update the _inter_sweep_estimate.
1651 _cmsGen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()),
1652 _inter_sweep_estimate.padded_average(),
1653 _intra_sweep_estimate.padded_average());
1654 }
1655
1656 GenMarkSweep::invoke_at_safepoint(_cmsGen->level(),
1657 ref_processor(), clear_all_soft_refs);
1658 #ifdef ASSERT
1659 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
1660 size_t free_size = cms_space->free();
1661 assert(free_size ==
1662 pointer_delta(cms_space->end(), cms_space->compaction_top())
1663 * HeapWordSize,
1664 "All the free space should be compacted into one chunk at top");
1665 assert(cms_space->dictionary()->total_chunk_size(
1666 debug_only(cms_space->freelistLock())) == 0 ||
1667 cms_space->totalSizeInIndexedFreeLists() == 0,
1668 "All the free space should be in a single chunk");
1669 size_t num = cms_space->totalCount();
1670 assert((free_size == 0 && num == 0) ||
1671 (free_size > 0 && (num == 1 || num == 2)),
1672 "There should be at most 2 free chunks after compaction");
1673 #endif // ASSERT
1674 _collectorState = Resetting;
1675 assert(_restart_addr == NULL,
1676 "Should have been NULL'd before baton was passed");
1677 reset(false /* == !concurrent */);
2417 } else {
2418 warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant",
2419 CMSRemarkVerifyVariant);
2420 }
2421 if (!silent) gclog_or_tty->print(" done] ");
2422 return true;
2423 }
2424
2425 void CMSCollector::verify_after_remark_work_1() {
2426 ResourceMark rm;
2427 HandleMark hm;
2428 GenCollectedHeap* gch = GenCollectedHeap::heap();
2429
2430 // Get a clear set of claim bits for the roots processing to work with.
2431 ClassLoaderDataGraph::clear_claimed_marks();
2432
2433 // Mark from roots one level into CMS
2434 MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
2435 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2436
2437 gch->gen_process_roots(_cmsGen->level(),
2438 true, // younger gens are roots
2439 true, // activate StrongRootsScope
2440 GenCollectedHeap::ScanningOption(roots_scanning_options()),
2441 should_unload_classes(),
2442 ¬Older,
2443 NULL,
2444 NULL); // SSS: Provide correct closure
2445
2446 // Now mark from the roots
2447 MarkFromRootsClosure markFromRootsClosure(this, _span,
2448 verification_mark_bm(), verification_mark_stack(),
2449 false /* don't yield */, true /* verifying */);
2450 assert(_restart_addr == NULL, "Expected pre-condition");
2451 verification_mark_bm()->iterate(&markFromRootsClosure);
2452 while (_restart_addr != NULL) {
2453 // Deal with stack overflow: by restarting at the indicated
2454 // address.
2455 HeapWord* ra = _restart_addr;
2456 markFromRootsClosure.reset(ra);
2457 _restart_addr = NULL;
2485 void do_klass(Klass* k) {
2486 k->oops_do(&_oop_closure);
2487 }
2488 };
2489
2490 void CMSCollector::verify_after_remark_work_2() {
2491 ResourceMark rm;
2492 HandleMark hm;
2493 GenCollectedHeap* gch = GenCollectedHeap::heap();
2494
2495 // Get a clear set of claim bits for the roots processing to work with.
2496 ClassLoaderDataGraph::clear_claimed_marks();
2497
2498 // Mark from roots one level into CMS
2499 MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(),
2500 markBitMap());
2501 CLDToOopClosure cld_closure(¬Older, true);
2502
2503 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2504
2505 gch->gen_process_roots(_cmsGen->level(),
2506 true, // younger gens are roots
2507 true, // activate StrongRootsScope
2508 GenCollectedHeap::ScanningOption(roots_scanning_options()),
2509 should_unload_classes(),
2510 ¬Older,
2511 NULL,
2512 &cld_closure);
2513
2514 // Now mark from the roots
2515 MarkFromRootsVerifyClosure markFromRootsClosure(this, _span,
2516 verification_mark_bm(), markBitMap(), verification_mark_stack());
2517 assert(_restart_addr == NULL, "Expected pre-condition");
2518 verification_mark_bm()->iterate(&markFromRootsClosure);
2519 while (_restart_addr != NULL) {
2520 // Deal with stack overflow: by restarting at the indicated
2521 // address.
2522 HeapWord* ra = _restart_addr;
2523 markFromRootsClosure.reset(ra);
2524 _restart_addr = NULL;
2525 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
3008 if (CMSParallelInitialMarkEnabled && CollectedHeap::use_parallel_gc_threads()) {
3009 // The parallel version.
3010 FlexibleWorkGang* workers = gch->workers();
3011 assert(workers != NULL, "Need parallel worker threads.");
3012 int n_workers = workers->active_workers();
3013 CMSParInitialMarkTask tsk(this, n_workers);
3014 gch->set_par_threads(n_workers);
3015 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
3016 if (n_workers > 1) {
3017 StrongRootsScope srs;
3018 workers->run_task(&tsk);
3019 } else {
3020 StrongRootsScope srs;
3021 tsk.work(0);
3022 }
3023 gch->set_par_threads(0);
3024 } else {
3025 // The serial version.
3026 CLDToOopClosure cld_closure(¬Older, true);
3027 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
3028 gch->gen_process_roots(_cmsGen->level(),
3029 true, // younger gens are roots
3030 true, // activate StrongRootsScope
3031 GenCollectedHeap::ScanningOption(roots_scanning_options()),
3032 should_unload_classes(),
3033 ¬Older,
3034 NULL,
3035 &cld_closure);
3036 }
3037 }
3038
3039 // Clear mod-union table; it will be dirtied in the prologue of
3040 // CMS generation per each younger generation collection.
3041
3042 assert(_modUnionTable.isAllClear(),
3043 "Was cleared in most recent final checkpoint phase"
3044 " or no bits are set in the gc_prologue before the start of the next "
3045 "subsequent marking phase.");
3046
3047 assert(_ct->klass_rem_set()->mod_union_is_clear(), "Must be");
3048
4261 assert(SafepointSynchronize::is_at_safepoint(),
4262 "world should be stopped");
4263 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
4264
4265 verify_work_stacks_empty();
4266 verify_overflow_empty();
4267
4268 if (PrintGCDetails) {
4269 gclog_or_tty->print("[YG occupancy: "SIZE_FORMAT" K ("SIZE_FORMAT" K)]",
4270 _young_gen->used() / K,
4271 _young_gen->capacity() / K);
4272 }
4273 {
4274 if (CMSScavengeBeforeRemark) {
4275 GenCollectedHeap* gch = GenCollectedHeap::heap();
4276 // Temporarily set flag to false, GCH->do_collection will
4277 // expect it to be false and set to true
4278 FlagSetting fl(gch->_is_gc_active, false);
4279 NOT_PRODUCT(GCTraceTime t("Scavenge-Before-Remark",
4280 PrintGCDetails && Verbose, true, _gc_timer_cm, _gc_tracer_cm->gc_id());)
4281 int level = _cmsGen->level() - 1;
4282 if (level >= 0) {
4283 gch->do_collection(true, // full (i.e. force, see below)
4284 false, // !clear_all_soft_refs
4285 0, // size
4286 false, // is_tlab
4287 level // max_level
4288 );
4289 }
4290 }
4291 FreelistLocker x(this);
4292 MutexLockerEx y(bitMapLock(),
4293 Mutex::_no_safepoint_check_flag);
4294 checkpointRootsFinalWork();
4295 }
4296 verify_work_stacks_empty();
4297 verify_overflow_empty();
4298 }
4299
4300 void CMSCollector::checkpointRootsFinalWork() {
4301 NOT_PRODUCT(GCTraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());)
4302
4303 assert(haveFreelistLocks(), "must have free list locks");
4304 assert_lock_strong(bitMapLock());
4305
4306 ResourceMark rm;
4307 HandleMark hm;
4308
4309 GenCollectedHeap* gch = GenCollectedHeap::heap();
4310
4442 GenCollectedHeap* gch = GenCollectedHeap::heap();
4443 Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap));
4444
4445 // ---------- young gen roots --------------
4446 {
4447 work_on_young_gen_roots(worker_id, &par_mri_cl);
4448 _timer.stop();
4449 if (PrintCMSStatistics != 0) {
4450 gclog_or_tty->print_cr(
4451 "Finished young gen initial mark scan work in %dth thread: %3.3f sec",
4452 worker_id, _timer.seconds());
4453 }
4454 }
4455
4456 // ---------- remaining roots --------------
4457 _timer.reset();
4458 _timer.start();
4459
4460 CLDToOopClosure cld_closure(&par_mri_cl, true);
4461
4462 gch->gen_process_roots(_collector->_cmsGen->level(),
4463 false, // yg was scanned above
4464 false, // this is parallel code
4465 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4466 _collector->should_unload_classes(),
4467 &par_mri_cl,
4468 NULL,
4469 &cld_closure);
4470 assert(_collector->should_unload_classes()
4471 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4472 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4473 _timer.stop();
4474 if (PrintCMSStatistics != 0) {
4475 gclog_or_tty->print_cr(
4476 "Finished remaining root initial mark scan work in %dth thread: %3.3f sec",
4477 worker_id, _timer.seconds());
4478 }
4479 }
4480
4481 // Parallel remark task
4482 class CMSParRemarkTask: public CMSParMarkTask {
4578 work_queue(worker_id));
4579
4580 // Rescan young gen roots first since these are likely
4581 // coarsely partitioned and may, on that account, constitute
4582 // the critical path; thus, it's best to start off that
4583 // work first.
4584 // ---------- young gen roots --------------
4585 {
4586 work_on_young_gen_roots(worker_id, &par_mrias_cl);
4587 _timer.stop();
4588 if (PrintCMSStatistics != 0) {
4589 gclog_or_tty->print_cr(
4590 "Finished young gen rescan work in %dth thread: %3.3f sec",
4591 worker_id, _timer.seconds());
4592 }
4593 }
4594
4595 // ---------- remaining roots --------------
4596 _timer.reset();
4597 _timer.start();
4598 gch->gen_process_roots(_collector->_cmsGen->level(),
4599 false, // yg was scanned above
4600 false, // this is parallel code
4601 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4602 _collector->should_unload_classes(),
4603 &par_mrias_cl,
4604 NULL,
4605 NULL); // The dirty klasses will be handled below
4606
4607 assert(_collector->should_unload_classes()
4608 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4609 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4610 _timer.stop();
4611 if (PrintCMSStatistics != 0) {
4612 gclog_or_tty->print_cr(
4613 "Finished remaining root rescan work in %dth thread: %3.3f sec",
4614 worker_id, _timer.seconds());
4615 }
4616
4617 // ---------- unhandled CLD scanning ----------
4618 if (worker_id == 0) { // Single threaded at the moment.
5169 verify_work_stacks_empty();
5170 if (PrintCMSStatistics != 0) {
5171 gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ",
5172 markFromDirtyCardsClosure.num_dirty_cards());
5173 }
5174 }
5175 }
5176 if (VerifyDuringGC &&
5177 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
5178 HandleMark hm; // Discard invalid handles created during verification
5179 Universe::verify();
5180 }
5181 {
5182 GCTraceTime t("root rescan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5183
5184 verify_work_stacks_empty();
5185
5186 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
5187 StrongRootsScope srs;
5188
5189 gch->gen_process_roots(_cmsGen->level(),
5190 true, // younger gens as roots
5191 false, // use the local StrongRootsScope
5192 GenCollectedHeap::ScanningOption(roots_scanning_options()),
5193 should_unload_classes(),
5194 &mrias_cl,
5195 NULL,
5196 NULL); // The dirty klasses will be handled below
5197
5198 assert(should_unload_classes()
5199 || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
5200 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
5201 }
5202
5203 {
5204 GCTraceTime t("visit unhandled CLDs", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5205
5206 verify_work_stacks_empty();
5207
5208 // Scan all class loader data objects that might have been introduced
5209 // during concurrent marking.
5638 size_t nearLargestOffset =
5639 (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
5640 if (PrintFLSStatistics != 0) {
5641 gclog_or_tty->print_cr(
5642 "CMS: Large Block: " PTR_FORMAT ";"
5643 " Proximity: " PTR_FORMAT " -> " PTR_FORMAT,
5644 p2i(largestAddr),
5645 p2i(_cmsSpace->nearLargestChunk()), p2i(minAddr + nearLargestOffset));
5646 }
5647 _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
5648 }
5649
5650 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
5651 return addr >= _cmsSpace->nearLargestChunk();
5652 }
5653
5654 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
5655 return _cmsSpace->find_chunk_at_end();
5656 }
5657
5658 void ConcurrentMarkSweepGeneration::update_gc_stats(int current_level,
5659 bool full) {
5660 // The next lower level has been collected. Gather any statistics
5661 // that are of interest at this point.
5662 if (!full && (current_level + 1) == level()) {
5663 // Gather statistics on the young generation collection.
5664 collector()->stats().record_gc0_end(used());
5665 }
5666 }
5667
5668 void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* gen) {
5669 // We iterate over the space(s) underlying this generation,
5670 // checking the mark bit map to see if the bits corresponding
5671 // to specific blocks are marked or not. Blocks that are
5672 // marked are live and are not swept up. All remaining blocks
5673 // are swept up, with coalescing on-the-fly as we sweep up
5674 // contiguous free and/or garbage blocks:
5675 // We need to ensure that the sweeper synchronizes with allocators
5676 // and stop-the-world collectors. In particular, the following
5677 // locks are used:
5678 // . CMS token: if this is held, a stop the world collection cannot occur
5679 // . freelistLock: if this is held no allocation can occur from this
5680 // generation by another thread
5681 // . bitMapLock: if this is held, no other thread can access or update
5682 //
|
171 //////////////////////////////////////////////////////////////////
172 // Concurrent Mark-Sweep Generation /////////////////////////////
173 //////////////////////////////////////////////////////////////////
174
175 NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;)
176
177 // This struct contains per-thread things necessary to support parallel
178 // young-gen collection.
179 class CMSParGCThreadState: public CHeapObj<mtGC> {
180 public:
181 CFLS_LAB lab;
182 PromotionInfo promo;
183
184 // Constructor.
185 CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) {
186 promo.setSpace(cfls);
187 }
188 };
189
190 ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration(
191 ReservedSpace rs, size_t initial_byte_size,
192 CardTableRS* ct, bool use_adaptive_freelists,
193 FreeBlockDictionary<FreeChunk>::DictionaryChoice dictionaryChoice) :
194 CardGeneration(rs, initial_byte_size, ct),
195 _dilatation_factor(((double)MinChunkSize)/((double)(CollectedHeap::min_fill_size()))),
196 _did_compact(false)
197 {
198 HeapWord* bottom = (HeapWord*) _virtual_space.low();
199 HeapWord* end = (HeapWord*) _virtual_space.high();
200
201 _direct_allocated_words = 0;
202 NOT_PRODUCT(
203 _numObjectsPromoted = 0;
204 _numWordsPromoted = 0;
205 _numObjectsAllocated = 0;
206 _numWordsAllocated = 0;
207 )
208
209 _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end),
210 use_adaptive_freelists,
211 dictionaryChoice);
212 NOT_PRODUCT(debug_cms_space = _cmsSpace;)
213 _cmsSpace->_gen = this;
214
668 _space_counters->update_used(used);
669 _space_counters->update_capacity();
670 _gen_counters->update_all();
671 }
672 }
673
674 void ConcurrentMarkSweepGeneration::print() const {
675 Generation::print();
676 cmsSpace()->print();
677 }
678
679 #ifndef PRODUCT
680 void ConcurrentMarkSweepGeneration::print_statistics() {
681 cmsSpace()->printFLCensus(0);
682 }
683 #endif
684
685 void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) {
686 GenCollectedHeap* gch = GenCollectedHeap::heap();
687 if (PrintGCDetails) {
688 // I didn't want to change the logging when removing the level concept,
689 // but I guess this logging could say "old" or something instead of "1".
690 assert(this == gch->old_gen(),
691 "The CMS generation should be the old generation");
692 uint level = 1;
693 if (Verbose) {
694 gclog_or_tty->print("[%u %s-%s: "SIZE_FORMAT"("SIZE_FORMAT")]",
695 level, short_name(), s, used(), capacity());
696 } else {
697 gclog_or_tty->print("[%u %s-%s: "SIZE_FORMAT"K("SIZE_FORMAT"K)]",
698 level, short_name(), s, used() / K, capacity() / K);
699 }
700 }
701 if (Verbose) {
702 gclog_or_tty->print(" "SIZE_FORMAT"("SIZE_FORMAT")",
703 gch->used(), gch->capacity());
704 } else {
705 gclog_or_tty->print(" "SIZE_FORMAT"K("SIZE_FORMAT"K)",
706 gch->used() / K, gch->capacity() / K);
707 }
708 }
709
710 size_t
711 ConcurrentMarkSweepGeneration::contiguous_available() const {
712 // dld proposes an improvement in precision here. If the committed
713 // part of the space ends in a free block we should add that to
714 // uncommitted size in the calculation below. Will make this
715 // change later, staying with the approximation below for the
716 // time being. -- ysr.
717 return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
718 }
787 // to the limit.
788 if (incremental_collection_failed()) {
789 clear_incremental_collection_failed();
790 grow_to_reserved();
791 return;
792 }
793
794 double free_percentage = ((double) free()) / capacity();
795 double desired_free_percentage = (double) MinHeapFreeRatio / 100;
796 double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
797
798 // compute expansion delta needed for reaching desired free percentage
799 if (free_percentage < desired_free_percentage) {
800 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
801 assert(desired_capacity >= capacity(), "invalid expansion size");
802 size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
803 if (PrintGCDetails && Verbose) {
804 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
805 gclog_or_tty->print_cr("\nFrom compute_new_size: ");
806 gclog_or_tty->print_cr(" Free fraction %f", free_percentage);
807 gclog_or_tty->print_cr(" Desired free fraction %f", desired_free_percentage);
808 gclog_or_tty->print_cr(" Maximum free fraction %f", maximum_free_percentage);
809 gclog_or_tty->print_cr(" Capacity "SIZE_FORMAT, capacity() / 1000);
810 gclog_or_tty->print_cr(" Desired capacity "SIZE_FORMAT, desired_capacity / 1000);
811 GenCollectedHeap* gch = GenCollectedHeap::heap();
812 assert(this == gch->_old_gen, "The CMS generation should always be the old generation");
813 size_t young_size = gch->_young_gen->capacity();
814 gclog_or_tty->print_cr(" Young gen size "SIZE_FORMAT, young_size / 1000);
815 gclog_or_tty->print_cr(" unsafe_max_alloc_nogc "SIZE_FORMAT, unsafe_max_alloc_nogc() / 1000);
816 gclog_or_tty->print_cr(" contiguous available "SIZE_FORMAT, contiguous_available() / 1000);
817 gclog_or_tty->print_cr(" Expand by "SIZE_FORMAT" (bytes)", expand_bytes);
818 }
819 // safe if expansion fails
820 expand_for_gc_cause(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
821 if (PrintGCDetails && Verbose) {
822 gclog_or_tty->print_cr(" Expanded free fraction %f",
823 ((double) free()) / capacity());
824 }
825 } else {
826 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
827 assert(desired_capacity <= capacity(), "invalid expansion size");
828 size_t shrink_bytes = capacity() - desired_capacity;
829 // Don't shrink unless the delta is greater than the minimum shrink we want
830 if (shrink_bytes >= MinHeapDeltaBytes) {
831 shrink_free_list_by(shrink_bytes);
832 }
833 }
834 }
835
836 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
837 return cmsSpace()->freelistLock();
1630 // collection, clear the _modUnionTable.
1631 assert(_collectorState != Idling || _modUnionTable.isAllClear(),
1632 "_modUnionTable should be clear if the baton was not passed");
1633 _modUnionTable.clear_all();
1634 assert(_collectorState != Idling || _ct->klass_rem_set()->mod_union_is_clear(),
1635 "mod union for klasses should be clear if the baton was passed");
1636 _ct->klass_rem_set()->clear_mod_union();
1637
1638 // We must adjust the allocation statistics being maintained
1639 // in the free list space. We do so by reading and clearing
1640 // the sweep timer and updating the block flux rate estimates below.
1641 assert(!_intra_sweep_timer.is_active(), "_intra_sweep_timer should be inactive");
1642 if (_inter_sweep_timer.is_active()) {
1643 _inter_sweep_timer.stop();
1644 // Note that we do not use this sample to update the _inter_sweep_estimate.
1645 _cmsGen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()),
1646 _inter_sweep_estimate.padded_average(),
1647 _intra_sweep_estimate.padded_average());
1648 }
1649
1650 GenMarkSweep::invoke_at_safepoint(ref_processor(), clear_all_soft_refs);
1651 #ifdef ASSERT
1652 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
1653 size_t free_size = cms_space->free();
1654 assert(free_size ==
1655 pointer_delta(cms_space->end(), cms_space->compaction_top())
1656 * HeapWordSize,
1657 "All the free space should be compacted into one chunk at top");
1658 assert(cms_space->dictionary()->total_chunk_size(
1659 debug_only(cms_space->freelistLock())) == 0 ||
1660 cms_space->totalSizeInIndexedFreeLists() == 0,
1661 "All the free space should be in a single chunk");
1662 size_t num = cms_space->totalCount();
1663 assert((free_size == 0 && num == 0) ||
1664 (free_size > 0 && (num == 1 || num == 2)),
1665 "There should be at most 2 free chunks after compaction");
1666 #endif // ASSERT
1667 _collectorState = Resetting;
1668 assert(_restart_addr == NULL,
1669 "Should have been NULL'd before baton was passed");
1670 reset(false /* == !concurrent */);
2410 } else {
2411 warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant",
2412 CMSRemarkVerifyVariant);
2413 }
2414 if (!silent) gclog_or_tty->print(" done] ");
2415 return true;
2416 }
2417
2418 void CMSCollector::verify_after_remark_work_1() {
2419 ResourceMark rm;
2420 HandleMark hm;
2421 GenCollectedHeap* gch = GenCollectedHeap::heap();
2422
2423 // Get a clear set of claim bits for the roots processing to work with.
2424 ClassLoaderDataGraph::clear_claimed_marks();
2425
2426 // Mark from roots one level into CMS
2427 MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
2428 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2429
2430 gch->gen_process_roots(Generation::Old,
2431 true, // younger gens are roots
2432 true, // activate StrongRootsScope
2433 GenCollectedHeap::ScanningOption(roots_scanning_options()),
2434 should_unload_classes(),
2435 ¬Older,
2436 NULL,
2437 NULL); // SSS: Provide correct closure
2438
2439 // Now mark from the roots
2440 MarkFromRootsClosure markFromRootsClosure(this, _span,
2441 verification_mark_bm(), verification_mark_stack(),
2442 false /* don't yield */, true /* verifying */);
2443 assert(_restart_addr == NULL, "Expected pre-condition");
2444 verification_mark_bm()->iterate(&markFromRootsClosure);
2445 while (_restart_addr != NULL) {
2446 // Deal with stack overflow: by restarting at the indicated
2447 // address.
2448 HeapWord* ra = _restart_addr;
2449 markFromRootsClosure.reset(ra);
2450 _restart_addr = NULL;
2478 void do_klass(Klass* k) {
2479 k->oops_do(&_oop_closure);
2480 }
2481 };
2482
2483 void CMSCollector::verify_after_remark_work_2() {
2484 ResourceMark rm;
2485 HandleMark hm;
2486 GenCollectedHeap* gch = GenCollectedHeap::heap();
2487
2488 // Get a clear set of claim bits for the roots processing to work with.
2489 ClassLoaderDataGraph::clear_claimed_marks();
2490
2491 // Mark from roots one level into CMS
2492 MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(),
2493 markBitMap());
2494 CLDToOopClosure cld_closure(¬Older, true);
2495
2496 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2497
2498 gch->gen_process_roots(Generation::Old,
2499 true, // younger gens are roots
2500 true, // activate StrongRootsScope
2501 GenCollectedHeap::ScanningOption(roots_scanning_options()),
2502 should_unload_classes(),
2503 ¬Older,
2504 NULL,
2505 &cld_closure);
2506
2507 // Now mark from the roots
2508 MarkFromRootsVerifyClosure markFromRootsClosure(this, _span,
2509 verification_mark_bm(), markBitMap(), verification_mark_stack());
2510 assert(_restart_addr == NULL, "Expected pre-condition");
2511 verification_mark_bm()->iterate(&markFromRootsClosure);
2512 while (_restart_addr != NULL) {
2513 // Deal with stack overflow: by restarting at the indicated
2514 // address.
2515 HeapWord* ra = _restart_addr;
2516 markFromRootsClosure.reset(ra);
2517 _restart_addr = NULL;
2518 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
3001 if (CMSParallelInitialMarkEnabled && CollectedHeap::use_parallel_gc_threads()) {
3002 // The parallel version.
3003 FlexibleWorkGang* workers = gch->workers();
3004 assert(workers != NULL, "Need parallel worker threads.");
3005 int n_workers = workers->active_workers();
3006 CMSParInitialMarkTask tsk(this, n_workers);
3007 gch->set_par_threads(n_workers);
3008 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
3009 if (n_workers > 1) {
3010 StrongRootsScope srs;
3011 workers->run_task(&tsk);
3012 } else {
3013 StrongRootsScope srs;
3014 tsk.work(0);
3015 }
3016 gch->set_par_threads(0);
3017 } else {
3018 // The serial version.
3019 CLDToOopClosure cld_closure(¬Older, true);
3020 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
3021 gch->gen_process_roots(Generation::Old,
3022 true, // younger gens are roots
3023 true, // activate StrongRootsScope
3024 GenCollectedHeap::ScanningOption(roots_scanning_options()),
3025 should_unload_classes(),
3026 ¬Older,
3027 NULL,
3028 &cld_closure);
3029 }
3030 }
3031
3032 // Clear mod-union table; it will be dirtied in the prologue of
3033 // CMS generation per each younger generation collection.
3034
3035 assert(_modUnionTable.isAllClear(),
3036 "Was cleared in most recent final checkpoint phase"
3037 " or no bits are set in the gc_prologue before the start of the next "
3038 "subsequent marking phase.");
3039
3040 assert(_ct->klass_rem_set()->mod_union_is_clear(), "Must be");
3041
4254 assert(SafepointSynchronize::is_at_safepoint(),
4255 "world should be stopped");
4256 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
4257
4258 verify_work_stacks_empty();
4259 verify_overflow_empty();
4260
4261 if (PrintGCDetails) {
4262 gclog_or_tty->print("[YG occupancy: "SIZE_FORMAT" K ("SIZE_FORMAT" K)]",
4263 _young_gen->used() / K,
4264 _young_gen->capacity() / K);
4265 }
4266 {
4267 if (CMSScavengeBeforeRemark) {
4268 GenCollectedHeap* gch = GenCollectedHeap::heap();
4269 // Temporarily set flag to false, GCH->do_collection will
4270 // expect it to be false and set to true
4271 FlagSetting fl(gch->_is_gc_active, false);
4272 NOT_PRODUCT(GCTraceTime t("Scavenge-Before-Remark",
4273 PrintGCDetails && Verbose, true, _gc_timer_cm, _gc_tracer_cm->gc_id());)
4274 gch->do_collection(true, // full (i.e. force, see below)
4275 false, // !clear_all_soft_refs
4276 0, // size
4277 false, // is_tlab
4278 Generation::Young // type
4279 );
4280 }
4281 FreelistLocker x(this);
4282 MutexLockerEx y(bitMapLock(),
4283 Mutex::_no_safepoint_check_flag);
4284 checkpointRootsFinalWork();
4285 }
4286 verify_work_stacks_empty();
4287 verify_overflow_empty();
4288 }
4289
4290 void CMSCollector::checkpointRootsFinalWork() {
4291 NOT_PRODUCT(GCTraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());)
4292
4293 assert(haveFreelistLocks(), "must have free list locks");
4294 assert_lock_strong(bitMapLock());
4295
4296 ResourceMark rm;
4297 HandleMark hm;
4298
4299 GenCollectedHeap* gch = GenCollectedHeap::heap();
4300
4432 GenCollectedHeap* gch = GenCollectedHeap::heap();
4433 Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap));
4434
4435 // ---------- young gen roots --------------
4436 {
4437 work_on_young_gen_roots(worker_id, &par_mri_cl);
4438 _timer.stop();
4439 if (PrintCMSStatistics != 0) {
4440 gclog_or_tty->print_cr(
4441 "Finished young gen initial mark scan work in %dth thread: %3.3f sec",
4442 worker_id, _timer.seconds());
4443 }
4444 }
4445
4446 // ---------- remaining roots --------------
4447 _timer.reset();
4448 _timer.start();
4449
4450 CLDToOopClosure cld_closure(&par_mri_cl, true);
4451
4452 gch->gen_process_roots(Generation::Old,
4453 false, // yg was scanned above
4454 false, // this is parallel code
4455 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4456 _collector->should_unload_classes(),
4457 &par_mri_cl,
4458 NULL,
4459 &cld_closure);
4460 assert(_collector->should_unload_classes()
4461 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4462 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4463 _timer.stop();
4464 if (PrintCMSStatistics != 0) {
4465 gclog_or_tty->print_cr(
4466 "Finished remaining root initial mark scan work in %dth thread: %3.3f sec",
4467 worker_id, _timer.seconds());
4468 }
4469 }
4470
4471 // Parallel remark task
4472 class CMSParRemarkTask: public CMSParMarkTask {
4568 work_queue(worker_id));
4569
4570 // Rescan young gen roots first since these are likely
4571 // coarsely partitioned and may, on that account, constitute
4572 // the critical path; thus, it's best to start off that
4573 // work first.
4574 // ---------- young gen roots --------------
4575 {
4576 work_on_young_gen_roots(worker_id, &par_mrias_cl);
4577 _timer.stop();
4578 if (PrintCMSStatistics != 0) {
4579 gclog_or_tty->print_cr(
4580 "Finished young gen rescan work in %dth thread: %3.3f sec",
4581 worker_id, _timer.seconds());
4582 }
4583 }
4584
4585 // ---------- remaining roots --------------
4586 _timer.reset();
4587 _timer.start();
4588 gch->gen_process_roots(Generation::Old,
4589 false, // yg was scanned above
4590 false, // this is parallel code
4591 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4592 _collector->should_unload_classes(),
4593 &par_mrias_cl,
4594 NULL,
4595 NULL); // The dirty klasses will be handled below
4596
4597 assert(_collector->should_unload_classes()
4598 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4599 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4600 _timer.stop();
4601 if (PrintCMSStatistics != 0) {
4602 gclog_or_tty->print_cr(
4603 "Finished remaining root rescan work in %dth thread: %3.3f sec",
4604 worker_id, _timer.seconds());
4605 }
4606
4607 // ---------- unhandled CLD scanning ----------
4608 if (worker_id == 0) { // Single threaded at the moment.
5159 verify_work_stacks_empty();
5160 if (PrintCMSStatistics != 0) {
5161 gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ",
5162 markFromDirtyCardsClosure.num_dirty_cards());
5163 }
5164 }
5165 }
5166 if (VerifyDuringGC &&
5167 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
5168 HandleMark hm; // Discard invalid handles created during verification
5169 Universe::verify();
5170 }
5171 {
5172 GCTraceTime t("root rescan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5173
5174 verify_work_stacks_empty();
5175
5176 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
5177 StrongRootsScope srs;
5178
5179 gch->gen_process_roots(Generation::Old,
5180 true, // younger gens as roots
5181 false, // use the local StrongRootsScope
5182 GenCollectedHeap::ScanningOption(roots_scanning_options()),
5183 should_unload_classes(),
5184 &mrias_cl,
5185 NULL,
5186 NULL); // The dirty klasses will be handled below
5187
5188 assert(should_unload_classes()
5189 || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
5190 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
5191 }
5192
5193 {
5194 GCTraceTime t("visit unhandled CLDs", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5195
5196 verify_work_stacks_empty();
5197
5198 // Scan all class loader data objects that might have been introduced
5199 // during concurrent marking.
5628 size_t nearLargestOffset =
5629 (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
5630 if (PrintFLSStatistics != 0) {
5631 gclog_or_tty->print_cr(
5632 "CMS: Large Block: " PTR_FORMAT ";"
5633 " Proximity: " PTR_FORMAT " -> " PTR_FORMAT,
5634 p2i(largestAddr),
5635 p2i(_cmsSpace->nearLargestChunk()), p2i(minAddr + nearLargestOffset));
5636 }
5637 _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
5638 }
5639
5640 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
5641 return addr >= _cmsSpace->nearLargestChunk();
5642 }
5643
5644 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
5645 return _cmsSpace->find_chunk_at_end();
5646 }
5647
5648 void ConcurrentMarkSweepGeneration::update_gc_stats(Generation* current_generation,
5649 bool full) {
5650 // If the young generation has been collected. Gather any statistics
5651 // that are of interest at this point.
5652 bool current_is_young = (current_generation == GenCollectedHeap::heap()->young_gen());
5653 if (!full && current_is_young) {
5654 // Gather statistics on the young generation collection.
5655 collector()->stats().record_gc0_end(used());
5656 }
5657 }
5658
5659 void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* gen) {
5660 // We iterate over the space(s) underlying this generation,
5661 // checking the mark bit map to see if the bits corresponding
5662 // to specific blocks are marked or not. Blocks that are
5663 // marked are live and are not swept up. All remaining blocks
5664 // are swept up, with coalescing on-the-fly as we sweep up
5665 // contiguous free and/or garbage blocks:
5666 // We need to ensure that the sweeper synchronizes with allocators
5667 // and stop-the-world collectors. In particular, the following
5668 // locks are used:
5669 // . CMS token: if this is held, a stop the world collection cannot occur
5670 // . freelistLock: if this is held no allocation can occur from this
5671 // generation by another thread
5672 // . bitMapLock: if this is held, no other thread can access or update
5673 //
|