29 #include "code/codeCache.hpp"
30 #include "gc/cms/cmsCollectorPolicy.hpp"
31 #include "gc/cms/cmsOopClosures.inline.hpp"
32 #include "gc/cms/compactibleFreeListSpace.hpp"
33 #include "gc/cms/concurrentMarkSweepGeneration.inline.hpp"
34 #include "gc/cms/concurrentMarkSweepThread.hpp"
35 #include "gc/cms/parNewGeneration.hpp"
36 #include "gc/cms/vmCMSOperations.hpp"
37 #include "gc/serial/genMarkSweep.hpp"
38 #include "gc/serial/tenuredGeneration.hpp"
39 #include "gc/shared/adaptiveSizePolicy.hpp"
40 #include "gc/shared/cardGeneration.inline.hpp"
41 #include "gc/shared/cardTableRS.hpp"
42 #include "gc/shared/collectedHeap.inline.hpp"
43 #include "gc/shared/collectorCounters.hpp"
44 #include "gc/shared/collectorPolicy.hpp"
45 #include "gc/shared/gcLocker.inline.hpp"
46 #include "gc/shared/gcPolicyCounters.hpp"
47 #include "gc/shared/gcTimer.hpp"
48 #include "gc/shared/gcTrace.hpp"
49 #include "gc/shared/gcTraceTime.hpp"
50 #include "gc/shared/genCollectedHeap.hpp"
51 #include "gc/shared/genOopClosures.inline.hpp"
52 #include "gc/shared/isGCActiveMark.hpp"
53 #include "gc/shared/referencePolicy.hpp"
54 #include "gc/shared/strongRootsScope.hpp"
55 #include "gc/shared/taskqueue.inline.hpp"
56 #include "memory/allocation.hpp"
57 #include "memory/iterator.inline.hpp"
58 #include "memory/padded.hpp"
59 #include "memory/resourceArea.hpp"
60 #include "oops/oop.inline.hpp"
61 #include "prims/jvmtiExport.hpp"
62 #include "runtime/atomic.inline.hpp"
63 #include "runtime/globals_extension.hpp"
64 #include "runtime/handles.inline.hpp"
65 #include "runtime/java.hpp"
66 #include "runtime/orderAccess.inline.hpp"
67 #include "runtime/vmThread.hpp"
68 #include "services/memoryService.hpp"
69 #include "services/runtimeService.hpp"
70 #include "utilities/stack.inline.hpp"
71
72 // statics
73 CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;
74 bool CMSCollector::_full_gc_requested = false;
75 GCCause::Cause CMSCollector::_full_gc_cause = GCCause::_no_gc;
76
77 //////////////////////////////////////////////////////////////////
78 // In support of CMS/VM thread synchronization
79 //////////////////////////////////////////////////////////////////
80 // We split use of the CGC_lock into 2 "levels".
81 // The low-level locking is of the usual CGC_lock monitor. We introduce
82 // a higher level "token" (hereafter "CMS token") built on top of the
83 // low level monitor (hereafter "CGC lock").
84 // The token-passing protocol gives priority to the VM thread. The
85 // CMS-lock doesn't provide any fairness guarantees, but clients
86 // should ensure that it is only held for very short, bounded
349 // young generation collection.
350 double CMSStats::time_until_cms_gen_full() const {
351 size_t cms_free = _cms_gen->cmsSpace()->free();
352 GenCollectedHeap* gch = GenCollectedHeap::heap();
353 size_t expected_promotion = MIN2(gch->young_gen()->capacity(),
354 (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average());
355 if (cms_free > expected_promotion) {
356 // Start a cms collection if there isn't enough space to promote
357 // for the next young collection. Use the padded average as
358 // a safety factor.
359 cms_free -= expected_promotion;
360
361 // Adjust by the safety factor.
362 double cms_free_dbl = (double)cms_free;
363 double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor) / 100.0;
364 // Apply a further correction factor which tries to adjust
365 // for recent occurance of concurrent mode failures.
366 cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free);
367 cms_free_dbl = cms_free_dbl * cms_adjustment;
368
369 if (PrintGCDetails && Verbose) {
370 gclog_or_tty->print_cr("CMSStats::time_until_cms_gen_full: cms_free "
371 SIZE_FORMAT " expected_promotion " SIZE_FORMAT,
372 cms_free, expected_promotion);
373 gclog_or_tty->print_cr(" cms_free_dbl %f cms_consumption_rate %f",
374 cms_free_dbl, cms_consumption_rate() + 1.0);
375 }
376 // Add 1 in case the consumption rate goes to zero.
377 return cms_free_dbl / (cms_consumption_rate() + 1.0);
378 }
379 return 0.0;
380 }
381
382 // Compare the duration of the cms collection to the
383 // time remaining before the cms generation is empty.
384 // Note that the time from the start of the cms collection
385 // to the start of the cms sweep (less than the total
386 // duration of the cms collection) can be used. This
387 // has been tried and some applications experienced
388 // promotion failures early in execution. This was
389 // possibly because the averages were not accurate
390 // enough at the beginning.
391 double CMSStats::time_until_cms_start() const {
392 // We add "gc0_period" to the "work" calculation
393 // below because this query is done (mostly) at the
394 // end of a scavenge, so we need to conservatively
395 // account for that much possible delay
396 // in the query so as to avoid concurrent mode failures
397 // due to starting the collection just a wee bit too
398 // late.
399 double work = cms_duration() + gc0_period();
400 double deadline = time_until_cms_gen_full();
401 // If a concurrent mode failure occurred recently, we want to be
402 // more conservative and halve our expected time_until_cms_gen_full()
403 if (work > deadline) {
404 if (Verbose && PrintGCDetails) {
405 gclog_or_tty->print(
406 " CMSCollector: collect because of anticipated promotion "
407 "before full %3.7f + %3.7f > %3.7f ", cms_duration(),
408 gc0_period(), time_until_cms_gen_full());
409 }
410 return 0.0;
411 }
412 return work - deadline;
413 }
414
415 #ifndef PRODUCT
416 void CMSStats::print_on(outputStream *st) const {
417 st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);
418 st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,
419 gc0_duration(), gc0_period(), gc0_promoted());
420 st->print(",cms_dur=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,
421 cms_duration(), cms_period(), cms_allocated());
422 st->print(",cms_since_beg=%g,cms_since_end=%g",
423 cms_time_since_begin(), cms_time_since_end());
424 st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,
425 _cms_used_at_gc0_begin, _cms_used_at_gc0_end);
426
427 if (valid()) {
428 st->print(",promo_rate=%g,cms_alloc_rate=%g",
429 promotion_rate(), cms_allocation_rate());
651 //
652 void ConcurrentMarkSweepGeneration::update_counters(size_t used) {
653 if (UsePerfData) {
654 _space_counters->update_used(used);
655 _space_counters->update_capacity();
656 _gen_counters->update_all();
657 }
658 }
659
660 void ConcurrentMarkSweepGeneration::print() const {
661 Generation::print();
662 cmsSpace()->print();
663 }
664
665 #ifndef PRODUCT
666 void ConcurrentMarkSweepGeneration::print_statistics() {
667 cmsSpace()->printFLCensus(0);
668 }
669 #endif
670
671 void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) {
672 GenCollectedHeap* gch = GenCollectedHeap::heap();
673 if (PrintGCDetails) {
674 // I didn't want to change the logging when removing the level concept,
675 // but I guess this logging could say "old" or something instead of "1".
676 assert(gch->is_old_gen(this),
677 "The CMS generation should be the old generation");
678 uint level = 1;
679 if (Verbose) {
680 gclog_or_tty->print("[%u %s-%s: " SIZE_FORMAT "(" SIZE_FORMAT ")]",
681 level, short_name(), s, used(), capacity());
682 } else {
683 gclog_or_tty->print("[%u %s-%s: " SIZE_FORMAT "K(" SIZE_FORMAT "K)]",
684 level, short_name(), s, used() / K, capacity() / K);
685 }
686 }
687 if (Verbose) {
688 gclog_or_tty->print(" " SIZE_FORMAT "(" SIZE_FORMAT ")",
689 gch->used(), gch->capacity());
690 } else {
691 gclog_or_tty->print(" " SIZE_FORMAT "K(" SIZE_FORMAT "K)",
692 gch->used() / K, gch->capacity() / K);
693 }
694 }
695
696 size_t
697 ConcurrentMarkSweepGeneration::contiguous_available() const {
698 // dld proposes an improvement in precision here. If the committed
699 // part of the space ends in a free block we should add that to
700 // uncommitted size in the calculation below. Will make this
701 // change later, staying with the approximation below for the
702 // time being. -- ysr.
703 return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
704 }
705
706 size_t
707 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
708 return _cmsSpace->max_alloc_in_words() * HeapWordSize;
709 }
710
711 size_t ConcurrentMarkSweepGeneration::max_available() const {
712 return free() + _virtual_space.uncommitted_size();
713 }
714
715 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
716 size_t available = max_available();
717 size_t av_promo = (size_t)gc_stats()->avg_promoted()->padded_average();
718 bool res = (available >= av_promo) || (available >= max_promotion_in_bytes);
719 if (Verbose && PrintGCDetails) {
720 gclog_or_tty->print_cr(
721 "CMS: promo attempt is%s safe: available(" SIZE_FORMAT ") %s av_promo(" SIZE_FORMAT "),"
722 "max_promo(" SIZE_FORMAT ")",
723 res? "":" not", available, res? ">=":"<",
724 av_promo, max_promotion_in_bytes);
725 }
726 return res;
727 }
728
729 // At a promotion failure dump information on block layout in heap
730 // (cms old generation).
731 void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {
732 if (CMSDumpAtPromotionFailure) {
733 cmsSpace()->dump_at_safepoint_with_locks(collector(), gclog_or_tty);
734 }
735 }
736
737 void ConcurrentMarkSweepGeneration::reset_after_compaction() {
738 // Clear the promotion information. These pointers can be adjusted
739 // along with all the other pointers into the heap but
740 // compaction is expected to be a rare event with
741 // a heap using cms so don't do it without seeing the need.
742 for (uint i = 0; i < ParallelGCThreads; i++) {
743 _par_gc_thread_states[i]->promo.reset();
744 }
745 }
746
747 void ConcurrentMarkSweepGeneration::compute_new_size() {
748 assert_locked_or_safepoint(Heap_lock);
749
750 // If incremental collection failed, we just want to expand
751 // to the limit.
752 if (incremental_collection_failed()) {
753 clear_incremental_collection_failed();
769 void ConcurrentMarkSweepGeneration::compute_new_size_free_list() {
770 assert_locked_or_safepoint(Heap_lock);
771
772 // If incremental collection failed, we just want to expand
773 // to the limit.
774 if (incremental_collection_failed()) {
775 clear_incremental_collection_failed();
776 grow_to_reserved();
777 return;
778 }
779
780 double free_percentage = ((double) free()) / capacity();
781 double desired_free_percentage = (double) MinHeapFreeRatio / 100;
782 double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
783
784 // compute expansion delta needed for reaching desired free percentage
785 if (free_percentage < desired_free_percentage) {
786 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
787 assert(desired_capacity >= capacity(), "invalid expansion size");
788 size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
789 if (PrintGCDetails && Verbose) {
790 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
791 gclog_or_tty->print_cr("\nFrom compute_new_size: ");
792 gclog_or_tty->print_cr(" Free fraction %f", free_percentage);
793 gclog_or_tty->print_cr(" Desired free fraction %f", desired_free_percentage);
794 gclog_or_tty->print_cr(" Maximum free fraction %f", maximum_free_percentage);
795 gclog_or_tty->print_cr(" Capacity " SIZE_FORMAT, capacity() / 1000);
796 gclog_or_tty->print_cr(" Desired capacity " SIZE_FORMAT, desired_capacity / 1000);
797 GenCollectedHeap* gch = GenCollectedHeap::heap();
798 assert(gch->is_old_gen(this), "The CMS generation should always be the old generation");
799 size_t young_size = gch->young_gen()->capacity();
800 gclog_or_tty->print_cr(" Young gen size " SIZE_FORMAT, young_size / 1000);
801 gclog_or_tty->print_cr(" unsafe_max_alloc_nogc " SIZE_FORMAT, unsafe_max_alloc_nogc() / 1000);
802 gclog_or_tty->print_cr(" contiguous available " SIZE_FORMAT, contiguous_available() / 1000);
803 gclog_or_tty->print_cr(" Expand by " SIZE_FORMAT " (bytes)", expand_bytes);
804 }
805 // safe if expansion fails
806 expand_for_gc_cause(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
807 if (PrintGCDetails && Verbose) {
808 gclog_or_tty->print_cr(" Expanded free fraction %f", ((double) free()) / capacity());
809 }
810 } else {
811 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
812 assert(desired_capacity <= capacity(), "invalid expansion size");
813 size_t shrink_bytes = capacity() - desired_capacity;
814 // Don't shrink unless the delta is greater than the minimum shrink we want
815 if (shrink_bytes >= MinHeapDeltaBytes) {
816 shrink_free_list_by(shrink_bytes);
817 }
818 }
819 }
820
821 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
822 return cmsSpace()->freelistLock();
823 }
824
825 HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size, bool tlab) {
826 CMSSynchronousYieldRequest yr;
827 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
828 return have_lock_and_allocate(size, tlab);
829 }
1127 ConcurrentMarkSweepGeneration::
1128 par_oop_since_save_marks_iterate_done(int thread_num) {
1129 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1130 ParScanWithoutBarrierClosure* dummy_cl = NULL;
1131 ps->promo.promoted_oops_iterate_nv(dummy_cl);
1132 }
1133
1134 bool ConcurrentMarkSweepGeneration::should_collect(bool full,
1135 size_t size,
1136 bool tlab)
1137 {
1138 // We allow a STW collection only if a full
1139 // collection was requested.
1140 return full || should_allocate(size, tlab); // FIX ME !!!
1141 // This and promotion failure handling are connected at the
1142 // hip and should be fixed by untying them.
1143 }
1144
1145 bool CMSCollector::shouldConcurrentCollect() {
1146 if (_full_gc_requested) {
1147 if (Verbose && PrintGCDetails) {
1148 gclog_or_tty->print_cr("CMSCollector: collect because of explicit "
1149 " gc request (or gc_locker)");
1150 }
1151 return true;
1152 }
1153
1154 FreelistLocker x(this);
1155 // ------------------------------------------------------------------
1156 // Print out lots of information which affects the initiation of
1157 // a collection.
1158 if (PrintCMSInitiationStatistics && stats().valid()) {
1159 gclog_or_tty->print("CMSCollector shouldConcurrentCollect: ");
1160 gclog_or_tty->stamp();
1161 gclog_or_tty->cr();
1162 stats().print_on(gclog_or_tty);
1163 gclog_or_tty->print_cr("time_until_cms_gen_full %3.7f",
1164 stats().time_until_cms_gen_full());
1165 gclog_or_tty->print_cr("free=" SIZE_FORMAT, _cmsGen->free());
1166 gclog_or_tty->print_cr("contiguous_available=" SIZE_FORMAT,
1167 _cmsGen->contiguous_available());
1168 gclog_or_tty->print_cr("promotion_rate=%g", stats().promotion_rate());
1169 gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate());
1170 gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy());
1171 gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy());
1172 gclog_or_tty->print_cr("cms_time_since_begin=%3.7f", stats().cms_time_since_begin());
1173 gclog_or_tty->print_cr("cms_time_since_end=%3.7f", stats().cms_time_since_end());
1174 gclog_or_tty->print_cr("metadata initialized %d",
1175 MetaspaceGC::should_concurrent_collect());
1176 }
1177 // ------------------------------------------------------------------
1178
1179 // If the estimated time to complete a cms collection (cms_duration())
1180 // is less than the estimated time remaining until the cms generation
1181 // is full, start a collection.
1182 if (!UseCMSInitiatingOccupancyOnly) {
1183 if (stats().valid()) {
1184 if (stats().time_until_cms_start() == 0.0) {
1185 return true;
1186 }
1187 } else {
1188 // We want to conservatively collect somewhat early in order
1189 // to try and "bootstrap" our CMS/promotion statistics;
1190 // this branch will not fire after the first successful CMS
1191 // collection because the stats should then be valid.
1192 if (_cmsGen->occupancy() >= _bootstrap_occupancy) {
1193 if (Verbose && PrintGCDetails) {
1194 gclog_or_tty->print_cr(
1195 " CMSCollector: collect for bootstrapping statistics:"
1196 " occupancy = %f, boot occupancy = %f", _cmsGen->occupancy(),
1197 _bootstrap_occupancy);
1198 }
1199 return true;
1200 }
1201 }
1202 }
1203
1204 // Otherwise, we start a collection cycle if
1205 // old gen want a collection cycle started. Each may use
1206 // an appropriate criterion for making this decision.
1207 // XXX We need to make sure that the gen expansion
1208 // criterion dovetails well with this. XXX NEED TO FIX THIS
1209 if (_cmsGen->should_concurrent_collect()) {
1210 if (Verbose && PrintGCDetails) {
1211 gclog_or_tty->print_cr("CMS old gen initiated");
1212 }
1213 return true;
1214 }
1215
1216 // We start a collection if we believe an incremental collection may fail;
1217 // this is not likely to be productive in practice because it's probably too
1218 // late anyway.
1219 GenCollectedHeap* gch = GenCollectedHeap::heap();
1220 assert(gch->collector_policy()->is_generation_policy(),
1221 "You may want to check the correctness of the following");
1222 if (gch->incremental_collection_will_fail(true /* consult_young */)) {
1223 if (Verbose && PrintGCDetails) {
1224 gclog_or_tty->print("CMSCollector: collect because incremental collection will fail ");
1225 }
1226 return true;
1227 }
1228
1229 if (MetaspaceGC::should_concurrent_collect()) {
1230 if (Verbose && PrintGCDetails) {
1231 gclog_or_tty->print("CMSCollector: collect for metadata allocation ");
1232 }
1233 return true;
1234 }
1235
1236 // CMSTriggerInterval starts a CMS cycle if enough time has passed.
1237 if (CMSTriggerInterval >= 0) {
1238 if (CMSTriggerInterval == 0) {
1239 // Trigger always
1240 return true;
1241 }
1242
1243 // Check the CMS time since begin (we do not check the stats validity
1244 // as we want to be able to trigger the first CMS cycle as well)
1245 if (stats().cms_time_since_begin() >= (CMSTriggerInterval / ((double) MILLIUNITS))) {
1246 if (Verbose && PrintGCDetails) {
1247 if (stats().valid()) {
1248 gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (time since last begin %3.7f secs)",
1249 stats().cms_time_since_begin());
1250 } else {
1251 gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (first collection)");
1252 }
1253 }
1254 return true;
1255 }
1256 }
1257
1258 return false;
1259 }
1260
1261 void CMSCollector::set_did_compact(bool v) { _cmsGen->set_did_compact(v); }
1262
1263 // Clear _expansion_cause fields of constituent generations
1264 void CMSCollector::clear_expansion_cause() {
1265 _cmsGen->clear_expansion_cause();
1266 }
1267
1268 // We should be conservative in starting a collection cycle. To
1269 // start too eagerly runs the risk of collecting too often in the
1270 // extreme. To collect too rarely falls back on full collections,
1271 // which works, even if not optimum in terms of concurrent work.
1272 // As a work around for too eagerly collecting, use the flag
1275 // collections.
1276 // We want to start a new collection cycle if any of the following
1277 // conditions hold:
1278 // . our current occupancy exceeds the configured initiating occupancy
1279 // for this generation, or
1280 // . we recently needed to expand this space and have not, since that
1281 // expansion, done a collection of this generation, or
1282 // . the underlying space believes that it may be a good idea to initiate
1283 // a concurrent collection (this may be based on criteria such as the
1284 // following: the space uses linear allocation and linear allocation is
1285 // going to fail, or there is believed to be excessive fragmentation in
1286 // the generation, etc... or ...
1287 // [.(currently done by CMSCollector::shouldConcurrentCollect() only for
1288 // the case of the old generation; see CR 6543076):
1289 // we may be approaching a point at which allocation requests may fail because
1290 // we will be out of sufficient free space given allocation rate estimates.]
1291 bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const {
1292
1293 assert_lock_strong(freelistLock());
1294 if (occupancy() > initiating_occupancy()) {
1295 if (PrintGCDetails && Verbose) {
1296 gclog_or_tty->print(" %s: collect because of occupancy %f / %f ",
1297 short_name(), occupancy(), initiating_occupancy());
1298 }
1299 return true;
1300 }
1301 if (UseCMSInitiatingOccupancyOnly) {
1302 return false;
1303 }
1304 if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) {
1305 if (PrintGCDetails && Verbose) {
1306 gclog_or_tty->print(" %s: collect because expanded for allocation ",
1307 short_name());
1308 }
1309 return true;
1310 }
1311 return false;
1312 }
1313
1314 void ConcurrentMarkSweepGeneration::collect(bool full,
1315 bool clear_all_soft_refs,
1316 size_t size,
1317 bool tlab)
1318 {
1319 collector()->collect(full, clear_all_soft_refs, size, tlab);
1320 }
1321
1322 void CMSCollector::collect(bool full,
1323 bool clear_all_soft_refs,
1324 size_t size,
1325 bool tlab)
1326 {
1327 // The following "if" branch is present for defensive reasons.
1328 // In the current uses of this interface, it can be replaced with:
1345 GenCollectedHeap* gch = GenCollectedHeap::heap();
1346 unsigned int gc_count = gch->total_full_collections();
1347 if (gc_count == full_gc_count) {
1348 MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag);
1349 _full_gc_requested = true;
1350 _full_gc_cause = cause;
1351 CGC_lock->notify(); // nudge CMS thread
1352 } else {
1353 assert(gc_count > full_gc_count, "Error: causal loop");
1354 }
1355 }
1356
1357 bool CMSCollector::is_external_interruption() {
1358 GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause();
1359 return GCCause::is_user_requested_gc(cause) ||
1360 GCCause::is_serviceability_requested_gc(cause);
1361 }
1362
1363 void CMSCollector::report_concurrent_mode_interruption() {
1364 if (is_external_interruption()) {
1365 if (PrintGCDetails) {
1366 gclog_or_tty->print(" (concurrent mode interrupted)");
1367 }
1368 } else {
1369 if (PrintGCDetails) {
1370 gclog_or_tty->print(" (concurrent mode failure)");
1371 }
1372 _gc_tracer_cm->report_concurrent_mode_failure();
1373 }
1374 }
1375
1376
1377 // The foreground and background collectors need to coordinate in order
1378 // to make sure that they do not mutually interfere with CMS collections.
1379 // When a background collection is active,
1380 // the foreground collector may need to take over (preempt) and
1381 // synchronously complete an ongoing collection. Depending on the
1382 // frequency of the background collections and the heap usage
1383 // of the application, this preemption can be seldom or frequent.
1384 // There are only certain
1385 // points in the background collection that the "collection-baton"
1386 // can be passed to the foreground collector.
1387 //
1388 // The foreground collector will wait for the baton before
1389 // starting any part of the collection. The foreground collector
1390 // will only wait at one location.
1391 //
1485 CGC_lock->notify();
1486 assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(),
1487 "Possible deadlock");
1488 while (_foregroundGCShouldWait) {
1489 // wait for notification
1490 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1491 // Possibility of delay/starvation here, since CMS token does
1492 // not know to give priority to VM thread? Actually, i think
1493 // there wouldn't be any delay/starvation, but the proof of
1494 // that "fact" (?) appears non-trivial. XXX 20011219YSR
1495 }
1496 ConcurrentMarkSweepThread::set_CMS_flag(
1497 ConcurrentMarkSweepThread::CMS_vm_has_token);
1498 }
1499 }
1500 // The CMS_token is already held. Get back the other locks.
1501 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1502 "VM thread should have CMS token");
1503 getFreelistLocks();
1504 bitMapLock()->lock_without_safepoint_check();
1505 if (TraceCMSState) {
1506 gclog_or_tty->print_cr("CMS foreground collector has asked for control "
1507 INTPTR_FORMAT " with first state %d", p2i(Thread::current()), first_state);
1508 gclog_or_tty->print_cr(" gets control with state %d", _collectorState);
1509 }
1510
1511 // Inform cms gen if this was due to partial collection failing.
1512 // The CMS gen may use this fact to determine its expansion policy.
1513 GenCollectedHeap* gch = GenCollectedHeap::heap();
1514 if (gch->incremental_collection_will_fail(false /* don't consult_young */)) {
1515 assert(!_cmsGen->incremental_collection_failed(),
1516 "Should have been noticed, reacted to and cleared");
1517 _cmsGen->set_incremental_collection_failed();
1518 }
1519
1520 if (first_state > Idling) {
1521 report_concurrent_mode_interruption();
1522 }
1523
1524 set_did_compact(true);
1525
1526 // If the collection is being acquired from the background
1527 // collector, there may be references on the discovered
1528 // references lists. Abandon those references, since some
1529 // of them may have become unreachable after concurrent
1563 // after obtaining the free list locks for the
1564 // two generations.
1565 void CMSCollector::compute_new_size() {
1566 assert_locked_or_safepoint(Heap_lock);
1567 FreelistLocker z(this);
1568 MetaspaceGC::compute_new_size();
1569 _cmsGen->compute_new_size_free_list();
1570 }
1571
1572 // A work method used by the foreground collector to do
1573 // a mark-sweep-compact.
1574 void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
1575 GenCollectedHeap* gch = GenCollectedHeap::heap();
1576
1577 STWGCTimer* gc_timer = GenMarkSweep::gc_timer();
1578 gc_timer->register_gc_start();
1579
1580 SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer();
1581 gc_tracer->report_gc_start(gch->gc_cause(), gc_timer->gc_start());
1582
1583 GCTraceTime t("CMS:MSC ", PrintGCDetails && Verbose, true, NULL);
1584
1585 // Temporarily widen the span of the weak reference processing to
1586 // the entire heap.
1587 MemRegion new_span(GenCollectedHeap::heap()->reserved_region());
1588 ReferenceProcessorSpanMutator rp_mut_span(ref_processor(), new_span);
1589 // Temporarily, clear the "is_alive_non_header" field of the
1590 // reference processor.
1591 ReferenceProcessorIsAliveMutator rp_mut_closure(ref_processor(), NULL);
1592 // Temporarily make reference _processing_ single threaded (non-MT).
1593 ReferenceProcessorMTProcMutator rp_mut_mt_processing(ref_processor(), false);
1594 // Temporarily make refs discovery atomic
1595 ReferenceProcessorAtomicMutator rp_mut_atomic(ref_processor(), true);
1596 // Temporarily make reference _discovery_ single threaded (non-MT)
1597 ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false);
1598
1599 ref_processor()->set_enqueuing_is_done(false);
1600 ref_processor()->enable_discovery();
1601 ref_processor()->setup_policy(clear_all_soft_refs);
1602 // If an asynchronous collection finishes, the _modUnionTable is
1603 // all clear. If we are assuming the collection from an asynchronous
1648 // Clear any data recorded in the PLAB chunk arrays.
1649 if (_survivor_plab_array != NULL) {
1650 reset_survivor_plab_arrays();
1651 }
1652
1653 // Adjust the per-size allocation stats for the next epoch.
1654 _cmsGen->cmsSpace()->endSweepFLCensus(sweep_count() /* fake */);
1655 // Restart the "inter sweep timer" for the next epoch.
1656 _inter_sweep_timer.reset();
1657 _inter_sweep_timer.start();
1658
1659 gc_timer->register_gc_end();
1660
1661 gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions());
1662
1663 // For a mark-sweep-compact, compute_new_size() will be called
1664 // in the heap's do_collection() method.
1665 }
1666
1667 void CMSCollector::print_eden_and_survivor_chunk_arrays() {
1668 ContiguousSpace* eden_space = _young_gen->eden();
1669 ContiguousSpace* from_space = _young_gen->from();
1670 ContiguousSpace* to_space = _young_gen->to();
1671 // Eden
1672 if (_eden_chunk_array != NULL) {
1673 gclog_or_tty->print_cr("eden " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1674 p2i(eden_space->bottom()), p2i(eden_space->top()),
1675 p2i(eden_space->end()), eden_space->capacity());
1676 gclog_or_tty->print_cr("_eden_chunk_index=" SIZE_FORMAT ", "
1677 "_eden_chunk_capacity=" SIZE_FORMAT,
1678 _eden_chunk_index, _eden_chunk_capacity);
1679 for (size_t i = 0; i < _eden_chunk_index; i++) {
1680 gclog_or_tty->print_cr("_eden_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT,
1681 i, p2i(_eden_chunk_array[i]));
1682 }
1683 }
1684 // Survivor
1685 if (_survivor_chunk_array != NULL) {
1686 gclog_or_tty->print_cr("survivor " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1687 p2i(from_space->bottom()), p2i(from_space->top()),
1688 p2i(from_space->end()), from_space->capacity());
1689 gclog_or_tty->print_cr("_survivor_chunk_index=" SIZE_FORMAT ", "
1690 "_survivor_chunk_capacity=" SIZE_FORMAT,
1691 _survivor_chunk_index, _survivor_chunk_capacity);
1692 for (size_t i = 0; i < _survivor_chunk_index; i++) {
1693 gclog_or_tty->print_cr("_survivor_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT,
1694 i, p2i(_survivor_chunk_array[i]));
1695 }
1696 }
1697 }
1698
1699 void CMSCollector::getFreelistLocks() const {
1700 // Get locks for all free lists in all generations that this
1701 // collector is responsible for
1702 _cmsGen->freelistLock()->lock_without_safepoint_check();
1703 }
1704
1705 void CMSCollector::releaseFreelistLocks() const {
1706 // Release locks for all free lists in all generations that this
1707 // collector is responsible for
1708 _cmsGen->freelistLock()->unlock();
1709 }
1710
1711 bool CMSCollector::haveFreelistLocks() const {
1712 // Check locks for all free lists in all generations that this
1713 // collector is responsible for
1714 assert_lock_strong(_cmsGen->freelistLock());
1763 _collectorState = InitialMarking;
1764 register_gc_start(cause);
1765 // Reset the expansion cause, now that we are about to begin
1766 // a new cycle.
1767 clear_expansion_cause();
1768
1769 // Clear the MetaspaceGC flag since a concurrent collection
1770 // is starting but also clear it after the collection.
1771 MetaspaceGC::set_should_concurrent_collect(false);
1772 }
1773 // Decide if we want to enable class unloading as part of the
1774 // ensuing concurrent GC cycle.
1775 update_should_unload_classes();
1776 _full_gc_requested = false; // acks all outstanding full gc requests
1777 _full_gc_cause = GCCause::_no_gc;
1778 // Signal that we are about to start a collection
1779 gch->increment_total_full_collections(); // ... starting a collection cycle
1780 _collection_count_start = gch->total_full_collections();
1781 }
1782
1783 // Used for PrintGC
1784 size_t prev_used = 0;
1785 if (PrintGC && Verbose) {
1786 prev_used = _cmsGen->used();
1787 }
1788
1789 // The change of the collection state is normally done at this level;
1790 // the exceptions are phases that are executed while the world is
1791 // stopped. For those phases the change of state is done while the
1792 // world is stopped. For baton passing purposes this allows the
1793 // background collector to finish the phase and change state atomically.
1794 // The foreground collector cannot wait on a phase that is done
1795 // while the world is stopped because the foreground collector already
1796 // has the world stopped and would deadlock.
1797 while (_collectorState != Idling) {
1798 if (TraceCMSState) {
1799 gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
1800 p2i(Thread::current()), _collectorState);
1801 }
1802 // The foreground collector
1803 // holds the Heap_lock throughout its collection.
1804 // holds the CMS token (but not the lock)
1805 // except while it is waiting for the background collector to yield.
1806 //
1807 // The foreground collector should be blocked (not for long)
1808 // if the background collector is about to start a phase
1809 // executed with world stopped. If the background
1810 // collector has already started such a phase, the
1811 // foreground collector is blocked waiting for the
1812 // Heap_lock. The stop-world phases (InitialMarking and FinalMarking)
1813 // are executed in the VM thread.
1814 //
1815 // The locking order is
1816 // PendingListLock (PLL) -- if applicable (FinalMarking)
1817 // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue())
1818 // CMS token (claimed in
1819 // stop_world_and_do() -->
1820 // safepoint_synchronize() -->
1821 // CMSThread::synchronize())
1822
1823 {
1824 // Check if the FG collector wants us to yield.
1825 CMSTokenSync x(true); // is cms thread
1826 if (waitForForegroundGC()) {
1827 // We yielded to a foreground GC, nothing more to be
1828 // done this round.
1829 assert(_foregroundGCShouldWait == false, "We set it to false in "
1830 "waitForForegroundGC()");
1831 if (TraceCMSState) {
1832 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1833 " exiting collection CMS state %d",
1834 p2i(Thread::current()), _collectorState);
1835 }
1836 return;
1837 } else {
1838 // The background collector can run but check to see if the
1839 // foreground collector has done a collection while the
1840 // background collector was waiting to get the CGC_lock
1841 // above. If yes, break so that _foregroundGCShouldWait
1842 // is cleared before returning.
1843 if (_collectorState == Idling) {
1844 break;
1845 }
1846 }
1847 }
1848
1849 assert(_foregroundGCShouldWait, "Foreground collector, if active, "
1850 "should be waiting");
1851
1852 switch (_collectorState) {
1853 case InitialMarking:
1854 {
1855 ReleaseForegroundGC x(this);
1919 break;
1920 }
1921 case Resetting:
1922 // CMS heap resizing has been completed
1923 reset_concurrent();
1924 assert(_collectorState == Idling, "Collector state should "
1925 "have changed");
1926
1927 MetaspaceGC::set_should_concurrent_collect(false);
1928
1929 stats().record_cms_end();
1930 // Don't move the concurrent_phases_end() and compute_new_size()
1931 // calls to here because a preempted background collection
1932 // has it's state set to "Resetting".
1933 break;
1934 case Idling:
1935 default:
1936 ShouldNotReachHere();
1937 break;
1938 }
1939 if (TraceCMSState) {
1940 gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
1941 p2i(Thread::current()), _collectorState);
1942 }
1943 assert(_foregroundGCShouldWait, "block post-condition");
1944 }
1945
1946 // Should this be in gc_epilogue?
1947 collector_policy()->counters()->update_counters();
1948
1949 {
1950 // Clear _foregroundGCShouldWait and, in the event that the
1951 // foreground collector is waiting, notify it, before
1952 // returning.
1953 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1954 _foregroundGCShouldWait = false;
1955 if (_foregroundGCIsActive) {
1956 CGC_lock->notify();
1957 }
1958 assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1959 "Possible deadlock");
1960 }
1961 if (TraceCMSState) {
1962 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1963 " exiting collection CMS state %d",
1964 p2i(Thread::current()), _collectorState);
1965 }
1966 if (PrintGC && Verbose) {
1967 _cmsGen->print_heap_change(prev_used);
1968 }
1969 }
1970
1971 void CMSCollector::register_gc_start(GCCause::Cause cause) {
1972 _cms_start_registered = true;
1973 _gc_timer_cm->register_gc_start();
1974 _gc_tracer_cm->report_gc_start(cause, _gc_timer_cm->gc_start());
1975 }
1976
1977 void CMSCollector::register_gc_end() {
1978 if (_cms_start_registered) {
1979 report_heap_summary(GCWhen::AfterGC);
1980
1981 _gc_timer_cm->register_gc_end();
1982 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
1983 _cms_start_registered = false;
1984 }
1985 }
1986
1987 void CMSCollector::save_heap_summary() {
1988 GenCollectedHeap* gch = GenCollectedHeap::heap();
2000 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
2001 "CMS thread should have CMS token");
2002 // Block the foreground collector until the
2003 // background collectors decides whether to
2004 // yield.
2005 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2006 _foregroundGCShouldWait = true;
2007 if (_foregroundGCIsActive) {
2008 // The background collector yields to the
2009 // foreground collector and returns a value
2010 // indicating that it has yielded. The foreground
2011 // collector can proceed.
2012 res = true;
2013 _foregroundGCShouldWait = false;
2014 ConcurrentMarkSweepThread::clear_CMS_flag(
2015 ConcurrentMarkSweepThread::CMS_cms_has_token);
2016 ConcurrentMarkSweepThread::set_CMS_flag(
2017 ConcurrentMarkSweepThread::CMS_cms_wants_token);
2018 // Get a possibly blocked foreground thread going
2019 CGC_lock->notify();
2020 if (TraceCMSState) {
2021 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d",
2022 p2i(Thread::current()), _collectorState);
2023 }
2024 while (_foregroundGCIsActive) {
2025 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
2026 }
2027 ConcurrentMarkSweepThread::set_CMS_flag(
2028 ConcurrentMarkSweepThread::CMS_cms_has_token);
2029 ConcurrentMarkSweepThread::clear_CMS_flag(
2030 ConcurrentMarkSweepThread::CMS_cms_wants_token);
2031 }
2032 if (TraceCMSState) {
2033 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d",
2034 p2i(Thread::current()), _collectorState);
2035 }
2036 return res;
2037 }
2038
2039 // Because of the need to lock the free lists and other structures in
2040 // the collector, common to all the generations that the collector is
2041 // collecting, we need the gc_prologues of individual CMS generations
2042 // delegate to their collector. It may have been simpler had the
2043 // current infrastructure allowed one to call a prologue on a
2044 // collector. In the absence of that we have the generation's
2045 // prologue delegate to the collector, which delegates back
2046 // some "local" work to a worker method in the individual generations
2047 // that it's responsible for collecting, while itself doing any
2048 // work common to all generations it's responsible for. A similar
2049 // comment applies to the gc_epilogue()'s.
2050 // The role of the variable _between_prologue_and_epilogue is to
2051 // enforce the invocation protocol.
2052 void CMSCollector::gc_prologue(bool full) {
2053 // Call gc_prologue_work() for the CMSGen
2054 // we are responsible for.
2055
2112 // collecting.
2113 collector()->gc_prologue(full);
2114 }
2115
2116 // This is a "private" interface for use by this generation's CMSCollector.
2117 // Not to be called directly by any other entity (for instance,
2118 // GenCollectedHeap, which calls the "public" gc_prologue method above).
2119 void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
2120 bool registerClosure, ModUnionClosure* modUnionClosure) {
2121 assert(!incremental_collection_failed(), "Shouldn't be set yet");
2122 assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,
2123 "Should be NULL");
2124 if (registerClosure) {
2125 cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure);
2126 }
2127 cmsSpace()->gc_prologue();
2128 // Clear stat counters
2129 NOT_PRODUCT(
2130 assert(_numObjectsPromoted == 0, "check");
2131 assert(_numWordsPromoted == 0, "check");
2132 if (Verbose && PrintGC) {
2133 gclog_or_tty->print("Allocated " SIZE_FORMAT " objects, "
2134 SIZE_FORMAT " bytes concurrently",
2135 _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord));
2136 }
2137 _numObjectsAllocated = 0;
2138 _numWordsAllocated = 0;
2139 )
2140 }
2141
2142 void CMSCollector::gc_epilogue(bool full) {
2143 // The following locking discipline assumes that we are only called
2144 // when the world is stopped.
2145 assert(SafepointSynchronize::is_at_safepoint(),
2146 "world is stopped assumption");
2147
2148 // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks
2149 // if linear allocation blocks need to be appropriately marked to allow the
2150 // the blocks to be parsable. We also check here whether we need to nudge the
2151 // CMS collector thread to start a new cycle (if it's not already active).
2152 assert( Thread::current()->is_VM_thread()
2153 || ( CMSScavengeBeforeRemark
2154 && Thread::current()->is_ConcurrentGC_thread()),
2155 "Incorrect thread type for epilogue execution");
2156
2193 _between_prologue_and_epilogue = false; // ready for next cycle
2194 }
2195
2196 void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
2197 collector()->gc_epilogue(full);
2198
2199 // Also reset promotion tracking in par gc thread states.
2200 for (uint i = 0; i < ParallelGCThreads; i++) {
2201 _par_gc_thread_states[i]->promo.stopTrackingPromotions(i);
2202 }
2203 }
2204
2205 void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) {
2206 assert(!incremental_collection_failed(), "Should have been cleared");
2207 cmsSpace()->setPreconsumptionDirtyCardClosure(NULL);
2208 cmsSpace()->gc_epilogue();
2209 // Print stat counters
2210 NOT_PRODUCT(
2211 assert(_numObjectsAllocated == 0, "check");
2212 assert(_numWordsAllocated == 0, "check");
2213 if (Verbose && PrintGC) {
2214 gclog_or_tty->print("Promoted " SIZE_FORMAT " objects, "
2215 SIZE_FORMAT " bytes",
2216 _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord));
2217 }
2218 _numObjectsPromoted = 0;
2219 _numWordsPromoted = 0;
2220 )
2221
2222 if (PrintGC && Verbose) {
2223 // Call down the chain in contiguous_available needs the freelistLock
2224 // so print this out before releasing the freeListLock.
2225 gclog_or_tty->print(" Contiguous available " SIZE_FORMAT " bytes ",
2226 contiguous_available());
2227 }
2228 }
2229
2230 #ifndef PRODUCT
2231 bool CMSCollector::have_cms_token() {
2232 Thread* thr = Thread::current();
2233 if (thr->is_VM_thread()) {
2234 return ConcurrentMarkSweepThread::vm_thread_has_cms_token();
2235 } else if (thr->is_ConcurrentGC_thread()) {
2236 return ConcurrentMarkSweepThread::cms_thread_has_cms_token();
2237 } else if (thr->is_GC_task_thread()) {
2238 return ConcurrentMarkSweepThread::vm_thread_has_cms_token() &&
2239 ParGCRareEvent_lock->owned_by_self();
2240 }
2241 return false;
2242 }
2243
2244 // Check reachability of the given heap address in CMS generation,
2245 // treating all other generations as roots.
2246 bool CMSCollector::is_cms_reachable(HeapWord* addr) {
2247 // We could "guarantee" below, rather than assert, but I'll
2291 }
2292 }
2293
2294 ////////////////////////////////////////////////////////
2295 // CMS Verification Support
2296 ////////////////////////////////////////////////////////
2297 // Following the remark phase, the following invariant
2298 // should hold -- each object in the CMS heap which is
2299 // marked in markBitMap() should be marked in the verification_mark_bm().
2300
2301 class VerifyMarkedClosure: public BitMapClosure {
2302 CMSBitMap* _marks;
2303 bool _failed;
2304
2305 public:
2306 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2307
2308 bool do_bit(size_t offset) {
2309 HeapWord* addr = _marks->offsetToHeapWord(offset);
2310 if (!_marks->isMarked(addr)) {
2311 oop(addr)->print_on(gclog_or_tty);
2312 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
2313 _failed = true;
2314 }
2315 return true;
2316 }
2317
2318 bool failed() { return _failed; }
2319 };
2320
2321 bool CMSCollector::verify_after_remark(bool silent) {
2322 if (!silent) gclog_or_tty->print(" [Verifying CMS Marking... ");
2323 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2324 static bool init = false;
2325
2326 assert(SafepointSynchronize::is_at_safepoint(),
2327 "Else mutations in object graph will make answer suspect");
2328 assert(have_cms_token(),
2329 "Else there may be mutual interference in use of "
2330 " verification data structures");
2331 assert(_collectorState > Marking && _collectorState <= Sweeping,
2332 "Else marking info checked here may be obsolete");
2333 assert(haveFreelistLocks(), "must hold free list locks");
2334 assert_lock_strong(bitMapLock());
2335
2336
2337 // Allocate marking bit map if not already allocated
2338 if (!init) { // first time
2339 if (!verification_mark_bm()->allocate(_span)) {
2340 return false;
2341 }
2342 init = true;
2365 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2366 // Update the saved marks which may affect the root scans.
2367 gch->save_marks();
2368
2369 if (CMSRemarkVerifyVariant == 1) {
2370 // In this first variant of verification, we complete
2371 // all marking, then check if the new marks-vector is
2372 // a subset of the CMS marks-vector.
2373 verify_after_remark_work_1();
2374 } else if (CMSRemarkVerifyVariant == 2) {
2375 // In this second variant of verification, we flag an error
2376 // (i.e. an object reachable in the new marks-vector not reachable
2377 // in the CMS marks-vector) immediately, also indicating the
2378 // identify of an object (A) that references the unmarked object (B) --
2379 // presumably, a mutation to A failed to be picked up by preclean/remark?
2380 verify_after_remark_work_2();
2381 } else {
2382 warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant",
2383 CMSRemarkVerifyVariant);
2384 }
2385 if (!silent) gclog_or_tty->print(" done] ");
2386 return true;
2387 }
2388
2389 void CMSCollector::verify_after_remark_work_1() {
2390 ResourceMark rm;
2391 HandleMark hm;
2392 GenCollectedHeap* gch = GenCollectedHeap::heap();
2393
2394 // Get a clear set of claim bits for the roots processing to work with.
2395 ClassLoaderDataGraph::clear_claimed_marks();
2396
2397 // Mark from roots one level into CMS
2398 MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
2399 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2400
2401 {
2402 StrongRootsScope srs(1);
2403
2404 gch->gen_process_roots(&srs,
2405 GenCollectedHeap::OldGen,
2417 false /* don't yield */, true /* verifying */);
2418 assert(_restart_addr == NULL, "Expected pre-condition");
2419 verification_mark_bm()->iterate(&markFromRootsClosure);
2420 while (_restart_addr != NULL) {
2421 // Deal with stack overflow: by restarting at the indicated
2422 // address.
2423 HeapWord* ra = _restart_addr;
2424 markFromRootsClosure.reset(ra);
2425 _restart_addr = NULL;
2426 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2427 }
2428 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2429 verify_work_stacks_empty();
2430
2431 // Marking completed -- now verify that each bit marked in
2432 // verification_mark_bm() is also marked in markBitMap(); flag all
2433 // errors by printing corresponding objects.
2434 VerifyMarkedClosure vcl(markBitMap());
2435 verification_mark_bm()->iterate(&vcl);
2436 if (vcl.failed()) {
2437 gclog_or_tty->print("Verification failed");
2438 gch->print_on(gclog_or_tty);
2439 fatal("CMS: failed marking verification after remark");
2440 }
2441 }
2442
2443 class VerifyKlassOopsKlassClosure : public KlassClosure {
2444 class VerifyKlassOopsClosure : public OopClosure {
2445 CMSBitMap* _bitmap;
2446 public:
2447 VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { }
2448 void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); }
2449 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2450 } _oop_closure;
2451 public:
2452 VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {}
2453 void do_klass(Klass* k) {
2454 k->oops_do(&_oop_closure);
2455 }
2456 };
2457
2458 void CMSCollector::verify_after_remark_work_2() {
2711 expand_for_gc_cause(word_size*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_allocation);
2712 if (GCExpandToAllocateDelayMillis > 0) {
2713 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2714 }
2715 return have_lock_and_allocate(word_size, tlab);
2716 }
2717
2718 void ConcurrentMarkSweepGeneration::expand_for_gc_cause(
2719 size_t bytes,
2720 size_t expand_bytes,
2721 CMSExpansionCause::Cause cause)
2722 {
2723
2724 bool success = expand(bytes, expand_bytes);
2725
2726 // remember why we expanded; this information is used
2727 // by shouldConcurrentCollect() when making decisions on whether to start
2728 // a new CMS cycle.
2729 if (success) {
2730 set_expansion_cause(cause);
2731 if (PrintGCDetails && Verbose) {
2732 gclog_or_tty->print_cr("Expanded CMS gen for %s",
2733 CMSExpansionCause::to_string(cause));
2734 }
2735 }
2736 }
2737
2738 HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) {
2739 HeapWord* res = NULL;
2740 MutexLocker x(ParGCRareEvent_lock);
2741 while (true) {
2742 // Expansion by some other thread might make alloc OK now:
2743 res = ps->lab.alloc(word_sz);
2744 if (res != NULL) return res;
2745 // If there's not enough expansion space available, give up.
2746 if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) {
2747 return NULL;
2748 }
2749 // Otherwise, we try expansion.
2750 expand_for_gc_cause(word_sz*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_allocate_par_lab);
2751 // Now go around the loop and try alloc again;
2752 // A competing par_promote might beat us to the expansion space,
2753 // so we may go around the loop again if promotion fails again.
2754 if (GCExpandToAllocateDelayMillis > 0) {
2782 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2783 }
2784 }
2785 }
2786
2787 void ConcurrentMarkSweepGeneration::shrink(size_t bytes) {
2788 // Only shrink if a compaction was done so that all the free space
2789 // in the generation is in a contiguous block at the end.
2790 if (did_compact()) {
2791 CardGeneration::shrink(bytes);
2792 }
2793 }
2794
2795 void ConcurrentMarkSweepGeneration::assert_correct_size_change_locking() {
2796 assert_locked_or_safepoint(Heap_lock);
2797 }
2798
2799 void ConcurrentMarkSweepGeneration::shrink_free_list_by(size_t bytes) {
2800 assert_locked_or_safepoint(Heap_lock);
2801 assert_lock_strong(freelistLock());
2802 if (PrintGCDetails && Verbose) {
2803 warning("Shrinking of CMS not yet implemented");
2804 }
2805 return;
2806 }
2807
2808
2809 // Simple ctor/dtor wrapper for accounting & timer chores around concurrent
2810 // phases.
2811 class CMSPhaseAccounting: public StackObj {
2812 public:
2813 CMSPhaseAccounting(CMSCollector *collector,
2814 const char *phase,
2815 bool print_cr = true);
2816 ~CMSPhaseAccounting();
2817
2818 private:
2819 CMSCollector *_collector;
2820 const char *_phase;
2821 elapsedTimer _wallclock;
2822 bool _print_cr;
2823
2824 public:
2825 // Not MT-safe; so do not pass around these StackObj's
2826 // where they may be accessed by other threads.
2827 jlong wallclock_millis() {
2828 assert(_wallclock.is_active(), "Wall clock should not stop");
2829 _wallclock.stop(); // to record time
2830 jlong ret = _wallclock.milliseconds();
2831 _wallclock.start(); // restart
2832 return ret;
2833 }
2834 };
2835
2836 CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector,
2837 const char *phase,
2838 bool print_cr) :
2839 _collector(collector), _phase(phase), _print_cr(print_cr) {
2840
2841 if (PrintCMSStatistics != 0) {
2842 _collector->resetYields();
2843 }
2844 if (PrintGCDetails) {
2845 gclog_or_tty->gclog_stamp();
2846 gclog_or_tty->print_cr("[%s-concurrent-%s-start]",
2847 _collector->cmsGen()->short_name(), _phase);
2848 }
2849 _collector->resetTimer();
2850 _wallclock.start();
2851 _collector->startTimer();
2852 }
2853
2854 CMSPhaseAccounting::~CMSPhaseAccounting() {
2855 assert(_wallclock.is_active(), "Wall clock should not have stopped");
2856 _collector->stopTimer();
2857 _wallclock.stop();
2858 if (PrintGCDetails) {
2859 gclog_or_tty->gclog_stamp();
2860 gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]",
2861 _collector->cmsGen()->short_name(),
2862 _phase, _collector->timerValue(), _wallclock.seconds());
2863 if (_print_cr) {
2864 gclog_or_tty->cr();
2865 }
2866 if (PrintCMSStatistics != 0) {
2867 gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase,
2868 _collector->yields());
2869 }
2870 }
2871 }
2872
2873 // CMS work
2874
2875 // The common parts of CMSParInitialMarkTask and CMSParRemarkTask.
2876 class CMSParMarkTask : public AbstractGangTask {
2877 protected:
2878 CMSCollector* _collector;
2879 uint _n_workers;
2880 CMSParMarkTask(const char* name, CMSCollector* collector, uint n_workers) :
2881 AbstractGangTask(name),
2882 _collector(collector),
2883 _n_workers(n_workers) {}
2884 // Work method in support of parallel rescan ... of young gen spaces
2885 void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl,
2886 ContiguousSpace* space,
2887 HeapWord** chunk_array, size_t chunk_top);
2888 void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl);
2889 };
2890
2917 Mutex::_no_safepoint_check_flag);
2918 checkpointRootsInitialWork();
2919 // enable ("weak") refs discovery
2920 rp->enable_discovery();
2921 _collectorState = Marking;
2922 }
2923 }
2924
2925 void CMSCollector::checkpointRootsInitialWork() {
2926 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
2927 assert(_collectorState == InitialMarking, "just checking");
2928
2929 // Already have locks.
2930 assert_lock_strong(bitMapLock());
2931 assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
2932
2933 // Setup the verification and class unloading state for this
2934 // CMS collection cycle.
2935 setup_cms_unloading_and_verification_state();
2936
2937 NOT_PRODUCT(GCTraceTime t("\ncheckpointRootsInitialWork",
2938 PrintGCDetails && Verbose, true, _gc_timer_cm);)
2939
2940 // Reset all the PLAB chunk arrays if necessary.
2941 if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
2942 reset_survivor_plab_arrays();
2943 }
2944
2945 ResourceMark rm;
2946 HandleMark hm;
2947
2948 MarkRefsIntoClosure notOlder(_span, &_markBitMap);
2949 GenCollectedHeap* gch = GenCollectedHeap::heap();
2950
2951 verify_work_stacks_empty();
2952 verify_overflow_empty();
2953
2954 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2955 // Update the saved marks which may affect the root scans.
2956 gch->save_marks();
2957
2958 // weak reference processing has not started yet.
2959 ref_processor()->set_enqueuing_is_done(false);
2960
2961 // Need to remember all newly created CLDs,
2962 // so that we can guarantee that the remark finds them.
2963 ClassLoaderDataGraph::remember_new_clds(true);
2964
2965 // Whenever a CLD is found, it will be claimed before proceeding to mark
2966 // the klasses. The claimed marks need to be cleared before marking starts.
2967 ClassLoaderDataGraph::clear_claimed_marks();
2968
2969 if (CMSPrintEdenSurvivorChunks) {
2970 print_eden_and_survivor_chunk_arrays();
2971 }
2972
2973 {
2974 #if defined(COMPILER2) || INCLUDE_JVMCI
2975 DerivedPointerTableDeactivate dpt_deact;
2976 #endif
2977 if (CMSParallelInitialMarkEnabled) {
2978 // The parallel version.
2979 WorkGang* workers = gch->workers();
2980 assert(workers != NULL, "Need parallel worker threads.");
2981 uint n_workers = workers->active_workers();
2982
2983 StrongRootsScope srs(n_workers);
2984
2985 CMSParInitialMarkTask tsk(this, &srs, n_workers);
2986 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
2987 if (n_workers > 1) {
2988 workers->run_task(&tsk);
2989 } else {
2990 tsk.work(0);
2991 }
3022 save_sweep_limits();
3023 verify_overflow_empty();
3024 }
3025
3026 bool CMSCollector::markFromRoots() {
3027 // we might be tempted to assert that:
3028 // assert(!SafepointSynchronize::is_at_safepoint(),
3029 // "inconsistent argument?");
3030 // However that wouldn't be right, because it's possible that
3031 // a safepoint is indeed in progress as a young generation
3032 // stop-the-world GC happens even as we mark in this generation.
3033 assert(_collectorState == Marking, "inconsistent state?");
3034 check_correct_thread_executing();
3035 verify_overflow_empty();
3036
3037 // Weak ref discovery note: We may be discovering weak
3038 // refs in this generation concurrent (but interleaved) with
3039 // weak ref discovery by the young generation collector.
3040
3041 CMSTokenSyncWithLocks ts(true, bitMapLock());
3042 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3043 CMSPhaseAccounting pa(this, "mark", !PrintGCDetails);
3044 bool res = markFromRootsWork();
3045 if (res) {
3046 _collectorState = Precleaning;
3047 } else { // We failed and a foreground collection wants to take over
3048 assert(_foregroundGCIsActive, "internal state inconsistency");
3049 assert(_restart_addr == NULL, "foreground will restart from scratch");
3050 if (PrintGCDetails) {
3051 gclog_or_tty->print_cr("bailing out to foreground collection");
3052 }
3053 }
3054 verify_overflow_empty();
3055 return res;
3056 }
3057
3058 bool CMSCollector::markFromRootsWork() {
3059 // iterate over marked bits in bit map, doing a full scan and mark
3060 // from these roots using the following algorithm:
3061 // . if oop is to the right of the current scan pointer,
3062 // mark corresponding bit (we'll process it later)
3063 // . else (oop is to left of current scan pointer)
3064 // push oop on marking stack
3065 // . drain the marking stack
3066
3067 // Note that when we do a marking step we need to hold the
3068 // bit map lock -- recall that direct allocation (by mutators)
3069 // and promotion (by the young generation collector) is also
3070 // marking the bit map. [the so-called allocate live policy.]
3071 // Because the implementation of bit map marking is not
3072 // robust wrt simultaneous marking of bits in the same word,
3237 // and local work queue empty,
3238 // then in a loop do:
3239 // . check global overflow stack; steal a batch of oops and trace
3240 // . try to steal from other threads oif GOS is empty
3241 // . if neither is available, offer termination
3242 // -- Terminate and return result
3243 //
3244 void CMSConcMarkingTask::work(uint worker_id) {
3245 elapsedTimer _timer;
3246 ResourceMark rm;
3247 HandleMark hm;
3248
3249 DEBUG_ONLY(_collector->verify_overflow_empty();)
3250
3251 // Before we begin work, our work queue should be empty
3252 assert(work_queue(worker_id)->size() == 0, "Expected to be empty");
3253 // Scan the bitmap covering _cms_space, tracing through grey objects.
3254 _timer.start();
3255 do_scan_and_mark(worker_id, _cms_space);
3256 _timer.stop();
3257 if (PrintCMSStatistics != 0) {
3258 gclog_or_tty->print_cr("Finished cms space scanning in %dth thread: %3.3f sec",
3259 worker_id, _timer.seconds());
3260 // XXX: need xxx/xxx type of notation, two timers
3261 }
3262
3263 // ... do work stealing
3264 _timer.reset();
3265 _timer.start();
3266 do_work_steal(worker_id);
3267 _timer.stop();
3268 if (PrintCMSStatistics != 0) {
3269 gclog_or_tty->print_cr("Finished work stealing in %dth thread: %3.3f sec",
3270 worker_id, _timer.seconds());
3271 // XXX: need xxx/xxx type of notation, two timers
3272 }
3273 assert(_collector->_markStack.isEmpty(), "Should have been emptied");
3274 assert(work_queue(worker_id)->size() == 0, "Should have been emptied");
3275 // Note that under the current task protocol, the
3276 // following assertion is true even of the spaces
3277 // expanded since the completion of the concurrent
3278 // marking. XXX This will likely change under a strict
3279 // ABORT semantics.
3280 // After perm removal the comparison was changed to
3281 // greater than or equal to from strictly greater than.
3282 // Before perm removal the highest address sweep would
3283 // have been at the end of perm gen but now is at the
3284 // end of the tenured gen.
3285 assert(_global_finger >= _cms_space->end(),
3286 "All tasks have been completed");
3287 DEBUG_ONLY(_collector->verify_overflow_empty();)
3288 }
3289
3290 void CMSConcMarkingTask::bump_global_finger(HeapWord* f) {
3291 HeapWord* read = _global_finger;
3292 HeapWord* cur = read;
3467 // Check if oop points into the CMS generation
3468 // and is not marked
3469 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
3470 // a white object ...
3471 // If we manage to "claim" the object, by being the
3472 // first thread to mark it, then we push it on our
3473 // marking stack
3474 if (_bit_map->par_mark(addr)) { // ... now grey
3475 // push on work queue (grey set)
3476 bool simulate_overflow = false;
3477 NOT_PRODUCT(
3478 if (CMSMarkStackOverflowALot &&
3479 _collector->simulate_overflow()) {
3480 // simulate a stack overflow
3481 simulate_overflow = true;
3482 }
3483 )
3484 if (simulate_overflow ||
3485 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
3486 // stack overflow
3487 if (PrintCMSStatistics != 0) {
3488 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
3489 SIZE_FORMAT, _overflow_stack->capacity());
3490 }
3491 // We cannot assert that the overflow stack is full because
3492 // it may have been emptied since.
3493 assert(simulate_overflow ||
3494 _work_queue->size() == _work_queue->max_elems(),
3495 "Else push should have succeeded");
3496 handle_stack_overflow(addr);
3497 }
3498 } // Else, some other thread got there first
3499 do_yield_check();
3500 }
3501 }
3502
3503 void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
3504 void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
3505
3506 void Par_ConcMarkingClosure::trim_queue(size_t max) {
3507 while (_work_queue->size() > max) {
3508 oop new_oop;
3509 if (_work_queue->pop_local(new_oop)) {
3510 assert(new_oop->is_oop(), "Should be an oop");
3555 assert(work_q->size() == 0, "Impossible!");
3556 break;
3557 } else if (yielding() || should_yield()) {
3558 yield();
3559 }
3560 }
3561 }
3562
3563 // This is run by the CMS (coordinator) thread.
3564 void CMSConcMarkingTask::coordinator_yield() {
3565 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
3566 "CMS thread should hold CMS token");
3567 // First give up the locks, then yield, then re-lock
3568 // We should probably use a constructor/destructor idiom to
3569 // do this unlock/lock or modify the MutexUnlocker class to
3570 // serve our purpose. XXX
3571 assert_lock_strong(_bit_map_lock);
3572 _bit_map_lock->unlock();
3573 ConcurrentMarkSweepThread::desynchronize(true);
3574 _collector->stopTimer();
3575 if (PrintCMSStatistics != 0) {
3576 _collector->incrementYields();
3577 }
3578
3579 // It is possible for whichever thread initiated the yield request
3580 // not to get a chance to wake up and take the bitmap lock between
3581 // this thread releasing it and reacquiring it. So, while the
3582 // should_yield() flag is on, let's sleep for a bit to give the
3583 // other thread a chance to wake up. The limit imposed on the number
3584 // of iterations is defensive, to avoid any unforseen circumstances
3585 // putting us into an infinite loop. Since it's always been this
3586 // (coordinator_yield()) method that was observed to cause the
3587 // problem, we are using a parameter (CMSCoordinatorYieldSleepCount)
3588 // which is by default non-zero. For the other seven methods that
3589 // also perform the yield operation, as are using a different
3590 // parameter (CMSYieldSleepCount) which is by default zero. This way we
3591 // can enable the sleeping for those methods too, if necessary.
3592 // See 6442774.
3593 //
3594 // We really need to reconsider the synchronization between the GC
3595 // thread and the yield-requesting threads in the future and we
3596 // should really use wait/notify, which is the recommended
3597 // way of doing this type of interaction. Additionally, we should
3719 void CMSCollector::preclean() {
3720 check_correct_thread_executing();
3721 assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
3722 verify_work_stacks_empty();
3723 verify_overflow_empty();
3724 _abort_preclean = false;
3725 if (CMSPrecleaningEnabled) {
3726 if (!CMSEdenChunksRecordAlways) {
3727 _eden_chunk_index = 0;
3728 }
3729 size_t used = get_eden_used();
3730 size_t capacity = get_eden_capacity();
3731 // Don't start sampling unless we will get sufficiently
3732 // many samples.
3733 if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
3734 * CMSScheduleRemarkEdenPenetration)) {
3735 _start_sampling = true;
3736 } else {
3737 _start_sampling = false;
3738 }
3739 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3740 CMSPhaseAccounting pa(this, "preclean", !PrintGCDetails);
3741 preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
3742 }
3743 CMSTokenSync x(true); // is cms thread
3744 if (CMSPrecleaningEnabled) {
3745 sample_eden();
3746 _collectorState = AbortablePreclean;
3747 } else {
3748 _collectorState = FinalMarking;
3749 }
3750 verify_work_stacks_empty();
3751 verify_overflow_empty();
3752 }
3753
3754 // Try and schedule the remark such that young gen
3755 // occupancy is CMSScheduleRemarkEdenPenetration %.
3756 void CMSCollector::abortable_preclean() {
3757 check_correct_thread_executing();
3758 assert(CMSPrecleaningEnabled, "Inconsistent control state");
3759 assert(_collectorState == AbortablePreclean, "Inconsistent control state");
3760
3761 // If Eden's current occupancy is below this threshold,
3762 // immediately schedule the remark; else preclean
3763 // past the next scavenge in an effort to
3764 // schedule the pause as described above. By choosing
3765 // CMSScheduleRemarkEdenSizeThreshold >= max eden size
3766 // we will never do an actual abortable preclean cycle.
3767 if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) {
3768 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3769 CMSPhaseAccounting pa(this, "abortable-preclean", !PrintGCDetails);
3770 // We need more smarts in the abortable preclean
3771 // loop below to deal with cases where allocation
3772 // in young gen is very very slow, and our precleaning
3773 // is running a losing race against a horde of
3774 // mutators intent on flooding us with CMS updates
3775 // (dirty cards).
3776 // One, admittedly dumb, strategy is to give up
3777 // after a certain number of abortable precleaning loops
3778 // or after a certain maximum time. We want to make
3779 // this smarter in the next iteration.
3780 // XXX FIX ME!!! YSR
3781 size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0;
3782 while (!(should_abort_preclean() ||
3783 ConcurrentMarkSweepThread::should_terminate())) {
3784 workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2);
3785 cumworkdone += workdone;
3786 loops++;
3787 // Voluntarily terminate abortable preclean phase if we have
3788 // been at it for too long.
3789 if ((CMSMaxAbortablePrecleanLoops != 0) &&
3790 loops >= CMSMaxAbortablePrecleanLoops) {
3791 if (PrintGCDetails) {
3792 gclog_or_tty->print(" CMS: abort preclean due to loops ");
3793 }
3794 break;
3795 }
3796 if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) {
3797 if (PrintGCDetails) {
3798 gclog_or_tty->print(" CMS: abort preclean due to time ");
3799 }
3800 break;
3801 }
3802 // If we are doing little work each iteration, we should
3803 // take a short break.
3804 if (workdone < CMSAbortablePrecleanMinWorkPerIteration) {
3805 // Sleep for some time, waiting for work to accumulate
3806 stopTimer();
3807 cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis);
3808 startTimer();
3809 waited++;
3810 }
3811 }
3812 if (PrintCMSStatistics > 0) {
3813 gclog_or_tty->print(" [" SIZE_FORMAT " iterations, " SIZE_FORMAT " waits, " SIZE_FORMAT " cards)] ",
3814 loops, waited, cumworkdone);
3815 }
3816 }
3817 CMSTokenSync x(true); // is cms thread
3818 if (_collectorState != Idling) {
3819 assert(_collectorState == AbortablePreclean,
3820 "Spontaneous state transition?");
3821 _collectorState = FinalMarking;
3822 } // Else, a foreground collection completed this CMS cycle.
3823 return;
3824 }
3825
3826 // Respond to an Eden sampling opportunity
3827 void CMSCollector::sample_eden() {
3828 // Make sure a young gc cannot sneak in between our
3829 // reading and recording of a sample.
3830 assert(Thread::current()->is_ConcurrentGC_thread(),
3831 "Only the cms thread may collect Eden samples");
3832 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
3833 "Should collect samples while holding CMS token");
3834 if (!_start_sampling) {
3835 return;
3836 }
3939 // processes.
3940 ScanMarkedObjectsAgainCarefullyClosure
3941 smoac_cl(this, _span,
3942 &_markBitMap, &_markStack, &mrias_cl, CMSYield);
3943
3944 // Preclean dirty cards in ModUnionTable and CardTable using
3945 // appropriate convergence criterion;
3946 // repeat CMSPrecleanIter times unless we find that
3947 // we are losing.
3948 assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large");
3949 assert(CMSPrecleanNumerator < CMSPrecleanDenominator,
3950 "Bad convergence multiplier");
3951 assert(CMSPrecleanThreshold >= 100,
3952 "Unreasonably low CMSPrecleanThreshold");
3953
3954 size_t numIter, cumNumCards, lastNumCards, curNumCards;
3955 for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0;
3956 numIter < CMSPrecleanIter;
3957 numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) {
3958 curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl);
3959 if (Verbose && PrintGCDetails) {
3960 gclog_or_tty->print(" (modUnionTable: " SIZE_FORMAT " cards)", curNumCards);
3961 }
3962 // Either there are very few dirty cards, so re-mark
3963 // pause will be small anyway, or our pre-cleaning isn't
3964 // that much faster than the rate at which cards are being
3965 // dirtied, so we might as well stop and re-mark since
3966 // precleaning won't improve our re-mark time by much.
3967 if (curNumCards <= CMSPrecleanThreshold ||
3968 (numIter > 0 &&
3969 (curNumCards * CMSPrecleanDenominator >
3970 lastNumCards * CMSPrecleanNumerator))) {
3971 numIter++;
3972 cumNumCards += curNumCards;
3973 break;
3974 }
3975 }
3976
3977 preclean_klasses(&mrias_cl, _cmsGen->freelistLock());
3978
3979 curNumCards = preclean_card_table(_cmsGen, &smoac_cl);
3980 cumNumCards += curNumCards;
3981 if (PrintGCDetails && PrintCMSStatistics != 0) {
3982 gclog_or_tty->print_cr(" (cardTable: " SIZE_FORMAT " cards, re-scanned " SIZE_FORMAT " cards, " SIZE_FORMAT " iterations)",
3983 curNumCards, cumNumCards, numIter);
3984 }
3985 return cumNumCards; // as a measure of useful work done
3986 }
3987
3988 // PRECLEANING NOTES:
3989 // Precleaning involves:
3990 // . reading the bits of the modUnionTable and clearing the set bits.
3991 // . For the cards corresponding to the set bits, we scan the
3992 // objects on those cards. This means we need the free_list_lock
3993 // so that we can safely iterate over the CMS space when scanning
3994 // for oops.
3995 // . When we scan the objects, we'll be both reading and setting
3996 // marks in the marking bit map, so we'll need the marking bit map.
3997 // . For protecting _collector_state transitions, we take the CGC_lock.
3998 // Note that any races in the reading of of card table entries by the
3999 // CMS thread on the one hand and the clearing of those entries by the
4000 // VM thread or the setting of those entries by the mutator threads on the
4001 // other are quite benign. However, for efficiency it makes sense to keep
4002 // the VM thread from racing with the CMS thread while the latter is
4003 // dirty card info to the modUnionTable. We therefore also use the
4004 // CGC_lock to protect the reading of the card table and the mod union
4218 // SSS: Add equivalent to ScanMarkedObjectsAgainCarefullyClosure::do_yield_check and should_abort_preclean?
4219 // SSS: We should probably check if precleaning should be aborted, at suitable intervals?
4220 PrecleanKlassClosure preclean_klass_closure(cl);
4221 ClassLoaderDataGraph::classes_do(&preclean_klass_closure);
4222
4223 verify_work_stacks_empty();
4224 verify_overflow_empty();
4225 }
4226
4227 void CMSCollector::checkpointRootsFinal() {
4228 assert(_collectorState == FinalMarking, "incorrect state transition?");
4229 check_correct_thread_executing();
4230 // world is stopped at this checkpoint
4231 assert(SafepointSynchronize::is_at_safepoint(),
4232 "world should be stopped");
4233 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
4234
4235 verify_work_stacks_empty();
4236 verify_overflow_empty();
4237
4238 if (PrintGCDetails) {
4239 gclog_or_tty->print("[YG occupancy: " SIZE_FORMAT " K (" SIZE_FORMAT " K)]",
4240 _young_gen->used() / K,
4241 _young_gen->capacity() / K);
4242 }
4243 {
4244 if (CMSScavengeBeforeRemark) {
4245 GenCollectedHeap* gch = GenCollectedHeap::heap();
4246 // Temporarily set flag to false, GCH->do_collection will
4247 // expect it to be false and set to true
4248 FlagSetting fl(gch->_is_gc_active, false);
4249 NOT_PRODUCT(GCTraceTime t("Scavenge-Before-Remark",
4250 PrintGCDetails && Verbose, true, _gc_timer_cm);)
4251 gch->do_collection(true, // full (i.e. force, see below)
4252 false, // !clear_all_soft_refs
4253 0, // size
4254 false, // is_tlab
4255 GenCollectedHeap::YoungGen // type
4256 );
4257 }
4258 FreelistLocker x(this);
4259 MutexLockerEx y(bitMapLock(),
4260 Mutex::_no_safepoint_check_flag);
4261 checkpointRootsFinalWork();
4262 }
4263 verify_work_stacks_empty();
4264 verify_overflow_empty();
4265 }
4266
4267 void CMSCollector::checkpointRootsFinalWork() {
4268 NOT_PRODUCT(GCTraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, _gc_timer_cm);)
4269
4270 assert(haveFreelistLocks(), "must have free list locks");
4271 assert_lock_strong(bitMapLock());
4272
4273 ResourceMark rm;
4274 HandleMark hm;
4275
4276 GenCollectedHeap* gch = GenCollectedHeap::heap();
4277
4278 if (should_unload_classes()) {
4279 CodeCache::gc_prologue();
4280 }
4281 assert(haveFreelistLocks(), "must have free list locks");
4282 assert_lock_strong(bitMapLock());
4283
4284 // We might assume that we need not fill TLAB's when
4285 // CMSScavengeBeforeRemark is set, because we may have just done
4286 // a scavenge which would have filled all TLAB's -- and besides
4287 // Eden would be empty. This however may not always be the case --
4288 // for instance although we asked for a scavenge, it may not have
4289 // happened because of a JNI critical section. We probably need
4290 // a policy for deciding whether we can in that case wait until
4291 // the critical section releases and then do the remark following
4292 // the scavenge, and skip it here. In the absence of that policy,
4293 // or of an indication of whether the scavenge did indeed occur,
4294 // we cannot rely on TLAB's having been filled and must do
4295 // so here just in case a scavenge did not happen.
4296 gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
4297 // Update the saved marks which may affect the root scans.
4298 gch->save_marks();
4299
4300 if (CMSPrintEdenSurvivorChunks) {
4301 print_eden_and_survivor_chunk_arrays();
4302 }
4303
4304 {
4305 #if defined(COMPILER2) || INCLUDE_JVMCI
4306 DerivedPointerTableDeactivate dpt_deact;
4307 #endif
4308
4309 // Note on the role of the mod union table:
4310 // Since the marker in "markFromRoots" marks concurrently with
4311 // mutators, it is possible for some reachable objects not to have been
4312 // scanned. For instance, an only reference to an object A was
4313 // placed in object B after the marker scanned B. Unless B is rescanned,
4314 // A would be collected. Such updates to references in marked objects
4315 // are detected via the mod union table which is the set of all cards
4316 // dirtied since the first checkpoint in this GC cycle and prior to
4317 // the most recent young generation GC, minus those cleaned up by the
4318 // concurrent precleaning.
4319 if (CMSParallelRemarkEnabled) {
4320 GCTraceTime t("Rescan (parallel) ", PrintGCDetails, false, _gc_timer_cm);
4321 do_remark_parallel();
4322 } else {
4323 GCTraceTime t("Rescan (non-parallel) ", PrintGCDetails, false, _gc_timer_cm);
4324 do_remark_non_parallel();
4325 }
4326 }
4327 verify_work_stacks_empty();
4328 verify_overflow_empty();
4329
4330 {
4331 NOT_PRODUCT(GCTraceTime ts("refProcessingWork", PrintGCDetails, false, _gc_timer_cm);)
4332 refProcessingWork();
4333 }
4334 verify_work_stacks_empty();
4335 verify_overflow_empty();
4336
4337 if (should_unload_classes()) {
4338 CodeCache::gc_epilogue();
4339 }
4340 JvmtiExport::gc_epilogue();
4341
4342 // If we encountered any (marking stack / work queue) overflow
4343 // events during the current CMS cycle, take appropriate
4344 // remedial measures, where possible, so as to try and avoid
4345 // recurrence of that condition.
4346 assert(_markStack.isEmpty(), "No grey objects");
4347 size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
4348 _ser_kac_ovflw + _ser_kac_preclean_ovflw;
4349 if (ser_ovflw > 0) {
4350 if (PrintCMSStatistics != 0) {
4351 gclog_or_tty->print_cr("Marking stack overflow (benign) "
4352 "(pmc_pc=" SIZE_FORMAT ", pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT
4353 ", kac_preclean=" SIZE_FORMAT ")",
4354 _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw,
4355 _ser_kac_ovflw, _ser_kac_preclean_ovflw);
4356 }
4357 _markStack.expand();
4358 _ser_pmc_remark_ovflw = 0;
4359 _ser_pmc_preclean_ovflw = 0;
4360 _ser_kac_preclean_ovflw = 0;
4361 _ser_kac_ovflw = 0;
4362 }
4363 if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
4364 if (PrintCMSStatistics != 0) {
4365 gclog_or_tty->print_cr("Work queue overflow (benign) "
4366 "(pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT ")",
4367 _par_pmc_remark_ovflw, _par_kac_ovflw);
4368 }
4369 _par_pmc_remark_ovflw = 0;
4370 _par_kac_ovflw = 0;
4371 }
4372 if (PrintCMSStatistics != 0) {
4373 if (_markStack._hit_limit > 0) {
4374 gclog_or_tty->print_cr(" (benign) Hit max stack size limit (" SIZE_FORMAT ")",
4375 _markStack._hit_limit);
4376 }
4377 if (_markStack._failed_double > 0) {
4378 gclog_or_tty->print_cr(" (benign) Failed stack doubling (" SIZE_FORMAT "),"
4379 " current capacity " SIZE_FORMAT,
4380 _markStack._failed_double,
4381 _markStack.capacity());
4382 }
4383 }
4384 _markStack._hit_limit = 0;
4385 _markStack._failed_double = 0;
4386
4387 if ((VerifyAfterGC || VerifyDuringGC) &&
4388 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4389 verify_after_remark();
4390 }
4391
4392 _gc_tracer_cm->report_object_count_after_gc(&_is_alive_closure);
4393
4394 // Change under the freelistLocks.
4395 _collectorState = Sweeping;
4396 // Call isAllClear() under bitMapLock
4397 assert(_modUnionTable.isAllClear(),
4398 "Should be clear by end of the final marking");
4399 assert(_ct->klass_rem_set()->mod_union_is_clear(),
4400 "Should be clear by end of the final marking");
4401 }
4402
4403 void CMSParInitialMarkTask::work(uint worker_id) {
4404 elapsedTimer _timer;
4405 ResourceMark rm;
4406 HandleMark hm;
4407
4408 // ---------- scan from roots --------------
4409 _timer.start();
4410 GenCollectedHeap* gch = GenCollectedHeap::heap();
4411 Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap));
4412
4413 // ---------- young gen roots --------------
4414 {
4415 work_on_young_gen_roots(worker_id, &par_mri_cl);
4416 _timer.stop();
4417 if (PrintCMSStatistics != 0) {
4418 gclog_or_tty->print_cr(
4419 "Finished young gen initial mark scan work in %dth thread: %3.3f sec",
4420 worker_id, _timer.seconds());
4421 }
4422 }
4423
4424 // ---------- remaining roots --------------
4425 _timer.reset();
4426 _timer.start();
4427
4428 CLDToOopClosure cld_closure(&par_mri_cl, true);
4429
4430 gch->gen_process_roots(_strong_roots_scope,
4431 GenCollectedHeap::OldGen,
4432 false, // yg was scanned above
4433 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4434 _collector->should_unload_classes(),
4435 &par_mri_cl,
4436 NULL,
4437 &cld_closure);
4438 assert(_collector->should_unload_classes()
4439 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4440 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4441 _timer.stop();
4442 if (PrintCMSStatistics != 0) {
4443 gclog_or_tty->print_cr(
4444 "Finished remaining root initial mark scan work in %dth thread: %3.3f sec",
4445 worker_id, _timer.seconds());
4446 }
4447 }
4448
4449 // Parallel remark task
4450 class CMSParRemarkTask: public CMSParMarkTask {
4451 CompactibleFreeListSpace* _cms_space;
4452
4453 // The per-thread work queues, available here for stealing.
4454 OopTaskQueueSet* _task_queues;
4455 ParallelTaskTerminator _term;
4456 StrongRootsScope* _strong_roots_scope;
4457
4458 public:
4459 // A value of 0 passed to n_workers will cause the number of
4460 // workers to be taken from the active workers in the work gang.
4461 CMSParRemarkTask(CMSCollector* collector,
4462 CompactibleFreeListSpace* cms_space,
4463 uint n_workers, WorkGang* workers,
4464 OopTaskQueueSet* task_queues,
4465 StrongRootsScope* strong_roots_scope):
4466 CMSParMarkTask("Rescan roots and grey objects in parallel",
4539 elapsedTimer _timer;
4540 ResourceMark rm;
4541 HandleMark hm;
4542
4543 // ---------- rescan from roots --------------
4544 _timer.start();
4545 GenCollectedHeap* gch = GenCollectedHeap::heap();
4546 Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
4547 _collector->_span, _collector->ref_processor(),
4548 &(_collector->_markBitMap),
4549 work_queue(worker_id));
4550
4551 // Rescan young gen roots first since these are likely
4552 // coarsely partitioned and may, on that account, constitute
4553 // the critical path; thus, it's best to start off that
4554 // work first.
4555 // ---------- young gen roots --------------
4556 {
4557 work_on_young_gen_roots(worker_id, &par_mrias_cl);
4558 _timer.stop();
4559 if (PrintCMSStatistics != 0) {
4560 gclog_or_tty->print_cr(
4561 "Finished young gen rescan work in %dth thread: %3.3f sec",
4562 worker_id, _timer.seconds());
4563 }
4564 }
4565
4566 // ---------- remaining roots --------------
4567 _timer.reset();
4568 _timer.start();
4569 gch->gen_process_roots(_strong_roots_scope,
4570 GenCollectedHeap::OldGen,
4571 false, // yg was scanned above
4572 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4573 _collector->should_unload_classes(),
4574 &par_mrias_cl,
4575 NULL,
4576 NULL); // The dirty klasses will be handled below
4577
4578 assert(_collector->should_unload_classes()
4579 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4580 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4581 _timer.stop();
4582 if (PrintCMSStatistics != 0) {
4583 gclog_or_tty->print_cr(
4584 "Finished remaining root rescan work in %dth thread: %3.3f sec",
4585 worker_id, _timer.seconds());
4586 }
4587
4588 // ---------- unhandled CLD scanning ----------
4589 if (worker_id == 0) { // Single threaded at the moment.
4590 _timer.reset();
4591 _timer.start();
4592
4593 // Scan all new class loader data objects and new dependencies that were
4594 // introduced during concurrent marking.
4595 ResourceMark rm;
4596 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
4597 for (int i = 0; i < array->length(); i++) {
4598 par_mrias_cl.do_cld_nv(array->at(i));
4599 }
4600
4601 // We don't need to keep track of new CLDs anymore.
4602 ClassLoaderDataGraph::remember_new_clds(false);
4603
4604 _timer.stop();
4605 if (PrintCMSStatistics != 0) {
4606 gclog_or_tty->print_cr(
4607 "Finished unhandled CLD scanning work in %dth thread: %3.3f sec",
4608 worker_id, _timer.seconds());
4609 }
4610 }
4611
4612 // ---------- dirty klass scanning ----------
4613 if (worker_id == 0) { // Single threaded at the moment.
4614 _timer.reset();
4615 _timer.start();
4616
4617 // Scan all classes that was dirtied during the concurrent marking phase.
4618 RemarkKlassClosure remark_klass_closure(&par_mrias_cl);
4619 ClassLoaderDataGraph::classes_do(&remark_klass_closure);
4620
4621 _timer.stop();
4622 if (PrintCMSStatistics != 0) {
4623 gclog_or_tty->print_cr(
4624 "Finished dirty klass scanning work in %dth thread: %3.3f sec",
4625 worker_id, _timer.seconds());
4626 }
4627 }
4628
4629 // We might have added oops to ClassLoaderData::_handles during the
4630 // concurrent marking phase. These oops point to newly allocated objects
4631 // that are guaranteed to be kept alive. Either by the direct allocation
4632 // code, or when the young collector processes the roots. Hence,
4633 // we don't have to revisit the _handles block during the remark phase.
4634
4635 // ---------- rescan dirty cards ------------
4636 _timer.reset();
4637 _timer.start();
4638
4639 // Do the rescan tasks for each of the two spaces
4640 // (cms_space) in turn.
4641 // "worker_id" is passed to select the task_queue for "worker_id"
4642 do_dirty_card_rescan_tasks(_cms_space, worker_id, &par_mrias_cl);
4643 _timer.stop();
4644 if (PrintCMSStatistics != 0) {
4645 gclog_or_tty->print_cr(
4646 "Finished dirty card rescan work in %dth thread: %3.3f sec",
4647 worker_id, _timer.seconds());
4648 }
4649
4650 // ---------- steal work from other threads ...
4651 // ---------- ... and drain overflow list.
4652 _timer.reset();
4653 _timer.start();
4654 do_work_steal(worker_id, &par_mrias_cl, _collector->hash_seed(worker_id));
4655 _timer.stop();
4656 if (PrintCMSStatistics != 0) {
4657 gclog_or_tty->print_cr(
4658 "Finished work stealing in %dth thread: %3.3f sec",
4659 worker_id, _timer.seconds());
4660 }
4661 }
4662
4663 // Note that parameter "i" is not used.
4664 void
4665 CMSParMarkTask::do_young_space_rescan(uint worker_id,
4666 OopsInGenClosure* cl, ContiguousSpace* space,
4667 HeapWord** chunk_array, size_t chunk_top) {
4668 // Until all tasks completed:
4669 // . claim an unclaimed task
4670 // . compute region boundaries corresponding to task claimed
4671 // using chunk_array
4672 // . par_oop_iterate(cl) over that region
4673
4674 ResourceMark rm;
4675 HandleMark hm;
4676
4677 SequentialSubTasksDone* pst = space->par_seq_tasks();
4678
4679 uint nth_task = 0;
4680 uint n_tasks = pst->n_tasks();
4834 // because we just took work from the overflow list,
4835 // but of course we can't since all of that could have
4836 // been already stolen from us.
4837 // "He giveth and He taketh away."
4838 continue;
4839 }
4840 // Verify that we have no work before we resort to stealing
4841 assert(work_q->size() == 0, "Have work, shouldn't steal");
4842 // Try to steal from other queues that have work
4843 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
4844 NOT_PRODUCT(num_steals++;)
4845 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
4846 assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
4847 // Do scanning work
4848 obj_to_scan->oop_iterate(cl);
4849 // Loop around, finish this work, and try to steal some more
4850 } else if (terminator()->offer_termination()) {
4851 break; // nirvana from the infinite cycle
4852 }
4853 }
4854 NOT_PRODUCT(
4855 if (PrintCMSStatistics != 0) {
4856 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
4857 }
4858 )
4859 assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
4860 "Else our work is not yet done");
4861 }
4862
4863 // Record object boundaries in _eden_chunk_array by sampling the eden
4864 // top in the slow-path eden object allocation code path and record
4865 // the boundaries, if CMSEdenChunksRecordAlways is true. If
4866 // CMSEdenChunksRecordAlways is false, we use the other asynchronous
4867 // sampling in sample_eden() that activates during the part of the
4868 // preclean phase.
4869 void CMSCollector::sample_eden_chunk() {
4870 if (CMSEdenChunksRecordAlways && _eden_chunk_array != NULL) {
4871 if (_eden_chunk_lock->try_lock()) {
4872 // Record a sample. This is the critical section. The contents
4873 // of the _eden_chunk_array have to be non-decreasing in the
4874 // address order.
4875 _eden_chunk_array[_eden_chunk_index] = *_top_addr;
4876 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
4877 "Unexpected state of Eden");
4878 if (_eden_chunk_index == 0 ||
4935 if (cur_val < min_val) {
4936 min_tid = j;
4937 min_val = cur_val;
4938 } else {
4939 assert(cur_val < top, "All recorded addresses should be less");
4940 }
4941 }
4942 // At this point min_val and min_tid are respectively
4943 // the least address in _survivor_plab_array[j]->nth(_cursor[j])
4944 // and the thread (j) that witnesses that address.
4945 // We record this address in the _survivor_chunk_array[i]
4946 // and increment _cursor[min_tid] prior to the next round i.
4947 if (min_val == top) {
4948 break;
4949 }
4950 _survivor_chunk_array[i] = min_val;
4951 _cursor[min_tid]++;
4952 }
4953 // We are all done; record the size of the _survivor_chunk_array
4954 _survivor_chunk_index = i; // exclusive: [0, i)
4955 if (PrintCMSStatistics > 0) {
4956 gclog_or_tty->print(" (Survivor:" SIZE_FORMAT "chunks) ", i);
4957 }
4958 // Verify that we used up all the recorded entries
4959 #ifdef ASSERT
4960 size_t total = 0;
4961 for (int j = 0; j < no_of_gc_threads; j++) {
4962 assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
4963 total += _cursor[j];
4964 }
4965 assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
4966 // Check that the merged array is in sorted order
4967 if (total > 0) {
4968 for (size_t i = 0; i < total - 1; i++) {
4969 if (PrintCMSStatistics > 0) {
4970 gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
4971 i, p2i(_survivor_chunk_array[i]));
4972 }
4973 assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
4974 "Not sorted");
4975 }
4976 }
4977 #endif // ASSERT
4978 }
4979
4980 // Set up the space's par_seq_tasks structure for work claiming
4981 // for parallel initial scan and rescan of young gen.
4982 // See ParRescanTask where this is currently used.
4983 void
4984 CMSCollector::
4985 initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
4986 assert(n_threads > 0, "Unexpected n_threads argument");
4987
4988 // Eden space
4989 if (!_young_gen->eden()->is_empty()) {
4990 SequentialSubTasksDone* pst = _young_gen->eden()->par_seq_tasks();
4991 assert(!pst->valid(), "Clobbering existing data?");
4992 // Each valid entry in [0, _eden_chunk_index) represents a task.
5086 // as a result of work_q overflow
5087 restore_preserved_marks_if_any();
5088 }
5089
5090 // Non-parallel version of remark
5091 void CMSCollector::do_remark_non_parallel() {
5092 ResourceMark rm;
5093 HandleMark hm;
5094 GenCollectedHeap* gch = GenCollectedHeap::heap();
5095 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
5096
5097 MarkRefsIntoAndScanClosure
5098 mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */,
5099 &_markStack, this,
5100 false /* should_yield */, false /* not precleaning */);
5101 MarkFromDirtyCardsClosure
5102 markFromDirtyCardsClosure(this, _span,
5103 NULL, // space is set further below
5104 &_markBitMap, &_markStack, &mrias_cl);
5105 {
5106 GCTraceTime t("grey object rescan", PrintGCDetails, false, _gc_timer_cm);
5107 // Iterate over the dirty cards, setting the corresponding bits in the
5108 // mod union table.
5109 {
5110 ModUnionClosure modUnionClosure(&_modUnionTable);
5111 _ct->ct_bs()->dirty_card_iterate(
5112 _cmsGen->used_region(),
5113 &modUnionClosure);
5114 }
5115 // Having transferred these marks into the modUnionTable, we just need
5116 // to rescan the marked objects on the dirty cards in the modUnionTable.
5117 // The initial marking may have been done during an asynchronous
5118 // collection so there may be dirty bits in the mod-union table.
5119 const int alignment =
5120 CardTableModRefBS::card_size * BitsPerWord;
5121 {
5122 // ... First handle dirty cards in CMS gen
5123 markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
5124 MemRegion ur = _cmsGen->used_region();
5125 HeapWord* lb = ur.start();
5126 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
5127 MemRegion cms_span(lb, ub);
5128 _modUnionTable.dirty_range_iterate_clear(cms_span,
5129 &markFromDirtyCardsClosure);
5130 verify_work_stacks_empty();
5131 if (PrintCMSStatistics != 0) {
5132 gclog_or_tty->print(" (re-scanned " SIZE_FORMAT " dirty cards in cms gen) ",
5133 markFromDirtyCardsClosure.num_dirty_cards());
5134 }
5135 }
5136 }
5137 if (VerifyDuringGC &&
5138 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
5139 HandleMark hm; // Discard invalid handles created during verification
5140 Universe::verify();
5141 }
5142 {
5143 GCTraceTime t("root rescan", PrintGCDetails, false, _gc_timer_cm);
5144
5145 verify_work_stacks_empty();
5146
5147 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
5148 StrongRootsScope srs(1);
5149
5150 gch->gen_process_roots(&srs,
5151 GenCollectedHeap::OldGen,
5152 true, // young gen as roots
5153 GenCollectedHeap::ScanningOption(roots_scanning_options()),
5154 should_unload_classes(),
5155 &mrias_cl,
5156 NULL,
5157 NULL); // The dirty klasses will be handled below
5158
5159 assert(should_unload_classes()
5160 || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
5161 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
5162 }
5163
5164 {
5165 GCTraceTime t("visit unhandled CLDs", PrintGCDetails, false, _gc_timer_cm);
5166
5167 verify_work_stacks_empty();
5168
5169 // Scan all class loader data objects that might have been introduced
5170 // during concurrent marking.
5171 ResourceMark rm;
5172 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
5173 for (int i = 0; i < array->length(); i++) {
5174 mrias_cl.do_cld_nv(array->at(i));
5175 }
5176
5177 // We don't need to keep track of new CLDs anymore.
5178 ClassLoaderDataGraph::remember_new_clds(false);
5179
5180 verify_work_stacks_empty();
5181 }
5182
5183 {
5184 GCTraceTime t("dirty klass scan", PrintGCDetails, false, _gc_timer_cm);
5185
5186 verify_work_stacks_empty();
5187
5188 RemarkKlassClosure remark_klass_closure(&mrias_cl);
5189 ClassLoaderDataGraph::classes_do(&remark_klass_closure);
5190
5191 verify_work_stacks_empty();
5192 }
5193
5194 // We might have added oops to ClassLoaderData::_handles during the
5195 // concurrent marking phase. These oops point to newly allocated objects
5196 // that are guaranteed to be kept alive. Either by the direct allocation
5197 // code, or when the young collector processes the roots. Hence,
5198 // we don't have to revisit the _handles block during the remark phase.
5199
5200 verify_work_stacks_empty();
5201 // Restore evacuated mark words, if any, used for overflow list links
5202 restore_preserved_marks_if_any();
5203
5204 verify_overflow_empty();
5326 // We'd like to assert(work_q->size() != 0, ...)
5327 // because we just took work from the overflow list,
5328 // but of course we can't, since all of that might have
5329 // been already stolen from us.
5330 continue;
5331 }
5332 // Verify that we have no work before we resort to stealing
5333 assert(work_q->size() == 0, "Have work, shouldn't steal");
5334 // Try to steal from other queues that have work
5335 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5336 NOT_PRODUCT(num_steals++;)
5337 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5338 assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5339 // Do scanning work
5340 obj_to_scan->oop_iterate(keep_alive);
5341 // Loop around, finish this work, and try to steal some more
5342 } else if (terminator()->offer_termination()) {
5343 break; // nirvana from the infinite cycle
5344 }
5345 }
5346 NOT_PRODUCT(
5347 if (PrintCMSStatistics != 0) {
5348 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
5349 }
5350 )
5351 }
5352
5353 void CMSRefProcTaskExecutor::execute(ProcessTask& task)
5354 {
5355 GenCollectedHeap* gch = GenCollectedHeap::heap();
5356 WorkGang* workers = gch->workers();
5357 assert(workers != NULL, "Need parallel worker threads.");
5358 CMSRefProcTaskProxy rp_task(task, &_collector,
5359 _collector.ref_processor()->span(),
5360 _collector.markBitMap(),
5361 workers, _collector.task_queues());
5362 workers->run_task(&rp_task);
5363 }
5364
5365 void CMSRefProcTaskExecutor::execute(EnqueueTask& task)
5366 {
5367
5368 GenCollectedHeap* gch = GenCollectedHeap::heap();
5369 WorkGang* workers = gch->workers();
5370 assert(workers != NULL, "Need parallel worker threads.");
5372 workers->run_task(&enq_task);
5373 }
5374
5375 void CMSCollector::refProcessingWork() {
5376 ResourceMark rm;
5377 HandleMark hm;
5378
5379 ReferenceProcessor* rp = ref_processor();
5380 assert(rp->span().equals(_span), "Spans should be equal");
5381 assert(!rp->enqueuing_is_done(), "Enqueuing should not be complete");
5382 // Process weak references.
5383 rp->setup_policy(false);
5384 verify_work_stacks_empty();
5385
5386 CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap,
5387 &_markStack, false /* !preclean */);
5388 CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this,
5389 _span, &_markBitMap, &_markStack,
5390 &cmsKeepAliveClosure, false /* !preclean */);
5391 {
5392 GCTraceTime t("weak refs processing", PrintGCDetails, false, _gc_timer_cm);
5393
5394 ReferenceProcessorStats stats;
5395 if (rp->processing_is_mt()) {
5396 // Set the degree of MT here. If the discovery is done MT, there
5397 // may have been a different number of threads doing the discovery
5398 // and a different number of discovered lists may have Ref objects.
5399 // That is OK as long as the Reference lists are balanced (see
5400 // balance_all_queues() and balance_queues()).
5401 GenCollectedHeap* gch = GenCollectedHeap::heap();
5402 uint active_workers = ParallelGCThreads;
5403 WorkGang* workers = gch->workers();
5404 if (workers != NULL) {
5405 active_workers = workers->active_workers();
5406 // The expectation is that active_workers will have already
5407 // been set to a reasonable value. If it has not been set,
5408 // investigate.
5409 assert(active_workers > 0, "Should have been set during scavenge");
5410 }
5411 rp->set_active_mt_degree(active_workers);
5412 CMSRefProcTaskExecutor task_executor(*this);
5414 &cmsKeepAliveClosure,
5415 &cmsDrainMarkingStackClosure,
5416 &task_executor,
5417 _gc_timer_cm);
5418 } else {
5419 stats = rp->process_discovered_references(&_is_alive_closure,
5420 &cmsKeepAliveClosure,
5421 &cmsDrainMarkingStackClosure,
5422 NULL,
5423 _gc_timer_cm);
5424 }
5425 _gc_tracer_cm->report_gc_reference_stats(stats);
5426
5427 }
5428
5429 // This is the point where the entire marking should have completed.
5430 verify_work_stacks_empty();
5431
5432 if (should_unload_classes()) {
5433 {
5434 GCTraceTime t("class unloading", PrintGCDetails, false, _gc_timer_cm);
5435
5436 // Unload classes and purge the SystemDictionary.
5437 bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure);
5438
5439 // Unload nmethods.
5440 CodeCache::do_unloading(&_is_alive_closure, purged_class);
5441
5442 // Prune dead klasses from subklass/sibling/implementor lists.
5443 Klass::clean_weak_klass_links(&_is_alive_closure);
5444 }
5445
5446 {
5447 GCTraceTime t("scrub symbol table", PrintGCDetails, false, _gc_timer_cm);
5448 // Clean up unreferenced symbols in symbol table.
5449 SymbolTable::unlink();
5450 }
5451
5452 {
5453 GCTraceTime t("scrub string table", PrintGCDetails, false, _gc_timer_cm);
5454 // Delete entries for dead interned strings.
5455 StringTable::unlink(&_is_alive_closure);
5456 }
5457 }
5458
5459
5460 // Restore any preserved marks as a result of mark stack or
5461 // work queue overflow
5462 restore_preserved_marks_if_any(); // done single-threaded for now
5463
5464 rp->set_enqueuing_is_done(true);
5465 if (rp->processing_is_mt()) {
5466 rp->balance_all_queues();
5467 CMSRefProcTaskExecutor task_executor(*this);
5468 rp->enqueue_discovered_references(&task_executor);
5469 } else {
5470 rp->enqueue_discovered_references(NULL);
5471 }
5472 rp->verify_no_references_recorded();
5473 assert(!rp->discovery_enabled(), "should have been disabled");
5500 }
5501 }
5502 }
5503 #endif
5504
5505 void CMSCollector::sweep() {
5506 assert(_collectorState == Sweeping, "just checking");
5507 check_correct_thread_executing();
5508 verify_work_stacks_empty();
5509 verify_overflow_empty();
5510 increment_sweep_count();
5511 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
5512
5513 _inter_sweep_timer.stop();
5514 _inter_sweep_estimate.sample(_inter_sweep_timer.seconds());
5515
5516 assert(!_intra_sweep_timer.is_active(), "Should not be active");
5517 _intra_sweep_timer.reset();
5518 _intra_sweep_timer.start();
5519 {
5520 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
5521 CMSPhaseAccounting pa(this, "sweep", !PrintGCDetails);
5522 // First sweep the old gen
5523 {
5524 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
5525 bitMapLock());
5526 sweepWork(_cmsGen);
5527 }
5528
5529 // Update Universe::_heap_*_at_gc figures.
5530 // We need all the free list locks to make the abstract state
5531 // transition from Sweeping to Resetting. See detailed note
5532 // further below.
5533 {
5534 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock());
5535 // Update heap occupancy information which is used as
5536 // input to soft ref clearing policy at the next gc.
5537 Universe::update_heap_info_at_gc();
5538 _collectorState = Resizing;
5539 }
5540 }
5541 verify_work_stacks_empty();
5584 GenCollectedHeap* gch = GenCollectedHeap::heap();
5585 gch->clear_incremental_collection_failed(); // Worth retrying as fresh space may have been freed up
5586 gch->update_full_collections_completed(_collection_count_start);
5587 }
5588
5589 // FIX ME!!! Looks like this belongs in CFLSpace, with
5590 // CMSGen merely delegating to it.
5591 void ConcurrentMarkSweepGeneration::setNearLargestChunk() {
5592 double nearLargestPercent = FLSLargestBlockCoalesceProximity;
5593 HeapWord* minAddr = _cmsSpace->bottom();
5594 HeapWord* largestAddr =
5595 (HeapWord*) _cmsSpace->dictionary()->find_largest_dict();
5596 if (largestAddr == NULL) {
5597 // The dictionary appears to be empty. In this case
5598 // try to coalesce at the end of the heap.
5599 largestAddr = _cmsSpace->end();
5600 }
5601 size_t largestOffset = pointer_delta(largestAddr, minAddr);
5602 size_t nearLargestOffset =
5603 (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
5604 if (PrintFLSStatistics != 0) {
5605 gclog_or_tty->print_cr(
5606 "CMS: Large Block: " PTR_FORMAT ";"
5607 " Proximity: " PTR_FORMAT " -> " PTR_FORMAT,
5608 p2i(largestAddr),
5609 p2i(_cmsSpace->nearLargestChunk()), p2i(minAddr + nearLargestOffset));
5610 }
5611 _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
5612 }
5613
5614 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
5615 return addr >= _cmsSpace->nearLargestChunk();
5616 }
5617
5618 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
5619 return _cmsSpace->find_chunk_at_end();
5620 }
5621
5622 void ConcurrentMarkSweepGeneration::update_gc_stats(Generation* current_generation,
5623 bool full) {
5624 // If the young generation has been collected, gather any statistics
5625 // that are of interest at this point.
5626 bool current_is_young = GenCollectedHeap::heap()->is_young_gen(current_generation);
5627 if (!full && current_is_young) {
5628 // Gather statistics on the young generation collection.
5629 collector()->stats().record_gc0_end(used());
5630 }
5684 } else { // did not unload classes,
5685 _concurrent_cycles_since_last_unload++; // ... increment count
5686 }
5687 }
5688
5689 // Reset CMS data structures (for now just the marking bit map)
5690 // preparatory for the next cycle.
5691 void CMSCollector::reset_concurrent() {
5692 CMSTokenSyncWithLocks ts(true, bitMapLock());
5693
5694 // If the state is not "Resetting", the foreground thread
5695 // has done a collection and the resetting.
5696 if (_collectorState != Resetting) {
5697 assert(_collectorState == Idling, "The state should only change"
5698 " because the foreground collector has finished the collection");
5699 return;
5700 }
5701
5702 // Clear the mark bitmap (no grey objects to start with)
5703 // for the next cycle.
5704 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
5705 CMSPhaseAccounting cmspa(this, "reset", !PrintGCDetails);
5706
5707 HeapWord* curAddr = _markBitMap.startWord();
5708 while (curAddr < _markBitMap.endWord()) {
5709 size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr);
5710 MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining));
5711 _markBitMap.clear_large_range(chunk);
5712 if (ConcurrentMarkSweepThread::should_yield() &&
5713 !foregroundGCIsActive() &&
5714 CMSYield) {
5715 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
5716 "CMS thread should hold CMS token");
5717 assert_lock_strong(bitMapLock());
5718 bitMapLock()->unlock();
5719 ConcurrentMarkSweepThread::desynchronize(true);
5720 stopTimer();
5721 if (PrintCMSStatistics != 0) {
5722 incrementYields();
5723 }
5724
5725 // See the comment in coordinator_yield()
5726 for (unsigned i = 0; i < CMSYieldSleepCount &&
5727 ConcurrentMarkSweepThread::should_yield() &&
5728 !CMSCollector::foregroundGCIsActive(); ++i) {
5729 os::sleep(Thread::current(), 1, false);
5730 }
5731
5732 ConcurrentMarkSweepThread::synchronize(true);
5733 bitMapLock()->lock_without_safepoint_check();
5734 startTimer();
5735 }
5736 curAddr = chunk.end();
5737 }
5738 // A successful mostly concurrent collection has been done.
5739 // Because only the full (i.e., concurrent mode failure) collections
5740 // are being measured for gc overhead limits, clean the "near" flag
5741 // and count.
5742 size_policy()->reset_gc_overhead_limit_count();
5743 _collectorState = Idling;
5744
5745 register_gc_end();
5746 }
5747
5748 // Same as above but for STW paths
5749 void CMSCollector::reset_stw() {
5750 // already have the lock
5751 assert(_collectorState == Resetting, "just checking");
5752 assert_lock_strong(bitMapLock());
5753 GCIdMarkAndRestore gc_id_mark(_cmsThread->gc_id());
5754 _markBitMap.clear_all();
5755 _collectorState = Idling;
5756 register_gc_end();
5757 }
5758
5759 void CMSCollector::do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause) {
5760 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
5761 GCTraceTime t(GCCauseString("GC", gc_cause), PrintGC, !PrintGCDetails, NULL);
5762 TraceCollectorStats tcs(counters());
5763
5764 switch (op) {
5765 case CMS_op_checkpointRootsInitial: {
5766 SvcGCMarker sgcm(SvcGCMarker::OTHER);
5767 checkpointRootsInitial();
5768 if (PrintGC) {
5769 _cmsGen->printOccupancy("initial-mark");
5770 }
5771 break;
5772 }
5773 case CMS_op_checkpointRootsFinal: {
5774 SvcGCMarker sgcm(SvcGCMarker::OTHER);
5775 checkpointRootsFinal();
5776 if (PrintGC) {
5777 _cmsGen->printOccupancy("remark");
5778 }
5779 break;
5780 }
5781 default:
5782 fatal("No such CMS_op");
5783 }
5784 }
5785
5786 #ifndef PRODUCT
5787 size_t const CMSCollector::skip_header_HeapWords() {
5788 return FreeChunk::header_size();
5789 }
5790
5791 // Try and collect here conditions that should hold when
5792 // CMS thread is exiting. The idea is that the foreground GC
5793 // thread should not be blocked if it wants to terminate
5794 // the CMS thread and yet continue to run the VM for a while
5795 // after that.
5796 void CMSCollector::verify_ok_to_terminate() const {
5797 assert(Thread::current()->is_ConcurrentGC_thread(),
5798 "should be called by CMS thread");
5971 }
5972 assert(_virtual_space.committed_size() == rs.size(),
5973 "didn't reserve backing store for all of CMS stack?");
5974 _base = (oop*)(_virtual_space.low());
5975 _index = 0;
5976 _capacity = size;
5977 NOT_PRODUCT(_max_depth = 0);
5978 return true;
5979 }
5980
5981 // XXX FIX ME !!! In the MT case we come in here holding a
5982 // leaf lock. For printing we need to take a further lock
5983 // which has lower rank. We need to recalibrate the two
5984 // lock-ranks involved in order to be able to print the
5985 // messages below. (Or defer the printing to the caller.
5986 // For now we take the expedient path of just disabling the
5987 // messages for the problematic case.)
5988 void CMSMarkStack::expand() {
5989 assert(_capacity <= MarkStackSizeMax, "stack bigger than permitted");
5990 if (_capacity == MarkStackSizeMax) {
5991 if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
5992 // We print a warning message only once per CMS cycle.
5993 gclog_or_tty->print_cr(" (benign) Hit CMSMarkStack max size limit");
5994 }
5995 return;
5996 }
5997 // Double capacity if possible
5998 size_t new_capacity = MIN2(_capacity*2, MarkStackSizeMax);
5999 // Do not give up existing stack until we have managed to
6000 // get the double capacity that we desired.
6001 ReservedSpace rs(ReservedSpace::allocation_align_size_up(
6002 new_capacity * sizeof(oop)));
6003 if (rs.is_reserved()) {
6004 // Release the backing store associated with old stack
6005 _virtual_space.release();
6006 // Reinitialize virtual space for new stack
6007 if (!_virtual_space.initialize(rs, rs.size())) {
6008 fatal("Not enough swap for expanded marking stack");
6009 }
6010 _base = (oop*)(_virtual_space.low());
6011 _index = 0;
6012 _capacity = new_capacity;
6013 } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
6014 // Failed to double capacity, continue;
6015 // we print a detail message only once per CMS cycle.
6016 gclog_or_tty->print(" (benign) Failed to expand marking stack from " SIZE_FORMAT "K to "
6017 SIZE_FORMAT "K",
6018 _capacity / K, new_capacity / K);
6019 }
6020 }
6021
6022
6023 // Closures
6024 // XXX: there seems to be a lot of code duplication here;
6025 // should refactor and consolidate common code.
6026
6027 // This closure is used to mark refs into the CMS generation in
6028 // the CMS bit map. Called at the first checkpoint. This closure
6029 // assumes that we do not need to re-mark dirty cards; if the CMS
6030 // generation on which this is used is not an oldest
6031 // generation then this will lose younger_gen cards!
6032
6033 MarkRefsIntoClosure::MarkRefsIntoClosure(
6034 MemRegion span, CMSBitMap* bitMap):
6035 _span(span),
6036 _bitMap(bitMap)
6037 {
6075 void Par_MarkRefsIntoClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); }
6076
6077 // A variant of the above, used for CMS marking verification.
6078 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
6079 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
6080 _span(span),
6081 _verification_bm(verification_bm),
6082 _cms_bm(cms_bm)
6083 {
6084 assert(ref_processor() == NULL, "deliberately left NULL");
6085 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
6086 }
6087
6088 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
6089 // if p points into _span, then mark corresponding bit in _markBitMap
6090 assert(obj->is_oop(), "expected an oop");
6091 HeapWord* addr = (HeapWord*)obj;
6092 if (_span.contains(addr)) {
6093 _verification_bm->mark(addr);
6094 if (!_cms_bm->isMarked(addr)) {
6095 oop(addr)->print();
6096 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
6097 fatal("... aborting");
6098 }
6099 }
6100 }
6101
6102 void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
6103 void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
6104
6105 //////////////////////////////////////////////////
6106 // MarkRefsIntoAndScanClosure
6107 //////////////////////////////////////////////////
6108
6109 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
6110 ReferenceProcessor* rp,
6111 CMSBitMap* bit_map,
6112 CMSBitMap* mod_union_table,
6113 CMSMarkStack* mark_stack,
6114 CMSCollector* collector,
6115 bool should_yield,
6116 bool concurrent_precleaning):
6172 "overflow list was drained above");
6173
6174 assert(_collector->no_preserved_marks(),
6175 "All preserved marks should have been restored above");
6176 }
6177 }
6178
6179 void MarkRefsIntoAndScanClosure::do_oop(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
6180 void MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
6181
6182 void MarkRefsIntoAndScanClosure::do_yield_work() {
6183 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6184 "CMS thread should hold CMS token");
6185 assert_lock_strong(_freelistLock);
6186 assert_lock_strong(_bit_map->lock());
6187 // relinquish the free_list_lock and bitMaplock()
6188 _bit_map->lock()->unlock();
6189 _freelistLock->unlock();
6190 ConcurrentMarkSweepThread::desynchronize(true);
6191 _collector->stopTimer();
6192 if (PrintCMSStatistics != 0) {
6193 _collector->incrementYields();
6194 }
6195
6196 // See the comment in coordinator_yield()
6197 for (unsigned i = 0;
6198 i < CMSYieldSleepCount &&
6199 ConcurrentMarkSweepThread::should_yield() &&
6200 !CMSCollector::foregroundGCIsActive();
6201 ++i) {
6202 os::sleep(Thread::current(), 1, false);
6203 }
6204
6205 ConcurrentMarkSweepThread::synchronize(true);
6206 _freelistLock->lock_without_safepoint_check();
6207 _bit_map->lock()->lock_without_safepoint_check();
6208 _collector->startTimer();
6209 }
6210
6211 ///////////////////////////////////////////////////////////
6212 // Par_MarkRefsIntoAndScanClosure: a parallel version of
6213 // MarkRefsIntoAndScanClosure
6214 ///////////////////////////////////////////////////////////
6330 // An object not (yet) reached by marking: we merely need to
6331 // compute its size so as to go look at the next block.
6332 assert(p->is_oop(true), "should be an oop");
6333 size = CompactibleFreeListSpace::adjustObjectSize(p->size());
6334 }
6335 }
6336 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6337 return size;
6338 }
6339
6340 void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
6341 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6342 "CMS thread should hold CMS token");
6343 assert_lock_strong(_freelistLock);
6344 assert_lock_strong(_bitMap->lock());
6345 // relinquish the free_list_lock and bitMaplock()
6346 _bitMap->lock()->unlock();
6347 _freelistLock->unlock();
6348 ConcurrentMarkSweepThread::desynchronize(true);
6349 _collector->stopTimer();
6350 if (PrintCMSStatistics != 0) {
6351 _collector->incrementYields();
6352 }
6353
6354 // See the comment in coordinator_yield()
6355 for (unsigned i = 0; i < CMSYieldSleepCount &&
6356 ConcurrentMarkSweepThread::should_yield() &&
6357 !CMSCollector::foregroundGCIsActive(); ++i) {
6358 os::sleep(Thread::current(), 1, false);
6359 }
6360
6361 ConcurrentMarkSweepThread::synchronize(true);
6362 _freelistLock->lock_without_safepoint_check();
6363 _bitMap->lock()->lock_without_safepoint_check();
6364 _collector->startTimer();
6365 }
6366
6367
6368 //////////////////////////////////////////////////////////////////
6369 // SurvivorSpacePrecleanClosure
6370 //////////////////////////////////////////////////////////////////
6371 // This (single-threaded) closure is used to preclean the oops in
6372 // the survivor spaces.
6399 // the ones in CMS heap (i.e. in _span).
6400 new_oop->oop_iterate(_scanning_closure);
6401 // check if it's time to yield
6402 do_yield_check();
6403 }
6404 unsigned int after_count =
6405 GenCollectedHeap::heap()->total_collections();
6406 bool abort = (_before_count != after_count) ||
6407 _collector->should_abort_preclean();
6408 return abort ? 0 : size;
6409 }
6410
6411 void SurvivorSpacePrecleanClosure::do_yield_work() {
6412 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6413 "CMS thread should hold CMS token");
6414 assert_lock_strong(_bit_map->lock());
6415 // Relinquish the bit map lock
6416 _bit_map->lock()->unlock();
6417 ConcurrentMarkSweepThread::desynchronize(true);
6418 _collector->stopTimer();
6419 if (PrintCMSStatistics != 0) {
6420 _collector->incrementYields();
6421 }
6422
6423 // See the comment in coordinator_yield()
6424 for (unsigned i = 0; i < CMSYieldSleepCount &&
6425 ConcurrentMarkSweepThread::should_yield() &&
6426 !CMSCollector::foregroundGCIsActive(); ++i) {
6427 os::sleep(Thread::current(), 1, false);
6428 }
6429
6430 ConcurrentMarkSweepThread::synchronize(true);
6431 _bit_map->lock()->lock_without_safepoint_check();
6432 _collector->startTimer();
6433 }
6434
6435 // This closure is used to rescan the marked objects on the dirty cards
6436 // in the mod union table and the card table proper. In the parallel
6437 // case, although the bitMap is shared, we do a single read so the
6438 // isMarked() query is "safe".
6439 bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) {
6440 // Ignore mark word because we are running concurrent with mutators
6441 assert(p->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(p));
6554 return true;
6555 }
6556 }
6557 scanOopsInOop(addr);
6558 return true;
6559 }
6560
6561 // We take a break if we've been at this for a while,
6562 // so as to avoid monopolizing the locks involved.
6563 void MarkFromRootsClosure::do_yield_work() {
6564 // First give up the locks, then yield, then re-lock
6565 // We should probably use a constructor/destructor idiom to
6566 // do this unlock/lock or modify the MutexUnlocker class to
6567 // serve our purpose. XXX
6568 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6569 "CMS thread should hold CMS token");
6570 assert_lock_strong(_bitMap->lock());
6571 _bitMap->lock()->unlock();
6572 ConcurrentMarkSweepThread::desynchronize(true);
6573 _collector->stopTimer();
6574 if (PrintCMSStatistics != 0) {
6575 _collector->incrementYields();
6576 }
6577
6578 // See the comment in coordinator_yield()
6579 for (unsigned i = 0; i < CMSYieldSleepCount &&
6580 ConcurrentMarkSweepThread::should_yield() &&
6581 !CMSCollector::foregroundGCIsActive(); ++i) {
6582 os::sleep(Thread::current(), 1, false);
6583 }
6584
6585 ConcurrentMarkSweepThread::synchronize(true);
6586 _bitMap->lock()->lock_without_safepoint_check();
6587 _collector->startTimer();
6588 }
6589
6590 void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) {
6591 assert(_bitMap->isMarked(ptr), "expected bit to be set");
6592 assert(_markStack->isEmpty(),
6593 "should drain stack to limit stack usage");
6594 // convert ptr to an oop preparatory to scanning
6595 oop obj = oop(ptr);
6596 // Ignore mark word in verification below, since we
6862 void PushAndMarkVerifyClosure::do_oop(narrowOop* p) { PushAndMarkVerifyClosure::do_oop_work(p); }
6863
6864 // Upon stack overflow, we discard (part of) the stack,
6865 // remembering the least address amongst those discarded
6866 // in CMSCollector's _restart_address.
6867 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
6868 // Remember the least grey address discarded
6869 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
6870 _collector->lower_restart_addr(ra);
6871 _mark_stack->reset(); // discard stack contents
6872 _mark_stack->expand(); // expand the stack if possible
6873 }
6874
6875 void PushAndMarkVerifyClosure::do_oop(oop obj) {
6876 assert(obj->is_oop_or_null(), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6877 HeapWord* addr = (HeapWord*)obj;
6878 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
6879 // Oop lies in _span and isn't yet grey or black
6880 _verification_bm->mark(addr); // now grey
6881 if (!_cms_bm->isMarked(addr)) {
6882 oop(addr)->print();
6883 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)",
6884 p2i(addr));
6885 fatal("... aborting");
6886 }
6887
6888 if (!_mark_stack->push(obj)) { // stack overflow
6889 if (PrintCMSStatistics != 0) {
6890 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
6891 SIZE_FORMAT, _mark_stack->capacity());
6892 }
6893 assert(_mark_stack->isFull(), "Else push should have succeeded");
6894 handle_stack_overflow(addr);
6895 }
6896 // anything including and to the right of _finger
6897 // will be scanned as we iterate over the remainder of the
6898 // bit map
6899 }
6900 }
6901
6902 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
6903 MemRegion span,
6904 CMSBitMap* bitMap, CMSMarkStack* markStack,
6905 HeapWord* finger, MarkFromRootsClosure* parent) :
6906 MetadataAwareOopClosure(collector->ref_processor()),
6907 _collector(collector),
6908 _span(span),
6909 _bitMap(bitMap),
6910 _markStack(markStack),
6911 _finger(finger),
6912 _parent(parent)
6972 void PushOrMarkClosure::do_oop(oop obj) {
6973 // Ignore mark word because we are running concurrent with mutators.
6974 assert(obj->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6975 HeapWord* addr = (HeapWord*)obj;
6976 if (_span.contains(addr) && !_bitMap->isMarked(addr)) {
6977 // Oop lies in _span and isn't yet grey or black
6978 _bitMap->mark(addr); // now grey
6979 if (addr < _finger) {
6980 // the bit map iteration has already either passed, or
6981 // sampled, this bit in the bit map; we'll need to
6982 // use the marking stack to scan this oop's oops.
6983 bool simulate_overflow = false;
6984 NOT_PRODUCT(
6985 if (CMSMarkStackOverflowALot &&
6986 _collector->simulate_overflow()) {
6987 // simulate a stack overflow
6988 simulate_overflow = true;
6989 }
6990 )
6991 if (simulate_overflow || !_markStack->push(obj)) { // stack overflow
6992 if (PrintCMSStatistics != 0) {
6993 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
6994 SIZE_FORMAT, _markStack->capacity());
6995 }
6996 assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded");
6997 handle_stack_overflow(addr);
6998 }
6999 }
7000 // anything including and to the right of _finger
7001 // will be scanned as we iterate over the remainder of the
7002 // bit map
7003 do_yield_check();
7004 }
7005 }
7006
7007 void PushOrMarkClosure::do_oop(oop* p) { PushOrMarkClosure::do_oop_work(p); }
7008 void PushOrMarkClosure::do_oop(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); }
7009
7010 void Par_PushOrMarkClosure::do_oop(oop obj) {
7011 // Ignore mark word because we are running concurrent with mutators.
7012 assert(obj->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
7013 HeapWord* addr = (HeapWord*)obj;
7014 if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) {
7015 // Oop lies in _span and isn't yet grey or black
7024 // -- else push on work queue
7025 if ( !res // someone else marked it, they will deal with it
7026 || (addr >= *gfa) // will be scanned in a later task
7027 || (_span.contains(addr) && addr >= _finger)) { // later in this chunk
7028 return;
7029 }
7030 // the bit map iteration has already either passed, or
7031 // sampled, this bit in the bit map; we'll need to
7032 // use the marking stack to scan this oop's oops.
7033 bool simulate_overflow = false;
7034 NOT_PRODUCT(
7035 if (CMSMarkStackOverflowALot &&
7036 _collector->simulate_overflow()) {
7037 // simulate a stack overflow
7038 simulate_overflow = true;
7039 }
7040 )
7041 if (simulate_overflow ||
7042 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
7043 // stack overflow
7044 if (PrintCMSStatistics != 0) {
7045 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
7046 SIZE_FORMAT, _overflow_stack->capacity());
7047 }
7048 // We cannot assert that the overflow stack is full because
7049 // it may have been emptied since.
7050 assert(simulate_overflow ||
7051 _work_queue->size() == _work_queue->max_elems(),
7052 "Else push should have succeeded");
7053 handle_stack_overflow(addr);
7054 }
7055 do_yield_check();
7056 }
7057 }
7058
7059 void Par_PushOrMarkClosure::do_oop(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
7060 void Par_PushOrMarkClosure::do_oop(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
7061
7062 PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector,
7063 MemRegion span,
7064 ReferenceProcessor* rp,
7065 CMSBitMap* bit_map,
7066 CMSBitMap* mod_union_table,
7067 CMSMarkStack* mark_stack,
7189 _collector->par_push_on_overflow_list(obj);
7190 _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS
7191 }
7192 } // Else, some other thread got there first
7193 }
7194 }
7195
7196 void Par_PushAndMarkClosure::do_oop(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
7197 void Par_PushAndMarkClosure::do_oop(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
7198
7199 void CMSPrecleanRefsYieldClosure::do_yield_work() {
7200 Mutex* bml = _collector->bitMapLock();
7201 assert_lock_strong(bml);
7202 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7203 "CMS thread should hold CMS token");
7204
7205 bml->unlock();
7206 ConcurrentMarkSweepThread::desynchronize(true);
7207
7208 _collector->stopTimer();
7209 if (PrintCMSStatistics != 0) {
7210 _collector->incrementYields();
7211 }
7212
7213 // See the comment in coordinator_yield()
7214 for (unsigned i = 0; i < CMSYieldSleepCount &&
7215 ConcurrentMarkSweepThread::should_yield() &&
7216 !CMSCollector::foregroundGCIsActive(); ++i) {
7217 os::sleep(Thread::current(), 1, false);
7218 }
7219
7220 ConcurrentMarkSweepThread::synchronize(true);
7221 bml->lock();
7222
7223 _collector->startTimer();
7224 }
7225
7226 bool CMSPrecleanRefsYieldClosure::should_return() {
7227 if (ConcurrentMarkSweepThread::should_yield()) {
7228 do_yield_work();
7229 }
7230 return _collector->foregroundGCIsActive();
7231 }
7232
7233 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
7234 assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
7235 "mr should be aligned to start at a card boundary");
7236 // We'd like to assert:
7237 // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
7238 // "mr should be a range of cards");
7239 // However, that would be too strong in one case -- the last
7240 // partition ends at _unallocated_block which, in general, can be
7241 // an arbitrary boundary, not necessarily card aligned.
7242 if (PrintCMSStatistics != 0) {
7243 _num_dirty_cards +=
7244 mr.word_size()/CardTableModRefBS::card_size_in_words;
7245 }
7246 _space->object_iterate_mem(mr, &_scan_cl);
7247 }
7248
7249 SweepClosure::SweepClosure(CMSCollector* collector,
7250 ConcurrentMarkSweepGeneration* g,
7251 CMSBitMap* bitMap, bool should_yield) :
7252 _collector(collector),
7253 _g(g),
7254 _sp(g->cmsSpace()),
7255 _limit(_sp->sweep_limit()),
7256 _freelistLock(_sp->freelistLock()),
7257 _bitMap(bitMap),
7258 _yield(should_yield),
7259 _inFreeRange(false), // No free range at beginning of sweep
7260 _freeRangeInFreeLists(false), // No free range at beginning of sweep
7261 _lastFreeRangeCoalesced(false),
7262 _freeFinger(g->used_region().start())
7263 {
7264 NOT_PRODUCT(
7265 _numObjectsFreed = 0;
7266 _numWordsFreed = 0;
7267 _numObjectsLive = 0;
7268 _numWordsLive = 0;
7269 _numObjectsAlreadyFree = 0;
7270 _numWordsAlreadyFree = 0;
7271 _last_fc = NULL;
7272
7273 _sp->initializeIndexedFreeListArrayReturnedBytes();
7274 _sp->dictionary()->initialize_dict_returned_bytes();
7275 )
7276 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7277 "sweep _limit out of bounds");
7278 if (CMSTraceSweeper) {
7279 gclog_or_tty->print_cr("\n====================\nStarting new sweep with limit " PTR_FORMAT,
7280 p2i(_limit));
7281 }
7282 }
7283
7284 void SweepClosure::print_on(outputStream* st) const {
7285 tty->print_cr("_sp = [" PTR_FORMAT "," PTR_FORMAT ")",
7286 p2i(_sp->bottom()), p2i(_sp->end()));
7287 tty->print_cr("_limit = " PTR_FORMAT, p2i(_limit));
7288 tty->print_cr("_freeFinger = " PTR_FORMAT, p2i(_freeFinger));
7289 NOT_PRODUCT(tty->print_cr("_last_fc = " PTR_FORMAT, p2i(_last_fc));)
7290 tty->print_cr("_inFreeRange = %d, _freeRangeInFreeLists = %d, _lastFreeRangeCoalesced = %d",
7291 _inFreeRange, _freeRangeInFreeLists, _lastFreeRangeCoalesced);
7292 }
7293
7294 #ifndef PRODUCT
7295 // Assertion checking only: no useful work in product mode --
7296 // however, if any of the flags below become product flags,
7297 // you may need to review this code to see if it needs to be
7298 // enabled in product mode.
7299 SweepClosure::~SweepClosure() {
7300 assert_lock_strong(_freelistLock);
7301 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7302 "sweep _limit out of bounds");
7303 if (inFreeRange()) {
7304 warning("inFreeRange() should have been reset; dumping state of SweepClosure");
7305 print();
7306 ShouldNotReachHere();
7307 }
7308 if (Verbose && PrintGC) {
7309 gclog_or_tty->print("Collected " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7310 _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
7311 gclog_or_tty->print_cr("\nLive " SIZE_FORMAT " objects, "
7312 SIZE_FORMAT " bytes "
7313 "Already free " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7314 _numObjectsLive, _numWordsLive*sizeof(HeapWord),
7315 _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
7316 size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree)
7317 * sizeof(HeapWord);
7318 gclog_or_tty->print_cr("Total sweep: " SIZE_FORMAT " bytes", totalBytes);
7319
7320 if (PrintCMSStatistics && CMSVerifyReturnedBytes) {
7321 size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
7322 size_t dict_returned_bytes = _sp->dictionary()->sum_dict_returned_bytes();
7323 size_t returned_bytes = indexListReturnedBytes + dict_returned_bytes;
7324 gclog_or_tty->print("Returned " SIZE_FORMAT " bytes", returned_bytes);
7325 gclog_or_tty->print(" Indexed List Returned " SIZE_FORMAT " bytes",
7326 indexListReturnedBytes);
7327 gclog_or_tty->print_cr(" Dictionary Returned " SIZE_FORMAT " bytes",
7328 dict_returned_bytes);
7329 }
7330 }
7331 if (CMSTraceSweeper) {
7332 gclog_or_tty->print_cr("end of sweep with _limit = " PTR_FORMAT "\n================",
7333 p2i(_limit));
7334 }
7335 }
7336 #endif // PRODUCT
7337
7338 void SweepClosure::initialize_free_range(HeapWord* freeFinger,
7339 bool freeRangeInFreeLists) {
7340 if (CMSTraceSweeper) {
7341 gclog_or_tty->print("---- Start free range at " PTR_FORMAT " with free block (%d)\n",
7342 p2i(freeFinger), freeRangeInFreeLists);
7343 }
7344 assert(!inFreeRange(), "Trampling existing free range");
7345 set_inFreeRange(true);
7346 set_lastFreeRangeCoalesced(false);
7347
7348 set_freeFinger(freeFinger);
7349 set_freeRangeInFreeLists(freeRangeInFreeLists);
7350 if (CMSTestInFreeList) {
7351 if (freeRangeInFreeLists) {
7352 FreeChunk* fc = (FreeChunk*) freeFinger;
7353 assert(fc->is_free(), "A chunk on the free list should be free.");
7354 assert(fc->size() > 0, "Free range should have a size");
7355 assert(_sp->verify_chunk_in_free_list(fc), "Chunk is not in free lists");
7356 }
7357 }
7358 }
7359
7360 // Note that the sweeper runs concurrently with mutators. Thus,
7361 // it is possible for direct allocation in this generation to happen
7362 // in the middle of the sweep. Note that the sweeper also coalesces
7363 // contiguous free blocks. Thus, unless the sweeper and the allocator
7389 size_t res;
7390
7391 // Check if we are done sweeping. Below we check "addr >= _limit" rather
7392 // than "addr == _limit" because although _limit was a block boundary when
7393 // we started the sweep, it may no longer be one because heap expansion
7394 // may have caused us to coalesce the block ending at the address _limit
7395 // with a newly expanded chunk (this happens when _limit was set to the
7396 // previous _end of the space), so we may have stepped past _limit:
7397 // see the following Zeno-like trail of CRs 6977970, 7008136, 7042740.
7398 if (addr >= _limit) { // we have swept up to or past the limit: finish up
7399 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7400 "sweep _limit out of bounds");
7401 assert(addr < _sp->end(), "addr out of bounds");
7402 // Flush any free range we might be holding as a single
7403 // coalesced chunk to the appropriate free list.
7404 if (inFreeRange()) {
7405 assert(freeFinger() >= _sp->bottom() && freeFinger() < _limit,
7406 "freeFinger() " PTR_FORMAT " is out-of-bounds", p2i(freeFinger()));
7407 flush_cur_free_chunk(freeFinger(),
7408 pointer_delta(addr, freeFinger()));
7409 if (CMSTraceSweeper) {
7410 gclog_or_tty->print("Sweep: last chunk: ");
7411 gclog_or_tty->print("put_free_blk " PTR_FORMAT " (" SIZE_FORMAT ") "
7412 "[coalesced:%d]\n",
7413 p2i(freeFinger()), pointer_delta(addr, freeFinger()),
7414 lastFreeRangeCoalesced() ? 1 : 0);
7415 }
7416 }
7417
7418 // help the iterator loop finish
7419 return pointer_delta(_sp->end(), addr);
7420 }
7421
7422 assert(addr < _limit, "sweep invariant");
7423 // check if we should yield
7424 do_yield_check(addr);
7425 if (fc->is_free()) {
7426 // Chunk that is already free
7427 res = fc->size();
7428 do_already_free_chunk(fc);
7429 debug_only(_sp->verifyFreeLists());
7430 // If we flush the chunk at hand in lookahead_and_flush()
7431 // and it's coalesced with a preceding chunk, then the
7432 // process of "mangling" the payload of the coalesced block
7433 // will cause erasure of the size information from the
7434 // (erstwhile) header of all the coalesced blocks but the
7435 // first, so the first disjunct in the assert will not hold
7436 // in that specific case (in which case the second disjunct
7606 // Verify that the bit map has no bits marked between
7607 // addr and purported end of this block.
7608 size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
7609 assert(size >= 3, "Necessary for Printezis marks to work");
7610 assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
7611 DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
7612 }
7613 return size;
7614 }
7615
7616 void SweepClosure::do_post_free_or_garbage_chunk(FreeChunk* fc,
7617 size_t chunkSize) {
7618 // do_post_free_or_garbage_chunk() should only be called in the case
7619 // of the adaptive free list allocator.
7620 const bool fcInFreeLists = fc->is_free();
7621 assert((HeapWord*)fc <= _limit, "sweep invariant");
7622 if (CMSTestInFreeList && fcInFreeLists) {
7623 assert(_sp->verify_chunk_in_free_list(fc), "free chunk is not in free lists");
7624 }
7625
7626 if (CMSTraceSweeper) {
7627 gclog_or_tty->print_cr(" -- pick up another chunk at " PTR_FORMAT " (" SIZE_FORMAT ")", p2i(fc), chunkSize);
7628 }
7629
7630 HeapWord* const fc_addr = (HeapWord*) fc;
7631
7632 bool coalesce = false;
7633 const size_t left = pointer_delta(fc_addr, freeFinger());
7634 const size_t right = chunkSize;
7635 switch (FLSCoalescePolicy) {
7636 // numeric value forms a coalition aggressiveness metric
7637 case 0: { // never coalesce
7638 coalesce = false;
7639 break;
7640 }
7641 case 1: { // coalesce if left & right chunks on overpopulated lists
7642 coalesce = _sp->coalOverPopulated(left) &&
7643 _sp->coalOverPopulated(right);
7644 break;
7645 }
7646 case 2: { // coalesce if left chunk on overpopulated list (default)
7647 coalesce = _sp->coalOverPopulated(left);
7648 break;
7709 // we'll look at because its end crosses past _limit, we'll preemptively
7710 // flush it along with any free range we may be holding on to. Note that
7711 // this can be the case only for an already free or freshly garbage
7712 // chunk. If this block is an object, it can never straddle
7713 // over _limit. The "straddling" occurs when _limit is set at
7714 // the previous end of the space when this cycle started, and
7715 // a subsequent heap expansion caused the previously co-terminal
7716 // free block to be coalesced with the newly expanded portion,
7717 // thus rendering _limit a non-block-boundary making it dangerous
7718 // for the sweeper to step over and examine.
7719 void SweepClosure::lookahead_and_flush(FreeChunk* fc, size_t chunk_size) {
7720 assert(inFreeRange(), "Should only be called if currently in a free range.");
7721 HeapWord* const eob = ((HeapWord*)fc) + chunk_size;
7722 assert(_sp->used_region().contains(eob - 1),
7723 "eob = " PTR_FORMAT " eob-1 = " PTR_FORMAT " _limit = " PTR_FORMAT
7724 " out of bounds wrt _sp = [" PTR_FORMAT "," PTR_FORMAT ")"
7725 " when examining fc = " PTR_FORMAT "(" SIZE_FORMAT ")",
7726 p2i(eob), p2i(eob-1), p2i(_limit), p2i(_sp->bottom()), p2i(_sp->end()), p2i(fc), chunk_size);
7727 if (eob >= _limit) {
7728 assert(eob == _limit || fc->is_free(), "Only a free chunk should allow us to cross over the limit");
7729 if (CMSTraceSweeper) {
7730 gclog_or_tty->print_cr("_limit " PTR_FORMAT " reached or crossed by block "
7731 "[" PTR_FORMAT "," PTR_FORMAT ") in space "
7732 "[" PTR_FORMAT "," PTR_FORMAT ")",
7733 p2i(_limit), p2i(fc), p2i(eob), p2i(_sp->bottom()), p2i(_sp->end()));
7734 }
7735 // Return the storage we are tracking back into the free lists.
7736 if (CMSTraceSweeper) {
7737 gclog_or_tty->print_cr("Flushing ... ");
7738 }
7739 assert(freeFinger() < eob, "Error");
7740 flush_cur_free_chunk( freeFinger(), pointer_delta(eob, freeFinger()));
7741 }
7742 }
7743
7744 void SweepClosure::flush_cur_free_chunk(HeapWord* chunk, size_t size) {
7745 assert(inFreeRange(), "Should only be called if currently in a free range.");
7746 assert(size > 0,
7747 "A zero sized chunk cannot be added to the free lists.");
7748 if (!freeRangeInFreeLists()) {
7749 if (CMSTestInFreeList) {
7750 FreeChunk* fc = (FreeChunk*) chunk;
7751 fc->set_size(size);
7752 assert(!_sp->verify_chunk_in_free_list(fc),
7753 "chunk should not be in free lists yet");
7754 }
7755 if (CMSTraceSweeper) {
7756 gclog_or_tty->print_cr(" -- add free block " PTR_FORMAT " (" SIZE_FORMAT ") to free lists",
7757 p2i(chunk), size);
7758 }
7759 // A new free range is going to be starting. The current
7760 // free range has not been added to the free lists yet or
7761 // was removed so add it back.
7762 // If the current free range was coalesced, then the death
7763 // of the free range was recorded. Record a birth now.
7764 if (lastFreeRangeCoalesced()) {
7765 _sp->coalBirth(size);
7766 }
7767 _sp->addChunkAndRepairOffsetTable(chunk, size,
7768 lastFreeRangeCoalesced());
7769 } else if (CMSTraceSweeper) {
7770 gclog_or_tty->print_cr("Already in free list: nothing to flush");
7771 }
7772 set_inFreeRange(false);
7773 set_freeRangeInFreeLists(false);
7774 }
7775
7776 // We take a break if we've been at this for a while,
7777 // so as to avoid monopolizing the locks involved.
7778 void SweepClosure::do_yield_work(HeapWord* addr) {
7779 // Return current free chunk being used for coalescing (if any)
7780 // to the appropriate freelist. After yielding, the next
7781 // free block encountered will start a coalescing range of
7782 // free blocks. If the next free block is adjacent to the
7783 // chunk just flushed, they will need to wait for the next
7784 // sweep to be coalesced.
7785 if (inFreeRange()) {
7786 flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger()));
7787 }
7788
7789 // First give up the locks, then yield, then re-lock.
7790 // We should probably use a constructor/destructor idiom to
7791 // do this unlock/lock or modify the MutexUnlocker class to
7792 // serve our purpose. XXX
7793 assert_lock_strong(_bitMap->lock());
7794 assert_lock_strong(_freelistLock);
7795 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7796 "CMS thread should hold CMS token");
7797 _bitMap->lock()->unlock();
7798 _freelistLock->unlock();
7799 ConcurrentMarkSweepThread::desynchronize(true);
7800 _collector->stopTimer();
7801 if (PrintCMSStatistics != 0) {
7802 _collector->incrementYields();
7803 }
7804
7805 // See the comment in coordinator_yield()
7806 for (unsigned i = 0; i < CMSYieldSleepCount &&
7807 ConcurrentMarkSweepThread::should_yield() &&
7808 !CMSCollector::foregroundGCIsActive(); ++i) {
7809 os::sleep(Thread::current(), 1, false);
7810 }
7811
7812 ConcurrentMarkSweepThread::synchronize(true);
7813 _freelistLock->lock();
7814 _bitMap->lock()->lock_without_safepoint_check();
7815 _collector->startTimer();
7816 }
7817
7818 #ifndef PRODUCT
7819 // This is actually very useful in a product build if it can
7820 // be called from the debugger. Compile it into the product
7821 // as needed.
7822 bool debug_verify_chunk_in_free_list(FreeChunk* fc) {
7823 return debug_cms_space->verify_chunk_in_free_list(fc);
7824 }
7825 #endif
7826
7827 void SweepClosure::print_free_block_coalesced(FreeChunk* fc) const {
7828 if (CMSTraceSweeper) {
7829 gclog_or_tty->print_cr("Sweep:coal_free_blk " PTR_FORMAT " (" SIZE_FORMAT ")",
7830 p2i(fc), fc->size());
7831 }
7832 }
7833
7834 // CMSIsAliveClosure
7835 bool CMSIsAliveClosure::do_object_b(oop obj) {
7836 HeapWord* addr = (HeapWord*)obj;
7837 return addr != NULL &&
7838 (!_span.contains(addr) || _bit_map->isMarked(addr));
7839 }
7840
7841
7842 CMSKeepAliveClosure::CMSKeepAliveClosure( CMSCollector* collector,
7843 MemRegion span,
7844 CMSBitMap* bit_map, CMSMarkStack* mark_stack,
7845 bool cpc):
7846 _collector(collector),
7847 _span(span),
7848 _bit_map(bit_map),
7849 _mark_stack(mark_stack),
7850 _concurrent_precleaning(cpc) {
7851 assert(!_span.is_empty(), "Empty span could spell trouble");
|
29 #include "code/codeCache.hpp"
30 #include "gc/cms/cmsCollectorPolicy.hpp"
31 #include "gc/cms/cmsOopClosures.inline.hpp"
32 #include "gc/cms/compactibleFreeListSpace.hpp"
33 #include "gc/cms/concurrentMarkSweepGeneration.inline.hpp"
34 #include "gc/cms/concurrentMarkSweepThread.hpp"
35 #include "gc/cms/parNewGeneration.hpp"
36 #include "gc/cms/vmCMSOperations.hpp"
37 #include "gc/serial/genMarkSweep.hpp"
38 #include "gc/serial/tenuredGeneration.hpp"
39 #include "gc/shared/adaptiveSizePolicy.hpp"
40 #include "gc/shared/cardGeneration.inline.hpp"
41 #include "gc/shared/cardTableRS.hpp"
42 #include "gc/shared/collectedHeap.inline.hpp"
43 #include "gc/shared/collectorCounters.hpp"
44 #include "gc/shared/collectorPolicy.hpp"
45 #include "gc/shared/gcLocker.inline.hpp"
46 #include "gc/shared/gcPolicyCounters.hpp"
47 #include "gc/shared/gcTimer.hpp"
48 #include "gc/shared/gcTrace.hpp"
49 #include "gc/shared/gcTraceTime.inline.hpp"
50 #include "gc/shared/genCollectedHeap.hpp"
51 #include "gc/shared/genOopClosures.inline.hpp"
52 #include "gc/shared/isGCActiveMark.hpp"
53 #include "gc/shared/referencePolicy.hpp"
54 #include "gc/shared/strongRootsScope.hpp"
55 #include "gc/shared/taskqueue.inline.hpp"
56 #include "logging/log.hpp"
57 #include "memory/allocation.hpp"
58 #include "memory/iterator.inline.hpp"
59 #include "memory/padded.hpp"
60 #include "memory/resourceArea.hpp"
61 #include "oops/oop.inline.hpp"
62 #include "prims/jvmtiExport.hpp"
63 #include "runtime/atomic.inline.hpp"
64 #include "runtime/globals_extension.hpp"
65 #include "runtime/handles.inline.hpp"
66 #include "runtime/java.hpp"
67 #include "runtime/orderAccess.inline.hpp"
68 #include "runtime/timer.hpp"
69 #include "runtime/vmThread.hpp"
70 #include "services/memoryService.hpp"
71 #include "services/runtimeService.hpp"
72 #include "utilities/stack.inline.hpp"
73
74 // statics
75 CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;
76 bool CMSCollector::_full_gc_requested = false;
77 GCCause::Cause CMSCollector::_full_gc_cause = GCCause::_no_gc;
78
79 //////////////////////////////////////////////////////////////////
80 // In support of CMS/VM thread synchronization
81 //////////////////////////////////////////////////////////////////
82 // We split use of the CGC_lock into 2 "levels".
83 // The low-level locking is of the usual CGC_lock monitor. We introduce
84 // a higher level "token" (hereafter "CMS token") built on top of the
85 // low level monitor (hereafter "CGC lock").
86 // The token-passing protocol gives priority to the VM thread. The
87 // CMS-lock doesn't provide any fairness guarantees, but clients
88 // should ensure that it is only held for very short, bounded
351 // young generation collection.
352 double CMSStats::time_until_cms_gen_full() const {
353 size_t cms_free = _cms_gen->cmsSpace()->free();
354 GenCollectedHeap* gch = GenCollectedHeap::heap();
355 size_t expected_promotion = MIN2(gch->young_gen()->capacity(),
356 (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average());
357 if (cms_free > expected_promotion) {
358 // Start a cms collection if there isn't enough space to promote
359 // for the next young collection. Use the padded average as
360 // a safety factor.
361 cms_free -= expected_promotion;
362
363 // Adjust by the safety factor.
364 double cms_free_dbl = (double)cms_free;
365 double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor) / 100.0;
366 // Apply a further correction factor which tries to adjust
367 // for recent occurance of concurrent mode failures.
368 cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free);
369 cms_free_dbl = cms_free_dbl * cms_adjustment;
370
371 log_trace(gc)("CMSStats::time_until_cms_gen_full: cms_free " SIZE_FORMAT " expected_promotion " SIZE_FORMAT,
372 cms_free, expected_promotion);
373 log_trace(gc)(" cms_free_dbl %f cms_consumption_rate %f", cms_free_dbl, cms_consumption_rate() + 1.0);
374 // Add 1 in case the consumption rate goes to zero.
375 return cms_free_dbl / (cms_consumption_rate() + 1.0);
376 }
377 return 0.0;
378 }
379
380 // Compare the duration of the cms collection to the
381 // time remaining before the cms generation is empty.
382 // Note that the time from the start of the cms collection
383 // to the start of the cms sweep (less than the total
384 // duration of the cms collection) can be used. This
385 // has been tried and some applications experienced
386 // promotion failures early in execution. This was
387 // possibly because the averages were not accurate
388 // enough at the beginning.
389 double CMSStats::time_until_cms_start() const {
390 // We add "gc0_period" to the "work" calculation
391 // below because this query is done (mostly) at the
392 // end of a scavenge, so we need to conservatively
393 // account for that much possible delay
394 // in the query so as to avoid concurrent mode failures
395 // due to starting the collection just a wee bit too
396 // late.
397 double work = cms_duration() + gc0_period();
398 double deadline = time_until_cms_gen_full();
399 // If a concurrent mode failure occurred recently, we want to be
400 // more conservative and halve our expected time_until_cms_gen_full()
401 if (work > deadline) {
402 log_develop_trace(gc)("CMSCollector: collect because of anticipated promotion before full %3.7f + %3.7f > %3.7f ",
403 cms_duration(), gc0_period(), time_until_cms_gen_full());
404 return 0.0;
405 }
406 return work - deadline;
407 }
408
409 #ifndef PRODUCT
410 void CMSStats::print_on(outputStream *st) const {
411 st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);
412 st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,
413 gc0_duration(), gc0_period(), gc0_promoted());
414 st->print(",cms_dur=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,
415 cms_duration(), cms_period(), cms_allocated());
416 st->print(",cms_since_beg=%g,cms_since_end=%g",
417 cms_time_since_begin(), cms_time_since_end());
418 st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,
419 _cms_used_at_gc0_begin, _cms_used_at_gc0_end);
420
421 if (valid()) {
422 st->print(",promo_rate=%g,cms_alloc_rate=%g",
423 promotion_rate(), cms_allocation_rate());
645 //
646 void ConcurrentMarkSweepGeneration::update_counters(size_t used) {
647 if (UsePerfData) {
648 _space_counters->update_used(used);
649 _space_counters->update_capacity();
650 _gen_counters->update_all();
651 }
652 }
653
654 void ConcurrentMarkSweepGeneration::print() const {
655 Generation::print();
656 cmsSpace()->print();
657 }
658
659 #ifndef PRODUCT
660 void ConcurrentMarkSweepGeneration::print_statistics() {
661 cmsSpace()->printFLCensus(0);
662 }
663 #endif
664
665 size_t
666 ConcurrentMarkSweepGeneration::contiguous_available() const {
667 // dld proposes an improvement in precision here. If the committed
668 // part of the space ends in a free block we should add that to
669 // uncommitted size in the calculation below. Will make this
670 // change later, staying with the approximation below for the
671 // time being. -- ysr.
672 return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
673 }
674
675 size_t
676 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
677 return _cmsSpace->max_alloc_in_words() * HeapWordSize;
678 }
679
680 size_t ConcurrentMarkSweepGeneration::max_available() const {
681 return free() + _virtual_space.uncommitted_size();
682 }
683
684 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
685 size_t available = max_available();
686 size_t av_promo = (size_t)gc_stats()->avg_promoted()->padded_average();
687 bool res = (available >= av_promo) || (available >= max_promotion_in_bytes);
688 log_trace(gc, promotion)("CMS: promo attempt is%s safe: available(" SIZE_FORMAT ") %s av_promo(" SIZE_FORMAT "), max_promo(" SIZE_FORMAT ")",
689 res? "":" not", available, res? ">=":"<", av_promo, max_promotion_in_bytes);
690 return res;
691 }
692
693 // At a promotion failure dump information on block layout in heap
694 // (cms old generation).
695 void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {
696 LogHandle(gc, promotion) log;
697 if (log.is_trace()) {
698 ResourceMark rm;
699 cmsSpace()->dump_at_safepoint_with_locks(collector(), log.trace_stream());
700 }
701 }
702
703 void ConcurrentMarkSweepGeneration::reset_after_compaction() {
704 // Clear the promotion information. These pointers can be adjusted
705 // along with all the other pointers into the heap but
706 // compaction is expected to be a rare event with
707 // a heap using cms so don't do it without seeing the need.
708 for (uint i = 0; i < ParallelGCThreads; i++) {
709 _par_gc_thread_states[i]->promo.reset();
710 }
711 }
712
713 void ConcurrentMarkSweepGeneration::compute_new_size() {
714 assert_locked_or_safepoint(Heap_lock);
715
716 // If incremental collection failed, we just want to expand
717 // to the limit.
718 if (incremental_collection_failed()) {
719 clear_incremental_collection_failed();
735 void ConcurrentMarkSweepGeneration::compute_new_size_free_list() {
736 assert_locked_or_safepoint(Heap_lock);
737
738 // If incremental collection failed, we just want to expand
739 // to the limit.
740 if (incremental_collection_failed()) {
741 clear_incremental_collection_failed();
742 grow_to_reserved();
743 return;
744 }
745
746 double free_percentage = ((double) free()) / capacity();
747 double desired_free_percentage = (double) MinHeapFreeRatio / 100;
748 double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
749
750 // compute expansion delta needed for reaching desired free percentage
751 if (free_percentage < desired_free_percentage) {
752 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
753 assert(desired_capacity >= capacity(), "invalid expansion size");
754 size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
755 LogHandle(gc) log;
756 if (log.is_trace()) {
757 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
758 log.trace("From compute_new_size: ");
759 log.trace(" Free fraction %f", free_percentage);
760 log.trace(" Desired free fraction %f", desired_free_percentage);
761 log.trace(" Maximum free fraction %f", maximum_free_percentage);
762 log.trace(" Capacity " SIZE_FORMAT, capacity() / 1000);
763 log.trace(" Desired capacity " SIZE_FORMAT, desired_capacity / 1000);
764 GenCollectedHeap* gch = GenCollectedHeap::heap();
765 assert(gch->is_old_gen(this), "The CMS generation should always be the old generation");
766 size_t young_size = gch->young_gen()->capacity();
767 log.trace(" Young gen size " SIZE_FORMAT, young_size / 1000);
768 log.trace(" unsafe_max_alloc_nogc " SIZE_FORMAT, unsafe_max_alloc_nogc() / 1000);
769 log.trace(" contiguous available " SIZE_FORMAT, contiguous_available() / 1000);
770 log.trace(" Expand by " SIZE_FORMAT " (bytes)", expand_bytes);
771 }
772 // safe if expansion fails
773 expand_for_gc_cause(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
774 log.trace(" Expanded free fraction %f", ((double) free()) / capacity());
775 } else {
776 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
777 assert(desired_capacity <= capacity(), "invalid expansion size");
778 size_t shrink_bytes = capacity() - desired_capacity;
779 // Don't shrink unless the delta is greater than the minimum shrink we want
780 if (shrink_bytes >= MinHeapDeltaBytes) {
781 shrink_free_list_by(shrink_bytes);
782 }
783 }
784 }
785
786 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
787 return cmsSpace()->freelistLock();
788 }
789
790 HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size, bool tlab) {
791 CMSSynchronousYieldRequest yr;
792 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
793 return have_lock_and_allocate(size, tlab);
794 }
1092 ConcurrentMarkSweepGeneration::
1093 par_oop_since_save_marks_iterate_done(int thread_num) {
1094 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1095 ParScanWithoutBarrierClosure* dummy_cl = NULL;
1096 ps->promo.promoted_oops_iterate_nv(dummy_cl);
1097 }
1098
1099 bool ConcurrentMarkSweepGeneration::should_collect(bool full,
1100 size_t size,
1101 bool tlab)
1102 {
1103 // We allow a STW collection only if a full
1104 // collection was requested.
1105 return full || should_allocate(size, tlab); // FIX ME !!!
1106 // This and promotion failure handling are connected at the
1107 // hip and should be fixed by untying them.
1108 }
1109
1110 bool CMSCollector::shouldConcurrentCollect() {
1111 if (_full_gc_requested) {
1112 log_trace(gc)("CMSCollector: collect because of explicit gc request (or gc_locker)");
1113 return true;
1114 }
1115
1116 FreelistLocker x(this);
1117 // ------------------------------------------------------------------
1118 // Print out lots of information which affects the initiation of
1119 // a collection.
1120 LogHandle(gc) log;
1121 if (log.is_trace() && stats().valid()) {
1122 log.trace("CMSCollector shouldConcurrentCollect: ");
1123 ResourceMark rm;
1124 stats().print_on(log.debug_stream());
1125 log.trace("time_until_cms_gen_full %3.7f", stats().time_until_cms_gen_full());
1126 log.trace("free=" SIZE_FORMAT, _cmsGen->free());
1127 log.trace("contiguous_available=" SIZE_FORMAT, _cmsGen->contiguous_available());
1128 log.trace("promotion_rate=%g", stats().promotion_rate());
1129 log.trace("cms_allocation_rate=%g", stats().cms_allocation_rate());
1130 log.trace("occupancy=%3.7f", _cmsGen->occupancy());
1131 log.trace("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy());
1132 log.trace("cms_time_since_begin=%3.7f", stats().cms_time_since_begin());
1133 log.trace("cms_time_since_end=%3.7f", stats().cms_time_since_end());
1134 log.trace("metadata initialized %d", MetaspaceGC::should_concurrent_collect());
1135 }
1136 // ------------------------------------------------------------------
1137
1138 // If the estimated time to complete a cms collection (cms_duration())
1139 // is less than the estimated time remaining until the cms generation
1140 // is full, start a collection.
1141 if (!UseCMSInitiatingOccupancyOnly) {
1142 if (stats().valid()) {
1143 if (stats().time_until_cms_start() == 0.0) {
1144 return true;
1145 }
1146 } else {
1147 // We want to conservatively collect somewhat early in order
1148 // to try and "bootstrap" our CMS/promotion statistics;
1149 // this branch will not fire after the first successful CMS
1150 // collection because the stats should then be valid.
1151 if (_cmsGen->occupancy() >= _bootstrap_occupancy) {
1152 log_trace(gc)(" CMSCollector: collect for bootstrapping statistics: occupancy = %f, boot occupancy = %f",
1153 _cmsGen->occupancy(), _bootstrap_occupancy);
1154 return true;
1155 }
1156 }
1157 }
1158
1159 // Otherwise, we start a collection cycle if
1160 // old gen want a collection cycle started. Each may use
1161 // an appropriate criterion for making this decision.
1162 // XXX We need to make sure that the gen expansion
1163 // criterion dovetails well with this. XXX NEED TO FIX THIS
1164 if (_cmsGen->should_concurrent_collect()) {
1165 log_trace(gc)("CMS old gen initiated");
1166 return true;
1167 }
1168
1169 // We start a collection if we believe an incremental collection may fail;
1170 // this is not likely to be productive in practice because it's probably too
1171 // late anyway.
1172 GenCollectedHeap* gch = GenCollectedHeap::heap();
1173 assert(gch->collector_policy()->is_generation_policy(),
1174 "You may want to check the correctness of the following");
1175 if (gch->incremental_collection_will_fail(true /* consult_young */)) {
1176 log_trace(gc)("CMSCollector: collect because incremental collection will fail ");
1177 return true;
1178 }
1179
1180 if (MetaspaceGC::should_concurrent_collect()) {
1181 log_trace(gc)("CMSCollector: collect for metadata allocation ");
1182 return true;
1183 }
1184
1185 // CMSTriggerInterval starts a CMS cycle if enough time has passed.
1186 if (CMSTriggerInterval >= 0) {
1187 if (CMSTriggerInterval == 0) {
1188 // Trigger always
1189 return true;
1190 }
1191
1192 // Check the CMS time since begin (we do not check the stats validity
1193 // as we want to be able to trigger the first CMS cycle as well)
1194 if (stats().cms_time_since_begin() >= (CMSTriggerInterval / ((double) MILLIUNITS))) {
1195 if (stats().valid()) {
1196 log_trace(gc)("CMSCollector: collect because of trigger interval (time since last begin %3.7f secs)",
1197 stats().cms_time_since_begin());
1198 } else {
1199 log_trace(gc)("CMSCollector: collect because of trigger interval (first collection)");
1200 }
1201 return true;
1202 }
1203 }
1204
1205 return false;
1206 }
1207
1208 void CMSCollector::set_did_compact(bool v) { _cmsGen->set_did_compact(v); }
1209
1210 // Clear _expansion_cause fields of constituent generations
1211 void CMSCollector::clear_expansion_cause() {
1212 _cmsGen->clear_expansion_cause();
1213 }
1214
1215 // We should be conservative in starting a collection cycle. To
1216 // start too eagerly runs the risk of collecting too often in the
1217 // extreme. To collect too rarely falls back on full collections,
1218 // which works, even if not optimum in terms of concurrent work.
1219 // As a work around for too eagerly collecting, use the flag
1222 // collections.
1223 // We want to start a new collection cycle if any of the following
1224 // conditions hold:
1225 // . our current occupancy exceeds the configured initiating occupancy
1226 // for this generation, or
1227 // . we recently needed to expand this space and have not, since that
1228 // expansion, done a collection of this generation, or
1229 // . the underlying space believes that it may be a good idea to initiate
1230 // a concurrent collection (this may be based on criteria such as the
1231 // following: the space uses linear allocation and linear allocation is
1232 // going to fail, or there is believed to be excessive fragmentation in
1233 // the generation, etc... or ...
1234 // [.(currently done by CMSCollector::shouldConcurrentCollect() only for
1235 // the case of the old generation; see CR 6543076):
1236 // we may be approaching a point at which allocation requests may fail because
1237 // we will be out of sufficient free space given allocation rate estimates.]
1238 bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const {
1239
1240 assert_lock_strong(freelistLock());
1241 if (occupancy() > initiating_occupancy()) {
1242 log_trace(gc)(" %s: collect because of occupancy %f / %f ",
1243 short_name(), occupancy(), initiating_occupancy());
1244 return true;
1245 }
1246 if (UseCMSInitiatingOccupancyOnly) {
1247 return false;
1248 }
1249 if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) {
1250 log_trace(gc)(" %s: collect because expanded for allocation ", short_name());
1251 return true;
1252 }
1253 return false;
1254 }
1255
1256 void ConcurrentMarkSweepGeneration::collect(bool full,
1257 bool clear_all_soft_refs,
1258 size_t size,
1259 bool tlab)
1260 {
1261 collector()->collect(full, clear_all_soft_refs, size, tlab);
1262 }
1263
1264 void CMSCollector::collect(bool full,
1265 bool clear_all_soft_refs,
1266 size_t size,
1267 bool tlab)
1268 {
1269 // The following "if" branch is present for defensive reasons.
1270 // In the current uses of this interface, it can be replaced with:
1287 GenCollectedHeap* gch = GenCollectedHeap::heap();
1288 unsigned int gc_count = gch->total_full_collections();
1289 if (gc_count == full_gc_count) {
1290 MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag);
1291 _full_gc_requested = true;
1292 _full_gc_cause = cause;
1293 CGC_lock->notify(); // nudge CMS thread
1294 } else {
1295 assert(gc_count > full_gc_count, "Error: causal loop");
1296 }
1297 }
1298
1299 bool CMSCollector::is_external_interruption() {
1300 GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause();
1301 return GCCause::is_user_requested_gc(cause) ||
1302 GCCause::is_serviceability_requested_gc(cause);
1303 }
1304
1305 void CMSCollector::report_concurrent_mode_interruption() {
1306 if (is_external_interruption()) {
1307 log_debug(gc)("Concurrent mode interrupted");
1308 } else {
1309 log_debug(gc)("Concurrent mode failure");
1310 _gc_tracer_cm->report_concurrent_mode_failure();
1311 }
1312 }
1313
1314
1315 // The foreground and background collectors need to coordinate in order
1316 // to make sure that they do not mutually interfere with CMS collections.
1317 // When a background collection is active,
1318 // the foreground collector may need to take over (preempt) and
1319 // synchronously complete an ongoing collection. Depending on the
1320 // frequency of the background collections and the heap usage
1321 // of the application, this preemption can be seldom or frequent.
1322 // There are only certain
1323 // points in the background collection that the "collection-baton"
1324 // can be passed to the foreground collector.
1325 //
1326 // The foreground collector will wait for the baton before
1327 // starting any part of the collection. The foreground collector
1328 // will only wait at one location.
1329 //
1423 CGC_lock->notify();
1424 assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(),
1425 "Possible deadlock");
1426 while (_foregroundGCShouldWait) {
1427 // wait for notification
1428 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1429 // Possibility of delay/starvation here, since CMS token does
1430 // not know to give priority to VM thread? Actually, i think
1431 // there wouldn't be any delay/starvation, but the proof of
1432 // that "fact" (?) appears non-trivial. XXX 20011219YSR
1433 }
1434 ConcurrentMarkSweepThread::set_CMS_flag(
1435 ConcurrentMarkSweepThread::CMS_vm_has_token);
1436 }
1437 }
1438 // The CMS_token is already held. Get back the other locks.
1439 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1440 "VM thread should have CMS token");
1441 getFreelistLocks();
1442 bitMapLock()->lock_without_safepoint_check();
1443 log_debug(gc, state)("CMS foreground collector has asked for control " INTPTR_FORMAT " with first state %d",
1444 p2i(Thread::current()), first_state);
1445 log_debug(gc, state)(" gets control with state %d", _collectorState);
1446
1447 // Inform cms gen if this was due to partial collection failing.
1448 // The CMS gen may use this fact to determine its expansion policy.
1449 GenCollectedHeap* gch = GenCollectedHeap::heap();
1450 if (gch->incremental_collection_will_fail(false /* don't consult_young */)) {
1451 assert(!_cmsGen->incremental_collection_failed(),
1452 "Should have been noticed, reacted to and cleared");
1453 _cmsGen->set_incremental_collection_failed();
1454 }
1455
1456 if (first_state > Idling) {
1457 report_concurrent_mode_interruption();
1458 }
1459
1460 set_did_compact(true);
1461
1462 // If the collection is being acquired from the background
1463 // collector, there may be references on the discovered
1464 // references lists. Abandon those references, since some
1465 // of them may have become unreachable after concurrent
1499 // after obtaining the free list locks for the
1500 // two generations.
1501 void CMSCollector::compute_new_size() {
1502 assert_locked_or_safepoint(Heap_lock);
1503 FreelistLocker z(this);
1504 MetaspaceGC::compute_new_size();
1505 _cmsGen->compute_new_size_free_list();
1506 }
1507
1508 // A work method used by the foreground collector to do
1509 // a mark-sweep-compact.
1510 void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
1511 GenCollectedHeap* gch = GenCollectedHeap::heap();
1512
1513 STWGCTimer* gc_timer = GenMarkSweep::gc_timer();
1514 gc_timer->register_gc_start();
1515
1516 SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer();
1517 gc_tracer->report_gc_start(gch->gc_cause(), gc_timer->gc_start());
1518
1519 GCTraceTime(Trace, gc) t("CMS:MSC");
1520
1521 // Temporarily widen the span of the weak reference processing to
1522 // the entire heap.
1523 MemRegion new_span(GenCollectedHeap::heap()->reserved_region());
1524 ReferenceProcessorSpanMutator rp_mut_span(ref_processor(), new_span);
1525 // Temporarily, clear the "is_alive_non_header" field of the
1526 // reference processor.
1527 ReferenceProcessorIsAliveMutator rp_mut_closure(ref_processor(), NULL);
1528 // Temporarily make reference _processing_ single threaded (non-MT).
1529 ReferenceProcessorMTProcMutator rp_mut_mt_processing(ref_processor(), false);
1530 // Temporarily make refs discovery atomic
1531 ReferenceProcessorAtomicMutator rp_mut_atomic(ref_processor(), true);
1532 // Temporarily make reference _discovery_ single threaded (non-MT)
1533 ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false);
1534
1535 ref_processor()->set_enqueuing_is_done(false);
1536 ref_processor()->enable_discovery();
1537 ref_processor()->setup_policy(clear_all_soft_refs);
1538 // If an asynchronous collection finishes, the _modUnionTable is
1539 // all clear. If we are assuming the collection from an asynchronous
1584 // Clear any data recorded in the PLAB chunk arrays.
1585 if (_survivor_plab_array != NULL) {
1586 reset_survivor_plab_arrays();
1587 }
1588
1589 // Adjust the per-size allocation stats for the next epoch.
1590 _cmsGen->cmsSpace()->endSweepFLCensus(sweep_count() /* fake */);
1591 // Restart the "inter sweep timer" for the next epoch.
1592 _inter_sweep_timer.reset();
1593 _inter_sweep_timer.start();
1594
1595 gc_timer->register_gc_end();
1596
1597 gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions());
1598
1599 // For a mark-sweep-compact, compute_new_size() will be called
1600 // in the heap's do_collection() method.
1601 }
1602
1603 void CMSCollector::print_eden_and_survivor_chunk_arrays() {
1604 LogHandle(gc, heap) log;
1605 if (!log.is_trace()) {
1606 return;
1607 }
1608
1609 ContiguousSpace* eden_space = _young_gen->eden();
1610 ContiguousSpace* from_space = _young_gen->from();
1611 ContiguousSpace* to_space = _young_gen->to();
1612 // Eden
1613 if (_eden_chunk_array != NULL) {
1614 log.trace("eden " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1615 p2i(eden_space->bottom()), p2i(eden_space->top()),
1616 p2i(eden_space->end()), eden_space->capacity());
1617 log.trace("_eden_chunk_index=" SIZE_FORMAT ", _eden_chunk_capacity=" SIZE_FORMAT,
1618 _eden_chunk_index, _eden_chunk_capacity);
1619 for (size_t i = 0; i < _eden_chunk_index; i++) {
1620 log.trace("_eden_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT, i, p2i(_eden_chunk_array[i]));
1621 }
1622 }
1623 // Survivor
1624 if (_survivor_chunk_array != NULL) {
1625 log.trace("survivor " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1626 p2i(from_space->bottom()), p2i(from_space->top()),
1627 p2i(from_space->end()), from_space->capacity());
1628 log.trace("_survivor_chunk_index=" SIZE_FORMAT ", _survivor_chunk_capacity=" SIZE_FORMAT,
1629 _survivor_chunk_index, _survivor_chunk_capacity);
1630 for (size_t i = 0; i < _survivor_chunk_index; i++) {
1631 log.trace("_survivor_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT, i, p2i(_survivor_chunk_array[i]));
1632 }
1633 }
1634 }
1635
1636 void CMSCollector::getFreelistLocks() const {
1637 // Get locks for all free lists in all generations that this
1638 // collector is responsible for
1639 _cmsGen->freelistLock()->lock_without_safepoint_check();
1640 }
1641
1642 void CMSCollector::releaseFreelistLocks() const {
1643 // Release locks for all free lists in all generations that this
1644 // collector is responsible for
1645 _cmsGen->freelistLock()->unlock();
1646 }
1647
1648 bool CMSCollector::haveFreelistLocks() const {
1649 // Check locks for all free lists in all generations that this
1650 // collector is responsible for
1651 assert_lock_strong(_cmsGen->freelistLock());
1700 _collectorState = InitialMarking;
1701 register_gc_start(cause);
1702 // Reset the expansion cause, now that we are about to begin
1703 // a new cycle.
1704 clear_expansion_cause();
1705
1706 // Clear the MetaspaceGC flag since a concurrent collection
1707 // is starting but also clear it after the collection.
1708 MetaspaceGC::set_should_concurrent_collect(false);
1709 }
1710 // Decide if we want to enable class unloading as part of the
1711 // ensuing concurrent GC cycle.
1712 update_should_unload_classes();
1713 _full_gc_requested = false; // acks all outstanding full gc requests
1714 _full_gc_cause = GCCause::_no_gc;
1715 // Signal that we are about to start a collection
1716 gch->increment_total_full_collections(); // ... starting a collection cycle
1717 _collection_count_start = gch->total_full_collections();
1718 }
1719
1720 size_t prev_used = _cmsGen->used();
1721
1722 // The change of the collection state is normally done at this level;
1723 // the exceptions are phases that are executed while the world is
1724 // stopped. For those phases the change of state is done while the
1725 // world is stopped. For baton passing purposes this allows the
1726 // background collector to finish the phase and change state atomically.
1727 // The foreground collector cannot wait on a phase that is done
1728 // while the world is stopped because the foreground collector already
1729 // has the world stopped and would deadlock.
1730 while (_collectorState != Idling) {
1731 log_debug(gc, state)("Thread " INTPTR_FORMAT " in CMS state %d",
1732 p2i(Thread::current()), _collectorState);
1733 // The foreground collector
1734 // holds the Heap_lock throughout its collection.
1735 // holds the CMS token (but not the lock)
1736 // except while it is waiting for the background collector to yield.
1737 //
1738 // The foreground collector should be blocked (not for long)
1739 // if the background collector is about to start a phase
1740 // executed with world stopped. If the background
1741 // collector has already started such a phase, the
1742 // foreground collector is blocked waiting for the
1743 // Heap_lock. The stop-world phases (InitialMarking and FinalMarking)
1744 // are executed in the VM thread.
1745 //
1746 // The locking order is
1747 // PendingListLock (PLL) -- if applicable (FinalMarking)
1748 // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue())
1749 // CMS token (claimed in
1750 // stop_world_and_do() -->
1751 // safepoint_synchronize() -->
1752 // CMSThread::synchronize())
1753
1754 {
1755 // Check if the FG collector wants us to yield.
1756 CMSTokenSync x(true); // is cms thread
1757 if (waitForForegroundGC()) {
1758 // We yielded to a foreground GC, nothing more to be
1759 // done this round.
1760 assert(_foregroundGCShouldWait == false, "We set it to false in "
1761 "waitForForegroundGC()");
1762 log_debug(gc, state)("CMS Thread " INTPTR_FORMAT " exiting collection CMS state %d",
1763 p2i(Thread::current()), _collectorState);
1764 return;
1765 } else {
1766 // The background collector can run but check to see if the
1767 // foreground collector has done a collection while the
1768 // background collector was waiting to get the CGC_lock
1769 // above. If yes, break so that _foregroundGCShouldWait
1770 // is cleared before returning.
1771 if (_collectorState == Idling) {
1772 break;
1773 }
1774 }
1775 }
1776
1777 assert(_foregroundGCShouldWait, "Foreground collector, if active, "
1778 "should be waiting");
1779
1780 switch (_collectorState) {
1781 case InitialMarking:
1782 {
1783 ReleaseForegroundGC x(this);
1847 break;
1848 }
1849 case Resetting:
1850 // CMS heap resizing has been completed
1851 reset_concurrent();
1852 assert(_collectorState == Idling, "Collector state should "
1853 "have changed");
1854
1855 MetaspaceGC::set_should_concurrent_collect(false);
1856
1857 stats().record_cms_end();
1858 // Don't move the concurrent_phases_end() and compute_new_size()
1859 // calls to here because a preempted background collection
1860 // has it's state set to "Resetting".
1861 break;
1862 case Idling:
1863 default:
1864 ShouldNotReachHere();
1865 break;
1866 }
1867 log_debug(gc, state)(" Thread " INTPTR_FORMAT " done - next CMS state %d",
1868 p2i(Thread::current()), _collectorState);
1869 assert(_foregroundGCShouldWait, "block post-condition");
1870 }
1871
1872 // Should this be in gc_epilogue?
1873 collector_policy()->counters()->update_counters();
1874
1875 {
1876 // Clear _foregroundGCShouldWait and, in the event that the
1877 // foreground collector is waiting, notify it, before
1878 // returning.
1879 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1880 _foregroundGCShouldWait = false;
1881 if (_foregroundGCIsActive) {
1882 CGC_lock->notify();
1883 }
1884 assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1885 "Possible deadlock");
1886 }
1887 log_debug(gc, state)("CMS Thread " INTPTR_FORMAT " exiting collection CMS state %d",
1888 p2i(Thread::current()), _collectorState);
1889 log_info(gc, heap)("Old: " SIZE_FORMAT "K->" SIZE_FORMAT "K(" SIZE_FORMAT "K)",
1890 prev_used / K, _cmsGen->used()/K, _cmsGen->capacity() /K);
1891 }
1892
1893 void CMSCollector::register_gc_start(GCCause::Cause cause) {
1894 _cms_start_registered = true;
1895 _gc_timer_cm->register_gc_start();
1896 _gc_tracer_cm->report_gc_start(cause, _gc_timer_cm->gc_start());
1897 }
1898
1899 void CMSCollector::register_gc_end() {
1900 if (_cms_start_registered) {
1901 report_heap_summary(GCWhen::AfterGC);
1902
1903 _gc_timer_cm->register_gc_end();
1904 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
1905 _cms_start_registered = false;
1906 }
1907 }
1908
1909 void CMSCollector::save_heap_summary() {
1910 GenCollectedHeap* gch = GenCollectedHeap::heap();
1922 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1923 "CMS thread should have CMS token");
1924 // Block the foreground collector until the
1925 // background collectors decides whether to
1926 // yield.
1927 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1928 _foregroundGCShouldWait = true;
1929 if (_foregroundGCIsActive) {
1930 // The background collector yields to the
1931 // foreground collector and returns a value
1932 // indicating that it has yielded. The foreground
1933 // collector can proceed.
1934 res = true;
1935 _foregroundGCShouldWait = false;
1936 ConcurrentMarkSweepThread::clear_CMS_flag(
1937 ConcurrentMarkSweepThread::CMS_cms_has_token);
1938 ConcurrentMarkSweepThread::set_CMS_flag(
1939 ConcurrentMarkSweepThread::CMS_cms_wants_token);
1940 // Get a possibly blocked foreground thread going
1941 CGC_lock->notify();
1942 log_debug(gc, state)("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d",
1943 p2i(Thread::current()), _collectorState);
1944 while (_foregroundGCIsActive) {
1945 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1946 }
1947 ConcurrentMarkSweepThread::set_CMS_flag(
1948 ConcurrentMarkSweepThread::CMS_cms_has_token);
1949 ConcurrentMarkSweepThread::clear_CMS_flag(
1950 ConcurrentMarkSweepThread::CMS_cms_wants_token);
1951 }
1952 log_debug(gc, state)("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d",
1953 p2i(Thread::current()), _collectorState);
1954 return res;
1955 }
1956
1957 // Because of the need to lock the free lists and other structures in
1958 // the collector, common to all the generations that the collector is
1959 // collecting, we need the gc_prologues of individual CMS generations
1960 // delegate to their collector. It may have been simpler had the
1961 // current infrastructure allowed one to call a prologue on a
1962 // collector. In the absence of that we have the generation's
1963 // prologue delegate to the collector, which delegates back
1964 // some "local" work to a worker method in the individual generations
1965 // that it's responsible for collecting, while itself doing any
1966 // work common to all generations it's responsible for. A similar
1967 // comment applies to the gc_epilogue()'s.
1968 // The role of the variable _between_prologue_and_epilogue is to
1969 // enforce the invocation protocol.
1970 void CMSCollector::gc_prologue(bool full) {
1971 // Call gc_prologue_work() for the CMSGen
1972 // we are responsible for.
1973
2030 // collecting.
2031 collector()->gc_prologue(full);
2032 }
2033
2034 // This is a "private" interface for use by this generation's CMSCollector.
2035 // Not to be called directly by any other entity (for instance,
2036 // GenCollectedHeap, which calls the "public" gc_prologue method above).
2037 void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
2038 bool registerClosure, ModUnionClosure* modUnionClosure) {
2039 assert(!incremental_collection_failed(), "Shouldn't be set yet");
2040 assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,
2041 "Should be NULL");
2042 if (registerClosure) {
2043 cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure);
2044 }
2045 cmsSpace()->gc_prologue();
2046 // Clear stat counters
2047 NOT_PRODUCT(
2048 assert(_numObjectsPromoted == 0, "check");
2049 assert(_numWordsPromoted == 0, "check");
2050 log_develop_trace(gc, alloc)("Allocated " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes concurrently",
2051 _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord));
2052 _numObjectsAllocated = 0;
2053 _numWordsAllocated = 0;
2054 )
2055 }
2056
2057 void CMSCollector::gc_epilogue(bool full) {
2058 // The following locking discipline assumes that we are only called
2059 // when the world is stopped.
2060 assert(SafepointSynchronize::is_at_safepoint(),
2061 "world is stopped assumption");
2062
2063 // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks
2064 // if linear allocation blocks need to be appropriately marked to allow the
2065 // the blocks to be parsable. We also check here whether we need to nudge the
2066 // CMS collector thread to start a new cycle (if it's not already active).
2067 assert( Thread::current()->is_VM_thread()
2068 || ( CMSScavengeBeforeRemark
2069 && Thread::current()->is_ConcurrentGC_thread()),
2070 "Incorrect thread type for epilogue execution");
2071
2108 _between_prologue_and_epilogue = false; // ready for next cycle
2109 }
2110
2111 void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
2112 collector()->gc_epilogue(full);
2113
2114 // Also reset promotion tracking in par gc thread states.
2115 for (uint i = 0; i < ParallelGCThreads; i++) {
2116 _par_gc_thread_states[i]->promo.stopTrackingPromotions(i);
2117 }
2118 }
2119
2120 void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) {
2121 assert(!incremental_collection_failed(), "Should have been cleared");
2122 cmsSpace()->setPreconsumptionDirtyCardClosure(NULL);
2123 cmsSpace()->gc_epilogue();
2124 // Print stat counters
2125 NOT_PRODUCT(
2126 assert(_numObjectsAllocated == 0, "check");
2127 assert(_numWordsAllocated == 0, "check");
2128 log_develop_trace(gc, promotion)("Promoted " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
2129 _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord));
2130 _numObjectsPromoted = 0;
2131 _numWordsPromoted = 0;
2132 )
2133
2134 // Call down the chain in contiguous_available needs the freelistLock
2135 // so print this out before releasing the freeListLock.
2136 log_develop_trace(gc)(" Contiguous available " SIZE_FORMAT " bytes ", contiguous_available());
2137 }
2138
2139 #ifndef PRODUCT
2140 bool CMSCollector::have_cms_token() {
2141 Thread* thr = Thread::current();
2142 if (thr->is_VM_thread()) {
2143 return ConcurrentMarkSweepThread::vm_thread_has_cms_token();
2144 } else if (thr->is_ConcurrentGC_thread()) {
2145 return ConcurrentMarkSweepThread::cms_thread_has_cms_token();
2146 } else if (thr->is_GC_task_thread()) {
2147 return ConcurrentMarkSweepThread::vm_thread_has_cms_token() &&
2148 ParGCRareEvent_lock->owned_by_self();
2149 }
2150 return false;
2151 }
2152
2153 // Check reachability of the given heap address in CMS generation,
2154 // treating all other generations as roots.
2155 bool CMSCollector::is_cms_reachable(HeapWord* addr) {
2156 // We could "guarantee" below, rather than assert, but I'll
2200 }
2201 }
2202
2203 ////////////////////////////////////////////////////////
2204 // CMS Verification Support
2205 ////////////////////////////////////////////////////////
2206 // Following the remark phase, the following invariant
2207 // should hold -- each object in the CMS heap which is
2208 // marked in markBitMap() should be marked in the verification_mark_bm().
2209
2210 class VerifyMarkedClosure: public BitMapClosure {
2211 CMSBitMap* _marks;
2212 bool _failed;
2213
2214 public:
2215 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2216
2217 bool do_bit(size_t offset) {
2218 HeapWord* addr = _marks->offsetToHeapWord(offset);
2219 if (!_marks->isMarked(addr)) {
2220 LogHandle(gc, verify) log;
2221 ResourceMark rm;
2222 oop(addr)->print_on(log.info_stream());
2223 log.info(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
2224 _failed = true;
2225 }
2226 return true;
2227 }
2228
2229 bool failed() { return _failed; }
2230 };
2231
2232 bool CMSCollector::verify_after_remark() {
2233 GCTraceTime(Info, gc, verify) tm("Verifying CMS Marking.");
2234 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2235 static bool init = false;
2236
2237 assert(SafepointSynchronize::is_at_safepoint(),
2238 "Else mutations in object graph will make answer suspect");
2239 assert(have_cms_token(),
2240 "Else there may be mutual interference in use of "
2241 " verification data structures");
2242 assert(_collectorState > Marking && _collectorState <= Sweeping,
2243 "Else marking info checked here may be obsolete");
2244 assert(haveFreelistLocks(), "must hold free list locks");
2245 assert_lock_strong(bitMapLock());
2246
2247
2248 // Allocate marking bit map if not already allocated
2249 if (!init) { // first time
2250 if (!verification_mark_bm()->allocate(_span)) {
2251 return false;
2252 }
2253 init = true;
2276 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2277 // Update the saved marks which may affect the root scans.
2278 gch->save_marks();
2279
2280 if (CMSRemarkVerifyVariant == 1) {
2281 // In this first variant of verification, we complete
2282 // all marking, then check if the new marks-vector is
2283 // a subset of the CMS marks-vector.
2284 verify_after_remark_work_1();
2285 } else if (CMSRemarkVerifyVariant == 2) {
2286 // In this second variant of verification, we flag an error
2287 // (i.e. an object reachable in the new marks-vector not reachable
2288 // in the CMS marks-vector) immediately, also indicating the
2289 // identify of an object (A) that references the unmarked object (B) --
2290 // presumably, a mutation to A failed to be picked up by preclean/remark?
2291 verify_after_remark_work_2();
2292 } else {
2293 warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant",
2294 CMSRemarkVerifyVariant);
2295 }
2296 return true;
2297 }
2298
2299 void CMSCollector::verify_after_remark_work_1() {
2300 ResourceMark rm;
2301 HandleMark hm;
2302 GenCollectedHeap* gch = GenCollectedHeap::heap();
2303
2304 // Get a clear set of claim bits for the roots processing to work with.
2305 ClassLoaderDataGraph::clear_claimed_marks();
2306
2307 // Mark from roots one level into CMS
2308 MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
2309 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2310
2311 {
2312 StrongRootsScope srs(1);
2313
2314 gch->gen_process_roots(&srs,
2315 GenCollectedHeap::OldGen,
2327 false /* don't yield */, true /* verifying */);
2328 assert(_restart_addr == NULL, "Expected pre-condition");
2329 verification_mark_bm()->iterate(&markFromRootsClosure);
2330 while (_restart_addr != NULL) {
2331 // Deal with stack overflow: by restarting at the indicated
2332 // address.
2333 HeapWord* ra = _restart_addr;
2334 markFromRootsClosure.reset(ra);
2335 _restart_addr = NULL;
2336 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2337 }
2338 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2339 verify_work_stacks_empty();
2340
2341 // Marking completed -- now verify that each bit marked in
2342 // verification_mark_bm() is also marked in markBitMap(); flag all
2343 // errors by printing corresponding objects.
2344 VerifyMarkedClosure vcl(markBitMap());
2345 verification_mark_bm()->iterate(&vcl);
2346 if (vcl.failed()) {
2347 LogHandle(gc, verify) log;
2348 log.info("Verification failed");
2349 ResourceMark rm;
2350 gch->print_on(log.info_stream());
2351 fatal("CMS: failed marking verification after remark");
2352 }
2353 }
2354
2355 class VerifyKlassOopsKlassClosure : public KlassClosure {
2356 class VerifyKlassOopsClosure : public OopClosure {
2357 CMSBitMap* _bitmap;
2358 public:
2359 VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { }
2360 void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); }
2361 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2362 } _oop_closure;
2363 public:
2364 VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {}
2365 void do_klass(Klass* k) {
2366 k->oops_do(&_oop_closure);
2367 }
2368 };
2369
2370 void CMSCollector::verify_after_remark_work_2() {
2623 expand_for_gc_cause(word_size*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_allocation);
2624 if (GCExpandToAllocateDelayMillis > 0) {
2625 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2626 }
2627 return have_lock_and_allocate(word_size, tlab);
2628 }
2629
2630 void ConcurrentMarkSweepGeneration::expand_for_gc_cause(
2631 size_t bytes,
2632 size_t expand_bytes,
2633 CMSExpansionCause::Cause cause)
2634 {
2635
2636 bool success = expand(bytes, expand_bytes);
2637
2638 // remember why we expanded; this information is used
2639 // by shouldConcurrentCollect() when making decisions on whether to start
2640 // a new CMS cycle.
2641 if (success) {
2642 set_expansion_cause(cause);
2643 log_trace(gc)("Expanded CMS gen for %s", CMSExpansionCause::to_string(cause));
2644 }
2645 }
2646
2647 HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) {
2648 HeapWord* res = NULL;
2649 MutexLocker x(ParGCRareEvent_lock);
2650 while (true) {
2651 // Expansion by some other thread might make alloc OK now:
2652 res = ps->lab.alloc(word_sz);
2653 if (res != NULL) return res;
2654 // If there's not enough expansion space available, give up.
2655 if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) {
2656 return NULL;
2657 }
2658 // Otherwise, we try expansion.
2659 expand_for_gc_cause(word_sz*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_allocate_par_lab);
2660 // Now go around the loop and try alloc again;
2661 // A competing par_promote might beat us to the expansion space,
2662 // so we may go around the loop again if promotion fails again.
2663 if (GCExpandToAllocateDelayMillis > 0) {
2691 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2692 }
2693 }
2694 }
2695
2696 void ConcurrentMarkSweepGeneration::shrink(size_t bytes) {
2697 // Only shrink if a compaction was done so that all the free space
2698 // in the generation is in a contiguous block at the end.
2699 if (did_compact()) {
2700 CardGeneration::shrink(bytes);
2701 }
2702 }
2703
2704 void ConcurrentMarkSweepGeneration::assert_correct_size_change_locking() {
2705 assert_locked_or_safepoint(Heap_lock);
2706 }
2707
2708 void ConcurrentMarkSweepGeneration::shrink_free_list_by(size_t bytes) {
2709 assert_locked_or_safepoint(Heap_lock);
2710 assert_lock_strong(freelistLock());
2711 log_trace(gc)("Shrinking of CMS not yet implemented");
2712 return;
2713 }
2714
2715
2716 // Simple ctor/dtor wrapper for accounting & timer chores around concurrent
2717 // phases.
2718 class CMSPhaseAccounting: public StackObj {
2719 public:
2720 CMSPhaseAccounting(CMSCollector *collector,
2721 const char *title);
2722 ~CMSPhaseAccounting();
2723
2724 private:
2725 CMSCollector *_collector;
2726 const char *_title;
2727 GCTraceConcTime(Info, gc) _trace_time;
2728
2729 public:
2730 // Not MT-safe; so do not pass around these StackObj's
2731 // where they may be accessed by other threads.
2732 jlong wallclock_millis() {
2733 return TimeHelper::counter_to_millis(os::elapsed_counter() - _trace_time.start_time());
2734 }
2735 };
2736
2737 CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector,
2738 const char *title) :
2739 _collector(collector), _title(title), _trace_time(title) {
2740
2741 _collector->resetYields();
2742 _collector->resetTimer();
2743 _collector->startTimer();
2744 }
2745
2746 CMSPhaseAccounting::~CMSPhaseAccounting() {
2747 _collector->stopTimer();
2748 log_debug(gc)("Concurrent active time: %.3fms", TimeHelper::counter_to_seconds(_collector->timerTicks()));
2749 log_trace(gc)(" (CMS %s yielded %d times)", _title, _collector->yields());
2750 }
2751
2752 // CMS work
2753
2754 // The common parts of CMSParInitialMarkTask and CMSParRemarkTask.
2755 class CMSParMarkTask : public AbstractGangTask {
2756 protected:
2757 CMSCollector* _collector;
2758 uint _n_workers;
2759 CMSParMarkTask(const char* name, CMSCollector* collector, uint n_workers) :
2760 AbstractGangTask(name),
2761 _collector(collector),
2762 _n_workers(n_workers) {}
2763 // Work method in support of parallel rescan ... of young gen spaces
2764 void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl,
2765 ContiguousSpace* space,
2766 HeapWord** chunk_array, size_t chunk_top);
2767 void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl);
2768 };
2769
2796 Mutex::_no_safepoint_check_flag);
2797 checkpointRootsInitialWork();
2798 // enable ("weak") refs discovery
2799 rp->enable_discovery();
2800 _collectorState = Marking;
2801 }
2802 }
2803
2804 void CMSCollector::checkpointRootsInitialWork() {
2805 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
2806 assert(_collectorState == InitialMarking, "just checking");
2807
2808 // Already have locks.
2809 assert_lock_strong(bitMapLock());
2810 assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
2811
2812 // Setup the verification and class unloading state for this
2813 // CMS collection cycle.
2814 setup_cms_unloading_and_verification_state();
2815
2816 GCTraceTime(Trace, gc) ts("checkpointRootsInitialWork", _gc_timer_cm);
2817
2818 // Reset all the PLAB chunk arrays if necessary.
2819 if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
2820 reset_survivor_plab_arrays();
2821 }
2822
2823 ResourceMark rm;
2824 HandleMark hm;
2825
2826 MarkRefsIntoClosure notOlder(_span, &_markBitMap);
2827 GenCollectedHeap* gch = GenCollectedHeap::heap();
2828
2829 verify_work_stacks_empty();
2830 verify_overflow_empty();
2831
2832 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2833 // Update the saved marks which may affect the root scans.
2834 gch->save_marks();
2835
2836 // weak reference processing has not started yet.
2837 ref_processor()->set_enqueuing_is_done(false);
2838
2839 // Need to remember all newly created CLDs,
2840 // so that we can guarantee that the remark finds them.
2841 ClassLoaderDataGraph::remember_new_clds(true);
2842
2843 // Whenever a CLD is found, it will be claimed before proceeding to mark
2844 // the klasses. The claimed marks need to be cleared before marking starts.
2845 ClassLoaderDataGraph::clear_claimed_marks();
2846
2847 print_eden_and_survivor_chunk_arrays();
2848
2849 {
2850 #if defined(COMPILER2) || INCLUDE_JVMCI
2851 DerivedPointerTableDeactivate dpt_deact;
2852 #endif
2853 if (CMSParallelInitialMarkEnabled) {
2854 // The parallel version.
2855 WorkGang* workers = gch->workers();
2856 assert(workers != NULL, "Need parallel worker threads.");
2857 uint n_workers = workers->active_workers();
2858
2859 StrongRootsScope srs(n_workers);
2860
2861 CMSParInitialMarkTask tsk(this, &srs, n_workers);
2862 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
2863 if (n_workers > 1) {
2864 workers->run_task(&tsk);
2865 } else {
2866 tsk.work(0);
2867 }
2898 save_sweep_limits();
2899 verify_overflow_empty();
2900 }
2901
2902 bool CMSCollector::markFromRoots() {
2903 // we might be tempted to assert that:
2904 // assert(!SafepointSynchronize::is_at_safepoint(),
2905 // "inconsistent argument?");
2906 // However that wouldn't be right, because it's possible that
2907 // a safepoint is indeed in progress as a young generation
2908 // stop-the-world GC happens even as we mark in this generation.
2909 assert(_collectorState == Marking, "inconsistent state?");
2910 check_correct_thread_executing();
2911 verify_overflow_empty();
2912
2913 // Weak ref discovery note: We may be discovering weak
2914 // refs in this generation concurrent (but interleaved) with
2915 // weak ref discovery by the young generation collector.
2916
2917 CMSTokenSyncWithLocks ts(true, bitMapLock());
2918 GCTraceCPUTime tcpu;
2919 CMSPhaseAccounting pa(this, "Concrurrent Mark");
2920 bool res = markFromRootsWork();
2921 if (res) {
2922 _collectorState = Precleaning;
2923 } else { // We failed and a foreground collection wants to take over
2924 assert(_foregroundGCIsActive, "internal state inconsistency");
2925 assert(_restart_addr == NULL, "foreground will restart from scratch");
2926 log_debug(gc)("bailing out to foreground collection");
2927 }
2928 verify_overflow_empty();
2929 return res;
2930 }
2931
2932 bool CMSCollector::markFromRootsWork() {
2933 // iterate over marked bits in bit map, doing a full scan and mark
2934 // from these roots using the following algorithm:
2935 // . if oop is to the right of the current scan pointer,
2936 // mark corresponding bit (we'll process it later)
2937 // . else (oop is to left of current scan pointer)
2938 // push oop on marking stack
2939 // . drain the marking stack
2940
2941 // Note that when we do a marking step we need to hold the
2942 // bit map lock -- recall that direct allocation (by mutators)
2943 // and promotion (by the young generation collector) is also
2944 // marking the bit map. [the so-called allocate live policy.]
2945 // Because the implementation of bit map marking is not
2946 // robust wrt simultaneous marking of bits in the same word,
3111 // and local work queue empty,
3112 // then in a loop do:
3113 // . check global overflow stack; steal a batch of oops and trace
3114 // . try to steal from other threads oif GOS is empty
3115 // . if neither is available, offer termination
3116 // -- Terminate and return result
3117 //
3118 void CMSConcMarkingTask::work(uint worker_id) {
3119 elapsedTimer _timer;
3120 ResourceMark rm;
3121 HandleMark hm;
3122
3123 DEBUG_ONLY(_collector->verify_overflow_empty();)
3124
3125 // Before we begin work, our work queue should be empty
3126 assert(work_queue(worker_id)->size() == 0, "Expected to be empty");
3127 // Scan the bitmap covering _cms_space, tracing through grey objects.
3128 _timer.start();
3129 do_scan_and_mark(worker_id, _cms_space);
3130 _timer.stop();
3131 log_trace(gc, task)("Finished cms space scanning in %dth thread: %3.3f sec", worker_id, _timer.seconds());
3132
3133 // ... do work stealing
3134 _timer.reset();
3135 _timer.start();
3136 do_work_steal(worker_id);
3137 _timer.stop();
3138 log_trace(gc, task)("Finished work stealing in %dth thread: %3.3f sec", worker_id, _timer.seconds());
3139 assert(_collector->_markStack.isEmpty(), "Should have been emptied");
3140 assert(work_queue(worker_id)->size() == 0, "Should have been emptied");
3141 // Note that under the current task protocol, the
3142 // following assertion is true even of the spaces
3143 // expanded since the completion of the concurrent
3144 // marking. XXX This will likely change under a strict
3145 // ABORT semantics.
3146 // After perm removal the comparison was changed to
3147 // greater than or equal to from strictly greater than.
3148 // Before perm removal the highest address sweep would
3149 // have been at the end of perm gen but now is at the
3150 // end of the tenured gen.
3151 assert(_global_finger >= _cms_space->end(),
3152 "All tasks have been completed");
3153 DEBUG_ONLY(_collector->verify_overflow_empty();)
3154 }
3155
3156 void CMSConcMarkingTask::bump_global_finger(HeapWord* f) {
3157 HeapWord* read = _global_finger;
3158 HeapWord* cur = read;
3333 // Check if oop points into the CMS generation
3334 // and is not marked
3335 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
3336 // a white object ...
3337 // If we manage to "claim" the object, by being the
3338 // first thread to mark it, then we push it on our
3339 // marking stack
3340 if (_bit_map->par_mark(addr)) { // ... now grey
3341 // push on work queue (grey set)
3342 bool simulate_overflow = false;
3343 NOT_PRODUCT(
3344 if (CMSMarkStackOverflowALot &&
3345 _collector->simulate_overflow()) {
3346 // simulate a stack overflow
3347 simulate_overflow = true;
3348 }
3349 )
3350 if (simulate_overflow ||
3351 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
3352 // stack overflow
3353 log_trace(gc)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _overflow_stack->capacity());
3354 // We cannot assert that the overflow stack is full because
3355 // it may have been emptied since.
3356 assert(simulate_overflow ||
3357 _work_queue->size() == _work_queue->max_elems(),
3358 "Else push should have succeeded");
3359 handle_stack_overflow(addr);
3360 }
3361 } // Else, some other thread got there first
3362 do_yield_check();
3363 }
3364 }
3365
3366 void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
3367 void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
3368
3369 void Par_ConcMarkingClosure::trim_queue(size_t max) {
3370 while (_work_queue->size() > max) {
3371 oop new_oop;
3372 if (_work_queue->pop_local(new_oop)) {
3373 assert(new_oop->is_oop(), "Should be an oop");
3418 assert(work_q->size() == 0, "Impossible!");
3419 break;
3420 } else if (yielding() || should_yield()) {
3421 yield();
3422 }
3423 }
3424 }
3425
3426 // This is run by the CMS (coordinator) thread.
3427 void CMSConcMarkingTask::coordinator_yield() {
3428 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
3429 "CMS thread should hold CMS token");
3430 // First give up the locks, then yield, then re-lock
3431 // We should probably use a constructor/destructor idiom to
3432 // do this unlock/lock or modify the MutexUnlocker class to
3433 // serve our purpose. XXX
3434 assert_lock_strong(_bit_map_lock);
3435 _bit_map_lock->unlock();
3436 ConcurrentMarkSweepThread::desynchronize(true);
3437 _collector->stopTimer();
3438 _collector->incrementYields();
3439
3440 // It is possible for whichever thread initiated the yield request
3441 // not to get a chance to wake up and take the bitmap lock between
3442 // this thread releasing it and reacquiring it. So, while the
3443 // should_yield() flag is on, let's sleep for a bit to give the
3444 // other thread a chance to wake up. The limit imposed on the number
3445 // of iterations is defensive, to avoid any unforseen circumstances
3446 // putting us into an infinite loop. Since it's always been this
3447 // (coordinator_yield()) method that was observed to cause the
3448 // problem, we are using a parameter (CMSCoordinatorYieldSleepCount)
3449 // which is by default non-zero. For the other seven methods that
3450 // also perform the yield operation, as are using a different
3451 // parameter (CMSYieldSleepCount) which is by default zero. This way we
3452 // can enable the sleeping for those methods too, if necessary.
3453 // See 6442774.
3454 //
3455 // We really need to reconsider the synchronization between the GC
3456 // thread and the yield-requesting threads in the future and we
3457 // should really use wait/notify, which is the recommended
3458 // way of doing this type of interaction. Additionally, we should
3580 void CMSCollector::preclean() {
3581 check_correct_thread_executing();
3582 assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
3583 verify_work_stacks_empty();
3584 verify_overflow_empty();
3585 _abort_preclean = false;
3586 if (CMSPrecleaningEnabled) {
3587 if (!CMSEdenChunksRecordAlways) {
3588 _eden_chunk_index = 0;
3589 }
3590 size_t used = get_eden_used();
3591 size_t capacity = get_eden_capacity();
3592 // Don't start sampling unless we will get sufficiently
3593 // many samples.
3594 if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
3595 * CMSScheduleRemarkEdenPenetration)) {
3596 _start_sampling = true;
3597 } else {
3598 _start_sampling = false;
3599 }
3600 GCTraceCPUTime tcpu;
3601 CMSPhaseAccounting pa(this, "Concurrent Preclean");
3602 preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
3603 }
3604 CMSTokenSync x(true); // is cms thread
3605 if (CMSPrecleaningEnabled) {
3606 sample_eden();
3607 _collectorState = AbortablePreclean;
3608 } else {
3609 _collectorState = FinalMarking;
3610 }
3611 verify_work_stacks_empty();
3612 verify_overflow_empty();
3613 }
3614
3615 // Try and schedule the remark such that young gen
3616 // occupancy is CMSScheduleRemarkEdenPenetration %.
3617 void CMSCollector::abortable_preclean() {
3618 check_correct_thread_executing();
3619 assert(CMSPrecleaningEnabled, "Inconsistent control state");
3620 assert(_collectorState == AbortablePreclean, "Inconsistent control state");
3621
3622 // If Eden's current occupancy is below this threshold,
3623 // immediately schedule the remark; else preclean
3624 // past the next scavenge in an effort to
3625 // schedule the pause as described above. By choosing
3626 // CMSScheduleRemarkEdenSizeThreshold >= max eden size
3627 // we will never do an actual abortable preclean cycle.
3628 if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) {
3629 GCTraceCPUTime tcpu;
3630 CMSPhaseAccounting pa(this, "Concurrent Abortable Preclean");
3631 // We need more smarts in the abortable preclean
3632 // loop below to deal with cases where allocation
3633 // in young gen is very very slow, and our precleaning
3634 // is running a losing race against a horde of
3635 // mutators intent on flooding us with CMS updates
3636 // (dirty cards).
3637 // One, admittedly dumb, strategy is to give up
3638 // after a certain number of abortable precleaning loops
3639 // or after a certain maximum time. We want to make
3640 // this smarter in the next iteration.
3641 // XXX FIX ME!!! YSR
3642 size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0;
3643 while (!(should_abort_preclean() ||
3644 ConcurrentMarkSweepThread::should_terminate())) {
3645 workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2);
3646 cumworkdone += workdone;
3647 loops++;
3648 // Voluntarily terminate abortable preclean phase if we have
3649 // been at it for too long.
3650 if ((CMSMaxAbortablePrecleanLoops != 0) &&
3651 loops >= CMSMaxAbortablePrecleanLoops) {
3652 log_debug(gc)(" CMS: abort preclean due to loops ");
3653 break;
3654 }
3655 if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) {
3656 log_debug(gc)(" CMS: abort preclean due to time ");
3657 break;
3658 }
3659 // If we are doing little work each iteration, we should
3660 // take a short break.
3661 if (workdone < CMSAbortablePrecleanMinWorkPerIteration) {
3662 // Sleep for some time, waiting for work to accumulate
3663 stopTimer();
3664 cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis);
3665 startTimer();
3666 waited++;
3667 }
3668 }
3669 log_trace(gc)(" [" SIZE_FORMAT " iterations, " SIZE_FORMAT " waits, " SIZE_FORMAT " cards)] ",
3670 loops, waited, cumworkdone);
3671 }
3672 CMSTokenSync x(true); // is cms thread
3673 if (_collectorState != Idling) {
3674 assert(_collectorState == AbortablePreclean,
3675 "Spontaneous state transition?");
3676 _collectorState = FinalMarking;
3677 } // Else, a foreground collection completed this CMS cycle.
3678 return;
3679 }
3680
3681 // Respond to an Eden sampling opportunity
3682 void CMSCollector::sample_eden() {
3683 // Make sure a young gc cannot sneak in between our
3684 // reading and recording of a sample.
3685 assert(Thread::current()->is_ConcurrentGC_thread(),
3686 "Only the cms thread may collect Eden samples");
3687 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
3688 "Should collect samples while holding CMS token");
3689 if (!_start_sampling) {
3690 return;
3691 }
3794 // processes.
3795 ScanMarkedObjectsAgainCarefullyClosure
3796 smoac_cl(this, _span,
3797 &_markBitMap, &_markStack, &mrias_cl, CMSYield);
3798
3799 // Preclean dirty cards in ModUnionTable and CardTable using
3800 // appropriate convergence criterion;
3801 // repeat CMSPrecleanIter times unless we find that
3802 // we are losing.
3803 assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large");
3804 assert(CMSPrecleanNumerator < CMSPrecleanDenominator,
3805 "Bad convergence multiplier");
3806 assert(CMSPrecleanThreshold >= 100,
3807 "Unreasonably low CMSPrecleanThreshold");
3808
3809 size_t numIter, cumNumCards, lastNumCards, curNumCards;
3810 for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0;
3811 numIter < CMSPrecleanIter;
3812 numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) {
3813 curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl);
3814 log_trace(gc)(" (modUnionTable: " SIZE_FORMAT " cards)", curNumCards);
3815 // Either there are very few dirty cards, so re-mark
3816 // pause will be small anyway, or our pre-cleaning isn't
3817 // that much faster than the rate at which cards are being
3818 // dirtied, so we might as well stop and re-mark since
3819 // precleaning won't improve our re-mark time by much.
3820 if (curNumCards <= CMSPrecleanThreshold ||
3821 (numIter > 0 &&
3822 (curNumCards * CMSPrecleanDenominator >
3823 lastNumCards * CMSPrecleanNumerator))) {
3824 numIter++;
3825 cumNumCards += curNumCards;
3826 break;
3827 }
3828 }
3829
3830 preclean_klasses(&mrias_cl, _cmsGen->freelistLock());
3831
3832 curNumCards = preclean_card_table(_cmsGen, &smoac_cl);
3833 cumNumCards += curNumCards;
3834 log_trace(gc)(" (cardTable: " SIZE_FORMAT " cards, re-scanned " SIZE_FORMAT " cards, " SIZE_FORMAT " iterations)",
3835 curNumCards, cumNumCards, numIter);
3836 return cumNumCards; // as a measure of useful work done
3837 }
3838
3839 // PRECLEANING NOTES:
3840 // Precleaning involves:
3841 // . reading the bits of the modUnionTable and clearing the set bits.
3842 // . For the cards corresponding to the set bits, we scan the
3843 // objects on those cards. This means we need the free_list_lock
3844 // so that we can safely iterate over the CMS space when scanning
3845 // for oops.
3846 // . When we scan the objects, we'll be both reading and setting
3847 // marks in the marking bit map, so we'll need the marking bit map.
3848 // . For protecting _collector_state transitions, we take the CGC_lock.
3849 // Note that any races in the reading of of card table entries by the
3850 // CMS thread on the one hand and the clearing of those entries by the
3851 // VM thread or the setting of those entries by the mutator threads on the
3852 // other are quite benign. However, for efficiency it makes sense to keep
3853 // the VM thread from racing with the CMS thread while the latter is
3854 // dirty card info to the modUnionTable. We therefore also use the
3855 // CGC_lock to protect the reading of the card table and the mod union
4069 // SSS: Add equivalent to ScanMarkedObjectsAgainCarefullyClosure::do_yield_check and should_abort_preclean?
4070 // SSS: We should probably check if precleaning should be aborted, at suitable intervals?
4071 PrecleanKlassClosure preclean_klass_closure(cl);
4072 ClassLoaderDataGraph::classes_do(&preclean_klass_closure);
4073
4074 verify_work_stacks_empty();
4075 verify_overflow_empty();
4076 }
4077
4078 void CMSCollector::checkpointRootsFinal() {
4079 assert(_collectorState == FinalMarking, "incorrect state transition?");
4080 check_correct_thread_executing();
4081 // world is stopped at this checkpoint
4082 assert(SafepointSynchronize::is_at_safepoint(),
4083 "world should be stopped");
4084 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
4085
4086 verify_work_stacks_empty();
4087 verify_overflow_empty();
4088
4089 log_debug(gc)("YG occupancy: " SIZE_FORMAT " K (" SIZE_FORMAT " K)",
4090 _young_gen->used() / K, _young_gen->capacity() / K);
4091 {
4092 if (CMSScavengeBeforeRemark) {
4093 GenCollectedHeap* gch = GenCollectedHeap::heap();
4094 // Temporarily set flag to false, GCH->do_collection will
4095 // expect it to be false and set to true
4096 FlagSetting fl(gch->_is_gc_active, false);
4097
4098 GCTraceTime(Trace, gc) tm("Pause Scavenge Before Remark", _gc_timer_cm);
4099
4100 gch->do_collection(true, // full (i.e. force, see below)
4101 false, // !clear_all_soft_refs
4102 0, // size
4103 false, // is_tlab
4104 GenCollectedHeap::YoungGen // type
4105 );
4106 }
4107 FreelistLocker x(this);
4108 MutexLockerEx y(bitMapLock(),
4109 Mutex::_no_safepoint_check_flag);
4110 checkpointRootsFinalWork();
4111 }
4112 verify_work_stacks_empty();
4113 verify_overflow_empty();
4114 }
4115
4116 void CMSCollector::checkpointRootsFinalWork() {
4117 GCTraceTime(Trace, gc) tm("checkpointRootsFinalWork", _gc_timer_cm);
4118
4119 assert(haveFreelistLocks(), "must have free list locks");
4120 assert_lock_strong(bitMapLock());
4121
4122 ResourceMark rm;
4123 HandleMark hm;
4124
4125 GenCollectedHeap* gch = GenCollectedHeap::heap();
4126
4127 if (should_unload_classes()) {
4128 CodeCache::gc_prologue();
4129 }
4130 assert(haveFreelistLocks(), "must have free list locks");
4131 assert_lock_strong(bitMapLock());
4132
4133 // We might assume that we need not fill TLAB's when
4134 // CMSScavengeBeforeRemark is set, because we may have just done
4135 // a scavenge which would have filled all TLAB's -- and besides
4136 // Eden would be empty. This however may not always be the case --
4137 // for instance although we asked for a scavenge, it may not have
4138 // happened because of a JNI critical section. We probably need
4139 // a policy for deciding whether we can in that case wait until
4140 // the critical section releases and then do the remark following
4141 // the scavenge, and skip it here. In the absence of that policy,
4142 // or of an indication of whether the scavenge did indeed occur,
4143 // we cannot rely on TLAB's having been filled and must do
4144 // so here just in case a scavenge did not happen.
4145 gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
4146 // Update the saved marks which may affect the root scans.
4147 gch->save_marks();
4148
4149 print_eden_and_survivor_chunk_arrays();
4150
4151 {
4152 #if defined(COMPILER2) || INCLUDE_JVMCI
4153 DerivedPointerTableDeactivate dpt_deact;
4154 #endif
4155
4156 // Note on the role of the mod union table:
4157 // Since the marker in "markFromRoots" marks concurrently with
4158 // mutators, it is possible for some reachable objects not to have been
4159 // scanned. For instance, an only reference to an object A was
4160 // placed in object B after the marker scanned B. Unless B is rescanned,
4161 // A would be collected. Such updates to references in marked objects
4162 // are detected via the mod union table which is the set of all cards
4163 // dirtied since the first checkpoint in this GC cycle and prior to
4164 // the most recent young generation GC, minus those cleaned up by the
4165 // concurrent precleaning.
4166 if (CMSParallelRemarkEnabled) {
4167 GCTraceTime(Debug, gc) t("Rescan (parallel)", _gc_timer_cm);
4168 do_remark_parallel();
4169 } else {
4170 GCTraceTime(Debug, gc) t("Rescan (non-parallel)", _gc_timer_cm);
4171 do_remark_non_parallel();
4172 }
4173 }
4174 verify_work_stacks_empty();
4175 verify_overflow_empty();
4176
4177 {
4178 GCTraceTime(Trace, gc) ts("refProcessingWork", _gc_timer_cm);
4179 refProcessingWork();
4180 }
4181 verify_work_stacks_empty();
4182 verify_overflow_empty();
4183
4184 if (should_unload_classes()) {
4185 CodeCache::gc_epilogue();
4186 }
4187 JvmtiExport::gc_epilogue();
4188
4189 // If we encountered any (marking stack / work queue) overflow
4190 // events during the current CMS cycle, take appropriate
4191 // remedial measures, where possible, so as to try and avoid
4192 // recurrence of that condition.
4193 assert(_markStack.isEmpty(), "No grey objects");
4194 size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
4195 _ser_kac_ovflw + _ser_kac_preclean_ovflw;
4196 if (ser_ovflw > 0) {
4197 log_trace(gc)("Marking stack overflow (benign) (pmc_pc=" SIZE_FORMAT ", pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT ", kac_preclean=" SIZE_FORMAT ")",
4198 _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw, _ser_kac_ovflw, _ser_kac_preclean_ovflw);
4199 _markStack.expand();
4200 _ser_pmc_remark_ovflw = 0;
4201 _ser_pmc_preclean_ovflw = 0;
4202 _ser_kac_preclean_ovflw = 0;
4203 _ser_kac_ovflw = 0;
4204 }
4205 if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
4206 log_trace(gc)("Work queue overflow (benign) (pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT ")",
4207 _par_pmc_remark_ovflw, _par_kac_ovflw);
4208 _par_pmc_remark_ovflw = 0;
4209 _par_kac_ovflw = 0;
4210 }
4211 if (_markStack._hit_limit > 0) {
4212 log_trace(gc)(" (benign) Hit max stack size limit (" SIZE_FORMAT ")",
4213 _markStack._hit_limit);
4214 }
4215 if (_markStack._failed_double > 0) {
4216 log_trace(gc)(" (benign) Failed stack doubling (" SIZE_FORMAT "), current capacity " SIZE_FORMAT,
4217 _markStack._failed_double, _markStack.capacity());
4218 }
4219 _markStack._hit_limit = 0;
4220 _markStack._failed_double = 0;
4221
4222 if ((VerifyAfterGC || VerifyDuringGC) &&
4223 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4224 verify_after_remark();
4225 }
4226
4227 _gc_tracer_cm->report_object_count_after_gc(&_is_alive_closure);
4228
4229 // Change under the freelistLocks.
4230 _collectorState = Sweeping;
4231 // Call isAllClear() under bitMapLock
4232 assert(_modUnionTable.isAllClear(),
4233 "Should be clear by end of the final marking");
4234 assert(_ct->klass_rem_set()->mod_union_is_clear(),
4235 "Should be clear by end of the final marking");
4236 }
4237
4238 void CMSParInitialMarkTask::work(uint worker_id) {
4239 elapsedTimer _timer;
4240 ResourceMark rm;
4241 HandleMark hm;
4242
4243 // ---------- scan from roots --------------
4244 _timer.start();
4245 GenCollectedHeap* gch = GenCollectedHeap::heap();
4246 Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap));
4247
4248 // ---------- young gen roots --------------
4249 {
4250 work_on_young_gen_roots(worker_id, &par_mri_cl);
4251 _timer.stop();
4252 log_trace(gc, task)("Finished young gen initial mark scan work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4253 }
4254
4255 // ---------- remaining roots --------------
4256 _timer.reset();
4257 _timer.start();
4258
4259 CLDToOopClosure cld_closure(&par_mri_cl, true);
4260
4261 gch->gen_process_roots(_strong_roots_scope,
4262 GenCollectedHeap::OldGen,
4263 false, // yg was scanned above
4264 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4265 _collector->should_unload_classes(),
4266 &par_mri_cl,
4267 NULL,
4268 &cld_closure);
4269 assert(_collector->should_unload_classes()
4270 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4271 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4272 _timer.stop();
4273 log_trace(gc, task)("Finished remaining root initial mark scan work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4274 }
4275
4276 // Parallel remark task
4277 class CMSParRemarkTask: public CMSParMarkTask {
4278 CompactibleFreeListSpace* _cms_space;
4279
4280 // The per-thread work queues, available here for stealing.
4281 OopTaskQueueSet* _task_queues;
4282 ParallelTaskTerminator _term;
4283 StrongRootsScope* _strong_roots_scope;
4284
4285 public:
4286 // A value of 0 passed to n_workers will cause the number of
4287 // workers to be taken from the active workers in the work gang.
4288 CMSParRemarkTask(CMSCollector* collector,
4289 CompactibleFreeListSpace* cms_space,
4290 uint n_workers, WorkGang* workers,
4291 OopTaskQueueSet* task_queues,
4292 StrongRootsScope* strong_roots_scope):
4293 CMSParMarkTask("Rescan roots and grey objects in parallel",
4366 elapsedTimer _timer;
4367 ResourceMark rm;
4368 HandleMark hm;
4369
4370 // ---------- rescan from roots --------------
4371 _timer.start();
4372 GenCollectedHeap* gch = GenCollectedHeap::heap();
4373 Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
4374 _collector->_span, _collector->ref_processor(),
4375 &(_collector->_markBitMap),
4376 work_queue(worker_id));
4377
4378 // Rescan young gen roots first since these are likely
4379 // coarsely partitioned and may, on that account, constitute
4380 // the critical path; thus, it's best to start off that
4381 // work first.
4382 // ---------- young gen roots --------------
4383 {
4384 work_on_young_gen_roots(worker_id, &par_mrias_cl);
4385 _timer.stop();
4386 log_trace(gc, task)("Finished young gen rescan work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4387 }
4388
4389 // ---------- remaining roots --------------
4390 _timer.reset();
4391 _timer.start();
4392 gch->gen_process_roots(_strong_roots_scope,
4393 GenCollectedHeap::OldGen,
4394 false, // yg was scanned above
4395 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4396 _collector->should_unload_classes(),
4397 &par_mrias_cl,
4398 NULL,
4399 NULL); // The dirty klasses will be handled below
4400
4401 assert(_collector->should_unload_classes()
4402 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4403 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4404 _timer.stop();
4405 log_trace(gc, task)("Finished remaining root rescan work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4406
4407 // ---------- unhandled CLD scanning ----------
4408 if (worker_id == 0) { // Single threaded at the moment.
4409 _timer.reset();
4410 _timer.start();
4411
4412 // Scan all new class loader data objects and new dependencies that were
4413 // introduced during concurrent marking.
4414 ResourceMark rm;
4415 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
4416 for (int i = 0; i < array->length(); i++) {
4417 par_mrias_cl.do_cld_nv(array->at(i));
4418 }
4419
4420 // We don't need to keep track of new CLDs anymore.
4421 ClassLoaderDataGraph::remember_new_clds(false);
4422
4423 _timer.stop();
4424 log_trace(gc, task)("Finished unhandled CLD scanning work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4425 }
4426
4427 // ---------- dirty klass scanning ----------
4428 if (worker_id == 0) { // Single threaded at the moment.
4429 _timer.reset();
4430 _timer.start();
4431
4432 // Scan all classes that was dirtied during the concurrent marking phase.
4433 RemarkKlassClosure remark_klass_closure(&par_mrias_cl);
4434 ClassLoaderDataGraph::classes_do(&remark_klass_closure);
4435
4436 _timer.stop();
4437 log_trace(gc, task)("Finished dirty klass scanning work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4438 }
4439
4440 // We might have added oops to ClassLoaderData::_handles during the
4441 // concurrent marking phase. These oops point to newly allocated objects
4442 // that are guaranteed to be kept alive. Either by the direct allocation
4443 // code, or when the young collector processes the roots. Hence,
4444 // we don't have to revisit the _handles block during the remark phase.
4445
4446 // ---------- rescan dirty cards ------------
4447 _timer.reset();
4448 _timer.start();
4449
4450 // Do the rescan tasks for each of the two spaces
4451 // (cms_space) in turn.
4452 // "worker_id" is passed to select the task_queue for "worker_id"
4453 do_dirty_card_rescan_tasks(_cms_space, worker_id, &par_mrias_cl);
4454 _timer.stop();
4455 log_trace(gc, task)("Finished dirty card rescan work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4456
4457 // ---------- steal work from other threads ...
4458 // ---------- ... and drain overflow list.
4459 _timer.reset();
4460 _timer.start();
4461 do_work_steal(worker_id, &par_mrias_cl, _collector->hash_seed(worker_id));
4462 _timer.stop();
4463 log_trace(gc, task)("Finished work stealing in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4464 }
4465
4466 // Note that parameter "i" is not used.
4467 void
4468 CMSParMarkTask::do_young_space_rescan(uint worker_id,
4469 OopsInGenClosure* cl, ContiguousSpace* space,
4470 HeapWord** chunk_array, size_t chunk_top) {
4471 // Until all tasks completed:
4472 // . claim an unclaimed task
4473 // . compute region boundaries corresponding to task claimed
4474 // using chunk_array
4475 // . par_oop_iterate(cl) over that region
4476
4477 ResourceMark rm;
4478 HandleMark hm;
4479
4480 SequentialSubTasksDone* pst = space->par_seq_tasks();
4481
4482 uint nth_task = 0;
4483 uint n_tasks = pst->n_tasks();
4637 // because we just took work from the overflow list,
4638 // but of course we can't since all of that could have
4639 // been already stolen from us.
4640 // "He giveth and He taketh away."
4641 continue;
4642 }
4643 // Verify that we have no work before we resort to stealing
4644 assert(work_q->size() == 0, "Have work, shouldn't steal");
4645 // Try to steal from other queues that have work
4646 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
4647 NOT_PRODUCT(num_steals++;)
4648 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
4649 assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
4650 // Do scanning work
4651 obj_to_scan->oop_iterate(cl);
4652 // Loop around, finish this work, and try to steal some more
4653 } else if (terminator()->offer_termination()) {
4654 break; // nirvana from the infinite cycle
4655 }
4656 }
4657 log_develop_trace(gc, task)("\t(%d: stole %d oops)", i, num_steals);
4658 assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
4659 "Else our work is not yet done");
4660 }
4661
4662 // Record object boundaries in _eden_chunk_array by sampling the eden
4663 // top in the slow-path eden object allocation code path and record
4664 // the boundaries, if CMSEdenChunksRecordAlways is true. If
4665 // CMSEdenChunksRecordAlways is false, we use the other asynchronous
4666 // sampling in sample_eden() that activates during the part of the
4667 // preclean phase.
4668 void CMSCollector::sample_eden_chunk() {
4669 if (CMSEdenChunksRecordAlways && _eden_chunk_array != NULL) {
4670 if (_eden_chunk_lock->try_lock()) {
4671 // Record a sample. This is the critical section. The contents
4672 // of the _eden_chunk_array have to be non-decreasing in the
4673 // address order.
4674 _eden_chunk_array[_eden_chunk_index] = *_top_addr;
4675 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
4676 "Unexpected state of Eden");
4677 if (_eden_chunk_index == 0 ||
4734 if (cur_val < min_val) {
4735 min_tid = j;
4736 min_val = cur_val;
4737 } else {
4738 assert(cur_val < top, "All recorded addresses should be less");
4739 }
4740 }
4741 // At this point min_val and min_tid are respectively
4742 // the least address in _survivor_plab_array[j]->nth(_cursor[j])
4743 // and the thread (j) that witnesses that address.
4744 // We record this address in the _survivor_chunk_array[i]
4745 // and increment _cursor[min_tid] prior to the next round i.
4746 if (min_val == top) {
4747 break;
4748 }
4749 _survivor_chunk_array[i] = min_val;
4750 _cursor[min_tid]++;
4751 }
4752 // We are all done; record the size of the _survivor_chunk_array
4753 _survivor_chunk_index = i; // exclusive: [0, i)
4754 log_trace(gc, survivor)(" (Survivor:" SIZE_FORMAT "chunks) ", i);
4755 // Verify that we used up all the recorded entries
4756 #ifdef ASSERT
4757 size_t total = 0;
4758 for (int j = 0; j < no_of_gc_threads; j++) {
4759 assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
4760 total += _cursor[j];
4761 }
4762 assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
4763 // Check that the merged array is in sorted order
4764 if (total > 0) {
4765 for (size_t i = 0; i < total - 1; i++) {
4766 log_develop_trace(gc, survivor)(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
4767 i, p2i(_survivor_chunk_array[i]));
4768 assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
4769 "Not sorted");
4770 }
4771 }
4772 #endif // ASSERT
4773 }
4774
4775 // Set up the space's par_seq_tasks structure for work claiming
4776 // for parallel initial scan and rescan of young gen.
4777 // See ParRescanTask where this is currently used.
4778 void
4779 CMSCollector::
4780 initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
4781 assert(n_threads > 0, "Unexpected n_threads argument");
4782
4783 // Eden space
4784 if (!_young_gen->eden()->is_empty()) {
4785 SequentialSubTasksDone* pst = _young_gen->eden()->par_seq_tasks();
4786 assert(!pst->valid(), "Clobbering existing data?");
4787 // Each valid entry in [0, _eden_chunk_index) represents a task.
4881 // as a result of work_q overflow
4882 restore_preserved_marks_if_any();
4883 }
4884
4885 // Non-parallel version of remark
4886 void CMSCollector::do_remark_non_parallel() {
4887 ResourceMark rm;
4888 HandleMark hm;
4889 GenCollectedHeap* gch = GenCollectedHeap::heap();
4890 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
4891
4892 MarkRefsIntoAndScanClosure
4893 mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */,
4894 &_markStack, this,
4895 false /* should_yield */, false /* not precleaning */);
4896 MarkFromDirtyCardsClosure
4897 markFromDirtyCardsClosure(this, _span,
4898 NULL, // space is set further below
4899 &_markBitMap, &_markStack, &mrias_cl);
4900 {
4901 GCTraceTime(Trace, gc) t("Grey Object Rescan", _gc_timer_cm);
4902 // Iterate over the dirty cards, setting the corresponding bits in the
4903 // mod union table.
4904 {
4905 ModUnionClosure modUnionClosure(&_modUnionTable);
4906 _ct->ct_bs()->dirty_card_iterate(
4907 _cmsGen->used_region(),
4908 &modUnionClosure);
4909 }
4910 // Having transferred these marks into the modUnionTable, we just need
4911 // to rescan the marked objects on the dirty cards in the modUnionTable.
4912 // The initial marking may have been done during an asynchronous
4913 // collection so there may be dirty bits in the mod-union table.
4914 const int alignment =
4915 CardTableModRefBS::card_size * BitsPerWord;
4916 {
4917 // ... First handle dirty cards in CMS gen
4918 markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
4919 MemRegion ur = _cmsGen->used_region();
4920 HeapWord* lb = ur.start();
4921 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
4922 MemRegion cms_span(lb, ub);
4923 _modUnionTable.dirty_range_iterate_clear(cms_span,
4924 &markFromDirtyCardsClosure);
4925 verify_work_stacks_empty();
4926 log_trace(gc)(" (re-scanned " SIZE_FORMAT " dirty cards in cms gen) ", markFromDirtyCardsClosure.num_dirty_cards());
4927 }
4928 }
4929 if (VerifyDuringGC &&
4930 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4931 HandleMark hm; // Discard invalid handles created during verification
4932 Universe::verify();
4933 }
4934 {
4935 GCTraceTime(Trace, gc) t("Root Rescan", _gc_timer_cm);
4936
4937 verify_work_stacks_empty();
4938
4939 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
4940 StrongRootsScope srs(1);
4941
4942 gch->gen_process_roots(&srs,
4943 GenCollectedHeap::OldGen,
4944 true, // young gen as roots
4945 GenCollectedHeap::ScanningOption(roots_scanning_options()),
4946 should_unload_classes(),
4947 &mrias_cl,
4948 NULL,
4949 NULL); // The dirty klasses will be handled below
4950
4951 assert(should_unload_classes()
4952 || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4953 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4954 }
4955
4956 {
4957 GCTraceTime(Trace, gc) t("Visit Unhandled CLDs", _gc_timer_cm);
4958
4959 verify_work_stacks_empty();
4960
4961 // Scan all class loader data objects that might have been introduced
4962 // during concurrent marking.
4963 ResourceMark rm;
4964 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
4965 for (int i = 0; i < array->length(); i++) {
4966 mrias_cl.do_cld_nv(array->at(i));
4967 }
4968
4969 // We don't need to keep track of new CLDs anymore.
4970 ClassLoaderDataGraph::remember_new_clds(false);
4971
4972 verify_work_stacks_empty();
4973 }
4974
4975 {
4976 GCTraceTime(Trace, gc) t("Dirty Klass Scan", _gc_timer_cm);
4977
4978 verify_work_stacks_empty();
4979
4980 RemarkKlassClosure remark_klass_closure(&mrias_cl);
4981 ClassLoaderDataGraph::classes_do(&remark_klass_closure);
4982
4983 verify_work_stacks_empty();
4984 }
4985
4986 // We might have added oops to ClassLoaderData::_handles during the
4987 // concurrent marking phase. These oops point to newly allocated objects
4988 // that are guaranteed to be kept alive. Either by the direct allocation
4989 // code, or when the young collector processes the roots. Hence,
4990 // we don't have to revisit the _handles block during the remark phase.
4991
4992 verify_work_stacks_empty();
4993 // Restore evacuated mark words, if any, used for overflow list links
4994 restore_preserved_marks_if_any();
4995
4996 verify_overflow_empty();
5118 // We'd like to assert(work_q->size() != 0, ...)
5119 // because we just took work from the overflow list,
5120 // but of course we can't, since all of that might have
5121 // been already stolen from us.
5122 continue;
5123 }
5124 // Verify that we have no work before we resort to stealing
5125 assert(work_q->size() == 0, "Have work, shouldn't steal");
5126 // Try to steal from other queues that have work
5127 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5128 NOT_PRODUCT(num_steals++;)
5129 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5130 assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5131 // Do scanning work
5132 obj_to_scan->oop_iterate(keep_alive);
5133 // Loop around, finish this work, and try to steal some more
5134 } else if (terminator()->offer_termination()) {
5135 break; // nirvana from the infinite cycle
5136 }
5137 }
5138 log_develop_trace(gc, task)("\t(%d: stole %d oops)", i, num_steals);
5139 }
5140
5141 void CMSRefProcTaskExecutor::execute(ProcessTask& task)
5142 {
5143 GenCollectedHeap* gch = GenCollectedHeap::heap();
5144 WorkGang* workers = gch->workers();
5145 assert(workers != NULL, "Need parallel worker threads.");
5146 CMSRefProcTaskProxy rp_task(task, &_collector,
5147 _collector.ref_processor()->span(),
5148 _collector.markBitMap(),
5149 workers, _collector.task_queues());
5150 workers->run_task(&rp_task);
5151 }
5152
5153 void CMSRefProcTaskExecutor::execute(EnqueueTask& task)
5154 {
5155
5156 GenCollectedHeap* gch = GenCollectedHeap::heap();
5157 WorkGang* workers = gch->workers();
5158 assert(workers != NULL, "Need parallel worker threads.");
5160 workers->run_task(&enq_task);
5161 }
5162
5163 void CMSCollector::refProcessingWork() {
5164 ResourceMark rm;
5165 HandleMark hm;
5166
5167 ReferenceProcessor* rp = ref_processor();
5168 assert(rp->span().equals(_span), "Spans should be equal");
5169 assert(!rp->enqueuing_is_done(), "Enqueuing should not be complete");
5170 // Process weak references.
5171 rp->setup_policy(false);
5172 verify_work_stacks_empty();
5173
5174 CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap,
5175 &_markStack, false /* !preclean */);
5176 CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this,
5177 _span, &_markBitMap, &_markStack,
5178 &cmsKeepAliveClosure, false /* !preclean */);
5179 {
5180 GCTraceTime(Debug, gc) t("Weak Refs Processing", _gc_timer_cm);
5181
5182 ReferenceProcessorStats stats;
5183 if (rp->processing_is_mt()) {
5184 // Set the degree of MT here. If the discovery is done MT, there
5185 // may have been a different number of threads doing the discovery
5186 // and a different number of discovered lists may have Ref objects.
5187 // That is OK as long as the Reference lists are balanced (see
5188 // balance_all_queues() and balance_queues()).
5189 GenCollectedHeap* gch = GenCollectedHeap::heap();
5190 uint active_workers = ParallelGCThreads;
5191 WorkGang* workers = gch->workers();
5192 if (workers != NULL) {
5193 active_workers = workers->active_workers();
5194 // The expectation is that active_workers will have already
5195 // been set to a reasonable value. If it has not been set,
5196 // investigate.
5197 assert(active_workers > 0, "Should have been set during scavenge");
5198 }
5199 rp->set_active_mt_degree(active_workers);
5200 CMSRefProcTaskExecutor task_executor(*this);
5202 &cmsKeepAliveClosure,
5203 &cmsDrainMarkingStackClosure,
5204 &task_executor,
5205 _gc_timer_cm);
5206 } else {
5207 stats = rp->process_discovered_references(&_is_alive_closure,
5208 &cmsKeepAliveClosure,
5209 &cmsDrainMarkingStackClosure,
5210 NULL,
5211 _gc_timer_cm);
5212 }
5213 _gc_tracer_cm->report_gc_reference_stats(stats);
5214
5215 }
5216
5217 // This is the point where the entire marking should have completed.
5218 verify_work_stacks_empty();
5219
5220 if (should_unload_classes()) {
5221 {
5222 GCTraceTime(Debug, gc) t("Class Unloading", _gc_timer_cm);
5223
5224 // Unload classes and purge the SystemDictionary.
5225 bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure);
5226
5227 // Unload nmethods.
5228 CodeCache::do_unloading(&_is_alive_closure, purged_class);
5229
5230 // Prune dead klasses from subklass/sibling/implementor lists.
5231 Klass::clean_weak_klass_links(&_is_alive_closure);
5232 }
5233
5234 {
5235 GCTraceTime(Debug, gc) t("Scrub Symbol Table", _gc_timer_cm);
5236 // Clean up unreferenced symbols in symbol table.
5237 SymbolTable::unlink();
5238 }
5239
5240 {
5241 GCTraceTime(Debug, gc) t("Scrub String Table", _gc_timer_cm);
5242 // Delete entries for dead interned strings.
5243 StringTable::unlink(&_is_alive_closure);
5244 }
5245 }
5246
5247
5248 // Restore any preserved marks as a result of mark stack or
5249 // work queue overflow
5250 restore_preserved_marks_if_any(); // done single-threaded for now
5251
5252 rp->set_enqueuing_is_done(true);
5253 if (rp->processing_is_mt()) {
5254 rp->balance_all_queues();
5255 CMSRefProcTaskExecutor task_executor(*this);
5256 rp->enqueue_discovered_references(&task_executor);
5257 } else {
5258 rp->enqueue_discovered_references(NULL);
5259 }
5260 rp->verify_no_references_recorded();
5261 assert(!rp->discovery_enabled(), "should have been disabled");
5288 }
5289 }
5290 }
5291 #endif
5292
5293 void CMSCollector::sweep() {
5294 assert(_collectorState == Sweeping, "just checking");
5295 check_correct_thread_executing();
5296 verify_work_stacks_empty();
5297 verify_overflow_empty();
5298 increment_sweep_count();
5299 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
5300
5301 _inter_sweep_timer.stop();
5302 _inter_sweep_estimate.sample(_inter_sweep_timer.seconds());
5303
5304 assert(!_intra_sweep_timer.is_active(), "Should not be active");
5305 _intra_sweep_timer.reset();
5306 _intra_sweep_timer.start();
5307 {
5308 GCTraceCPUTime tcpu;
5309 CMSPhaseAccounting pa(this, "Concurrent Sweep");
5310 // First sweep the old gen
5311 {
5312 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
5313 bitMapLock());
5314 sweepWork(_cmsGen);
5315 }
5316
5317 // Update Universe::_heap_*_at_gc figures.
5318 // We need all the free list locks to make the abstract state
5319 // transition from Sweeping to Resetting. See detailed note
5320 // further below.
5321 {
5322 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock());
5323 // Update heap occupancy information which is used as
5324 // input to soft ref clearing policy at the next gc.
5325 Universe::update_heap_info_at_gc();
5326 _collectorState = Resizing;
5327 }
5328 }
5329 verify_work_stacks_empty();
5372 GenCollectedHeap* gch = GenCollectedHeap::heap();
5373 gch->clear_incremental_collection_failed(); // Worth retrying as fresh space may have been freed up
5374 gch->update_full_collections_completed(_collection_count_start);
5375 }
5376
5377 // FIX ME!!! Looks like this belongs in CFLSpace, with
5378 // CMSGen merely delegating to it.
5379 void ConcurrentMarkSweepGeneration::setNearLargestChunk() {
5380 double nearLargestPercent = FLSLargestBlockCoalesceProximity;
5381 HeapWord* minAddr = _cmsSpace->bottom();
5382 HeapWord* largestAddr =
5383 (HeapWord*) _cmsSpace->dictionary()->find_largest_dict();
5384 if (largestAddr == NULL) {
5385 // The dictionary appears to be empty. In this case
5386 // try to coalesce at the end of the heap.
5387 largestAddr = _cmsSpace->end();
5388 }
5389 size_t largestOffset = pointer_delta(largestAddr, minAddr);
5390 size_t nearLargestOffset =
5391 (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
5392 log_debug(gc, freelist)("CMS: Large Block: " PTR_FORMAT "; Proximity: " PTR_FORMAT " -> " PTR_FORMAT,
5393 p2i(largestAddr), p2i(_cmsSpace->nearLargestChunk()), p2i(minAddr + nearLargestOffset));
5394 _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
5395 }
5396
5397 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
5398 return addr >= _cmsSpace->nearLargestChunk();
5399 }
5400
5401 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
5402 return _cmsSpace->find_chunk_at_end();
5403 }
5404
5405 void ConcurrentMarkSweepGeneration::update_gc_stats(Generation* current_generation,
5406 bool full) {
5407 // If the young generation has been collected, gather any statistics
5408 // that are of interest at this point.
5409 bool current_is_young = GenCollectedHeap::heap()->is_young_gen(current_generation);
5410 if (!full && current_is_young) {
5411 // Gather statistics on the young generation collection.
5412 collector()->stats().record_gc0_end(used());
5413 }
5467 } else { // did not unload classes,
5468 _concurrent_cycles_since_last_unload++; // ... increment count
5469 }
5470 }
5471
5472 // Reset CMS data structures (for now just the marking bit map)
5473 // preparatory for the next cycle.
5474 void CMSCollector::reset_concurrent() {
5475 CMSTokenSyncWithLocks ts(true, bitMapLock());
5476
5477 // If the state is not "Resetting", the foreground thread
5478 // has done a collection and the resetting.
5479 if (_collectorState != Resetting) {
5480 assert(_collectorState == Idling, "The state should only change"
5481 " because the foreground collector has finished the collection");
5482 return;
5483 }
5484
5485 // Clear the mark bitmap (no grey objects to start with)
5486 // for the next cycle.
5487 GCTraceCPUTime tcpu;
5488 CMSPhaseAccounting cmspa(this, "Concurrent Reset");
5489
5490 HeapWord* curAddr = _markBitMap.startWord();
5491 while (curAddr < _markBitMap.endWord()) {
5492 size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr);
5493 MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining));
5494 _markBitMap.clear_large_range(chunk);
5495 if (ConcurrentMarkSweepThread::should_yield() &&
5496 !foregroundGCIsActive() &&
5497 CMSYield) {
5498 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
5499 "CMS thread should hold CMS token");
5500 assert_lock_strong(bitMapLock());
5501 bitMapLock()->unlock();
5502 ConcurrentMarkSweepThread::desynchronize(true);
5503 stopTimer();
5504 incrementYields();
5505
5506 // See the comment in coordinator_yield()
5507 for (unsigned i = 0; i < CMSYieldSleepCount &&
5508 ConcurrentMarkSweepThread::should_yield() &&
5509 !CMSCollector::foregroundGCIsActive(); ++i) {
5510 os::sleep(Thread::current(), 1, false);
5511 }
5512
5513 ConcurrentMarkSweepThread::synchronize(true);
5514 bitMapLock()->lock_without_safepoint_check();
5515 startTimer();
5516 }
5517 curAddr = chunk.end();
5518 }
5519 // A successful mostly concurrent collection has been done.
5520 // Because only the full (i.e., concurrent mode failure) collections
5521 // are being measured for gc overhead limits, clean the "near" flag
5522 // and count.
5523 size_policy()->reset_gc_overhead_limit_count();
5524 _collectorState = Idling;
5525
5526 register_gc_end();
5527 }
5528
5529 // Same as above but for STW paths
5530 void CMSCollector::reset_stw() {
5531 // already have the lock
5532 assert(_collectorState == Resetting, "just checking");
5533 assert_lock_strong(bitMapLock());
5534 GCIdMarkAndRestore gc_id_mark(_cmsThread->gc_id());
5535 _markBitMap.clear_all();
5536 _collectorState = Idling;
5537 register_gc_end();
5538 }
5539
5540 void CMSCollector::do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause) {
5541 GCTraceCPUTime tcpu;
5542 TraceCollectorStats tcs(counters());
5543
5544 switch (op) {
5545 case CMS_op_checkpointRootsInitial: {
5546 GCTraceTime(Info, gc) t("Pause Initial Mark", NULL, GCCause::_no_gc, true);
5547 SvcGCMarker sgcm(SvcGCMarker::OTHER);
5548 checkpointRootsInitial();
5549 break;
5550 }
5551 case CMS_op_checkpointRootsFinal: {
5552 GCTraceTime(Info, gc) t("Pause Remark", NULL, GCCause::_no_gc, true);
5553 SvcGCMarker sgcm(SvcGCMarker::OTHER);
5554 checkpointRootsFinal();
5555 break;
5556 }
5557 default:
5558 fatal("No such CMS_op");
5559 }
5560 }
5561
5562 #ifndef PRODUCT
5563 size_t const CMSCollector::skip_header_HeapWords() {
5564 return FreeChunk::header_size();
5565 }
5566
5567 // Try and collect here conditions that should hold when
5568 // CMS thread is exiting. The idea is that the foreground GC
5569 // thread should not be blocked if it wants to terminate
5570 // the CMS thread and yet continue to run the VM for a while
5571 // after that.
5572 void CMSCollector::verify_ok_to_terminate() const {
5573 assert(Thread::current()->is_ConcurrentGC_thread(),
5574 "should be called by CMS thread");
5747 }
5748 assert(_virtual_space.committed_size() == rs.size(),
5749 "didn't reserve backing store for all of CMS stack?");
5750 _base = (oop*)(_virtual_space.low());
5751 _index = 0;
5752 _capacity = size;
5753 NOT_PRODUCT(_max_depth = 0);
5754 return true;
5755 }
5756
5757 // XXX FIX ME !!! In the MT case we come in here holding a
5758 // leaf lock. For printing we need to take a further lock
5759 // which has lower rank. We need to recalibrate the two
5760 // lock-ranks involved in order to be able to print the
5761 // messages below. (Or defer the printing to the caller.
5762 // For now we take the expedient path of just disabling the
5763 // messages for the problematic case.)
5764 void CMSMarkStack::expand() {
5765 assert(_capacity <= MarkStackSizeMax, "stack bigger than permitted");
5766 if (_capacity == MarkStackSizeMax) {
5767 if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled) {
5768 // We print a warning message only once per CMS cycle.
5769 log_debug(gc)(" (benign) Hit CMSMarkStack max size limit");
5770 }
5771 return;
5772 }
5773 // Double capacity if possible
5774 size_t new_capacity = MIN2(_capacity*2, MarkStackSizeMax);
5775 // Do not give up existing stack until we have managed to
5776 // get the double capacity that we desired.
5777 ReservedSpace rs(ReservedSpace::allocation_align_size_up(
5778 new_capacity * sizeof(oop)));
5779 if (rs.is_reserved()) {
5780 // Release the backing store associated with old stack
5781 _virtual_space.release();
5782 // Reinitialize virtual space for new stack
5783 if (!_virtual_space.initialize(rs, rs.size())) {
5784 fatal("Not enough swap for expanded marking stack");
5785 }
5786 _base = (oop*)(_virtual_space.low());
5787 _index = 0;
5788 _capacity = new_capacity;
5789 } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled) {
5790 // Failed to double capacity, continue;
5791 // we print a detail message only once per CMS cycle.
5792 log_debug(gc)(" (benign) Failed to expand marking stack from " SIZE_FORMAT "K to " SIZE_FORMAT "K",
5793 _capacity / K, new_capacity / K);
5794 }
5795 }
5796
5797
5798 // Closures
5799 // XXX: there seems to be a lot of code duplication here;
5800 // should refactor and consolidate common code.
5801
5802 // This closure is used to mark refs into the CMS generation in
5803 // the CMS bit map. Called at the first checkpoint. This closure
5804 // assumes that we do not need to re-mark dirty cards; if the CMS
5805 // generation on which this is used is not an oldest
5806 // generation then this will lose younger_gen cards!
5807
5808 MarkRefsIntoClosure::MarkRefsIntoClosure(
5809 MemRegion span, CMSBitMap* bitMap):
5810 _span(span),
5811 _bitMap(bitMap)
5812 {
5850 void Par_MarkRefsIntoClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); }
5851
5852 // A variant of the above, used for CMS marking verification.
5853 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
5854 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
5855 _span(span),
5856 _verification_bm(verification_bm),
5857 _cms_bm(cms_bm)
5858 {
5859 assert(ref_processor() == NULL, "deliberately left NULL");
5860 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
5861 }
5862
5863 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
5864 // if p points into _span, then mark corresponding bit in _markBitMap
5865 assert(obj->is_oop(), "expected an oop");
5866 HeapWord* addr = (HeapWord*)obj;
5867 if (_span.contains(addr)) {
5868 _verification_bm->mark(addr);
5869 if (!_cms_bm->isMarked(addr)) {
5870 LogHandle(gc, verify) log;
5871 ResourceMark rm;
5872 oop(addr)->print_on(log.info_stream());
5873 log.info(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
5874 fatal("... aborting");
5875 }
5876 }
5877 }
5878
5879 void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5880 void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5881
5882 //////////////////////////////////////////////////
5883 // MarkRefsIntoAndScanClosure
5884 //////////////////////////////////////////////////
5885
5886 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
5887 ReferenceProcessor* rp,
5888 CMSBitMap* bit_map,
5889 CMSBitMap* mod_union_table,
5890 CMSMarkStack* mark_stack,
5891 CMSCollector* collector,
5892 bool should_yield,
5893 bool concurrent_precleaning):
5949 "overflow list was drained above");
5950
5951 assert(_collector->no_preserved_marks(),
5952 "All preserved marks should have been restored above");
5953 }
5954 }
5955
5956 void MarkRefsIntoAndScanClosure::do_oop(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
5957 void MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
5958
5959 void MarkRefsIntoAndScanClosure::do_yield_work() {
5960 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
5961 "CMS thread should hold CMS token");
5962 assert_lock_strong(_freelistLock);
5963 assert_lock_strong(_bit_map->lock());
5964 // relinquish the free_list_lock and bitMaplock()
5965 _bit_map->lock()->unlock();
5966 _freelistLock->unlock();
5967 ConcurrentMarkSweepThread::desynchronize(true);
5968 _collector->stopTimer();
5969 _collector->incrementYields();
5970
5971 // See the comment in coordinator_yield()
5972 for (unsigned i = 0;
5973 i < CMSYieldSleepCount &&
5974 ConcurrentMarkSweepThread::should_yield() &&
5975 !CMSCollector::foregroundGCIsActive();
5976 ++i) {
5977 os::sleep(Thread::current(), 1, false);
5978 }
5979
5980 ConcurrentMarkSweepThread::synchronize(true);
5981 _freelistLock->lock_without_safepoint_check();
5982 _bit_map->lock()->lock_without_safepoint_check();
5983 _collector->startTimer();
5984 }
5985
5986 ///////////////////////////////////////////////////////////
5987 // Par_MarkRefsIntoAndScanClosure: a parallel version of
5988 // MarkRefsIntoAndScanClosure
5989 ///////////////////////////////////////////////////////////
6105 // An object not (yet) reached by marking: we merely need to
6106 // compute its size so as to go look at the next block.
6107 assert(p->is_oop(true), "should be an oop");
6108 size = CompactibleFreeListSpace::adjustObjectSize(p->size());
6109 }
6110 }
6111 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6112 return size;
6113 }
6114
6115 void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
6116 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6117 "CMS thread should hold CMS token");
6118 assert_lock_strong(_freelistLock);
6119 assert_lock_strong(_bitMap->lock());
6120 // relinquish the free_list_lock and bitMaplock()
6121 _bitMap->lock()->unlock();
6122 _freelistLock->unlock();
6123 ConcurrentMarkSweepThread::desynchronize(true);
6124 _collector->stopTimer();
6125 _collector->incrementYields();
6126
6127 // See the comment in coordinator_yield()
6128 for (unsigned i = 0; i < CMSYieldSleepCount &&
6129 ConcurrentMarkSweepThread::should_yield() &&
6130 !CMSCollector::foregroundGCIsActive(); ++i) {
6131 os::sleep(Thread::current(), 1, false);
6132 }
6133
6134 ConcurrentMarkSweepThread::synchronize(true);
6135 _freelistLock->lock_without_safepoint_check();
6136 _bitMap->lock()->lock_without_safepoint_check();
6137 _collector->startTimer();
6138 }
6139
6140
6141 //////////////////////////////////////////////////////////////////
6142 // SurvivorSpacePrecleanClosure
6143 //////////////////////////////////////////////////////////////////
6144 // This (single-threaded) closure is used to preclean the oops in
6145 // the survivor spaces.
6172 // the ones in CMS heap (i.e. in _span).
6173 new_oop->oop_iterate(_scanning_closure);
6174 // check if it's time to yield
6175 do_yield_check();
6176 }
6177 unsigned int after_count =
6178 GenCollectedHeap::heap()->total_collections();
6179 bool abort = (_before_count != after_count) ||
6180 _collector->should_abort_preclean();
6181 return abort ? 0 : size;
6182 }
6183
6184 void SurvivorSpacePrecleanClosure::do_yield_work() {
6185 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6186 "CMS thread should hold CMS token");
6187 assert_lock_strong(_bit_map->lock());
6188 // Relinquish the bit map lock
6189 _bit_map->lock()->unlock();
6190 ConcurrentMarkSweepThread::desynchronize(true);
6191 _collector->stopTimer();
6192 _collector->incrementYields();
6193
6194 // See the comment in coordinator_yield()
6195 for (unsigned i = 0; i < CMSYieldSleepCount &&
6196 ConcurrentMarkSweepThread::should_yield() &&
6197 !CMSCollector::foregroundGCIsActive(); ++i) {
6198 os::sleep(Thread::current(), 1, false);
6199 }
6200
6201 ConcurrentMarkSweepThread::synchronize(true);
6202 _bit_map->lock()->lock_without_safepoint_check();
6203 _collector->startTimer();
6204 }
6205
6206 // This closure is used to rescan the marked objects on the dirty cards
6207 // in the mod union table and the card table proper. In the parallel
6208 // case, although the bitMap is shared, we do a single read so the
6209 // isMarked() query is "safe".
6210 bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) {
6211 // Ignore mark word because we are running concurrent with mutators
6212 assert(p->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(p));
6325 return true;
6326 }
6327 }
6328 scanOopsInOop(addr);
6329 return true;
6330 }
6331
6332 // We take a break if we've been at this for a while,
6333 // so as to avoid monopolizing the locks involved.
6334 void MarkFromRootsClosure::do_yield_work() {
6335 // First give up the locks, then yield, then re-lock
6336 // We should probably use a constructor/destructor idiom to
6337 // do this unlock/lock or modify the MutexUnlocker class to
6338 // serve our purpose. XXX
6339 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6340 "CMS thread should hold CMS token");
6341 assert_lock_strong(_bitMap->lock());
6342 _bitMap->lock()->unlock();
6343 ConcurrentMarkSweepThread::desynchronize(true);
6344 _collector->stopTimer();
6345 _collector->incrementYields();
6346
6347 // See the comment in coordinator_yield()
6348 for (unsigned i = 0; i < CMSYieldSleepCount &&
6349 ConcurrentMarkSweepThread::should_yield() &&
6350 !CMSCollector::foregroundGCIsActive(); ++i) {
6351 os::sleep(Thread::current(), 1, false);
6352 }
6353
6354 ConcurrentMarkSweepThread::synchronize(true);
6355 _bitMap->lock()->lock_without_safepoint_check();
6356 _collector->startTimer();
6357 }
6358
6359 void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) {
6360 assert(_bitMap->isMarked(ptr), "expected bit to be set");
6361 assert(_markStack->isEmpty(),
6362 "should drain stack to limit stack usage");
6363 // convert ptr to an oop preparatory to scanning
6364 oop obj = oop(ptr);
6365 // Ignore mark word in verification below, since we
6631 void PushAndMarkVerifyClosure::do_oop(narrowOop* p) { PushAndMarkVerifyClosure::do_oop_work(p); }
6632
6633 // Upon stack overflow, we discard (part of) the stack,
6634 // remembering the least address amongst those discarded
6635 // in CMSCollector's _restart_address.
6636 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
6637 // Remember the least grey address discarded
6638 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
6639 _collector->lower_restart_addr(ra);
6640 _mark_stack->reset(); // discard stack contents
6641 _mark_stack->expand(); // expand the stack if possible
6642 }
6643
6644 void PushAndMarkVerifyClosure::do_oop(oop obj) {
6645 assert(obj->is_oop_or_null(), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6646 HeapWord* addr = (HeapWord*)obj;
6647 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
6648 // Oop lies in _span and isn't yet grey or black
6649 _verification_bm->mark(addr); // now grey
6650 if (!_cms_bm->isMarked(addr)) {
6651 LogHandle(gc, verify) log;
6652 ResourceMark rm;
6653 oop(addr)->print_on(log.info_stream());
6654 log.info(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
6655 fatal("... aborting");
6656 }
6657
6658 if (!_mark_stack->push(obj)) { // stack overflow
6659 log_trace(gc)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _mark_stack->capacity());
6660 assert(_mark_stack->isFull(), "Else push should have succeeded");
6661 handle_stack_overflow(addr);
6662 }
6663 // anything including and to the right of _finger
6664 // will be scanned as we iterate over the remainder of the
6665 // bit map
6666 }
6667 }
6668
6669 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
6670 MemRegion span,
6671 CMSBitMap* bitMap, CMSMarkStack* markStack,
6672 HeapWord* finger, MarkFromRootsClosure* parent) :
6673 MetadataAwareOopClosure(collector->ref_processor()),
6674 _collector(collector),
6675 _span(span),
6676 _bitMap(bitMap),
6677 _markStack(markStack),
6678 _finger(finger),
6679 _parent(parent)
6739 void PushOrMarkClosure::do_oop(oop obj) {
6740 // Ignore mark word because we are running concurrent with mutators.
6741 assert(obj->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6742 HeapWord* addr = (HeapWord*)obj;
6743 if (_span.contains(addr) && !_bitMap->isMarked(addr)) {
6744 // Oop lies in _span and isn't yet grey or black
6745 _bitMap->mark(addr); // now grey
6746 if (addr < _finger) {
6747 // the bit map iteration has already either passed, or
6748 // sampled, this bit in the bit map; we'll need to
6749 // use the marking stack to scan this oop's oops.
6750 bool simulate_overflow = false;
6751 NOT_PRODUCT(
6752 if (CMSMarkStackOverflowALot &&
6753 _collector->simulate_overflow()) {
6754 // simulate a stack overflow
6755 simulate_overflow = true;
6756 }
6757 )
6758 if (simulate_overflow || !_markStack->push(obj)) { // stack overflow
6759 log_trace(gc)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _markStack->capacity());
6760 assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded");
6761 handle_stack_overflow(addr);
6762 }
6763 }
6764 // anything including and to the right of _finger
6765 // will be scanned as we iterate over the remainder of the
6766 // bit map
6767 do_yield_check();
6768 }
6769 }
6770
6771 void PushOrMarkClosure::do_oop(oop* p) { PushOrMarkClosure::do_oop_work(p); }
6772 void PushOrMarkClosure::do_oop(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); }
6773
6774 void Par_PushOrMarkClosure::do_oop(oop obj) {
6775 // Ignore mark word because we are running concurrent with mutators.
6776 assert(obj->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6777 HeapWord* addr = (HeapWord*)obj;
6778 if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) {
6779 // Oop lies in _span and isn't yet grey or black
6788 // -- else push on work queue
6789 if ( !res // someone else marked it, they will deal with it
6790 || (addr >= *gfa) // will be scanned in a later task
6791 || (_span.contains(addr) && addr >= _finger)) { // later in this chunk
6792 return;
6793 }
6794 // the bit map iteration has already either passed, or
6795 // sampled, this bit in the bit map; we'll need to
6796 // use the marking stack to scan this oop's oops.
6797 bool simulate_overflow = false;
6798 NOT_PRODUCT(
6799 if (CMSMarkStackOverflowALot &&
6800 _collector->simulate_overflow()) {
6801 // simulate a stack overflow
6802 simulate_overflow = true;
6803 }
6804 )
6805 if (simulate_overflow ||
6806 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
6807 // stack overflow
6808 log_trace(gc)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _overflow_stack->capacity());
6809 // We cannot assert that the overflow stack is full because
6810 // it may have been emptied since.
6811 assert(simulate_overflow ||
6812 _work_queue->size() == _work_queue->max_elems(),
6813 "Else push should have succeeded");
6814 handle_stack_overflow(addr);
6815 }
6816 do_yield_check();
6817 }
6818 }
6819
6820 void Par_PushOrMarkClosure::do_oop(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
6821 void Par_PushOrMarkClosure::do_oop(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
6822
6823 PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector,
6824 MemRegion span,
6825 ReferenceProcessor* rp,
6826 CMSBitMap* bit_map,
6827 CMSBitMap* mod_union_table,
6828 CMSMarkStack* mark_stack,
6950 _collector->par_push_on_overflow_list(obj);
6951 _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS
6952 }
6953 } // Else, some other thread got there first
6954 }
6955 }
6956
6957 void Par_PushAndMarkClosure::do_oop(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
6958 void Par_PushAndMarkClosure::do_oop(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
6959
6960 void CMSPrecleanRefsYieldClosure::do_yield_work() {
6961 Mutex* bml = _collector->bitMapLock();
6962 assert_lock_strong(bml);
6963 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6964 "CMS thread should hold CMS token");
6965
6966 bml->unlock();
6967 ConcurrentMarkSweepThread::desynchronize(true);
6968
6969 _collector->stopTimer();
6970 _collector->incrementYields();
6971
6972 // See the comment in coordinator_yield()
6973 for (unsigned i = 0; i < CMSYieldSleepCount &&
6974 ConcurrentMarkSweepThread::should_yield() &&
6975 !CMSCollector::foregroundGCIsActive(); ++i) {
6976 os::sleep(Thread::current(), 1, false);
6977 }
6978
6979 ConcurrentMarkSweepThread::synchronize(true);
6980 bml->lock();
6981
6982 _collector->startTimer();
6983 }
6984
6985 bool CMSPrecleanRefsYieldClosure::should_return() {
6986 if (ConcurrentMarkSweepThread::should_yield()) {
6987 do_yield_work();
6988 }
6989 return _collector->foregroundGCIsActive();
6990 }
6991
6992 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
6993 assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
6994 "mr should be aligned to start at a card boundary");
6995 // We'd like to assert:
6996 // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
6997 // "mr should be a range of cards");
6998 // However, that would be too strong in one case -- the last
6999 // partition ends at _unallocated_block which, in general, can be
7000 // an arbitrary boundary, not necessarily card aligned.
7001 _num_dirty_cards += mr.word_size()/CardTableModRefBS::card_size_in_words;
7002 _space->object_iterate_mem(mr, &_scan_cl);
7003 }
7004
7005 SweepClosure::SweepClosure(CMSCollector* collector,
7006 ConcurrentMarkSweepGeneration* g,
7007 CMSBitMap* bitMap, bool should_yield) :
7008 _collector(collector),
7009 _g(g),
7010 _sp(g->cmsSpace()),
7011 _limit(_sp->sweep_limit()),
7012 _freelistLock(_sp->freelistLock()),
7013 _bitMap(bitMap),
7014 _yield(should_yield),
7015 _inFreeRange(false), // No free range at beginning of sweep
7016 _freeRangeInFreeLists(false), // No free range at beginning of sweep
7017 _lastFreeRangeCoalesced(false),
7018 _freeFinger(g->used_region().start())
7019 {
7020 NOT_PRODUCT(
7021 _numObjectsFreed = 0;
7022 _numWordsFreed = 0;
7023 _numObjectsLive = 0;
7024 _numWordsLive = 0;
7025 _numObjectsAlreadyFree = 0;
7026 _numWordsAlreadyFree = 0;
7027 _last_fc = NULL;
7028
7029 _sp->initializeIndexedFreeListArrayReturnedBytes();
7030 _sp->dictionary()->initialize_dict_returned_bytes();
7031 )
7032 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7033 "sweep _limit out of bounds");
7034 log_develop_trace(gc, sweep)("====================");
7035 log_develop_trace(gc, sweep)("Starting new sweep with limit " PTR_FORMAT, p2i(_limit));
7036 }
7037
7038 void SweepClosure::print_on(outputStream* st) const {
7039 tty->print_cr("_sp = [" PTR_FORMAT "," PTR_FORMAT ")",
7040 p2i(_sp->bottom()), p2i(_sp->end()));
7041 tty->print_cr("_limit = " PTR_FORMAT, p2i(_limit));
7042 tty->print_cr("_freeFinger = " PTR_FORMAT, p2i(_freeFinger));
7043 NOT_PRODUCT(tty->print_cr("_last_fc = " PTR_FORMAT, p2i(_last_fc));)
7044 tty->print_cr("_inFreeRange = %d, _freeRangeInFreeLists = %d, _lastFreeRangeCoalesced = %d",
7045 _inFreeRange, _freeRangeInFreeLists, _lastFreeRangeCoalesced);
7046 }
7047
7048 #ifndef PRODUCT
7049 // Assertion checking only: no useful work in product mode --
7050 // however, if any of the flags below become product flags,
7051 // you may need to review this code to see if it needs to be
7052 // enabled in product mode.
7053 SweepClosure::~SweepClosure() {
7054 assert_lock_strong(_freelistLock);
7055 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7056 "sweep _limit out of bounds");
7057 if (inFreeRange()) {
7058 warning("inFreeRange() should have been reset; dumping state of SweepClosure");
7059 print();
7060 ShouldNotReachHere();
7061 }
7062
7063 if (log_is_enabled(Debug, gc, sweep)) {
7064 log_debug(gc, sweep)("Collected " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7065 _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
7066 log_debug(gc, sweep)("Live " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes Already free " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7067 _numObjectsLive, _numWordsLive*sizeof(HeapWord), _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
7068 size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree) * sizeof(HeapWord);
7069 log_debug(gc, sweep)("Total sweep: " SIZE_FORMAT " bytes", totalBytes);
7070 }
7071
7072 if (log_is_enabled(Trace, gc, sweep) && CMSVerifyReturnedBytes) {
7073 size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
7074 size_t dict_returned_bytes = _sp->dictionary()->sum_dict_returned_bytes();
7075 size_t returned_bytes = indexListReturnedBytes + dict_returned_bytes;
7076 log_trace(gc, sweep)("Returned " SIZE_FORMAT " bytes Indexed List Returned " SIZE_FORMAT " bytes Dictionary Returned " SIZE_FORMAT " bytes",
7077 returned_bytes, indexListReturnedBytes, dict_returned_bytes);
7078 }
7079 log_develop_trace(gc, sweep)("end of sweep with _limit = " PTR_FORMAT, p2i(_limit));
7080 log_develop_trace(gc, sweep)("================");
7081 }
7082 #endif // PRODUCT
7083
7084 void SweepClosure::initialize_free_range(HeapWord* freeFinger,
7085 bool freeRangeInFreeLists) {
7086 log_develop_trace(gc, sweep)("---- Start free range at " PTR_FORMAT " with free block (%d)",
7087 p2i(freeFinger), freeRangeInFreeLists);
7088 assert(!inFreeRange(), "Trampling existing free range");
7089 set_inFreeRange(true);
7090 set_lastFreeRangeCoalesced(false);
7091
7092 set_freeFinger(freeFinger);
7093 set_freeRangeInFreeLists(freeRangeInFreeLists);
7094 if (CMSTestInFreeList) {
7095 if (freeRangeInFreeLists) {
7096 FreeChunk* fc = (FreeChunk*) freeFinger;
7097 assert(fc->is_free(), "A chunk on the free list should be free.");
7098 assert(fc->size() > 0, "Free range should have a size");
7099 assert(_sp->verify_chunk_in_free_list(fc), "Chunk is not in free lists");
7100 }
7101 }
7102 }
7103
7104 // Note that the sweeper runs concurrently with mutators. Thus,
7105 // it is possible for direct allocation in this generation to happen
7106 // in the middle of the sweep. Note that the sweeper also coalesces
7107 // contiguous free blocks. Thus, unless the sweeper and the allocator
7133 size_t res;
7134
7135 // Check if we are done sweeping. Below we check "addr >= _limit" rather
7136 // than "addr == _limit" because although _limit was a block boundary when
7137 // we started the sweep, it may no longer be one because heap expansion
7138 // may have caused us to coalesce the block ending at the address _limit
7139 // with a newly expanded chunk (this happens when _limit was set to the
7140 // previous _end of the space), so we may have stepped past _limit:
7141 // see the following Zeno-like trail of CRs 6977970, 7008136, 7042740.
7142 if (addr >= _limit) { // we have swept up to or past the limit: finish up
7143 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7144 "sweep _limit out of bounds");
7145 assert(addr < _sp->end(), "addr out of bounds");
7146 // Flush any free range we might be holding as a single
7147 // coalesced chunk to the appropriate free list.
7148 if (inFreeRange()) {
7149 assert(freeFinger() >= _sp->bottom() && freeFinger() < _limit,
7150 "freeFinger() " PTR_FORMAT " is out-of-bounds", p2i(freeFinger()));
7151 flush_cur_free_chunk(freeFinger(),
7152 pointer_delta(addr, freeFinger()));
7153 log_develop_trace(gc, sweep)("Sweep: last chunk: put_free_blk " PTR_FORMAT " (" SIZE_FORMAT ") [coalesced:%d]",
7154 p2i(freeFinger()), pointer_delta(addr, freeFinger()),
7155 lastFreeRangeCoalesced() ? 1 : 0);
7156 }
7157
7158 // help the iterator loop finish
7159 return pointer_delta(_sp->end(), addr);
7160 }
7161
7162 assert(addr < _limit, "sweep invariant");
7163 // check if we should yield
7164 do_yield_check(addr);
7165 if (fc->is_free()) {
7166 // Chunk that is already free
7167 res = fc->size();
7168 do_already_free_chunk(fc);
7169 debug_only(_sp->verifyFreeLists());
7170 // If we flush the chunk at hand in lookahead_and_flush()
7171 // and it's coalesced with a preceding chunk, then the
7172 // process of "mangling" the payload of the coalesced block
7173 // will cause erasure of the size information from the
7174 // (erstwhile) header of all the coalesced blocks but the
7175 // first, so the first disjunct in the assert will not hold
7176 // in that specific case (in which case the second disjunct
7346 // Verify that the bit map has no bits marked between
7347 // addr and purported end of this block.
7348 size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
7349 assert(size >= 3, "Necessary for Printezis marks to work");
7350 assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
7351 DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
7352 }
7353 return size;
7354 }
7355
7356 void SweepClosure::do_post_free_or_garbage_chunk(FreeChunk* fc,
7357 size_t chunkSize) {
7358 // do_post_free_or_garbage_chunk() should only be called in the case
7359 // of the adaptive free list allocator.
7360 const bool fcInFreeLists = fc->is_free();
7361 assert((HeapWord*)fc <= _limit, "sweep invariant");
7362 if (CMSTestInFreeList && fcInFreeLists) {
7363 assert(_sp->verify_chunk_in_free_list(fc), "free chunk is not in free lists");
7364 }
7365
7366 log_develop_trace(gc, sweep)(" -- pick up another chunk at " PTR_FORMAT " (" SIZE_FORMAT ")", p2i(fc), chunkSize);
7367
7368 HeapWord* const fc_addr = (HeapWord*) fc;
7369
7370 bool coalesce = false;
7371 const size_t left = pointer_delta(fc_addr, freeFinger());
7372 const size_t right = chunkSize;
7373 switch (FLSCoalescePolicy) {
7374 // numeric value forms a coalition aggressiveness metric
7375 case 0: { // never coalesce
7376 coalesce = false;
7377 break;
7378 }
7379 case 1: { // coalesce if left & right chunks on overpopulated lists
7380 coalesce = _sp->coalOverPopulated(left) &&
7381 _sp->coalOverPopulated(right);
7382 break;
7383 }
7384 case 2: { // coalesce if left chunk on overpopulated list (default)
7385 coalesce = _sp->coalOverPopulated(left);
7386 break;
7447 // we'll look at because its end crosses past _limit, we'll preemptively
7448 // flush it along with any free range we may be holding on to. Note that
7449 // this can be the case only for an already free or freshly garbage
7450 // chunk. If this block is an object, it can never straddle
7451 // over _limit. The "straddling" occurs when _limit is set at
7452 // the previous end of the space when this cycle started, and
7453 // a subsequent heap expansion caused the previously co-terminal
7454 // free block to be coalesced with the newly expanded portion,
7455 // thus rendering _limit a non-block-boundary making it dangerous
7456 // for the sweeper to step over and examine.
7457 void SweepClosure::lookahead_and_flush(FreeChunk* fc, size_t chunk_size) {
7458 assert(inFreeRange(), "Should only be called if currently in a free range.");
7459 HeapWord* const eob = ((HeapWord*)fc) + chunk_size;
7460 assert(_sp->used_region().contains(eob - 1),
7461 "eob = " PTR_FORMAT " eob-1 = " PTR_FORMAT " _limit = " PTR_FORMAT
7462 " out of bounds wrt _sp = [" PTR_FORMAT "," PTR_FORMAT ")"
7463 " when examining fc = " PTR_FORMAT "(" SIZE_FORMAT ")",
7464 p2i(eob), p2i(eob-1), p2i(_limit), p2i(_sp->bottom()), p2i(_sp->end()), p2i(fc), chunk_size);
7465 if (eob >= _limit) {
7466 assert(eob == _limit || fc->is_free(), "Only a free chunk should allow us to cross over the limit");
7467 log_develop_trace(gc, sweep)("_limit " PTR_FORMAT " reached or crossed by block "
7468 "[" PTR_FORMAT "," PTR_FORMAT ") in space "
7469 "[" PTR_FORMAT "," PTR_FORMAT ")",
7470 p2i(_limit), p2i(fc), p2i(eob), p2i(_sp->bottom()), p2i(_sp->end()));
7471 // Return the storage we are tracking back into the free lists.
7472 log_develop_trace(gc, sweep)("Flushing ... ");
7473 assert(freeFinger() < eob, "Error");
7474 flush_cur_free_chunk( freeFinger(), pointer_delta(eob, freeFinger()));
7475 }
7476 }
7477
7478 void SweepClosure::flush_cur_free_chunk(HeapWord* chunk, size_t size) {
7479 assert(inFreeRange(), "Should only be called if currently in a free range.");
7480 assert(size > 0,
7481 "A zero sized chunk cannot be added to the free lists.");
7482 if (!freeRangeInFreeLists()) {
7483 if (CMSTestInFreeList) {
7484 FreeChunk* fc = (FreeChunk*) chunk;
7485 fc->set_size(size);
7486 assert(!_sp->verify_chunk_in_free_list(fc),
7487 "chunk should not be in free lists yet");
7488 }
7489 log_develop_trace(gc, sweep)(" -- add free block " PTR_FORMAT " (" SIZE_FORMAT ") to free lists", p2i(chunk), size);
7490 // A new free range is going to be starting. The current
7491 // free range has not been added to the free lists yet or
7492 // was removed so add it back.
7493 // If the current free range was coalesced, then the death
7494 // of the free range was recorded. Record a birth now.
7495 if (lastFreeRangeCoalesced()) {
7496 _sp->coalBirth(size);
7497 }
7498 _sp->addChunkAndRepairOffsetTable(chunk, size,
7499 lastFreeRangeCoalesced());
7500 } else {
7501 log_develop_trace(gc, sweep)("Already in free list: nothing to flush");
7502 }
7503 set_inFreeRange(false);
7504 set_freeRangeInFreeLists(false);
7505 }
7506
7507 // We take a break if we've been at this for a while,
7508 // so as to avoid monopolizing the locks involved.
7509 void SweepClosure::do_yield_work(HeapWord* addr) {
7510 // Return current free chunk being used for coalescing (if any)
7511 // to the appropriate freelist. After yielding, the next
7512 // free block encountered will start a coalescing range of
7513 // free blocks. If the next free block is adjacent to the
7514 // chunk just flushed, they will need to wait for the next
7515 // sweep to be coalesced.
7516 if (inFreeRange()) {
7517 flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger()));
7518 }
7519
7520 // First give up the locks, then yield, then re-lock.
7521 // We should probably use a constructor/destructor idiom to
7522 // do this unlock/lock or modify the MutexUnlocker class to
7523 // serve our purpose. XXX
7524 assert_lock_strong(_bitMap->lock());
7525 assert_lock_strong(_freelistLock);
7526 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7527 "CMS thread should hold CMS token");
7528 _bitMap->lock()->unlock();
7529 _freelistLock->unlock();
7530 ConcurrentMarkSweepThread::desynchronize(true);
7531 _collector->stopTimer();
7532 _collector->incrementYields();
7533
7534 // See the comment in coordinator_yield()
7535 for (unsigned i = 0; i < CMSYieldSleepCount &&
7536 ConcurrentMarkSweepThread::should_yield() &&
7537 !CMSCollector::foregroundGCIsActive(); ++i) {
7538 os::sleep(Thread::current(), 1, false);
7539 }
7540
7541 ConcurrentMarkSweepThread::synchronize(true);
7542 _freelistLock->lock();
7543 _bitMap->lock()->lock_without_safepoint_check();
7544 _collector->startTimer();
7545 }
7546
7547 #ifndef PRODUCT
7548 // This is actually very useful in a product build if it can
7549 // be called from the debugger. Compile it into the product
7550 // as needed.
7551 bool debug_verify_chunk_in_free_list(FreeChunk* fc) {
7552 return debug_cms_space->verify_chunk_in_free_list(fc);
7553 }
7554 #endif
7555
7556 void SweepClosure::print_free_block_coalesced(FreeChunk* fc) const {
7557 log_develop_trace(gc, sweep)("Sweep:coal_free_blk " PTR_FORMAT " (" SIZE_FORMAT ")",
7558 p2i(fc), fc->size());
7559 }
7560
7561 // CMSIsAliveClosure
7562 bool CMSIsAliveClosure::do_object_b(oop obj) {
7563 HeapWord* addr = (HeapWord*)obj;
7564 return addr != NULL &&
7565 (!_span.contains(addr) || _bit_map->isMarked(addr));
7566 }
7567
7568
7569 CMSKeepAliveClosure::CMSKeepAliveClosure( CMSCollector* collector,
7570 MemRegion span,
7571 CMSBitMap* bit_map, CMSMarkStack* mark_stack,
7572 bool cpc):
7573 _collector(collector),
7574 _span(span),
7575 _bit_map(bit_map),
7576 _mark_stack(mark_stack),
7577 _concurrent_precleaning(cpc) {
7578 assert(!_span.is_empty(), "Empty span could spell trouble");
|