36 #include "gc/cms/vmCMSOperations.hpp"
37 #include "gc/serial/genMarkSweep.hpp"
38 #include "gc/serial/tenuredGeneration.hpp"
39 #include "gc/shared/adaptiveSizePolicy.hpp"
40 #include "gc/shared/cardGeneration.inline.hpp"
41 #include "gc/shared/cardTableRS.hpp"
42 #include "gc/shared/collectedHeap.inline.hpp"
43 #include "gc/shared/collectorCounters.hpp"
44 #include "gc/shared/collectorPolicy.hpp"
45 #include "gc/shared/gcLocker.inline.hpp"
46 #include "gc/shared/gcPolicyCounters.hpp"
47 #include "gc/shared/gcTimer.hpp"
48 #include "gc/shared/gcTrace.hpp"
49 #include "gc/shared/gcTraceTime.hpp"
50 #include "gc/shared/genCollectedHeap.hpp"
51 #include "gc/shared/genOopClosures.inline.hpp"
52 #include "gc/shared/isGCActiveMark.hpp"
53 #include "gc/shared/referencePolicy.hpp"
54 #include "gc/shared/strongRootsScope.hpp"
55 #include "gc/shared/taskqueue.inline.hpp"
56 #include "memory/allocation.hpp"
57 #include "memory/iterator.inline.hpp"
58 #include "memory/padded.hpp"
59 #include "memory/resourceArea.hpp"
60 #include "oops/oop.inline.hpp"
61 #include "prims/jvmtiExport.hpp"
62 #include "runtime/atomic.inline.hpp"
63 #include "runtime/globals_extension.hpp"
64 #include "runtime/handles.inline.hpp"
65 #include "runtime/java.hpp"
66 #include "runtime/orderAccess.inline.hpp"
67 #include "runtime/vmThread.hpp"
68 #include "services/memoryService.hpp"
69 #include "services/runtimeService.hpp"
70 #include "utilities/stack.inline.hpp"
71
72 // statics
73 CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;
74 bool CMSCollector::_full_gc_requested = false;
75 GCCause::Cause CMSCollector::_full_gc_cause = GCCause::_no_gc;
349 // young generation collection.
350 double CMSStats::time_until_cms_gen_full() const {
351 size_t cms_free = _cms_gen->cmsSpace()->free();
352 GenCollectedHeap* gch = GenCollectedHeap::heap();
353 size_t expected_promotion = MIN2(gch->young_gen()->capacity(),
354 (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average());
355 if (cms_free > expected_promotion) {
356 // Start a cms collection if there isn't enough space to promote
357 // for the next young collection. Use the padded average as
358 // a safety factor.
359 cms_free -= expected_promotion;
360
361 // Adjust by the safety factor.
362 double cms_free_dbl = (double)cms_free;
363 double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor) / 100.0;
364 // Apply a further correction factor which tries to adjust
365 // for recent occurance of concurrent mode failures.
366 cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free);
367 cms_free_dbl = cms_free_dbl * cms_adjustment;
368
369 if (PrintGCDetails && Verbose) {
370 gclog_or_tty->print_cr("CMSStats::time_until_cms_gen_full: cms_free "
371 SIZE_FORMAT " expected_promotion " SIZE_FORMAT,
372 cms_free, expected_promotion);
373 gclog_or_tty->print_cr(" cms_free_dbl %f cms_consumption_rate %f",
374 cms_free_dbl, cms_consumption_rate() + 1.0);
375 }
376 // Add 1 in case the consumption rate goes to zero.
377 return cms_free_dbl / (cms_consumption_rate() + 1.0);
378 }
379 return 0.0;
380 }
381
382 // Compare the duration of the cms collection to the
383 // time remaining before the cms generation is empty.
384 // Note that the time from the start of the cms collection
385 // to the start of the cms sweep (less than the total
386 // duration of the cms collection) can be used. This
387 // has been tried and some applications experienced
388 // promotion failures early in execution. This was
389 // possibly because the averages were not accurate
390 // enough at the beginning.
391 double CMSStats::time_until_cms_start() const {
392 // We add "gc0_period" to the "work" calculation
393 // below because this query is done (mostly) at the
394 // end of a scavenge, so we need to conservatively
395 // account for that much possible delay
396 // in the query so as to avoid concurrent mode failures
397 // due to starting the collection just a wee bit too
398 // late.
399 double work = cms_duration() + gc0_period();
400 double deadline = time_until_cms_gen_full();
401 // If a concurrent mode failure occurred recently, we want to be
402 // more conservative and halve our expected time_until_cms_gen_full()
403 if (work > deadline) {
404 if (Verbose && PrintGCDetails) {
405 gclog_or_tty->print(
406 " CMSCollector: collect because of anticipated promotion "
407 "before full %3.7f + %3.7f > %3.7f ", cms_duration(),
408 gc0_period(), time_until_cms_gen_full());
409 }
410 return 0.0;
411 }
412 return work - deadline;
413 }
414
415 #ifndef PRODUCT
416 void CMSStats::print_on(outputStream *st) const {
417 st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);
418 st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,
419 gc0_duration(), gc0_period(), gc0_promoted());
420 st->print(",cms_dur=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,
421 cms_duration(), cms_period(), cms_allocated());
422 st->print(",cms_since_beg=%g,cms_since_end=%g",
423 cms_time_since_begin(), cms_time_since_end());
424 st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,
425 _cms_used_at_gc0_begin, _cms_used_at_gc0_end);
426
427 if (valid()) {
428 st->print(",promo_rate=%g,cms_alloc_rate=%g",
429 promotion_rate(), cms_allocation_rate());
651 //
652 void ConcurrentMarkSweepGeneration::update_counters(size_t used) {
653 if (UsePerfData) {
654 _space_counters->update_used(used);
655 _space_counters->update_capacity();
656 _gen_counters->update_all();
657 }
658 }
659
660 void ConcurrentMarkSweepGeneration::print() const {
661 Generation::print();
662 cmsSpace()->print();
663 }
664
665 #ifndef PRODUCT
666 void ConcurrentMarkSweepGeneration::print_statistics() {
667 cmsSpace()->printFLCensus(0);
668 }
669 #endif
670
671 void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) {
672 GenCollectedHeap* gch = GenCollectedHeap::heap();
673 if (PrintGCDetails) {
674 // I didn't want to change the logging when removing the level concept,
675 // but I guess this logging could say "old" or something instead of "1".
676 assert(gch->is_old_gen(this),
677 "The CMS generation should be the old generation");
678 uint level = 1;
679 if (Verbose) {
680 gclog_or_tty->print("[%u %s-%s: " SIZE_FORMAT "(" SIZE_FORMAT ")]",
681 level, short_name(), s, used(), capacity());
682 } else {
683 gclog_or_tty->print("[%u %s-%s: " SIZE_FORMAT "K(" SIZE_FORMAT "K)]",
684 level, short_name(), s, used() / K, capacity() / K);
685 }
686 }
687 if (Verbose) {
688 gclog_or_tty->print(" " SIZE_FORMAT "(" SIZE_FORMAT ")",
689 gch->used(), gch->capacity());
690 } else {
691 gclog_or_tty->print(" " SIZE_FORMAT "K(" SIZE_FORMAT "K)",
692 gch->used() / K, gch->capacity() / K);
693 }
694 }
695
696 size_t
697 ConcurrentMarkSweepGeneration::contiguous_available() const {
698 // dld proposes an improvement in precision here. If the committed
699 // part of the space ends in a free block we should add that to
700 // uncommitted size in the calculation below. Will make this
701 // change later, staying with the approximation below for the
702 // time being. -- ysr.
703 return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
704 }
705
706 size_t
707 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
708 return _cmsSpace->max_alloc_in_words() * HeapWordSize;
709 }
710
711 size_t ConcurrentMarkSweepGeneration::max_available() const {
712 return free() + _virtual_space.uncommitted_size();
713 }
714
715 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
716 size_t available = max_available();
717 size_t av_promo = (size_t)gc_stats()->avg_promoted()->padded_average();
718 bool res = (available >= av_promo) || (available >= max_promotion_in_bytes);
719 if (Verbose && PrintGCDetails) {
720 gclog_or_tty->print_cr(
721 "CMS: promo attempt is%s safe: available(" SIZE_FORMAT ") %s av_promo(" SIZE_FORMAT "),"
722 "max_promo(" SIZE_FORMAT ")",
723 res? "":" not", available, res? ">=":"<",
724 av_promo, max_promotion_in_bytes);
725 }
726 return res;
727 }
728
729 // At a promotion failure dump information on block layout in heap
730 // (cms old generation).
731 void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {
732 if (CMSDumpAtPromotionFailure) {
733 cmsSpace()->dump_at_safepoint_with_locks(collector(), gclog_or_tty);
734 }
735 }
736
737 void ConcurrentMarkSweepGeneration::reset_after_compaction() {
738 // Clear the promotion information. These pointers can be adjusted
739 // along with all the other pointers into the heap but
740 // compaction is expected to be a rare event with
741 // a heap using cms so don't do it without seeing the need.
742 for (uint i = 0; i < ParallelGCThreads; i++) {
743 _par_gc_thread_states[i]->promo.reset();
744 }
745 }
746
747 void ConcurrentMarkSweepGeneration::compute_new_size() {
748 assert_locked_or_safepoint(Heap_lock);
749
750 // If incremental collection failed, we just want to expand
751 // to the limit.
752 if (incremental_collection_failed()) {
753 clear_incremental_collection_failed();
769 void ConcurrentMarkSweepGeneration::compute_new_size_free_list() {
770 assert_locked_or_safepoint(Heap_lock);
771
772 // If incremental collection failed, we just want to expand
773 // to the limit.
774 if (incremental_collection_failed()) {
775 clear_incremental_collection_failed();
776 grow_to_reserved();
777 return;
778 }
779
780 double free_percentage = ((double) free()) / capacity();
781 double desired_free_percentage = (double) MinHeapFreeRatio / 100;
782 double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
783
784 // compute expansion delta needed for reaching desired free percentage
785 if (free_percentage < desired_free_percentage) {
786 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
787 assert(desired_capacity >= capacity(), "invalid expansion size");
788 size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
789 if (PrintGCDetails && Verbose) {
790 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
791 gclog_or_tty->print_cr("\nFrom compute_new_size: ");
792 gclog_or_tty->print_cr(" Free fraction %f", free_percentage);
793 gclog_or_tty->print_cr(" Desired free fraction %f", desired_free_percentage);
794 gclog_or_tty->print_cr(" Maximum free fraction %f", maximum_free_percentage);
795 gclog_or_tty->print_cr(" Capacity " SIZE_FORMAT, capacity() / 1000);
796 gclog_or_tty->print_cr(" Desired capacity " SIZE_FORMAT, desired_capacity / 1000);
797 GenCollectedHeap* gch = GenCollectedHeap::heap();
798 assert(gch->is_old_gen(this), "The CMS generation should always be the old generation");
799 size_t young_size = gch->young_gen()->capacity();
800 gclog_or_tty->print_cr(" Young gen size " SIZE_FORMAT, young_size / 1000);
801 gclog_or_tty->print_cr(" unsafe_max_alloc_nogc " SIZE_FORMAT, unsafe_max_alloc_nogc() / 1000);
802 gclog_or_tty->print_cr(" contiguous available " SIZE_FORMAT, contiguous_available() / 1000);
803 gclog_or_tty->print_cr(" Expand by " SIZE_FORMAT " (bytes)", expand_bytes);
804 }
805 // safe if expansion fails
806 expand_for_gc_cause(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
807 if (PrintGCDetails && Verbose) {
808 gclog_or_tty->print_cr(" Expanded free fraction %f", ((double) free()) / capacity());
809 }
810 } else {
811 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
812 assert(desired_capacity <= capacity(), "invalid expansion size");
813 size_t shrink_bytes = capacity() - desired_capacity;
814 // Don't shrink unless the delta is greater than the minimum shrink we want
815 if (shrink_bytes >= MinHeapDeltaBytes) {
816 shrink_free_list_by(shrink_bytes);
817 }
818 }
819 }
820
821 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
822 return cmsSpace()->freelistLock();
823 }
824
825 HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size, bool tlab) {
826 CMSSynchronousYieldRequest yr;
827 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
828 return have_lock_and_allocate(size, tlab);
829 }
1127 ConcurrentMarkSweepGeneration::
1128 par_oop_since_save_marks_iterate_done(int thread_num) {
1129 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1130 ParScanWithoutBarrierClosure* dummy_cl = NULL;
1131 ps->promo.promoted_oops_iterate_nv(dummy_cl);
1132 }
1133
1134 bool ConcurrentMarkSweepGeneration::should_collect(bool full,
1135 size_t size,
1136 bool tlab)
1137 {
1138 // We allow a STW collection only if a full
1139 // collection was requested.
1140 return full || should_allocate(size, tlab); // FIX ME !!!
1141 // This and promotion failure handling are connected at the
1142 // hip and should be fixed by untying them.
1143 }
1144
1145 bool CMSCollector::shouldConcurrentCollect() {
1146 if (_full_gc_requested) {
1147 if (Verbose && PrintGCDetails) {
1148 gclog_or_tty->print_cr("CMSCollector: collect because of explicit "
1149 " gc request (or gc_locker)");
1150 }
1151 return true;
1152 }
1153
1154 FreelistLocker x(this);
1155 // ------------------------------------------------------------------
1156 // Print out lots of information which affects the initiation of
1157 // a collection.
1158 if (PrintCMSInitiationStatistics && stats().valid()) {
1159 gclog_or_tty->print("CMSCollector shouldConcurrentCollect: ");
1160 gclog_or_tty->stamp();
1161 gclog_or_tty->cr();
1162 stats().print_on(gclog_or_tty);
1163 gclog_or_tty->print_cr("time_until_cms_gen_full %3.7f",
1164 stats().time_until_cms_gen_full());
1165 gclog_or_tty->print_cr("free=" SIZE_FORMAT, _cmsGen->free());
1166 gclog_or_tty->print_cr("contiguous_available=" SIZE_FORMAT,
1167 _cmsGen->contiguous_available());
1168 gclog_or_tty->print_cr("promotion_rate=%g", stats().promotion_rate());
1169 gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate());
1170 gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy());
1171 gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy());
1172 gclog_or_tty->print_cr("cms_time_since_begin=%3.7f", stats().cms_time_since_begin());
1173 gclog_or_tty->print_cr("cms_time_since_end=%3.7f", stats().cms_time_since_end());
1174 gclog_or_tty->print_cr("metadata initialized %d",
1175 MetaspaceGC::should_concurrent_collect());
1176 }
1177 // ------------------------------------------------------------------
1178
1179 // If the estimated time to complete a cms collection (cms_duration())
1180 // is less than the estimated time remaining until the cms generation
1181 // is full, start a collection.
1182 if (!UseCMSInitiatingOccupancyOnly) {
1183 if (stats().valid()) {
1184 if (stats().time_until_cms_start() == 0.0) {
1185 return true;
1186 }
1187 } else {
1188 // We want to conservatively collect somewhat early in order
1189 // to try and "bootstrap" our CMS/promotion statistics;
1190 // this branch will not fire after the first successful CMS
1191 // collection because the stats should then be valid.
1192 if (_cmsGen->occupancy() >= _bootstrap_occupancy) {
1193 if (Verbose && PrintGCDetails) {
1194 gclog_or_tty->print_cr(
1195 " CMSCollector: collect for bootstrapping statistics:"
1196 " occupancy = %f, boot occupancy = %f", _cmsGen->occupancy(),
1197 _bootstrap_occupancy);
1198 }
1199 return true;
1200 }
1201 }
1202 }
1203
1204 // Otherwise, we start a collection cycle if
1205 // old gen want a collection cycle started. Each may use
1206 // an appropriate criterion for making this decision.
1207 // XXX We need to make sure that the gen expansion
1208 // criterion dovetails well with this. XXX NEED TO FIX THIS
1209 if (_cmsGen->should_concurrent_collect()) {
1210 if (Verbose && PrintGCDetails) {
1211 gclog_or_tty->print_cr("CMS old gen initiated");
1212 }
1213 return true;
1214 }
1215
1216 // We start a collection if we believe an incremental collection may fail;
1217 // this is not likely to be productive in practice because it's probably too
1218 // late anyway.
1219 GenCollectedHeap* gch = GenCollectedHeap::heap();
1220 assert(gch->collector_policy()->is_generation_policy(),
1221 "You may want to check the correctness of the following");
1222 if (gch->incremental_collection_will_fail(true /* consult_young */)) {
1223 if (Verbose && PrintGCDetails) {
1224 gclog_or_tty->print("CMSCollector: collect because incremental collection will fail ");
1225 }
1226 return true;
1227 }
1228
1229 if (MetaspaceGC::should_concurrent_collect()) {
1230 if (Verbose && PrintGCDetails) {
1231 gclog_or_tty->print("CMSCollector: collect for metadata allocation ");
1232 }
1233 return true;
1234 }
1235
1236 // CMSTriggerInterval starts a CMS cycle if enough time has passed.
1237 if (CMSTriggerInterval >= 0) {
1238 if (CMSTriggerInterval == 0) {
1239 // Trigger always
1240 return true;
1241 }
1242
1243 // Check the CMS time since begin (we do not check the stats validity
1244 // as we want to be able to trigger the first CMS cycle as well)
1245 if (stats().cms_time_since_begin() >= (CMSTriggerInterval / ((double) MILLIUNITS))) {
1246 if (Verbose && PrintGCDetails) {
1247 if (stats().valid()) {
1248 gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (time since last begin %3.7f secs)",
1249 stats().cms_time_since_begin());
1250 } else {
1251 gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (first collection)");
1252 }
1253 }
1254 return true;
1255 }
1256 }
1257
1258 return false;
1259 }
1260
1261 void CMSCollector::set_did_compact(bool v) { _cmsGen->set_did_compact(v); }
1262
1263 // Clear _expansion_cause fields of constituent generations
1264 void CMSCollector::clear_expansion_cause() {
1265 _cmsGen->clear_expansion_cause();
1266 }
1267
1268 // We should be conservative in starting a collection cycle. To
1269 // start too eagerly runs the risk of collecting too often in the
1270 // extreme. To collect too rarely falls back on full collections,
1271 // which works, even if not optimum in terms of concurrent work.
1272 // As a work around for too eagerly collecting, use the flag
1275 // collections.
1276 // We want to start a new collection cycle if any of the following
1277 // conditions hold:
1278 // . our current occupancy exceeds the configured initiating occupancy
1279 // for this generation, or
1280 // . we recently needed to expand this space and have not, since that
1281 // expansion, done a collection of this generation, or
1282 // . the underlying space believes that it may be a good idea to initiate
1283 // a concurrent collection (this may be based on criteria such as the
1284 // following: the space uses linear allocation and linear allocation is
1285 // going to fail, or there is believed to be excessive fragmentation in
1286 // the generation, etc... or ...
1287 // [.(currently done by CMSCollector::shouldConcurrentCollect() only for
1288 // the case of the old generation; see CR 6543076):
1289 // we may be approaching a point at which allocation requests may fail because
1290 // we will be out of sufficient free space given allocation rate estimates.]
1291 bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const {
1292
1293 assert_lock_strong(freelistLock());
1294 if (occupancy() > initiating_occupancy()) {
1295 if (PrintGCDetails && Verbose) {
1296 gclog_or_tty->print(" %s: collect because of occupancy %f / %f ",
1297 short_name(), occupancy(), initiating_occupancy());
1298 }
1299 return true;
1300 }
1301 if (UseCMSInitiatingOccupancyOnly) {
1302 return false;
1303 }
1304 if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) {
1305 if (PrintGCDetails && Verbose) {
1306 gclog_or_tty->print(" %s: collect because expanded for allocation ",
1307 short_name());
1308 }
1309 return true;
1310 }
1311 return false;
1312 }
1313
1314 void ConcurrentMarkSweepGeneration::collect(bool full,
1315 bool clear_all_soft_refs,
1316 size_t size,
1317 bool tlab)
1318 {
1319 collector()->collect(full, clear_all_soft_refs, size, tlab);
1320 }
1321
1322 void CMSCollector::collect(bool full,
1323 bool clear_all_soft_refs,
1324 size_t size,
1325 bool tlab)
1326 {
1327 // The following "if" branch is present for defensive reasons.
1328 // In the current uses of this interface, it can be replaced with:
1345 GenCollectedHeap* gch = GenCollectedHeap::heap();
1346 unsigned int gc_count = gch->total_full_collections();
1347 if (gc_count == full_gc_count) {
1348 MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag);
1349 _full_gc_requested = true;
1350 _full_gc_cause = cause;
1351 CGC_lock->notify(); // nudge CMS thread
1352 } else {
1353 assert(gc_count > full_gc_count, "Error: causal loop");
1354 }
1355 }
1356
1357 bool CMSCollector::is_external_interruption() {
1358 GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause();
1359 return GCCause::is_user_requested_gc(cause) ||
1360 GCCause::is_serviceability_requested_gc(cause);
1361 }
1362
1363 void CMSCollector::report_concurrent_mode_interruption() {
1364 if (is_external_interruption()) {
1365 if (PrintGCDetails) {
1366 gclog_or_tty->print(" (concurrent mode interrupted)");
1367 }
1368 } else {
1369 if (PrintGCDetails) {
1370 gclog_or_tty->print(" (concurrent mode failure)");
1371 }
1372 _gc_tracer_cm->report_concurrent_mode_failure();
1373 }
1374 }
1375
1376
1377 // The foreground and background collectors need to coordinate in order
1378 // to make sure that they do not mutually interfere with CMS collections.
1379 // When a background collection is active,
1380 // the foreground collector may need to take over (preempt) and
1381 // synchronously complete an ongoing collection. Depending on the
1382 // frequency of the background collections and the heap usage
1383 // of the application, this preemption can be seldom or frequent.
1384 // There are only certain
1385 // points in the background collection that the "collection-baton"
1386 // can be passed to the foreground collector.
1387 //
1388 // The foreground collector will wait for the baton before
1389 // starting any part of the collection. The foreground collector
1390 // will only wait at one location.
1391 //
1485 CGC_lock->notify();
1486 assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(),
1487 "Possible deadlock");
1488 while (_foregroundGCShouldWait) {
1489 // wait for notification
1490 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1491 // Possibility of delay/starvation here, since CMS token does
1492 // not know to give priority to VM thread? Actually, i think
1493 // there wouldn't be any delay/starvation, but the proof of
1494 // that "fact" (?) appears non-trivial. XXX 20011219YSR
1495 }
1496 ConcurrentMarkSweepThread::set_CMS_flag(
1497 ConcurrentMarkSweepThread::CMS_vm_has_token);
1498 }
1499 }
1500 // The CMS_token is already held. Get back the other locks.
1501 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1502 "VM thread should have CMS token");
1503 getFreelistLocks();
1504 bitMapLock()->lock_without_safepoint_check();
1505 if (TraceCMSState) {
1506 gclog_or_tty->print_cr("CMS foreground collector has asked for control "
1507 INTPTR_FORMAT " with first state %d", p2i(Thread::current()), first_state);
1508 gclog_or_tty->print_cr(" gets control with state %d", _collectorState);
1509 }
1510
1511 // Inform cms gen if this was due to partial collection failing.
1512 // The CMS gen may use this fact to determine its expansion policy.
1513 GenCollectedHeap* gch = GenCollectedHeap::heap();
1514 if (gch->incremental_collection_will_fail(false /* don't consult_young */)) {
1515 assert(!_cmsGen->incremental_collection_failed(),
1516 "Should have been noticed, reacted to and cleared");
1517 _cmsGen->set_incremental_collection_failed();
1518 }
1519
1520 if (first_state > Idling) {
1521 report_concurrent_mode_interruption();
1522 }
1523
1524 set_did_compact(true);
1525
1526 // If the collection is being acquired from the background
1527 // collector, there may be references on the discovered
1528 // references lists. Abandon those references, since some
1529 // of them may have become unreachable after concurrent
1563 // after obtaining the free list locks for the
1564 // two generations.
1565 void CMSCollector::compute_new_size() {
1566 assert_locked_or_safepoint(Heap_lock);
1567 FreelistLocker z(this);
1568 MetaspaceGC::compute_new_size();
1569 _cmsGen->compute_new_size_free_list();
1570 }
1571
1572 // A work method used by the foreground collector to do
1573 // a mark-sweep-compact.
1574 void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
1575 GenCollectedHeap* gch = GenCollectedHeap::heap();
1576
1577 STWGCTimer* gc_timer = GenMarkSweep::gc_timer();
1578 gc_timer->register_gc_start();
1579
1580 SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer();
1581 gc_tracer->report_gc_start(gch->gc_cause(), gc_timer->gc_start());
1582
1583 GCTraceTime t("CMS:MSC ", PrintGCDetails && Verbose, true, NULL);
1584
1585 // Temporarily widen the span of the weak reference processing to
1586 // the entire heap.
1587 MemRegion new_span(GenCollectedHeap::heap()->reserved_region());
1588 ReferenceProcessorSpanMutator rp_mut_span(ref_processor(), new_span);
1589 // Temporarily, clear the "is_alive_non_header" field of the
1590 // reference processor.
1591 ReferenceProcessorIsAliveMutator rp_mut_closure(ref_processor(), NULL);
1592 // Temporarily make reference _processing_ single threaded (non-MT).
1593 ReferenceProcessorMTProcMutator rp_mut_mt_processing(ref_processor(), false);
1594 // Temporarily make refs discovery atomic
1595 ReferenceProcessorAtomicMutator rp_mut_atomic(ref_processor(), true);
1596 // Temporarily make reference _discovery_ single threaded (non-MT)
1597 ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false);
1598
1599 ref_processor()->set_enqueuing_is_done(false);
1600 ref_processor()->enable_discovery();
1601 ref_processor()->setup_policy(clear_all_soft_refs);
1602 // If an asynchronous collection finishes, the _modUnionTable is
1603 // all clear. If we are assuming the collection from an asynchronous
1648 // Clear any data recorded in the PLAB chunk arrays.
1649 if (_survivor_plab_array != NULL) {
1650 reset_survivor_plab_arrays();
1651 }
1652
1653 // Adjust the per-size allocation stats for the next epoch.
1654 _cmsGen->cmsSpace()->endSweepFLCensus(sweep_count() /* fake */);
1655 // Restart the "inter sweep timer" for the next epoch.
1656 _inter_sweep_timer.reset();
1657 _inter_sweep_timer.start();
1658
1659 gc_timer->register_gc_end();
1660
1661 gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions());
1662
1663 // For a mark-sweep-compact, compute_new_size() will be called
1664 // in the heap's do_collection() method.
1665 }
1666
1667 void CMSCollector::print_eden_and_survivor_chunk_arrays() {
1668 ContiguousSpace* eden_space = _young_gen->eden();
1669 ContiguousSpace* from_space = _young_gen->from();
1670 ContiguousSpace* to_space = _young_gen->to();
1671 // Eden
1672 if (_eden_chunk_array != NULL) {
1673 gclog_or_tty->print_cr("eden " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1674 p2i(eden_space->bottom()), p2i(eden_space->top()),
1675 p2i(eden_space->end()), eden_space->capacity());
1676 gclog_or_tty->print_cr("_eden_chunk_index=" SIZE_FORMAT ", "
1677 "_eden_chunk_capacity=" SIZE_FORMAT,
1678 _eden_chunk_index, _eden_chunk_capacity);
1679 for (size_t i = 0; i < _eden_chunk_index; i++) {
1680 gclog_or_tty->print_cr("_eden_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT,
1681 i, p2i(_eden_chunk_array[i]));
1682 }
1683 }
1684 // Survivor
1685 if (_survivor_chunk_array != NULL) {
1686 gclog_or_tty->print_cr("survivor " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1687 p2i(from_space->bottom()), p2i(from_space->top()),
1688 p2i(from_space->end()), from_space->capacity());
1689 gclog_or_tty->print_cr("_survivor_chunk_index=" SIZE_FORMAT ", "
1690 "_survivor_chunk_capacity=" SIZE_FORMAT,
1691 _survivor_chunk_index, _survivor_chunk_capacity);
1692 for (size_t i = 0; i < _survivor_chunk_index; i++) {
1693 gclog_or_tty->print_cr("_survivor_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT,
1694 i, p2i(_survivor_chunk_array[i]));
1695 }
1696 }
1697 }
1698
1699 void CMSCollector::getFreelistLocks() const {
1700 // Get locks for all free lists in all generations that this
1701 // collector is responsible for
1702 _cmsGen->freelistLock()->lock_without_safepoint_check();
1703 }
1704
1705 void CMSCollector::releaseFreelistLocks() const {
1706 // Release locks for all free lists in all generations that this
1707 // collector is responsible for
1708 _cmsGen->freelistLock()->unlock();
1709 }
1710
1711 bool CMSCollector::haveFreelistLocks() const {
1712 // Check locks for all free lists in all generations that this
1713 // collector is responsible for
1714 assert_lock_strong(_cmsGen->freelistLock());
1763 _collectorState = InitialMarking;
1764 register_gc_start(cause);
1765 // Reset the expansion cause, now that we are about to begin
1766 // a new cycle.
1767 clear_expansion_cause();
1768
1769 // Clear the MetaspaceGC flag since a concurrent collection
1770 // is starting but also clear it after the collection.
1771 MetaspaceGC::set_should_concurrent_collect(false);
1772 }
1773 // Decide if we want to enable class unloading as part of the
1774 // ensuing concurrent GC cycle.
1775 update_should_unload_classes();
1776 _full_gc_requested = false; // acks all outstanding full gc requests
1777 _full_gc_cause = GCCause::_no_gc;
1778 // Signal that we are about to start a collection
1779 gch->increment_total_full_collections(); // ... starting a collection cycle
1780 _collection_count_start = gch->total_full_collections();
1781 }
1782
1783 // Used for PrintGC
1784 size_t prev_used = 0;
1785 if (PrintGC && Verbose) {
1786 prev_used = _cmsGen->used();
1787 }
1788
1789 // The change of the collection state is normally done at this level;
1790 // the exceptions are phases that are executed while the world is
1791 // stopped. For those phases the change of state is done while the
1792 // world is stopped. For baton passing purposes this allows the
1793 // background collector to finish the phase and change state atomically.
1794 // The foreground collector cannot wait on a phase that is done
1795 // while the world is stopped because the foreground collector already
1796 // has the world stopped and would deadlock.
1797 while (_collectorState != Idling) {
1798 if (TraceCMSState) {
1799 gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
1800 p2i(Thread::current()), _collectorState);
1801 }
1802 // The foreground collector
1803 // holds the Heap_lock throughout its collection.
1804 // holds the CMS token (but not the lock)
1805 // except while it is waiting for the background collector to yield.
1806 //
1807 // The foreground collector should be blocked (not for long)
1808 // if the background collector is about to start a phase
1809 // executed with world stopped. If the background
1810 // collector has already started such a phase, the
1811 // foreground collector is blocked waiting for the
1812 // Heap_lock. The stop-world phases (InitialMarking and FinalMarking)
1813 // are executed in the VM thread.
1814 //
1815 // The locking order is
1816 // PendingListLock (PLL) -- if applicable (FinalMarking)
1817 // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue())
1818 // CMS token (claimed in
1819 // stop_world_and_do() -->
1820 // safepoint_synchronize() -->
1821 // CMSThread::synchronize())
1822
1823 {
1824 // Check if the FG collector wants us to yield.
1825 CMSTokenSync x(true); // is cms thread
1826 if (waitForForegroundGC()) {
1827 // We yielded to a foreground GC, nothing more to be
1828 // done this round.
1829 assert(_foregroundGCShouldWait == false, "We set it to false in "
1830 "waitForForegroundGC()");
1831 if (TraceCMSState) {
1832 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1833 " exiting collection CMS state %d",
1834 p2i(Thread::current()), _collectorState);
1835 }
1836 return;
1837 } else {
1838 // The background collector can run but check to see if the
1839 // foreground collector has done a collection while the
1840 // background collector was waiting to get the CGC_lock
1841 // above. If yes, break so that _foregroundGCShouldWait
1842 // is cleared before returning.
1843 if (_collectorState == Idling) {
1844 break;
1845 }
1846 }
1847 }
1848
1849 assert(_foregroundGCShouldWait, "Foreground collector, if active, "
1850 "should be waiting");
1851
1852 switch (_collectorState) {
1853 case InitialMarking:
1854 {
1855 ReleaseForegroundGC x(this);
1919 break;
1920 }
1921 case Resetting:
1922 // CMS heap resizing has been completed
1923 reset_concurrent();
1924 assert(_collectorState == Idling, "Collector state should "
1925 "have changed");
1926
1927 MetaspaceGC::set_should_concurrent_collect(false);
1928
1929 stats().record_cms_end();
1930 // Don't move the concurrent_phases_end() and compute_new_size()
1931 // calls to here because a preempted background collection
1932 // has it's state set to "Resetting".
1933 break;
1934 case Idling:
1935 default:
1936 ShouldNotReachHere();
1937 break;
1938 }
1939 if (TraceCMSState) {
1940 gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
1941 p2i(Thread::current()), _collectorState);
1942 }
1943 assert(_foregroundGCShouldWait, "block post-condition");
1944 }
1945
1946 // Should this be in gc_epilogue?
1947 collector_policy()->counters()->update_counters();
1948
1949 {
1950 // Clear _foregroundGCShouldWait and, in the event that the
1951 // foreground collector is waiting, notify it, before
1952 // returning.
1953 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1954 _foregroundGCShouldWait = false;
1955 if (_foregroundGCIsActive) {
1956 CGC_lock->notify();
1957 }
1958 assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1959 "Possible deadlock");
1960 }
1961 if (TraceCMSState) {
1962 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1963 " exiting collection CMS state %d",
1964 p2i(Thread::current()), _collectorState);
1965 }
1966 if (PrintGC && Verbose) {
1967 _cmsGen->print_heap_change(prev_used);
1968 }
1969 }
1970
1971 void CMSCollector::register_gc_start(GCCause::Cause cause) {
1972 _cms_start_registered = true;
1973 _gc_timer_cm->register_gc_start();
1974 _gc_tracer_cm->report_gc_start(cause, _gc_timer_cm->gc_start());
1975 }
1976
1977 void CMSCollector::register_gc_end() {
1978 if (_cms_start_registered) {
1979 report_heap_summary(GCWhen::AfterGC);
1980
1981 _gc_timer_cm->register_gc_end();
1982 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
1983 _cms_start_registered = false;
1984 }
1985 }
1986
1987 void CMSCollector::save_heap_summary() {
1988 GenCollectedHeap* gch = GenCollectedHeap::heap();
2000 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
2001 "CMS thread should have CMS token");
2002 // Block the foreground collector until the
2003 // background collectors decides whether to
2004 // yield.
2005 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2006 _foregroundGCShouldWait = true;
2007 if (_foregroundGCIsActive) {
2008 // The background collector yields to the
2009 // foreground collector and returns a value
2010 // indicating that it has yielded. The foreground
2011 // collector can proceed.
2012 res = true;
2013 _foregroundGCShouldWait = false;
2014 ConcurrentMarkSweepThread::clear_CMS_flag(
2015 ConcurrentMarkSweepThread::CMS_cms_has_token);
2016 ConcurrentMarkSweepThread::set_CMS_flag(
2017 ConcurrentMarkSweepThread::CMS_cms_wants_token);
2018 // Get a possibly blocked foreground thread going
2019 CGC_lock->notify();
2020 if (TraceCMSState) {
2021 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d",
2022 p2i(Thread::current()), _collectorState);
2023 }
2024 while (_foregroundGCIsActive) {
2025 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
2026 }
2027 ConcurrentMarkSweepThread::set_CMS_flag(
2028 ConcurrentMarkSweepThread::CMS_cms_has_token);
2029 ConcurrentMarkSweepThread::clear_CMS_flag(
2030 ConcurrentMarkSweepThread::CMS_cms_wants_token);
2031 }
2032 if (TraceCMSState) {
2033 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d",
2034 p2i(Thread::current()), _collectorState);
2035 }
2036 return res;
2037 }
2038
2039 // Because of the need to lock the free lists and other structures in
2040 // the collector, common to all the generations that the collector is
2041 // collecting, we need the gc_prologues of individual CMS generations
2042 // delegate to their collector. It may have been simpler had the
2043 // current infrastructure allowed one to call a prologue on a
2044 // collector. In the absence of that we have the generation's
2045 // prologue delegate to the collector, which delegates back
2046 // some "local" work to a worker method in the individual generations
2047 // that it's responsible for collecting, while itself doing any
2048 // work common to all generations it's responsible for. A similar
2049 // comment applies to the gc_epilogue()'s.
2050 // The role of the variable _between_prologue_and_epilogue is to
2051 // enforce the invocation protocol.
2052 void CMSCollector::gc_prologue(bool full) {
2053 // Call gc_prologue_work() for the CMSGen
2054 // we are responsible for.
2055
2112 // collecting.
2113 collector()->gc_prologue(full);
2114 }
2115
2116 // This is a "private" interface for use by this generation's CMSCollector.
2117 // Not to be called directly by any other entity (for instance,
2118 // GenCollectedHeap, which calls the "public" gc_prologue method above).
2119 void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
2120 bool registerClosure, ModUnionClosure* modUnionClosure) {
2121 assert(!incremental_collection_failed(), "Shouldn't be set yet");
2122 assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,
2123 "Should be NULL");
2124 if (registerClosure) {
2125 cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure);
2126 }
2127 cmsSpace()->gc_prologue();
2128 // Clear stat counters
2129 NOT_PRODUCT(
2130 assert(_numObjectsPromoted == 0, "check");
2131 assert(_numWordsPromoted == 0, "check");
2132 if (Verbose && PrintGC) {
2133 gclog_or_tty->print("Allocated " SIZE_FORMAT " objects, "
2134 SIZE_FORMAT " bytes concurrently",
2135 _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord));
2136 }
2137 _numObjectsAllocated = 0;
2138 _numWordsAllocated = 0;
2139 )
2140 }
2141
2142 void CMSCollector::gc_epilogue(bool full) {
2143 // The following locking discipline assumes that we are only called
2144 // when the world is stopped.
2145 assert(SafepointSynchronize::is_at_safepoint(),
2146 "world is stopped assumption");
2147
2148 // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks
2149 // if linear allocation blocks need to be appropriately marked to allow the
2150 // the blocks to be parsable. We also check here whether we need to nudge the
2151 // CMS collector thread to start a new cycle (if it's not already active).
2152 assert( Thread::current()->is_VM_thread()
2153 || ( CMSScavengeBeforeRemark
2154 && Thread::current()->is_ConcurrentGC_thread()),
2155 "Incorrect thread type for epilogue execution");
2156
2193 _between_prologue_and_epilogue = false; // ready for next cycle
2194 }
2195
2196 void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
2197 collector()->gc_epilogue(full);
2198
2199 // Also reset promotion tracking in par gc thread states.
2200 for (uint i = 0; i < ParallelGCThreads; i++) {
2201 _par_gc_thread_states[i]->promo.stopTrackingPromotions(i);
2202 }
2203 }
2204
2205 void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) {
2206 assert(!incremental_collection_failed(), "Should have been cleared");
2207 cmsSpace()->setPreconsumptionDirtyCardClosure(NULL);
2208 cmsSpace()->gc_epilogue();
2209 // Print stat counters
2210 NOT_PRODUCT(
2211 assert(_numObjectsAllocated == 0, "check");
2212 assert(_numWordsAllocated == 0, "check");
2213 if (Verbose && PrintGC) {
2214 gclog_or_tty->print("Promoted " SIZE_FORMAT " objects, "
2215 SIZE_FORMAT " bytes",
2216 _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord));
2217 }
2218 _numObjectsPromoted = 0;
2219 _numWordsPromoted = 0;
2220 )
2221
2222 if (PrintGC && Verbose) {
2223 // Call down the chain in contiguous_available needs the freelistLock
2224 // so print this out before releasing the freeListLock.
2225 gclog_or_tty->print(" Contiguous available " SIZE_FORMAT " bytes ",
2226 contiguous_available());
2227 }
2228 }
2229
2230 #ifndef PRODUCT
2231 bool CMSCollector::have_cms_token() {
2232 Thread* thr = Thread::current();
2233 if (thr->is_VM_thread()) {
2234 return ConcurrentMarkSweepThread::vm_thread_has_cms_token();
2235 } else if (thr->is_ConcurrentGC_thread()) {
2236 return ConcurrentMarkSweepThread::cms_thread_has_cms_token();
2237 } else if (thr->is_GC_task_thread()) {
2238 return ConcurrentMarkSweepThread::vm_thread_has_cms_token() &&
2239 ParGCRareEvent_lock->owned_by_self();
2240 }
2241 return false;
2242 }
2243 #endif
2244
2245 // Check reachability of the given heap address in CMS generation,
2246 // treating all other generations as roots.
2247 bool CMSCollector::is_cms_reachable(HeapWord* addr) {
2248 // We could "guarantee" below, rather than assert, but I'll
2249 // leave these as "asserts" so that an adventurous debugger
2250 // could try this in the product build provided some subset of
2251 // the conditions were met, provided they were interested in the
2252 // results and knew that the computation below wouldn't interfere
2253 // with other concurrent computations mutating the structures
2254 // being read or written.
2255 assert(SafepointSynchronize::is_at_safepoint(),
2256 "Else mutations in object graph will make answer suspect");
2257 assert(have_cms_token(), "Should hold cms token");
2258 assert(haveFreelistLocks(), "must hold free list locks");
2259 assert_lock_strong(bitMapLock());
2260
2261 // Clear the marking bit map array before starting, but, just
2262 // for kicks, first report if the given address is already marked
2263 gclog_or_tty->print_cr("Start: Address " PTR_FORMAT " is%s marked", p2i(addr),
2264 _markBitMap.isMarked(addr) ? "" : " not");
2265
2266 if (verify_after_remark()) {
2267 MutexLockerEx x(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2268 bool result = verification_mark_bm()->isMarked(addr);
2269 gclog_or_tty->print_cr("TransitiveMark: Address " PTR_FORMAT " %s marked", p2i(addr),
2270 result ? "IS" : "is NOT");
2271 return result;
2272 } else {
2273 gclog_or_tty->print_cr("Could not compute result");
2274 return false;
2275 }
2276 }
2277
2278
2279 void
2280 CMSCollector::print_on_error(outputStream* st) {
2281 CMSCollector* collector = ConcurrentMarkSweepGeneration::_collector;
2282 if (collector != NULL) {
2283 CMSBitMap* bitmap = &collector->_markBitMap;
2284 st->print_cr("Marking Bits: (CMSBitMap*) " PTR_FORMAT, p2i(bitmap));
2285 bitmap->print_on_error(st, " Bits: ");
2286
2287 st->cr();
2288
2289 CMSBitMap* mut_bitmap = &collector->_modUnionTable;
2290 st->print_cr("Mod Union Table: (CMSBitMap*) " PTR_FORMAT, p2i(mut_bitmap));
2291 mut_bitmap->print_on_error(st, " Bits: ");
2292 }
2293 }
2294
2295 ////////////////////////////////////////////////////////
2296 // CMS Verification Support
2297 ////////////////////////////////////////////////////////
2298 // Following the remark phase, the following invariant
2299 // should hold -- each object in the CMS heap which is
2300 // marked in markBitMap() should be marked in the verification_mark_bm().
2301
2302 class VerifyMarkedClosure: public BitMapClosure {
2303 CMSBitMap* _marks;
2304 bool _failed;
2305
2306 public:
2307 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2308
2309 bool do_bit(size_t offset) {
2310 HeapWord* addr = _marks->offsetToHeapWord(offset);
2311 if (!_marks->isMarked(addr)) {
2312 oop(addr)->print_on(gclog_or_tty);
2313 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
2314 _failed = true;
2315 }
2316 return true;
2317 }
2318
2319 bool failed() { return _failed; }
2320 };
2321
2322 bool CMSCollector::verify_after_remark(bool silent) {
2323 if (!silent) gclog_or_tty->print(" [Verifying CMS Marking... ");
2324 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2325 static bool init = false;
2326
2327 assert(SafepointSynchronize::is_at_safepoint(),
2328 "Else mutations in object graph will make answer suspect");
2329 assert(have_cms_token(),
2330 "Else there may be mutual interference in use of "
2331 " verification data structures");
2332 assert(_collectorState > Marking && _collectorState <= Sweeping,
2333 "Else marking info checked here may be obsolete");
2334 assert(haveFreelistLocks(), "must hold free list locks");
2335 assert_lock_strong(bitMapLock());
2336
2337
2338 // Allocate marking bit map if not already allocated
2339 if (!init) { // first time
2340 if (!verification_mark_bm()->allocate(_span)) {
2341 return false;
2342 }
2343 init = true;
2366 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2367 // Update the saved marks which may affect the root scans.
2368 gch->save_marks();
2369
2370 if (CMSRemarkVerifyVariant == 1) {
2371 // In this first variant of verification, we complete
2372 // all marking, then check if the new marks-vector is
2373 // a subset of the CMS marks-vector.
2374 verify_after_remark_work_1();
2375 } else if (CMSRemarkVerifyVariant == 2) {
2376 // In this second variant of verification, we flag an error
2377 // (i.e. an object reachable in the new marks-vector not reachable
2378 // in the CMS marks-vector) immediately, also indicating the
2379 // identify of an object (A) that references the unmarked object (B) --
2380 // presumably, a mutation to A failed to be picked up by preclean/remark?
2381 verify_after_remark_work_2();
2382 } else {
2383 warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant",
2384 CMSRemarkVerifyVariant);
2385 }
2386 if (!silent) gclog_or_tty->print(" done] ");
2387 return true;
2388 }
2389
2390 void CMSCollector::verify_after_remark_work_1() {
2391 ResourceMark rm;
2392 HandleMark hm;
2393 GenCollectedHeap* gch = GenCollectedHeap::heap();
2394
2395 // Get a clear set of claim bits for the roots processing to work with.
2396 ClassLoaderDataGraph::clear_claimed_marks();
2397
2398 // Mark from roots one level into CMS
2399 MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
2400 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2401
2402 {
2403 StrongRootsScope srs(1);
2404
2405 gch->gen_process_roots(&srs,
2406 GenCollectedHeap::OldGen,
2418 false /* don't yield */, true /* verifying */);
2419 assert(_restart_addr == NULL, "Expected pre-condition");
2420 verification_mark_bm()->iterate(&markFromRootsClosure);
2421 while (_restart_addr != NULL) {
2422 // Deal with stack overflow: by restarting at the indicated
2423 // address.
2424 HeapWord* ra = _restart_addr;
2425 markFromRootsClosure.reset(ra);
2426 _restart_addr = NULL;
2427 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2428 }
2429 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2430 verify_work_stacks_empty();
2431
2432 // Marking completed -- now verify that each bit marked in
2433 // verification_mark_bm() is also marked in markBitMap(); flag all
2434 // errors by printing corresponding objects.
2435 VerifyMarkedClosure vcl(markBitMap());
2436 verification_mark_bm()->iterate(&vcl);
2437 if (vcl.failed()) {
2438 gclog_or_tty->print("Verification failed");
2439 gch->print_on(gclog_or_tty);
2440 fatal("CMS: failed marking verification after remark");
2441 }
2442 }
2443
2444 class VerifyKlassOopsKlassClosure : public KlassClosure {
2445 class VerifyKlassOopsClosure : public OopClosure {
2446 CMSBitMap* _bitmap;
2447 public:
2448 VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { }
2449 void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); }
2450 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2451 } _oop_closure;
2452 public:
2453 VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {}
2454 void do_klass(Klass* k) {
2455 k->oops_do(&_oop_closure);
2456 }
2457 };
2458
2459 void CMSCollector::verify_after_remark_work_2() {
2712 expand_for_gc_cause(word_size*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_allocation);
2713 if (GCExpandToAllocateDelayMillis > 0) {
2714 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2715 }
2716 return have_lock_and_allocate(word_size, tlab);
2717 }
2718
2719 void ConcurrentMarkSweepGeneration::expand_for_gc_cause(
2720 size_t bytes,
2721 size_t expand_bytes,
2722 CMSExpansionCause::Cause cause)
2723 {
2724
2725 bool success = expand(bytes, expand_bytes);
2726
2727 // remember why we expanded; this information is used
2728 // by shouldConcurrentCollect() when making decisions on whether to start
2729 // a new CMS cycle.
2730 if (success) {
2731 set_expansion_cause(cause);
2732 if (PrintGCDetails && Verbose) {
2733 gclog_or_tty->print_cr("Expanded CMS gen for %s",
2734 CMSExpansionCause::to_string(cause));
2735 }
2736 }
2737 }
2738
2739 HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) {
2740 HeapWord* res = NULL;
2741 MutexLocker x(ParGCRareEvent_lock);
2742 while (true) {
2743 // Expansion by some other thread might make alloc OK now:
2744 res = ps->lab.alloc(word_sz);
2745 if (res != NULL) return res;
2746 // If there's not enough expansion space available, give up.
2747 if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) {
2748 return NULL;
2749 }
2750 // Otherwise, we try expansion.
2751 expand_for_gc_cause(word_sz*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_allocate_par_lab);
2752 // Now go around the loop and try alloc again;
2753 // A competing par_promote might beat us to the expansion space,
2754 // so we may go around the loop again if promotion fails again.
2755 if (GCExpandToAllocateDelayMillis > 0) {
2783 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2784 }
2785 }
2786 }
2787
2788 void ConcurrentMarkSweepGeneration::shrink(size_t bytes) {
2789 // Only shrink if a compaction was done so that all the free space
2790 // in the generation is in a contiguous block at the end.
2791 if (did_compact()) {
2792 CardGeneration::shrink(bytes);
2793 }
2794 }
2795
2796 void ConcurrentMarkSweepGeneration::assert_correct_size_change_locking() {
2797 assert_locked_or_safepoint(Heap_lock);
2798 }
2799
2800 void ConcurrentMarkSweepGeneration::shrink_free_list_by(size_t bytes) {
2801 assert_locked_or_safepoint(Heap_lock);
2802 assert_lock_strong(freelistLock());
2803 if (PrintGCDetails && Verbose) {
2804 warning("Shrinking of CMS not yet implemented");
2805 }
2806 return;
2807 }
2808
2809
2810 // Simple ctor/dtor wrapper for accounting & timer chores around concurrent
2811 // phases.
2812 class CMSPhaseAccounting: public StackObj {
2813 public:
2814 CMSPhaseAccounting(CMSCollector *collector,
2815 const char *phase,
2816 bool print_cr = true);
2817 ~CMSPhaseAccounting();
2818
2819 private:
2820 CMSCollector *_collector;
2821 const char *_phase;
2822 elapsedTimer _wallclock;
2823 bool _print_cr;
2824
2825 public:
2826 // Not MT-safe; so do not pass around these StackObj's
2827 // where they may be accessed by other threads.
2828 jlong wallclock_millis() {
2829 assert(_wallclock.is_active(), "Wall clock should not stop");
2830 _wallclock.stop(); // to record time
2831 jlong ret = _wallclock.milliseconds();
2832 _wallclock.start(); // restart
2833 return ret;
2834 }
2835 };
2836
2837 CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector,
2838 const char *phase,
2839 bool print_cr) :
2840 _collector(collector), _phase(phase), _print_cr(print_cr) {
2841
2842 if (PrintCMSStatistics != 0) {
2843 _collector->resetYields();
2844 }
2845 if (PrintGCDetails) {
2846 gclog_or_tty->gclog_stamp();
2847 gclog_or_tty->print_cr("[%s-concurrent-%s-start]",
2848 _collector->cmsGen()->short_name(), _phase);
2849 }
2850 _collector->resetTimer();
2851 _wallclock.start();
2852 _collector->startTimer();
2853 }
2854
2855 CMSPhaseAccounting::~CMSPhaseAccounting() {
2856 assert(_wallclock.is_active(), "Wall clock should not have stopped");
2857 _collector->stopTimer();
2858 _wallclock.stop();
2859 if (PrintGCDetails) {
2860 gclog_or_tty->gclog_stamp();
2861 gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]",
2862 _collector->cmsGen()->short_name(),
2863 _phase, _collector->timerValue(), _wallclock.seconds());
2864 if (_print_cr) {
2865 gclog_or_tty->cr();
2866 }
2867 if (PrintCMSStatistics != 0) {
2868 gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase,
2869 _collector->yields());
2870 }
2871 }
2872 }
2873
2874 // CMS work
2875
2876 // The common parts of CMSParInitialMarkTask and CMSParRemarkTask.
2877 class CMSParMarkTask : public AbstractGangTask {
2878 protected:
2879 CMSCollector* _collector;
2880 uint _n_workers;
2881 CMSParMarkTask(const char* name, CMSCollector* collector, uint n_workers) :
2882 AbstractGangTask(name),
2883 _collector(collector),
2884 _n_workers(n_workers) {}
2885 // Work method in support of parallel rescan ... of young gen spaces
2886 void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl,
2887 ContiguousSpace* space,
2888 HeapWord** chunk_array, size_t chunk_top);
2889 void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl);
2890 };
2891
2918 Mutex::_no_safepoint_check_flag);
2919 checkpointRootsInitialWork();
2920 // enable ("weak") refs discovery
2921 rp->enable_discovery();
2922 _collectorState = Marking;
2923 }
2924 }
2925
2926 void CMSCollector::checkpointRootsInitialWork() {
2927 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
2928 assert(_collectorState == InitialMarking, "just checking");
2929
2930 // Already have locks.
2931 assert_lock_strong(bitMapLock());
2932 assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
2933
2934 // Setup the verification and class unloading state for this
2935 // CMS collection cycle.
2936 setup_cms_unloading_and_verification_state();
2937
2938 NOT_PRODUCT(GCTraceTime t("\ncheckpointRootsInitialWork",
2939 PrintGCDetails && Verbose, true, _gc_timer_cm);)
2940
2941 // Reset all the PLAB chunk arrays if necessary.
2942 if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
2943 reset_survivor_plab_arrays();
2944 }
2945
2946 ResourceMark rm;
2947 HandleMark hm;
2948
2949 MarkRefsIntoClosure notOlder(_span, &_markBitMap);
2950 GenCollectedHeap* gch = GenCollectedHeap::heap();
2951
2952 verify_work_stacks_empty();
2953 verify_overflow_empty();
2954
2955 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2956 // Update the saved marks which may affect the root scans.
2957 gch->save_marks();
2958
2959 // weak reference processing has not started yet.
2960 ref_processor()->set_enqueuing_is_done(false);
2961
2962 // Need to remember all newly created CLDs,
2963 // so that we can guarantee that the remark finds them.
2964 ClassLoaderDataGraph::remember_new_clds(true);
2965
2966 // Whenever a CLD is found, it will be claimed before proceeding to mark
2967 // the klasses. The claimed marks need to be cleared before marking starts.
2968 ClassLoaderDataGraph::clear_claimed_marks();
2969
2970 if (CMSPrintEdenSurvivorChunks) {
2971 print_eden_and_survivor_chunk_arrays();
2972 }
2973
2974 {
2975 #if defined(COMPILER2) || INCLUDE_JVMCI
2976 DerivedPointerTableDeactivate dpt_deact;
2977 #endif
2978 if (CMSParallelInitialMarkEnabled) {
2979 // The parallel version.
2980 WorkGang* workers = gch->workers();
2981 assert(workers != NULL, "Need parallel worker threads.");
2982 uint n_workers = workers->active_workers();
2983
2984 StrongRootsScope srs(n_workers);
2985
2986 CMSParInitialMarkTask tsk(this, &srs, n_workers);
2987 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
2988 if (n_workers > 1) {
2989 workers->run_task(&tsk);
2990 } else {
2991 tsk.work(0);
2992 }
3023 save_sweep_limits();
3024 verify_overflow_empty();
3025 }
3026
3027 bool CMSCollector::markFromRoots() {
3028 // we might be tempted to assert that:
3029 // assert(!SafepointSynchronize::is_at_safepoint(),
3030 // "inconsistent argument?");
3031 // However that wouldn't be right, because it's possible that
3032 // a safepoint is indeed in progress as a young generation
3033 // stop-the-world GC happens even as we mark in this generation.
3034 assert(_collectorState == Marking, "inconsistent state?");
3035 check_correct_thread_executing();
3036 verify_overflow_empty();
3037
3038 // Weak ref discovery note: We may be discovering weak
3039 // refs in this generation concurrent (but interleaved) with
3040 // weak ref discovery by the young generation collector.
3041
3042 CMSTokenSyncWithLocks ts(true, bitMapLock());
3043 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3044 CMSPhaseAccounting pa(this, "mark", !PrintGCDetails);
3045 bool res = markFromRootsWork();
3046 if (res) {
3047 _collectorState = Precleaning;
3048 } else { // We failed and a foreground collection wants to take over
3049 assert(_foregroundGCIsActive, "internal state inconsistency");
3050 assert(_restart_addr == NULL, "foreground will restart from scratch");
3051 if (PrintGCDetails) {
3052 gclog_or_tty->print_cr("bailing out to foreground collection");
3053 }
3054 }
3055 verify_overflow_empty();
3056 return res;
3057 }
3058
3059 bool CMSCollector::markFromRootsWork() {
3060 // iterate over marked bits in bit map, doing a full scan and mark
3061 // from these roots using the following algorithm:
3062 // . if oop is to the right of the current scan pointer,
3063 // mark corresponding bit (we'll process it later)
3064 // . else (oop is to left of current scan pointer)
3065 // push oop on marking stack
3066 // . drain the marking stack
3067
3068 // Note that when we do a marking step we need to hold the
3069 // bit map lock -- recall that direct allocation (by mutators)
3070 // and promotion (by the young generation collector) is also
3071 // marking the bit map. [the so-called allocate live policy.]
3072 // Because the implementation of bit map marking is not
3073 // robust wrt simultaneous marking of bits in the same word,
3238 // and local work queue empty,
3239 // then in a loop do:
3240 // . check global overflow stack; steal a batch of oops and trace
3241 // . try to steal from other threads oif GOS is empty
3242 // . if neither is available, offer termination
3243 // -- Terminate and return result
3244 //
3245 void CMSConcMarkingTask::work(uint worker_id) {
3246 elapsedTimer _timer;
3247 ResourceMark rm;
3248 HandleMark hm;
3249
3250 DEBUG_ONLY(_collector->verify_overflow_empty();)
3251
3252 // Before we begin work, our work queue should be empty
3253 assert(work_queue(worker_id)->size() == 0, "Expected to be empty");
3254 // Scan the bitmap covering _cms_space, tracing through grey objects.
3255 _timer.start();
3256 do_scan_and_mark(worker_id, _cms_space);
3257 _timer.stop();
3258 if (PrintCMSStatistics != 0) {
3259 gclog_or_tty->print_cr("Finished cms space scanning in %dth thread: %3.3f sec",
3260 worker_id, _timer.seconds());
3261 // XXX: need xxx/xxx type of notation, two timers
3262 }
3263
3264 // ... do work stealing
3265 _timer.reset();
3266 _timer.start();
3267 do_work_steal(worker_id);
3268 _timer.stop();
3269 if (PrintCMSStatistics != 0) {
3270 gclog_or_tty->print_cr("Finished work stealing in %dth thread: %3.3f sec",
3271 worker_id, _timer.seconds());
3272 // XXX: need xxx/xxx type of notation, two timers
3273 }
3274 assert(_collector->_markStack.isEmpty(), "Should have been emptied");
3275 assert(work_queue(worker_id)->size() == 0, "Should have been emptied");
3276 // Note that under the current task protocol, the
3277 // following assertion is true even of the spaces
3278 // expanded since the completion of the concurrent
3279 // marking. XXX This will likely change under a strict
3280 // ABORT semantics.
3281 // After perm removal the comparison was changed to
3282 // greater than or equal to from strictly greater than.
3283 // Before perm removal the highest address sweep would
3284 // have been at the end of perm gen but now is at the
3285 // end of the tenured gen.
3286 assert(_global_finger >= _cms_space->end(),
3287 "All tasks have been completed");
3288 DEBUG_ONLY(_collector->verify_overflow_empty();)
3289 }
3290
3291 void CMSConcMarkingTask::bump_global_finger(HeapWord* f) {
3292 HeapWord* read = _global_finger;
3293 HeapWord* cur = read;
3468 // Check if oop points into the CMS generation
3469 // and is not marked
3470 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
3471 // a white object ...
3472 // If we manage to "claim" the object, by being the
3473 // first thread to mark it, then we push it on our
3474 // marking stack
3475 if (_bit_map->par_mark(addr)) { // ... now grey
3476 // push on work queue (grey set)
3477 bool simulate_overflow = false;
3478 NOT_PRODUCT(
3479 if (CMSMarkStackOverflowALot &&
3480 _collector->simulate_overflow()) {
3481 // simulate a stack overflow
3482 simulate_overflow = true;
3483 }
3484 )
3485 if (simulate_overflow ||
3486 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
3487 // stack overflow
3488 if (PrintCMSStatistics != 0) {
3489 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
3490 SIZE_FORMAT, _overflow_stack->capacity());
3491 }
3492 // We cannot assert that the overflow stack is full because
3493 // it may have been emptied since.
3494 assert(simulate_overflow ||
3495 _work_queue->size() == _work_queue->max_elems(),
3496 "Else push should have succeeded");
3497 handle_stack_overflow(addr);
3498 }
3499 } // Else, some other thread got there first
3500 do_yield_check();
3501 }
3502 }
3503
3504 void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
3505 void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
3506
3507 void Par_ConcMarkingClosure::trim_queue(size_t max) {
3508 while (_work_queue->size() > max) {
3509 oop new_oop;
3510 if (_work_queue->pop_local(new_oop)) {
3511 assert(new_oop->is_oop(), "Should be an oop");
3556 assert(work_q->size() == 0, "Impossible!");
3557 break;
3558 } else if (yielding() || should_yield()) {
3559 yield();
3560 }
3561 }
3562 }
3563
3564 // This is run by the CMS (coordinator) thread.
3565 void CMSConcMarkingTask::coordinator_yield() {
3566 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
3567 "CMS thread should hold CMS token");
3568 // First give up the locks, then yield, then re-lock
3569 // We should probably use a constructor/destructor idiom to
3570 // do this unlock/lock or modify the MutexUnlocker class to
3571 // serve our purpose. XXX
3572 assert_lock_strong(_bit_map_lock);
3573 _bit_map_lock->unlock();
3574 ConcurrentMarkSweepThread::desynchronize(true);
3575 _collector->stopTimer();
3576 if (PrintCMSStatistics != 0) {
3577 _collector->incrementYields();
3578 }
3579
3580 // It is possible for whichever thread initiated the yield request
3581 // not to get a chance to wake up and take the bitmap lock between
3582 // this thread releasing it and reacquiring it. So, while the
3583 // should_yield() flag is on, let's sleep for a bit to give the
3584 // other thread a chance to wake up. The limit imposed on the number
3585 // of iterations is defensive, to avoid any unforseen circumstances
3586 // putting us into an infinite loop. Since it's always been this
3587 // (coordinator_yield()) method that was observed to cause the
3588 // problem, we are using a parameter (CMSCoordinatorYieldSleepCount)
3589 // which is by default non-zero. For the other seven methods that
3590 // also perform the yield operation, as are using a different
3591 // parameter (CMSYieldSleepCount) which is by default zero. This way we
3592 // can enable the sleeping for those methods too, if necessary.
3593 // See 6442774.
3594 //
3595 // We really need to reconsider the synchronization between the GC
3596 // thread and the yield-requesting threads in the future and we
3597 // should really use wait/notify, which is the recommended
3598 // way of doing this type of interaction. Additionally, we should
3720 void CMSCollector::preclean() {
3721 check_correct_thread_executing();
3722 assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
3723 verify_work_stacks_empty();
3724 verify_overflow_empty();
3725 _abort_preclean = false;
3726 if (CMSPrecleaningEnabled) {
3727 if (!CMSEdenChunksRecordAlways) {
3728 _eden_chunk_index = 0;
3729 }
3730 size_t used = get_eden_used();
3731 size_t capacity = get_eden_capacity();
3732 // Don't start sampling unless we will get sufficiently
3733 // many samples.
3734 if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
3735 * CMSScheduleRemarkEdenPenetration)) {
3736 _start_sampling = true;
3737 } else {
3738 _start_sampling = false;
3739 }
3740 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3741 CMSPhaseAccounting pa(this, "preclean", !PrintGCDetails);
3742 preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
3743 }
3744 CMSTokenSync x(true); // is cms thread
3745 if (CMSPrecleaningEnabled) {
3746 sample_eden();
3747 _collectorState = AbortablePreclean;
3748 } else {
3749 _collectorState = FinalMarking;
3750 }
3751 verify_work_stacks_empty();
3752 verify_overflow_empty();
3753 }
3754
3755 // Try and schedule the remark such that young gen
3756 // occupancy is CMSScheduleRemarkEdenPenetration %.
3757 void CMSCollector::abortable_preclean() {
3758 check_correct_thread_executing();
3759 assert(CMSPrecleaningEnabled, "Inconsistent control state");
3760 assert(_collectorState == AbortablePreclean, "Inconsistent control state");
3761
3762 // If Eden's current occupancy is below this threshold,
3763 // immediately schedule the remark; else preclean
3764 // past the next scavenge in an effort to
3765 // schedule the pause as described above. By choosing
3766 // CMSScheduleRemarkEdenSizeThreshold >= max eden size
3767 // we will never do an actual abortable preclean cycle.
3768 if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) {
3769 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3770 CMSPhaseAccounting pa(this, "abortable-preclean", !PrintGCDetails);
3771 // We need more smarts in the abortable preclean
3772 // loop below to deal with cases where allocation
3773 // in young gen is very very slow, and our precleaning
3774 // is running a losing race against a horde of
3775 // mutators intent on flooding us with CMS updates
3776 // (dirty cards).
3777 // One, admittedly dumb, strategy is to give up
3778 // after a certain number of abortable precleaning loops
3779 // or after a certain maximum time. We want to make
3780 // this smarter in the next iteration.
3781 // XXX FIX ME!!! YSR
3782 size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0;
3783 while (!(should_abort_preclean() ||
3784 ConcurrentMarkSweepThread::should_terminate())) {
3785 workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2);
3786 cumworkdone += workdone;
3787 loops++;
3788 // Voluntarily terminate abortable preclean phase if we have
3789 // been at it for too long.
3790 if ((CMSMaxAbortablePrecleanLoops != 0) &&
3791 loops >= CMSMaxAbortablePrecleanLoops) {
3792 if (PrintGCDetails) {
3793 gclog_or_tty->print(" CMS: abort preclean due to loops ");
3794 }
3795 break;
3796 }
3797 if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) {
3798 if (PrintGCDetails) {
3799 gclog_or_tty->print(" CMS: abort preclean due to time ");
3800 }
3801 break;
3802 }
3803 // If we are doing little work each iteration, we should
3804 // take a short break.
3805 if (workdone < CMSAbortablePrecleanMinWorkPerIteration) {
3806 // Sleep for some time, waiting for work to accumulate
3807 stopTimer();
3808 cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis);
3809 startTimer();
3810 waited++;
3811 }
3812 }
3813 if (PrintCMSStatistics > 0) {
3814 gclog_or_tty->print(" [" SIZE_FORMAT " iterations, " SIZE_FORMAT " waits, " SIZE_FORMAT " cards)] ",
3815 loops, waited, cumworkdone);
3816 }
3817 }
3818 CMSTokenSync x(true); // is cms thread
3819 if (_collectorState != Idling) {
3820 assert(_collectorState == AbortablePreclean,
3821 "Spontaneous state transition?");
3822 _collectorState = FinalMarking;
3823 } // Else, a foreground collection completed this CMS cycle.
3824 return;
3825 }
3826
3827 // Respond to an Eden sampling opportunity
3828 void CMSCollector::sample_eden() {
3829 // Make sure a young gc cannot sneak in between our
3830 // reading and recording of a sample.
3831 assert(Thread::current()->is_ConcurrentGC_thread(),
3832 "Only the cms thread may collect Eden samples");
3833 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
3834 "Should collect samples while holding CMS token");
3835 if (!_start_sampling) {
3836 return;
3837 }
3940 // processes.
3941 ScanMarkedObjectsAgainCarefullyClosure
3942 smoac_cl(this, _span,
3943 &_markBitMap, &_markStack, &mrias_cl, CMSYield);
3944
3945 // Preclean dirty cards in ModUnionTable and CardTable using
3946 // appropriate convergence criterion;
3947 // repeat CMSPrecleanIter times unless we find that
3948 // we are losing.
3949 assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large");
3950 assert(CMSPrecleanNumerator < CMSPrecleanDenominator,
3951 "Bad convergence multiplier");
3952 assert(CMSPrecleanThreshold >= 100,
3953 "Unreasonably low CMSPrecleanThreshold");
3954
3955 size_t numIter, cumNumCards, lastNumCards, curNumCards;
3956 for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0;
3957 numIter < CMSPrecleanIter;
3958 numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) {
3959 curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl);
3960 if (Verbose && PrintGCDetails) {
3961 gclog_or_tty->print(" (modUnionTable: " SIZE_FORMAT " cards)", curNumCards);
3962 }
3963 // Either there are very few dirty cards, so re-mark
3964 // pause will be small anyway, or our pre-cleaning isn't
3965 // that much faster than the rate at which cards are being
3966 // dirtied, so we might as well stop and re-mark since
3967 // precleaning won't improve our re-mark time by much.
3968 if (curNumCards <= CMSPrecleanThreshold ||
3969 (numIter > 0 &&
3970 (curNumCards * CMSPrecleanDenominator >
3971 lastNumCards * CMSPrecleanNumerator))) {
3972 numIter++;
3973 cumNumCards += curNumCards;
3974 break;
3975 }
3976 }
3977
3978 preclean_klasses(&mrias_cl, _cmsGen->freelistLock());
3979
3980 curNumCards = preclean_card_table(_cmsGen, &smoac_cl);
3981 cumNumCards += curNumCards;
3982 if (PrintGCDetails && PrintCMSStatistics != 0) {
3983 gclog_or_tty->print_cr(" (cardTable: " SIZE_FORMAT " cards, re-scanned " SIZE_FORMAT " cards, " SIZE_FORMAT " iterations)",
3984 curNumCards, cumNumCards, numIter);
3985 }
3986 return cumNumCards; // as a measure of useful work done
3987 }
3988
3989 // PRECLEANING NOTES:
3990 // Precleaning involves:
3991 // . reading the bits of the modUnionTable and clearing the set bits.
3992 // . For the cards corresponding to the set bits, we scan the
3993 // objects on those cards. This means we need the free_list_lock
3994 // so that we can safely iterate over the CMS space when scanning
3995 // for oops.
3996 // . When we scan the objects, we'll be both reading and setting
3997 // marks in the marking bit map, so we'll need the marking bit map.
3998 // . For protecting _collector_state transitions, we take the CGC_lock.
3999 // Note that any races in the reading of of card table entries by the
4000 // CMS thread on the one hand and the clearing of those entries by the
4001 // VM thread or the setting of those entries by the mutator threads on the
4002 // other are quite benign. However, for efficiency it makes sense to keep
4003 // the VM thread from racing with the CMS thread while the latter is
4004 // dirty card info to the modUnionTable. We therefore also use the
4005 // CGC_lock to protect the reading of the card table and the mod union
4219 // SSS: Add equivalent to ScanMarkedObjectsAgainCarefullyClosure::do_yield_check and should_abort_preclean?
4220 // SSS: We should probably check if precleaning should be aborted, at suitable intervals?
4221 PrecleanKlassClosure preclean_klass_closure(cl);
4222 ClassLoaderDataGraph::classes_do(&preclean_klass_closure);
4223
4224 verify_work_stacks_empty();
4225 verify_overflow_empty();
4226 }
4227
4228 void CMSCollector::checkpointRootsFinal() {
4229 assert(_collectorState == FinalMarking, "incorrect state transition?");
4230 check_correct_thread_executing();
4231 // world is stopped at this checkpoint
4232 assert(SafepointSynchronize::is_at_safepoint(),
4233 "world should be stopped");
4234 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
4235
4236 verify_work_stacks_empty();
4237 verify_overflow_empty();
4238
4239 if (PrintGCDetails) {
4240 gclog_or_tty->print("[YG occupancy: " SIZE_FORMAT " K (" SIZE_FORMAT " K)]",
4241 _young_gen->used() / K,
4242 _young_gen->capacity() / K);
4243 }
4244 {
4245 if (CMSScavengeBeforeRemark) {
4246 GenCollectedHeap* gch = GenCollectedHeap::heap();
4247 // Temporarily set flag to false, GCH->do_collection will
4248 // expect it to be false and set to true
4249 FlagSetting fl(gch->_is_gc_active, false);
4250 NOT_PRODUCT(GCTraceTime t("Scavenge-Before-Remark",
4251 PrintGCDetails && Verbose, true, _gc_timer_cm);)
4252 gch->do_collection(true, // full (i.e. force, see below)
4253 false, // !clear_all_soft_refs
4254 0, // size
4255 false, // is_tlab
4256 GenCollectedHeap::YoungGen // type
4257 );
4258 }
4259 FreelistLocker x(this);
4260 MutexLockerEx y(bitMapLock(),
4261 Mutex::_no_safepoint_check_flag);
4262 checkpointRootsFinalWork();
4263 }
4264 verify_work_stacks_empty();
4265 verify_overflow_empty();
4266 }
4267
4268 void CMSCollector::checkpointRootsFinalWork() {
4269 NOT_PRODUCT(GCTraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, _gc_timer_cm);)
4270
4271 assert(haveFreelistLocks(), "must have free list locks");
4272 assert_lock_strong(bitMapLock());
4273
4274 ResourceMark rm;
4275 HandleMark hm;
4276
4277 GenCollectedHeap* gch = GenCollectedHeap::heap();
4278
4279 if (should_unload_classes()) {
4280 CodeCache::gc_prologue();
4281 }
4282 assert(haveFreelistLocks(), "must have free list locks");
4283 assert_lock_strong(bitMapLock());
4284
4285 // We might assume that we need not fill TLAB's when
4286 // CMSScavengeBeforeRemark is set, because we may have just done
4287 // a scavenge which would have filled all TLAB's -- and besides
4288 // Eden would be empty. This however may not always be the case --
4289 // for instance although we asked for a scavenge, it may not have
4290 // happened because of a JNI critical section. We probably need
4291 // a policy for deciding whether we can in that case wait until
4292 // the critical section releases and then do the remark following
4293 // the scavenge, and skip it here. In the absence of that policy,
4294 // or of an indication of whether the scavenge did indeed occur,
4295 // we cannot rely on TLAB's having been filled and must do
4296 // so here just in case a scavenge did not happen.
4297 gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
4298 // Update the saved marks which may affect the root scans.
4299 gch->save_marks();
4300
4301 if (CMSPrintEdenSurvivorChunks) {
4302 print_eden_and_survivor_chunk_arrays();
4303 }
4304
4305 {
4306 #if defined(COMPILER2) || INCLUDE_JVMCI
4307 DerivedPointerTableDeactivate dpt_deact;
4308 #endif
4309
4310 // Note on the role of the mod union table:
4311 // Since the marker in "markFromRoots" marks concurrently with
4312 // mutators, it is possible for some reachable objects not to have been
4313 // scanned. For instance, an only reference to an object A was
4314 // placed in object B after the marker scanned B. Unless B is rescanned,
4315 // A would be collected. Such updates to references in marked objects
4316 // are detected via the mod union table which is the set of all cards
4317 // dirtied since the first checkpoint in this GC cycle and prior to
4318 // the most recent young generation GC, minus those cleaned up by the
4319 // concurrent precleaning.
4320 if (CMSParallelRemarkEnabled) {
4321 GCTraceTime t("Rescan (parallel) ", PrintGCDetails, false, _gc_timer_cm);
4322 do_remark_parallel();
4323 } else {
4324 GCTraceTime t("Rescan (non-parallel) ", PrintGCDetails, false, _gc_timer_cm);
4325 do_remark_non_parallel();
4326 }
4327 }
4328 verify_work_stacks_empty();
4329 verify_overflow_empty();
4330
4331 {
4332 NOT_PRODUCT(GCTraceTime ts("refProcessingWork", PrintGCDetails, false, _gc_timer_cm);)
4333 refProcessingWork();
4334 }
4335 verify_work_stacks_empty();
4336 verify_overflow_empty();
4337
4338 if (should_unload_classes()) {
4339 CodeCache::gc_epilogue();
4340 }
4341 JvmtiExport::gc_epilogue();
4342
4343 // If we encountered any (marking stack / work queue) overflow
4344 // events during the current CMS cycle, take appropriate
4345 // remedial measures, where possible, so as to try and avoid
4346 // recurrence of that condition.
4347 assert(_markStack.isEmpty(), "No grey objects");
4348 size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
4349 _ser_kac_ovflw + _ser_kac_preclean_ovflw;
4350 if (ser_ovflw > 0) {
4351 if (PrintCMSStatistics != 0) {
4352 gclog_or_tty->print_cr("Marking stack overflow (benign) "
4353 "(pmc_pc=" SIZE_FORMAT ", pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT
4354 ", kac_preclean=" SIZE_FORMAT ")",
4355 _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw,
4356 _ser_kac_ovflw, _ser_kac_preclean_ovflw);
4357 }
4358 _markStack.expand();
4359 _ser_pmc_remark_ovflw = 0;
4360 _ser_pmc_preclean_ovflw = 0;
4361 _ser_kac_preclean_ovflw = 0;
4362 _ser_kac_ovflw = 0;
4363 }
4364 if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
4365 if (PrintCMSStatistics != 0) {
4366 gclog_or_tty->print_cr("Work queue overflow (benign) "
4367 "(pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT ")",
4368 _par_pmc_remark_ovflw, _par_kac_ovflw);
4369 }
4370 _par_pmc_remark_ovflw = 0;
4371 _par_kac_ovflw = 0;
4372 }
4373 if (PrintCMSStatistics != 0) {
4374 if (_markStack._hit_limit > 0) {
4375 gclog_or_tty->print_cr(" (benign) Hit max stack size limit (" SIZE_FORMAT ")",
4376 _markStack._hit_limit);
4377 }
4378 if (_markStack._failed_double > 0) {
4379 gclog_or_tty->print_cr(" (benign) Failed stack doubling (" SIZE_FORMAT "),"
4380 " current capacity " SIZE_FORMAT,
4381 _markStack._failed_double,
4382 _markStack.capacity());
4383 }
4384 }
4385 _markStack._hit_limit = 0;
4386 _markStack._failed_double = 0;
4387
4388 if ((VerifyAfterGC || VerifyDuringGC) &&
4389 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4390 verify_after_remark();
4391 }
4392
4393 _gc_tracer_cm->report_object_count_after_gc(&_is_alive_closure);
4394
4395 // Change under the freelistLocks.
4396 _collectorState = Sweeping;
4397 // Call isAllClear() under bitMapLock
4398 assert(_modUnionTable.isAllClear(),
4399 "Should be clear by end of the final marking");
4400 assert(_ct->klass_rem_set()->mod_union_is_clear(),
4401 "Should be clear by end of the final marking");
4402 }
4403
4404 void CMSParInitialMarkTask::work(uint worker_id) {
4405 elapsedTimer _timer;
4406 ResourceMark rm;
4407 HandleMark hm;
4408
4409 // ---------- scan from roots --------------
4410 _timer.start();
4411 GenCollectedHeap* gch = GenCollectedHeap::heap();
4412 Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap));
4413
4414 // ---------- young gen roots --------------
4415 {
4416 work_on_young_gen_roots(worker_id, &par_mri_cl);
4417 _timer.stop();
4418 if (PrintCMSStatistics != 0) {
4419 gclog_or_tty->print_cr(
4420 "Finished young gen initial mark scan work in %dth thread: %3.3f sec",
4421 worker_id, _timer.seconds());
4422 }
4423 }
4424
4425 // ---------- remaining roots --------------
4426 _timer.reset();
4427 _timer.start();
4428
4429 CLDToOopClosure cld_closure(&par_mri_cl, true);
4430
4431 gch->gen_process_roots(_strong_roots_scope,
4432 GenCollectedHeap::OldGen,
4433 false, // yg was scanned above
4434 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4435 _collector->should_unload_classes(),
4436 &par_mri_cl,
4437 NULL,
4438 &cld_closure);
4439 assert(_collector->should_unload_classes()
4440 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4441 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4442 _timer.stop();
4443 if (PrintCMSStatistics != 0) {
4444 gclog_or_tty->print_cr(
4445 "Finished remaining root initial mark scan work in %dth thread: %3.3f sec",
4446 worker_id, _timer.seconds());
4447 }
4448 }
4449
4450 // Parallel remark task
4451 class CMSParRemarkTask: public CMSParMarkTask {
4452 CompactibleFreeListSpace* _cms_space;
4453
4454 // The per-thread work queues, available here for stealing.
4455 OopTaskQueueSet* _task_queues;
4456 ParallelTaskTerminator _term;
4457 StrongRootsScope* _strong_roots_scope;
4458
4459 public:
4460 // A value of 0 passed to n_workers will cause the number of
4461 // workers to be taken from the active workers in the work gang.
4462 CMSParRemarkTask(CMSCollector* collector,
4463 CompactibleFreeListSpace* cms_space,
4464 uint n_workers, WorkGang* workers,
4465 OopTaskQueueSet* task_queues,
4466 StrongRootsScope* strong_roots_scope):
4467 CMSParMarkTask("Rescan roots and grey objects in parallel",
4540 elapsedTimer _timer;
4541 ResourceMark rm;
4542 HandleMark hm;
4543
4544 // ---------- rescan from roots --------------
4545 _timer.start();
4546 GenCollectedHeap* gch = GenCollectedHeap::heap();
4547 Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
4548 _collector->_span, _collector->ref_processor(),
4549 &(_collector->_markBitMap),
4550 work_queue(worker_id));
4551
4552 // Rescan young gen roots first since these are likely
4553 // coarsely partitioned and may, on that account, constitute
4554 // the critical path; thus, it's best to start off that
4555 // work first.
4556 // ---------- young gen roots --------------
4557 {
4558 work_on_young_gen_roots(worker_id, &par_mrias_cl);
4559 _timer.stop();
4560 if (PrintCMSStatistics != 0) {
4561 gclog_or_tty->print_cr(
4562 "Finished young gen rescan work in %dth thread: %3.3f sec",
4563 worker_id, _timer.seconds());
4564 }
4565 }
4566
4567 // ---------- remaining roots --------------
4568 _timer.reset();
4569 _timer.start();
4570 gch->gen_process_roots(_strong_roots_scope,
4571 GenCollectedHeap::OldGen,
4572 false, // yg was scanned above
4573 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4574 _collector->should_unload_classes(),
4575 &par_mrias_cl,
4576 NULL,
4577 NULL); // The dirty klasses will be handled below
4578
4579 assert(_collector->should_unload_classes()
4580 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4581 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4582 _timer.stop();
4583 if (PrintCMSStatistics != 0) {
4584 gclog_or_tty->print_cr(
4585 "Finished remaining root rescan work in %dth thread: %3.3f sec",
4586 worker_id, _timer.seconds());
4587 }
4588
4589 // ---------- unhandled CLD scanning ----------
4590 if (worker_id == 0) { // Single threaded at the moment.
4591 _timer.reset();
4592 _timer.start();
4593
4594 // Scan all new class loader data objects and new dependencies that were
4595 // introduced during concurrent marking.
4596 ResourceMark rm;
4597 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
4598 for (int i = 0; i < array->length(); i++) {
4599 par_mrias_cl.do_cld_nv(array->at(i));
4600 }
4601
4602 // We don't need to keep track of new CLDs anymore.
4603 ClassLoaderDataGraph::remember_new_clds(false);
4604
4605 _timer.stop();
4606 if (PrintCMSStatistics != 0) {
4607 gclog_or_tty->print_cr(
4608 "Finished unhandled CLD scanning work in %dth thread: %3.3f sec",
4609 worker_id, _timer.seconds());
4610 }
4611 }
4612
4613 // ---------- dirty klass scanning ----------
4614 if (worker_id == 0) { // Single threaded at the moment.
4615 _timer.reset();
4616 _timer.start();
4617
4618 // Scan all classes that was dirtied during the concurrent marking phase.
4619 RemarkKlassClosure remark_klass_closure(&par_mrias_cl);
4620 ClassLoaderDataGraph::classes_do(&remark_klass_closure);
4621
4622 _timer.stop();
4623 if (PrintCMSStatistics != 0) {
4624 gclog_or_tty->print_cr(
4625 "Finished dirty klass scanning work in %dth thread: %3.3f sec",
4626 worker_id, _timer.seconds());
4627 }
4628 }
4629
4630 // We might have added oops to ClassLoaderData::_handles during the
4631 // concurrent marking phase. These oops point to newly allocated objects
4632 // that are guaranteed to be kept alive. Either by the direct allocation
4633 // code, or when the young collector processes the roots. Hence,
4634 // we don't have to revisit the _handles block during the remark phase.
4635
4636 // ---------- rescan dirty cards ------------
4637 _timer.reset();
4638 _timer.start();
4639
4640 // Do the rescan tasks for each of the two spaces
4641 // (cms_space) in turn.
4642 // "worker_id" is passed to select the task_queue for "worker_id"
4643 do_dirty_card_rescan_tasks(_cms_space, worker_id, &par_mrias_cl);
4644 _timer.stop();
4645 if (PrintCMSStatistics != 0) {
4646 gclog_or_tty->print_cr(
4647 "Finished dirty card rescan work in %dth thread: %3.3f sec",
4648 worker_id, _timer.seconds());
4649 }
4650
4651 // ---------- steal work from other threads ...
4652 // ---------- ... and drain overflow list.
4653 _timer.reset();
4654 _timer.start();
4655 do_work_steal(worker_id, &par_mrias_cl, _collector->hash_seed(worker_id));
4656 _timer.stop();
4657 if (PrintCMSStatistics != 0) {
4658 gclog_or_tty->print_cr(
4659 "Finished work stealing in %dth thread: %3.3f sec",
4660 worker_id, _timer.seconds());
4661 }
4662 }
4663
4664 // Note that parameter "i" is not used.
4665 void
4666 CMSParMarkTask::do_young_space_rescan(uint worker_id,
4667 OopsInGenClosure* cl, ContiguousSpace* space,
4668 HeapWord** chunk_array, size_t chunk_top) {
4669 // Until all tasks completed:
4670 // . claim an unclaimed task
4671 // . compute region boundaries corresponding to task claimed
4672 // using chunk_array
4673 // . par_oop_iterate(cl) over that region
4674
4675 ResourceMark rm;
4676 HandleMark hm;
4677
4678 SequentialSubTasksDone* pst = space->par_seq_tasks();
4679
4680 uint nth_task = 0;
4681 uint n_tasks = pst->n_tasks();
4835 // because we just took work from the overflow list,
4836 // but of course we can't since all of that could have
4837 // been already stolen from us.
4838 // "He giveth and He taketh away."
4839 continue;
4840 }
4841 // Verify that we have no work before we resort to stealing
4842 assert(work_q->size() == 0, "Have work, shouldn't steal");
4843 // Try to steal from other queues that have work
4844 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
4845 NOT_PRODUCT(num_steals++;)
4846 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
4847 assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
4848 // Do scanning work
4849 obj_to_scan->oop_iterate(cl);
4850 // Loop around, finish this work, and try to steal some more
4851 } else if (terminator()->offer_termination()) {
4852 break; // nirvana from the infinite cycle
4853 }
4854 }
4855 NOT_PRODUCT(
4856 if (PrintCMSStatistics != 0) {
4857 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
4858 }
4859 )
4860 assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
4861 "Else our work is not yet done");
4862 }
4863
4864 // Record object boundaries in _eden_chunk_array by sampling the eden
4865 // top in the slow-path eden object allocation code path and record
4866 // the boundaries, if CMSEdenChunksRecordAlways is true. If
4867 // CMSEdenChunksRecordAlways is false, we use the other asynchronous
4868 // sampling in sample_eden() that activates during the part of the
4869 // preclean phase.
4870 void CMSCollector::sample_eden_chunk() {
4871 if (CMSEdenChunksRecordAlways && _eden_chunk_array != NULL) {
4872 if (_eden_chunk_lock->try_lock()) {
4873 // Record a sample. This is the critical section. The contents
4874 // of the _eden_chunk_array have to be non-decreasing in the
4875 // address order.
4876 _eden_chunk_array[_eden_chunk_index] = *_top_addr;
4877 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
4878 "Unexpected state of Eden");
4879 if (_eden_chunk_index == 0 ||
4936 if (cur_val < min_val) {
4937 min_tid = j;
4938 min_val = cur_val;
4939 } else {
4940 assert(cur_val < top, "All recorded addresses should be less");
4941 }
4942 }
4943 // At this point min_val and min_tid are respectively
4944 // the least address in _survivor_plab_array[j]->nth(_cursor[j])
4945 // and the thread (j) that witnesses that address.
4946 // We record this address in the _survivor_chunk_array[i]
4947 // and increment _cursor[min_tid] prior to the next round i.
4948 if (min_val == top) {
4949 break;
4950 }
4951 _survivor_chunk_array[i] = min_val;
4952 _cursor[min_tid]++;
4953 }
4954 // We are all done; record the size of the _survivor_chunk_array
4955 _survivor_chunk_index = i; // exclusive: [0, i)
4956 if (PrintCMSStatistics > 0) {
4957 gclog_or_tty->print(" (Survivor:" SIZE_FORMAT "chunks) ", i);
4958 }
4959 // Verify that we used up all the recorded entries
4960 #ifdef ASSERT
4961 size_t total = 0;
4962 for (int j = 0; j < no_of_gc_threads; j++) {
4963 assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
4964 total += _cursor[j];
4965 }
4966 assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
4967 // Check that the merged array is in sorted order
4968 if (total > 0) {
4969 for (size_t i = 0; i < total - 1; i++) {
4970 if (PrintCMSStatistics > 0) {
4971 gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
4972 i, p2i(_survivor_chunk_array[i]));
4973 }
4974 assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
4975 "Not sorted");
4976 }
4977 }
4978 #endif // ASSERT
4979 }
4980
4981 // Set up the space's par_seq_tasks structure for work claiming
4982 // for parallel initial scan and rescan of young gen.
4983 // See ParRescanTask where this is currently used.
4984 void
4985 CMSCollector::
4986 initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
4987 assert(n_threads > 0, "Unexpected n_threads argument");
4988
4989 // Eden space
4990 if (!_young_gen->eden()->is_empty()) {
4991 SequentialSubTasksDone* pst = _young_gen->eden()->par_seq_tasks();
4992 assert(!pst->valid(), "Clobbering existing data?");
4993 // Each valid entry in [0, _eden_chunk_index) represents a task.
5087 // as a result of work_q overflow
5088 restore_preserved_marks_if_any();
5089 }
5090
5091 // Non-parallel version of remark
5092 void CMSCollector::do_remark_non_parallel() {
5093 ResourceMark rm;
5094 HandleMark hm;
5095 GenCollectedHeap* gch = GenCollectedHeap::heap();
5096 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
5097
5098 MarkRefsIntoAndScanClosure
5099 mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */,
5100 &_markStack, this,
5101 false /* should_yield */, false /* not precleaning */);
5102 MarkFromDirtyCardsClosure
5103 markFromDirtyCardsClosure(this, _span,
5104 NULL, // space is set further below
5105 &_markBitMap, &_markStack, &mrias_cl);
5106 {
5107 GCTraceTime t("grey object rescan", PrintGCDetails, false, _gc_timer_cm);
5108 // Iterate over the dirty cards, setting the corresponding bits in the
5109 // mod union table.
5110 {
5111 ModUnionClosure modUnionClosure(&_modUnionTable);
5112 _ct->ct_bs()->dirty_card_iterate(
5113 _cmsGen->used_region(),
5114 &modUnionClosure);
5115 }
5116 // Having transferred these marks into the modUnionTable, we just need
5117 // to rescan the marked objects on the dirty cards in the modUnionTable.
5118 // The initial marking may have been done during an asynchronous
5119 // collection so there may be dirty bits in the mod-union table.
5120 const int alignment =
5121 CardTableModRefBS::card_size * BitsPerWord;
5122 {
5123 // ... First handle dirty cards in CMS gen
5124 markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
5125 MemRegion ur = _cmsGen->used_region();
5126 HeapWord* lb = ur.start();
5127 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
5128 MemRegion cms_span(lb, ub);
5129 _modUnionTable.dirty_range_iterate_clear(cms_span,
5130 &markFromDirtyCardsClosure);
5131 verify_work_stacks_empty();
5132 if (PrintCMSStatistics != 0) {
5133 gclog_or_tty->print(" (re-scanned " SIZE_FORMAT " dirty cards in cms gen) ",
5134 markFromDirtyCardsClosure.num_dirty_cards());
5135 }
5136 }
5137 }
5138 if (VerifyDuringGC &&
5139 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
5140 HandleMark hm; // Discard invalid handles created during verification
5141 Universe::verify();
5142 }
5143 {
5144 GCTraceTime t("root rescan", PrintGCDetails, false, _gc_timer_cm);
5145
5146 verify_work_stacks_empty();
5147
5148 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
5149 StrongRootsScope srs(1);
5150
5151 gch->gen_process_roots(&srs,
5152 GenCollectedHeap::OldGen,
5153 true, // young gen as roots
5154 GenCollectedHeap::ScanningOption(roots_scanning_options()),
5155 should_unload_classes(),
5156 &mrias_cl,
5157 NULL,
5158 NULL); // The dirty klasses will be handled below
5159
5160 assert(should_unload_classes()
5161 || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
5162 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
5163 }
5164
5165 {
5166 GCTraceTime t("visit unhandled CLDs", PrintGCDetails, false, _gc_timer_cm);
5167
5168 verify_work_stacks_empty();
5169
5170 // Scan all class loader data objects that might have been introduced
5171 // during concurrent marking.
5172 ResourceMark rm;
5173 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
5174 for (int i = 0; i < array->length(); i++) {
5175 mrias_cl.do_cld_nv(array->at(i));
5176 }
5177
5178 // We don't need to keep track of new CLDs anymore.
5179 ClassLoaderDataGraph::remember_new_clds(false);
5180
5181 verify_work_stacks_empty();
5182 }
5183
5184 {
5185 GCTraceTime t("dirty klass scan", PrintGCDetails, false, _gc_timer_cm);
5186
5187 verify_work_stacks_empty();
5188
5189 RemarkKlassClosure remark_klass_closure(&mrias_cl);
5190 ClassLoaderDataGraph::classes_do(&remark_klass_closure);
5191
5192 verify_work_stacks_empty();
5193 }
5194
5195 // We might have added oops to ClassLoaderData::_handles during the
5196 // concurrent marking phase. These oops point to newly allocated objects
5197 // that are guaranteed to be kept alive. Either by the direct allocation
5198 // code, or when the young collector processes the roots. Hence,
5199 // we don't have to revisit the _handles block during the remark phase.
5200
5201 verify_work_stacks_empty();
5202 // Restore evacuated mark words, if any, used for overflow list links
5203 restore_preserved_marks_if_any();
5204
5205 verify_overflow_empty();
5327 // We'd like to assert(work_q->size() != 0, ...)
5328 // because we just took work from the overflow list,
5329 // but of course we can't, since all of that might have
5330 // been already stolen from us.
5331 continue;
5332 }
5333 // Verify that we have no work before we resort to stealing
5334 assert(work_q->size() == 0, "Have work, shouldn't steal");
5335 // Try to steal from other queues that have work
5336 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5337 NOT_PRODUCT(num_steals++;)
5338 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5339 assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5340 // Do scanning work
5341 obj_to_scan->oop_iterate(keep_alive);
5342 // Loop around, finish this work, and try to steal some more
5343 } else if (terminator()->offer_termination()) {
5344 break; // nirvana from the infinite cycle
5345 }
5346 }
5347 NOT_PRODUCT(
5348 if (PrintCMSStatistics != 0) {
5349 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
5350 }
5351 )
5352 }
5353
5354 void CMSRefProcTaskExecutor::execute(ProcessTask& task)
5355 {
5356 GenCollectedHeap* gch = GenCollectedHeap::heap();
5357 WorkGang* workers = gch->workers();
5358 assert(workers != NULL, "Need parallel worker threads.");
5359 CMSRefProcTaskProxy rp_task(task, &_collector,
5360 _collector.ref_processor()->span(),
5361 _collector.markBitMap(),
5362 workers, _collector.task_queues());
5363 workers->run_task(&rp_task);
5364 }
5365
5366 void CMSRefProcTaskExecutor::execute(EnqueueTask& task)
5367 {
5368
5369 GenCollectedHeap* gch = GenCollectedHeap::heap();
5370 WorkGang* workers = gch->workers();
5371 assert(workers != NULL, "Need parallel worker threads.");
5373 workers->run_task(&enq_task);
5374 }
5375
5376 void CMSCollector::refProcessingWork() {
5377 ResourceMark rm;
5378 HandleMark hm;
5379
5380 ReferenceProcessor* rp = ref_processor();
5381 assert(rp->span().equals(_span), "Spans should be equal");
5382 assert(!rp->enqueuing_is_done(), "Enqueuing should not be complete");
5383 // Process weak references.
5384 rp->setup_policy(false);
5385 verify_work_stacks_empty();
5386
5387 CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap,
5388 &_markStack, false /* !preclean */);
5389 CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this,
5390 _span, &_markBitMap, &_markStack,
5391 &cmsKeepAliveClosure, false /* !preclean */);
5392 {
5393 GCTraceTime t("weak refs processing", PrintGCDetails, false, _gc_timer_cm);
5394
5395 ReferenceProcessorStats stats;
5396 if (rp->processing_is_mt()) {
5397 // Set the degree of MT here. If the discovery is done MT, there
5398 // may have been a different number of threads doing the discovery
5399 // and a different number of discovered lists may have Ref objects.
5400 // That is OK as long as the Reference lists are balanced (see
5401 // balance_all_queues() and balance_queues()).
5402 GenCollectedHeap* gch = GenCollectedHeap::heap();
5403 uint active_workers = ParallelGCThreads;
5404 WorkGang* workers = gch->workers();
5405 if (workers != NULL) {
5406 active_workers = workers->active_workers();
5407 // The expectation is that active_workers will have already
5408 // been set to a reasonable value. If it has not been set,
5409 // investigate.
5410 assert(active_workers > 0, "Should have been set during scavenge");
5411 }
5412 rp->set_active_mt_degree(active_workers);
5413 CMSRefProcTaskExecutor task_executor(*this);
5415 &cmsKeepAliveClosure,
5416 &cmsDrainMarkingStackClosure,
5417 &task_executor,
5418 _gc_timer_cm);
5419 } else {
5420 stats = rp->process_discovered_references(&_is_alive_closure,
5421 &cmsKeepAliveClosure,
5422 &cmsDrainMarkingStackClosure,
5423 NULL,
5424 _gc_timer_cm);
5425 }
5426 _gc_tracer_cm->report_gc_reference_stats(stats);
5427
5428 }
5429
5430 // This is the point where the entire marking should have completed.
5431 verify_work_stacks_empty();
5432
5433 if (should_unload_classes()) {
5434 {
5435 GCTraceTime t("class unloading", PrintGCDetails, false, _gc_timer_cm);
5436
5437 // Unload classes and purge the SystemDictionary.
5438 bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure);
5439
5440 // Unload nmethods.
5441 CodeCache::do_unloading(&_is_alive_closure, purged_class);
5442
5443 // Prune dead klasses from subklass/sibling/implementor lists.
5444 Klass::clean_weak_klass_links(&_is_alive_closure);
5445 }
5446
5447 {
5448 GCTraceTime t("scrub symbol table", PrintGCDetails, false, _gc_timer_cm);
5449 // Clean up unreferenced symbols in symbol table.
5450 SymbolTable::unlink();
5451 }
5452
5453 {
5454 GCTraceTime t("scrub string table", PrintGCDetails, false, _gc_timer_cm);
5455 // Delete entries for dead interned strings.
5456 StringTable::unlink(&_is_alive_closure);
5457 }
5458 }
5459
5460
5461 // Restore any preserved marks as a result of mark stack or
5462 // work queue overflow
5463 restore_preserved_marks_if_any(); // done single-threaded for now
5464
5465 rp->set_enqueuing_is_done(true);
5466 if (rp->processing_is_mt()) {
5467 rp->balance_all_queues();
5468 CMSRefProcTaskExecutor task_executor(*this);
5469 rp->enqueue_discovered_references(&task_executor);
5470 } else {
5471 rp->enqueue_discovered_references(NULL);
5472 }
5473 rp->verify_no_references_recorded();
5474 assert(!rp->discovery_enabled(), "should have been disabled");
5501 }
5502 }
5503 }
5504 #endif
5505
5506 void CMSCollector::sweep() {
5507 assert(_collectorState == Sweeping, "just checking");
5508 check_correct_thread_executing();
5509 verify_work_stacks_empty();
5510 verify_overflow_empty();
5511 increment_sweep_count();
5512 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
5513
5514 _inter_sweep_timer.stop();
5515 _inter_sweep_estimate.sample(_inter_sweep_timer.seconds());
5516
5517 assert(!_intra_sweep_timer.is_active(), "Should not be active");
5518 _intra_sweep_timer.reset();
5519 _intra_sweep_timer.start();
5520 {
5521 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
5522 CMSPhaseAccounting pa(this, "sweep", !PrintGCDetails);
5523 // First sweep the old gen
5524 {
5525 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
5526 bitMapLock());
5527 sweepWork(_cmsGen);
5528 }
5529
5530 // Update Universe::_heap_*_at_gc figures.
5531 // We need all the free list locks to make the abstract state
5532 // transition from Sweeping to Resetting. See detailed note
5533 // further below.
5534 {
5535 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock());
5536 // Update heap occupancy information which is used as
5537 // input to soft ref clearing policy at the next gc.
5538 Universe::update_heap_info_at_gc();
5539 _collectorState = Resizing;
5540 }
5541 }
5542 verify_work_stacks_empty();
5585 GenCollectedHeap* gch = GenCollectedHeap::heap();
5586 gch->clear_incremental_collection_failed(); // Worth retrying as fresh space may have been freed up
5587 gch->update_full_collections_completed(_collection_count_start);
5588 }
5589
5590 // FIX ME!!! Looks like this belongs in CFLSpace, with
5591 // CMSGen merely delegating to it.
5592 void ConcurrentMarkSweepGeneration::setNearLargestChunk() {
5593 double nearLargestPercent = FLSLargestBlockCoalesceProximity;
5594 HeapWord* minAddr = _cmsSpace->bottom();
5595 HeapWord* largestAddr =
5596 (HeapWord*) _cmsSpace->dictionary()->find_largest_dict();
5597 if (largestAddr == NULL) {
5598 // The dictionary appears to be empty. In this case
5599 // try to coalesce at the end of the heap.
5600 largestAddr = _cmsSpace->end();
5601 }
5602 size_t largestOffset = pointer_delta(largestAddr, minAddr);
5603 size_t nearLargestOffset =
5604 (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
5605 if (PrintFLSStatistics != 0) {
5606 gclog_or_tty->print_cr(
5607 "CMS: Large Block: " PTR_FORMAT ";"
5608 " Proximity: " PTR_FORMAT " -> " PTR_FORMAT,
5609 p2i(largestAddr),
5610 p2i(_cmsSpace->nearLargestChunk()), p2i(minAddr + nearLargestOffset));
5611 }
5612 _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
5613 }
5614
5615 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
5616 return addr >= _cmsSpace->nearLargestChunk();
5617 }
5618
5619 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
5620 return _cmsSpace->find_chunk_at_end();
5621 }
5622
5623 void ConcurrentMarkSweepGeneration::update_gc_stats(Generation* current_generation,
5624 bool full) {
5625 // If the young generation has been collected, gather any statistics
5626 // that are of interest at this point.
5627 bool current_is_young = GenCollectedHeap::heap()->is_young_gen(current_generation);
5628 if (!full && current_is_young) {
5629 // Gather statistics on the young generation collection.
5630 collector()->stats().record_gc0_end(used());
5631 }
5685 } else { // did not unload classes,
5686 _concurrent_cycles_since_last_unload++; // ... increment count
5687 }
5688 }
5689
5690 // Reset CMS data structures (for now just the marking bit map)
5691 // preparatory for the next cycle.
5692 void CMSCollector::reset_concurrent() {
5693 CMSTokenSyncWithLocks ts(true, bitMapLock());
5694
5695 // If the state is not "Resetting", the foreground thread
5696 // has done a collection and the resetting.
5697 if (_collectorState != Resetting) {
5698 assert(_collectorState == Idling, "The state should only change"
5699 " because the foreground collector has finished the collection");
5700 return;
5701 }
5702
5703 // Clear the mark bitmap (no grey objects to start with)
5704 // for the next cycle.
5705 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
5706 CMSPhaseAccounting cmspa(this, "reset", !PrintGCDetails);
5707
5708 HeapWord* curAddr = _markBitMap.startWord();
5709 while (curAddr < _markBitMap.endWord()) {
5710 size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr);
5711 MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining));
5712 _markBitMap.clear_large_range(chunk);
5713 if (ConcurrentMarkSweepThread::should_yield() &&
5714 !foregroundGCIsActive() &&
5715 CMSYield) {
5716 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
5717 "CMS thread should hold CMS token");
5718 assert_lock_strong(bitMapLock());
5719 bitMapLock()->unlock();
5720 ConcurrentMarkSweepThread::desynchronize(true);
5721 stopTimer();
5722 if (PrintCMSStatistics != 0) {
5723 incrementYields();
5724 }
5725
5726 // See the comment in coordinator_yield()
5727 for (unsigned i = 0; i < CMSYieldSleepCount &&
5728 ConcurrentMarkSweepThread::should_yield() &&
5729 !CMSCollector::foregroundGCIsActive(); ++i) {
5730 os::sleep(Thread::current(), 1, false);
5731 }
5732
5733 ConcurrentMarkSweepThread::synchronize(true);
5734 bitMapLock()->lock_without_safepoint_check();
5735 startTimer();
5736 }
5737 curAddr = chunk.end();
5738 }
5739 // A successful mostly concurrent collection has been done.
5740 // Because only the full (i.e., concurrent mode failure) collections
5741 // are being measured for gc overhead limits, clean the "near" flag
5742 // and count.
5743 size_policy()->reset_gc_overhead_limit_count();
5744 _collectorState = Idling;
5745
5746 register_gc_end();
5747 }
5748
5749 // Same as above but for STW paths
5750 void CMSCollector::reset_stw() {
5751 // already have the lock
5752 assert(_collectorState == Resetting, "just checking");
5753 assert_lock_strong(bitMapLock());
5754 GCIdMarkAndRestore gc_id_mark(_cmsThread->gc_id());
5755 _markBitMap.clear_all();
5756 _collectorState = Idling;
5757 register_gc_end();
5758 }
5759
5760 void CMSCollector::do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause) {
5761 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
5762 GCTraceTime t(GCCauseString("GC", gc_cause), PrintGC, !PrintGCDetails, NULL);
5763 TraceCollectorStats tcs(counters());
5764
5765 switch (op) {
5766 case CMS_op_checkpointRootsInitial: {
5767 SvcGCMarker sgcm(SvcGCMarker::OTHER);
5768 checkpointRootsInitial();
5769 if (PrintGC) {
5770 _cmsGen->printOccupancy("initial-mark");
5771 }
5772 break;
5773 }
5774 case CMS_op_checkpointRootsFinal: {
5775 SvcGCMarker sgcm(SvcGCMarker::OTHER);
5776 checkpointRootsFinal();
5777 if (PrintGC) {
5778 _cmsGen->printOccupancy("remark");
5779 }
5780 break;
5781 }
5782 default:
5783 fatal("No such CMS_op");
5784 }
5785 }
5786
5787 #ifndef PRODUCT
5788 size_t const CMSCollector::skip_header_HeapWords() {
5789 return FreeChunk::header_size();
5790 }
5791
5792 // Try and collect here conditions that should hold when
5793 // CMS thread is exiting. The idea is that the foreground GC
5794 // thread should not be blocked if it wants to terminate
5795 // the CMS thread and yet continue to run the VM for a while
5796 // after that.
5797 void CMSCollector::verify_ok_to_terminate() const {
5798 assert(Thread::current()->is_ConcurrentGC_thread(),
5799 "should be called by CMS thread");
5972 }
5973 assert(_virtual_space.committed_size() == rs.size(),
5974 "didn't reserve backing store for all of CMS stack?");
5975 _base = (oop*)(_virtual_space.low());
5976 _index = 0;
5977 _capacity = size;
5978 NOT_PRODUCT(_max_depth = 0);
5979 return true;
5980 }
5981
5982 // XXX FIX ME !!! In the MT case we come in here holding a
5983 // leaf lock. For printing we need to take a further lock
5984 // which has lower rank. We need to recalibrate the two
5985 // lock-ranks involved in order to be able to print the
5986 // messages below. (Or defer the printing to the caller.
5987 // For now we take the expedient path of just disabling the
5988 // messages for the problematic case.)
5989 void CMSMarkStack::expand() {
5990 assert(_capacity <= MarkStackSizeMax, "stack bigger than permitted");
5991 if (_capacity == MarkStackSizeMax) {
5992 if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
5993 // We print a warning message only once per CMS cycle.
5994 gclog_or_tty->print_cr(" (benign) Hit CMSMarkStack max size limit");
5995 }
5996 return;
5997 }
5998 // Double capacity if possible
5999 size_t new_capacity = MIN2(_capacity*2, MarkStackSizeMax);
6000 // Do not give up existing stack until we have managed to
6001 // get the double capacity that we desired.
6002 ReservedSpace rs(ReservedSpace::allocation_align_size_up(
6003 new_capacity * sizeof(oop)));
6004 if (rs.is_reserved()) {
6005 // Release the backing store associated with old stack
6006 _virtual_space.release();
6007 // Reinitialize virtual space for new stack
6008 if (!_virtual_space.initialize(rs, rs.size())) {
6009 fatal("Not enough swap for expanded marking stack");
6010 }
6011 _base = (oop*)(_virtual_space.low());
6012 _index = 0;
6013 _capacity = new_capacity;
6014 } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
6015 // Failed to double capacity, continue;
6016 // we print a detail message only once per CMS cycle.
6017 gclog_or_tty->print(" (benign) Failed to expand marking stack from " SIZE_FORMAT "K to "
6018 SIZE_FORMAT "K",
6019 _capacity / K, new_capacity / K);
6020 }
6021 }
6022
6023
6024 // Closures
6025 // XXX: there seems to be a lot of code duplication here;
6026 // should refactor and consolidate common code.
6027
6028 // This closure is used to mark refs into the CMS generation in
6029 // the CMS bit map. Called at the first checkpoint. This closure
6030 // assumes that we do not need to re-mark dirty cards; if the CMS
6031 // generation on which this is used is not an oldest
6032 // generation then this will lose younger_gen cards!
6033
6034 MarkRefsIntoClosure::MarkRefsIntoClosure(
6035 MemRegion span, CMSBitMap* bitMap):
6036 _span(span),
6037 _bitMap(bitMap)
6038 {
6076 void Par_MarkRefsIntoClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); }
6077
6078 // A variant of the above, used for CMS marking verification.
6079 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
6080 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
6081 _span(span),
6082 _verification_bm(verification_bm),
6083 _cms_bm(cms_bm)
6084 {
6085 assert(ref_processor() == NULL, "deliberately left NULL");
6086 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
6087 }
6088
6089 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
6090 // if p points into _span, then mark corresponding bit in _markBitMap
6091 assert(obj->is_oop(), "expected an oop");
6092 HeapWord* addr = (HeapWord*)obj;
6093 if (_span.contains(addr)) {
6094 _verification_bm->mark(addr);
6095 if (!_cms_bm->isMarked(addr)) {
6096 oop(addr)->print();
6097 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
6098 fatal("... aborting");
6099 }
6100 }
6101 }
6102
6103 void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
6104 void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
6105
6106 //////////////////////////////////////////////////
6107 // MarkRefsIntoAndScanClosure
6108 //////////////////////////////////////////////////
6109
6110 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
6111 ReferenceProcessor* rp,
6112 CMSBitMap* bit_map,
6113 CMSBitMap* mod_union_table,
6114 CMSMarkStack* mark_stack,
6115 CMSCollector* collector,
6116 bool should_yield,
6117 bool concurrent_precleaning):
6173 "overflow list was drained above");
6174
6175 assert(_collector->no_preserved_marks(),
6176 "All preserved marks should have been restored above");
6177 }
6178 }
6179
6180 void MarkRefsIntoAndScanClosure::do_oop(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
6181 void MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
6182
6183 void MarkRefsIntoAndScanClosure::do_yield_work() {
6184 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6185 "CMS thread should hold CMS token");
6186 assert_lock_strong(_freelistLock);
6187 assert_lock_strong(_bit_map->lock());
6188 // relinquish the free_list_lock and bitMaplock()
6189 _bit_map->lock()->unlock();
6190 _freelistLock->unlock();
6191 ConcurrentMarkSweepThread::desynchronize(true);
6192 _collector->stopTimer();
6193 if (PrintCMSStatistics != 0) {
6194 _collector->incrementYields();
6195 }
6196
6197 // See the comment in coordinator_yield()
6198 for (unsigned i = 0;
6199 i < CMSYieldSleepCount &&
6200 ConcurrentMarkSweepThread::should_yield() &&
6201 !CMSCollector::foregroundGCIsActive();
6202 ++i) {
6203 os::sleep(Thread::current(), 1, false);
6204 }
6205
6206 ConcurrentMarkSweepThread::synchronize(true);
6207 _freelistLock->lock_without_safepoint_check();
6208 _bit_map->lock()->lock_without_safepoint_check();
6209 _collector->startTimer();
6210 }
6211
6212 ///////////////////////////////////////////////////////////
6213 // Par_MarkRefsIntoAndScanClosure: a parallel version of
6214 // MarkRefsIntoAndScanClosure
6215 ///////////////////////////////////////////////////////////
6331 // An object not (yet) reached by marking: we merely need to
6332 // compute its size so as to go look at the next block.
6333 assert(p->is_oop(true), "should be an oop");
6334 size = CompactibleFreeListSpace::adjustObjectSize(p->size());
6335 }
6336 }
6337 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6338 return size;
6339 }
6340
6341 void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
6342 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6343 "CMS thread should hold CMS token");
6344 assert_lock_strong(_freelistLock);
6345 assert_lock_strong(_bitMap->lock());
6346 // relinquish the free_list_lock and bitMaplock()
6347 _bitMap->lock()->unlock();
6348 _freelistLock->unlock();
6349 ConcurrentMarkSweepThread::desynchronize(true);
6350 _collector->stopTimer();
6351 if (PrintCMSStatistics != 0) {
6352 _collector->incrementYields();
6353 }
6354
6355 // See the comment in coordinator_yield()
6356 for (unsigned i = 0; i < CMSYieldSleepCount &&
6357 ConcurrentMarkSweepThread::should_yield() &&
6358 !CMSCollector::foregroundGCIsActive(); ++i) {
6359 os::sleep(Thread::current(), 1, false);
6360 }
6361
6362 ConcurrentMarkSweepThread::synchronize(true);
6363 _freelistLock->lock_without_safepoint_check();
6364 _bitMap->lock()->lock_without_safepoint_check();
6365 _collector->startTimer();
6366 }
6367
6368
6369 //////////////////////////////////////////////////////////////////
6370 // SurvivorSpacePrecleanClosure
6371 //////////////////////////////////////////////////////////////////
6372 // This (single-threaded) closure is used to preclean the oops in
6373 // the survivor spaces.
6400 // the ones in CMS heap (i.e. in _span).
6401 new_oop->oop_iterate(_scanning_closure);
6402 // check if it's time to yield
6403 do_yield_check();
6404 }
6405 unsigned int after_count =
6406 GenCollectedHeap::heap()->total_collections();
6407 bool abort = (_before_count != after_count) ||
6408 _collector->should_abort_preclean();
6409 return abort ? 0 : size;
6410 }
6411
6412 void SurvivorSpacePrecleanClosure::do_yield_work() {
6413 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6414 "CMS thread should hold CMS token");
6415 assert_lock_strong(_bit_map->lock());
6416 // Relinquish the bit map lock
6417 _bit_map->lock()->unlock();
6418 ConcurrentMarkSweepThread::desynchronize(true);
6419 _collector->stopTimer();
6420 if (PrintCMSStatistics != 0) {
6421 _collector->incrementYields();
6422 }
6423
6424 // See the comment in coordinator_yield()
6425 for (unsigned i = 0; i < CMSYieldSleepCount &&
6426 ConcurrentMarkSweepThread::should_yield() &&
6427 !CMSCollector::foregroundGCIsActive(); ++i) {
6428 os::sleep(Thread::current(), 1, false);
6429 }
6430
6431 ConcurrentMarkSweepThread::synchronize(true);
6432 _bit_map->lock()->lock_without_safepoint_check();
6433 _collector->startTimer();
6434 }
6435
6436 // This closure is used to rescan the marked objects on the dirty cards
6437 // in the mod union table and the card table proper. In the parallel
6438 // case, although the bitMap is shared, we do a single read so the
6439 // isMarked() query is "safe".
6440 bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) {
6441 // Ignore mark word because we are running concurrent with mutators
6442 assert(p->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(p));
6555 return true;
6556 }
6557 }
6558 scanOopsInOop(addr);
6559 return true;
6560 }
6561
6562 // We take a break if we've been at this for a while,
6563 // so as to avoid monopolizing the locks involved.
6564 void MarkFromRootsClosure::do_yield_work() {
6565 // First give up the locks, then yield, then re-lock
6566 // We should probably use a constructor/destructor idiom to
6567 // do this unlock/lock or modify the MutexUnlocker class to
6568 // serve our purpose. XXX
6569 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6570 "CMS thread should hold CMS token");
6571 assert_lock_strong(_bitMap->lock());
6572 _bitMap->lock()->unlock();
6573 ConcurrentMarkSweepThread::desynchronize(true);
6574 _collector->stopTimer();
6575 if (PrintCMSStatistics != 0) {
6576 _collector->incrementYields();
6577 }
6578
6579 // See the comment in coordinator_yield()
6580 for (unsigned i = 0; i < CMSYieldSleepCount &&
6581 ConcurrentMarkSweepThread::should_yield() &&
6582 !CMSCollector::foregroundGCIsActive(); ++i) {
6583 os::sleep(Thread::current(), 1, false);
6584 }
6585
6586 ConcurrentMarkSweepThread::synchronize(true);
6587 _bitMap->lock()->lock_without_safepoint_check();
6588 _collector->startTimer();
6589 }
6590
6591 void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) {
6592 assert(_bitMap->isMarked(ptr), "expected bit to be set");
6593 assert(_markStack->isEmpty(),
6594 "should drain stack to limit stack usage");
6595 // convert ptr to an oop preparatory to scanning
6596 oop obj = oop(ptr);
6597 // Ignore mark word in verification below, since we
6863 void PushAndMarkVerifyClosure::do_oop(narrowOop* p) { PushAndMarkVerifyClosure::do_oop_work(p); }
6864
6865 // Upon stack overflow, we discard (part of) the stack,
6866 // remembering the least address amongst those discarded
6867 // in CMSCollector's _restart_address.
6868 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
6869 // Remember the least grey address discarded
6870 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
6871 _collector->lower_restart_addr(ra);
6872 _mark_stack->reset(); // discard stack contents
6873 _mark_stack->expand(); // expand the stack if possible
6874 }
6875
6876 void PushAndMarkVerifyClosure::do_oop(oop obj) {
6877 assert(obj->is_oop_or_null(), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6878 HeapWord* addr = (HeapWord*)obj;
6879 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
6880 // Oop lies in _span and isn't yet grey or black
6881 _verification_bm->mark(addr); // now grey
6882 if (!_cms_bm->isMarked(addr)) {
6883 oop(addr)->print();
6884 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)",
6885 p2i(addr));
6886 fatal("... aborting");
6887 }
6888
6889 if (!_mark_stack->push(obj)) { // stack overflow
6890 if (PrintCMSStatistics != 0) {
6891 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
6892 SIZE_FORMAT, _mark_stack->capacity());
6893 }
6894 assert(_mark_stack->isFull(), "Else push should have succeeded");
6895 handle_stack_overflow(addr);
6896 }
6897 // anything including and to the right of _finger
6898 // will be scanned as we iterate over the remainder of the
6899 // bit map
6900 }
6901 }
6902
6903 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
6904 MemRegion span,
6905 CMSBitMap* bitMap, CMSMarkStack* markStack,
6906 HeapWord* finger, MarkFromRootsClosure* parent) :
6907 MetadataAwareOopClosure(collector->ref_processor()),
6908 _collector(collector),
6909 _span(span),
6910 _bitMap(bitMap),
6911 _markStack(markStack),
6912 _finger(finger),
6913 _parent(parent)
6973 void PushOrMarkClosure::do_oop(oop obj) {
6974 // Ignore mark word because we are running concurrent with mutators.
6975 assert(obj->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6976 HeapWord* addr = (HeapWord*)obj;
6977 if (_span.contains(addr) && !_bitMap->isMarked(addr)) {
6978 // Oop lies in _span and isn't yet grey or black
6979 _bitMap->mark(addr); // now grey
6980 if (addr < _finger) {
6981 // the bit map iteration has already either passed, or
6982 // sampled, this bit in the bit map; we'll need to
6983 // use the marking stack to scan this oop's oops.
6984 bool simulate_overflow = false;
6985 NOT_PRODUCT(
6986 if (CMSMarkStackOverflowALot &&
6987 _collector->simulate_overflow()) {
6988 // simulate a stack overflow
6989 simulate_overflow = true;
6990 }
6991 )
6992 if (simulate_overflow || !_markStack->push(obj)) { // stack overflow
6993 if (PrintCMSStatistics != 0) {
6994 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
6995 SIZE_FORMAT, _markStack->capacity());
6996 }
6997 assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded");
6998 handle_stack_overflow(addr);
6999 }
7000 }
7001 // anything including and to the right of _finger
7002 // will be scanned as we iterate over the remainder of the
7003 // bit map
7004 do_yield_check();
7005 }
7006 }
7007
7008 void PushOrMarkClosure::do_oop(oop* p) { PushOrMarkClosure::do_oop_work(p); }
7009 void PushOrMarkClosure::do_oop(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); }
7010
7011 void Par_PushOrMarkClosure::do_oop(oop obj) {
7012 // Ignore mark word because we are running concurrent with mutators.
7013 assert(obj->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
7014 HeapWord* addr = (HeapWord*)obj;
7015 if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) {
7016 // Oop lies in _span and isn't yet grey or black
7025 // -- else push on work queue
7026 if ( !res // someone else marked it, they will deal with it
7027 || (addr >= *gfa) // will be scanned in a later task
7028 || (_span.contains(addr) && addr >= _finger)) { // later in this chunk
7029 return;
7030 }
7031 // the bit map iteration has already either passed, or
7032 // sampled, this bit in the bit map; we'll need to
7033 // use the marking stack to scan this oop's oops.
7034 bool simulate_overflow = false;
7035 NOT_PRODUCT(
7036 if (CMSMarkStackOverflowALot &&
7037 _collector->simulate_overflow()) {
7038 // simulate a stack overflow
7039 simulate_overflow = true;
7040 }
7041 )
7042 if (simulate_overflow ||
7043 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
7044 // stack overflow
7045 if (PrintCMSStatistics != 0) {
7046 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
7047 SIZE_FORMAT, _overflow_stack->capacity());
7048 }
7049 // We cannot assert that the overflow stack is full because
7050 // it may have been emptied since.
7051 assert(simulate_overflow ||
7052 _work_queue->size() == _work_queue->max_elems(),
7053 "Else push should have succeeded");
7054 handle_stack_overflow(addr);
7055 }
7056 do_yield_check();
7057 }
7058 }
7059
7060 void Par_PushOrMarkClosure::do_oop(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
7061 void Par_PushOrMarkClosure::do_oop(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
7062
7063 PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector,
7064 MemRegion span,
7065 ReferenceProcessor* rp,
7066 CMSBitMap* bit_map,
7067 CMSBitMap* mod_union_table,
7068 CMSMarkStack* mark_stack,
7190 _collector->par_push_on_overflow_list(obj);
7191 _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS
7192 }
7193 } // Else, some other thread got there first
7194 }
7195 }
7196
7197 void Par_PushAndMarkClosure::do_oop(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
7198 void Par_PushAndMarkClosure::do_oop(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
7199
7200 void CMSPrecleanRefsYieldClosure::do_yield_work() {
7201 Mutex* bml = _collector->bitMapLock();
7202 assert_lock_strong(bml);
7203 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7204 "CMS thread should hold CMS token");
7205
7206 bml->unlock();
7207 ConcurrentMarkSweepThread::desynchronize(true);
7208
7209 _collector->stopTimer();
7210 if (PrintCMSStatistics != 0) {
7211 _collector->incrementYields();
7212 }
7213
7214 // See the comment in coordinator_yield()
7215 for (unsigned i = 0; i < CMSYieldSleepCount &&
7216 ConcurrentMarkSweepThread::should_yield() &&
7217 !CMSCollector::foregroundGCIsActive(); ++i) {
7218 os::sleep(Thread::current(), 1, false);
7219 }
7220
7221 ConcurrentMarkSweepThread::synchronize(true);
7222 bml->lock();
7223
7224 _collector->startTimer();
7225 }
7226
7227 bool CMSPrecleanRefsYieldClosure::should_return() {
7228 if (ConcurrentMarkSweepThread::should_yield()) {
7229 do_yield_work();
7230 }
7231 return _collector->foregroundGCIsActive();
7232 }
7233
7234 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
7235 assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
7236 "mr should be aligned to start at a card boundary");
7237 // We'd like to assert:
7238 // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
7239 // "mr should be a range of cards");
7240 // However, that would be too strong in one case -- the last
7241 // partition ends at _unallocated_block which, in general, can be
7242 // an arbitrary boundary, not necessarily card aligned.
7243 if (PrintCMSStatistics != 0) {
7244 _num_dirty_cards +=
7245 mr.word_size()/CardTableModRefBS::card_size_in_words;
7246 }
7247 _space->object_iterate_mem(mr, &_scan_cl);
7248 }
7249
7250 SweepClosure::SweepClosure(CMSCollector* collector,
7251 ConcurrentMarkSweepGeneration* g,
7252 CMSBitMap* bitMap, bool should_yield) :
7253 _collector(collector),
7254 _g(g),
7255 _sp(g->cmsSpace()),
7256 _limit(_sp->sweep_limit()),
7257 _freelistLock(_sp->freelistLock()),
7258 _bitMap(bitMap),
7259 _yield(should_yield),
7260 _inFreeRange(false), // No free range at beginning of sweep
7261 _freeRangeInFreeLists(false), // No free range at beginning of sweep
7262 _lastFreeRangeCoalesced(false),
7263 _freeFinger(g->used_region().start())
7264 {
7265 NOT_PRODUCT(
7266 _numObjectsFreed = 0;
7267 _numWordsFreed = 0;
7268 _numObjectsLive = 0;
7269 _numWordsLive = 0;
7270 _numObjectsAlreadyFree = 0;
7271 _numWordsAlreadyFree = 0;
7272 _last_fc = NULL;
7273
7274 _sp->initializeIndexedFreeListArrayReturnedBytes();
7275 _sp->dictionary()->initialize_dict_returned_bytes();
7276 )
7277 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7278 "sweep _limit out of bounds");
7279 if (CMSTraceSweeper) {
7280 gclog_or_tty->print_cr("\n====================\nStarting new sweep with limit " PTR_FORMAT,
7281 p2i(_limit));
7282 }
7283 }
7284
7285 void SweepClosure::print_on(outputStream* st) const {
7286 tty->print_cr("_sp = [" PTR_FORMAT "," PTR_FORMAT ")",
7287 p2i(_sp->bottom()), p2i(_sp->end()));
7288 tty->print_cr("_limit = " PTR_FORMAT, p2i(_limit));
7289 tty->print_cr("_freeFinger = " PTR_FORMAT, p2i(_freeFinger));
7290 NOT_PRODUCT(tty->print_cr("_last_fc = " PTR_FORMAT, p2i(_last_fc));)
7291 tty->print_cr("_inFreeRange = %d, _freeRangeInFreeLists = %d, _lastFreeRangeCoalesced = %d",
7292 _inFreeRange, _freeRangeInFreeLists, _lastFreeRangeCoalesced);
7293 }
7294
7295 #ifndef PRODUCT
7296 // Assertion checking only: no useful work in product mode --
7297 // however, if any of the flags below become product flags,
7298 // you may need to review this code to see if it needs to be
7299 // enabled in product mode.
7300 SweepClosure::~SweepClosure() {
7301 assert_lock_strong(_freelistLock);
7302 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7303 "sweep _limit out of bounds");
7304 if (inFreeRange()) {
7305 warning("inFreeRange() should have been reset; dumping state of SweepClosure");
7306 print();
7307 ShouldNotReachHere();
7308 }
7309 if (Verbose && PrintGC) {
7310 gclog_or_tty->print("Collected " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7311 _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
7312 gclog_or_tty->print_cr("\nLive " SIZE_FORMAT " objects, "
7313 SIZE_FORMAT " bytes "
7314 "Already free " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7315 _numObjectsLive, _numWordsLive*sizeof(HeapWord),
7316 _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
7317 size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree)
7318 * sizeof(HeapWord);
7319 gclog_or_tty->print_cr("Total sweep: " SIZE_FORMAT " bytes", totalBytes);
7320
7321 if (PrintCMSStatistics && CMSVerifyReturnedBytes) {
7322 size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
7323 size_t dict_returned_bytes = _sp->dictionary()->sum_dict_returned_bytes();
7324 size_t returned_bytes = indexListReturnedBytes + dict_returned_bytes;
7325 gclog_or_tty->print("Returned " SIZE_FORMAT " bytes", returned_bytes);
7326 gclog_or_tty->print(" Indexed List Returned " SIZE_FORMAT " bytes",
7327 indexListReturnedBytes);
7328 gclog_or_tty->print_cr(" Dictionary Returned " SIZE_FORMAT " bytes",
7329 dict_returned_bytes);
7330 }
7331 }
7332 if (CMSTraceSweeper) {
7333 gclog_or_tty->print_cr("end of sweep with _limit = " PTR_FORMAT "\n================",
7334 p2i(_limit));
7335 }
7336 }
7337 #endif // PRODUCT
7338
7339 void SweepClosure::initialize_free_range(HeapWord* freeFinger,
7340 bool freeRangeInFreeLists) {
7341 if (CMSTraceSweeper) {
7342 gclog_or_tty->print("---- Start free range at " PTR_FORMAT " with free block (%d)\n",
7343 p2i(freeFinger), freeRangeInFreeLists);
7344 }
7345 assert(!inFreeRange(), "Trampling existing free range");
7346 set_inFreeRange(true);
7347 set_lastFreeRangeCoalesced(false);
7348
7349 set_freeFinger(freeFinger);
7350 set_freeRangeInFreeLists(freeRangeInFreeLists);
7351 if (CMSTestInFreeList) {
7352 if (freeRangeInFreeLists) {
7353 FreeChunk* fc = (FreeChunk*) freeFinger;
7354 assert(fc->is_free(), "A chunk on the free list should be free.");
7355 assert(fc->size() > 0, "Free range should have a size");
7356 assert(_sp->verify_chunk_in_free_list(fc), "Chunk is not in free lists");
7357 }
7358 }
7359 }
7360
7361 // Note that the sweeper runs concurrently with mutators. Thus,
7362 // it is possible for direct allocation in this generation to happen
7363 // in the middle of the sweep. Note that the sweeper also coalesces
7364 // contiguous free blocks. Thus, unless the sweeper and the allocator
7390 size_t res;
7391
7392 // Check if we are done sweeping. Below we check "addr >= _limit" rather
7393 // than "addr == _limit" because although _limit was a block boundary when
7394 // we started the sweep, it may no longer be one because heap expansion
7395 // may have caused us to coalesce the block ending at the address _limit
7396 // with a newly expanded chunk (this happens when _limit was set to the
7397 // previous _end of the space), so we may have stepped past _limit:
7398 // see the following Zeno-like trail of CRs 6977970, 7008136, 7042740.
7399 if (addr >= _limit) { // we have swept up to or past the limit: finish up
7400 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7401 "sweep _limit out of bounds");
7402 assert(addr < _sp->end(), "addr out of bounds");
7403 // Flush any free range we might be holding as a single
7404 // coalesced chunk to the appropriate free list.
7405 if (inFreeRange()) {
7406 assert(freeFinger() >= _sp->bottom() && freeFinger() < _limit,
7407 "freeFinger() " PTR_FORMAT " is out-of-bounds", p2i(freeFinger()));
7408 flush_cur_free_chunk(freeFinger(),
7409 pointer_delta(addr, freeFinger()));
7410 if (CMSTraceSweeper) {
7411 gclog_or_tty->print("Sweep: last chunk: ");
7412 gclog_or_tty->print("put_free_blk " PTR_FORMAT " (" SIZE_FORMAT ") "
7413 "[coalesced:%d]\n",
7414 p2i(freeFinger()), pointer_delta(addr, freeFinger()),
7415 lastFreeRangeCoalesced() ? 1 : 0);
7416 }
7417 }
7418
7419 // help the iterator loop finish
7420 return pointer_delta(_sp->end(), addr);
7421 }
7422
7423 assert(addr < _limit, "sweep invariant");
7424 // check if we should yield
7425 do_yield_check(addr);
7426 if (fc->is_free()) {
7427 // Chunk that is already free
7428 res = fc->size();
7429 do_already_free_chunk(fc);
7430 debug_only(_sp->verifyFreeLists());
7431 // If we flush the chunk at hand in lookahead_and_flush()
7432 // and it's coalesced with a preceding chunk, then the
7433 // process of "mangling" the payload of the coalesced block
7434 // will cause erasure of the size information from the
7435 // (erstwhile) header of all the coalesced blocks but the
7436 // first, so the first disjunct in the assert will not hold
7437 // in that specific case (in which case the second disjunct
7607 // Verify that the bit map has no bits marked between
7608 // addr and purported end of this block.
7609 size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
7610 assert(size >= 3, "Necessary for Printezis marks to work");
7611 assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
7612 DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
7613 }
7614 return size;
7615 }
7616
7617 void SweepClosure::do_post_free_or_garbage_chunk(FreeChunk* fc,
7618 size_t chunkSize) {
7619 // do_post_free_or_garbage_chunk() should only be called in the case
7620 // of the adaptive free list allocator.
7621 const bool fcInFreeLists = fc->is_free();
7622 assert((HeapWord*)fc <= _limit, "sweep invariant");
7623 if (CMSTestInFreeList && fcInFreeLists) {
7624 assert(_sp->verify_chunk_in_free_list(fc), "free chunk is not in free lists");
7625 }
7626
7627 if (CMSTraceSweeper) {
7628 gclog_or_tty->print_cr(" -- pick up another chunk at " PTR_FORMAT " (" SIZE_FORMAT ")", p2i(fc), chunkSize);
7629 }
7630
7631 HeapWord* const fc_addr = (HeapWord*) fc;
7632
7633 bool coalesce = false;
7634 const size_t left = pointer_delta(fc_addr, freeFinger());
7635 const size_t right = chunkSize;
7636 switch (FLSCoalescePolicy) {
7637 // numeric value forms a coalition aggressiveness metric
7638 case 0: { // never coalesce
7639 coalesce = false;
7640 break;
7641 }
7642 case 1: { // coalesce if left & right chunks on overpopulated lists
7643 coalesce = _sp->coalOverPopulated(left) &&
7644 _sp->coalOverPopulated(right);
7645 break;
7646 }
7647 case 2: { // coalesce if left chunk on overpopulated list (default)
7648 coalesce = _sp->coalOverPopulated(left);
7649 break;
7710 // we'll look at because its end crosses past _limit, we'll preemptively
7711 // flush it along with any free range we may be holding on to. Note that
7712 // this can be the case only for an already free or freshly garbage
7713 // chunk. If this block is an object, it can never straddle
7714 // over _limit. The "straddling" occurs when _limit is set at
7715 // the previous end of the space when this cycle started, and
7716 // a subsequent heap expansion caused the previously co-terminal
7717 // free block to be coalesced with the newly expanded portion,
7718 // thus rendering _limit a non-block-boundary making it dangerous
7719 // for the sweeper to step over and examine.
7720 void SweepClosure::lookahead_and_flush(FreeChunk* fc, size_t chunk_size) {
7721 assert(inFreeRange(), "Should only be called if currently in a free range.");
7722 HeapWord* const eob = ((HeapWord*)fc) + chunk_size;
7723 assert(_sp->used_region().contains(eob - 1),
7724 "eob = " PTR_FORMAT " eob-1 = " PTR_FORMAT " _limit = " PTR_FORMAT
7725 " out of bounds wrt _sp = [" PTR_FORMAT "," PTR_FORMAT ")"
7726 " when examining fc = " PTR_FORMAT "(" SIZE_FORMAT ")",
7727 p2i(eob), p2i(eob-1), p2i(_limit), p2i(_sp->bottom()), p2i(_sp->end()), p2i(fc), chunk_size);
7728 if (eob >= _limit) {
7729 assert(eob == _limit || fc->is_free(), "Only a free chunk should allow us to cross over the limit");
7730 if (CMSTraceSweeper) {
7731 gclog_or_tty->print_cr("_limit " PTR_FORMAT " reached or crossed by block "
7732 "[" PTR_FORMAT "," PTR_FORMAT ") in space "
7733 "[" PTR_FORMAT "," PTR_FORMAT ")",
7734 p2i(_limit), p2i(fc), p2i(eob), p2i(_sp->bottom()), p2i(_sp->end()));
7735 }
7736 // Return the storage we are tracking back into the free lists.
7737 if (CMSTraceSweeper) {
7738 gclog_or_tty->print_cr("Flushing ... ");
7739 }
7740 assert(freeFinger() < eob, "Error");
7741 flush_cur_free_chunk( freeFinger(), pointer_delta(eob, freeFinger()));
7742 }
7743 }
7744
7745 void SweepClosure::flush_cur_free_chunk(HeapWord* chunk, size_t size) {
7746 assert(inFreeRange(), "Should only be called if currently in a free range.");
7747 assert(size > 0,
7748 "A zero sized chunk cannot be added to the free lists.");
7749 if (!freeRangeInFreeLists()) {
7750 if (CMSTestInFreeList) {
7751 FreeChunk* fc = (FreeChunk*) chunk;
7752 fc->set_size(size);
7753 assert(!_sp->verify_chunk_in_free_list(fc),
7754 "chunk should not be in free lists yet");
7755 }
7756 if (CMSTraceSweeper) {
7757 gclog_or_tty->print_cr(" -- add free block " PTR_FORMAT " (" SIZE_FORMAT ") to free lists",
7758 p2i(chunk), size);
7759 }
7760 // A new free range is going to be starting. The current
7761 // free range has not been added to the free lists yet or
7762 // was removed so add it back.
7763 // If the current free range was coalesced, then the death
7764 // of the free range was recorded. Record a birth now.
7765 if (lastFreeRangeCoalesced()) {
7766 _sp->coalBirth(size);
7767 }
7768 _sp->addChunkAndRepairOffsetTable(chunk, size,
7769 lastFreeRangeCoalesced());
7770 } else if (CMSTraceSweeper) {
7771 gclog_or_tty->print_cr("Already in free list: nothing to flush");
7772 }
7773 set_inFreeRange(false);
7774 set_freeRangeInFreeLists(false);
7775 }
7776
7777 // We take a break if we've been at this for a while,
7778 // so as to avoid monopolizing the locks involved.
7779 void SweepClosure::do_yield_work(HeapWord* addr) {
7780 // Return current free chunk being used for coalescing (if any)
7781 // to the appropriate freelist. After yielding, the next
7782 // free block encountered will start a coalescing range of
7783 // free blocks. If the next free block is adjacent to the
7784 // chunk just flushed, they will need to wait for the next
7785 // sweep to be coalesced.
7786 if (inFreeRange()) {
7787 flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger()));
7788 }
7789
7790 // First give up the locks, then yield, then re-lock.
7791 // We should probably use a constructor/destructor idiom to
7792 // do this unlock/lock or modify the MutexUnlocker class to
7793 // serve our purpose. XXX
7794 assert_lock_strong(_bitMap->lock());
7795 assert_lock_strong(_freelistLock);
7796 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7797 "CMS thread should hold CMS token");
7798 _bitMap->lock()->unlock();
7799 _freelistLock->unlock();
7800 ConcurrentMarkSweepThread::desynchronize(true);
7801 _collector->stopTimer();
7802 if (PrintCMSStatistics != 0) {
7803 _collector->incrementYields();
7804 }
7805
7806 // See the comment in coordinator_yield()
7807 for (unsigned i = 0; i < CMSYieldSleepCount &&
7808 ConcurrentMarkSweepThread::should_yield() &&
7809 !CMSCollector::foregroundGCIsActive(); ++i) {
7810 os::sleep(Thread::current(), 1, false);
7811 }
7812
7813 ConcurrentMarkSweepThread::synchronize(true);
7814 _freelistLock->lock();
7815 _bitMap->lock()->lock_without_safepoint_check();
7816 _collector->startTimer();
7817 }
7818
7819 #ifndef PRODUCT
7820 // This is actually very useful in a product build if it can
7821 // be called from the debugger. Compile it into the product
7822 // as needed.
7823 bool debug_verify_chunk_in_free_list(FreeChunk* fc) {
7824 return debug_cms_space->verify_chunk_in_free_list(fc);
7825 }
7826 #endif
7827
7828 void SweepClosure::print_free_block_coalesced(FreeChunk* fc) const {
7829 if (CMSTraceSweeper) {
7830 gclog_or_tty->print_cr("Sweep:coal_free_blk " PTR_FORMAT " (" SIZE_FORMAT ")",
7831 p2i(fc), fc->size());
7832 }
7833 }
7834
7835 // CMSIsAliveClosure
7836 bool CMSIsAliveClosure::do_object_b(oop obj) {
7837 HeapWord* addr = (HeapWord*)obj;
7838 return addr != NULL &&
7839 (!_span.contains(addr) || _bit_map->isMarked(addr));
7840 }
7841
7842
7843 CMSKeepAliveClosure::CMSKeepAliveClosure( CMSCollector* collector,
7844 MemRegion span,
7845 CMSBitMap* bit_map, CMSMarkStack* mark_stack,
7846 bool cpc):
7847 _collector(collector),
7848 _span(span),
7849 _bit_map(bit_map),
7850 _mark_stack(mark_stack),
7851 _concurrent_precleaning(cpc) {
7852 assert(!_span.is_empty(), "Empty span could spell trouble");
|
36 #include "gc/cms/vmCMSOperations.hpp"
37 #include "gc/serial/genMarkSweep.hpp"
38 #include "gc/serial/tenuredGeneration.hpp"
39 #include "gc/shared/adaptiveSizePolicy.hpp"
40 #include "gc/shared/cardGeneration.inline.hpp"
41 #include "gc/shared/cardTableRS.hpp"
42 #include "gc/shared/collectedHeap.inline.hpp"
43 #include "gc/shared/collectorCounters.hpp"
44 #include "gc/shared/collectorPolicy.hpp"
45 #include "gc/shared/gcLocker.inline.hpp"
46 #include "gc/shared/gcPolicyCounters.hpp"
47 #include "gc/shared/gcTimer.hpp"
48 #include "gc/shared/gcTrace.hpp"
49 #include "gc/shared/gcTraceTime.hpp"
50 #include "gc/shared/genCollectedHeap.hpp"
51 #include "gc/shared/genOopClosures.inline.hpp"
52 #include "gc/shared/isGCActiveMark.hpp"
53 #include "gc/shared/referencePolicy.hpp"
54 #include "gc/shared/strongRootsScope.hpp"
55 #include "gc/shared/taskqueue.inline.hpp"
56 #include "logging/log.hpp"
57 #include "memory/allocation.hpp"
58 #include "memory/iterator.inline.hpp"
59 #include "memory/padded.hpp"
60 #include "memory/resourceArea.hpp"
61 #include "oops/oop.inline.hpp"
62 #include "prims/jvmtiExport.hpp"
63 #include "runtime/atomic.inline.hpp"
64 #include "runtime/globals_extension.hpp"
65 #include "runtime/handles.inline.hpp"
66 #include "runtime/java.hpp"
67 #include "runtime/orderAccess.inline.hpp"
68 #include "runtime/vmThread.hpp"
69 #include "services/memoryService.hpp"
70 #include "services/runtimeService.hpp"
71 #include "utilities/stack.inline.hpp"
72
73 // statics
74 CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;
75 bool CMSCollector::_full_gc_requested = false;
76 GCCause::Cause CMSCollector::_full_gc_cause = GCCause::_no_gc;
350 // young generation collection.
351 double CMSStats::time_until_cms_gen_full() const {
352 size_t cms_free = _cms_gen->cmsSpace()->free();
353 GenCollectedHeap* gch = GenCollectedHeap::heap();
354 size_t expected_promotion = MIN2(gch->young_gen()->capacity(),
355 (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average());
356 if (cms_free > expected_promotion) {
357 // Start a cms collection if there isn't enough space to promote
358 // for the next young collection. Use the padded average as
359 // a safety factor.
360 cms_free -= expected_promotion;
361
362 // Adjust by the safety factor.
363 double cms_free_dbl = (double)cms_free;
364 double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor) / 100.0;
365 // Apply a further correction factor which tries to adjust
366 // for recent occurance of concurrent mode failures.
367 cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free);
368 cms_free_dbl = cms_free_dbl * cms_adjustment;
369
370 log_trace(gc, stats)("CMSStats::time_until_cms_gen_full: cms_free " SIZE_FORMAT " expected_promotion " SIZE_FORMAT,
371 cms_free, expected_promotion);
372 log_trace(gc, stats)(" cms_free_dbl %f cms_consumption_rate %f",
373 cms_free_dbl, cms_consumption_rate() + 1.0);
374 // Add 1 in case the consumption rate goes to zero.
375 return cms_free_dbl / (cms_consumption_rate() + 1.0);
376 }
377 return 0.0;
378 }
379
380 // Compare the duration of the cms collection to the
381 // time remaining before the cms generation is empty.
382 // Note that the time from the start of the cms collection
383 // to the start of the cms sweep (less than the total
384 // duration of the cms collection) can be used. This
385 // has been tried and some applications experienced
386 // promotion failures early in execution. This was
387 // possibly because the averages were not accurate
388 // enough at the beginning.
389 double CMSStats::time_until_cms_start() const {
390 // We add "gc0_period" to the "work" calculation
391 // below because this query is done (mostly) at the
392 // end of a scavenge, so we need to conservatively
393 // account for that much possible delay
394 // in the query so as to avoid concurrent mode failures
395 // due to starting the collection just a wee bit too
396 // late.
397 double work = cms_duration() + gc0_period();
398 double deadline = time_until_cms_gen_full();
399 // If a concurrent mode failure occurred recently, we want to be
400 // more conservative and halve our expected time_until_cms_gen_full()
401 if (work > deadline) {
402 log_develop(gc)("CMSCollector: collect because of anticipated promotion before full %3.7f + %3.7f > %3.7f ",
403 cms_duration(), gc0_period(), time_until_cms_gen_full());
404 return 0.0;
405 }
406 return work - deadline;
407 }
408
409 #ifndef PRODUCT
410 void CMSStats::print_on(outputStream *st) const {
411 st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);
412 st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,
413 gc0_duration(), gc0_period(), gc0_promoted());
414 st->print(",cms_dur=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,
415 cms_duration(), cms_period(), cms_allocated());
416 st->print(",cms_since_beg=%g,cms_since_end=%g",
417 cms_time_since_begin(), cms_time_since_end());
418 st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,
419 _cms_used_at_gc0_begin, _cms_used_at_gc0_end);
420
421 if (valid()) {
422 st->print(",promo_rate=%g,cms_alloc_rate=%g",
423 promotion_rate(), cms_allocation_rate());
645 //
646 void ConcurrentMarkSweepGeneration::update_counters(size_t used) {
647 if (UsePerfData) {
648 _space_counters->update_used(used);
649 _space_counters->update_capacity();
650 _gen_counters->update_all();
651 }
652 }
653
654 void ConcurrentMarkSweepGeneration::print() const {
655 Generation::print();
656 cmsSpace()->print();
657 }
658
659 #ifndef PRODUCT
660 void ConcurrentMarkSweepGeneration::print_statistics() {
661 cmsSpace()->printFLCensus(0);
662 }
663 #endif
664
665 size_t
666 ConcurrentMarkSweepGeneration::contiguous_available() const {
667 // dld proposes an improvement in precision here. If the committed
668 // part of the space ends in a free block we should add that to
669 // uncommitted size in the calculation below. Will make this
670 // change later, staying with the approximation below for the
671 // time being. -- ysr.
672 return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
673 }
674
675 size_t
676 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
677 return _cmsSpace->max_alloc_in_words() * HeapWordSize;
678 }
679
680 size_t ConcurrentMarkSweepGeneration::max_available() const {
681 return free() + _virtual_space.uncommitted_size();
682 }
683
684 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
685 size_t available = max_available();
686 size_t av_promo = (size_t)gc_stats()->avg_promoted()->padded_average();
687 bool res = (available >= av_promo) || (available >= max_promotion_in_bytes);
688 log_trace(gc, promotion)("CMS: promo attempt is%s safe: available(" SIZE_FORMAT ") %s av_promo(" SIZE_FORMAT "), max_promo(" SIZE_FORMAT ")",
689 res? "":" not", available, res? ">=":"<", av_promo, max_promotion_in_bytes);
690 return res;
691 }
692
693 // At a promotion failure dump information on block layout in heap
694 // (cms old generation).
695 void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {
696 LogHandle(gc, promotion) log;
697 if (log.is_trace()) {
698 ResourceMark rm;
699 cmsSpace()->dump_at_safepoint_with_locks(collector(), log.trace_stream());
700 }
701 }
702
703 void ConcurrentMarkSweepGeneration::reset_after_compaction() {
704 // Clear the promotion information. These pointers can be adjusted
705 // along with all the other pointers into the heap but
706 // compaction is expected to be a rare event with
707 // a heap using cms so don't do it without seeing the need.
708 for (uint i = 0; i < ParallelGCThreads; i++) {
709 _par_gc_thread_states[i]->promo.reset();
710 }
711 }
712
713 void ConcurrentMarkSweepGeneration::compute_new_size() {
714 assert_locked_or_safepoint(Heap_lock);
715
716 // If incremental collection failed, we just want to expand
717 // to the limit.
718 if (incremental_collection_failed()) {
719 clear_incremental_collection_failed();
735 void ConcurrentMarkSweepGeneration::compute_new_size_free_list() {
736 assert_locked_or_safepoint(Heap_lock);
737
738 // If incremental collection failed, we just want to expand
739 // to the limit.
740 if (incremental_collection_failed()) {
741 clear_incremental_collection_failed();
742 grow_to_reserved();
743 return;
744 }
745
746 double free_percentage = ((double) free()) / capacity();
747 double desired_free_percentage = (double) MinHeapFreeRatio / 100;
748 double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
749
750 // compute expansion delta needed for reaching desired free percentage
751 if (free_percentage < desired_free_percentage) {
752 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
753 assert(desired_capacity >= capacity(), "invalid expansion size");
754 size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
755 LogHandle(gc) log;
756 if (log.is_trace()) {
757 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
758 log.trace("From compute_new_size: ");
759 log.trace(" Free fraction %f", free_percentage);
760 log.trace(" Desired free fraction %f", desired_free_percentage);
761 log.trace(" Maximum free fraction %f", maximum_free_percentage);
762 log.trace(" Capacity " SIZE_FORMAT, capacity() / 1000);
763 log.trace(" Desired capacity " SIZE_FORMAT, desired_capacity / 1000);
764 GenCollectedHeap* gch = GenCollectedHeap::heap();
765 assert(gch->is_old_gen(this), "The CMS generation should always be the old generation");
766 size_t young_size = gch->young_gen()->capacity();
767 log.trace(" Young gen size " SIZE_FORMAT, young_size / 1000);
768 log.trace(" unsafe_max_alloc_nogc " SIZE_FORMAT, unsafe_max_alloc_nogc() / 1000);
769 log.trace(" contiguous available " SIZE_FORMAT, contiguous_available() / 1000);
770 log.trace(" Expand by " SIZE_FORMAT " (bytes)", expand_bytes);
771 }
772 // safe if expansion fails
773 expand_for_gc_cause(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
774 log.trace(" Expanded free fraction %f", ((double) free()) / capacity());
775 } else {
776 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
777 assert(desired_capacity <= capacity(), "invalid expansion size");
778 size_t shrink_bytes = capacity() - desired_capacity;
779 // Don't shrink unless the delta is greater than the minimum shrink we want
780 if (shrink_bytes >= MinHeapDeltaBytes) {
781 shrink_free_list_by(shrink_bytes);
782 }
783 }
784 }
785
786 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
787 return cmsSpace()->freelistLock();
788 }
789
790 HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size, bool tlab) {
791 CMSSynchronousYieldRequest yr;
792 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
793 return have_lock_and_allocate(size, tlab);
794 }
1092 ConcurrentMarkSweepGeneration::
1093 par_oop_since_save_marks_iterate_done(int thread_num) {
1094 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1095 ParScanWithoutBarrierClosure* dummy_cl = NULL;
1096 ps->promo.promoted_oops_iterate_nv(dummy_cl);
1097 }
1098
1099 bool ConcurrentMarkSweepGeneration::should_collect(bool full,
1100 size_t size,
1101 bool tlab)
1102 {
1103 // We allow a STW collection only if a full
1104 // collection was requested.
1105 return full || should_allocate(size, tlab); // FIX ME !!!
1106 // This and promotion failure handling are connected at the
1107 // hip and should be fixed by untying them.
1108 }
1109
1110 bool CMSCollector::shouldConcurrentCollect() {
1111 if (_full_gc_requested) {
1112 log_trace(gc)("CMSCollector: collect because of explicit gc request (or gc_locker)");
1113 return true;
1114 }
1115
1116 FreelistLocker x(this);
1117 // ------------------------------------------------------------------
1118 // Print out lots of information which affects the initiation of
1119 // a collection.
1120 LogHandle(gc, stats) log;
1121 if (log.is_debug() && stats().valid()) {
1122 log.debug("CMSCollector shouldConcurrentCollect: ");
1123 ResourceMark rm;
1124 stats().print_on(log.debug_stream());
1125 log.debug("time_until_cms_gen_full %3.7f", stats().time_until_cms_gen_full());
1126 log.debug("free=" SIZE_FORMAT, _cmsGen->free());
1127 log.debug("contiguous_available=" SIZE_FORMAT, _cmsGen->contiguous_available());
1128 log.debug("promotion_rate=%g", stats().promotion_rate());
1129 log.debug("cms_allocation_rate=%g", stats().cms_allocation_rate());
1130 log.debug("occupancy=%3.7f", _cmsGen->occupancy());
1131 log.debug("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy());
1132 log.debug("cms_time_since_begin=%3.7f", stats().cms_time_since_begin());
1133 log.debug("cms_time_since_end=%3.7f", stats().cms_time_since_end());
1134 log.debug("metadata initialized %d", MetaspaceGC::should_concurrent_collect());
1135 }
1136 // ------------------------------------------------------------------
1137
1138 // If the estimated time to complete a cms collection (cms_duration())
1139 // is less than the estimated time remaining until the cms generation
1140 // is full, start a collection.
1141 if (!UseCMSInitiatingOccupancyOnly) {
1142 if (stats().valid()) {
1143 if (stats().time_until_cms_start() == 0.0) {
1144 return true;
1145 }
1146 } else {
1147 // We want to conservatively collect somewhat early in order
1148 // to try and "bootstrap" our CMS/promotion statistics;
1149 // this branch will not fire after the first successful CMS
1150 // collection because the stats should then be valid.
1151 if (_cmsGen->occupancy() >= _bootstrap_occupancy) {
1152 log_trace(gc)(" CMSCollector: collect for bootstrapping statistics: occupancy = %f, boot occupancy = %f",
1153 _cmsGen->occupancy(), _bootstrap_occupancy);
1154 return true;
1155 }
1156 }
1157 }
1158
1159 // Otherwise, we start a collection cycle if
1160 // old gen want a collection cycle started. Each may use
1161 // an appropriate criterion for making this decision.
1162 // XXX We need to make sure that the gen expansion
1163 // criterion dovetails well with this. XXX NEED TO FIX THIS
1164 if (_cmsGen->should_concurrent_collect()) {
1165 log_trace(gc)("CMS old gen initiated");
1166 return true;
1167 }
1168
1169 // We start a collection if we believe an incremental collection may fail;
1170 // this is not likely to be productive in practice because it's probably too
1171 // late anyway.
1172 GenCollectedHeap* gch = GenCollectedHeap::heap();
1173 assert(gch->collector_policy()->is_generation_policy(),
1174 "You may want to check the correctness of the following");
1175 if (gch->incremental_collection_will_fail(true /* consult_young */)) {
1176 log_trace(gc)("CMSCollector: collect because incremental collection will fail ");
1177 return true;
1178 }
1179
1180 if (MetaspaceGC::should_concurrent_collect()) {
1181 log_trace(gc)("CMSCollector: collect for metadata allocation ");
1182 return true;
1183 }
1184
1185 // CMSTriggerInterval starts a CMS cycle if enough time has passed.
1186 if (CMSTriggerInterval >= 0) {
1187 if (CMSTriggerInterval == 0) {
1188 // Trigger always
1189 return true;
1190 }
1191
1192 // Check the CMS time since begin (we do not check the stats validity
1193 // as we want to be able to trigger the first CMS cycle as well)
1194 if (stats().cms_time_since_begin() >= (CMSTriggerInterval / ((double) MILLIUNITS))) {
1195 if (stats().valid()) {
1196 log_trace(gc)("CMSCollector: collect because of trigger interval (time since last begin %3.7f secs)",
1197 stats().cms_time_since_begin());
1198 } else {
1199 log_trace(gc)("CMSCollector: collect because of trigger interval (first collection)");
1200 }
1201 return true;
1202 }
1203 }
1204
1205 return false;
1206 }
1207
1208 void CMSCollector::set_did_compact(bool v) { _cmsGen->set_did_compact(v); }
1209
1210 // Clear _expansion_cause fields of constituent generations
1211 void CMSCollector::clear_expansion_cause() {
1212 _cmsGen->clear_expansion_cause();
1213 }
1214
1215 // We should be conservative in starting a collection cycle. To
1216 // start too eagerly runs the risk of collecting too often in the
1217 // extreme. To collect too rarely falls back on full collections,
1218 // which works, even if not optimum in terms of concurrent work.
1219 // As a work around for too eagerly collecting, use the flag
1222 // collections.
1223 // We want to start a new collection cycle if any of the following
1224 // conditions hold:
1225 // . our current occupancy exceeds the configured initiating occupancy
1226 // for this generation, or
1227 // . we recently needed to expand this space and have not, since that
1228 // expansion, done a collection of this generation, or
1229 // . the underlying space believes that it may be a good idea to initiate
1230 // a concurrent collection (this may be based on criteria such as the
1231 // following: the space uses linear allocation and linear allocation is
1232 // going to fail, or there is believed to be excessive fragmentation in
1233 // the generation, etc... or ...
1234 // [.(currently done by CMSCollector::shouldConcurrentCollect() only for
1235 // the case of the old generation; see CR 6543076):
1236 // we may be approaching a point at which allocation requests may fail because
1237 // we will be out of sufficient free space given allocation rate estimates.]
1238 bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const {
1239
1240 assert_lock_strong(freelistLock());
1241 if (occupancy() > initiating_occupancy()) {
1242 log_trace(gc)(" %s: collect because of occupancy %f / %f ",
1243 short_name(), occupancy(), initiating_occupancy());
1244 return true;
1245 }
1246 if (UseCMSInitiatingOccupancyOnly) {
1247 return false;
1248 }
1249 if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) {
1250 log_trace(gc)(" %s: collect because expanded for allocation ", short_name());
1251 return true;
1252 }
1253 return false;
1254 }
1255
1256 void ConcurrentMarkSweepGeneration::collect(bool full,
1257 bool clear_all_soft_refs,
1258 size_t size,
1259 bool tlab)
1260 {
1261 collector()->collect(full, clear_all_soft_refs, size, tlab);
1262 }
1263
1264 void CMSCollector::collect(bool full,
1265 bool clear_all_soft_refs,
1266 size_t size,
1267 bool tlab)
1268 {
1269 // The following "if" branch is present for defensive reasons.
1270 // In the current uses of this interface, it can be replaced with:
1287 GenCollectedHeap* gch = GenCollectedHeap::heap();
1288 unsigned int gc_count = gch->total_full_collections();
1289 if (gc_count == full_gc_count) {
1290 MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag);
1291 _full_gc_requested = true;
1292 _full_gc_cause = cause;
1293 CGC_lock->notify(); // nudge CMS thread
1294 } else {
1295 assert(gc_count > full_gc_count, "Error: causal loop");
1296 }
1297 }
1298
1299 bool CMSCollector::is_external_interruption() {
1300 GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause();
1301 return GCCause::is_user_requested_gc(cause) ||
1302 GCCause::is_serviceability_requested_gc(cause);
1303 }
1304
1305 void CMSCollector::report_concurrent_mode_interruption() {
1306 if (is_external_interruption()) {
1307 log_debug(gc)("Concurrent mode interrupted");
1308 } else {
1309 log_debug(gc)("Concurrent mode failure");
1310 _gc_tracer_cm->report_concurrent_mode_failure();
1311 }
1312 }
1313
1314
1315 // The foreground and background collectors need to coordinate in order
1316 // to make sure that they do not mutually interfere with CMS collections.
1317 // When a background collection is active,
1318 // the foreground collector may need to take over (preempt) and
1319 // synchronously complete an ongoing collection. Depending on the
1320 // frequency of the background collections and the heap usage
1321 // of the application, this preemption can be seldom or frequent.
1322 // There are only certain
1323 // points in the background collection that the "collection-baton"
1324 // can be passed to the foreground collector.
1325 //
1326 // The foreground collector will wait for the baton before
1327 // starting any part of the collection. The foreground collector
1328 // will only wait at one location.
1329 //
1423 CGC_lock->notify();
1424 assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(),
1425 "Possible deadlock");
1426 while (_foregroundGCShouldWait) {
1427 // wait for notification
1428 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1429 // Possibility of delay/starvation here, since CMS token does
1430 // not know to give priority to VM thread? Actually, i think
1431 // there wouldn't be any delay/starvation, but the proof of
1432 // that "fact" (?) appears non-trivial. XXX 20011219YSR
1433 }
1434 ConcurrentMarkSweepThread::set_CMS_flag(
1435 ConcurrentMarkSweepThread::CMS_vm_has_token);
1436 }
1437 }
1438 // The CMS_token is already held. Get back the other locks.
1439 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1440 "VM thread should have CMS token");
1441 getFreelistLocks();
1442 bitMapLock()->lock_without_safepoint_check();
1443 log_debug(gc, state)("CMS foreground collector has asked for control " INTPTR_FORMAT " with first state %d",
1444 p2i(Thread::current()), first_state);
1445 log_debug(gc, state)(" gets control with state %d", _collectorState);
1446
1447 // Inform cms gen if this was due to partial collection failing.
1448 // The CMS gen may use this fact to determine its expansion policy.
1449 GenCollectedHeap* gch = GenCollectedHeap::heap();
1450 if (gch->incremental_collection_will_fail(false /* don't consult_young */)) {
1451 assert(!_cmsGen->incremental_collection_failed(),
1452 "Should have been noticed, reacted to and cleared");
1453 _cmsGen->set_incremental_collection_failed();
1454 }
1455
1456 if (first_state > Idling) {
1457 report_concurrent_mode_interruption();
1458 }
1459
1460 set_did_compact(true);
1461
1462 // If the collection is being acquired from the background
1463 // collector, there may be references on the discovered
1464 // references lists. Abandon those references, since some
1465 // of them may have become unreachable after concurrent
1499 // after obtaining the free list locks for the
1500 // two generations.
1501 void CMSCollector::compute_new_size() {
1502 assert_locked_or_safepoint(Heap_lock);
1503 FreelistLocker z(this);
1504 MetaspaceGC::compute_new_size();
1505 _cmsGen->compute_new_size_free_list();
1506 }
1507
1508 // A work method used by the foreground collector to do
1509 // a mark-sweep-compact.
1510 void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
1511 GenCollectedHeap* gch = GenCollectedHeap::heap();
1512
1513 STWGCTimer* gc_timer = GenMarkSweep::gc_timer();
1514 gc_timer->register_gc_start();
1515
1516 SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer();
1517 gc_tracer->report_gc_start(gch->gc_cause(), gc_timer->gc_start());
1518
1519 GCTraceTime(Trace, gc) t("CMS:MSC");
1520
1521 // Temporarily widen the span of the weak reference processing to
1522 // the entire heap.
1523 MemRegion new_span(GenCollectedHeap::heap()->reserved_region());
1524 ReferenceProcessorSpanMutator rp_mut_span(ref_processor(), new_span);
1525 // Temporarily, clear the "is_alive_non_header" field of the
1526 // reference processor.
1527 ReferenceProcessorIsAliveMutator rp_mut_closure(ref_processor(), NULL);
1528 // Temporarily make reference _processing_ single threaded (non-MT).
1529 ReferenceProcessorMTProcMutator rp_mut_mt_processing(ref_processor(), false);
1530 // Temporarily make refs discovery atomic
1531 ReferenceProcessorAtomicMutator rp_mut_atomic(ref_processor(), true);
1532 // Temporarily make reference _discovery_ single threaded (non-MT)
1533 ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false);
1534
1535 ref_processor()->set_enqueuing_is_done(false);
1536 ref_processor()->enable_discovery();
1537 ref_processor()->setup_policy(clear_all_soft_refs);
1538 // If an asynchronous collection finishes, the _modUnionTable is
1539 // all clear. If we are assuming the collection from an asynchronous
1584 // Clear any data recorded in the PLAB chunk arrays.
1585 if (_survivor_plab_array != NULL) {
1586 reset_survivor_plab_arrays();
1587 }
1588
1589 // Adjust the per-size allocation stats for the next epoch.
1590 _cmsGen->cmsSpace()->endSweepFLCensus(sweep_count() /* fake */);
1591 // Restart the "inter sweep timer" for the next epoch.
1592 _inter_sweep_timer.reset();
1593 _inter_sweep_timer.start();
1594
1595 gc_timer->register_gc_end();
1596
1597 gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions());
1598
1599 // For a mark-sweep-compact, compute_new_size() will be called
1600 // in the heap's do_collection() method.
1601 }
1602
1603 void CMSCollector::print_eden_and_survivor_chunk_arrays() {
1604 LogHandle(gc, heap) log;
1605 if (!log.is_trace()) {
1606 return;
1607 }
1608
1609 ContiguousSpace* eden_space = _young_gen->eden();
1610 ContiguousSpace* from_space = _young_gen->from();
1611 ContiguousSpace* to_space = _young_gen->to();
1612 // Eden
1613 if (_eden_chunk_array != NULL) {
1614 log.trace("eden " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1615 p2i(eden_space->bottom()), p2i(eden_space->top()),
1616 p2i(eden_space->end()), eden_space->capacity());
1617 log.trace("_eden_chunk_index=" SIZE_FORMAT ", _eden_chunk_capacity=" SIZE_FORMAT,
1618 _eden_chunk_index, _eden_chunk_capacity);
1619 for (size_t i = 0; i < _eden_chunk_index; i++) {
1620 log.trace("_eden_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT, i, p2i(_eden_chunk_array[i]));
1621 }
1622 }
1623 // Survivor
1624 if (_survivor_chunk_array != NULL) {
1625 log.trace("survivor " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1626 p2i(from_space->bottom()), p2i(from_space->top()),
1627 p2i(from_space->end()), from_space->capacity());
1628 log.trace("_survivor_chunk_index=" SIZE_FORMAT ", _survivor_chunk_capacity=" SIZE_FORMAT,
1629 _survivor_chunk_index, _survivor_chunk_capacity);
1630 for (size_t i = 0; i < _survivor_chunk_index; i++) {
1631 log.trace("_survivor_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT, i, p2i(_survivor_chunk_array[i]));
1632 }
1633 }
1634 }
1635
1636 void CMSCollector::getFreelistLocks() const {
1637 // Get locks for all free lists in all generations that this
1638 // collector is responsible for
1639 _cmsGen->freelistLock()->lock_without_safepoint_check();
1640 }
1641
1642 void CMSCollector::releaseFreelistLocks() const {
1643 // Release locks for all free lists in all generations that this
1644 // collector is responsible for
1645 _cmsGen->freelistLock()->unlock();
1646 }
1647
1648 bool CMSCollector::haveFreelistLocks() const {
1649 // Check locks for all free lists in all generations that this
1650 // collector is responsible for
1651 assert_lock_strong(_cmsGen->freelistLock());
1700 _collectorState = InitialMarking;
1701 register_gc_start(cause);
1702 // Reset the expansion cause, now that we are about to begin
1703 // a new cycle.
1704 clear_expansion_cause();
1705
1706 // Clear the MetaspaceGC flag since a concurrent collection
1707 // is starting but also clear it after the collection.
1708 MetaspaceGC::set_should_concurrent_collect(false);
1709 }
1710 // Decide if we want to enable class unloading as part of the
1711 // ensuing concurrent GC cycle.
1712 update_should_unload_classes();
1713 _full_gc_requested = false; // acks all outstanding full gc requests
1714 _full_gc_cause = GCCause::_no_gc;
1715 // Signal that we are about to start a collection
1716 gch->increment_total_full_collections(); // ... starting a collection cycle
1717 _collection_count_start = gch->total_full_collections();
1718 }
1719
1720 size_t prev_used = _cmsGen->used();
1721
1722 // The change of the collection state is normally done at this level;
1723 // the exceptions are phases that are executed while the world is
1724 // stopped. For those phases the change of state is done while the
1725 // world is stopped. For baton passing purposes this allows the
1726 // background collector to finish the phase and change state atomically.
1727 // The foreground collector cannot wait on a phase that is done
1728 // while the world is stopped because the foreground collector already
1729 // has the world stopped and would deadlock.
1730 while (_collectorState != Idling) {
1731 log_debug(gc, state)("Thread " INTPTR_FORMAT " in CMS state %d",
1732 p2i(Thread::current()), _collectorState);
1733 // The foreground collector
1734 // holds the Heap_lock throughout its collection.
1735 // holds the CMS token (but not the lock)
1736 // except while it is waiting for the background collector to yield.
1737 //
1738 // The foreground collector should be blocked (not for long)
1739 // if the background collector is about to start a phase
1740 // executed with world stopped. If the background
1741 // collector has already started such a phase, the
1742 // foreground collector is blocked waiting for the
1743 // Heap_lock. The stop-world phases (InitialMarking and FinalMarking)
1744 // are executed in the VM thread.
1745 //
1746 // The locking order is
1747 // PendingListLock (PLL) -- if applicable (FinalMarking)
1748 // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue())
1749 // CMS token (claimed in
1750 // stop_world_and_do() -->
1751 // safepoint_synchronize() -->
1752 // CMSThread::synchronize())
1753
1754 {
1755 // Check if the FG collector wants us to yield.
1756 CMSTokenSync x(true); // is cms thread
1757 if (waitForForegroundGC()) {
1758 // We yielded to a foreground GC, nothing more to be
1759 // done this round.
1760 assert(_foregroundGCShouldWait == false, "We set it to false in "
1761 "waitForForegroundGC()");
1762 log_debug(gc, state)("CMS Thread " INTPTR_FORMAT " exiting collection CMS state %d",
1763 p2i(Thread::current()), _collectorState);
1764 return;
1765 } else {
1766 // The background collector can run but check to see if the
1767 // foreground collector has done a collection while the
1768 // background collector was waiting to get the CGC_lock
1769 // above. If yes, break so that _foregroundGCShouldWait
1770 // is cleared before returning.
1771 if (_collectorState == Idling) {
1772 break;
1773 }
1774 }
1775 }
1776
1777 assert(_foregroundGCShouldWait, "Foreground collector, if active, "
1778 "should be waiting");
1779
1780 switch (_collectorState) {
1781 case InitialMarking:
1782 {
1783 ReleaseForegroundGC x(this);
1847 break;
1848 }
1849 case Resetting:
1850 // CMS heap resizing has been completed
1851 reset_concurrent();
1852 assert(_collectorState == Idling, "Collector state should "
1853 "have changed");
1854
1855 MetaspaceGC::set_should_concurrent_collect(false);
1856
1857 stats().record_cms_end();
1858 // Don't move the concurrent_phases_end() and compute_new_size()
1859 // calls to here because a preempted background collection
1860 // has it's state set to "Resetting".
1861 break;
1862 case Idling:
1863 default:
1864 ShouldNotReachHere();
1865 break;
1866 }
1867 log_debug(gc, state)(" Thread " INTPTR_FORMAT " done - next CMS state %d",
1868 p2i(Thread::current()), _collectorState);
1869 assert(_foregroundGCShouldWait, "block post-condition");
1870 }
1871
1872 // Should this be in gc_epilogue?
1873 collector_policy()->counters()->update_counters();
1874
1875 {
1876 // Clear _foregroundGCShouldWait and, in the event that the
1877 // foreground collector is waiting, notify it, before
1878 // returning.
1879 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1880 _foregroundGCShouldWait = false;
1881 if (_foregroundGCIsActive) {
1882 CGC_lock->notify();
1883 }
1884 assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1885 "Possible deadlock");
1886 }
1887 log_debug(gc, state)("CMS Thread " INTPTR_FORMAT " exiting collection CMS state %d",
1888 p2i(Thread::current()), _collectorState);
1889 log_info(gc, heap)("Old: " SIZE_FORMAT "K->" SIZE_FORMAT "K(" SIZE_FORMAT "K)",
1890 prev_used / K, _cmsGen->used()/K, _cmsGen->capacity() /K);
1891 }
1892
1893 void CMSCollector::register_gc_start(GCCause::Cause cause) {
1894 _cms_start_registered = true;
1895 _gc_timer_cm->register_gc_start();
1896 _gc_tracer_cm->report_gc_start(cause, _gc_timer_cm->gc_start());
1897 }
1898
1899 void CMSCollector::register_gc_end() {
1900 if (_cms_start_registered) {
1901 report_heap_summary(GCWhen::AfterGC);
1902
1903 _gc_timer_cm->register_gc_end();
1904 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
1905 _cms_start_registered = false;
1906 }
1907 }
1908
1909 void CMSCollector::save_heap_summary() {
1910 GenCollectedHeap* gch = GenCollectedHeap::heap();
1922 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1923 "CMS thread should have CMS token");
1924 // Block the foreground collector until the
1925 // background collectors decides whether to
1926 // yield.
1927 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1928 _foregroundGCShouldWait = true;
1929 if (_foregroundGCIsActive) {
1930 // The background collector yields to the
1931 // foreground collector and returns a value
1932 // indicating that it has yielded. The foreground
1933 // collector can proceed.
1934 res = true;
1935 _foregroundGCShouldWait = false;
1936 ConcurrentMarkSweepThread::clear_CMS_flag(
1937 ConcurrentMarkSweepThread::CMS_cms_has_token);
1938 ConcurrentMarkSweepThread::set_CMS_flag(
1939 ConcurrentMarkSweepThread::CMS_cms_wants_token);
1940 // Get a possibly blocked foreground thread going
1941 CGC_lock->notify();
1942 log_debug(gc, state)("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d",
1943 p2i(Thread::current()), _collectorState);
1944 while (_foregroundGCIsActive) {
1945 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1946 }
1947 ConcurrentMarkSweepThread::set_CMS_flag(
1948 ConcurrentMarkSweepThread::CMS_cms_has_token);
1949 ConcurrentMarkSweepThread::clear_CMS_flag(
1950 ConcurrentMarkSweepThread::CMS_cms_wants_token);
1951 }
1952 log_debug(gc, state)("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d",
1953 p2i(Thread::current()), _collectorState);
1954 return res;
1955 }
1956
1957 // Because of the need to lock the free lists and other structures in
1958 // the collector, common to all the generations that the collector is
1959 // collecting, we need the gc_prologues of individual CMS generations
1960 // delegate to their collector. It may have been simpler had the
1961 // current infrastructure allowed one to call a prologue on a
1962 // collector. In the absence of that we have the generation's
1963 // prologue delegate to the collector, which delegates back
1964 // some "local" work to a worker method in the individual generations
1965 // that it's responsible for collecting, while itself doing any
1966 // work common to all generations it's responsible for. A similar
1967 // comment applies to the gc_epilogue()'s.
1968 // The role of the variable _between_prologue_and_epilogue is to
1969 // enforce the invocation protocol.
1970 void CMSCollector::gc_prologue(bool full) {
1971 // Call gc_prologue_work() for the CMSGen
1972 // we are responsible for.
1973
2030 // collecting.
2031 collector()->gc_prologue(full);
2032 }
2033
2034 // This is a "private" interface for use by this generation's CMSCollector.
2035 // Not to be called directly by any other entity (for instance,
2036 // GenCollectedHeap, which calls the "public" gc_prologue method above).
2037 void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
2038 bool registerClosure, ModUnionClosure* modUnionClosure) {
2039 assert(!incremental_collection_failed(), "Shouldn't be set yet");
2040 assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,
2041 "Should be NULL");
2042 if (registerClosure) {
2043 cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure);
2044 }
2045 cmsSpace()->gc_prologue();
2046 // Clear stat counters
2047 NOT_PRODUCT(
2048 assert(_numObjectsPromoted == 0, "check");
2049 assert(_numWordsPromoted == 0, "check");
2050 log_develop(gc, alloc)("Allocated " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes concurrently",
2051 _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord));
2052 _numObjectsAllocated = 0;
2053 _numWordsAllocated = 0;
2054 )
2055 }
2056
2057 void CMSCollector::gc_epilogue(bool full) {
2058 // The following locking discipline assumes that we are only called
2059 // when the world is stopped.
2060 assert(SafepointSynchronize::is_at_safepoint(),
2061 "world is stopped assumption");
2062
2063 // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks
2064 // if linear allocation blocks need to be appropriately marked to allow the
2065 // the blocks to be parsable. We also check here whether we need to nudge the
2066 // CMS collector thread to start a new cycle (if it's not already active).
2067 assert( Thread::current()->is_VM_thread()
2068 || ( CMSScavengeBeforeRemark
2069 && Thread::current()->is_ConcurrentGC_thread()),
2070 "Incorrect thread type for epilogue execution");
2071
2108 _between_prologue_and_epilogue = false; // ready for next cycle
2109 }
2110
2111 void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
2112 collector()->gc_epilogue(full);
2113
2114 // Also reset promotion tracking in par gc thread states.
2115 for (uint i = 0; i < ParallelGCThreads; i++) {
2116 _par_gc_thread_states[i]->promo.stopTrackingPromotions(i);
2117 }
2118 }
2119
2120 void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) {
2121 assert(!incremental_collection_failed(), "Should have been cleared");
2122 cmsSpace()->setPreconsumptionDirtyCardClosure(NULL);
2123 cmsSpace()->gc_epilogue();
2124 // Print stat counters
2125 NOT_PRODUCT(
2126 assert(_numObjectsAllocated == 0, "check");
2127 assert(_numWordsAllocated == 0, "check");
2128 log_develop(gc, promotion)("Promoted " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
2129 _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord));
2130 _numObjectsPromoted = 0;
2131 _numWordsPromoted = 0;
2132 )
2133
2134 // Call down the chain in contiguous_available needs the freelistLock
2135 // so print this out before releasing the freeListLock.
2136 log_develop(gc)(" Contiguous available " SIZE_FORMAT " bytes ", contiguous_available());
2137 }
2138
2139 #ifndef PRODUCT
2140 bool CMSCollector::have_cms_token() {
2141 Thread* thr = Thread::current();
2142 if (thr->is_VM_thread()) {
2143 return ConcurrentMarkSweepThread::vm_thread_has_cms_token();
2144 } else if (thr->is_ConcurrentGC_thread()) {
2145 return ConcurrentMarkSweepThread::cms_thread_has_cms_token();
2146 } else if (thr->is_GC_task_thread()) {
2147 return ConcurrentMarkSweepThread::vm_thread_has_cms_token() &&
2148 ParGCRareEvent_lock->owned_by_self();
2149 }
2150 return false;
2151 }
2152 #endif
2153
2154 void
2155 CMSCollector::print_on_error(outputStream* st) {
2156 CMSCollector* collector = ConcurrentMarkSweepGeneration::_collector;
2157 if (collector != NULL) {
2158 CMSBitMap* bitmap = &collector->_markBitMap;
2159 st->print_cr("Marking Bits: (CMSBitMap*) " PTR_FORMAT, p2i(bitmap));
2160 bitmap->print_on_error(st, " Bits: ");
2161
2162 st->cr();
2163
2164 CMSBitMap* mut_bitmap = &collector->_modUnionTable;
2165 st->print_cr("Mod Union Table: (CMSBitMap*) " PTR_FORMAT, p2i(mut_bitmap));
2166 mut_bitmap->print_on_error(st, " Bits: ");
2167 }
2168 }
2169
2170 ////////////////////////////////////////////////////////
2171 // CMS Verification Support
2172 ////////////////////////////////////////////////////////
2173 // Following the remark phase, the following invariant
2174 // should hold -- each object in the CMS heap which is
2175 // marked in markBitMap() should be marked in the verification_mark_bm().
2176
2177 class VerifyMarkedClosure: public BitMapClosure {
2178 CMSBitMap* _marks;
2179 bool _failed;
2180
2181 public:
2182 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2183
2184 bool do_bit(size_t offset) {
2185 HeapWord* addr = _marks->offsetToHeapWord(offset);
2186 if (!_marks->isMarked(addr)) {
2187 LogHandle(gc, verify) log;
2188 ResourceMark rm;
2189 oop(addr)->print_on(log.info_stream());
2190 log.info(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
2191 _failed = true;
2192 }
2193 return true;
2194 }
2195
2196 bool failed() { return _failed; }
2197 };
2198
2199 bool CMSCollector::verify_after_remark() {
2200 GCTraceTime(Info, gc, verify) tm("Verifying CMS Marking.");
2201 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2202 static bool init = false;
2203
2204 assert(SafepointSynchronize::is_at_safepoint(),
2205 "Else mutations in object graph will make answer suspect");
2206 assert(have_cms_token(),
2207 "Else there may be mutual interference in use of "
2208 " verification data structures");
2209 assert(_collectorState > Marking && _collectorState <= Sweeping,
2210 "Else marking info checked here may be obsolete");
2211 assert(haveFreelistLocks(), "must hold free list locks");
2212 assert_lock_strong(bitMapLock());
2213
2214
2215 // Allocate marking bit map if not already allocated
2216 if (!init) { // first time
2217 if (!verification_mark_bm()->allocate(_span)) {
2218 return false;
2219 }
2220 init = true;
2243 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2244 // Update the saved marks which may affect the root scans.
2245 gch->save_marks();
2246
2247 if (CMSRemarkVerifyVariant == 1) {
2248 // In this first variant of verification, we complete
2249 // all marking, then check if the new marks-vector is
2250 // a subset of the CMS marks-vector.
2251 verify_after_remark_work_1();
2252 } else if (CMSRemarkVerifyVariant == 2) {
2253 // In this second variant of verification, we flag an error
2254 // (i.e. an object reachable in the new marks-vector not reachable
2255 // in the CMS marks-vector) immediately, also indicating the
2256 // identify of an object (A) that references the unmarked object (B) --
2257 // presumably, a mutation to A failed to be picked up by preclean/remark?
2258 verify_after_remark_work_2();
2259 } else {
2260 warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant",
2261 CMSRemarkVerifyVariant);
2262 }
2263 return true;
2264 }
2265
2266 void CMSCollector::verify_after_remark_work_1() {
2267 ResourceMark rm;
2268 HandleMark hm;
2269 GenCollectedHeap* gch = GenCollectedHeap::heap();
2270
2271 // Get a clear set of claim bits for the roots processing to work with.
2272 ClassLoaderDataGraph::clear_claimed_marks();
2273
2274 // Mark from roots one level into CMS
2275 MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
2276 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2277
2278 {
2279 StrongRootsScope srs(1);
2280
2281 gch->gen_process_roots(&srs,
2282 GenCollectedHeap::OldGen,
2294 false /* don't yield */, true /* verifying */);
2295 assert(_restart_addr == NULL, "Expected pre-condition");
2296 verification_mark_bm()->iterate(&markFromRootsClosure);
2297 while (_restart_addr != NULL) {
2298 // Deal with stack overflow: by restarting at the indicated
2299 // address.
2300 HeapWord* ra = _restart_addr;
2301 markFromRootsClosure.reset(ra);
2302 _restart_addr = NULL;
2303 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2304 }
2305 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2306 verify_work_stacks_empty();
2307
2308 // Marking completed -- now verify that each bit marked in
2309 // verification_mark_bm() is also marked in markBitMap(); flag all
2310 // errors by printing corresponding objects.
2311 VerifyMarkedClosure vcl(markBitMap());
2312 verification_mark_bm()->iterate(&vcl);
2313 if (vcl.failed()) {
2314 LogHandle(gc, verify) log;
2315 log.info("Verification failed");
2316 ResourceMark rm;
2317 gch->print_on(log.info_stream());
2318 fatal("CMS: failed marking verification after remark");
2319 }
2320 }
2321
2322 class VerifyKlassOopsKlassClosure : public KlassClosure {
2323 class VerifyKlassOopsClosure : public OopClosure {
2324 CMSBitMap* _bitmap;
2325 public:
2326 VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { }
2327 void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); }
2328 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2329 } _oop_closure;
2330 public:
2331 VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {}
2332 void do_klass(Klass* k) {
2333 k->oops_do(&_oop_closure);
2334 }
2335 };
2336
2337 void CMSCollector::verify_after_remark_work_2() {
2590 expand_for_gc_cause(word_size*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_allocation);
2591 if (GCExpandToAllocateDelayMillis > 0) {
2592 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2593 }
2594 return have_lock_and_allocate(word_size, tlab);
2595 }
2596
2597 void ConcurrentMarkSweepGeneration::expand_for_gc_cause(
2598 size_t bytes,
2599 size_t expand_bytes,
2600 CMSExpansionCause::Cause cause)
2601 {
2602
2603 bool success = expand(bytes, expand_bytes);
2604
2605 // remember why we expanded; this information is used
2606 // by shouldConcurrentCollect() when making decisions on whether to start
2607 // a new CMS cycle.
2608 if (success) {
2609 set_expansion_cause(cause);
2610 log_trace(gc)("Expanded CMS gen for %s", CMSExpansionCause::to_string(cause));
2611 }
2612 }
2613
2614 HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) {
2615 HeapWord* res = NULL;
2616 MutexLocker x(ParGCRareEvent_lock);
2617 while (true) {
2618 // Expansion by some other thread might make alloc OK now:
2619 res = ps->lab.alloc(word_sz);
2620 if (res != NULL) return res;
2621 // If there's not enough expansion space available, give up.
2622 if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) {
2623 return NULL;
2624 }
2625 // Otherwise, we try expansion.
2626 expand_for_gc_cause(word_sz*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_allocate_par_lab);
2627 // Now go around the loop and try alloc again;
2628 // A competing par_promote might beat us to the expansion space,
2629 // so we may go around the loop again if promotion fails again.
2630 if (GCExpandToAllocateDelayMillis > 0) {
2658 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2659 }
2660 }
2661 }
2662
2663 void ConcurrentMarkSweepGeneration::shrink(size_t bytes) {
2664 // Only shrink if a compaction was done so that all the free space
2665 // in the generation is in a contiguous block at the end.
2666 if (did_compact()) {
2667 CardGeneration::shrink(bytes);
2668 }
2669 }
2670
2671 void ConcurrentMarkSweepGeneration::assert_correct_size_change_locking() {
2672 assert_locked_or_safepoint(Heap_lock);
2673 }
2674
2675 void ConcurrentMarkSweepGeneration::shrink_free_list_by(size_t bytes) {
2676 assert_locked_or_safepoint(Heap_lock);
2677 assert_lock_strong(freelistLock());
2678 log_trace(gc)("Shrinking of CMS not yet implemented");
2679 return;
2680 }
2681
2682
2683 // Simple ctor/dtor wrapper for accounting & timer chores around concurrent
2684 // phases.
2685 class CMSPhaseAccounting: public StackObj {
2686 public:
2687 CMSPhaseAccounting(CMSCollector *collector,
2688 const char *title);
2689 ~CMSPhaseAccounting();
2690
2691 private:
2692 CMSCollector *_collector;
2693 const char *_title;
2694 GCTraceConcTime(Info, gc) _trace_time;
2695
2696 public:
2697 // Not MT-safe; so do not pass around these StackObj's
2698 // where they may be accessed by other threads.
2699 jlong wallclock_millis() {
2700 return TimeHelper::counter_to_millis(os::elapsed_counter() - _trace_time.start_time());
2701 }
2702 };
2703
2704 CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector,
2705 const char *title) :
2706 _collector(collector), _title(title), _trace_time(title) {
2707
2708 _collector->resetYields();
2709 _collector->resetTimer();
2710 _collector->startTimer();
2711 }
2712
2713 CMSPhaseAccounting::~CMSPhaseAccounting() {
2714 _collector->stopTimer();
2715 jlong end_counter = os::elapsed_counter();
2716 log_debug(gc)("Concurrent active time: %.3fms", _collector->timerValue());
2717 log_debug(gc, stats)(" (CMS %s yielded %d times)", _title, _collector->yields());
2718 }
2719
2720 // CMS work
2721
2722 // The common parts of CMSParInitialMarkTask and CMSParRemarkTask.
2723 class CMSParMarkTask : public AbstractGangTask {
2724 protected:
2725 CMSCollector* _collector;
2726 uint _n_workers;
2727 CMSParMarkTask(const char* name, CMSCollector* collector, uint n_workers) :
2728 AbstractGangTask(name),
2729 _collector(collector),
2730 _n_workers(n_workers) {}
2731 // Work method in support of parallel rescan ... of young gen spaces
2732 void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl,
2733 ContiguousSpace* space,
2734 HeapWord** chunk_array, size_t chunk_top);
2735 void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl);
2736 };
2737
2764 Mutex::_no_safepoint_check_flag);
2765 checkpointRootsInitialWork();
2766 // enable ("weak") refs discovery
2767 rp->enable_discovery();
2768 _collectorState = Marking;
2769 }
2770 }
2771
2772 void CMSCollector::checkpointRootsInitialWork() {
2773 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
2774 assert(_collectorState == InitialMarking, "just checking");
2775
2776 // Already have locks.
2777 assert_lock_strong(bitMapLock());
2778 assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
2779
2780 // Setup the verification and class unloading state for this
2781 // CMS collection cycle.
2782 setup_cms_unloading_and_verification_state();
2783
2784 GCTraceTime(Trace, gc) ts("checkpointRootsInitialWork", _gc_timer_cm);
2785
2786 // Reset all the PLAB chunk arrays if necessary.
2787 if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
2788 reset_survivor_plab_arrays();
2789 }
2790
2791 ResourceMark rm;
2792 HandleMark hm;
2793
2794 MarkRefsIntoClosure notOlder(_span, &_markBitMap);
2795 GenCollectedHeap* gch = GenCollectedHeap::heap();
2796
2797 verify_work_stacks_empty();
2798 verify_overflow_empty();
2799
2800 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2801 // Update the saved marks which may affect the root scans.
2802 gch->save_marks();
2803
2804 // weak reference processing has not started yet.
2805 ref_processor()->set_enqueuing_is_done(false);
2806
2807 // Need to remember all newly created CLDs,
2808 // so that we can guarantee that the remark finds them.
2809 ClassLoaderDataGraph::remember_new_clds(true);
2810
2811 // Whenever a CLD is found, it will be claimed before proceeding to mark
2812 // the klasses. The claimed marks need to be cleared before marking starts.
2813 ClassLoaderDataGraph::clear_claimed_marks();
2814
2815 print_eden_and_survivor_chunk_arrays();
2816
2817 {
2818 #if defined(COMPILER2) || INCLUDE_JVMCI
2819 DerivedPointerTableDeactivate dpt_deact;
2820 #endif
2821 if (CMSParallelInitialMarkEnabled) {
2822 // The parallel version.
2823 WorkGang* workers = gch->workers();
2824 assert(workers != NULL, "Need parallel worker threads.");
2825 uint n_workers = workers->active_workers();
2826
2827 StrongRootsScope srs(n_workers);
2828
2829 CMSParInitialMarkTask tsk(this, &srs, n_workers);
2830 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
2831 if (n_workers > 1) {
2832 workers->run_task(&tsk);
2833 } else {
2834 tsk.work(0);
2835 }
2866 save_sweep_limits();
2867 verify_overflow_empty();
2868 }
2869
2870 bool CMSCollector::markFromRoots() {
2871 // we might be tempted to assert that:
2872 // assert(!SafepointSynchronize::is_at_safepoint(),
2873 // "inconsistent argument?");
2874 // However that wouldn't be right, because it's possible that
2875 // a safepoint is indeed in progress as a young generation
2876 // stop-the-world GC happens even as we mark in this generation.
2877 assert(_collectorState == Marking, "inconsistent state?");
2878 check_correct_thread_executing();
2879 verify_overflow_empty();
2880
2881 // Weak ref discovery note: We may be discovering weak
2882 // refs in this generation concurrent (but interleaved) with
2883 // weak ref discovery by the young generation collector.
2884
2885 CMSTokenSyncWithLocks ts(true, bitMapLock());
2886 GCTraceCPUTime tcpu;
2887 CMSPhaseAccounting pa(this, "Concrurrent Mark");
2888 bool res = markFromRootsWork();
2889 if (res) {
2890 _collectorState = Precleaning;
2891 } else { // We failed and a foreground collection wants to take over
2892 assert(_foregroundGCIsActive, "internal state inconsistency");
2893 assert(_restart_addr == NULL, "foreground will restart from scratch");
2894 log_debug(gc)("bailing out to foreground collection");
2895 }
2896 verify_overflow_empty();
2897 return res;
2898 }
2899
2900 bool CMSCollector::markFromRootsWork() {
2901 // iterate over marked bits in bit map, doing a full scan and mark
2902 // from these roots using the following algorithm:
2903 // . if oop is to the right of the current scan pointer,
2904 // mark corresponding bit (we'll process it later)
2905 // . else (oop is to left of current scan pointer)
2906 // push oop on marking stack
2907 // . drain the marking stack
2908
2909 // Note that when we do a marking step we need to hold the
2910 // bit map lock -- recall that direct allocation (by mutators)
2911 // and promotion (by the young generation collector) is also
2912 // marking the bit map. [the so-called allocate live policy.]
2913 // Because the implementation of bit map marking is not
2914 // robust wrt simultaneous marking of bits in the same word,
3079 // and local work queue empty,
3080 // then in a loop do:
3081 // . check global overflow stack; steal a batch of oops and trace
3082 // . try to steal from other threads oif GOS is empty
3083 // . if neither is available, offer termination
3084 // -- Terminate and return result
3085 //
3086 void CMSConcMarkingTask::work(uint worker_id) {
3087 elapsedTimer _timer;
3088 ResourceMark rm;
3089 HandleMark hm;
3090
3091 DEBUG_ONLY(_collector->verify_overflow_empty();)
3092
3093 // Before we begin work, our work queue should be empty
3094 assert(work_queue(worker_id)->size() == 0, "Expected to be empty");
3095 // Scan the bitmap covering _cms_space, tracing through grey objects.
3096 _timer.start();
3097 do_scan_and_mark(worker_id, _cms_space);
3098 _timer.stop();
3099 log_debug(gc, stats)("Finished cms space scanning in %dth thread: %3.3f sec", worker_id, _timer.seconds());
3100
3101 // ... do work stealing
3102 _timer.reset();
3103 _timer.start();
3104 do_work_steal(worker_id);
3105 _timer.stop();
3106 log_debug(gc, stats)("Finished work stealing in %dth thread: %3.3f sec", worker_id, _timer.seconds());
3107 assert(_collector->_markStack.isEmpty(), "Should have been emptied");
3108 assert(work_queue(worker_id)->size() == 0, "Should have been emptied");
3109 // Note that under the current task protocol, the
3110 // following assertion is true even of the spaces
3111 // expanded since the completion of the concurrent
3112 // marking. XXX This will likely change under a strict
3113 // ABORT semantics.
3114 // After perm removal the comparison was changed to
3115 // greater than or equal to from strictly greater than.
3116 // Before perm removal the highest address sweep would
3117 // have been at the end of perm gen but now is at the
3118 // end of the tenured gen.
3119 assert(_global_finger >= _cms_space->end(),
3120 "All tasks have been completed");
3121 DEBUG_ONLY(_collector->verify_overflow_empty();)
3122 }
3123
3124 void CMSConcMarkingTask::bump_global_finger(HeapWord* f) {
3125 HeapWord* read = _global_finger;
3126 HeapWord* cur = read;
3301 // Check if oop points into the CMS generation
3302 // and is not marked
3303 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
3304 // a white object ...
3305 // If we manage to "claim" the object, by being the
3306 // first thread to mark it, then we push it on our
3307 // marking stack
3308 if (_bit_map->par_mark(addr)) { // ... now grey
3309 // push on work queue (grey set)
3310 bool simulate_overflow = false;
3311 NOT_PRODUCT(
3312 if (CMSMarkStackOverflowALot &&
3313 _collector->simulate_overflow()) {
3314 // simulate a stack overflow
3315 simulate_overflow = true;
3316 }
3317 )
3318 if (simulate_overflow ||
3319 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
3320 // stack overflow
3321 log_debug(gc, stats)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _overflow_stack->capacity());
3322 // We cannot assert that the overflow stack is full because
3323 // it may have been emptied since.
3324 assert(simulate_overflow ||
3325 _work_queue->size() == _work_queue->max_elems(),
3326 "Else push should have succeeded");
3327 handle_stack_overflow(addr);
3328 }
3329 } // Else, some other thread got there first
3330 do_yield_check();
3331 }
3332 }
3333
3334 void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
3335 void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
3336
3337 void Par_ConcMarkingClosure::trim_queue(size_t max) {
3338 while (_work_queue->size() > max) {
3339 oop new_oop;
3340 if (_work_queue->pop_local(new_oop)) {
3341 assert(new_oop->is_oop(), "Should be an oop");
3386 assert(work_q->size() == 0, "Impossible!");
3387 break;
3388 } else if (yielding() || should_yield()) {
3389 yield();
3390 }
3391 }
3392 }
3393
3394 // This is run by the CMS (coordinator) thread.
3395 void CMSConcMarkingTask::coordinator_yield() {
3396 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
3397 "CMS thread should hold CMS token");
3398 // First give up the locks, then yield, then re-lock
3399 // We should probably use a constructor/destructor idiom to
3400 // do this unlock/lock or modify the MutexUnlocker class to
3401 // serve our purpose. XXX
3402 assert_lock_strong(_bit_map_lock);
3403 _bit_map_lock->unlock();
3404 ConcurrentMarkSweepThread::desynchronize(true);
3405 _collector->stopTimer();
3406 _collector->incrementYields();
3407
3408 // It is possible for whichever thread initiated the yield request
3409 // not to get a chance to wake up and take the bitmap lock between
3410 // this thread releasing it and reacquiring it. So, while the
3411 // should_yield() flag is on, let's sleep for a bit to give the
3412 // other thread a chance to wake up. The limit imposed on the number
3413 // of iterations is defensive, to avoid any unforseen circumstances
3414 // putting us into an infinite loop. Since it's always been this
3415 // (coordinator_yield()) method that was observed to cause the
3416 // problem, we are using a parameter (CMSCoordinatorYieldSleepCount)
3417 // which is by default non-zero. For the other seven methods that
3418 // also perform the yield operation, as are using a different
3419 // parameter (CMSYieldSleepCount) which is by default zero. This way we
3420 // can enable the sleeping for those methods too, if necessary.
3421 // See 6442774.
3422 //
3423 // We really need to reconsider the synchronization between the GC
3424 // thread and the yield-requesting threads in the future and we
3425 // should really use wait/notify, which is the recommended
3426 // way of doing this type of interaction. Additionally, we should
3548 void CMSCollector::preclean() {
3549 check_correct_thread_executing();
3550 assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
3551 verify_work_stacks_empty();
3552 verify_overflow_empty();
3553 _abort_preclean = false;
3554 if (CMSPrecleaningEnabled) {
3555 if (!CMSEdenChunksRecordAlways) {
3556 _eden_chunk_index = 0;
3557 }
3558 size_t used = get_eden_used();
3559 size_t capacity = get_eden_capacity();
3560 // Don't start sampling unless we will get sufficiently
3561 // many samples.
3562 if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
3563 * CMSScheduleRemarkEdenPenetration)) {
3564 _start_sampling = true;
3565 } else {
3566 _start_sampling = false;
3567 }
3568 GCTraceCPUTime tcpu;
3569 CMSPhaseAccounting pa(this, "Concurrent Preclean");
3570 preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
3571 }
3572 CMSTokenSync x(true); // is cms thread
3573 if (CMSPrecleaningEnabled) {
3574 sample_eden();
3575 _collectorState = AbortablePreclean;
3576 } else {
3577 _collectorState = FinalMarking;
3578 }
3579 verify_work_stacks_empty();
3580 verify_overflow_empty();
3581 }
3582
3583 // Try and schedule the remark such that young gen
3584 // occupancy is CMSScheduleRemarkEdenPenetration %.
3585 void CMSCollector::abortable_preclean() {
3586 check_correct_thread_executing();
3587 assert(CMSPrecleaningEnabled, "Inconsistent control state");
3588 assert(_collectorState == AbortablePreclean, "Inconsistent control state");
3589
3590 // If Eden's current occupancy is below this threshold,
3591 // immediately schedule the remark; else preclean
3592 // past the next scavenge in an effort to
3593 // schedule the pause as described above. By choosing
3594 // CMSScheduleRemarkEdenSizeThreshold >= max eden size
3595 // we will never do an actual abortable preclean cycle.
3596 if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) {
3597 GCTraceCPUTime tcpu;
3598 CMSPhaseAccounting pa(this, "Concurrent Abortable Preclean");
3599 // We need more smarts in the abortable preclean
3600 // loop below to deal with cases where allocation
3601 // in young gen is very very slow, and our precleaning
3602 // is running a losing race against a horde of
3603 // mutators intent on flooding us with CMS updates
3604 // (dirty cards).
3605 // One, admittedly dumb, strategy is to give up
3606 // after a certain number of abortable precleaning loops
3607 // or after a certain maximum time. We want to make
3608 // this smarter in the next iteration.
3609 // XXX FIX ME!!! YSR
3610 size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0;
3611 while (!(should_abort_preclean() ||
3612 ConcurrentMarkSweepThread::should_terminate())) {
3613 workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2);
3614 cumworkdone += workdone;
3615 loops++;
3616 // Voluntarily terminate abortable preclean phase if we have
3617 // been at it for too long.
3618 if ((CMSMaxAbortablePrecleanLoops != 0) &&
3619 loops >= CMSMaxAbortablePrecleanLoops) {
3620 log_debug(gc)(" CMS: abort preclean due to loops ");
3621 break;
3622 }
3623 if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) {
3624 log_debug(gc)(" CMS: abort preclean due to time ");
3625 break;
3626 }
3627 // If we are doing little work each iteration, we should
3628 // take a short break.
3629 if (workdone < CMSAbortablePrecleanMinWorkPerIteration) {
3630 // Sleep for some time, waiting for work to accumulate
3631 stopTimer();
3632 cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis);
3633 startTimer();
3634 waited++;
3635 }
3636 }
3637 log_debug(gc, stats)(" [" SIZE_FORMAT " iterations, " SIZE_FORMAT " waits, " SIZE_FORMAT " cards)] ",
3638 loops, waited, cumworkdone);
3639 }
3640 CMSTokenSync x(true); // is cms thread
3641 if (_collectorState != Idling) {
3642 assert(_collectorState == AbortablePreclean,
3643 "Spontaneous state transition?");
3644 _collectorState = FinalMarking;
3645 } // Else, a foreground collection completed this CMS cycle.
3646 return;
3647 }
3648
3649 // Respond to an Eden sampling opportunity
3650 void CMSCollector::sample_eden() {
3651 // Make sure a young gc cannot sneak in between our
3652 // reading and recording of a sample.
3653 assert(Thread::current()->is_ConcurrentGC_thread(),
3654 "Only the cms thread may collect Eden samples");
3655 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
3656 "Should collect samples while holding CMS token");
3657 if (!_start_sampling) {
3658 return;
3659 }
3762 // processes.
3763 ScanMarkedObjectsAgainCarefullyClosure
3764 smoac_cl(this, _span,
3765 &_markBitMap, &_markStack, &mrias_cl, CMSYield);
3766
3767 // Preclean dirty cards in ModUnionTable and CardTable using
3768 // appropriate convergence criterion;
3769 // repeat CMSPrecleanIter times unless we find that
3770 // we are losing.
3771 assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large");
3772 assert(CMSPrecleanNumerator < CMSPrecleanDenominator,
3773 "Bad convergence multiplier");
3774 assert(CMSPrecleanThreshold >= 100,
3775 "Unreasonably low CMSPrecleanThreshold");
3776
3777 size_t numIter, cumNumCards, lastNumCards, curNumCards;
3778 for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0;
3779 numIter < CMSPrecleanIter;
3780 numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) {
3781 curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl);
3782 log_trace(gc)(" (modUnionTable: " SIZE_FORMAT " cards)", curNumCards);
3783 // Either there are very few dirty cards, so re-mark
3784 // pause will be small anyway, or our pre-cleaning isn't
3785 // that much faster than the rate at which cards are being
3786 // dirtied, so we might as well stop and re-mark since
3787 // precleaning won't improve our re-mark time by much.
3788 if (curNumCards <= CMSPrecleanThreshold ||
3789 (numIter > 0 &&
3790 (curNumCards * CMSPrecleanDenominator >
3791 lastNumCards * CMSPrecleanNumerator))) {
3792 numIter++;
3793 cumNumCards += curNumCards;
3794 break;
3795 }
3796 }
3797
3798 preclean_klasses(&mrias_cl, _cmsGen->freelistLock());
3799
3800 curNumCards = preclean_card_table(_cmsGen, &smoac_cl);
3801 cumNumCards += curNumCards;
3802 log_debug(gc, stats)(" (cardTable: " SIZE_FORMAT " cards, re-scanned " SIZE_FORMAT " cards, " SIZE_FORMAT " iterations)",
3803 curNumCards, cumNumCards, numIter);
3804 return cumNumCards; // as a measure of useful work done
3805 }
3806
3807 // PRECLEANING NOTES:
3808 // Precleaning involves:
3809 // . reading the bits of the modUnionTable and clearing the set bits.
3810 // . For the cards corresponding to the set bits, we scan the
3811 // objects on those cards. This means we need the free_list_lock
3812 // so that we can safely iterate over the CMS space when scanning
3813 // for oops.
3814 // . When we scan the objects, we'll be both reading and setting
3815 // marks in the marking bit map, so we'll need the marking bit map.
3816 // . For protecting _collector_state transitions, we take the CGC_lock.
3817 // Note that any races in the reading of of card table entries by the
3818 // CMS thread on the one hand and the clearing of those entries by the
3819 // VM thread or the setting of those entries by the mutator threads on the
3820 // other are quite benign. However, for efficiency it makes sense to keep
3821 // the VM thread from racing with the CMS thread while the latter is
3822 // dirty card info to the modUnionTable. We therefore also use the
3823 // CGC_lock to protect the reading of the card table and the mod union
4037 // SSS: Add equivalent to ScanMarkedObjectsAgainCarefullyClosure::do_yield_check and should_abort_preclean?
4038 // SSS: We should probably check if precleaning should be aborted, at suitable intervals?
4039 PrecleanKlassClosure preclean_klass_closure(cl);
4040 ClassLoaderDataGraph::classes_do(&preclean_klass_closure);
4041
4042 verify_work_stacks_empty();
4043 verify_overflow_empty();
4044 }
4045
4046 void CMSCollector::checkpointRootsFinal() {
4047 assert(_collectorState == FinalMarking, "incorrect state transition?");
4048 check_correct_thread_executing();
4049 // world is stopped at this checkpoint
4050 assert(SafepointSynchronize::is_at_safepoint(),
4051 "world should be stopped");
4052 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
4053
4054 verify_work_stacks_empty();
4055 verify_overflow_empty();
4056
4057 log_debug(gc)("YG occupancy: " SIZE_FORMAT " K (" SIZE_FORMAT " K)",
4058 _young_gen->used() / K, _young_gen->capacity() / K);
4059 {
4060 if (CMSScavengeBeforeRemark) {
4061 GenCollectedHeap* gch = GenCollectedHeap::heap();
4062 // Temporarily set flag to false, GCH->do_collection will
4063 // expect it to be false and set to true
4064 FlagSetting fl(gch->_is_gc_active, false);
4065
4066 GCTraceTime(Trace, gc) tm("Scavenge Before Remark", _gc_timer_cm);
4067
4068 gch->do_collection(true, // full (i.e. force, see below)
4069 false, // !clear_all_soft_refs
4070 0, // size
4071 false, // is_tlab
4072 GenCollectedHeap::YoungGen // type
4073 );
4074 }
4075 FreelistLocker x(this);
4076 MutexLockerEx y(bitMapLock(),
4077 Mutex::_no_safepoint_check_flag);
4078 checkpointRootsFinalWork();
4079 }
4080 verify_work_stacks_empty();
4081 verify_overflow_empty();
4082 }
4083
4084 void CMSCollector::checkpointRootsFinalWork() {
4085 GCTraceTime(Trace, gc) tm("checkpointRootsFinalWork", _gc_timer_cm);
4086
4087 assert(haveFreelistLocks(), "must have free list locks");
4088 assert_lock_strong(bitMapLock());
4089
4090 ResourceMark rm;
4091 HandleMark hm;
4092
4093 GenCollectedHeap* gch = GenCollectedHeap::heap();
4094
4095 if (should_unload_classes()) {
4096 CodeCache::gc_prologue();
4097 }
4098 assert(haveFreelistLocks(), "must have free list locks");
4099 assert_lock_strong(bitMapLock());
4100
4101 // We might assume that we need not fill TLAB's when
4102 // CMSScavengeBeforeRemark is set, because we may have just done
4103 // a scavenge which would have filled all TLAB's -- and besides
4104 // Eden would be empty. This however may not always be the case --
4105 // for instance although we asked for a scavenge, it may not have
4106 // happened because of a JNI critical section. We probably need
4107 // a policy for deciding whether we can in that case wait until
4108 // the critical section releases and then do the remark following
4109 // the scavenge, and skip it here. In the absence of that policy,
4110 // or of an indication of whether the scavenge did indeed occur,
4111 // we cannot rely on TLAB's having been filled and must do
4112 // so here just in case a scavenge did not happen.
4113 gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
4114 // Update the saved marks which may affect the root scans.
4115 gch->save_marks();
4116
4117 print_eden_and_survivor_chunk_arrays();
4118
4119 {
4120 #if defined(COMPILER2) || INCLUDE_JVMCI
4121 DerivedPointerTableDeactivate dpt_deact;
4122 #endif
4123
4124 // Note on the role of the mod union table:
4125 // Since the marker in "markFromRoots" marks concurrently with
4126 // mutators, it is possible for some reachable objects not to have been
4127 // scanned. For instance, an only reference to an object A was
4128 // placed in object B after the marker scanned B. Unless B is rescanned,
4129 // A would be collected. Such updates to references in marked objects
4130 // are detected via the mod union table which is the set of all cards
4131 // dirtied since the first checkpoint in this GC cycle and prior to
4132 // the most recent young generation GC, minus those cleaned up by the
4133 // concurrent precleaning.
4134 if (CMSParallelRemarkEnabled) {
4135 GCTraceTime(Debug, gc) t("Rescan (parallel)", _gc_timer_cm);
4136 do_remark_parallel();
4137 } else {
4138 GCTraceTime(Debug, gc) t("Rescan (non-parallel)", _gc_timer_cm);
4139 do_remark_non_parallel();
4140 }
4141 }
4142 verify_work_stacks_empty();
4143 verify_overflow_empty();
4144
4145 {
4146 GCTraceTime(Trace, gc) ts("refProcessingWork", _gc_timer_cm);
4147 refProcessingWork();
4148 }
4149 verify_work_stacks_empty();
4150 verify_overflow_empty();
4151
4152 if (should_unload_classes()) {
4153 CodeCache::gc_epilogue();
4154 }
4155 JvmtiExport::gc_epilogue();
4156
4157 // If we encountered any (marking stack / work queue) overflow
4158 // events during the current CMS cycle, take appropriate
4159 // remedial measures, where possible, so as to try and avoid
4160 // recurrence of that condition.
4161 assert(_markStack.isEmpty(), "No grey objects");
4162 size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
4163 _ser_kac_ovflw + _ser_kac_preclean_ovflw;
4164 if (ser_ovflw > 0) {
4165 log_debug(gc, stats)("Marking stack overflow (benign) (pmc_pc=" SIZE_FORMAT ", pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT ", kac_preclean=" SIZE_FORMAT ")",
4166 _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw, _ser_kac_ovflw, _ser_kac_preclean_ovflw);
4167 _markStack.expand();
4168 _ser_pmc_remark_ovflw = 0;
4169 _ser_pmc_preclean_ovflw = 0;
4170 _ser_kac_preclean_ovflw = 0;
4171 _ser_kac_ovflw = 0;
4172 }
4173 if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
4174 log_debug(gc, stats)("Work queue overflow (benign) (pmc_rm=" SIZE_FORMAT ", kac=" SIZE_FORMAT ")",
4175 _par_pmc_remark_ovflw, _par_kac_ovflw);
4176 _par_pmc_remark_ovflw = 0;
4177 _par_kac_ovflw = 0;
4178 }
4179 if (_markStack._hit_limit > 0) {
4180 log_debug(gc, stats)(" (benign) Hit max stack size limit (" SIZE_FORMAT ")",
4181 _markStack._hit_limit);
4182 }
4183 if (_markStack._failed_double > 0) {
4184 log_debug(gc, stats)(" (benign) Failed stack doubling (" SIZE_FORMAT "), current capacity " SIZE_FORMAT,
4185 _markStack._failed_double, _markStack.capacity());
4186 }
4187 _markStack._hit_limit = 0;
4188 _markStack._failed_double = 0;
4189
4190 if ((VerifyAfterGC || VerifyDuringGC) &&
4191 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4192 verify_after_remark();
4193 }
4194
4195 _gc_tracer_cm->report_object_count_after_gc(&_is_alive_closure);
4196
4197 // Change under the freelistLocks.
4198 _collectorState = Sweeping;
4199 // Call isAllClear() under bitMapLock
4200 assert(_modUnionTable.isAllClear(),
4201 "Should be clear by end of the final marking");
4202 assert(_ct->klass_rem_set()->mod_union_is_clear(),
4203 "Should be clear by end of the final marking");
4204 }
4205
4206 void CMSParInitialMarkTask::work(uint worker_id) {
4207 elapsedTimer _timer;
4208 ResourceMark rm;
4209 HandleMark hm;
4210
4211 // ---------- scan from roots --------------
4212 _timer.start();
4213 GenCollectedHeap* gch = GenCollectedHeap::heap();
4214 Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap));
4215
4216 // ---------- young gen roots --------------
4217 {
4218 work_on_young_gen_roots(worker_id, &par_mri_cl);
4219 _timer.stop();
4220 log_debug(gc, stats)("Finished young gen initial mark scan work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4221 }
4222
4223 // ---------- remaining roots --------------
4224 _timer.reset();
4225 _timer.start();
4226
4227 CLDToOopClosure cld_closure(&par_mri_cl, true);
4228
4229 gch->gen_process_roots(_strong_roots_scope,
4230 GenCollectedHeap::OldGen,
4231 false, // yg was scanned above
4232 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4233 _collector->should_unload_classes(),
4234 &par_mri_cl,
4235 NULL,
4236 &cld_closure);
4237 assert(_collector->should_unload_classes()
4238 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4239 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4240 _timer.stop();
4241 log_debug(gc, stats)("Finished remaining root initial mark scan work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4242 }
4243
4244 // Parallel remark task
4245 class CMSParRemarkTask: public CMSParMarkTask {
4246 CompactibleFreeListSpace* _cms_space;
4247
4248 // The per-thread work queues, available here for stealing.
4249 OopTaskQueueSet* _task_queues;
4250 ParallelTaskTerminator _term;
4251 StrongRootsScope* _strong_roots_scope;
4252
4253 public:
4254 // A value of 0 passed to n_workers will cause the number of
4255 // workers to be taken from the active workers in the work gang.
4256 CMSParRemarkTask(CMSCollector* collector,
4257 CompactibleFreeListSpace* cms_space,
4258 uint n_workers, WorkGang* workers,
4259 OopTaskQueueSet* task_queues,
4260 StrongRootsScope* strong_roots_scope):
4261 CMSParMarkTask("Rescan roots and grey objects in parallel",
4334 elapsedTimer _timer;
4335 ResourceMark rm;
4336 HandleMark hm;
4337
4338 // ---------- rescan from roots --------------
4339 _timer.start();
4340 GenCollectedHeap* gch = GenCollectedHeap::heap();
4341 Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
4342 _collector->_span, _collector->ref_processor(),
4343 &(_collector->_markBitMap),
4344 work_queue(worker_id));
4345
4346 // Rescan young gen roots first since these are likely
4347 // coarsely partitioned and may, on that account, constitute
4348 // the critical path; thus, it's best to start off that
4349 // work first.
4350 // ---------- young gen roots --------------
4351 {
4352 work_on_young_gen_roots(worker_id, &par_mrias_cl);
4353 _timer.stop();
4354 log_debug(gc, stats)("Finished young gen rescan work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4355 }
4356
4357 // ---------- remaining roots --------------
4358 _timer.reset();
4359 _timer.start();
4360 gch->gen_process_roots(_strong_roots_scope,
4361 GenCollectedHeap::OldGen,
4362 false, // yg was scanned above
4363 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4364 _collector->should_unload_classes(),
4365 &par_mrias_cl,
4366 NULL,
4367 NULL); // The dirty klasses will be handled below
4368
4369 assert(_collector->should_unload_classes()
4370 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4371 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4372 _timer.stop();
4373 log_debug(gc, stats)("Finished remaining root rescan work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4374
4375 // ---------- unhandled CLD scanning ----------
4376 if (worker_id == 0) { // Single threaded at the moment.
4377 _timer.reset();
4378 _timer.start();
4379
4380 // Scan all new class loader data objects and new dependencies that were
4381 // introduced during concurrent marking.
4382 ResourceMark rm;
4383 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
4384 for (int i = 0; i < array->length(); i++) {
4385 par_mrias_cl.do_cld_nv(array->at(i));
4386 }
4387
4388 // We don't need to keep track of new CLDs anymore.
4389 ClassLoaderDataGraph::remember_new_clds(false);
4390
4391 _timer.stop();
4392 log_debug(gc, stats)("Finished unhandled CLD scanning work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4393 }
4394
4395 // ---------- dirty klass scanning ----------
4396 if (worker_id == 0) { // Single threaded at the moment.
4397 _timer.reset();
4398 _timer.start();
4399
4400 // Scan all classes that was dirtied during the concurrent marking phase.
4401 RemarkKlassClosure remark_klass_closure(&par_mrias_cl);
4402 ClassLoaderDataGraph::classes_do(&remark_klass_closure);
4403
4404 _timer.stop();
4405 log_debug(gc, stats)("Finished dirty klass scanning work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4406 }
4407
4408 // We might have added oops to ClassLoaderData::_handles during the
4409 // concurrent marking phase. These oops point to newly allocated objects
4410 // that are guaranteed to be kept alive. Either by the direct allocation
4411 // code, or when the young collector processes the roots. Hence,
4412 // we don't have to revisit the _handles block during the remark phase.
4413
4414 // ---------- rescan dirty cards ------------
4415 _timer.reset();
4416 _timer.start();
4417
4418 // Do the rescan tasks for each of the two spaces
4419 // (cms_space) in turn.
4420 // "worker_id" is passed to select the task_queue for "worker_id"
4421 do_dirty_card_rescan_tasks(_cms_space, worker_id, &par_mrias_cl);
4422 _timer.stop();
4423 log_debug(gc, stats)("Finished dirty card rescan work in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4424
4425 // ---------- steal work from other threads ...
4426 // ---------- ... and drain overflow list.
4427 _timer.reset();
4428 _timer.start();
4429 do_work_steal(worker_id, &par_mrias_cl, _collector->hash_seed(worker_id));
4430 _timer.stop();
4431 log_debug(gc, stats)("Finished work stealing in %dth thread: %3.3f sec", worker_id, _timer.seconds());
4432 }
4433
4434 // Note that parameter "i" is not used.
4435 void
4436 CMSParMarkTask::do_young_space_rescan(uint worker_id,
4437 OopsInGenClosure* cl, ContiguousSpace* space,
4438 HeapWord** chunk_array, size_t chunk_top) {
4439 // Until all tasks completed:
4440 // . claim an unclaimed task
4441 // . compute region boundaries corresponding to task claimed
4442 // using chunk_array
4443 // . par_oop_iterate(cl) over that region
4444
4445 ResourceMark rm;
4446 HandleMark hm;
4447
4448 SequentialSubTasksDone* pst = space->par_seq_tasks();
4449
4450 uint nth_task = 0;
4451 uint n_tasks = pst->n_tasks();
4605 // because we just took work from the overflow list,
4606 // but of course we can't since all of that could have
4607 // been already stolen from us.
4608 // "He giveth and He taketh away."
4609 continue;
4610 }
4611 // Verify that we have no work before we resort to stealing
4612 assert(work_q->size() == 0, "Have work, shouldn't steal");
4613 // Try to steal from other queues that have work
4614 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
4615 NOT_PRODUCT(num_steals++;)
4616 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
4617 assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
4618 // Do scanning work
4619 obj_to_scan->oop_iterate(cl);
4620 // Loop around, finish this work, and try to steal some more
4621 } else if (terminator()->offer_termination()) {
4622 break; // nirvana from the infinite cycle
4623 }
4624 }
4625 log_develop(gc, stats)("\t(%d: stole %d oops)", i, num_steals);
4626 assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
4627 "Else our work is not yet done");
4628 }
4629
4630 // Record object boundaries in _eden_chunk_array by sampling the eden
4631 // top in the slow-path eden object allocation code path and record
4632 // the boundaries, if CMSEdenChunksRecordAlways is true. If
4633 // CMSEdenChunksRecordAlways is false, we use the other asynchronous
4634 // sampling in sample_eden() that activates during the part of the
4635 // preclean phase.
4636 void CMSCollector::sample_eden_chunk() {
4637 if (CMSEdenChunksRecordAlways && _eden_chunk_array != NULL) {
4638 if (_eden_chunk_lock->try_lock()) {
4639 // Record a sample. This is the critical section. The contents
4640 // of the _eden_chunk_array have to be non-decreasing in the
4641 // address order.
4642 _eden_chunk_array[_eden_chunk_index] = *_top_addr;
4643 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
4644 "Unexpected state of Eden");
4645 if (_eden_chunk_index == 0 ||
4702 if (cur_val < min_val) {
4703 min_tid = j;
4704 min_val = cur_val;
4705 } else {
4706 assert(cur_val < top, "All recorded addresses should be less");
4707 }
4708 }
4709 // At this point min_val and min_tid are respectively
4710 // the least address in _survivor_plab_array[j]->nth(_cursor[j])
4711 // and the thread (j) that witnesses that address.
4712 // We record this address in the _survivor_chunk_array[i]
4713 // and increment _cursor[min_tid] prior to the next round i.
4714 if (min_val == top) {
4715 break;
4716 }
4717 _survivor_chunk_array[i] = min_val;
4718 _cursor[min_tid]++;
4719 }
4720 // We are all done; record the size of the _survivor_chunk_array
4721 _survivor_chunk_index = i; // exclusive: [0, i)
4722 log_debug(gc, stats)(" (Survivor:" SIZE_FORMAT "chunks) ", i);
4723 // Verify that we used up all the recorded entries
4724 #ifdef ASSERT
4725 size_t total = 0;
4726 for (int j = 0; j < no_of_gc_threads; j++) {
4727 assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
4728 total += _cursor[j];
4729 }
4730 assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
4731 // Check that the merged array is in sorted order
4732 if (total > 0) {
4733 for (size_t i = 0; i < total - 1; i++) {
4734 log_develop(gc, stats)(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
4735 i, p2i(_survivor_chunk_array[i]));
4736 assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
4737 "Not sorted");
4738 }
4739 }
4740 #endif // ASSERT
4741 }
4742
4743 // Set up the space's par_seq_tasks structure for work claiming
4744 // for parallel initial scan and rescan of young gen.
4745 // See ParRescanTask where this is currently used.
4746 void
4747 CMSCollector::
4748 initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
4749 assert(n_threads > 0, "Unexpected n_threads argument");
4750
4751 // Eden space
4752 if (!_young_gen->eden()->is_empty()) {
4753 SequentialSubTasksDone* pst = _young_gen->eden()->par_seq_tasks();
4754 assert(!pst->valid(), "Clobbering existing data?");
4755 // Each valid entry in [0, _eden_chunk_index) represents a task.
4849 // as a result of work_q overflow
4850 restore_preserved_marks_if_any();
4851 }
4852
4853 // Non-parallel version of remark
4854 void CMSCollector::do_remark_non_parallel() {
4855 ResourceMark rm;
4856 HandleMark hm;
4857 GenCollectedHeap* gch = GenCollectedHeap::heap();
4858 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
4859
4860 MarkRefsIntoAndScanClosure
4861 mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */,
4862 &_markStack, this,
4863 false /* should_yield */, false /* not precleaning */);
4864 MarkFromDirtyCardsClosure
4865 markFromDirtyCardsClosure(this, _span,
4866 NULL, // space is set further below
4867 &_markBitMap, &_markStack, &mrias_cl);
4868 {
4869 GCTraceTime(Trace, gc) t("Grey Object Rescan", _gc_timer_cm);
4870 // Iterate over the dirty cards, setting the corresponding bits in the
4871 // mod union table.
4872 {
4873 ModUnionClosure modUnionClosure(&_modUnionTable);
4874 _ct->ct_bs()->dirty_card_iterate(
4875 _cmsGen->used_region(),
4876 &modUnionClosure);
4877 }
4878 // Having transferred these marks into the modUnionTable, we just need
4879 // to rescan the marked objects on the dirty cards in the modUnionTable.
4880 // The initial marking may have been done during an asynchronous
4881 // collection so there may be dirty bits in the mod-union table.
4882 const int alignment =
4883 CardTableModRefBS::card_size * BitsPerWord;
4884 {
4885 // ... First handle dirty cards in CMS gen
4886 markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
4887 MemRegion ur = _cmsGen->used_region();
4888 HeapWord* lb = ur.start();
4889 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
4890 MemRegion cms_span(lb, ub);
4891 _modUnionTable.dirty_range_iterate_clear(cms_span,
4892 &markFromDirtyCardsClosure);
4893 verify_work_stacks_empty();
4894 log_debug(gc, stats)(" (re-scanned " SIZE_FORMAT " dirty cards in cms gen) ", markFromDirtyCardsClosure.num_dirty_cards());
4895 }
4896 }
4897 if (VerifyDuringGC &&
4898 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4899 HandleMark hm; // Discard invalid handles created during verification
4900 Universe::verify();
4901 }
4902 {
4903 GCTraceTime(Trace, gc) t("Root Rescan", _gc_timer_cm);
4904
4905 verify_work_stacks_empty();
4906
4907 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
4908 StrongRootsScope srs(1);
4909
4910 gch->gen_process_roots(&srs,
4911 GenCollectedHeap::OldGen,
4912 true, // young gen as roots
4913 GenCollectedHeap::ScanningOption(roots_scanning_options()),
4914 should_unload_classes(),
4915 &mrias_cl,
4916 NULL,
4917 NULL); // The dirty klasses will be handled below
4918
4919 assert(should_unload_classes()
4920 || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4921 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4922 }
4923
4924 {
4925 GCTraceTime(Trace, gc) t("Visit Unhandled CLDs", _gc_timer_cm);
4926
4927 verify_work_stacks_empty();
4928
4929 // Scan all class loader data objects that might have been introduced
4930 // during concurrent marking.
4931 ResourceMark rm;
4932 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
4933 for (int i = 0; i < array->length(); i++) {
4934 mrias_cl.do_cld_nv(array->at(i));
4935 }
4936
4937 // We don't need to keep track of new CLDs anymore.
4938 ClassLoaderDataGraph::remember_new_clds(false);
4939
4940 verify_work_stacks_empty();
4941 }
4942
4943 {
4944 GCTraceTime(Trace, gc) t("Dirty Klass Scan", _gc_timer_cm);
4945
4946 verify_work_stacks_empty();
4947
4948 RemarkKlassClosure remark_klass_closure(&mrias_cl);
4949 ClassLoaderDataGraph::classes_do(&remark_klass_closure);
4950
4951 verify_work_stacks_empty();
4952 }
4953
4954 // We might have added oops to ClassLoaderData::_handles during the
4955 // concurrent marking phase. These oops point to newly allocated objects
4956 // that are guaranteed to be kept alive. Either by the direct allocation
4957 // code, or when the young collector processes the roots. Hence,
4958 // we don't have to revisit the _handles block during the remark phase.
4959
4960 verify_work_stacks_empty();
4961 // Restore evacuated mark words, if any, used for overflow list links
4962 restore_preserved_marks_if_any();
4963
4964 verify_overflow_empty();
5086 // We'd like to assert(work_q->size() != 0, ...)
5087 // because we just took work from the overflow list,
5088 // but of course we can't, since all of that might have
5089 // been already stolen from us.
5090 continue;
5091 }
5092 // Verify that we have no work before we resort to stealing
5093 assert(work_q->size() == 0, "Have work, shouldn't steal");
5094 // Try to steal from other queues that have work
5095 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5096 NOT_PRODUCT(num_steals++;)
5097 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5098 assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5099 // Do scanning work
5100 obj_to_scan->oop_iterate(keep_alive);
5101 // Loop around, finish this work, and try to steal some more
5102 } else if (terminator()->offer_termination()) {
5103 break; // nirvana from the infinite cycle
5104 }
5105 }
5106 log_develop(gc, stats)("\t(%d: stole %d oops)", i, num_steals);
5107 }
5108
5109 void CMSRefProcTaskExecutor::execute(ProcessTask& task)
5110 {
5111 GenCollectedHeap* gch = GenCollectedHeap::heap();
5112 WorkGang* workers = gch->workers();
5113 assert(workers != NULL, "Need parallel worker threads.");
5114 CMSRefProcTaskProxy rp_task(task, &_collector,
5115 _collector.ref_processor()->span(),
5116 _collector.markBitMap(),
5117 workers, _collector.task_queues());
5118 workers->run_task(&rp_task);
5119 }
5120
5121 void CMSRefProcTaskExecutor::execute(EnqueueTask& task)
5122 {
5123
5124 GenCollectedHeap* gch = GenCollectedHeap::heap();
5125 WorkGang* workers = gch->workers();
5126 assert(workers != NULL, "Need parallel worker threads.");
5128 workers->run_task(&enq_task);
5129 }
5130
5131 void CMSCollector::refProcessingWork() {
5132 ResourceMark rm;
5133 HandleMark hm;
5134
5135 ReferenceProcessor* rp = ref_processor();
5136 assert(rp->span().equals(_span), "Spans should be equal");
5137 assert(!rp->enqueuing_is_done(), "Enqueuing should not be complete");
5138 // Process weak references.
5139 rp->setup_policy(false);
5140 verify_work_stacks_empty();
5141
5142 CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap,
5143 &_markStack, false /* !preclean */);
5144 CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this,
5145 _span, &_markBitMap, &_markStack,
5146 &cmsKeepAliveClosure, false /* !preclean */);
5147 {
5148 GCTraceTime(Debug, gc) t("Weak Refs Processing", _gc_timer_cm);
5149
5150 ReferenceProcessorStats stats;
5151 if (rp->processing_is_mt()) {
5152 // Set the degree of MT here. If the discovery is done MT, there
5153 // may have been a different number of threads doing the discovery
5154 // and a different number of discovered lists may have Ref objects.
5155 // That is OK as long as the Reference lists are balanced (see
5156 // balance_all_queues() and balance_queues()).
5157 GenCollectedHeap* gch = GenCollectedHeap::heap();
5158 uint active_workers = ParallelGCThreads;
5159 WorkGang* workers = gch->workers();
5160 if (workers != NULL) {
5161 active_workers = workers->active_workers();
5162 // The expectation is that active_workers will have already
5163 // been set to a reasonable value. If it has not been set,
5164 // investigate.
5165 assert(active_workers > 0, "Should have been set during scavenge");
5166 }
5167 rp->set_active_mt_degree(active_workers);
5168 CMSRefProcTaskExecutor task_executor(*this);
5170 &cmsKeepAliveClosure,
5171 &cmsDrainMarkingStackClosure,
5172 &task_executor,
5173 _gc_timer_cm);
5174 } else {
5175 stats = rp->process_discovered_references(&_is_alive_closure,
5176 &cmsKeepAliveClosure,
5177 &cmsDrainMarkingStackClosure,
5178 NULL,
5179 _gc_timer_cm);
5180 }
5181 _gc_tracer_cm->report_gc_reference_stats(stats);
5182
5183 }
5184
5185 // This is the point where the entire marking should have completed.
5186 verify_work_stacks_empty();
5187
5188 if (should_unload_classes()) {
5189 {
5190 GCTraceTime(Debug, gc) t("Class Unloading", _gc_timer_cm);
5191
5192 // Unload classes and purge the SystemDictionary.
5193 bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure);
5194
5195 // Unload nmethods.
5196 CodeCache::do_unloading(&_is_alive_closure, purged_class);
5197
5198 // Prune dead klasses from subklass/sibling/implementor lists.
5199 Klass::clean_weak_klass_links(&_is_alive_closure);
5200 }
5201
5202 {
5203 GCTraceTime(Debug, gc) t("Scrub Symbol Table", _gc_timer_cm);
5204 // Clean up unreferenced symbols in symbol table.
5205 SymbolTable::unlink();
5206 }
5207
5208 {
5209 GCTraceTime(Debug, gc) t("Scrub String Table", _gc_timer_cm);
5210 // Delete entries for dead interned strings.
5211 StringTable::unlink(&_is_alive_closure);
5212 }
5213 }
5214
5215
5216 // Restore any preserved marks as a result of mark stack or
5217 // work queue overflow
5218 restore_preserved_marks_if_any(); // done single-threaded for now
5219
5220 rp->set_enqueuing_is_done(true);
5221 if (rp->processing_is_mt()) {
5222 rp->balance_all_queues();
5223 CMSRefProcTaskExecutor task_executor(*this);
5224 rp->enqueue_discovered_references(&task_executor);
5225 } else {
5226 rp->enqueue_discovered_references(NULL);
5227 }
5228 rp->verify_no_references_recorded();
5229 assert(!rp->discovery_enabled(), "should have been disabled");
5256 }
5257 }
5258 }
5259 #endif
5260
5261 void CMSCollector::sweep() {
5262 assert(_collectorState == Sweeping, "just checking");
5263 check_correct_thread_executing();
5264 verify_work_stacks_empty();
5265 verify_overflow_empty();
5266 increment_sweep_count();
5267 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
5268
5269 _inter_sweep_timer.stop();
5270 _inter_sweep_estimate.sample(_inter_sweep_timer.seconds());
5271
5272 assert(!_intra_sweep_timer.is_active(), "Should not be active");
5273 _intra_sweep_timer.reset();
5274 _intra_sweep_timer.start();
5275 {
5276 GCTraceCPUTime tcpu;
5277 CMSPhaseAccounting pa(this, "Concurrent Sweep");
5278 // First sweep the old gen
5279 {
5280 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
5281 bitMapLock());
5282 sweepWork(_cmsGen);
5283 }
5284
5285 // Update Universe::_heap_*_at_gc figures.
5286 // We need all the free list locks to make the abstract state
5287 // transition from Sweeping to Resetting. See detailed note
5288 // further below.
5289 {
5290 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock());
5291 // Update heap occupancy information which is used as
5292 // input to soft ref clearing policy at the next gc.
5293 Universe::update_heap_info_at_gc();
5294 _collectorState = Resizing;
5295 }
5296 }
5297 verify_work_stacks_empty();
5340 GenCollectedHeap* gch = GenCollectedHeap::heap();
5341 gch->clear_incremental_collection_failed(); // Worth retrying as fresh space may have been freed up
5342 gch->update_full_collections_completed(_collection_count_start);
5343 }
5344
5345 // FIX ME!!! Looks like this belongs in CFLSpace, with
5346 // CMSGen merely delegating to it.
5347 void ConcurrentMarkSweepGeneration::setNearLargestChunk() {
5348 double nearLargestPercent = FLSLargestBlockCoalesceProximity;
5349 HeapWord* minAddr = _cmsSpace->bottom();
5350 HeapWord* largestAddr =
5351 (HeapWord*) _cmsSpace->dictionary()->find_largest_dict();
5352 if (largestAddr == NULL) {
5353 // The dictionary appears to be empty. In this case
5354 // try to coalesce at the end of the heap.
5355 largestAddr = _cmsSpace->end();
5356 }
5357 size_t largestOffset = pointer_delta(largestAddr, minAddr);
5358 size_t nearLargestOffset =
5359 (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
5360 log_debug(gc, freelist, stats)("CMS: Large Block: " PTR_FORMAT "; Proximity: " PTR_FORMAT " -> " PTR_FORMAT,
5361 p2i(largestAddr), p2i(_cmsSpace->nearLargestChunk()), p2i(minAddr + nearLargestOffset));
5362 _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
5363 }
5364
5365 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
5366 return addr >= _cmsSpace->nearLargestChunk();
5367 }
5368
5369 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
5370 return _cmsSpace->find_chunk_at_end();
5371 }
5372
5373 void ConcurrentMarkSweepGeneration::update_gc_stats(Generation* current_generation,
5374 bool full) {
5375 // If the young generation has been collected, gather any statistics
5376 // that are of interest at this point.
5377 bool current_is_young = GenCollectedHeap::heap()->is_young_gen(current_generation);
5378 if (!full && current_is_young) {
5379 // Gather statistics on the young generation collection.
5380 collector()->stats().record_gc0_end(used());
5381 }
5435 } else { // did not unload classes,
5436 _concurrent_cycles_since_last_unload++; // ... increment count
5437 }
5438 }
5439
5440 // Reset CMS data structures (for now just the marking bit map)
5441 // preparatory for the next cycle.
5442 void CMSCollector::reset_concurrent() {
5443 CMSTokenSyncWithLocks ts(true, bitMapLock());
5444
5445 // If the state is not "Resetting", the foreground thread
5446 // has done a collection and the resetting.
5447 if (_collectorState != Resetting) {
5448 assert(_collectorState == Idling, "The state should only change"
5449 " because the foreground collector has finished the collection");
5450 return;
5451 }
5452
5453 // Clear the mark bitmap (no grey objects to start with)
5454 // for the next cycle.
5455 GCTraceCPUTime tcpu;
5456 CMSPhaseAccounting cmspa(this, "Concurrent Reset");
5457
5458 HeapWord* curAddr = _markBitMap.startWord();
5459 while (curAddr < _markBitMap.endWord()) {
5460 size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr);
5461 MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining));
5462 _markBitMap.clear_large_range(chunk);
5463 if (ConcurrentMarkSweepThread::should_yield() &&
5464 !foregroundGCIsActive() &&
5465 CMSYield) {
5466 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
5467 "CMS thread should hold CMS token");
5468 assert_lock_strong(bitMapLock());
5469 bitMapLock()->unlock();
5470 ConcurrentMarkSweepThread::desynchronize(true);
5471 stopTimer();
5472 incrementYields();
5473
5474 // See the comment in coordinator_yield()
5475 for (unsigned i = 0; i < CMSYieldSleepCount &&
5476 ConcurrentMarkSweepThread::should_yield() &&
5477 !CMSCollector::foregroundGCIsActive(); ++i) {
5478 os::sleep(Thread::current(), 1, false);
5479 }
5480
5481 ConcurrentMarkSweepThread::synchronize(true);
5482 bitMapLock()->lock_without_safepoint_check();
5483 startTimer();
5484 }
5485 curAddr = chunk.end();
5486 }
5487 // A successful mostly concurrent collection has been done.
5488 // Because only the full (i.e., concurrent mode failure) collections
5489 // are being measured for gc overhead limits, clean the "near" flag
5490 // and count.
5491 size_policy()->reset_gc_overhead_limit_count();
5492 _collectorState = Idling;
5493
5494 register_gc_end();
5495 }
5496
5497 // Same as above but for STW paths
5498 void CMSCollector::reset_stw() {
5499 // already have the lock
5500 assert(_collectorState == Resetting, "just checking");
5501 assert_lock_strong(bitMapLock());
5502 GCIdMarkAndRestore gc_id_mark(_cmsThread->gc_id());
5503 _markBitMap.clear_all();
5504 _collectorState = Idling;
5505 register_gc_end();
5506 }
5507
5508 void CMSCollector::do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause) {
5509 GCTraceCPUTime tcpu;
5510 TraceCollectorStats tcs(counters());
5511
5512 switch (op) {
5513 case CMS_op_checkpointRootsInitial: {
5514 GCTraceTime(Info, gc) t("Pause Initial Mark", NULL, GCCause::_no_gc, true);
5515 SvcGCMarker sgcm(SvcGCMarker::OTHER);
5516 checkpointRootsInitial();
5517 break;
5518 }
5519 case CMS_op_checkpointRootsFinal: {
5520 GCTraceTime(Info, gc) t("Pause Remark", NULL, GCCause::_no_gc, true);
5521 SvcGCMarker sgcm(SvcGCMarker::OTHER);
5522 checkpointRootsFinal();
5523 break;
5524 }
5525 default:
5526 fatal("No such CMS_op");
5527 }
5528 }
5529
5530 #ifndef PRODUCT
5531 size_t const CMSCollector::skip_header_HeapWords() {
5532 return FreeChunk::header_size();
5533 }
5534
5535 // Try and collect here conditions that should hold when
5536 // CMS thread is exiting. The idea is that the foreground GC
5537 // thread should not be blocked if it wants to terminate
5538 // the CMS thread and yet continue to run the VM for a while
5539 // after that.
5540 void CMSCollector::verify_ok_to_terminate() const {
5541 assert(Thread::current()->is_ConcurrentGC_thread(),
5542 "should be called by CMS thread");
5715 }
5716 assert(_virtual_space.committed_size() == rs.size(),
5717 "didn't reserve backing store for all of CMS stack?");
5718 _base = (oop*)(_virtual_space.low());
5719 _index = 0;
5720 _capacity = size;
5721 NOT_PRODUCT(_max_depth = 0);
5722 return true;
5723 }
5724
5725 // XXX FIX ME !!! In the MT case we come in here holding a
5726 // leaf lock. For printing we need to take a further lock
5727 // which has lower rank. We need to recalibrate the two
5728 // lock-ranks involved in order to be able to print the
5729 // messages below. (Or defer the printing to the caller.
5730 // For now we take the expedient path of just disabling the
5731 // messages for the problematic case.)
5732 void CMSMarkStack::expand() {
5733 assert(_capacity <= MarkStackSizeMax, "stack bigger than permitted");
5734 if (_capacity == MarkStackSizeMax) {
5735 if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled) {
5736 // We print a warning message only once per CMS cycle.
5737 log_debug(gc)(" (benign) Hit CMSMarkStack max size limit");
5738 }
5739 return;
5740 }
5741 // Double capacity if possible
5742 size_t new_capacity = MIN2(_capacity*2, MarkStackSizeMax);
5743 // Do not give up existing stack until we have managed to
5744 // get the double capacity that we desired.
5745 ReservedSpace rs(ReservedSpace::allocation_align_size_up(
5746 new_capacity * sizeof(oop)));
5747 if (rs.is_reserved()) {
5748 // Release the backing store associated with old stack
5749 _virtual_space.release();
5750 // Reinitialize virtual space for new stack
5751 if (!_virtual_space.initialize(rs, rs.size())) {
5752 fatal("Not enough swap for expanded marking stack");
5753 }
5754 _base = (oop*)(_virtual_space.low());
5755 _index = 0;
5756 _capacity = new_capacity;
5757 } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled) {
5758 // Failed to double capacity, continue;
5759 // we print a detail message only once per CMS cycle.
5760 log_debug(gc)(" (benign) Failed to expand marking stack from " SIZE_FORMAT "K to " SIZE_FORMAT "K",
5761 _capacity / K, new_capacity / K);
5762 }
5763 }
5764
5765
5766 // Closures
5767 // XXX: there seems to be a lot of code duplication here;
5768 // should refactor and consolidate common code.
5769
5770 // This closure is used to mark refs into the CMS generation in
5771 // the CMS bit map. Called at the first checkpoint. This closure
5772 // assumes that we do not need to re-mark dirty cards; if the CMS
5773 // generation on which this is used is not an oldest
5774 // generation then this will lose younger_gen cards!
5775
5776 MarkRefsIntoClosure::MarkRefsIntoClosure(
5777 MemRegion span, CMSBitMap* bitMap):
5778 _span(span),
5779 _bitMap(bitMap)
5780 {
5818 void Par_MarkRefsIntoClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); }
5819
5820 // A variant of the above, used for CMS marking verification.
5821 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
5822 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
5823 _span(span),
5824 _verification_bm(verification_bm),
5825 _cms_bm(cms_bm)
5826 {
5827 assert(ref_processor() == NULL, "deliberately left NULL");
5828 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
5829 }
5830
5831 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
5832 // if p points into _span, then mark corresponding bit in _markBitMap
5833 assert(obj->is_oop(), "expected an oop");
5834 HeapWord* addr = (HeapWord*)obj;
5835 if (_span.contains(addr)) {
5836 _verification_bm->mark(addr);
5837 if (!_cms_bm->isMarked(addr)) {
5838 LogHandle(gc, verify) log;
5839 ResourceMark rm;
5840 oop(addr)->print_on(log.info_stream());
5841 log.info(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
5842 fatal("... aborting");
5843 }
5844 }
5845 }
5846
5847 void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5848 void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
5849
5850 //////////////////////////////////////////////////
5851 // MarkRefsIntoAndScanClosure
5852 //////////////////////////////////////////////////
5853
5854 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
5855 ReferenceProcessor* rp,
5856 CMSBitMap* bit_map,
5857 CMSBitMap* mod_union_table,
5858 CMSMarkStack* mark_stack,
5859 CMSCollector* collector,
5860 bool should_yield,
5861 bool concurrent_precleaning):
5917 "overflow list was drained above");
5918
5919 assert(_collector->no_preserved_marks(),
5920 "All preserved marks should have been restored above");
5921 }
5922 }
5923
5924 void MarkRefsIntoAndScanClosure::do_oop(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
5925 void MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
5926
5927 void MarkRefsIntoAndScanClosure::do_yield_work() {
5928 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
5929 "CMS thread should hold CMS token");
5930 assert_lock_strong(_freelistLock);
5931 assert_lock_strong(_bit_map->lock());
5932 // relinquish the free_list_lock and bitMaplock()
5933 _bit_map->lock()->unlock();
5934 _freelistLock->unlock();
5935 ConcurrentMarkSweepThread::desynchronize(true);
5936 _collector->stopTimer();
5937 _collector->incrementYields();
5938
5939 // See the comment in coordinator_yield()
5940 for (unsigned i = 0;
5941 i < CMSYieldSleepCount &&
5942 ConcurrentMarkSweepThread::should_yield() &&
5943 !CMSCollector::foregroundGCIsActive();
5944 ++i) {
5945 os::sleep(Thread::current(), 1, false);
5946 }
5947
5948 ConcurrentMarkSweepThread::synchronize(true);
5949 _freelistLock->lock_without_safepoint_check();
5950 _bit_map->lock()->lock_without_safepoint_check();
5951 _collector->startTimer();
5952 }
5953
5954 ///////////////////////////////////////////////////////////
5955 // Par_MarkRefsIntoAndScanClosure: a parallel version of
5956 // MarkRefsIntoAndScanClosure
5957 ///////////////////////////////////////////////////////////
6073 // An object not (yet) reached by marking: we merely need to
6074 // compute its size so as to go look at the next block.
6075 assert(p->is_oop(true), "should be an oop");
6076 size = CompactibleFreeListSpace::adjustObjectSize(p->size());
6077 }
6078 }
6079 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6080 return size;
6081 }
6082
6083 void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
6084 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6085 "CMS thread should hold CMS token");
6086 assert_lock_strong(_freelistLock);
6087 assert_lock_strong(_bitMap->lock());
6088 // relinquish the free_list_lock and bitMaplock()
6089 _bitMap->lock()->unlock();
6090 _freelistLock->unlock();
6091 ConcurrentMarkSweepThread::desynchronize(true);
6092 _collector->stopTimer();
6093 _collector->incrementYields();
6094
6095 // See the comment in coordinator_yield()
6096 for (unsigned i = 0; i < CMSYieldSleepCount &&
6097 ConcurrentMarkSweepThread::should_yield() &&
6098 !CMSCollector::foregroundGCIsActive(); ++i) {
6099 os::sleep(Thread::current(), 1, false);
6100 }
6101
6102 ConcurrentMarkSweepThread::synchronize(true);
6103 _freelistLock->lock_without_safepoint_check();
6104 _bitMap->lock()->lock_without_safepoint_check();
6105 _collector->startTimer();
6106 }
6107
6108
6109 //////////////////////////////////////////////////////////////////
6110 // SurvivorSpacePrecleanClosure
6111 //////////////////////////////////////////////////////////////////
6112 // This (single-threaded) closure is used to preclean the oops in
6113 // the survivor spaces.
6140 // the ones in CMS heap (i.e. in _span).
6141 new_oop->oop_iterate(_scanning_closure);
6142 // check if it's time to yield
6143 do_yield_check();
6144 }
6145 unsigned int after_count =
6146 GenCollectedHeap::heap()->total_collections();
6147 bool abort = (_before_count != after_count) ||
6148 _collector->should_abort_preclean();
6149 return abort ? 0 : size;
6150 }
6151
6152 void SurvivorSpacePrecleanClosure::do_yield_work() {
6153 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6154 "CMS thread should hold CMS token");
6155 assert_lock_strong(_bit_map->lock());
6156 // Relinquish the bit map lock
6157 _bit_map->lock()->unlock();
6158 ConcurrentMarkSweepThread::desynchronize(true);
6159 _collector->stopTimer();
6160 _collector->incrementYields();
6161
6162 // See the comment in coordinator_yield()
6163 for (unsigned i = 0; i < CMSYieldSleepCount &&
6164 ConcurrentMarkSweepThread::should_yield() &&
6165 !CMSCollector::foregroundGCIsActive(); ++i) {
6166 os::sleep(Thread::current(), 1, false);
6167 }
6168
6169 ConcurrentMarkSweepThread::synchronize(true);
6170 _bit_map->lock()->lock_without_safepoint_check();
6171 _collector->startTimer();
6172 }
6173
6174 // This closure is used to rescan the marked objects on the dirty cards
6175 // in the mod union table and the card table proper. In the parallel
6176 // case, although the bitMap is shared, we do a single read so the
6177 // isMarked() query is "safe".
6178 bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) {
6179 // Ignore mark word because we are running concurrent with mutators
6180 assert(p->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(p));
6293 return true;
6294 }
6295 }
6296 scanOopsInOop(addr);
6297 return true;
6298 }
6299
6300 // We take a break if we've been at this for a while,
6301 // so as to avoid monopolizing the locks involved.
6302 void MarkFromRootsClosure::do_yield_work() {
6303 // First give up the locks, then yield, then re-lock
6304 // We should probably use a constructor/destructor idiom to
6305 // do this unlock/lock or modify the MutexUnlocker class to
6306 // serve our purpose. XXX
6307 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6308 "CMS thread should hold CMS token");
6309 assert_lock_strong(_bitMap->lock());
6310 _bitMap->lock()->unlock();
6311 ConcurrentMarkSweepThread::desynchronize(true);
6312 _collector->stopTimer();
6313 _collector->incrementYields();
6314
6315 // See the comment in coordinator_yield()
6316 for (unsigned i = 0; i < CMSYieldSleepCount &&
6317 ConcurrentMarkSweepThread::should_yield() &&
6318 !CMSCollector::foregroundGCIsActive(); ++i) {
6319 os::sleep(Thread::current(), 1, false);
6320 }
6321
6322 ConcurrentMarkSweepThread::synchronize(true);
6323 _bitMap->lock()->lock_without_safepoint_check();
6324 _collector->startTimer();
6325 }
6326
6327 void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) {
6328 assert(_bitMap->isMarked(ptr), "expected bit to be set");
6329 assert(_markStack->isEmpty(),
6330 "should drain stack to limit stack usage");
6331 // convert ptr to an oop preparatory to scanning
6332 oop obj = oop(ptr);
6333 // Ignore mark word in verification below, since we
6599 void PushAndMarkVerifyClosure::do_oop(narrowOop* p) { PushAndMarkVerifyClosure::do_oop_work(p); }
6600
6601 // Upon stack overflow, we discard (part of) the stack,
6602 // remembering the least address amongst those discarded
6603 // in CMSCollector's _restart_address.
6604 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
6605 // Remember the least grey address discarded
6606 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
6607 _collector->lower_restart_addr(ra);
6608 _mark_stack->reset(); // discard stack contents
6609 _mark_stack->expand(); // expand the stack if possible
6610 }
6611
6612 void PushAndMarkVerifyClosure::do_oop(oop obj) {
6613 assert(obj->is_oop_or_null(), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6614 HeapWord* addr = (HeapWord*)obj;
6615 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
6616 // Oop lies in _span and isn't yet grey or black
6617 _verification_bm->mark(addr); // now grey
6618 if (!_cms_bm->isMarked(addr)) {
6619 LogHandle(gc, verify) log;
6620 ResourceMark rm;
6621 oop(addr)->print_on(log.info_stream());
6622 log.info(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
6623 fatal("... aborting");
6624 }
6625
6626 if (!_mark_stack->push(obj)) { // stack overflow
6627 log_debug(gc, stats)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _mark_stack->capacity());
6628 assert(_mark_stack->isFull(), "Else push should have succeeded");
6629 handle_stack_overflow(addr);
6630 }
6631 // anything including and to the right of _finger
6632 // will be scanned as we iterate over the remainder of the
6633 // bit map
6634 }
6635 }
6636
6637 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
6638 MemRegion span,
6639 CMSBitMap* bitMap, CMSMarkStack* markStack,
6640 HeapWord* finger, MarkFromRootsClosure* parent) :
6641 MetadataAwareOopClosure(collector->ref_processor()),
6642 _collector(collector),
6643 _span(span),
6644 _bitMap(bitMap),
6645 _markStack(markStack),
6646 _finger(finger),
6647 _parent(parent)
6707 void PushOrMarkClosure::do_oop(oop obj) {
6708 // Ignore mark word because we are running concurrent with mutators.
6709 assert(obj->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6710 HeapWord* addr = (HeapWord*)obj;
6711 if (_span.contains(addr) && !_bitMap->isMarked(addr)) {
6712 // Oop lies in _span and isn't yet grey or black
6713 _bitMap->mark(addr); // now grey
6714 if (addr < _finger) {
6715 // the bit map iteration has already either passed, or
6716 // sampled, this bit in the bit map; we'll need to
6717 // use the marking stack to scan this oop's oops.
6718 bool simulate_overflow = false;
6719 NOT_PRODUCT(
6720 if (CMSMarkStackOverflowALot &&
6721 _collector->simulate_overflow()) {
6722 // simulate a stack overflow
6723 simulate_overflow = true;
6724 }
6725 )
6726 if (simulate_overflow || !_markStack->push(obj)) { // stack overflow
6727 log_debug(gc, stats)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _markStack->capacity());
6728 assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded");
6729 handle_stack_overflow(addr);
6730 }
6731 }
6732 // anything including and to the right of _finger
6733 // will be scanned as we iterate over the remainder of the
6734 // bit map
6735 do_yield_check();
6736 }
6737 }
6738
6739 void PushOrMarkClosure::do_oop(oop* p) { PushOrMarkClosure::do_oop_work(p); }
6740 void PushOrMarkClosure::do_oop(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); }
6741
6742 void Par_PushOrMarkClosure::do_oop(oop obj) {
6743 // Ignore mark word because we are running concurrent with mutators.
6744 assert(obj->is_oop_or_null(true), "Expected an oop or NULL at " PTR_FORMAT, p2i(obj));
6745 HeapWord* addr = (HeapWord*)obj;
6746 if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) {
6747 // Oop lies in _span and isn't yet grey or black
6756 // -- else push on work queue
6757 if ( !res // someone else marked it, they will deal with it
6758 || (addr >= *gfa) // will be scanned in a later task
6759 || (_span.contains(addr) && addr >= _finger)) { // later in this chunk
6760 return;
6761 }
6762 // the bit map iteration has already either passed, or
6763 // sampled, this bit in the bit map; we'll need to
6764 // use the marking stack to scan this oop's oops.
6765 bool simulate_overflow = false;
6766 NOT_PRODUCT(
6767 if (CMSMarkStackOverflowALot &&
6768 _collector->simulate_overflow()) {
6769 // simulate a stack overflow
6770 simulate_overflow = true;
6771 }
6772 )
6773 if (simulate_overflow ||
6774 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
6775 // stack overflow
6776 log_debug(gc, stats)("CMS marking stack overflow (benign) at " SIZE_FORMAT, _overflow_stack->capacity());
6777 // We cannot assert that the overflow stack is full because
6778 // it may have been emptied since.
6779 assert(simulate_overflow ||
6780 _work_queue->size() == _work_queue->max_elems(),
6781 "Else push should have succeeded");
6782 handle_stack_overflow(addr);
6783 }
6784 do_yield_check();
6785 }
6786 }
6787
6788 void Par_PushOrMarkClosure::do_oop(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
6789 void Par_PushOrMarkClosure::do_oop(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
6790
6791 PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector,
6792 MemRegion span,
6793 ReferenceProcessor* rp,
6794 CMSBitMap* bit_map,
6795 CMSBitMap* mod_union_table,
6796 CMSMarkStack* mark_stack,
6918 _collector->par_push_on_overflow_list(obj);
6919 _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS
6920 }
6921 } // Else, some other thread got there first
6922 }
6923 }
6924
6925 void Par_PushAndMarkClosure::do_oop(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
6926 void Par_PushAndMarkClosure::do_oop(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
6927
6928 void CMSPrecleanRefsYieldClosure::do_yield_work() {
6929 Mutex* bml = _collector->bitMapLock();
6930 assert_lock_strong(bml);
6931 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6932 "CMS thread should hold CMS token");
6933
6934 bml->unlock();
6935 ConcurrentMarkSweepThread::desynchronize(true);
6936
6937 _collector->stopTimer();
6938 _collector->incrementYields();
6939
6940 // See the comment in coordinator_yield()
6941 for (unsigned i = 0; i < CMSYieldSleepCount &&
6942 ConcurrentMarkSweepThread::should_yield() &&
6943 !CMSCollector::foregroundGCIsActive(); ++i) {
6944 os::sleep(Thread::current(), 1, false);
6945 }
6946
6947 ConcurrentMarkSweepThread::synchronize(true);
6948 bml->lock();
6949
6950 _collector->startTimer();
6951 }
6952
6953 bool CMSPrecleanRefsYieldClosure::should_return() {
6954 if (ConcurrentMarkSweepThread::should_yield()) {
6955 do_yield_work();
6956 }
6957 return _collector->foregroundGCIsActive();
6958 }
6959
6960 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
6961 assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
6962 "mr should be aligned to start at a card boundary");
6963 // We'd like to assert:
6964 // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
6965 // "mr should be a range of cards");
6966 // However, that would be too strong in one case -- the last
6967 // partition ends at _unallocated_block which, in general, can be
6968 // an arbitrary boundary, not necessarily card aligned.
6969 _num_dirty_cards += mr.word_size()/CardTableModRefBS::card_size_in_words;
6970 _space->object_iterate_mem(mr, &_scan_cl);
6971 }
6972
6973 SweepClosure::SweepClosure(CMSCollector* collector,
6974 ConcurrentMarkSweepGeneration* g,
6975 CMSBitMap* bitMap, bool should_yield) :
6976 _collector(collector),
6977 _g(g),
6978 _sp(g->cmsSpace()),
6979 _limit(_sp->sweep_limit()),
6980 _freelistLock(_sp->freelistLock()),
6981 _bitMap(bitMap),
6982 _yield(should_yield),
6983 _inFreeRange(false), // No free range at beginning of sweep
6984 _freeRangeInFreeLists(false), // No free range at beginning of sweep
6985 _lastFreeRangeCoalesced(false),
6986 _freeFinger(g->used_region().start())
6987 {
6988 NOT_PRODUCT(
6989 _numObjectsFreed = 0;
6990 _numWordsFreed = 0;
6991 _numObjectsLive = 0;
6992 _numWordsLive = 0;
6993 _numObjectsAlreadyFree = 0;
6994 _numWordsAlreadyFree = 0;
6995 _last_fc = NULL;
6996
6997 _sp->initializeIndexedFreeListArrayReturnedBytes();
6998 _sp->dictionary()->initialize_dict_returned_bytes();
6999 )
7000 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7001 "sweep _limit out of bounds");
7002 log_develop(gc, sweep)("====================");
7003 log_develop(gc, sweep)("Starting new sweep with limit " PTR_FORMAT, p2i(_limit));
7004 }
7005
7006 void SweepClosure::print_on(outputStream* st) const {
7007 tty->print_cr("_sp = [" PTR_FORMAT "," PTR_FORMAT ")",
7008 p2i(_sp->bottom()), p2i(_sp->end()));
7009 tty->print_cr("_limit = " PTR_FORMAT, p2i(_limit));
7010 tty->print_cr("_freeFinger = " PTR_FORMAT, p2i(_freeFinger));
7011 NOT_PRODUCT(tty->print_cr("_last_fc = " PTR_FORMAT, p2i(_last_fc));)
7012 tty->print_cr("_inFreeRange = %d, _freeRangeInFreeLists = %d, _lastFreeRangeCoalesced = %d",
7013 _inFreeRange, _freeRangeInFreeLists, _lastFreeRangeCoalesced);
7014 }
7015
7016 #ifndef PRODUCT
7017 // Assertion checking only: no useful work in product mode --
7018 // however, if any of the flags below become product flags,
7019 // you may need to review this code to see if it needs to be
7020 // enabled in product mode.
7021 SweepClosure::~SweepClosure() {
7022 assert_lock_strong(_freelistLock);
7023 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7024 "sweep _limit out of bounds");
7025 if (inFreeRange()) {
7026 warning("inFreeRange() should have been reset; dumping state of SweepClosure");
7027 print();
7028 ShouldNotReachHere();
7029 }
7030
7031 if (log_is_enabled(Debug, gc, sweep)) {
7032 log_debug(gc, sweep)("Collected " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7033 _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
7034 log_debug(gc, sweep)("Live " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes Already free " SIZE_FORMAT " objects, " SIZE_FORMAT " bytes",
7035 _numObjectsLive, _numWordsLive*sizeof(HeapWord), _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
7036 size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree) * sizeof(HeapWord);
7037 log_debug(gc, sweep)("Total sweep: " SIZE_FORMAT " bytes", totalBytes);
7038 }
7039
7040 if (log_is_enabled(Debug, gc, stats) && CMSVerifyReturnedBytes) {
7041 size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
7042 size_t dict_returned_bytes = _sp->dictionary()->sum_dict_returned_bytes();
7043 size_t returned_bytes = indexListReturnedBytes + dict_returned_bytes;
7044 log_debug(gc, stats)("Returned " SIZE_FORMAT " bytes Indexed List Returned " SIZE_FORMAT " bytes Dictionary Returned " SIZE_FORMAT " bytes",
7045 returned_bytes, indexListReturnedBytes, dict_returned_bytes);
7046 }
7047 log_develop(gc, sweep)("end of sweep with _limit = " PTR_FORMAT, p2i(_limit));
7048 log_develop(gc, sweep)("================");
7049 }
7050 #endif // PRODUCT
7051
7052 void SweepClosure::initialize_free_range(HeapWord* freeFinger,
7053 bool freeRangeInFreeLists) {
7054 log_develop(gc, sweep)("---- Start free range at " PTR_FORMAT " with free block (%d)",
7055 p2i(freeFinger), freeRangeInFreeLists);
7056 assert(!inFreeRange(), "Trampling existing free range");
7057 set_inFreeRange(true);
7058 set_lastFreeRangeCoalesced(false);
7059
7060 set_freeFinger(freeFinger);
7061 set_freeRangeInFreeLists(freeRangeInFreeLists);
7062 if (CMSTestInFreeList) {
7063 if (freeRangeInFreeLists) {
7064 FreeChunk* fc = (FreeChunk*) freeFinger;
7065 assert(fc->is_free(), "A chunk on the free list should be free.");
7066 assert(fc->size() > 0, "Free range should have a size");
7067 assert(_sp->verify_chunk_in_free_list(fc), "Chunk is not in free lists");
7068 }
7069 }
7070 }
7071
7072 // Note that the sweeper runs concurrently with mutators. Thus,
7073 // it is possible for direct allocation in this generation to happen
7074 // in the middle of the sweep. Note that the sweeper also coalesces
7075 // contiguous free blocks. Thus, unless the sweeper and the allocator
7101 size_t res;
7102
7103 // Check if we are done sweeping. Below we check "addr >= _limit" rather
7104 // than "addr == _limit" because although _limit was a block boundary when
7105 // we started the sweep, it may no longer be one because heap expansion
7106 // may have caused us to coalesce the block ending at the address _limit
7107 // with a newly expanded chunk (this happens when _limit was set to the
7108 // previous _end of the space), so we may have stepped past _limit:
7109 // see the following Zeno-like trail of CRs 6977970, 7008136, 7042740.
7110 if (addr >= _limit) { // we have swept up to or past the limit: finish up
7111 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7112 "sweep _limit out of bounds");
7113 assert(addr < _sp->end(), "addr out of bounds");
7114 // Flush any free range we might be holding as a single
7115 // coalesced chunk to the appropriate free list.
7116 if (inFreeRange()) {
7117 assert(freeFinger() >= _sp->bottom() && freeFinger() < _limit,
7118 "freeFinger() " PTR_FORMAT " is out-of-bounds", p2i(freeFinger()));
7119 flush_cur_free_chunk(freeFinger(),
7120 pointer_delta(addr, freeFinger()));
7121 log_develop(gc, sweep)("Sweep: last chunk: put_free_blk " PTR_FORMAT " (" SIZE_FORMAT ") [coalesced:%d]",
7122 p2i(freeFinger()), pointer_delta(addr, freeFinger()),
7123 lastFreeRangeCoalesced() ? 1 : 0);
7124 }
7125
7126 // help the iterator loop finish
7127 return pointer_delta(_sp->end(), addr);
7128 }
7129
7130 assert(addr < _limit, "sweep invariant");
7131 // check if we should yield
7132 do_yield_check(addr);
7133 if (fc->is_free()) {
7134 // Chunk that is already free
7135 res = fc->size();
7136 do_already_free_chunk(fc);
7137 debug_only(_sp->verifyFreeLists());
7138 // If we flush the chunk at hand in lookahead_and_flush()
7139 // and it's coalesced with a preceding chunk, then the
7140 // process of "mangling" the payload of the coalesced block
7141 // will cause erasure of the size information from the
7142 // (erstwhile) header of all the coalesced blocks but the
7143 // first, so the first disjunct in the assert will not hold
7144 // in that specific case (in which case the second disjunct
7314 // Verify that the bit map has no bits marked between
7315 // addr and purported end of this block.
7316 size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
7317 assert(size >= 3, "Necessary for Printezis marks to work");
7318 assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
7319 DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
7320 }
7321 return size;
7322 }
7323
7324 void SweepClosure::do_post_free_or_garbage_chunk(FreeChunk* fc,
7325 size_t chunkSize) {
7326 // do_post_free_or_garbage_chunk() should only be called in the case
7327 // of the adaptive free list allocator.
7328 const bool fcInFreeLists = fc->is_free();
7329 assert((HeapWord*)fc <= _limit, "sweep invariant");
7330 if (CMSTestInFreeList && fcInFreeLists) {
7331 assert(_sp->verify_chunk_in_free_list(fc), "free chunk is not in free lists");
7332 }
7333
7334 log_develop(gc, sweep)(" -- pick up another chunk at " PTR_FORMAT " (" SIZE_FORMAT ")", p2i(fc), chunkSize);
7335
7336 HeapWord* const fc_addr = (HeapWord*) fc;
7337
7338 bool coalesce = false;
7339 const size_t left = pointer_delta(fc_addr, freeFinger());
7340 const size_t right = chunkSize;
7341 switch (FLSCoalescePolicy) {
7342 // numeric value forms a coalition aggressiveness metric
7343 case 0: { // never coalesce
7344 coalesce = false;
7345 break;
7346 }
7347 case 1: { // coalesce if left & right chunks on overpopulated lists
7348 coalesce = _sp->coalOverPopulated(left) &&
7349 _sp->coalOverPopulated(right);
7350 break;
7351 }
7352 case 2: { // coalesce if left chunk on overpopulated list (default)
7353 coalesce = _sp->coalOverPopulated(left);
7354 break;
7415 // we'll look at because its end crosses past _limit, we'll preemptively
7416 // flush it along with any free range we may be holding on to. Note that
7417 // this can be the case only for an already free or freshly garbage
7418 // chunk. If this block is an object, it can never straddle
7419 // over _limit. The "straddling" occurs when _limit is set at
7420 // the previous end of the space when this cycle started, and
7421 // a subsequent heap expansion caused the previously co-terminal
7422 // free block to be coalesced with the newly expanded portion,
7423 // thus rendering _limit a non-block-boundary making it dangerous
7424 // for the sweeper to step over and examine.
7425 void SweepClosure::lookahead_and_flush(FreeChunk* fc, size_t chunk_size) {
7426 assert(inFreeRange(), "Should only be called if currently in a free range.");
7427 HeapWord* const eob = ((HeapWord*)fc) + chunk_size;
7428 assert(_sp->used_region().contains(eob - 1),
7429 "eob = " PTR_FORMAT " eob-1 = " PTR_FORMAT " _limit = " PTR_FORMAT
7430 " out of bounds wrt _sp = [" PTR_FORMAT "," PTR_FORMAT ")"
7431 " when examining fc = " PTR_FORMAT "(" SIZE_FORMAT ")",
7432 p2i(eob), p2i(eob-1), p2i(_limit), p2i(_sp->bottom()), p2i(_sp->end()), p2i(fc), chunk_size);
7433 if (eob >= _limit) {
7434 assert(eob == _limit || fc->is_free(), "Only a free chunk should allow us to cross over the limit");
7435 log_develop(gc, sweep)("_limit " PTR_FORMAT " reached or crossed by block "
7436 "[" PTR_FORMAT "," PTR_FORMAT ") in space "
7437 "[" PTR_FORMAT "," PTR_FORMAT ")",
7438 p2i(_limit), p2i(fc), p2i(eob), p2i(_sp->bottom()), p2i(_sp->end()));
7439 // Return the storage we are tracking back into the free lists.
7440 log_develop(gc, sweep)("Flushing ... ");
7441 assert(freeFinger() < eob, "Error");
7442 flush_cur_free_chunk( freeFinger(), pointer_delta(eob, freeFinger()));
7443 }
7444 }
7445
7446 void SweepClosure::flush_cur_free_chunk(HeapWord* chunk, size_t size) {
7447 assert(inFreeRange(), "Should only be called if currently in a free range.");
7448 assert(size > 0,
7449 "A zero sized chunk cannot be added to the free lists.");
7450 if (!freeRangeInFreeLists()) {
7451 if (CMSTestInFreeList) {
7452 FreeChunk* fc = (FreeChunk*) chunk;
7453 fc->set_size(size);
7454 assert(!_sp->verify_chunk_in_free_list(fc),
7455 "chunk should not be in free lists yet");
7456 }
7457 log_develop(gc, sweep)(" -- add free block " PTR_FORMAT " (" SIZE_FORMAT ") to free lists", p2i(chunk), size);
7458 // A new free range is going to be starting. The current
7459 // free range has not been added to the free lists yet or
7460 // was removed so add it back.
7461 // If the current free range was coalesced, then the death
7462 // of the free range was recorded. Record a birth now.
7463 if (lastFreeRangeCoalesced()) {
7464 _sp->coalBirth(size);
7465 }
7466 _sp->addChunkAndRepairOffsetTable(chunk, size,
7467 lastFreeRangeCoalesced());
7468 } else {
7469 log_develop(gc, sweep)("Already in free list: nothing to flush");
7470 }
7471 set_inFreeRange(false);
7472 set_freeRangeInFreeLists(false);
7473 }
7474
7475 // We take a break if we've been at this for a while,
7476 // so as to avoid monopolizing the locks involved.
7477 void SweepClosure::do_yield_work(HeapWord* addr) {
7478 // Return current free chunk being used for coalescing (if any)
7479 // to the appropriate freelist. After yielding, the next
7480 // free block encountered will start a coalescing range of
7481 // free blocks. If the next free block is adjacent to the
7482 // chunk just flushed, they will need to wait for the next
7483 // sweep to be coalesced.
7484 if (inFreeRange()) {
7485 flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger()));
7486 }
7487
7488 // First give up the locks, then yield, then re-lock.
7489 // We should probably use a constructor/destructor idiom to
7490 // do this unlock/lock or modify the MutexUnlocker class to
7491 // serve our purpose. XXX
7492 assert_lock_strong(_bitMap->lock());
7493 assert_lock_strong(_freelistLock);
7494 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7495 "CMS thread should hold CMS token");
7496 _bitMap->lock()->unlock();
7497 _freelistLock->unlock();
7498 ConcurrentMarkSweepThread::desynchronize(true);
7499 _collector->stopTimer();
7500 _collector->incrementYields();
7501
7502 // See the comment in coordinator_yield()
7503 for (unsigned i = 0; i < CMSYieldSleepCount &&
7504 ConcurrentMarkSweepThread::should_yield() &&
7505 !CMSCollector::foregroundGCIsActive(); ++i) {
7506 os::sleep(Thread::current(), 1, false);
7507 }
7508
7509 ConcurrentMarkSweepThread::synchronize(true);
7510 _freelistLock->lock();
7511 _bitMap->lock()->lock_without_safepoint_check();
7512 _collector->startTimer();
7513 }
7514
7515 #ifndef PRODUCT
7516 // This is actually very useful in a product build if it can
7517 // be called from the debugger. Compile it into the product
7518 // as needed.
7519 bool debug_verify_chunk_in_free_list(FreeChunk* fc) {
7520 return debug_cms_space->verify_chunk_in_free_list(fc);
7521 }
7522 #endif
7523
7524 void SweepClosure::print_free_block_coalesced(FreeChunk* fc) const {
7525 log_develop(gc, sweep)("Sweep:coal_free_blk " PTR_FORMAT " (" SIZE_FORMAT ")",
7526 p2i(fc), fc->size());
7527 }
7528
7529 // CMSIsAliveClosure
7530 bool CMSIsAliveClosure::do_object_b(oop obj) {
7531 HeapWord* addr = (HeapWord*)obj;
7532 return addr != NULL &&
7533 (!_span.contains(addr) || _bit_map->isMarked(addr));
7534 }
7535
7536
7537 CMSKeepAliveClosure::CMSKeepAliveClosure( CMSCollector* collector,
7538 MemRegion span,
7539 CMSBitMap* bit_map, CMSMarkStack* mark_stack,
7540 bool cpc):
7541 _collector(collector),
7542 _span(span),
7543 _bit_map(bit_map),
7544 _mark_stack(mark_stack),
7545 _concurrent_precleaning(cpc) {
7546 assert(!_span.is_empty(), "Empty span could spell trouble");
|