594 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
595 if (_cmThread->osthread() == NULL) {
596 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
597 }
598
599 assert(CGC_lock != NULL, "Where's the CGC_lock?");
600 assert(_markBitMap1.covers(g1h->reserved_region()), "_markBitMap1 inconsistency");
601 assert(_markBitMap2.covers(g1h->reserved_region()), "_markBitMap2 inconsistency");
602
603 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
604 satb_qs.set_buffer_size(G1SATBBufferSize);
605
606 _root_regions.init(_g1h, this);
607
608 if (ConcGCThreads > ParallelGCThreads) {
609 warning("Can't have more ConcGCThreads (" UINTX_FORMAT ") "
610 "than ParallelGCThreads (" UINTX_FORMAT ").",
611 ConcGCThreads, ParallelGCThreads);
612 return;
613 }
614 if (ParallelGCThreads == 0) {
615 // if we are not running with any parallel GC threads we will not
616 // spawn any marking threads either
617 _parallel_marking_threads = 0;
618 _max_parallel_marking_threads = 0;
619 _sleep_factor = 0.0;
620 _marking_task_overhead = 1.0;
621 } else {
622 if (!FLAG_IS_DEFAULT(ConcGCThreads) && ConcGCThreads > 0) {
623 // Note: ConcGCThreads has precedence over G1MarkingOverheadPercent
624 // if both are set
625 _sleep_factor = 0.0;
626 _marking_task_overhead = 1.0;
627 } else if (G1MarkingOverheadPercent > 0) {
628 // We will calculate the number of parallel marking threads based
629 // on a target overhead with respect to the soft real-time goal
630 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0;
631 double overall_cm_overhead =
632 (double) MaxGCPauseMillis * marking_overhead /
633 (double) GCPauseIntervalMillis;
634 double cpu_ratio = 1.0 / (double) os::processor_count();
635 double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio);
636 double marking_task_overhead =
637 overall_cm_overhead / marking_thread_num *
638 (double) os::processor_count();
639 double sleep_factor =
640 (1.0 - marking_task_overhead) / marking_task_overhead;
641
654 assert(ConcGCThreads > 0, "Should have been set");
655 _parallel_marking_threads = (uint) ConcGCThreads;
656 _max_parallel_marking_threads = _parallel_marking_threads;
657
658 if (parallel_marking_threads() > 1) {
659 _cleanup_task_overhead = 1.0;
660 } else {
661 _cleanup_task_overhead = marking_task_overhead();
662 }
663 _cleanup_sleep_factor =
664 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
665
666 #if 0
667 gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads());
668 gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead());
669 gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor());
670 gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead());
671 gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor());
672 #endif
673
674 guarantee(parallel_marking_threads() > 0, "peace of mind");
675 _parallel_workers = new FlexibleWorkGang("G1 Parallel Marking Threads",
676 _max_parallel_marking_threads, false, true);
677 if (_parallel_workers == NULL) {
678 vm_exit_during_initialization("Failed necessary allocation.");
679 } else {
680 _parallel_workers->initialize_workers();
681 }
682 }
683
684 if (FLAG_IS_DEFAULT(MarkStackSize)) {
685 uintx mark_stack_size =
686 MIN2(MarkStackSizeMax,
687 MAX2(MarkStackSize, (uintx) (parallel_marking_threads() * TASKQUEUE_SIZE)));
688 // Verify that the calculated value for MarkStackSize is in range.
689 // It would be nice to use the private utility routine from Arguments.
690 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
691 warning("Invalid value calculated for MarkStackSize (" UINTX_FORMAT "): "
692 "must be between " UINTX_FORMAT " and " UINTX_FORMAT,
693 mark_stack_size, (uintx) 1, MarkStackSizeMax);
694 return;
695 }
696 FLAG_SET_ERGO(uintx, MarkStackSize, mark_stack_size);
697 } else {
698 // Verify MarkStackSize is in range.
699 if (FLAG_IS_CMDLINE(MarkStackSize)) {
700 if (FLAG_IS_DEFAULT(MarkStackSizeMax)) {
701 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
702 warning("Invalid value specified for MarkStackSize (" UINTX_FORMAT "): "
1149 }
1150 the_task->record_end_time();
1151 guarantee(!the_task->has_aborted() || _cm->has_aborted(), "invariant");
1152
1153 SuspendibleThreadSet::leave();
1154
1155 double end_vtime = os::elapsedVTime();
1156 _cm->update_accum_task_vtime(worker_id, end_vtime - start_vtime);
1157 }
1158
1159 CMConcurrentMarkingTask(ConcurrentMark* cm,
1160 ConcurrentMarkThread* cmt) :
1161 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { }
1162
1163 ~CMConcurrentMarkingTask() { }
1164 };
1165
1166 // Calculates the number of active workers for a concurrent
1167 // phase.
1168 uint ConcurrentMark::calc_parallel_marking_threads() {
1169 if (G1CollectedHeap::use_parallel_gc_threads()) {
1170 uint n_conc_workers = 0;
1171 if (!UseDynamicNumberOfGCThreads ||
1172 (!FLAG_IS_DEFAULT(ConcGCThreads) &&
1173 !ForceDynamicNumberOfGCThreads)) {
1174 n_conc_workers = max_parallel_marking_threads();
1175 } else {
1176 n_conc_workers =
1177 AdaptiveSizePolicy::calc_default_active_workers(
1178 max_parallel_marking_threads(),
1179 1, /* Minimum workers */
1180 parallel_marking_threads(),
1181 Threads::number_of_non_daemon_threads());
1182 // Don't scale down "n_conc_workers" by scale_parallel_threads() because
1183 // that scaling has already gone into "_max_parallel_marking_threads".
1184 }
1185 assert(n_conc_workers > 0, "Always need at least 1");
1186 return n_conc_workers;
1187 }
1188 // If we are not running with any parallel GC threads we will not
1189 // have spawned any marking threads either. Hence the number of
1190 // concurrent workers should be 0.
1191 return 0;
1192 }
1193
1194 void ConcurrentMark::scanRootRegion(HeapRegion* hr, uint worker_id) {
1195 // Currently, only survivors can be root regions.
1196 assert(hr->next_top_at_mark_start() == hr->bottom(), "invariant");
1197 G1RootRegionScanClosure cl(_g1h, this, worker_id);
1198
1199 const uintx interval = PrefetchScanIntervalInBytes;
1200 HeapWord* curr = hr->bottom();
1201 const HeapWord* end = hr->top();
1202 while (curr < end) {
1203 Prefetch::read(curr, interval);
1204 oop obj = oop(curr);
1205 int size = obj->oop_iterate(&cl);
1206 assert(size == obj->size(), "sanity");
1207 curr += size;
1208 }
1209 }
1210
1211 class CMRootRegionScanTask : public AbstractGangTask {
1226 _cm->scanRootRegion(hr, worker_id);
1227 hr = root_regions->claim_next();
1228 }
1229 }
1230 };
1231
1232 void ConcurrentMark::scanRootRegions() {
1233 // Start of concurrent marking.
1234 ClassLoaderDataGraph::clear_claimed_marks();
1235
1236 // scan_in_progress() will have been set to true only if there was
1237 // at least one root region to scan. So, if it's false, we
1238 // should not attempt to do any further work.
1239 if (root_regions()->scan_in_progress()) {
1240 _parallel_marking_threads = calc_parallel_marking_threads();
1241 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1242 "Maximum number of marking threads exceeded");
1243 uint active_workers = MAX2(1U, parallel_marking_threads());
1244
1245 CMRootRegionScanTask task(this);
1246 if (use_parallel_marking_threads()) {
1247 _parallel_workers->set_active_workers((int) active_workers);
1248 _parallel_workers->run_task(&task);
1249 } else {
1250 task.work(0);
1251 }
1252
1253 // It's possible that has_aborted() is true here without actually
1254 // aborting the survivor scan earlier. This is OK as it's
1255 // mainly used for sanity checking.
1256 root_regions()->scan_finished();
1257 }
1258 }
1259
1260 void ConcurrentMark::markFromRoots() {
1261 // we might be tempted to assert that:
1262 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1263 // "inconsistent argument?");
1264 // However that wouldn't be right, because it's possible that
1265 // a safepoint is indeed in progress as a younger generation
1266 // stop-the-world GC happens even as we mark in this generation.
1267
1268 _restart_for_overflow = false;
1269 force_overflow_conc()->init();
1270
1271 // _g1h has _n_par_threads
1272 _parallel_marking_threads = calc_parallel_marking_threads();
1273 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1274 "Maximum number of marking threads exceeded");
1275
1276 uint active_workers = MAX2(1U, parallel_marking_threads());
1277
1278 // Parallel task terminator is set in "set_concurrency_and_phase()"
1279 set_concurrency_and_phase(active_workers, true /* concurrent */);
1280
1281 CMConcurrentMarkingTask markingTask(this, cmThread());
1282 if (use_parallel_marking_threads()) {
1283 _parallel_workers->set_active_workers((int)active_workers);
1284 // Don't set _n_par_threads because it affects MT in process_roots()
1285 // and the decisions on that MT processing is made elsewhere.
1286 assert(_parallel_workers->active_workers() > 0, "Should have been set");
1287 _parallel_workers->run_task(&markingTask);
1288 } else {
1289 markingTask.work(0);
1290 }
1291 print_stats();
1292 }
1293
1294 // Helper class to get rid of some boilerplate code.
1295 class G1CMTraceTime : public GCTraceTime {
1296 static bool doit_and_prepend(bool doit) {
1297 if (doit) {
1298 gclog_or_tty->put(' ');
1299 }
1300 return doit;
1301 }
1302
1303 public:
1304 G1CMTraceTime(const char* title, bool doit)
1305 : GCTraceTime(title, doit_and_prepend(doit), false, G1CollectedHeap::heap()->gc_timer_cm(),
1306 G1CollectedHeap::heap()->concurrent_mark()->concurrent_gc_id()) {
1307 }
1308 };
1309
1310 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1697 _actual_region_bm(region_bm), _actual_card_bm(card_bm),
1698 _expected_region_bm(expected_region_bm), _expected_card_bm(expected_card_bm),
1699 _failures(0), _verbose(false),
1700 _n_workers(_g1h->workers()->active_workers()), _hrclaimer(_n_workers) {
1701 assert(VerifyDuringGC, "don't call this otherwise");
1702 assert(_expected_card_bm->size() == _actual_card_bm->size(), "sanity");
1703 assert(_expected_region_bm->size() == _actual_region_bm->size(), "sanity");
1704
1705 _verbose = _cm->verbose_medium();
1706 }
1707
1708 void work(uint worker_id) {
1709 assert(worker_id < _n_workers, "invariant");
1710
1711 VerifyLiveObjectDataHRClosure verify_cl(_g1h,
1712 _actual_region_bm, _actual_card_bm,
1713 _expected_region_bm,
1714 _expected_card_bm,
1715 _verbose);
1716
1717 if (G1CollectedHeap::use_parallel_gc_threads()) {
1718 _g1h->heap_region_par_iterate(&verify_cl, worker_id, &_hrclaimer);
1719 } else {
1720 _g1h->heap_region_iterate(&verify_cl);
1721 }
1722
1723 Atomic::add(verify_cl.failures(), &_failures);
1724 }
1725
1726 int failures() const { return _failures; }
1727 };
1728
1729 // Closure that finalizes the liveness counting data.
1730 // Used during the cleanup pause.
1731 // Sets the bits corresponding to the interval [NTAMS, top]
1732 // (which contains the implicitly live objects) in the
1733 // card liveness bitmap. Also sets the bit for each region,
1734 // containing live data, in the region liveness bitmap.
1735
1736 class FinalCountDataUpdateClosure: public CMCountDataClosureBase {
1737 public:
1738 FinalCountDataUpdateClosure(G1CollectedHeap* g1h,
1739 BitMap* region_bm,
1740 BitMap* card_bm) :
1741 CMCountDataClosureBase(g1h, region_bm, card_bm) { }
1804 BitMap* _actual_card_bm;
1805
1806 uint _n_workers;
1807 HeapRegionClaimer _hrclaimer;
1808
1809 public:
1810 G1ParFinalCountTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm)
1811 : AbstractGangTask("G1 final counting"),
1812 _g1h(g1h), _cm(_g1h->concurrent_mark()),
1813 _actual_region_bm(region_bm), _actual_card_bm(card_bm),
1814 _n_workers(_g1h->workers()->active_workers()), _hrclaimer(_n_workers) {
1815 }
1816
1817 void work(uint worker_id) {
1818 assert(worker_id < _n_workers, "invariant");
1819
1820 FinalCountDataUpdateClosure final_update_cl(_g1h,
1821 _actual_region_bm,
1822 _actual_card_bm);
1823
1824 if (G1CollectedHeap::use_parallel_gc_threads()) {
1825 _g1h->heap_region_par_iterate(&final_update_cl, worker_id, &_hrclaimer);
1826 } else {
1827 _g1h->heap_region_iterate(&final_update_cl);
1828 }
1829 }
1830 };
1831
1832 class G1ParNoteEndTask;
1833
1834 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1835 G1CollectedHeap* _g1;
1836 size_t _max_live_bytes;
1837 uint _regions_claimed;
1838 size_t _freed_bytes;
1839 FreeRegionList* _local_cleanup_list;
1840 HeapRegionSetCount _old_regions_removed;
1841 HeapRegionSetCount _humongous_regions_removed;
1842 HRRSCleanupTask* _hrrs_cleanup_task;
1843 double _claimed_region_time;
1844 double _max_region_time;
1845
1846 public:
1847 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1848 FreeRegionList* local_cleanup_list,
1905 friend class G1NoteEndOfConcMarkClosure;
1906
1907 protected:
1908 G1CollectedHeap* _g1h;
1909 size_t _max_live_bytes;
1910 size_t _freed_bytes;
1911 FreeRegionList* _cleanup_list;
1912 HeapRegionClaimer _hrclaimer;
1913
1914 public:
1915 G1ParNoteEndTask(G1CollectedHeap* g1h, FreeRegionList* cleanup_list, uint n_workers) :
1916 AbstractGangTask("G1 note end"), _g1h(g1h), _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list), _hrclaimer(n_workers) {
1917 }
1918
1919 void work(uint worker_id) {
1920 double start = os::elapsedTime();
1921 FreeRegionList local_cleanup_list("Local Cleanup List");
1922 HRRSCleanupTask hrrs_cleanup_task;
1923 G1NoteEndOfConcMarkClosure g1_note_end(_g1h, &local_cleanup_list,
1924 &hrrs_cleanup_task);
1925 if (G1CollectedHeap::use_parallel_gc_threads()) {
1926 _g1h->heap_region_par_iterate(&g1_note_end, worker_id, &_hrclaimer);
1927 } else {
1928 _g1h->heap_region_iterate(&g1_note_end);
1929 }
1930 assert(g1_note_end.complete(), "Shouldn't have yielded!");
1931
1932 // Now update the lists
1933 _g1h->remove_from_old_sets(g1_note_end.old_regions_removed(), g1_note_end.humongous_regions_removed());
1934 {
1935 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1936 _g1h->decrement_summary_bytes(g1_note_end.freed_bytes());
1937 _max_live_bytes += g1_note_end.max_live_bytes();
1938 _freed_bytes += g1_note_end.freed_bytes();
1939
1940 // If we iterate over the global cleanup list at the end of
1941 // cleanup to do this printing we will not guarantee to only
1942 // generate output for the newly-reclaimed regions (the list
1943 // might not be empty at the beginning of cleanup; we might
1944 // still be working on its previous contents). So we do the
1945 // printing here, before we append the new regions to the global
1946 // cleanup list.
1947
1948 G1HRPrinter* hr_printer = _g1h->hr_printer();
1949 if (hr_printer->is_active()) {
1960 HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task);
1961 }
1962 }
1963 size_t max_live_bytes() { return _max_live_bytes; }
1964 size_t freed_bytes() { return _freed_bytes; }
1965 };
1966
1967 class G1ParScrubRemSetTask: public AbstractGangTask {
1968 protected:
1969 G1RemSet* _g1rs;
1970 BitMap* _region_bm;
1971 BitMap* _card_bm;
1972 HeapRegionClaimer _hrclaimer;
1973
1974 public:
1975 G1ParScrubRemSetTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm, uint n_workers) :
1976 AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()), _region_bm(region_bm), _card_bm(card_bm), _hrclaimer(n_workers) {
1977 }
1978
1979 void work(uint worker_id) {
1980 if (G1CollectedHeap::use_parallel_gc_threads()) {
1981 _g1rs->scrub_par(_region_bm, _card_bm, worker_id, &_hrclaimer);
1982 } else {
1983 _g1rs->scrub(_region_bm, _card_bm);
1984 }
1985 }
1986
1987 };
1988
1989 void ConcurrentMark::cleanup() {
1990 // world is stopped at this checkpoint
1991 assert(SafepointSynchronize::is_at_safepoint(),
1992 "world should be stopped");
1993 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1994
1995 // If a full collection has happened, we shouldn't do this.
1996 if (has_aborted()) {
1997 g1h->set_marking_complete(); // So bitmap clearing isn't confused
1998 return;
1999 }
2000
2001 g1h->verify_region_sets_optional();
2002
2003 if (VerifyDuringGC) {
2004 HandleMark hm; // handle scope
2005 Universe::heap()->prepare_for_verify();
2006 Universe::verify(VerifyOption_G1UsePrevMarking,
2007 " VerifyDuringGC:(before)");
2008 }
2009 g1h->check_bitmaps("Cleanup Start");
2010
2011 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
2012 g1p->record_concurrent_mark_cleanup_start();
2013
2014 double start = os::elapsedTime();
2015
2016 HeapRegionRemSet::reset_for_cleanup_tasks();
2017
2018 uint n_workers;
2019
2020 // Do counting once more with the world stopped for good measure.
2021 G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
2022
2023 if (G1CollectedHeap::use_parallel_gc_threads()) {
2024 g1h->set_par_threads();
2025 n_workers = g1h->n_par_threads();
2026 assert(g1h->n_par_threads() == n_workers,
2027 "Should not have been reset");
2028 g1h->workers()->run_task(&g1_par_count_task);
2029 // Done with the parallel phase so reset to 0.
2030 g1h->set_par_threads(0);
2031 } else {
2032 n_workers = 1;
2033 g1_par_count_task.work(0);
2034 }
2035
2036 if (VerifyDuringGC) {
2037 // Verify that the counting data accumulated during marking matches
2038 // that calculated by walking the marking bitmap.
2039
2040 // Bitmaps to hold expected values
2041 BitMap expected_region_bm(_region_bm.size(), true);
2042 BitMap expected_card_bm(_card_bm.size(), true);
2043
2044 G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
2045 &_region_bm,
2046 &_card_bm,
2047 &expected_region_bm,
2048 &expected_card_bm);
2049
2050 if (G1CollectedHeap::use_parallel_gc_threads()) {
2051 g1h->set_par_threads((int)n_workers);
2052 g1h->workers()->run_task(&g1_par_verify_task);
2053 // Done with the parallel phase so reset to 0.
2054 g1h->set_par_threads(0);
2055 } else {
2056 g1_par_verify_task.work(0);
2057 }
2058
2059 guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
2060 }
2061
2062 size_t start_used_bytes = g1h->used();
2063 g1h->set_marking_complete();
2064
2065 double count_end = os::elapsedTime();
2066 double this_final_counting_time = (count_end - start);
2067 _total_counting_time += this_final_counting_time;
2068
2069 if (G1PrintRegionLivenessInfo) {
2070 G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking");
2071 _g1h->heap_region_iterate(&cl);
2072 }
2073
2074 // Install newly created mark bitMap as "prev".
2075 swapMarkBitMaps();
2076
2077 g1h->reset_gc_time_stamp();
2078
2079 // Note end of marking in all heap regions.
2080 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
2081 if (G1CollectedHeap::use_parallel_gc_threads()) {
2082 g1h->set_par_threads((int)n_workers);
2083 g1h->workers()->run_task(&g1_par_note_end_task);
2084 g1h->set_par_threads(0);
2085 } else {
2086 g1_par_note_end_task.work(0);
2087 }
2088 g1h->check_gc_time_stamps();
2089
2090 if (!cleanup_list_is_empty()) {
2091 // The cleanup list is not empty, so we'll have to process it
2092 // concurrently. Notify anyone else that might be wanting free
2093 // regions that there will be more free regions coming soon.
2094 g1h->set_free_regions_coming();
2095 }
2096
2097 // call below, since it affects the metric by which we sort the heap
2098 // regions.
2099 if (G1ScrubRemSets) {
2100 double rs_scrub_start = os::elapsedTime();
2101 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers);
2102 if (G1CollectedHeap::use_parallel_gc_threads()) {
2103 g1h->set_par_threads((int)n_workers);
2104 g1h->workers()->run_task(&g1_par_scrub_rs_task);
2105 g1h->set_par_threads(0);
2106 } else {
2107 g1_par_scrub_rs_task.work(0);
2108 }
2109
2110 double rs_scrub_end = os::elapsedTime();
2111 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
2112 _total_rs_scrub_time += this_rs_scrub_time;
2113 }
2114
2115 // this will also free any regions totally full of garbage objects,
2116 // and sort the regions.
2117 g1h->g1_policy()->record_concurrent_mark_cleanup_end((int)n_workers);
2118
2119 // Statistics.
2120 double end = os::elapsedTime();
2121 _cleanup_times.add((end - start) * 1000.0);
2122
2123 if (G1Log::fine()) {
2124 g1h->print_size_transition(gclog_or_tty,
2125 start_used_bytes,
2126 g1h->used(),
2127 g1h->capacity());
2128 }
2485 // in serial reference processing. Note these closures are also
2486 // used for serially processing (by the the current thread) the
2487 // JNI references during parallel reference processing.
2488 //
2489 // These closures do not need to synchronize with the worker
2490 // threads involved in parallel reference processing as these
2491 // instances are executed serially by the current thread (e.g.
2492 // reference processing is not multi-threaded and is thus
2493 // performed by the current thread instead of a gang worker).
2494 //
2495 // The gang tasks involved in parallel reference processing create
2496 // their own instances of these closures, which do their own
2497 // synchronization among themselves.
2498 G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */);
2499 G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */);
2500
2501 // We need at least one active thread. If reference processing
2502 // is not multi-threaded we use the current (VMThread) thread,
2503 // otherwise we use the work gang from the G1CollectedHeap and
2504 // we utilize all the worker threads we can.
2505 bool processing_is_mt = rp->processing_is_mt() && g1h->workers() != NULL;
2506 uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U);
2507 active_workers = MAX2(MIN2(active_workers, _max_worker_id), 1U);
2508
2509 // Parallel processing task executor.
2510 G1CMRefProcTaskExecutor par_task_executor(g1h, this,
2511 g1h->workers(), active_workers);
2512 AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL);
2513
2514 // Set the concurrency level. The phase was already set prior to
2515 // executing the remark task.
2516 set_concurrency(active_workers);
2517
2518 // Set the degree of MT processing here. If the discovery was done MT,
2519 // the number of threads involved during discovery could differ from
2520 // the number of active workers. This is OK as long as the discovered
2521 // Reference lists are balanced (see balance_all_queues() and balance_queues()).
2522 rp->set_active_mt_degree(active_workers);
2523
2524 // Process the weak references.
2525 const ReferenceProcessorStats& stats =
2594
2595 // Closure for iterating over objects, currently only used for
2596 // processing SATB buffers.
2597 class CMObjectClosure : public ObjectClosure {
2598 private:
2599 CMTask* _task;
2600
2601 public:
2602 void do_object(oop obj) {
2603 _task->deal_with_reference(obj);
2604 }
2605
2606 CMObjectClosure(CMTask* task) : _task(task) { }
2607 };
2608
2609 class G1RemarkThreadsClosure : public ThreadClosure {
2610 CMObjectClosure _cm_obj;
2611 G1CMOopClosure _cm_cl;
2612 MarkingCodeBlobClosure _code_cl;
2613 int _thread_parity;
2614 bool _is_par;
2615
2616 public:
2617 G1RemarkThreadsClosure(G1CollectedHeap* g1h, CMTask* task, bool is_par) :
2618 _cm_obj(task), _cm_cl(g1h, g1h->concurrent_mark(), task), _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations),
2619 _thread_parity(SharedHeap::heap()->strong_roots_parity()), _is_par(is_par) {}
2620
2621 void do_thread(Thread* thread) {
2622 if (thread->is_Java_thread()) {
2623 if (thread->claim_oops_do(_is_par, _thread_parity)) {
2624 JavaThread* jt = (JavaThread*)thread;
2625
2626 // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
2627 // however the liveness of oops reachable from nmethods have very complex lifecycles:
2628 // * Alive if on the stack of an executing method
2629 // * Weakly reachable otherwise
2630 // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
2631 // live by the SATB invariant but other oops recorded in nmethods may behave differently.
2632 jt->nmethods_do(&_code_cl);
2633
2634 jt->satb_mark_queue().apply_closure_and_empty(&_cm_obj);
2635 }
2636 } else if (thread->is_VM_thread()) {
2637 if (thread->claim_oops_do(_is_par, _thread_parity)) {
2638 JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(&_cm_obj);
2639 }
2640 }
2641 }
2642 };
2643
2644 class CMRemarkTask: public AbstractGangTask {
2645 private:
2646 ConcurrentMark* _cm;
2647 bool _is_serial;
2648 public:
2649 void work(uint worker_id) {
2650 // Since all available tasks are actually started, we should
2651 // only proceed if we're supposed to be active.
2652 if (worker_id < _cm->active_tasks()) {
2653 CMTask* task = _cm->task(worker_id);
2654 task->record_start_time();
2655 {
2656 ResourceMark rm;
2657 HandleMark hm;
2658
2659 G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task, !_is_serial);
2660 Threads::threads_do(&threads_f);
2661 }
2662
2663 do {
2664 task->do_marking_step(1000000000.0 /* something very large */,
2665 true /* do_termination */,
2666 _is_serial);
2667 } while (task->has_aborted() && !_cm->has_overflown());
2668 // If we overflow, then we do not want to restart. We instead
2669 // want to abort remark and do concurrent marking again.
2670 task->record_end_time();
2671 }
2672 }
2673
2674 CMRemarkTask(ConcurrentMark* cm, int active_workers, bool is_serial) :
2675 AbstractGangTask("Par Remark"), _cm(cm), _is_serial(is_serial) {
2676 _cm->terminator()->reset_for_reuse(active_workers);
2677 }
2678 };
2679
2680 void ConcurrentMark::checkpointRootsFinalWork() {
2681 ResourceMark rm;
2682 HandleMark hm;
2683 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2684
2685 G1CMTraceTime trace("Finalize Marking", G1Log::finer());
2686
2687 g1h->ensure_parsability(false);
2688
2689 if (G1CollectedHeap::use_parallel_gc_threads()) {
2690 G1CollectedHeap::StrongRootsScope srs(g1h);
2691 // this is remark, so we'll use up all active threads
2692 uint active_workers = g1h->workers()->active_workers();
2693 if (active_workers == 0) {
2694 assert(active_workers > 0, "Should have been set earlier");
2695 active_workers = (uint) ParallelGCThreads;
2696 g1h->workers()->set_active_workers(active_workers);
2697 }
2698 set_concurrency_and_phase(active_workers, false /* concurrent */);
2699 // Leave _parallel_marking_threads at it's
2700 // value originally calculated in the ConcurrentMark
2701 // constructor and pass values of the active workers
2702 // through the gang in the task.
2703
2704 CMRemarkTask remarkTask(this, active_workers, false /* is_serial */);
2705 // We will start all available threads, even if we decide that the
2706 // active_workers will be fewer. The extra ones will just bail out
2707 // immediately.
2708 g1h->set_par_threads(active_workers);
2709 g1h->workers()->run_task(&remarkTask);
2710 g1h->set_par_threads(0);
2711 } else {
2712 G1CollectedHeap::StrongRootsScope srs(g1h);
2713 uint active_workers = 1;
2714 set_concurrency_and_phase(active_workers, false /* concurrent */);
2715
2716 // Note - if there's no work gang then the VMThread will be
2717 // the thread to execute the remark - serially. We have
2718 // to pass true for the is_serial parameter so that
2719 // CMTask::do_marking_step() doesn't enter the sync
2720 // barriers in the event of an overflow. Doing so will
2721 // cause an assert that the current thread is not a
2722 // concurrent GC thread.
2723 CMRemarkTask remarkTask(this, active_workers, true /* is_serial*/);
2724 remarkTask.work(0);
2725 }
2726 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2727 guarantee(has_overflown() ||
2728 satb_mq_set.completed_buffers_num() == 0,
2729 err_msg("Invariant: has_overflown = %s, num buffers = %d",
2730 BOOL_TO_STR(has_overflown()),
2731 satb_mq_set.completed_buffers_num()));
2732
2733 print_stats();
2734 }
2735
2736 #ifndef PRODUCT
2737
2738 class PrintReachableOopClosure: public OopClosure {
2739 private:
2740 G1CollectedHeap* _g1h;
2741 outputStream* _out;
2742 VerifyOption _vo;
2743 bool _all;
2744
2745 public:
3251 uint _max_worker_id;
3252 int _active_workers;
3253 HeapRegionClaimer _hrclaimer;
3254
3255 public:
3256 G1AggregateCountDataTask(G1CollectedHeap* g1h,
3257 ConcurrentMark* cm,
3258 BitMap* cm_card_bm,
3259 uint max_worker_id,
3260 int n_workers) :
3261 AbstractGangTask("Count Aggregation"),
3262 _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm),
3263 _max_worker_id(max_worker_id),
3264 _active_workers(n_workers),
3265 _hrclaimer(_active_workers) {
3266 }
3267
3268 void work(uint worker_id) {
3269 AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_worker_id);
3270
3271 if (G1CollectedHeap::use_parallel_gc_threads()) {
3272 _g1h->heap_region_par_iterate(&cl, worker_id, &_hrclaimer);
3273 } else {
3274 _g1h->heap_region_iterate(&cl);
3275 }
3276 }
3277 };
3278
3279
3280 void ConcurrentMark::aggregate_count_data() {
3281 int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
3282 _g1h->workers()->active_workers() :
3283 1);
3284
3285 G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm,
3286 _max_worker_id, n_workers);
3287
3288 if (G1CollectedHeap::use_parallel_gc_threads()) {
3289 _g1h->set_par_threads(n_workers);
3290 _g1h->workers()->run_task(&g1_par_agg_task);
3291 _g1h->set_par_threads(0);
3292 } else {
3293 g1_par_agg_task.work(0);
3294 }
3295 _g1h->allocation_context_stats().update_at_remark();
3296 }
3297
3298 // Clear the per-worker arrays used to store the per-region counting data
3299 void ConcurrentMark::clear_all_count_data() {
3300 // Clear the global card bitmap - it will be filled during
3301 // liveness count aggregation (during remark) and the
3302 // final counting task.
3303 _card_bm.clear();
3304
3305 // Clear the global region bitmap - it will be filled as part
3306 // of the final counting task.
3307 _region_bm.clear();
3308
3309 uint max_regions = _g1h->max_regions();
3310 assert(_max_worker_id > 0, "uninitialized");
3311
3312 for (uint i = 0; i < _max_worker_id; i += 1) {
3313 BitMap* task_card_bm = count_card_bitmap_for(i);
3314 size_t* marked_bytes_array = count_marked_bytes_array_for(i);
3402 (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 /
3403 (double)_cleanup_times.num()
3404 : 0.0));
3405 if (G1ScrubRemSets) {
3406 gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
3407 _total_rs_scrub_time,
3408 (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 /
3409 (double)_cleanup_times.num()
3410 : 0.0));
3411 }
3412 gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.",
3413 (_init_times.sum() + _remark_times.sum() +
3414 _cleanup_times.sum())/1000.0);
3415 gclog_or_tty->print_cr(" Total concurrent time = %8.2f s "
3416 "(%8.2f s marking).",
3417 cmThread()->vtime_accum(),
3418 cmThread()->vtime_mark_accum());
3419 }
3420
3421 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
3422 if (use_parallel_marking_threads()) {
3423 _parallel_workers->print_worker_threads_on(st);
3424 }
3425 }
3426
3427 void ConcurrentMark::print_on_error(outputStream* st) const {
3428 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
3429 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
3430 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
3431 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
3432 }
3433
3434 // We take a break if someone is trying to stop the world.
3435 bool ConcurrentMark::do_yield_check(uint worker_id) {
3436 if (SuspendibleThreadSet::should_yield()) {
3437 if (worker_id == 0) {
3438 _g1h->g1_policy()->record_concurrent_pause();
3439 }
3440 SuspendibleThreadSet::yield();
3441 return true;
3442 } else {
3443 return false;
3444 }
3925 _worker_id, _cm->mark_stack_size());
3926 }
3927 }
3928 }
3929
3930 // SATB Queue has several assumptions on whether to call the par or
3931 // non-par versions of the methods. this is why some of the code is
3932 // replicated. We should really get rid of the single-threaded version
3933 // of the code to simplify things.
3934 void CMTask::drain_satb_buffers() {
3935 if (has_aborted()) return;
3936
3937 // We set this so that the regular clock knows that we're in the
3938 // middle of draining buffers and doesn't set the abort flag when it
3939 // notices that SATB buffers are available for draining. It'd be
3940 // very counter productive if it did that. :-)
3941 _draining_satb_buffers = true;
3942
3943 CMObjectClosure oc(this);
3944 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3945 if (G1CollectedHeap::use_parallel_gc_threads()) {
3946 satb_mq_set.set_par_closure(_worker_id, &oc);
3947 } else {
3948 satb_mq_set.set_closure(&oc);
3949 }
3950
3951 // This keeps claiming and applying the closure to completed buffers
3952 // until we run out of buffers or we need to abort.
3953 if (G1CollectedHeap::use_parallel_gc_threads()) {
3954 while (!has_aborted() &&
3955 satb_mq_set.par_apply_closure_to_completed_buffer(_worker_id)) {
3956 if (_cm->verbose_medium()) {
3957 gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id);
3958 }
3959 statsOnly( ++_satb_buffers_processed );
3960 regular_clock_call();
3961 }
3962 } else {
3963 while (!has_aborted() &&
3964 satb_mq_set.apply_closure_to_completed_buffer()) {
3965 if (_cm->verbose_medium()) {
3966 gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id);
3967 }
3968 statsOnly( ++_satb_buffers_processed );
3969 regular_clock_call();
3970 }
3971 }
3972
3973 _draining_satb_buffers = false;
3974
3975 assert(has_aborted() ||
3976 concurrent() ||
3977 satb_mq_set.completed_buffers_num() == 0, "invariant");
3978
3979 if (G1CollectedHeap::use_parallel_gc_threads()) {
3980 satb_mq_set.set_par_closure(_worker_id, NULL);
3981 } else {
3982 satb_mq_set.set_closure(NULL);
3983 }
3984
3985 // again, this was a potentially expensive operation, decrease the
3986 // limits to get the regular clock call early
3987 decrease_limits();
3988 }
3989
3990 void CMTask::print_stats() {
3991 gclog_or_tty->print_cr("Marking Stats, task = %u, calls = %d",
3992 _worker_id, _calls);
3993 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3994 _elapsed_time_ms, _termination_time_ms);
3995 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3996 _step_times_ms.num(), _step_times_ms.avg(),
3997 _step_times_ms.sd());
3998 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
3999 _step_times_ms.maximum(), _step_times_ms.sum());
4000
4001 #if _MARKING_STATS_
4002 gclog_or_tty->print_cr(" Clock Intervals (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
4003 _all_clock_intervals_ms.num(), _all_clock_intervals_ms.avg(),
|
594 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
595 if (_cmThread->osthread() == NULL) {
596 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
597 }
598
599 assert(CGC_lock != NULL, "Where's the CGC_lock?");
600 assert(_markBitMap1.covers(g1h->reserved_region()), "_markBitMap1 inconsistency");
601 assert(_markBitMap2.covers(g1h->reserved_region()), "_markBitMap2 inconsistency");
602
603 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
604 satb_qs.set_buffer_size(G1SATBBufferSize);
605
606 _root_regions.init(_g1h, this);
607
608 if (ConcGCThreads > ParallelGCThreads) {
609 warning("Can't have more ConcGCThreads (" UINTX_FORMAT ") "
610 "than ParallelGCThreads (" UINTX_FORMAT ").",
611 ConcGCThreads, ParallelGCThreads);
612 return;
613 }
614 if (!FLAG_IS_DEFAULT(ConcGCThreads) && ConcGCThreads > 0) {
615 // Note: ConcGCThreads has precedence over G1MarkingOverheadPercent
616 // if both are set
617 _sleep_factor = 0.0;
618 _marking_task_overhead = 1.0;
619 } else if (G1MarkingOverheadPercent > 0) {
620 // We will calculate the number of parallel marking threads based
621 // on a target overhead with respect to the soft real-time goal
622 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0;
623 double overall_cm_overhead =
624 (double) MaxGCPauseMillis * marking_overhead /
625 (double) GCPauseIntervalMillis;
626 double cpu_ratio = 1.0 / (double) os::processor_count();
627 double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio);
628 double marking_task_overhead =
629 overall_cm_overhead / marking_thread_num *
630 (double) os::processor_count();
631 double sleep_factor =
632 (1.0 - marking_task_overhead) / marking_task_overhead;
633
646 assert(ConcGCThreads > 0, "Should have been set");
647 _parallel_marking_threads = (uint) ConcGCThreads;
648 _max_parallel_marking_threads = _parallel_marking_threads;
649
650 if (parallel_marking_threads() > 1) {
651 _cleanup_task_overhead = 1.0;
652 } else {
653 _cleanup_task_overhead = marking_task_overhead();
654 }
655 _cleanup_sleep_factor =
656 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
657
658 #if 0
659 gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads());
660 gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead());
661 gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor());
662 gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead());
663 gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor());
664 #endif
665
666 _parallel_workers = new FlexibleWorkGang("G1 Parallel Marking Threads",
667 _max_parallel_marking_threads, false, true);
668 if (_parallel_workers == NULL) {
669 vm_exit_during_initialization("Failed necessary allocation.");
670 } else {
671 _parallel_workers->initialize_workers();
672 }
673
674 if (FLAG_IS_DEFAULT(MarkStackSize)) {
675 uintx mark_stack_size =
676 MIN2(MarkStackSizeMax,
677 MAX2(MarkStackSize, (uintx) (parallel_marking_threads() * TASKQUEUE_SIZE)));
678 // Verify that the calculated value for MarkStackSize is in range.
679 // It would be nice to use the private utility routine from Arguments.
680 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
681 warning("Invalid value calculated for MarkStackSize (" UINTX_FORMAT "): "
682 "must be between " UINTX_FORMAT " and " UINTX_FORMAT,
683 mark_stack_size, (uintx) 1, MarkStackSizeMax);
684 return;
685 }
686 FLAG_SET_ERGO(uintx, MarkStackSize, mark_stack_size);
687 } else {
688 // Verify MarkStackSize is in range.
689 if (FLAG_IS_CMDLINE(MarkStackSize)) {
690 if (FLAG_IS_DEFAULT(MarkStackSizeMax)) {
691 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
692 warning("Invalid value specified for MarkStackSize (" UINTX_FORMAT "): "
1139 }
1140 the_task->record_end_time();
1141 guarantee(!the_task->has_aborted() || _cm->has_aborted(), "invariant");
1142
1143 SuspendibleThreadSet::leave();
1144
1145 double end_vtime = os::elapsedVTime();
1146 _cm->update_accum_task_vtime(worker_id, end_vtime - start_vtime);
1147 }
1148
1149 CMConcurrentMarkingTask(ConcurrentMark* cm,
1150 ConcurrentMarkThread* cmt) :
1151 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { }
1152
1153 ~CMConcurrentMarkingTask() { }
1154 };
1155
1156 // Calculates the number of active workers for a concurrent
1157 // phase.
1158 uint ConcurrentMark::calc_parallel_marking_threads() {
1159 uint n_conc_workers = 0;
1160 if (!UseDynamicNumberOfGCThreads ||
1161 (!FLAG_IS_DEFAULT(ConcGCThreads) &&
1162 !ForceDynamicNumberOfGCThreads)) {
1163 n_conc_workers = max_parallel_marking_threads();
1164 } else {
1165 n_conc_workers =
1166 AdaptiveSizePolicy::calc_default_active_workers(
1167 max_parallel_marking_threads(),
1168 1, /* Minimum workers */
1169 parallel_marking_threads(),
1170 Threads::number_of_non_daemon_threads());
1171 // Don't scale down "n_conc_workers" by scale_parallel_threads() because
1172 // that scaling has already gone into "_max_parallel_marking_threads".
1173 }
1174 assert(n_conc_workers > 0, "Always need at least 1");
1175 return n_conc_workers;
1176 }
1177
1178 void ConcurrentMark::scanRootRegion(HeapRegion* hr, uint worker_id) {
1179 // Currently, only survivors can be root regions.
1180 assert(hr->next_top_at_mark_start() == hr->bottom(), "invariant");
1181 G1RootRegionScanClosure cl(_g1h, this, worker_id);
1182
1183 const uintx interval = PrefetchScanIntervalInBytes;
1184 HeapWord* curr = hr->bottom();
1185 const HeapWord* end = hr->top();
1186 while (curr < end) {
1187 Prefetch::read(curr, interval);
1188 oop obj = oop(curr);
1189 int size = obj->oop_iterate(&cl);
1190 assert(size == obj->size(), "sanity");
1191 curr += size;
1192 }
1193 }
1194
1195 class CMRootRegionScanTask : public AbstractGangTask {
1210 _cm->scanRootRegion(hr, worker_id);
1211 hr = root_regions->claim_next();
1212 }
1213 }
1214 };
1215
1216 void ConcurrentMark::scanRootRegions() {
1217 // Start of concurrent marking.
1218 ClassLoaderDataGraph::clear_claimed_marks();
1219
1220 // scan_in_progress() will have been set to true only if there was
1221 // at least one root region to scan. So, if it's false, we
1222 // should not attempt to do any further work.
1223 if (root_regions()->scan_in_progress()) {
1224 _parallel_marking_threads = calc_parallel_marking_threads();
1225 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1226 "Maximum number of marking threads exceeded");
1227 uint active_workers = MAX2(1U, parallel_marking_threads());
1228
1229 CMRootRegionScanTask task(this);
1230 _parallel_workers->set_active_workers(active_workers);
1231 _parallel_workers->run_task(&task);
1232
1233 // It's possible that has_aborted() is true here without actually
1234 // aborting the survivor scan earlier. This is OK as it's
1235 // mainly used for sanity checking.
1236 root_regions()->scan_finished();
1237 }
1238 }
1239
1240 void ConcurrentMark::markFromRoots() {
1241 // we might be tempted to assert that:
1242 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1243 // "inconsistent argument?");
1244 // However that wouldn't be right, because it's possible that
1245 // a safepoint is indeed in progress as a younger generation
1246 // stop-the-world GC happens even as we mark in this generation.
1247
1248 _restart_for_overflow = false;
1249 force_overflow_conc()->init();
1250
1251 // _g1h has _n_par_threads
1252 _parallel_marking_threads = calc_parallel_marking_threads();
1253 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1254 "Maximum number of marking threads exceeded");
1255
1256 uint active_workers = MAX2(1U, parallel_marking_threads());
1257
1258 // Parallel task terminator is set in "set_concurrency_and_phase()"
1259 set_concurrency_and_phase(active_workers, true /* concurrent */);
1260
1261 CMConcurrentMarkingTask markingTask(this, cmThread());
1262 _parallel_workers->set_active_workers(active_workers);
1263 // Don't set _n_par_threads because it affects MT in process_roots()
1264 // and the decisions on that MT processing is made elsewhere.
1265 assert(_parallel_workers->active_workers() > 0, "Should have been set");
1266 _parallel_workers->run_task(&markingTask);
1267 print_stats();
1268 }
1269
1270 // Helper class to get rid of some boilerplate code.
1271 class G1CMTraceTime : public GCTraceTime {
1272 static bool doit_and_prepend(bool doit) {
1273 if (doit) {
1274 gclog_or_tty->put(' ');
1275 }
1276 return doit;
1277 }
1278
1279 public:
1280 G1CMTraceTime(const char* title, bool doit)
1281 : GCTraceTime(title, doit_and_prepend(doit), false, G1CollectedHeap::heap()->gc_timer_cm(),
1282 G1CollectedHeap::heap()->concurrent_mark()->concurrent_gc_id()) {
1283 }
1284 };
1285
1286 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1673 _actual_region_bm(region_bm), _actual_card_bm(card_bm),
1674 _expected_region_bm(expected_region_bm), _expected_card_bm(expected_card_bm),
1675 _failures(0), _verbose(false),
1676 _n_workers(_g1h->workers()->active_workers()), _hrclaimer(_n_workers) {
1677 assert(VerifyDuringGC, "don't call this otherwise");
1678 assert(_expected_card_bm->size() == _actual_card_bm->size(), "sanity");
1679 assert(_expected_region_bm->size() == _actual_region_bm->size(), "sanity");
1680
1681 _verbose = _cm->verbose_medium();
1682 }
1683
1684 void work(uint worker_id) {
1685 assert(worker_id < _n_workers, "invariant");
1686
1687 VerifyLiveObjectDataHRClosure verify_cl(_g1h,
1688 _actual_region_bm, _actual_card_bm,
1689 _expected_region_bm,
1690 _expected_card_bm,
1691 _verbose);
1692
1693 _g1h->heap_region_par_iterate(&verify_cl, worker_id, &_hrclaimer);
1694
1695 Atomic::add(verify_cl.failures(), &_failures);
1696 }
1697
1698 int failures() const { return _failures; }
1699 };
1700
1701 // Closure that finalizes the liveness counting data.
1702 // Used during the cleanup pause.
1703 // Sets the bits corresponding to the interval [NTAMS, top]
1704 // (which contains the implicitly live objects) in the
1705 // card liveness bitmap. Also sets the bit for each region,
1706 // containing live data, in the region liveness bitmap.
1707
1708 class FinalCountDataUpdateClosure: public CMCountDataClosureBase {
1709 public:
1710 FinalCountDataUpdateClosure(G1CollectedHeap* g1h,
1711 BitMap* region_bm,
1712 BitMap* card_bm) :
1713 CMCountDataClosureBase(g1h, region_bm, card_bm) { }
1776 BitMap* _actual_card_bm;
1777
1778 uint _n_workers;
1779 HeapRegionClaimer _hrclaimer;
1780
1781 public:
1782 G1ParFinalCountTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm)
1783 : AbstractGangTask("G1 final counting"),
1784 _g1h(g1h), _cm(_g1h->concurrent_mark()),
1785 _actual_region_bm(region_bm), _actual_card_bm(card_bm),
1786 _n_workers(_g1h->workers()->active_workers()), _hrclaimer(_n_workers) {
1787 }
1788
1789 void work(uint worker_id) {
1790 assert(worker_id < _n_workers, "invariant");
1791
1792 FinalCountDataUpdateClosure final_update_cl(_g1h,
1793 _actual_region_bm,
1794 _actual_card_bm);
1795
1796 _g1h->heap_region_par_iterate(&final_update_cl, worker_id, &_hrclaimer);
1797 }
1798 };
1799
1800 class G1ParNoteEndTask;
1801
1802 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1803 G1CollectedHeap* _g1;
1804 size_t _max_live_bytes;
1805 uint _regions_claimed;
1806 size_t _freed_bytes;
1807 FreeRegionList* _local_cleanup_list;
1808 HeapRegionSetCount _old_regions_removed;
1809 HeapRegionSetCount _humongous_regions_removed;
1810 HRRSCleanupTask* _hrrs_cleanup_task;
1811 double _claimed_region_time;
1812 double _max_region_time;
1813
1814 public:
1815 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1816 FreeRegionList* local_cleanup_list,
1873 friend class G1NoteEndOfConcMarkClosure;
1874
1875 protected:
1876 G1CollectedHeap* _g1h;
1877 size_t _max_live_bytes;
1878 size_t _freed_bytes;
1879 FreeRegionList* _cleanup_list;
1880 HeapRegionClaimer _hrclaimer;
1881
1882 public:
1883 G1ParNoteEndTask(G1CollectedHeap* g1h, FreeRegionList* cleanup_list, uint n_workers) :
1884 AbstractGangTask("G1 note end"), _g1h(g1h), _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list), _hrclaimer(n_workers) {
1885 }
1886
1887 void work(uint worker_id) {
1888 double start = os::elapsedTime();
1889 FreeRegionList local_cleanup_list("Local Cleanup List");
1890 HRRSCleanupTask hrrs_cleanup_task;
1891 G1NoteEndOfConcMarkClosure g1_note_end(_g1h, &local_cleanup_list,
1892 &hrrs_cleanup_task);
1893 _g1h->heap_region_par_iterate(&g1_note_end, worker_id, &_hrclaimer);
1894 assert(g1_note_end.complete(), "Shouldn't have yielded!");
1895
1896 // Now update the lists
1897 _g1h->remove_from_old_sets(g1_note_end.old_regions_removed(), g1_note_end.humongous_regions_removed());
1898 {
1899 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1900 _g1h->decrement_summary_bytes(g1_note_end.freed_bytes());
1901 _max_live_bytes += g1_note_end.max_live_bytes();
1902 _freed_bytes += g1_note_end.freed_bytes();
1903
1904 // If we iterate over the global cleanup list at the end of
1905 // cleanup to do this printing we will not guarantee to only
1906 // generate output for the newly-reclaimed regions (the list
1907 // might not be empty at the beginning of cleanup; we might
1908 // still be working on its previous contents). So we do the
1909 // printing here, before we append the new regions to the global
1910 // cleanup list.
1911
1912 G1HRPrinter* hr_printer = _g1h->hr_printer();
1913 if (hr_printer->is_active()) {
1924 HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task);
1925 }
1926 }
1927 size_t max_live_bytes() { return _max_live_bytes; }
1928 size_t freed_bytes() { return _freed_bytes; }
1929 };
1930
1931 class G1ParScrubRemSetTask: public AbstractGangTask {
1932 protected:
1933 G1RemSet* _g1rs;
1934 BitMap* _region_bm;
1935 BitMap* _card_bm;
1936 HeapRegionClaimer _hrclaimer;
1937
1938 public:
1939 G1ParScrubRemSetTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm, uint n_workers) :
1940 AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()), _region_bm(region_bm), _card_bm(card_bm), _hrclaimer(n_workers) {
1941 }
1942
1943 void work(uint worker_id) {
1944 _g1rs->scrub(_region_bm, _card_bm, worker_id, &_hrclaimer);
1945 }
1946
1947 };
1948
1949 void ConcurrentMark::cleanup() {
1950 // world is stopped at this checkpoint
1951 assert(SafepointSynchronize::is_at_safepoint(),
1952 "world should be stopped");
1953 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1954
1955 // If a full collection has happened, we shouldn't do this.
1956 if (has_aborted()) {
1957 g1h->set_marking_complete(); // So bitmap clearing isn't confused
1958 return;
1959 }
1960
1961 g1h->verify_region_sets_optional();
1962
1963 if (VerifyDuringGC) {
1964 HandleMark hm; // handle scope
1965 Universe::heap()->prepare_for_verify();
1966 Universe::verify(VerifyOption_G1UsePrevMarking,
1967 " VerifyDuringGC:(before)");
1968 }
1969 g1h->check_bitmaps("Cleanup Start");
1970
1971 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
1972 g1p->record_concurrent_mark_cleanup_start();
1973
1974 double start = os::elapsedTime();
1975
1976 HeapRegionRemSet::reset_for_cleanup_tasks();
1977
1978 uint n_workers;
1979
1980 // Do counting once more with the world stopped for good measure.
1981 G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
1982
1983 g1h->set_par_threads();
1984 n_workers = g1h->n_par_threads();
1985 assert(g1h->n_par_threads() == n_workers,
1986 "Should not have been reset");
1987 g1h->workers()->run_task(&g1_par_count_task);
1988 // Done with the parallel phase so reset to 0.
1989 g1h->set_par_threads(0);
1990
1991 if (VerifyDuringGC) {
1992 // Verify that the counting data accumulated during marking matches
1993 // that calculated by walking the marking bitmap.
1994
1995 // Bitmaps to hold expected values
1996 BitMap expected_region_bm(_region_bm.size(), true);
1997 BitMap expected_card_bm(_card_bm.size(), true);
1998
1999 G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
2000 &_region_bm,
2001 &_card_bm,
2002 &expected_region_bm,
2003 &expected_card_bm);
2004
2005 g1h->set_par_threads((int)n_workers);
2006 g1h->workers()->run_task(&g1_par_verify_task);
2007 // Done with the parallel phase so reset to 0.
2008 g1h->set_par_threads(0);
2009
2010 guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
2011 }
2012
2013 size_t start_used_bytes = g1h->used();
2014 g1h->set_marking_complete();
2015
2016 double count_end = os::elapsedTime();
2017 double this_final_counting_time = (count_end - start);
2018 _total_counting_time += this_final_counting_time;
2019
2020 if (G1PrintRegionLivenessInfo) {
2021 G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking");
2022 _g1h->heap_region_iterate(&cl);
2023 }
2024
2025 // Install newly created mark bitMap as "prev".
2026 swapMarkBitMaps();
2027
2028 g1h->reset_gc_time_stamp();
2029
2030 // Note end of marking in all heap regions.
2031 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
2032 g1h->set_par_threads((int)n_workers);
2033 g1h->workers()->run_task(&g1_par_note_end_task);
2034 g1h->set_par_threads(0);
2035 g1h->check_gc_time_stamps();
2036
2037 if (!cleanup_list_is_empty()) {
2038 // The cleanup list is not empty, so we'll have to process it
2039 // concurrently. Notify anyone else that might be wanting free
2040 // regions that there will be more free regions coming soon.
2041 g1h->set_free_regions_coming();
2042 }
2043
2044 // call below, since it affects the metric by which we sort the heap
2045 // regions.
2046 if (G1ScrubRemSets) {
2047 double rs_scrub_start = os::elapsedTime();
2048 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers);
2049 g1h->set_par_threads((int)n_workers);
2050 g1h->workers()->run_task(&g1_par_scrub_rs_task);
2051 g1h->set_par_threads(0);
2052
2053 double rs_scrub_end = os::elapsedTime();
2054 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
2055 _total_rs_scrub_time += this_rs_scrub_time;
2056 }
2057
2058 // this will also free any regions totally full of garbage objects,
2059 // and sort the regions.
2060 g1h->g1_policy()->record_concurrent_mark_cleanup_end((int)n_workers);
2061
2062 // Statistics.
2063 double end = os::elapsedTime();
2064 _cleanup_times.add((end - start) * 1000.0);
2065
2066 if (G1Log::fine()) {
2067 g1h->print_size_transition(gclog_or_tty,
2068 start_used_bytes,
2069 g1h->used(),
2070 g1h->capacity());
2071 }
2428 // in serial reference processing. Note these closures are also
2429 // used for serially processing (by the the current thread) the
2430 // JNI references during parallel reference processing.
2431 //
2432 // These closures do not need to synchronize with the worker
2433 // threads involved in parallel reference processing as these
2434 // instances are executed serially by the current thread (e.g.
2435 // reference processing is not multi-threaded and is thus
2436 // performed by the current thread instead of a gang worker).
2437 //
2438 // The gang tasks involved in parallel reference processing create
2439 // their own instances of these closures, which do their own
2440 // synchronization among themselves.
2441 G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */);
2442 G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */);
2443
2444 // We need at least one active thread. If reference processing
2445 // is not multi-threaded we use the current (VMThread) thread,
2446 // otherwise we use the work gang from the G1CollectedHeap and
2447 // we utilize all the worker threads we can.
2448 bool processing_is_mt = rp->processing_is_mt();
2449 uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U);
2450 active_workers = MAX2(MIN2(active_workers, _max_worker_id), 1U);
2451
2452 // Parallel processing task executor.
2453 G1CMRefProcTaskExecutor par_task_executor(g1h, this,
2454 g1h->workers(), active_workers);
2455 AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL);
2456
2457 // Set the concurrency level. The phase was already set prior to
2458 // executing the remark task.
2459 set_concurrency(active_workers);
2460
2461 // Set the degree of MT processing here. If the discovery was done MT,
2462 // the number of threads involved during discovery could differ from
2463 // the number of active workers. This is OK as long as the discovered
2464 // Reference lists are balanced (see balance_all_queues() and balance_queues()).
2465 rp->set_active_mt_degree(active_workers);
2466
2467 // Process the weak references.
2468 const ReferenceProcessorStats& stats =
2537
2538 // Closure for iterating over objects, currently only used for
2539 // processing SATB buffers.
2540 class CMObjectClosure : public ObjectClosure {
2541 private:
2542 CMTask* _task;
2543
2544 public:
2545 void do_object(oop obj) {
2546 _task->deal_with_reference(obj);
2547 }
2548
2549 CMObjectClosure(CMTask* task) : _task(task) { }
2550 };
2551
2552 class G1RemarkThreadsClosure : public ThreadClosure {
2553 CMObjectClosure _cm_obj;
2554 G1CMOopClosure _cm_cl;
2555 MarkingCodeBlobClosure _code_cl;
2556 int _thread_parity;
2557
2558 public:
2559 G1RemarkThreadsClosure(G1CollectedHeap* g1h, CMTask* task) :
2560 _cm_obj(task), _cm_cl(g1h, g1h->concurrent_mark(), task), _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations),
2561 _thread_parity(SharedHeap::heap()->strong_roots_parity()) {}
2562
2563 void do_thread(Thread* thread) {
2564 if (thread->is_Java_thread()) {
2565 if (thread->claim_oops_do(true, _thread_parity)) {
2566 JavaThread* jt = (JavaThread*)thread;
2567
2568 // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
2569 // however the liveness of oops reachable from nmethods have very complex lifecycles:
2570 // * Alive if on the stack of an executing method
2571 // * Weakly reachable otherwise
2572 // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
2573 // live by the SATB invariant but other oops recorded in nmethods may behave differently.
2574 jt->nmethods_do(&_code_cl);
2575
2576 jt->satb_mark_queue().apply_closure_and_empty(&_cm_obj);
2577 }
2578 } else if (thread->is_VM_thread()) {
2579 if (thread->claim_oops_do(true, _thread_parity)) {
2580 JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(&_cm_obj);
2581 }
2582 }
2583 }
2584 };
2585
2586 class CMRemarkTask: public AbstractGangTask {
2587 private:
2588 ConcurrentMark* _cm;
2589 public:
2590 void work(uint worker_id) {
2591 // Since all available tasks are actually started, we should
2592 // only proceed if we're supposed to be active.
2593 if (worker_id < _cm->active_tasks()) {
2594 CMTask* task = _cm->task(worker_id);
2595 task->record_start_time();
2596 {
2597 ResourceMark rm;
2598 HandleMark hm;
2599
2600 G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task);
2601 Threads::threads_do(&threads_f);
2602 }
2603
2604 do {
2605 task->do_marking_step(1000000000.0 /* something very large */,
2606 true /* do_termination */,
2607 false /* is_serial */);
2608 } while (task->has_aborted() && !_cm->has_overflown());
2609 // If we overflow, then we do not want to restart. We instead
2610 // want to abort remark and do concurrent marking again.
2611 task->record_end_time();
2612 }
2613 }
2614
2615 CMRemarkTask(ConcurrentMark* cm, int active_workers) :
2616 AbstractGangTask("Par Remark"), _cm(cm) {
2617 _cm->terminator()->reset_for_reuse(active_workers);
2618 }
2619 };
2620
2621 void ConcurrentMark::checkpointRootsFinalWork() {
2622 ResourceMark rm;
2623 HandleMark hm;
2624 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2625
2626 G1CMTraceTime trace("Finalize Marking", G1Log::finer());
2627
2628 g1h->ensure_parsability(false);
2629
2630 G1CollectedHeap::StrongRootsScope srs(g1h);
2631 // this is remark, so we'll use up all active threads
2632 uint active_workers = g1h->workers()->active_workers();
2633 if (active_workers == 0) {
2634 assert(active_workers > 0, "Should have been set earlier");
2635 active_workers = (uint) ParallelGCThreads;
2636 g1h->workers()->set_active_workers(active_workers);
2637 }
2638 set_concurrency_and_phase(active_workers, false /* concurrent */);
2639 // Leave _parallel_marking_threads at it's
2640 // value originally calculated in the ConcurrentMark
2641 // constructor and pass values of the active workers
2642 // through the gang in the task.
2643
2644 CMRemarkTask remarkTask(this, active_workers);
2645 // We will start all available threads, even if we decide that the
2646 // active_workers will be fewer. The extra ones will just bail out
2647 // immediately.
2648 g1h->set_par_threads(active_workers);
2649 g1h->workers()->run_task(&remarkTask);
2650 g1h->set_par_threads(0);
2651
2652 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2653 guarantee(has_overflown() ||
2654 satb_mq_set.completed_buffers_num() == 0,
2655 err_msg("Invariant: has_overflown = %s, num buffers = %d",
2656 BOOL_TO_STR(has_overflown()),
2657 satb_mq_set.completed_buffers_num()));
2658
2659 print_stats();
2660 }
2661
2662 #ifndef PRODUCT
2663
2664 class PrintReachableOopClosure: public OopClosure {
2665 private:
2666 G1CollectedHeap* _g1h;
2667 outputStream* _out;
2668 VerifyOption _vo;
2669 bool _all;
2670
2671 public:
3177 uint _max_worker_id;
3178 int _active_workers;
3179 HeapRegionClaimer _hrclaimer;
3180
3181 public:
3182 G1AggregateCountDataTask(G1CollectedHeap* g1h,
3183 ConcurrentMark* cm,
3184 BitMap* cm_card_bm,
3185 uint max_worker_id,
3186 int n_workers) :
3187 AbstractGangTask("Count Aggregation"),
3188 _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm),
3189 _max_worker_id(max_worker_id),
3190 _active_workers(n_workers),
3191 _hrclaimer(_active_workers) {
3192 }
3193
3194 void work(uint worker_id) {
3195 AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_worker_id);
3196
3197 _g1h->heap_region_par_iterate(&cl, worker_id, &_hrclaimer);
3198 }
3199 };
3200
3201
3202 void ConcurrentMark::aggregate_count_data() {
3203 int n_workers = _g1h->workers()->active_workers();
3204
3205 G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm,
3206 _max_worker_id, n_workers);
3207
3208 _g1h->set_par_threads(n_workers);
3209 _g1h->workers()->run_task(&g1_par_agg_task);
3210 _g1h->set_par_threads(0);
3211 _g1h->allocation_context_stats().update_at_remark();
3212 }
3213
3214 // Clear the per-worker arrays used to store the per-region counting data
3215 void ConcurrentMark::clear_all_count_data() {
3216 // Clear the global card bitmap - it will be filled during
3217 // liveness count aggregation (during remark) and the
3218 // final counting task.
3219 _card_bm.clear();
3220
3221 // Clear the global region bitmap - it will be filled as part
3222 // of the final counting task.
3223 _region_bm.clear();
3224
3225 uint max_regions = _g1h->max_regions();
3226 assert(_max_worker_id > 0, "uninitialized");
3227
3228 for (uint i = 0; i < _max_worker_id; i += 1) {
3229 BitMap* task_card_bm = count_card_bitmap_for(i);
3230 size_t* marked_bytes_array = count_marked_bytes_array_for(i);
3318 (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 /
3319 (double)_cleanup_times.num()
3320 : 0.0));
3321 if (G1ScrubRemSets) {
3322 gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
3323 _total_rs_scrub_time,
3324 (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 /
3325 (double)_cleanup_times.num()
3326 : 0.0));
3327 }
3328 gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.",
3329 (_init_times.sum() + _remark_times.sum() +
3330 _cleanup_times.sum())/1000.0);
3331 gclog_or_tty->print_cr(" Total concurrent time = %8.2f s "
3332 "(%8.2f s marking).",
3333 cmThread()->vtime_accum(),
3334 cmThread()->vtime_mark_accum());
3335 }
3336
3337 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
3338 _parallel_workers->print_worker_threads_on(st);
3339 }
3340
3341 void ConcurrentMark::print_on_error(outputStream* st) const {
3342 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
3343 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
3344 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
3345 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
3346 }
3347
3348 // We take a break if someone is trying to stop the world.
3349 bool ConcurrentMark::do_yield_check(uint worker_id) {
3350 if (SuspendibleThreadSet::should_yield()) {
3351 if (worker_id == 0) {
3352 _g1h->g1_policy()->record_concurrent_pause();
3353 }
3354 SuspendibleThreadSet::yield();
3355 return true;
3356 } else {
3357 return false;
3358 }
3839 _worker_id, _cm->mark_stack_size());
3840 }
3841 }
3842 }
3843
3844 // SATB Queue has several assumptions on whether to call the par or
3845 // non-par versions of the methods. this is why some of the code is
3846 // replicated. We should really get rid of the single-threaded version
3847 // of the code to simplify things.
3848 void CMTask::drain_satb_buffers() {
3849 if (has_aborted()) return;
3850
3851 // We set this so that the regular clock knows that we're in the
3852 // middle of draining buffers and doesn't set the abort flag when it
3853 // notices that SATB buffers are available for draining. It'd be
3854 // very counter productive if it did that. :-)
3855 _draining_satb_buffers = true;
3856
3857 CMObjectClosure oc(this);
3858 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3859 satb_mq_set.set_closure(_worker_id, &oc);
3860
3861 // This keeps claiming and applying the closure to completed buffers
3862 // until we run out of buffers or we need to abort.
3863 while (!has_aborted() &&
3864 satb_mq_set.apply_closure_to_completed_buffer(_worker_id)) {
3865 if (_cm->verbose_medium()) {
3866 gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id);
3867 }
3868 statsOnly( ++_satb_buffers_processed );
3869 regular_clock_call();
3870 }
3871
3872 _draining_satb_buffers = false;
3873
3874 assert(has_aborted() ||
3875 concurrent() ||
3876 satb_mq_set.completed_buffers_num() == 0, "invariant");
3877
3878 satb_mq_set.set_closure(_worker_id, NULL);
3879
3880 // again, this was a potentially expensive operation, decrease the
3881 // limits to get the regular clock call early
3882 decrease_limits();
3883 }
3884
3885 void CMTask::print_stats() {
3886 gclog_or_tty->print_cr("Marking Stats, task = %u, calls = %d",
3887 _worker_id, _calls);
3888 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3889 _elapsed_time_ms, _termination_time_ms);
3890 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3891 _step_times_ms.num(), _step_times_ms.avg(),
3892 _step_times_ms.sd());
3893 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
3894 _step_times_ms.maximum(), _step_times_ms.sum());
3895
3896 #if _MARKING_STATS_
3897 gclog_or_tty->print_cr(" Clock Intervals (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3898 _all_clock_intervals_ms.num(), _all_clock_intervals_ms.avg(),
|