14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/metadataOnStackMark.hpp"
27 #include "classfile/symbolTable.hpp"
28 #include "code/codeCache.hpp"
29 #include "gc/g1/concurrentMark.inline.hpp"
30 #include "gc/g1/concurrentMarkThread.inline.hpp"
31 #include "gc/g1/g1CollectedHeap.inline.hpp"
32 #include "gc/g1/g1CollectorPolicy.hpp"
33 #include "gc/g1/g1CollectorState.hpp"
34 #include "gc/g1/g1ErgoVerbose.hpp"
35 #include "gc/g1/g1Log.hpp"
36 #include "gc/g1/g1OopClosures.inline.hpp"
37 #include "gc/g1/g1RemSet.hpp"
38 #include "gc/g1/g1StringDedup.hpp"
39 #include "gc/g1/heapRegion.inline.hpp"
40 #include "gc/g1/heapRegionManager.inline.hpp"
41 #include "gc/g1/heapRegionRemSet.hpp"
42 #include "gc/g1/heapRegionSet.inline.hpp"
43 #include "gc/g1/suspendibleThreadSet.hpp"
44 #include "gc/shared/gcId.hpp"
45 #include "gc/shared/gcTimer.hpp"
46 #include "gc/shared/gcTrace.hpp"
47 #include "gc/shared/gcTraceTime.hpp"
48 #include "gc/shared/genOopClosures.inline.hpp"
49 #include "gc/shared/referencePolicy.hpp"
50 #include "gc/shared/strongRootsScope.hpp"
51 #include "gc/shared/taskqueue.inline.hpp"
52 #include "gc/shared/vmGCOperations.hpp"
53 #include "memory/allocation.hpp"
54 #include "memory/resourceArea.hpp"
55 #include "oops/oop.inline.hpp"
56 #include "runtime/atomic.inline.hpp"
57 #include "runtime/handles.inline.hpp"
58 #include "runtime/java.hpp"
59 #include "runtime/prefetch.inline.hpp"
60 #include "services/memTracker.hpp"
61
62 // Concurrent marking bit map wrapper
63
64 CMBitMapRO::CMBitMapRO(int shifter) :
65 _bm(),
66 _shifter(shifter) {
67 _bmStartWord = 0;
68 _bmWordSize = 0;
69 }
70
71 HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
72 const HeapWord* limit) const {
262 rs.release();
263 return false;
264 }
265 assert(_virtual_space.committed_size() == rs.size(),
266 "Didn't reserve backing store for all of ConcurrentMark stack?");
267 _base = (oop*) _virtual_space.low();
268 setEmpty();
269 _capacity = (jint) capacity;
270 _saved_index = -1;
271 _should_expand = false;
272 return true;
273 }
274
275 void CMMarkStack::expand() {
276 // Called, during remark, if we've overflown the marking stack during marking.
277 assert(isEmpty(), "stack should been emptied while handling overflow");
278 assert(_capacity <= (jint) MarkStackSizeMax, "stack bigger than permitted");
279 // Clear expansion flag
280 _should_expand = false;
281 if (_capacity == (jint) MarkStackSizeMax) {
282 if (PrintGCDetails && Verbose) {
283 gclog_or_tty->print_cr(" (benign) Can't expand marking stack capacity, at max size limit");
284 }
285 return;
286 }
287 // Double capacity if possible
288 jint new_capacity = MIN2(_capacity*2, (jint) MarkStackSizeMax);
289 // Do not give up existing stack until we have managed to
290 // get the double capacity that we desired.
291 ReservedSpace rs(ReservedSpace::allocation_align_size_up(new_capacity *
292 sizeof(oop)));
293 if (rs.is_reserved()) {
294 // Release the backing store associated with old stack
295 _virtual_space.release();
296 // Reinitialize virtual space for new stack
297 if (!_virtual_space.initialize(rs, rs.size())) {
298 fatal("Not enough swap for expanded marking stack capacity");
299 }
300 _base = (oop*)(_virtual_space.low());
301 _index = 0;
302 _capacity = new_capacity;
303 } else {
304 if (PrintGCDetails && Verbose) {
305 // Failed to double capacity, continue;
306 gclog_or_tty->print(" (benign) Failed to expand marking stack capacity from "
307 SIZE_FORMAT "K to " SIZE_FORMAT "K",
308 _capacity / K, new_capacity / K);
309 }
310 }
311 }
312
313 void CMMarkStack::set_should_expand() {
314 // If we're resetting the marking state because of an
315 // marking stack overflow, record that we should, if
316 // possible, expand the stack.
317 _should_expand = _cm->has_overflown();
318 }
319
320 CMMarkStack::~CMMarkStack() {
321 if (_base != NULL) {
322 _base = NULL;
323 _virtual_space.release();
324 }
325 }
326
327 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
328 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
329 jint start = _index;
330 jint next_index = start + n;
559 // Calculate the number of parallel marking threads by scaling
560 // the number of parallel GC threads.
561 uint marking_thread_num = scale_parallel_threads(ParallelGCThreads);
562 FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num);
563 _sleep_factor = 0.0;
564 _marking_task_overhead = 1.0;
565 }
566
567 assert(ConcGCThreads > 0, "Should have been set");
568 _parallel_marking_threads = ConcGCThreads;
569 _max_parallel_marking_threads = _parallel_marking_threads;
570
571 if (parallel_marking_threads() > 1) {
572 _cleanup_task_overhead = 1.0;
573 } else {
574 _cleanup_task_overhead = marking_task_overhead();
575 }
576 _cleanup_sleep_factor =
577 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
578
579 #if 0
580 gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads());
581 gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead());
582 gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor());
583 gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead());
584 gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor());
585 #endif
586
587 _parallel_workers = new WorkGang("G1 Marker",
588 _max_parallel_marking_threads, false, true);
589 if (_parallel_workers == NULL) {
590 vm_exit_during_initialization("Failed necessary allocation.");
591 } else {
592 _parallel_workers->initialize_workers();
593 }
594
595 if (FLAG_IS_DEFAULT(MarkStackSize)) {
596 size_t mark_stack_size =
597 MIN2(MarkStackSizeMax,
598 MAX2(MarkStackSize, (size_t) (parallel_marking_threads() * TASKQUEUE_SIZE)));
599 // Verify that the calculated value for MarkStackSize is in range.
600 // It would be nice to use the private utility routine from Arguments.
601 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
602 warning("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): "
603 "must be between 1 and " SIZE_FORMAT,
604 mark_stack_size, MarkStackSizeMax);
605 return;
606 }
905 return;
906 }
907
908 // If we're executing the concurrent phase of marking, reset the marking
909 // state; otherwise the marking state is reset after reference processing,
910 // during the remark pause.
911 // If we reset here as a result of an overflow during the remark we will
912 // see assertion failures from any subsequent set_concurrency_and_phase()
913 // calls.
914 if (concurrent()) {
915 // let the task associated with with worker 0 do this
916 if (worker_id == 0) {
917 // task 0 is responsible for clearing the global data structures
918 // We should be here because of an overflow. During STW we should
919 // not clear the overflow flag since we rely on it being true when
920 // we exit this method to abort the pause and restart concurrent
921 // marking.
922 reset_marking_state(true /* clear_overflow */);
923 force_overflow()->update();
924
925 if (G1Log::fine()) {
926 gclog_or_tty->gclog_stamp();
927 gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]");
928 }
929 }
930 }
931
932 // after this, each task should reset its own data structures then
933 // then go into the second barrier
934 }
935
936 void ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
937 SuspendibleThreadSetLeaver sts_leave(concurrent());
938 _second_overflow_barrier_sync.enter();
939
940 // at this point everything should be re-initialized and ready to go
941 }
942
943 #ifndef PRODUCT
944 void ForceOverflowSettings::init() {
945 _num_remaining = G1ConcMarkForceOverflow;
946 _force = false;
947 update();
948 }
1070 ConcurrentMark* _cm;
1071
1072 public:
1073 CMRootRegionScanTask(ConcurrentMark* cm) :
1074 AbstractGangTask("Root Region Scan"), _cm(cm) { }
1075
1076 void work(uint worker_id) {
1077 assert(Thread::current()->is_ConcurrentGC_thread(),
1078 "this should only be done by a conc GC thread");
1079
1080 CMRootRegions* root_regions = _cm->root_regions();
1081 HeapRegion* hr = root_regions->claim_next();
1082 while (hr != NULL) {
1083 _cm->scanRootRegion(hr, worker_id);
1084 hr = root_regions->claim_next();
1085 }
1086 }
1087 };
1088
1089 void ConcurrentMark::scanRootRegions() {
1090 double scan_start = os::elapsedTime();
1091
1092 // Start of concurrent marking.
1093 ClassLoaderDataGraph::clear_claimed_marks();
1094
1095 // scan_in_progress() will have been set to true only if there was
1096 // at least one root region to scan. So, if it's false, we
1097 // should not attempt to do any further work.
1098 if (root_regions()->scan_in_progress()) {
1099 if (G1Log::fine()) {
1100 gclog_or_tty->gclog_stamp();
1101 gclog_or_tty->print_cr("[GC concurrent-root-region-scan-start]");
1102 }
1103
1104 _parallel_marking_threads = calc_parallel_marking_threads();
1105 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1106 "Maximum number of marking threads exceeded");
1107 uint active_workers = MAX2(1U, parallel_marking_threads());
1108
1109 CMRootRegionScanTask task(this);
1110 _parallel_workers->set_active_workers(active_workers);
1111 _parallel_workers->run_task(&task);
1112
1113 if (G1Log::fine()) {
1114 gclog_or_tty->gclog_stamp();
1115 gclog_or_tty->print_cr("[GC concurrent-root-region-scan-end, %1.7lf secs]", os::elapsedTime() - scan_start);
1116 }
1117
1118 // It's possible that has_aborted() is true here without actually
1119 // aborting the survivor scan earlier. This is OK as it's
1120 // mainly used for sanity checking.
1121 root_regions()->scan_finished();
1122 }
1123 }
1124
1125 void ConcurrentMark::markFromRoots() {
1126 // we might be tempted to assert that:
1127 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1128 // "inconsistent argument?");
1129 // However that wouldn't be right, because it's possible that
1130 // a safepoint is indeed in progress as a younger generation
1131 // stop-the-world GC happens even as we mark in this generation.
1132
1133 _restart_for_overflow = false;
1134 force_overflow_conc()->init();
1135
1136 // _g1h has _n_par_threads
1137 _parallel_marking_threads = calc_parallel_marking_threads();
1138 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1139 "Maximum number of marking threads exceeded");
1140
1141 uint active_workers = MAX2(1U, parallel_marking_threads());
1142 assert(active_workers > 0, "Should have been set");
1143
1144 // Parallel task terminator is set in "set_concurrency_and_phase()"
1145 set_concurrency_and_phase(active_workers, true /* concurrent */);
1146
1147 CMConcurrentMarkingTask markingTask(this, cmThread());
1148 _parallel_workers->set_active_workers(active_workers);
1149 _parallel_workers->run_task(&markingTask);
1150 print_stats();
1151 }
1152
1153 // Helper class to get rid of some boilerplate code.
1154 class G1CMTraceTime : public StackObj {
1155 GCTraceTimeImpl _gc_trace_time;
1156 static bool doit_and_prepend(bool doit) {
1157 if (doit) {
1158 gclog_or_tty->put(' ');
1159 }
1160 return doit;
1161 }
1162
1163 public:
1164 G1CMTraceTime(const char* title, bool doit)
1165 : _gc_trace_time(title, doit_and_prepend(doit), false, G1CollectedHeap::heap()->gc_timer_cm()) {
1166 }
1167 };
1168
1169 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1170 // world is stopped at this checkpoint
1171 assert(SafepointSynchronize::is_at_safepoint(),
1172 "world should be stopped");
1173
1174 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1175
1176 // If a full collection has happened, we shouldn't do this.
1177 if (has_aborted()) {
1178 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1179 return;
1180 }
1181
1182 SvcGCMarker sgcm(SvcGCMarker::OTHER);
1183
1184 if (VerifyDuringGC) {
1185 HandleMark hm; // handle scope
1186 g1h->prepare_for_verify();
1187 Universe::verify(VerifyOption_G1UsePrevMarking,
1188 " VerifyDuringGC:(before)");
1189 }
1190 g1h->check_bitmaps("Remark Start");
1191
1192 G1CollectorPolicy* g1p = g1h->g1_policy();
1193 g1p->record_concurrent_mark_remark_start();
1194
1195 double start = os::elapsedTime();
1196
1197 checkpointRootsFinalWork();
1198
1199 double mark_work_end = os::elapsedTime();
1200
1201 weakRefsWork(clear_all_soft_refs);
1202
1203 if (has_overflown()) {
1204 // Oops. We overflowed. Restart concurrent marking.
1205 _restart_for_overflow = true;
1206 if (G1TraceMarkStackOverflow) {
1207 gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
1208 }
1209
1210 // Verify the heap w.r.t. the previous marking bitmap.
1211 if (VerifyDuringGC) {
1212 HandleMark hm; // handle scope
1213 g1h->prepare_for_verify();
1214 Universe::verify(VerifyOption_G1UsePrevMarking,
1215 " VerifyDuringGC:(overflow)");
1216 }
1217
1218 // Clear the marking state because we will be restarting
1219 // marking due to overflowing the global mark stack.
1220 reset_marking_state();
1221 } else {
1222 {
1223 G1CMTraceTime trace("GC aggregate-data", G1Log::finer());
1224
1225 // Aggregate the per-task counting data that we have accumulated
1226 // while marking.
1227 aggregate_count_data();
1228 }
1229
1230 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1231 // We're done with marking.
1232 // This is the end of the marking cycle, we're expected all
1233 // threads to have SATB queues with active set to true.
1234 satb_mq_set.set_active_all_threads(false, /* new active value */
1235 true /* expected_active */);
1236
1237 if (VerifyDuringGC) {
1238 HandleMark hm; // handle scope
1239 g1h->prepare_for_verify();
1240 Universe::verify(VerifyOption_G1UseNextMarking,
1241 " VerifyDuringGC:(after)");
1242 }
1243 g1h->check_bitmaps("Remark End");
1244 assert(!restart_for_overflow(), "sanity");
1245 // Completely reset the marking state since marking completed
1246 set_non_marking_state();
1247 }
1248
1249 // Expand the marking stack, if we have to and if we can.
1250 if (_markStack.should_expand()) {
1251 _markStack.expand();
1252 }
1253
1254 // Statistics
1255 double now = os::elapsedTime();
1256 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1257 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1258 _remark_times.add((now - start) * 1000.0);
1259
1260 g1p->record_concurrent_mark_remark_end();
1261
1740
1741 };
1742
1743 void ConcurrentMark::cleanup() {
1744 // world is stopped at this checkpoint
1745 assert(SafepointSynchronize::is_at_safepoint(),
1746 "world should be stopped");
1747 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1748
1749 // If a full collection has happened, we shouldn't do this.
1750 if (has_aborted()) {
1751 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1752 return;
1753 }
1754
1755 g1h->verify_region_sets_optional();
1756
1757 if (VerifyDuringGC) {
1758 HandleMark hm; // handle scope
1759 g1h->prepare_for_verify();
1760 Universe::verify(VerifyOption_G1UsePrevMarking,
1761 " VerifyDuringGC:(before)");
1762 }
1763 g1h->check_bitmaps("Cleanup Start");
1764
1765 G1CollectorPolicy* g1p = g1h->g1_policy();
1766 g1p->record_concurrent_mark_cleanup_start();
1767
1768 double start = os::elapsedTime();
1769
1770 HeapRegionRemSet::reset_for_cleanup_tasks();
1771
1772 // Do counting once more with the world stopped for good measure.
1773 G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
1774
1775 g1h->workers()->run_task(&g1_par_count_task);
1776
1777 if (VerifyDuringGC) {
1778 // Verify that the counting data accumulated during marking matches
1779 // that calculated by walking the marking bitmap.
1780
1781 // Bitmaps to hold expected values
1783 BitMap expected_card_bm(_card_bm.size(), true);
1784
1785 G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
1786 &_region_bm,
1787 &_card_bm,
1788 &expected_region_bm,
1789 &expected_card_bm);
1790
1791 g1h->workers()->run_task(&g1_par_verify_task);
1792
1793 guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
1794 }
1795
1796 size_t start_used_bytes = g1h->used();
1797 g1h->collector_state()->set_mark_in_progress(false);
1798
1799 double count_end = os::elapsedTime();
1800 double this_final_counting_time = (count_end - start);
1801 _total_counting_time += this_final_counting_time;
1802
1803 if (G1PrintRegionLivenessInfo) {
1804 G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking");
1805 _g1h->heap_region_iterate(&cl);
1806 }
1807
1808 // Install newly created mark bitMap as "prev".
1809 swapMarkBitMaps();
1810
1811 g1h->reset_gc_time_stamp();
1812
1813 uint n_workers = _g1h->workers()->active_workers();
1814
1815 // Note end of marking in all heap regions.
1816 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1817 g1h->workers()->run_task(&g1_par_note_end_task);
1818 g1h->check_gc_time_stamps();
1819
1820 if (!cleanup_list_is_empty()) {
1821 // The cleanup list is not empty, so we'll have to process it
1822 // concurrently. Notify anyone else that might be wanting free
1823 // regions that there will be more free regions coming soon.
1824 g1h->set_free_regions_coming();
1827 // call below, since it affects the metric by which we sort the heap
1828 // regions.
1829 if (G1ScrubRemSets) {
1830 double rs_scrub_start = os::elapsedTime();
1831 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers);
1832 g1h->workers()->run_task(&g1_par_scrub_rs_task);
1833
1834 double rs_scrub_end = os::elapsedTime();
1835 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
1836 _total_rs_scrub_time += this_rs_scrub_time;
1837 }
1838
1839 // this will also free any regions totally full of garbage objects,
1840 // and sort the regions.
1841 g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1842
1843 // Statistics.
1844 double end = os::elapsedTime();
1845 _cleanup_times.add((end - start) * 1000.0);
1846
1847 if (G1Log::fine()) {
1848 g1h->g1_policy()->print_heap_transition(start_used_bytes);
1849 }
1850
1851 // Clean up will have freed any regions completely full of garbage.
1852 // Update the soft reference policy with the new heap occupancy.
1853 Universe::update_heap_info_at_gc();
1854
1855 if (VerifyDuringGC) {
1856 HandleMark hm; // handle scope
1857 g1h->prepare_for_verify();
1858 Universe::verify(VerifyOption_G1UsePrevMarking,
1859 " VerifyDuringGC:(after)");
1860 }
1861
1862 g1h->check_bitmaps("Cleanup End");
1863
1864 g1h->verify_region_sets_optional();
1865
1866 // We need to make this be a "collection" so any collection pause that
1867 // races with it goes around and waits for completeCleanup to finish.
1868 g1h->increment_total_collections();
1869
1870 // Clean out dead classes and update Metaspace sizes.
1871 if (ClassUnloadingWithConcurrentMark) {
1872 ClassLoaderDataGraph::purge();
1873 }
1874 MetaspaceGC::compute_new_size();
1875
1876 // We reclaimed old regions so we should calculate the sizes to make
1877 // sure we update the old gen/space data.
1878 g1h->g1mm()->update_sizes();
1879 g1h->allocation_context_stats().update_after_mark();
1880
1881 g1h->trace_heap_after_concurrent_cycle();
1882 }
1883
1884 void ConcurrentMark::completeCleanup() {
1885 if (has_aborted()) return;
1886
1887 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1888
1889 _cleanup_list.verify_optional();
1890 FreeRegionList tmp_free_list("Tmp Free List");
1891
1892 if (G1ConcRegionFreeingVerbose) {
1893 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1894 "cleanup list has %u entries",
1895 _cleanup_list.length());
1896 }
1897
1898 // No one else should be accessing the _cleanup_list at this point,
1899 // so it is not necessary to take any locks
1900 while (!_cleanup_list.is_empty()) {
1901 HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */);
1902 assert(hr != NULL, "Got NULL from a non-empty list");
1903 hr->par_clear();
1904 tmp_free_list.add_ordered(hr);
1905
1906 // Instead of adding one region at a time to the secondary_free_list,
1907 // we accumulate them in the local list and move them a few at a
1908 // time. This also cuts down on the number of notify_all() calls
1909 // we do during this process. We'll also append the local list when
1910 // _cleanup_list is empty (which means we just removed the last
1911 // region from the _cleanup_list).
1912 if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1913 _cleanup_list.is_empty()) {
1914 if (G1ConcRegionFreeingVerbose) {
1915 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1916 "appending %u entries to the secondary_free_list, "
1917 "cleanup list still has %u entries",
1918 tmp_free_list.length(),
1919 _cleanup_list.length());
1920 }
1921
1922 {
1923 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1924 g1h->secondary_free_list_add(&tmp_free_list);
1925 SecondaryFreeList_lock->notify_all();
1926 }
1927 #ifndef PRODUCT
1928 if (G1StressConcRegionFreeing) {
1929 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1930 os::sleep(Thread::current(), (jlong) 1, false);
1931 }
1932 }
1933 #endif
1934 }
1935 }
1936 assert(tmp_free_list.is_empty(), "post-condition");
1937 }
1938
1939 // Supporting Object and Oop closures for reference discovery
1940 // and processing in during marking
2157 // Skip processing the discovered references if we have
2158 // overflown the global marking stack. Reference objects
2159 // only get discovered once so it is OK to not
2160 // de-populate the discovered reference lists. We could have,
2161 // but the only benefit would be that, when marking restarts,
2162 // less reference objects are discovered.
2163 return;
2164 }
2165
2166 ResourceMark rm;
2167 HandleMark hm;
2168
2169 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2170
2171 // Is alive closure.
2172 G1CMIsAliveClosure g1_is_alive(g1h);
2173
2174 // Inner scope to exclude the cleaning of the string and symbol
2175 // tables from the displayed time.
2176 {
2177 G1CMTraceTime t("GC ref-proc", G1Log::finer());
2178
2179 ReferenceProcessor* rp = g1h->ref_processor_cm();
2180
2181 // See the comment in G1CollectedHeap::ref_processing_init()
2182 // about how reference processing currently works in G1.
2183
2184 // Set the soft reference policy
2185 rp->setup_policy(clear_all_soft_refs);
2186 assert(_markStack.isEmpty(), "mark stack should be empty");
2187
2188 // Instances of the 'Keep Alive' and 'Complete GC' closures used
2189 // in serial reference processing. Note these closures are also
2190 // used for serially processing (by the the current thread) the
2191 // JNI references during parallel reference processing.
2192 //
2193 // These closures do not need to synchronize with the worker
2194 // threads involved in parallel reference processing as these
2195 // instances are executed serially by the current thread (e.g.
2196 // reference processing is not multi-threaded and is thus
2197 // performed by the current thread instead of a gang worker).
2247 set_has_overflown();
2248 }
2249
2250 assert(rp->num_q() == active_workers, "why not");
2251
2252 rp->enqueue_discovered_references(executor);
2253
2254 rp->verify_no_references_recorded();
2255 assert(!rp->discovery_enabled(), "Post condition");
2256 }
2257
2258 if (has_overflown()) {
2259 // We can not trust g1_is_alive if the marking stack overflowed
2260 return;
2261 }
2262
2263 assert(_markStack.isEmpty(), "Marking should have completed");
2264
2265 // Unload Klasses, String, Symbols, Code Cache, etc.
2266 {
2267 G1CMTraceTime trace("Unloading", G1Log::finer());
2268
2269 if (ClassUnloadingWithConcurrentMark) {
2270 bool purged_classes;
2271
2272 {
2273 G1CMTraceTime trace("System Dictionary Unloading", G1Log::finest());
2274 purged_classes = SystemDictionary::do_unloading(&g1_is_alive, false /* Defer klass cleaning */);
2275 }
2276
2277 {
2278 G1CMTraceTime trace("Parallel Unloading", G1Log::finest());
2279 weakRefsWorkParallelPart(&g1_is_alive, purged_classes);
2280 }
2281 }
2282
2283 if (G1StringDedup::is_enabled()) {
2284 G1CMTraceTime trace("String Deduplication Unlink", G1Log::finest());
2285 G1StringDedup::unlink(&g1_is_alive);
2286 }
2287 }
2288 }
2289
2290 void ConcurrentMark::swapMarkBitMaps() {
2291 CMBitMapRO* temp = _prevMarkBitMap;
2292 _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap;
2293 _nextMarkBitMap = (CMBitMap*) temp;
2294 }
2295
2296 // Closure for marking entries in SATB buffers.
2297 class CMSATBBufferClosure : public SATBBufferClosure {
2298 private:
2299 CMTask* _task;
2300 G1CollectedHeap* _g1h;
2301
2302 // This is very similar to CMTask::deal_with_reference, but with
2303 // more relaxed requirements for the argument, so this must be more
2304 // circumspect about treating the argument as an object.
2385 true /* do_termination */,
2386 false /* is_serial */);
2387 } while (task->has_aborted() && !_cm->has_overflown());
2388 // If we overflow, then we do not want to restart. We instead
2389 // want to abort remark and do concurrent marking again.
2390 task->record_end_time();
2391 }
2392 }
2393
2394 CMRemarkTask(ConcurrentMark* cm, uint active_workers) :
2395 AbstractGangTask("Par Remark"), _cm(cm) {
2396 _cm->terminator()->reset_for_reuse(active_workers);
2397 }
2398 };
2399
2400 void ConcurrentMark::checkpointRootsFinalWork() {
2401 ResourceMark rm;
2402 HandleMark hm;
2403 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2404
2405 G1CMTraceTime trace("Finalize Marking", G1Log::finer());
2406
2407 g1h->ensure_parsability(false);
2408
2409 // this is remark, so we'll use up all active threads
2410 uint active_workers = g1h->workers()->active_workers();
2411 set_concurrency_and_phase(active_workers, false /* concurrent */);
2412 // Leave _parallel_marking_threads at it's
2413 // value originally calculated in the ConcurrentMark
2414 // constructor and pass values of the active workers
2415 // through the gang in the task.
2416
2417 {
2418 StrongRootsScope srs(active_workers);
2419
2420 CMRemarkTask remarkTask(this, active_workers);
2421 // We will start all available threads, even if we decide that the
2422 // active_workers will be fewer. The extra ones will just bail out
2423 // immediately.
2424 g1h->workers()->run_task(&remarkTask);
2425 }
2702 // Clear the global region bitmap - it will be filled as part
2703 // of the final counting task.
2704 _region_bm.clear();
2705
2706 uint max_regions = _g1h->max_regions();
2707 assert(_max_worker_id > 0, "uninitialized");
2708
2709 for (uint i = 0; i < _max_worker_id; i += 1) {
2710 BitMap* task_card_bm = count_card_bitmap_for(i);
2711 size_t* marked_bytes_array = count_marked_bytes_array_for(i);
2712
2713 assert(task_card_bm->size() == _card_bm.size(), "size mismatch");
2714 assert(marked_bytes_array != NULL, "uninitialized");
2715
2716 memset(marked_bytes_array, 0, (size_t) max_regions * sizeof(size_t));
2717 task_card_bm->clear();
2718 }
2719 }
2720
2721 void ConcurrentMark::print_stats() {
2722 if (G1MarkingVerboseLevel > 0) {
2723 gclog_or_tty->print_cr("---------------------------------------------------------------------");
2724 for (size_t i = 0; i < _active_tasks; ++i) {
2725 _tasks[i]->print_stats();
2726 gclog_or_tty->print_cr("---------------------------------------------------------------------");
2727 }
2728 }
2729 }
2730
2731 // abandon current marking iteration due to a Full GC
2732 void ConcurrentMark::abort() {
2733 if (!cmThread()->during_cycle() || _has_aborted) {
2734 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
2735 return;
2736 }
2737
2738 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
2739 // concurrent bitmap clearing.
2740 _nextMarkBitMap->clearAll();
2741
2742 // Note we cannot clear the previous marking bitmap here
2743 // since VerifyDuringGC verifies the objects marked during
2744 // a full GC against the previous bitmap.
2745
2746 // Clear the liveness counting data
2747 clear_all_count_data();
2751 _tasks[i]->clear_region_fields();
2752 }
2753 _first_overflow_barrier_sync.abort();
2754 _second_overflow_barrier_sync.abort();
2755 _has_aborted = true;
2756
2757 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2758 satb_mq_set.abandon_partial_marking();
2759 // This can be called either during or outside marking, we'll read
2760 // the expected_active value from the SATB queue set.
2761 satb_mq_set.set_active_all_threads(
2762 false, /* new active value */
2763 satb_mq_set.is_active() /* expected_active */);
2764
2765 _g1h->trace_heap_after_concurrent_cycle();
2766 _g1h->register_concurrent_cycle_end();
2767 }
2768
2769 static void print_ms_time_info(const char* prefix, const char* name,
2770 NumberSeq& ns) {
2771 gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2772 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2773 if (ns.num() > 0) {
2774 gclog_or_tty->print_cr("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2775 prefix, ns.sd(), ns.maximum());
2776 }
2777 }
2778
2779 void ConcurrentMark::print_summary_info() {
2780 gclog_or_tty->print_cr(" Concurrent marking:");
2781 print_ms_time_info(" ", "init marks", _init_times);
2782 print_ms_time_info(" ", "remarks", _remark_times);
2783 {
2784 print_ms_time_info(" ", "final marks", _remark_mark_times);
2785 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2786
2787 }
2788 print_ms_time_info(" ", "cleanups", _cleanup_times);
2789 gclog_or_tty->print_cr(" Final counting total time = %8.2f s (avg = %8.2f ms).",
2790 _total_counting_time,
2791 (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 /
2792 (double)_cleanup_times.num()
2793 : 0.0));
2794 if (G1ScrubRemSets) {
2795 gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
2796 _total_rs_scrub_time,
2797 (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 /
2798 (double)_cleanup_times.num()
2799 : 0.0));
2800 }
2801 gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.",
2802 (_init_times.sum() + _remark_times.sum() +
2803 _cleanup_times.sum())/1000.0);
2804 gclog_or_tty->print_cr(" Total concurrent time = %8.2f s "
2805 "(%8.2f s marking).",
2806 cmThread()->vtime_accum(),
2807 cmThread()->vtime_mark_accum());
2808 }
2809
2810 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2811 _parallel_workers->print_worker_threads_on(st);
2812 }
2813
2814 void ConcurrentMark::print_on_error(outputStream* st) const {
2815 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2816 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2817 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2818 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
2819 }
2820
2821 // We take a break if someone is trying to stop the world.
2822 bool ConcurrentMark::do_yield_check(uint worker_id) {
2823 if (SuspendibleThreadSet::should_yield()) {
2824 if (worker_id == 0) {
2825 _g1h->g1_policy()->record_concurrent_pause();
2826 }
2827 SuspendibleThreadSet::yield();
3167
3168 // This keeps claiming and applying the closure to completed buffers
3169 // until we run out of buffers or we need to abort.
3170 while (!has_aborted() &&
3171 satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
3172 regular_clock_call();
3173 }
3174
3175 _draining_satb_buffers = false;
3176
3177 assert(has_aborted() ||
3178 concurrent() ||
3179 satb_mq_set.completed_buffers_num() == 0, "invariant");
3180
3181 // again, this was a potentially expensive operation, decrease the
3182 // limits to get the regular clock call early
3183 decrease_limits();
3184 }
3185
3186 void CMTask::print_stats() {
3187 gclog_or_tty->print_cr("Marking Stats, task = %u, calls = %d",
3188 _worker_id, _calls);
3189 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3190 _elapsed_time_ms, _termination_time_ms);
3191 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3192 _step_times_ms.num(), _step_times_ms.avg(),
3193 _step_times_ms.sd());
3194 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
3195 _step_times_ms.maximum(), _step_times_ms.sum());
3196 }
3197
3198 bool ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, oop& obj) {
3199 return _task_queues->steal(worker_id, hash_seed, obj);
3200 }
3201
3202 /*****************************************************************************
3203
3204 The do_marking_step(time_target_ms, ...) method is the building
3205 block of the parallel marking framework. It can be called in parallel
3206 with other invocations of do_marking_step() on different tasks
3207 (but only one per task, obviously) and concurrently with the
3208 mutator threads, or during remark, hence it eliminates the need
3209 for two versions of the code. When called during remark, it will
3210 pick up from where the task left off during the concurrent marking
3211 phase. Interestingly, tasks are also claimable during evacuation
3212 pauses too, since do_marking_step() ensures that it aborts before
3213 it needs to yield.
3214
3684 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
3685 #else // _LP64
3686 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s"
3687 #endif // _LP64
3688
3689 // For per-region info
3690 #define G1PPRL_TYPE_FORMAT " %-4s"
3691 #define G1PPRL_TYPE_H_FORMAT " %4s"
3692 #define G1PPRL_BYTE_FORMAT " " SIZE_FORMAT_W(9)
3693 #define G1PPRL_BYTE_H_FORMAT " %9s"
3694 #define G1PPRL_DOUBLE_FORMAT " %14.1f"
3695 #define G1PPRL_DOUBLE_H_FORMAT " %14s"
3696
3697 // For summary info
3698 #define G1PPRL_SUM_ADDR_FORMAT(tag) " " tag ":" G1PPRL_ADDR_BASE_FORMAT
3699 #define G1PPRL_SUM_BYTE_FORMAT(tag) " " tag ": " SIZE_FORMAT
3700 #define G1PPRL_SUM_MB_FORMAT(tag) " " tag ": %1.2f MB"
3701 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
3702
3703 G1PrintRegionLivenessInfoClosure::
3704 G1PrintRegionLivenessInfoClosure(outputStream* out, const char* phase_name)
3705 : _out(out),
3706 _total_used_bytes(0), _total_capacity_bytes(0),
3707 _total_prev_live_bytes(0), _total_next_live_bytes(0),
3708 _hum_used_bytes(0), _hum_capacity_bytes(0),
3709 _hum_prev_live_bytes(0), _hum_next_live_bytes(0),
3710 _total_remset_bytes(0), _total_strong_code_roots_bytes(0) {
3711 G1CollectedHeap* g1h = G1CollectedHeap::heap();
3712 MemRegion g1_reserved = g1h->g1_reserved();
3713 double now = os::elapsedTime();
3714
3715 // Print the header of the output.
3716 _out->cr();
3717 _out->print_cr(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
3718 _out->print_cr(G1PPRL_LINE_PREFIX" HEAP"
3719 G1PPRL_SUM_ADDR_FORMAT("reserved")
3720 G1PPRL_SUM_BYTE_FORMAT("region-size"),
3721 p2i(g1_reserved.start()), p2i(g1_reserved.end()),
3722 HeapRegion::GrainBytes);
3723 _out->print_cr(G1PPRL_LINE_PREFIX);
3724 _out->print_cr(G1PPRL_LINE_PREFIX
3725 G1PPRL_TYPE_H_FORMAT
3726 G1PPRL_ADDR_BASE_H_FORMAT
3727 G1PPRL_BYTE_H_FORMAT
3728 G1PPRL_BYTE_H_FORMAT
3729 G1PPRL_BYTE_H_FORMAT
3730 G1PPRL_DOUBLE_H_FORMAT
3731 G1PPRL_BYTE_H_FORMAT
3732 G1PPRL_BYTE_H_FORMAT,
3733 "type", "address-range",
3734 "used", "prev-live", "next-live", "gc-eff",
3735 "remset", "code-roots");
3736 _out->print_cr(G1PPRL_LINE_PREFIX
3737 G1PPRL_TYPE_H_FORMAT
3738 G1PPRL_ADDR_BASE_H_FORMAT
3739 G1PPRL_BYTE_H_FORMAT
3740 G1PPRL_BYTE_H_FORMAT
3741 G1PPRL_BYTE_H_FORMAT
3742 G1PPRL_DOUBLE_H_FORMAT
3743 G1PPRL_BYTE_H_FORMAT
3744 G1PPRL_BYTE_H_FORMAT,
3745 "", "",
3746 "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)",
3747 "(bytes)", "(bytes)");
3748 }
3749
3750 // It takes as a parameter a reference to one of the _hum_* fields, it
3751 // deduces the corresponding value for a region in a humongous region
3752 // series (either the region size, or what's left if the _hum_* field
3753 // is < the region size), and updates the _hum_* field accordingly.
3754 size_t G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* hum_bytes) {
3755 size_t bytes = 0;
3756 // The > 0 check is to deal with the prev and next live bytes which
3798 _hum_used_bytes = used_bytes;
3799 _hum_prev_live_bytes = prev_live_bytes;
3800 _hum_next_live_bytes = next_live_bytes;
3801 get_hum_bytes(&used_bytes, &capacity_bytes,
3802 &prev_live_bytes, &next_live_bytes);
3803 end = bottom + HeapRegion::GrainWords;
3804 } else if (r->is_continues_humongous()) {
3805 get_hum_bytes(&used_bytes, &capacity_bytes,
3806 &prev_live_bytes, &next_live_bytes);
3807 assert(end == bottom + HeapRegion::GrainWords, "invariant");
3808 }
3809
3810 _total_used_bytes += used_bytes;
3811 _total_capacity_bytes += capacity_bytes;
3812 _total_prev_live_bytes += prev_live_bytes;
3813 _total_next_live_bytes += next_live_bytes;
3814 _total_remset_bytes += remset_bytes;
3815 _total_strong_code_roots_bytes += strong_code_roots_bytes;
3816
3817 // Print a line for this particular region.
3818 _out->print_cr(G1PPRL_LINE_PREFIX
3819 G1PPRL_TYPE_FORMAT
3820 G1PPRL_ADDR_BASE_FORMAT
3821 G1PPRL_BYTE_FORMAT
3822 G1PPRL_BYTE_FORMAT
3823 G1PPRL_BYTE_FORMAT
3824 G1PPRL_DOUBLE_FORMAT
3825 G1PPRL_BYTE_FORMAT
3826 G1PPRL_BYTE_FORMAT,
3827 type, p2i(bottom), p2i(end),
3828 used_bytes, prev_live_bytes, next_live_bytes, gc_eff,
3829 remset_bytes, strong_code_roots_bytes);
3830
3831 return false;
3832 }
3833
3834 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
3835 // add static memory usages to remembered set sizes
3836 _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
3837 // Print the footer of the output.
3838 _out->print_cr(G1PPRL_LINE_PREFIX);
3839 _out->print_cr(G1PPRL_LINE_PREFIX
3840 " SUMMARY"
3841 G1PPRL_SUM_MB_FORMAT("capacity")
3842 G1PPRL_SUM_MB_PERC_FORMAT("used")
3843 G1PPRL_SUM_MB_PERC_FORMAT("prev-live")
3844 G1PPRL_SUM_MB_PERC_FORMAT("next-live")
3845 G1PPRL_SUM_MB_FORMAT("remset")
3846 G1PPRL_SUM_MB_FORMAT("code-roots"),
3847 bytes_to_mb(_total_capacity_bytes),
3848 bytes_to_mb(_total_used_bytes),
3849 perc(_total_used_bytes, _total_capacity_bytes),
3850 bytes_to_mb(_total_prev_live_bytes),
3851 perc(_total_prev_live_bytes, _total_capacity_bytes),
3852 bytes_to_mb(_total_next_live_bytes),
3853 perc(_total_next_live_bytes, _total_capacity_bytes),
3854 bytes_to_mb(_total_remset_bytes),
3855 bytes_to_mb(_total_strong_code_roots_bytes));
3856 _out->cr();
3857 }
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/metadataOnStackMark.hpp"
27 #include "classfile/symbolTable.hpp"
28 #include "code/codeCache.hpp"
29 #include "gc/g1/concurrentMark.inline.hpp"
30 #include "gc/g1/concurrentMarkThread.inline.hpp"
31 #include "gc/g1/g1CollectedHeap.inline.hpp"
32 #include "gc/g1/g1CollectorPolicy.hpp"
33 #include "gc/g1/g1CollectorState.hpp"
34 #include "gc/g1/g1OopClosures.inline.hpp"
35 #include "gc/g1/g1RemSet.hpp"
36 #include "gc/g1/g1StringDedup.hpp"
37 #include "gc/g1/heapRegion.inline.hpp"
38 #include "gc/g1/heapRegionManager.inline.hpp"
39 #include "gc/g1/heapRegionRemSet.hpp"
40 #include "gc/g1/heapRegionSet.inline.hpp"
41 #include "gc/g1/suspendibleThreadSet.hpp"
42 #include "gc/shared/gcId.hpp"
43 #include "gc/shared/gcTimer.hpp"
44 #include "gc/shared/gcTrace.hpp"
45 #include "gc/shared/gcTraceTime.hpp"
46 #include "gc/shared/genOopClosures.inline.hpp"
47 #include "gc/shared/referencePolicy.hpp"
48 #include "gc/shared/strongRootsScope.hpp"
49 #include "gc/shared/taskqueue.inline.hpp"
50 #include "gc/shared/vmGCOperations.hpp"
51 #include "logging/log.hpp"
52 #include "memory/allocation.hpp"
53 #include "memory/resourceArea.hpp"
54 #include "oops/oop.inline.hpp"
55 #include "runtime/atomic.inline.hpp"
56 #include "runtime/handles.inline.hpp"
57 #include "runtime/java.hpp"
58 #include "runtime/prefetch.inline.hpp"
59 #include "services/memTracker.hpp"
60
61 // Concurrent marking bit map wrapper
62
63 CMBitMapRO::CMBitMapRO(int shifter) :
64 _bm(),
65 _shifter(shifter) {
66 _bmStartWord = 0;
67 _bmWordSize = 0;
68 }
69
70 HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
71 const HeapWord* limit) const {
261 rs.release();
262 return false;
263 }
264 assert(_virtual_space.committed_size() == rs.size(),
265 "Didn't reserve backing store for all of ConcurrentMark stack?");
266 _base = (oop*) _virtual_space.low();
267 setEmpty();
268 _capacity = (jint) capacity;
269 _saved_index = -1;
270 _should_expand = false;
271 return true;
272 }
273
274 void CMMarkStack::expand() {
275 // Called, during remark, if we've overflown the marking stack during marking.
276 assert(isEmpty(), "stack should been emptied while handling overflow");
277 assert(_capacity <= (jint) MarkStackSizeMax, "stack bigger than permitted");
278 // Clear expansion flag
279 _should_expand = false;
280 if (_capacity == (jint) MarkStackSizeMax) {
281 log_trace(gc)("(benign) Can't expand marking stack capacity, at max size limit");
282 return;
283 }
284 // Double capacity if possible
285 jint new_capacity = MIN2(_capacity*2, (jint) MarkStackSizeMax);
286 // Do not give up existing stack until we have managed to
287 // get the double capacity that we desired.
288 ReservedSpace rs(ReservedSpace::allocation_align_size_up(new_capacity *
289 sizeof(oop)));
290 if (rs.is_reserved()) {
291 // Release the backing store associated with old stack
292 _virtual_space.release();
293 // Reinitialize virtual space for new stack
294 if (!_virtual_space.initialize(rs, rs.size())) {
295 fatal("Not enough swap for expanded marking stack capacity");
296 }
297 _base = (oop*)(_virtual_space.low());
298 _index = 0;
299 _capacity = new_capacity;
300 } else {
301 // Failed to double capacity, continue;
302 log_trace(gc)("(benign) Failed to expand marking stack capacity from " SIZE_FORMAT "K to " SIZE_FORMAT "K",
303 _capacity / K, new_capacity / K);
304 }
305 }
306
307 void CMMarkStack::set_should_expand() {
308 // If we're resetting the marking state because of an
309 // marking stack overflow, record that we should, if
310 // possible, expand the stack.
311 _should_expand = _cm->has_overflown();
312 }
313
314 CMMarkStack::~CMMarkStack() {
315 if (_base != NULL) {
316 _base = NULL;
317 _virtual_space.release();
318 }
319 }
320
321 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
322 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
323 jint start = _index;
324 jint next_index = start + n;
553 // Calculate the number of parallel marking threads by scaling
554 // the number of parallel GC threads.
555 uint marking_thread_num = scale_parallel_threads(ParallelGCThreads);
556 FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num);
557 _sleep_factor = 0.0;
558 _marking_task_overhead = 1.0;
559 }
560
561 assert(ConcGCThreads > 0, "Should have been set");
562 _parallel_marking_threads = ConcGCThreads;
563 _max_parallel_marking_threads = _parallel_marking_threads;
564
565 if (parallel_marking_threads() > 1) {
566 _cleanup_task_overhead = 1.0;
567 } else {
568 _cleanup_task_overhead = marking_task_overhead();
569 }
570 _cleanup_sleep_factor =
571 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
572
573 _parallel_workers = new WorkGang("G1 Marker",
574 _max_parallel_marking_threads, false, true);
575 if (_parallel_workers == NULL) {
576 vm_exit_during_initialization("Failed necessary allocation.");
577 } else {
578 _parallel_workers->initialize_workers();
579 }
580
581 if (FLAG_IS_DEFAULT(MarkStackSize)) {
582 size_t mark_stack_size =
583 MIN2(MarkStackSizeMax,
584 MAX2(MarkStackSize, (size_t) (parallel_marking_threads() * TASKQUEUE_SIZE)));
585 // Verify that the calculated value for MarkStackSize is in range.
586 // It would be nice to use the private utility routine from Arguments.
587 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
588 warning("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): "
589 "must be between 1 and " SIZE_FORMAT,
590 mark_stack_size, MarkStackSizeMax);
591 return;
592 }
891 return;
892 }
893
894 // If we're executing the concurrent phase of marking, reset the marking
895 // state; otherwise the marking state is reset after reference processing,
896 // during the remark pause.
897 // If we reset here as a result of an overflow during the remark we will
898 // see assertion failures from any subsequent set_concurrency_and_phase()
899 // calls.
900 if (concurrent()) {
901 // let the task associated with with worker 0 do this
902 if (worker_id == 0) {
903 // task 0 is responsible for clearing the global data structures
904 // We should be here because of an overflow. During STW we should
905 // not clear the overflow flag since we rely on it being true when
906 // we exit this method to abort the pause and restart concurrent
907 // marking.
908 reset_marking_state(true /* clear_overflow */);
909 force_overflow()->update();
910
911 log_info(gc)("Concurrent Mark reset for overflow");
912 }
913 }
914
915 // after this, each task should reset its own data structures then
916 // then go into the second barrier
917 }
918
919 void ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
920 SuspendibleThreadSetLeaver sts_leave(concurrent());
921 _second_overflow_barrier_sync.enter();
922
923 // at this point everything should be re-initialized and ready to go
924 }
925
926 #ifndef PRODUCT
927 void ForceOverflowSettings::init() {
928 _num_remaining = G1ConcMarkForceOverflow;
929 _force = false;
930 update();
931 }
1053 ConcurrentMark* _cm;
1054
1055 public:
1056 CMRootRegionScanTask(ConcurrentMark* cm) :
1057 AbstractGangTask("Root Region Scan"), _cm(cm) { }
1058
1059 void work(uint worker_id) {
1060 assert(Thread::current()->is_ConcurrentGC_thread(),
1061 "this should only be done by a conc GC thread");
1062
1063 CMRootRegions* root_regions = _cm->root_regions();
1064 HeapRegion* hr = root_regions->claim_next();
1065 while (hr != NULL) {
1066 _cm->scanRootRegion(hr, worker_id);
1067 hr = root_regions->claim_next();
1068 }
1069 }
1070 };
1071
1072 void ConcurrentMark::scanRootRegions() {
1073 // Start of concurrent marking.
1074 ClassLoaderDataGraph::clear_claimed_marks();
1075
1076 // scan_in_progress() will have been set to true only if there was
1077 // at least one root region to scan. So, if it's false, we
1078 // should not attempt to do any further work.
1079 if (root_regions()->scan_in_progress()) {
1080 GCTraceConcTime(Info, gc) tt("Concurrent Root Region Scan");
1081
1082 _parallel_marking_threads = calc_parallel_marking_threads();
1083 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1084 "Maximum number of marking threads exceeded");
1085 uint active_workers = MAX2(1U, parallel_marking_threads());
1086
1087 CMRootRegionScanTask task(this);
1088 _parallel_workers->set_active_workers(active_workers);
1089 _parallel_workers->run_task(&task);
1090
1091 // It's possible that has_aborted() is true here without actually
1092 // aborting the survivor scan earlier. This is OK as it's
1093 // mainly used for sanity checking.
1094 root_regions()->scan_finished();
1095 }
1096 }
1097
1098 void ConcurrentMark::markFromRoots() {
1099 // we might be tempted to assert that:
1100 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1101 // "inconsistent argument?");
1102 // However that wouldn't be right, because it's possible that
1103 // a safepoint is indeed in progress as a younger generation
1104 // stop-the-world GC happens even as we mark in this generation.
1105
1106 _restart_for_overflow = false;
1107 force_overflow_conc()->init();
1108
1109 // _g1h has _n_par_threads
1110 _parallel_marking_threads = calc_parallel_marking_threads();
1111 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1112 "Maximum number of marking threads exceeded");
1113
1114 uint active_workers = MAX2(1U, parallel_marking_threads());
1115 assert(active_workers > 0, "Should have been set");
1116
1117 // Parallel task terminator is set in "set_concurrency_and_phase()"
1118 set_concurrency_and_phase(active_workers, true /* concurrent */);
1119
1120 CMConcurrentMarkingTask markingTask(this, cmThread());
1121 _parallel_workers->set_active_workers(active_workers);
1122 _parallel_workers->run_task(&markingTask);
1123 print_stats();
1124 }
1125
1126 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1127 // world is stopped at this checkpoint
1128 assert(SafepointSynchronize::is_at_safepoint(),
1129 "world should be stopped");
1130
1131 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1132
1133 // If a full collection has happened, we shouldn't do this.
1134 if (has_aborted()) {
1135 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1136 return;
1137 }
1138
1139 SvcGCMarker sgcm(SvcGCMarker::OTHER);
1140
1141 if (VerifyDuringGC) {
1142 HandleMark hm; // handle scope
1143 g1h->prepare_for_verify();
1144 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)");
1145 }
1146 g1h->check_bitmaps("Remark Start");
1147
1148 G1CollectorPolicy* g1p = g1h->g1_policy();
1149 g1p->record_concurrent_mark_remark_start();
1150
1151 double start = os::elapsedTime();
1152
1153 checkpointRootsFinalWork();
1154
1155 double mark_work_end = os::elapsedTime();
1156
1157 weakRefsWork(clear_all_soft_refs);
1158
1159 if (has_overflown()) {
1160 // Oops. We overflowed. Restart concurrent marking.
1161 _restart_for_overflow = true;
1162 log_develop(gc)("Remark led to restart for overflow.");
1163
1164 // Verify the heap w.r.t. the previous marking bitmap.
1165 if (VerifyDuringGC) {
1166 HandleMark hm; // handle scope
1167 g1h->prepare_for_verify();
1168 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (overflow)");
1169 }
1170
1171 // Clear the marking state because we will be restarting
1172 // marking due to overflowing the global mark stack.
1173 reset_marking_state();
1174 } else {
1175 {
1176 GCTraceTime(Debug, gc) trace("GC Aggregate Data", g1h->gc_timer_cm());
1177
1178 // Aggregate the per-task counting data that we have accumulated
1179 // while marking.
1180 aggregate_count_data();
1181 }
1182
1183 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1184 // We're done with marking.
1185 // This is the end of the marking cycle, we're expected all
1186 // threads to have SATB queues with active set to true.
1187 satb_mq_set.set_active_all_threads(false, /* new active value */
1188 true /* expected_active */);
1189
1190 if (VerifyDuringGC) {
1191 HandleMark hm; // handle scope
1192 g1h->prepare_for_verify();
1193 Universe::verify(VerifyOption_G1UseNextMarking, "During GC (after)");
1194 }
1195 g1h->check_bitmaps("Remark End");
1196 assert(!restart_for_overflow(), "sanity");
1197 // Completely reset the marking state since marking completed
1198 set_non_marking_state();
1199 }
1200
1201 // Expand the marking stack, if we have to and if we can.
1202 if (_markStack.should_expand()) {
1203 _markStack.expand();
1204 }
1205
1206 // Statistics
1207 double now = os::elapsedTime();
1208 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1209 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1210 _remark_times.add((now - start) * 1000.0);
1211
1212 g1p->record_concurrent_mark_remark_end();
1213
1692
1693 };
1694
1695 void ConcurrentMark::cleanup() {
1696 // world is stopped at this checkpoint
1697 assert(SafepointSynchronize::is_at_safepoint(),
1698 "world should be stopped");
1699 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1700
1701 // If a full collection has happened, we shouldn't do this.
1702 if (has_aborted()) {
1703 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1704 return;
1705 }
1706
1707 g1h->verify_region_sets_optional();
1708
1709 if (VerifyDuringGC) {
1710 HandleMark hm; // handle scope
1711 g1h->prepare_for_verify();
1712 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)");
1713 }
1714 g1h->check_bitmaps("Cleanup Start");
1715
1716 G1CollectorPolicy* g1p = g1h->g1_policy();
1717 g1p->record_concurrent_mark_cleanup_start();
1718
1719 double start = os::elapsedTime();
1720
1721 HeapRegionRemSet::reset_for_cleanup_tasks();
1722
1723 // Do counting once more with the world stopped for good measure.
1724 G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
1725
1726 g1h->workers()->run_task(&g1_par_count_task);
1727
1728 if (VerifyDuringGC) {
1729 // Verify that the counting data accumulated during marking matches
1730 // that calculated by walking the marking bitmap.
1731
1732 // Bitmaps to hold expected values
1734 BitMap expected_card_bm(_card_bm.size(), true);
1735
1736 G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
1737 &_region_bm,
1738 &_card_bm,
1739 &expected_region_bm,
1740 &expected_card_bm);
1741
1742 g1h->workers()->run_task(&g1_par_verify_task);
1743
1744 guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
1745 }
1746
1747 size_t start_used_bytes = g1h->used();
1748 g1h->collector_state()->set_mark_in_progress(false);
1749
1750 double count_end = os::elapsedTime();
1751 double this_final_counting_time = (count_end - start);
1752 _total_counting_time += this_final_counting_time;
1753
1754 if (Log<LOG_TAGS(gc, liveness)>::is_trace()) {
1755 G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1756 _g1h->heap_region_iterate(&cl);
1757 }
1758
1759 // Install newly created mark bitMap as "prev".
1760 swapMarkBitMaps();
1761
1762 g1h->reset_gc_time_stamp();
1763
1764 uint n_workers = _g1h->workers()->active_workers();
1765
1766 // Note end of marking in all heap regions.
1767 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1768 g1h->workers()->run_task(&g1_par_note_end_task);
1769 g1h->check_gc_time_stamps();
1770
1771 if (!cleanup_list_is_empty()) {
1772 // The cleanup list is not empty, so we'll have to process it
1773 // concurrently. Notify anyone else that might be wanting free
1774 // regions that there will be more free regions coming soon.
1775 g1h->set_free_regions_coming();
1778 // call below, since it affects the metric by which we sort the heap
1779 // regions.
1780 if (G1ScrubRemSets) {
1781 double rs_scrub_start = os::elapsedTime();
1782 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers);
1783 g1h->workers()->run_task(&g1_par_scrub_rs_task);
1784
1785 double rs_scrub_end = os::elapsedTime();
1786 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
1787 _total_rs_scrub_time += this_rs_scrub_time;
1788 }
1789
1790 // this will also free any regions totally full of garbage objects,
1791 // and sort the regions.
1792 g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1793
1794 // Statistics.
1795 double end = os::elapsedTime();
1796 _cleanup_times.add((end - start) * 1000.0);
1797
1798 // Clean up will have freed any regions completely full of garbage.
1799 // Update the soft reference policy with the new heap occupancy.
1800 Universe::update_heap_info_at_gc();
1801
1802 if (VerifyDuringGC) {
1803 HandleMark hm; // handle scope
1804 g1h->prepare_for_verify();
1805 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (after)");
1806 }
1807
1808 g1h->check_bitmaps("Cleanup End");
1809
1810 g1h->verify_region_sets_optional();
1811
1812 // We need to make this be a "collection" so any collection pause that
1813 // races with it goes around and waits for completeCleanup to finish.
1814 g1h->increment_total_collections();
1815
1816 // Clean out dead classes and update Metaspace sizes.
1817 if (ClassUnloadingWithConcurrentMark) {
1818 ClassLoaderDataGraph::purge();
1819 }
1820 MetaspaceGC::compute_new_size();
1821
1822 // We reclaimed old regions so we should calculate the sizes to make
1823 // sure we update the old gen/space data.
1824 g1h->g1mm()->update_sizes();
1825 g1h->allocation_context_stats().update_after_mark();
1826
1827 g1h->trace_heap_after_concurrent_cycle();
1828 }
1829
1830 void ConcurrentMark::completeCleanup() {
1831 if (has_aborted()) return;
1832
1833 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1834
1835 _cleanup_list.verify_optional();
1836 FreeRegionList tmp_free_list("Tmp Free List");
1837
1838 log_develop(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "
1839 "cleanup list has %u entries",
1840 _cleanup_list.length());
1841
1842 // No one else should be accessing the _cleanup_list at this point,
1843 // so it is not necessary to take any locks
1844 while (!_cleanup_list.is_empty()) {
1845 HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */);
1846 assert(hr != NULL, "Got NULL from a non-empty list");
1847 hr->par_clear();
1848 tmp_free_list.add_ordered(hr);
1849
1850 // Instead of adding one region at a time to the secondary_free_list,
1851 // we accumulate them in the local list and move them a few at a
1852 // time. This also cuts down on the number of notify_all() calls
1853 // we do during this process. We'll also append the local list when
1854 // _cleanup_list is empty (which means we just removed the last
1855 // region from the _cleanup_list).
1856 if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1857 _cleanup_list.is_empty()) {
1858 log_develop(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "
1859 "appending %u entries to the secondary_free_list, "
1860 "cleanup list still has %u entries",
1861 tmp_free_list.length(),
1862 _cleanup_list.length());
1863
1864 {
1865 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1866 g1h->secondary_free_list_add(&tmp_free_list);
1867 SecondaryFreeList_lock->notify_all();
1868 }
1869 #ifndef PRODUCT
1870 if (G1StressConcRegionFreeing) {
1871 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1872 os::sleep(Thread::current(), (jlong) 1, false);
1873 }
1874 }
1875 #endif
1876 }
1877 }
1878 assert(tmp_free_list.is_empty(), "post-condition");
1879 }
1880
1881 // Supporting Object and Oop closures for reference discovery
1882 // and processing in during marking
2099 // Skip processing the discovered references if we have
2100 // overflown the global marking stack. Reference objects
2101 // only get discovered once so it is OK to not
2102 // de-populate the discovered reference lists. We could have,
2103 // but the only benefit would be that, when marking restarts,
2104 // less reference objects are discovered.
2105 return;
2106 }
2107
2108 ResourceMark rm;
2109 HandleMark hm;
2110
2111 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2112
2113 // Is alive closure.
2114 G1CMIsAliveClosure g1_is_alive(g1h);
2115
2116 // Inner scope to exclude the cleaning of the string and symbol
2117 // tables from the displayed time.
2118 {
2119 GCTraceTime(Debug, gc) trace("GC Ref Proc", g1h->gc_timer_cm());
2120
2121 ReferenceProcessor* rp = g1h->ref_processor_cm();
2122
2123 // See the comment in G1CollectedHeap::ref_processing_init()
2124 // about how reference processing currently works in G1.
2125
2126 // Set the soft reference policy
2127 rp->setup_policy(clear_all_soft_refs);
2128 assert(_markStack.isEmpty(), "mark stack should be empty");
2129
2130 // Instances of the 'Keep Alive' and 'Complete GC' closures used
2131 // in serial reference processing. Note these closures are also
2132 // used for serially processing (by the the current thread) the
2133 // JNI references during parallel reference processing.
2134 //
2135 // These closures do not need to synchronize with the worker
2136 // threads involved in parallel reference processing as these
2137 // instances are executed serially by the current thread (e.g.
2138 // reference processing is not multi-threaded and is thus
2139 // performed by the current thread instead of a gang worker).
2189 set_has_overflown();
2190 }
2191
2192 assert(rp->num_q() == active_workers, "why not");
2193
2194 rp->enqueue_discovered_references(executor);
2195
2196 rp->verify_no_references_recorded();
2197 assert(!rp->discovery_enabled(), "Post condition");
2198 }
2199
2200 if (has_overflown()) {
2201 // We can not trust g1_is_alive if the marking stack overflowed
2202 return;
2203 }
2204
2205 assert(_markStack.isEmpty(), "Marking should have completed");
2206
2207 // Unload Klasses, String, Symbols, Code Cache, etc.
2208 {
2209 GCTraceTime(Debug, gc) trace("Unloading", g1h->gc_timer_cm());
2210
2211 if (ClassUnloadingWithConcurrentMark) {
2212 bool purged_classes;
2213
2214 {
2215 GCTraceTime(Trace, gc) trace("System Dictionary Unloading", g1h->gc_timer_cm());
2216 purged_classes = SystemDictionary::do_unloading(&g1_is_alive, false /* Defer klass cleaning */);
2217 }
2218
2219 {
2220 GCTraceTime(Trace, gc) trace("Parallel Unloading", g1h->gc_timer_cm());
2221 weakRefsWorkParallelPart(&g1_is_alive, purged_classes);
2222 }
2223 }
2224
2225 if (G1StringDedup::is_enabled()) {
2226 GCTraceTime(Trace, gc) trace("String Deduplication Unlink", g1h->gc_timer_cm());
2227 G1StringDedup::unlink(&g1_is_alive);
2228 }
2229 }
2230 }
2231
2232 void ConcurrentMark::swapMarkBitMaps() {
2233 CMBitMapRO* temp = _prevMarkBitMap;
2234 _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap;
2235 _nextMarkBitMap = (CMBitMap*) temp;
2236 }
2237
2238 // Closure for marking entries in SATB buffers.
2239 class CMSATBBufferClosure : public SATBBufferClosure {
2240 private:
2241 CMTask* _task;
2242 G1CollectedHeap* _g1h;
2243
2244 // This is very similar to CMTask::deal_with_reference, but with
2245 // more relaxed requirements for the argument, so this must be more
2246 // circumspect about treating the argument as an object.
2327 true /* do_termination */,
2328 false /* is_serial */);
2329 } while (task->has_aborted() && !_cm->has_overflown());
2330 // If we overflow, then we do not want to restart. We instead
2331 // want to abort remark and do concurrent marking again.
2332 task->record_end_time();
2333 }
2334 }
2335
2336 CMRemarkTask(ConcurrentMark* cm, uint active_workers) :
2337 AbstractGangTask("Par Remark"), _cm(cm) {
2338 _cm->terminator()->reset_for_reuse(active_workers);
2339 }
2340 };
2341
2342 void ConcurrentMark::checkpointRootsFinalWork() {
2343 ResourceMark rm;
2344 HandleMark hm;
2345 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2346
2347 GCTraceTime(Debug, gc) trace("Finalize Marking", g1h->gc_timer_cm());
2348
2349 g1h->ensure_parsability(false);
2350
2351 // this is remark, so we'll use up all active threads
2352 uint active_workers = g1h->workers()->active_workers();
2353 set_concurrency_and_phase(active_workers, false /* concurrent */);
2354 // Leave _parallel_marking_threads at it's
2355 // value originally calculated in the ConcurrentMark
2356 // constructor and pass values of the active workers
2357 // through the gang in the task.
2358
2359 {
2360 StrongRootsScope srs(active_workers);
2361
2362 CMRemarkTask remarkTask(this, active_workers);
2363 // We will start all available threads, even if we decide that the
2364 // active_workers will be fewer. The extra ones will just bail out
2365 // immediately.
2366 g1h->workers()->run_task(&remarkTask);
2367 }
2644 // Clear the global region bitmap - it will be filled as part
2645 // of the final counting task.
2646 _region_bm.clear();
2647
2648 uint max_regions = _g1h->max_regions();
2649 assert(_max_worker_id > 0, "uninitialized");
2650
2651 for (uint i = 0; i < _max_worker_id; i += 1) {
2652 BitMap* task_card_bm = count_card_bitmap_for(i);
2653 size_t* marked_bytes_array = count_marked_bytes_array_for(i);
2654
2655 assert(task_card_bm->size() == _card_bm.size(), "size mismatch");
2656 assert(marked_bytes_array != NULL, "uninitialized");
2657
2658 memset(marked_bytes_array, 0, (size_t) max_regions * sizeof(size_t));
2659 task_card_bm->clear();
2660 }
2661 }
2662
2663 void ConcurrentMark::print_stats() {
2664 if (!log_is_enabled(Debug, gc, stats)) {
2665 return;
2666 }
2667 log_debug(gc, stats)("---------------------------------------------------------------------");
2668 for (size_t i = 0; i < _active_tasks; ++i) {
2669 _tasks[i]->print_stats();
2670 log_debug(gc, stats)("---------------------------------------------------------------------");
2671 }
2672 }
2673
2674 // abandon current marking iteration due to a Full GC
2675 void ConcurrentMark::abort() {
2676 if (!cmThread()->during_cycle() || _has_aborted) {
2677 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
2678 return;
2679 }
2680
2681 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
2682 // concurrent bitmap clearing.
2683 _nextMarkBitMap->clearAll();
2684
2685 // Note we cannot clear the previous marking bitmap here
2686 // since VerifyDuringGC verifies the objects marked during
2687 // a full GC against the previous bitmap.
2688
2689 // Clear the liveness counting data
2690 clear_all_count_data();
2694 _tasks[i]->clear_region_fields();
2695 }
2696 _first_overflow_barrier_sync.abort();
2697 _second_overflow_barrier_sync.abort();
2698 _has_aborted = true;
2699
2700 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2701 satb_mq_set.abandon_partial_marking();
2702 // This can be called either during or outside marking, we'll read
2703 // the expected_active value from the SATB queue set.
2704 satb_mq_set.set_active_all_threads(
2705 false, /* new active value */
2706 satb_mq_set.is_active() /* expected_active */);
2707
2708 _g1h->trace_heap_after_concurrent_cycle();
2709 _g1h->register_concurrent_cycle_end();
2710 }
2711
2712 static void print_ms_time_info(const char* prefix, const char* name,
2713 NumberSeq& ns) {
2714 log_trace(gc, marking, stats, exit)("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2715 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2716 if (ns.num() > 0) {
2717 log_trace(gc, marking, stats, exit)("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2718 prefix, ns.sd(), ns.maximum());
2719 }
2720 }
2721
2722 void ConcurrentMark::print_summary_info() {
2723 LogHandle(gc, marking, stats, exit) log;
2724 if (!log.is_trace()) {
2725 return;
2726 }
2727
2728 log.trace(" Concurrent marking:");
2729 print_ms_time_info(" ", "init marks", _init_times);
2730 print_ms_time_info(" ", "remarks", _remark_times);
2731 {
2732 print_ms_time_info(" ", "final marks", _remark_mark_times);
2733 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2734
2735 }
2736 print_ms_time_info(" ", "cleanups", _cleanup_times);
2737 log.trace(" Final counting total time = %8.2f s (avg = %8.2f ms).",
2738 _total_counting_time, (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2739 if (G1ScrubRemSets) {
2740 log.trace(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
2741 _total_rs_scrub_time, (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2742 }
2743 log.trace(" Total stop_world time = %8.2f s.",
2744 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2745 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2746 cmThread()->vtime_accum(), cmThread()->vtime_mark_accum());
2747 }
2748
2749 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2750 _parallel_workers->print_worker_threads_on(st);
2751 }
2752
2753 void ConcurrentMark::print_on_error(outputStream* st) const {
2754 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2755 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2756 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2757 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
2758 }
2759
2760 // We take a break if someone is trying to stop the world.
2761 bool ConcurrentMark::do_yield_check(uint worker_id) {
2762 if (SuspendibleThreadSet::should_yield()) {
2763 if (worker_id == 0) {
2764 _g1h->g1_policy()->record_concurrent_pause();
2765 }
2766 SuspendibleThreadSet::yield();
3106
3107 // This keeps claiming and applying the closure to completed buffers
3108 // until we run out of buffers or we need to abort.
3109 while (!has_aborted() &&
3110 satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
3111 regular_clock_call();
3112 }
3113
3114 _draining_satb_buffers = false;
3115
3116 assert(has_aborted() ||
3117 concurrent() ||
3118 satb_mq_set.completed_buffers_num() == 0, "invariant");
3119
3120 // again, this was a potentially expensive operation, decrease the
3121 // limits to get the regular clock call early
3122 decrease_limits();
3123 }
3124
3125 void CMTask::print_stats() {
3126 log_debug(gc, stats)("Marking Stats, task = %u, calls = %d",
3127 _worker_id, _calls);
3128 log_debug(gc, stats)(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3129 _elapsed_time_ms, _termination_time_ms);
3130 log_debug(gc, stats)(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3131 _step_times_ms.num(), _step_times_ms.avg(),
3132 _step_times_ms.sd());
3133 log_debug(gc, stats)(" max = %1.2lfms, total = %1.2lfms",
3134 _step_times_ms.maximum(), _step_times_ms.sum());
3135 }
3136
3137 bool ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, oop& obj) {
3138 return _task_queues->steal(worker_id, hash_seed, obj);
3139 }
3140
3141 /*****************************************************************************
3142
3143 The do_marking_step(time_target_ms, ...) method is the building
3144 block of the parallel marking framework. It can be called in parallel
3145 with other invocations of do_marking_step() on different tasks
3146 (but only one per task, obviously) and concurrently with the
3147 mutator threads, or during remark, hence it eliminates the need
3148 for two versions of the code. When called during remark, it will
3149 pick up from where the task left off during the concurrent marking
3150 phase. Interestingly, tasks are also claimable during evacuation
3151 pauses too, since do_marking_step() ensures that it aborts before
3152 it needs to yield.
3153
3623 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
3624 #else // _LP64
3625 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s"
3626 #endif // _LP64
3627
3628 // For per-region info
3629 #define G1PPRL_TYPE_FORMAT " %-4s"
3630 #define G1PPRL_TYPE_H_FORMAT " %4s"
3631 #define G1PPRL_BYTE_FORMAT " " SIZE_FORMAT_W(9)
3632 #define G1PPRL_BYTE_H_FORMAT " %9s"
3633 #define G1PPRL_DOUBLE_FORMAT " %14.1f"
3634 #define G1PPRL_DOUBLE_H_FORMAT " %14s"
3635
3636 // For summary info
3637 #define G1PPRL_SUM_ADDR_FORMAT(tag) " " tag ":" G1PPRL_ADDR_BASE_FORMAT
3638 #define G1PPRL_SUM_BYTE_FORMAT(tag) " " tag ": " SIZE_FORMAT
3639 #define G1PPRL_SUM_MB_FORMAT(tag) " " tag ": %1.2f MB"
3640 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
3641
3642 G1PrintRegionLivenessInfoClosure::
3643 G1PrintRegionLivenessInfoClosure(const char* phase_name)
3644 : _total_used_bytes(0), _total_capacity_bytes(0),
3645 _total_prev_live_bytes(0), _total_next_live_bytes(0),
3646 _hum_used_bytes(0), _hum_capacity_bytes(0),
3647 _hum_prev_live_bytes(0), _hum_next_live_bytes(0),
3648 _total_remset_bytes(0), _total_strong_code_roots_bytes(0) {
3649 G1CollectedHeap* g1h = G1CollectedHeap::heap();
3650 MemRegion g1_reserved = g1h->g1_reserved();
3651 double now = os::elapsedTime();
3652
3653 // Print the header of the output.
3654 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
3655 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" HEAP"
3656 G1PPRL_SUM_ADDR_FORMAT("reserved")
3657 G1PPRL_SUM_BYTE_FORMAT("region-size"),
3658 p2i(g1_reserved.start()), p2i(g1_reserved.end()),
3659 HeapRegion::GrainBytes);
3660 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
3661 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3662 G1PPRL_TYPE_H_FORMAT
3663 G1PPRL_ADDR_BASE_H_FORMAT
3664 G1PPRL_BYTE_H_FORMAT
3665 G1PPRL_BYTE_H_FORMAT
3666 G1PPRL_BYTE_H_FORMAT
3667 G1PPRL_DOUBLE_H_FORMAT
3668 G1PPRL_BYTE_H_FORMAT
3669 G1PPRL_BYTE_H_FORMAT,
3670 "type", "address-range",
3671 "used", "prev-live", "next-live", "gc-eff",
3672 "remset", "code-roots");
3673 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3674 G1PPRL_TYPE_H_FORMAT
3675 G1PPRL_ADDR_BASE_H_FORMAT
3676 G1PPRL_BYTE_H_FORMAT
3677 G1PPRL_BYTE_H_FORMAT
3678 G1PPRL_BYTE_H_FORMAT
3679 G1PPRL_DOUBLE_H_FORMAT
3680 G1PPRL_BYTE_H_FORMAT
3681 G1PPRL_BYTE_H_FORMAT,
3682 "", "",
3683 "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)",
3684 "(bytes)", "(bytes)");
3685 }
3686
3687 // It takes as a parameter a reference to one of the _hum_* fields, it
3688 // deduces the corresponding value for a region in a humongous region
3689 // series (either the region size, or what's left if the _hum_* field
3690 // is < the region size), and updates the _hum_* field accordingly.
3691 size_t G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* hum_bytes) {
3692 size_t bytes = 0;
3693 // The > 0 check is to deal with the prev and next live bytes which
3735 _hum_used_bytes = used_bytes;
3736 _hum_prev_live_bytes = prev_live_bytes;
3737 _hum_next_live_bytes = next_live_bytes;
3738 get_hum_bytes(&used_bytes, &capacity_bytes,
3739 &prev_live_bytes, &next_live_bytes);
3740 end = bottom + HeapRegion::GrainWords;
3741 } else if (r->is_continues_humongous()) {
3742 get_hum_bytes(&used_bytes, &capacity_bytes,
3743 &prev_live_bytes, &next_live_bytes);
3744 assert(end == bottom + HeapRegion::GrainWords, "invariant");
3745 }
3746
3747 _total_used_bytes += used_bytes;
3748 _total_capacity_bytes += capacity_bytes;
3749 _total_prev_live_bytes += prev_live_bytes;
3750 _total_next_live_bytes += next_live_bytes;
3751 _total_remset_bytes += remset_bytes;
3752 _total_strong_code_roots_bytes += strong_code_roots_bytes;
3753
3754 // Print a line for this particular region.
3755 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3756 G1PPRL_TYPE_FORMAT
3757 G1PPRL_ADDR_BASE_FORMAT
3758 G1PPRL_BYTE_FORMAT
3759 G1PPRL_BYTE_FORMAT
3760 G1PPRL_BYTE_FORMAT
3761 G1PPRL_DOUBLE_FORMAT
3762 G1PPRL_BYTE_FORMAT
3763 G1PPRL_BYTE_FORMAT,
3764 type, p2i(bottom), p2i(end),
3765 used_bytes, prev_live_bytes, next_live_bytes, gc_eff,
3766 remset_bytes, strong_code_roots_bytes);
3767
3768 return false;
3769 }
3770
3771 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
3772 // add static memory usages to remembered set sizes
3773 _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
3774 // Print the footer of the output.
3775 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
3776 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3777 " SUMMARY"
3778 G1PPRL_SUM_MB_FORMAT("capacity")
3779 G1PPRL_SUM_MB_PERC_FORMAT("used")
3780 G1PPRL_SUM_MB_PERC_FORMAT("prev-live")
3781 G1PPRL_SUM_MB_PERC_FORMAT("next-live")
3782 G1PPRL_SUM_MB_FORMAT("remset")
3783 G1PPRL_SUM_MB_FORMAT("code-roots"),
3784 bytes_to_mb(_total_capacity_bytes),
3785 bytes_to_mb(_total_used_bytes),
3786 perc(_total_used_bytes, _total_capacity_bytes),
3787 bytes_to_mb(_total_prev_live_bytes),
3788 perc(_total_prev_live_bytes, _total_capacity_bytes),
3789 bytes_to_mb(_total_next_live_bytes),
3790 perc(_total_next_live_bytes, _total_capacity_bytes),
3791 bytes_to_mb(_total_remset_bytes),
3792 bytes_to_mb(_total_strong_code_roots_bytes));
3793 }
|