14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/metadataOnStackMark.hpp"
27 #include "classfile/symbolTable.hpp"
28 #include "code/codeCache.hpp"
29 #include "gc/g1/concurrentMark.inline.hpp"
30 #include "gc/g1/concurrentMarkThread.inline.hpp"
31 #include "gc/g1/g1CollectedHeap.inline.hpp"
32 #include "gc/g1/g1CollectorPolicy.hpp"
33 #include "gc/g1/g1CollectorState.hpp"
34 #include "gc/g1/g1ErgoVerbose.hpp"
35 #include "gc/g1/g1Log.hpp"
36 #include "gc/g1/g1OopClosures.inline.hpp"
37 #include "gc/g1/g1RemSet.hpp"
38 #include "gc/g1/g1StringDedup.hpp"
39 #include "gc/g1/heapRegion.inline.hpp"
40 #include "gc/g1/heapRegionManager.inline.hpp"
41 #include "gc/g1/heapRegionRemSet.hpp"
42 #include "gc/g1/heapRegionSet.inline.hpp"
43 #include "gc/g1/suspendibleThreadSet.hpp"
44 #include "gc/shared/gcId.hpp"
45 #include "gc/shared/gcTimer.hpp"
46 #include "gc/shared/gcTrace.hpp"
47 #include "gc/shared/gcTraceTime.hpp"
48 #include "gc/shared/genOopClosures.inline.hpp"
49 #include "gc/shared/referencePolicy.hpp"
50 #include "gc/shared/strongRootsScope.hpp"
51 #include "gc/shared/taskqueue.inline.hpp"
52 #include "gc/shared/vmGCOperations.hpp"
53 #include "memory/allocation.hpp"
54 #include "memory/resourceArea.hpp"
55 #include "oops/oop.inline.hpp"
56 #include "runtime/atomic.inline.hpp"
57 #include "runtime/handles.inline.hpp"
58 #include "runtime/java.hpp"
59 #include "runtime/prefetch.inline.hpp"
60 #include "services/memTracker.hpp"
61
62 // Concurrent marking bit map wrapper
63
64 CMBitMapRO::CMBitMapRO(int shifter) :
65 _bm(),
66 _shifter(shifter) {
67 _bmStartWord = 0;
68 _bmWordSize = 0;
69 }
70
71 HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
72 const HeapWord* limit) const {
262 rs.release();
263 return false;
264 }
265 assert(_virtual_space.committed_size() == rs.size(),
266 "Didn't reserve backing store for all of ConcurrentMark stack?");
267 _base = (oop*) _virtual_space.low();
268 setEmpty();
269 _capacity = (jint) capacity;
270 _saved_index = -1;
271 _should_expand = false;
272 return true;
273 }
274
275 void CMMarkStack::expand() {
276 // Called, during remark, if we've overflown the marking stack during marking.
277 assert(isEmpty(), "stack should been emptied while handling overflow");
278 assert(_capacity <= (jint) MarkStackSizeMax, "stack bigger than permitted");
279 // Clear expansion flag
280 _should_expand = false;
281 if (_capacity == (jint) MarkStackSizeMax) {
282 if (PrintGCDetails && Verbose) {
283 gclog_or_tty->print_cr(" (benign) Can't expand marking stack capacity, at max size limit");
284 }
285 return;
286 }
287 // Double capacity if possible
288 jint new_capacity = MIN2(_capacity*2, (jint) MarkStackSizeMax);
289 // Do not give up existing stack until we have managed to
290 // get the double capacity that we desired.
291 ReservedSpace rs(ReservedSpace::allocation_align_size_up(new_capacity *
292 sizeof(oop)));
293 if (rs.is_reserved()) {
294 // Release the backing store associated with old stack
295 _virtual_space.release();
296 // Reinitialize virtual space for new stack
297 if (!_virtual_space.initialize(rs, rs.size())) {
298 fatal("Not enough swap for expanded marking stack capacity");
299 }
300 _base = (oop*)(_virtual_space.low());
301 _index = 0;
302 _capacity = new_capacity;
303 } else {
304 if (PrintGCDetails && Verbose) {
305 // Failed to double capacity, continue;
306 gclog_or_tty->print(" (benign) Failed to expand marking stack capacity from "
307 SIZE_FORMAT "K to " SIZE_FORMAT "K",
308 _capacity / K, new_capacity / K);
309 }
310 }
311 }
312
313 void CMMarkStack::set_should_expand() {
314 // If we're resetting the marking state because of an
315 // marking stack overflow, record that we should, if
316 // possible, expand the stack.
317 _should_expand = _cm->has_overflown();
318 }
319
320 CMMarkStack::~CMMarkStack() {
321 if (_base != NULL) {
322 _base = NULL;
323 _virtual_space.release();
324 }
325 }
326
327 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
328 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
329 jint start = _index;
330 jint next_index = start + n;
559 // Calculate the number of parallel marking threads by scaling
560 // the number of parallel GC threads.
561 uint marking_thread_num = scale_parallel_threads(ParallelGCThreads);
562 FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num);
563 _sleep_factor = 0.0;
564 _marking_task_overhead = 1.0;
565 }
566
567 assert(ConcGCThreads > 0, "Should have been set");
568 _parallel_marking_threads = ConcGCThreads;
569 _max_parallel_marking_threads = _parallel_marking_threads;
570
571 if (parallel_marking_threads() > 1) {
572 _cleanup_task_overhead = 1.0;
573 } else {
574 _cleanup_task_overhead = marking_task_overhead();
575 }
576 _cleanup_sleep_factor =
577 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
578
579 #if 0
580 gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads());
581 gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead());
582 gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor());
583 gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead());
584 gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor());
585 #endif
586
587 _parallel_workers = new WorkGang("G1 Marker",
588 _max_parallel_marking_threads, false, true);
589 if (_parallel_workers == NULL) {
590 vm_exit_during_initialization("Failed necessary allocation.");
591 } else {
592 _parallel_workers->initialize_workers();
593 }
594
595 if (FLAG_IS_DEFAULT(MarkStackSize)) {
596 size_t mark_stack_size =
597 MIN2(MarkStackSizeMax,
598 MAX2(MarkStackSize, (size_t) (parallel_marking_threads() * TASKQUEUE_SIZE)));
599 // Verify that the calculated value for MarkStackSize is in range.
600 // It would be nice to use the private utility routine from Arguments.
601 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
602 warning("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): "
603 "must be between 1 and " SIZE_FORMAT,
604 mark_stack_size, MarkStackSizeMax);
605 return;
606 }
905 return;
906 }
907
908 // If we're executing the concurrent phase of marking, reset the marking
909 // state; otherwise the marking state is reset after reference processing,
910 // during the remark pause.
911 // If we reset here as a result of an overflow during the remark we will
912 // see assertion failures from any subsequent set_concurrency_and_phase()
913 // calls.
914 if (concurrent()) {
915 // let the task associated with with worker 0 do this
916 if (worker_id == 0) {
917 // task 0 is responsible for clearing the global data structures
918 // We should be here because of an overflow. During STW we should
919 // not clear the overflow flag since we rely on it being true when
920 // we exit this method to abort the pause and restart concurrent
921 // marking.
922 reset_marking_state(true /* clear_overflow */);
923 force_overflow()->update();
924
925 if (G1Log::fine()) {
926 gclog_or_tty->gclog_stamp();
927 gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]");
928 }
929 }
930 }
931
932 // after this, each task should reset its own data structures then
933 // then go into the second barrier
934 }
935
936 void ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
937 SuspendibleThreadSetLeaver sts_leave(concurrent());
938 _second_overflow_barrier_sync.enter();
939
940 // at this point everything should be re-initialized and ready to go
941 }
942
943 #ifndef PRODUCT
944 void ForceOverflowSettings::init() {
945 _num_remaining = G1ConcMarkForceOverflow;
946 _force = false;
947 update();
948 }
1070 ConcurrentMark* _cm;
1071
1072 public:
1073 CMRootRegionScanTask(ConcurrentMark* cm) :
1074 AbstractGangTask("Root Region Scan"), _cm(cm) { }
1075
1076 void work(uint worker_id) {
1077 assert(Thread::current()->is_ConcurrentGC_thread(),
1078 "this should only be done by a conc GC thread");
1079
1080 CMRootRegions* root_regions = _cm->root_regions();
1081 HeapRegion* hr = root_regions->claim_next();
1082 while (hr != NULL) {
1083 _cm->scanRootRegion(hr, worker_id);
1084 hr = root_regions->claim_next();
1085 }
1086 }
1087 };
1088
1089 void ConcurrentMark::scanRootRegions() {
1090 double scan_start = os::elapsedTime();
1091
1092 // Start of concurrent marking.
1093 ClassLoaderDataGraph::clear_claimed_marks();
1094
1095 // scan_in_progress() will have been set to true only if there was
1096 // at least one root region to scan. So, if it's false, we
1097 // should not attempt to do any further work.
1098 if (root_regions()->scan_in_progress()) {
1099 if (G1Log::fine()) {
1100 gclog_or_tty->gclog_stamp();
1101 gclog_or_tty->print_cr("[GC concurrent-root-region-scan-start]");
1102 }
1103
1104 _parallel_marking_threads = calc_parallel_marking_threads();
1105 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1106 "Maximum number of marking threads exceeded");
1107 uint active_workers = MAX2(1U, parallel_marking_threads());
1108
1109 CMRootRegionScanTask task(this);
1110 _parallel_workers->set_active_workers(active_workers);
1111 _parallel_workers->run_task(&task);
1112
1113 if (G1Log::fine()) {
1114 gclog_or_tty->gclog_stamp();
1115 gclog_or_tty->print_cr("[GC concurrent-root-region-scan-end, %1.7lf secs]", os::elapsedTime() - scan_start);
1116 }
1117
1118 // It's possible that has_aborted() is true here without actually
1119 // aborting the survivor scan earlier. This is OK as it's
1120 // mainly used for sanity checking.
1121 root_regions()->scan_finished();
1122 }
1123 }
1124
1125 void ConcurrentMark::markFromRoots() {
1126 // we might be tempted to assert that:
1127 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1128 // "inconsistent argument?");
1129 // However that wouldn't be right, because it's possible that
1130 // a safepoint is indeed in progress as a younger generation
1131 // stop-the-world GC happens even as we mark in this generation.
1132
1133 _restart_for_overflow = false;
1134 force_overflow_conc()->init();
1135
1136 // _g1h has _n_par_threads
1137 _parallel_marking_threads = calc_parallel_marking_threads();
1138 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1139 "Maximum number of marking threads exceeded");
1140
1141 uint active_workers = MAX2(1U, parallel_marking_threads());
1142 assert(active_workers > 0, "Should have been set");
1143
1144 // Parallel task terminator is set in "set_concurrency_and_phase()"
1145 set_concurrency_and_phase(active_workers, true /* concurrent */);
1146
1147 CMConcurrentMarkingTask markingTask(this, cmThread());
1148 _parallel_workers->set_active_workers(active_workers);
1149 _parallel_workers->run_task(&markingTask);
1150 print_stats();
1151 }
1152
1153 // Helper class to get rid of some boilerplate code.
1154 class G1CMTraceTime : public StackObj {
1155 GCTraceTimeImpl _gc_trace_time;
1156 static bool doit_and_prepend(bool doit) {
1157 if (doit) {
1158 gclog_or_tty->put(' ');
1159 }
1160 return doit;
1161 }
1162
1163 public:
1164 G1CMTraceTime(const char* title, bool doit)
1165 : _gc_trace_time(title, doit_and_prepend(doit), false, G1CollectedHeap::heap()->gc_timer_cm()) {
1166 }
1167 };
1168
1169 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1170 // world is stopped at this checkpoint
1171 assert(SafepointSynchronize::is_at_safepoint(),
1172 "world should be stopped");
1173
1174 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1175
1176 // If a full collection has happened, we shouldn't do this.
1177 if (has_aborted()) {
1178 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1179 return;
1180 }
1181
1182 SvcGCMarker sgcm(SvcGCMarker::OTHER);
1183
1184 if (VerifyDuringGC) {
1185 HandleMark hm; // handle scope
1186 g1h->prepare_for_verify();
1187 Universe::verify(VerifyOption_G1UsePrevMarking,
1188 " VerifyDuringGC:(before)");
1189 }
1190 g1h->check_bitmaps("Remark Start");
1191
1192 G1CollectorPolicy* g1p = g1h->g1_policy();
1193 g1p->record_concurrent_mark_remark_start();
1194
1195 double start = os::elapsedTime();
1196
1197 checkpointRootsFinalWork();
1198
1199 double mark_work_end = os::elapsedTime();
1200
1201 weakRefsWork(clear_all_soft_refs);
1202
1203 if (has_overflown()) {
1204 // Oops. We overflowed. Restart concurrent marking.
1205 _restart_for_overflow = true;
1206 if (G1TraceMarkStackOverflow) {
1207 gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
1208 }
1209
1210 // Verify the heap w.r.t. the previous marking bitmap.
1211 if (VerifyDuringGC) {
1212 HandleMark hm; // handle scope
1213 g1h->prepare_for_verify();
1214 Universe::verify(VerifyOption_G1UsePrevMarking,
1215 " VerifyDuringGC:(overflow)");
1216 }
1217
1218 // Clear the marking state because we will be restarting
1219 // marking due to overflowing the global mark stack.
1220 reset_marking_state();
1221 } else {
1222 {
1223 G1CMTraceTime trace("GC aggregate-data", G1Log::finer());
1224
1225 // Aggregate the per-task counting data that we have accumulated
1226 // while marking.
1227 aggregate_count_data();
1228 }
1229
1230 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1231 // We're done with marking.
1232 // This is the end of the marking cycle, we're expected all
1233 // threads to have SATB queues with active set to true.
1234 satb_mq_set.set_active_all_threads(false, /* new active value */
1235 true /* expected_active */);
1236
1237 if (VerifyDuringGC) {
1238 HandleMark hm; // handle scope
1239 g1h->prepare_for_verify();
1240 Universe::verify(VerifyOption_G1UseNextMarking,
1241 " VerifyDuringGC:(after)");
1242 }
1243 g1h->check_bitmaps("Remark End");
1244 assert(!restart_for_overflow(), "sanity");
1245 // Completely reset the marking state since marking completed
1246 set_non_marking_state();
1247 }
1248
1249 // Expand the marking stack, if we have to and if we can.
1250 if (_markStack.should_expand()) {
1251 _markStack.expand();
1252 }
1253
1254 // Statistics
1255 double now = os::elapsedTime();
1256 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1257 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1258 _remark_times.add((now - start) * 1000.0);
1259
1260 g1p->record_concurrent_mark_remark_end();
1261
1740
1741 };
1742
1743 void ConcurrentMark::cleanup() {
1744 // world is stopped at this checkpoint
1745 assert(SafepointSynchronize::is_at_safepoint(),
1746 "world should be stopped");
1747 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1748
1749 // If a full collection has happened, we shouldn't do this.
1750 if (has_aborted()) {
1751 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1752 return;
1753 }
1754
1755 g1h->verify_region_sets_optional();
1756
1757 if (VerifyDuringGC) {
1758 HandleMark hm; // handle scope
1759 g1h->prepare_for_verify();
1760 Universe::verify(VerifyOption_G1UsePrevMarking,
1761 " VerifyDuringGC:(before)");
1762 }
1763 g1h->check_bitmaps("Cleanup Start");
1764
1765 G1CollectorPolicy* g1p = g1h->g1_policy();
1766 g1p->record_concurrent_mark_cleanup_start();
1767
1768 double start = os::elapsedTime();
1769
1770 HeapRegionRemSet::reset_for_cleanup_tasks();
1771
1772 // Do counting once more with the world stopped for good measure.
1773 G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
1774
1775 g1h->workers()->run_task(&g1_par_count_task);
1776
1777 if (VerifyDuringGC) {
1778 // Verify that the counting data accumulated during marking matches
1779 // that calculated by walking the marking bitmap.
1780
1781 // Bitmaps to hold expected values
1783 BitMap expected_card_bm(_card_bm.size(), true);
1784
1785 G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
1786 &_region_bm,
1787 &_card_bm,
1788 &expected_region_bm,
1789 &expected_card_bm);
1790
1791 g1h->workers()->run_task(&g1_par_verify_task);
1792
1793 guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
1794 }
1795
1796 size_t start_used_bytes = g1h->used();
1797 g1h->collector_state()->set_mark_in_progress(false);
1798
1799 double count_end = os::elapsedTime();
1800 double this_final_counting_time = (count_end - start);
1801 _total_counting_time += this_final_counting_time;
1802
1803 if (G1PrintRegionLivenessInfo) {
1804 G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking");
1805 _g1h->heap_region_iterate(&cl);
1806 }
1807
1808 // Install newly created mark bitMap as "prev".
1809 swapMarkBitMaps();
1810
1811 g1h->reset_gc_time_stamp();
1812
1813 uint n_workers = _g1h->workers()->active_workers();
1814
1815 // Note end of marking in all heap regions.
1816 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1817 g1h->workers()->run_task(&g1_par_note_end_task);
1818 g1h->check_gc_time_stamps();
1819
1820 if (!cleanup_list_is_empty()) {
1821 // The cleanup list is not empty, so we'll have to process it
1822 // concurrently. Notify anyone else that might be wanting free
1823 // regions that there will be more free regions coming soon.
1824 g1h->set_free_regions_coming();
1827 // call below, since it affects the metric by which we sort the heap
1828 // regions.
1829 if (G1ScrubRemSets) {
1830 double rs_scrub_start = os::elapsedTime();
1831 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers);
1832 g1h->workers()->run_task(&g1_par_scrub_rs_task);
1833
1834 double rs_scrub_end = os::elapsedTime();
1835 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
1836 _total_rs_scrub_time += this_rs_scrub_time;
1837 }
1838
1839 // this will also free any regions totally full of garbage objects,
1840 // and sort the regions.
1841 g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1842
1843 // Statistics.
1844 double end = os::elapsedTime();
1845 _cleanup_times.add((end - start) * 1000.0);
1846
1847 if (G1Log::fine()) {
1848 g1h->g1_policy()->print_heap_transition(start_used_bytes);
1849 }
1850
1851 // Clean up will have freed any regions completely full of garbage.
1852 // Update the soft reference policy with the new heap occupancy.
1853 Universe::update_heap_info_at_gc();
1854
1855 if (VerifyDuringGC) {
1856 HandleMark hm; // handle scope
1857 g1h->prepare_for_verify();
1858 Universe::verify(VerifyOption_G1UsePrevMarking,
1859 " VerifyDuringGC:(after)");
1860 }
1861
1862 g1h->check_bitmaps("Cleanup End");
1863
1864 g1h->verify_region_sets_optional();
1865
1866 // We need to make this be a "collection" so any collection pause that
1867 // races with it goes around and waits for completeCleanup to finish.
1868 g1h->increment_total_collections();
1869
1870 // Clean out dead classes and update Metaspace sizes.
1871 if (ClassUnloadingWithConcurrentMark) {
1872 ClassLoaderDataGraph::purge();
1873 }
1874 MetaspaceGC::compute_new_size();
1875
1876 // We reclaimed old regions so we should calculate the sizes to make
1877 // sure we update the old gen/space data.
1878 g1h->g1mm()->update_sizes();
1879 g1h->allocation_context_stats().update_after_mark();
1880
1881 g1h->trace_heap_after_concurrent_cycle();
1882 }
1883
1884 void ConcurrentMark::completeCleanup() {
1885 if (has_aborted()) return;
1886
1887 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1888
1889 _cleanup_list.verify_optional();
1890 FreeRegionList tmp_free_list("Tmp Free List");
1891
1892 if (G1ConcRegionFreeingVerbose) {
1893 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1894 "cleanup list has %u entries",
1895 _cleanup_list.length());
1896 }
1897
1898 // No one else should be accessing the _cleanup_list at this point,
1899 // so it is not necessary to take any locks
1900 while (!_cleanup_list.is_empty()) {
1901 HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */);
1902 assert(hr != NULL, "Got NULL from a non-empty list");
1903 hr->par_clear();
1904 tmp_free_list.add_ordered(hr);
1905
1906 // Instead of adding one region at a time to the secondary_free_list,
1907 // we accumulate them in the local list and move them a few at a
1908 // time. This also cuts down on the number of notify_all() calls
1909 // we do during this process. We'll also append the local list when
1910 // _cleanup_list is empty (which means we just removed the last
1911 // region from the _cleanup_list).
1912 if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1913 _cleanup_list.is_empty()) {
1914 if (G1ConcRegionFreeingVerbose) {
1915 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1916 "appending %u entries to the secondary_free_list, "
1917 "cleanup list still has %u entries",
1918 tmp_free_list.length(),
1919 _cleanup_list.length());
1920 }
1921
1922 {
1923 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1924 g1h->secondary_free_list_add(&tmp_free_list);
1925 SecondaryFreeList_lock->notify_all();
1926 }
1927 #ifndef PRODUCT
1928 if (G1StressConcRegionFreeing) {
1929 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1930 os::sleep(Thread::current(), (jlong) 1, false);
1931 }
1932 }
1933 #endif
1934 }
1935 }
1936 assert(tmp_free_list.is_empty(), "post-condition");
1937 }
1938
1939 // Supporting Object and Oop closures for reference discovery
1940 // and processing in during marking
2157 // Skip processing the discovered references if we have
2158 // overflown the global marking stack. Reference objects
2159 // only get discovered once so it is OK to not
2160 // de-populate the discovered reference lists. We could have,
2161 // but the only benefit would be that, when marking restarts,
2162 // less reference objects are discovered.
2163 return;
2164 }
2165
2166 ResourceMark rm;
2167 HandleMark hm;
2168
2169 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2170
2171 // Is alive closure.
2172 G1CMIsAliveClosure g1_is_alive(g1h);
2173
2174 // Inner scope to exclude the cleaning of the string and symbol
2175 // tables from the displayed time.
2176 {
2177 G1CMTraceTime t("GC ref-proc", G1Log::finer());
2178
2179 ReferenceProcessor* rp = g1h->ref_processor_cm();
2180
2181 // See the comment in G1CollectedHeap::ref_processing_init()
2182 // about how reference processing currently works in G1.
2183
2184 // Set the soft reference policy
2185 rp->setup_policy(clear_all_soft_refs);
2186 assert(_markStack.isEmpty(), "mark stack should be empty");
2187
2188 // Instances of the 'Keep Alive' and 'Complete GC' closures used
2189 // in serial reference processing. Note these closures are also
2190 // used for serially processing (by the the current thread) the
2191 // JNI references during parallel reference processing.
2192 //
2193 // These closures do not need to synchronize with the worker
2194 // threads involved in parallel reference processing as these
2195 // instances are executed serially by the current thread (e.g.
2196 // reference processing is not multi-threaded and is thus
2197 // performed by the current thread instead of a gang worker).
2247 set_has_overflown();
2248 }
2249
2250 assert(rp->num_q() == active_workers, "why not");
2251
2252 rp->enqueue_discovered_references(executor);
2253
2254 rp->verify_no_references_recorded();
2255 assert(!rp->discovery_enabled(), "Post condition");
2256 }
2257
2258 if (has_overflown()) {
2259 // We can not trust g1_is_alive if the marking stack overflowed
2260 return;
2261 }
2262
2263 assert(_markStack.isEmpty(), "Marking should have completed");
2264
2265 // Unload Klasses, String, Symbols, Code Cache, etc.
2266 {
2267 G1CMTraceTime trace("Unloading", G1Log::finer());
2268
2269 if (ClassUnloadingWithConcurrentMark) {
2270 bool purged_classes;
2271
2272 {
2273 G1CMTraceTime trace("System Dictionary Unloading", G1Log::finest());
2274 purged_classes = SystemDictionary::do_unloading(&g1_is_alive, false /* Defer klass cleaning */);
2275 }
2276
2277 {
2278 G1CMTraceTime trace("Parallel Unloading", G1Log::finest());
2279 weakRefsWorkParallelPart(&g1_is_alive, purged_classes);
2280 }
2281 }
2282
2283 if (G1StringDedup::is_enabled()) {
2284 G1CMTraceTime trace("String Deduplication Unlink", G1Log::finest());
2285 G1StringDedup::unlink(&g1_is_alive);
2286 }
2287 }
2288 }
2289
2290 void ConcurrentMark::swapMarkBitMaps() {
2291 CMBitMapRO* temp = _prevMarkBitMap;
2292 _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap;
2293 _nextMarkBitMap = (CMBitMap*) temp;
2294 }
2295
2296 // Closure for marking entries in SATB buffers.
2297 class CMSATBBufferClosure : public SATBBufferClosure {
2298 private:
2299 CMTask* _task;
2300 G1CollectedHeap* _g1h;
2301
2302 // This is very similar to CMTask::deal_with_reference, but with
2303 // more relaxed requirements for the argument, so this must be more
2304 // circumspect about treating the argument as an object.
2385 true /* do_termination */,
2386 false /* is_serial */);
2387 } while (task->has_aborted() && !_cm->has_overflown());
2388 // If we overflow, then we do not want to restart. We instead
2389 // want to abort remark and do concurrent marking again.
2390 task->record_end_time();
2391 }
2392 }
2393
2394 CMRemarkTask(ConcurrentMark* cm, uint active_workers) :
2395 AbstractGangTask("Par Remark"), _cm(cm) {
2396 _cm->terminator()->reset_for_reuse(active_workers);
2397 }
2398 };
2399
2400 void ConcurrentMark::checkpointRootsFinalWork() {
2401 ResourceMark rm;
2402 HandleMark hm;
2403 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2404
2405 G1CMTraceTime trace("Finalize Marking", G1Log::finer());
2406
2407 g1h->ensure_parsability(false);
2408
2409 // this is remark, so we'll use up all active threads
2410 uint active_workers = g1h->workers()->active_workers();
2411 set_concurrency_and_phase(active_workers, false /* concurrent */);
2412 // Leave _parallel_marking_threads at it's
2413 // value originally calculated in the ConcurrentMark
2414 // constructor and pass values of the active workers
2415 // through the gang in the task.
2416
2417 {
2418 StrongRootsScope srs(active_workers);
2419
2420 CMRemarkTask remarkTask(this, active_workers);
2421 // We will start all available threads, even if we decide that the
2422 // active_workers will be fewer. The extra ones will just bail out
2423 // immediately.
2424 g1h->workers()->run_task(&remarkTask);
2425 }
2702 // Clear the global region bitmap - it will be filled as part
2703 // of the final counting task.
2704 _region_bm.clear();
2705
2706 uint max_regions = _g1h->max_regions();
2707 assert(_max_worker_id > 0, "uninitialized");
2708
2709 for (uint i = 0; i < _max_worker_id; i += 1) {
2710 BitMap* task_card_bm = count_card_bitmap_for(i);
2711 size_t* marked_bytes_array = count_marked_bytes_array_for(i);
2712
2713 assert(task_card_bm->size() == _card_bm.size(), "size mismatch");
2714 assert(marked_bytes_array != NULL, "uninitialized");
2715
2716 memset(marked_bytes_array, 0, (size_t) max_regions * sizeof(size_t));
2717 task_card_bm->clear();
2718 }
2719 }
2720
2721 void ConcurrentMark::print_stats() {
2722 if (G1MarkingVerboseLevel > 0) {
2723 gclog_or_tty->print_cr("---------------------------------------------------------------------");
2724 for (size_t i = 0; i < _active_tasks; ++i) {
2725 _tasks[i]->print_stats();
2726 gclog_or_tty->print_cr("---------------------------------------------------------------------");
2727 }
2728 }
2729 }
2730
2731 // abandon current marking iteration due to a Full GC
2732 void ConcurrentMark::abort() {
2733 if (!cmThread()->during_cycle() || _has_aborted) {
2734 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
2735 return;
2736 }
2737
2738 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
2739 // concurrent bitmap clearing.
2740 _nextMarkBitMap->clearAll();
2741
2742 // Note we cannot clear the previous marking bitmap here
2743 // since VerifyDuringGC verifies the objects marked during
2744 // a full GC against the previous bitmap.
2745
2746 // Clear the liveness counting data
2747 clear_all_count_data();
2751 _tasks[i]->clear_region_fields();
2752 }
2753 _first_overflow_barrier_sync.abort();
2754 _second_overflow_barrier_sync.abort();
2755 _has_aborted = true;
2756
2757 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2758 satb_mq_set.abandon_partial_marking();
2759 // This can be called either during or outside marking, we'll read
2760 // the expected_active value from the SATB queue set.
2761 satb_mq_set.set_active_all_threads(
2762 false, /* new active value */
2763 satb_mq_set.is_active() /* expected_active */);
2764
2765 _g1h->trace_heap_after_concurrent_cycle();
2766 _g1h->register_concurrent_cycle_end();
2767 }
2768
2769 static void print_ms_time_info(const char* prefix, const char* name,
2770 NumberSeq& ns) {
2771 gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2772 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2773 if (ns.num() > 0) {
2774 gclog_or_tty->print_cr("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2775 prefix, ns.sd(), ns.maximum());
2776 }
2777 }
2778
2779 void ConcurrentMark::print_summary_info() {
2780 gclog_or_tty->print_cr(" Concurrent marking:");
2781 print_ms_time_info(" ", "init marks", _init_times);
2782 print_ms_time_info(" ", "remarks", _remark_times);
2783 {
2784 print_ms_time_info(" ", "final marks", _remark_mark_times);
2785 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2786
2787 }
2788 print_ms_time_info(" ", "cleanups", _cleanup_times);
2789 gclog_or_tty->print_cr(" Final counting total time = %8.2f s (avg = %8.2f ms).",
2790 _total_counting_time,
2791 (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 /
2792 (double)_cleanup_times.num()
2793 : 0.0));
2794 if (G1ScrubRemSets) {
2795 gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
2796 _total_rs_scrub_time,
2797 (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 /
2798 (double)_cleanup_times.num()
2799 : 0.0));
2800 }
2801 gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.",
2802 (_init_times.sum() + _remark_times.sum() +
2803 _cleanup_times.sum())/1000.0);
2804 gclog_or_tty->print_cr(" Total concurrent time = %8.2f s "
2805 "(%8.2f s marking).",
2806 cmThread()->vtime_accum(),
2807 cmThread()->vtime_mark_accum());
2808 }
2809
2810 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2811 _parallel_workers->print_worker_threads_on(st);
2812 }
2813
2814 void ConcurrentMark::print_on_error(outputStream* st) const {
2815 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2816 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2817 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2818 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
2819 }
2820
2821 // We take a break if someone is trying to stop the world.
2822 bool ConcurrentMark::do_yield_check(uint worker_id) {
2823 if (SuspendibleThreadSet::should_yield()) {
2824 if (worker_id == 0) {
2825 _g1h->g1_policy()->record_concurrent_pause();
2826 }
2827 SuspendibleThreadSet::yield();
3167
3168 // This keeps claiming and applying the closure to completed buffers
3169 // until we run out of buffers or we need to abort.
3170 while (!has_aborted() &&
3171 satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
3172 regular_clock_call();
3173 }
3174
3175 _draining_satb_buffers = false;
3176
3177 assert(has_aborted() ||
3178 concurrent() ||
3179 satb_mq_set.completed_buffers_num() == 0, "invariant");
3180
3181 // again, this was a potentially expensive operation, decrease the
3182 // limits to get the regular clock call early
3183 decrease_limits();
3184 }
3185
3186 void CMTask::print_stats() {
3187 gclog_or_tty->print_cr("Marking Stats, task = %u, calls = %d",
3188 _worker_id, _calls);
3189 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3190 _elapsed_time_ms, _termination_time_ms);
3191 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3192 _step_times_ms.num(), _step_times_ms.avg(),
3193 _step_times_ms.sd());
3194 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
3195 _step_times_ms.maximum(), _step_times_ms.sum());
3196 }
3197
3198 bool ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, oop& obj) {
3199 return _task_queues->steal(worker_id, hash_seed, obj);
3200 }
3201
3202 /*****************************************************************************
3203
3204 The do_marking_step(time_target_ms, ...) method is the building
3205 block of the parallel marking framework. It can be called in parallel
3206 with other invocations of do_marking_step() on different tasks
3207 (but only one per task, obviously) and concurrently with the
3208 mutator threads, or during remark, hence it eliminates the need
3209 for two versions of the code. When called during remark, it will
3210 pick up from where the task left off during the concurrent marking
3211 phase. Interestingly, tasks are also claimable during evacuation
3212 pauses too, since do_marking_step() ensures that it aborts before
3213 it needs to yield.
3214
3684 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
3685 #else // _LP64
3686 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s"
3687 #endif // _LP64
3688
3689 // For per-region info
3690 #define G1PPRL_TYPE_FORMAT " %-4s"
3691 #define G1PPRL_TYPE_H_FORMAT " %4s"
3692 #define G1PPRL_BYTE_FORMAT " " SIZE_FORMAT_W(9)
3693 #define G1PPRL_BYTE_H_FORMAT " %9s"
3694 #define G1PPRL_DOUBLE_FORMAT " %14.1f"
3695 #define G1PPRL_DOUBLE_H_FORMAT " %14s"
3696
3697 // For summary info
3698 #define G1PPRL_SUM_ADDR_FORMAT(tag) " " tag ":" G1PPRL_ADDR_BASE_FORMAT
3699 #define G1PPRL_SUM_BYTE_FORMAT(tag) " " tag ": " SIZE_FORMAT
3700 #define G1PPRL_SUM_MB_FORMAT(tag) " " tag ": %1.2f MB"
3701 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
3702
3703 G1PrintRegionLivenessInfoClosure::
3704 G1PrintRegionLivenessInfoClosure(outputStream* out, const char* phase_name)
3705 : _out(out),
3706 _total_used_bytes(0), _total_capacity_bytes(0),
3707 _total_prev_live_bytes(0), _total_next_live_bytes(0),
3708 _hum_used_bytes(0), _hum_capacity_bytes(0),
3709 _hum_prev_live_bytes(0), _hum_next_live_bytes(0),
3710 _total_remset_bytes(0), _total_strong_code_roots_bytes(0) {
3711 G1CollectedHeap* g1h = G1CollectedHeap::heap();
3712 MemRegion g1_reserved = g1h->g1_reserved();
3713 double now = os::elapsedTime();
3714
3715 // Print the header of the output.
3716 _out->cr();
3717 _out->print_cr(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
3718 _out->print_cr(G1PPRL_LINE_PREFIX" HEAP"
3719 G1PPRL_SUM_ADDR_FORMAT("reserved")
3720 G1PPRL_SUM_BYTE_FORMAT("region-size"),
3721 p2i(g1_reserved.start()), p2i(g1_reserved.end()),
3722 HeapRegion::GrainBytes);
3723 _out->print_cr(G1PPRL_LINE_PREFIX);
3724 _out->print_cr(G1PPRL_LINE_PREFIX
3725 G1PPRL_TYPE_H_FORMAT
3726 G1PPRL_ADDR_BASE_H_FORMAT
3727 G1PPRL_BYTE_H_FORMAT
3728 G1PPRL_BYTE_H_FORMAT
3729 G1PPRL_BYTE_H_FORMAT
3730 G1PPRL_DOUBLE_H_FORMAT
3731 G1PPRL_BYTE_H_FORMAT
3732 G1PPRL_BYTE_H_FORMAT,
3733 "type", "address-range",
3734 "used", "prev-live", "next-live", "gc-eff",
3735 "remset", "code-roots");
3736 _out->print_cr(G1PPRL_LINE_PREFIX
3737 G1PPRL_TYPE_H_FORMAT
3738 G1PPRL_ADDR_BASE_H_FORMAT
3739 G1PPRL_BYTE_H_FORMAT
3740 G1PPRL_BYTE_H_FORMAT
3741 G1PPRL_BYTE_H_FORMAT
3742 G1PPRL_DOUBLE_H_FORMAT
3743 G1PPRL_BYTE_H_FORMAT
3744 G1PPRL_BYTE_H_FORMAT,
3745 "", "",
3746 "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)",
3747 "(bytes)", "(bytes)");
3748 }
3749
3750 // It takes as a parameter a reference to one of the _hum_* fields, it
3751 // deduces the corresponding value for a region in a humongous region
3752 // series (either the region size, or what's left if the _hum_* field
3753 // is < the region size), and updates the _hum_* field accordingly.
3754 size_t G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* hum_bytes) {
3755 size_t bytes = 0;
3756 // The > 0 check is to deal with the prev and next live bytes which
3798 _hum_used_bytes = used_bytes;
3799 _hum_prev_live_bytes = prev_live_bytes;
3800 _hum_next_live_bytes = next_live_bytes;
3801 get_hum_bytes(&used_bytes, &capacity_bytes,
3802 &prev_live_bytes, &next_live_bytes);
3803 end = bottom + HeapRegion::GrainWords;
3804 } else if (r->is_continues_humongous()) {
3805 get_hum_bytes(&used_bytes, &capacity_bytes,
3806 &prev_live_bytes, &next_live_bytes);
3807 assert(end == bottom + HeapRegion::GrainWords, "invariant");
3808 }
3809
3810 _total_used_bytes += used_bytes;
3811 _total_capacity_bytes += capacity_bytes;
3812 _total_prev_live_bytes += prev_live_bytes;
3813 _total_next_live_bytes += next_live_bytes;
3814 _total_remset_bytes += remset_bytes;
3815 _total_strong_code_roots_bytes += strong_code_roots_bytes;
3816
3817 // Print a line for this particular region.
3818 _out->print_cr(G1PPRL_LINE_PREFIX
3819 G1PPRL_TYPE_FORMAT
3820 G1PPRL_ADDR_BASE_FORMAT
3821 G1PPRL_BYTE_FORMAT
3822 G1PPRL_BYTE_FORMAT
3823 G1PPRL_BYTE_FORMAT
3824 G1PPRL_DOUBLE_FORMAT
3825 G1PPRL_BYTE_FORMAT
3826 G1PPRL_BYTE_FORMAT,
3827 type, p2i(bottom), p2i(end),
3828 used_bytes, prev_live_bytes, next_live_bytes, gc_eff,
3829 remset_bytes, strong_code_roots_bytes);
3830
3831 return false;
3832 }
3833
3834 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
3835 // add static memory usages to remembered set sizes
3836 _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
3837 // Print the footer of the output.
3838 _out->print_cr(G1PPRL_LINE_PREFIX);
3839 _out->print_cr(G1PPRL_LINE_PREFIX
3840 " SUMMARY"
3841 G1PPRL_SUM_MB_FORMAT("capacity")
3842 G1PPRL_SUM_MB_PERC_FORMAT("used")
3843 G1PPRL_SUM_MB_PERC_FORMAT("prev-live")
3844 G1PPRL_SUM_MB_PERC_FORMAT("next-live")
3845 G1PPRL_SUM_MB_FORMAT("remset")
3846 G1PPRL_SUM_MB_FORMAT("code-roots"),
3847 bytes_to_mb(_total_capacity_bytes),
3848 bytes_to_mb(_total_used_bytes),
3849 perc(_total_used_bytes, _total_capacity_bytes),
3850 bytes_to_mb(_total_prev_live_bytes),
3851 perc(_total_prev_live_bytes, _total_capacity_bytes),
3852 bytes_to_mb(_total_next_live_bytes),
3853 perc(_total_next_live_bytes, _total_capacity_bytes),
3854 bytes_to_mb(_total_remset_bytes),
3855 bytes_to_mb(_total_strong_code_roots_bytes));
3856 _out->cr();
3857 }
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/metadataOnStackMark.hpp"
27 #include "classfile/symbolTable.hpp"
28 #include "code/codeCache.hpp"
29 #include "gc/g1/concurrentMark.inline.hpp"
30 #include "gc/g1/concurrentMarkThread.inline.hpp"
31 #include "gc/g1/g1CollectedHeap.inline.hpp"
32 #include "gc/g1/g1CollectorPolicy.hpp"
33 #include "gc/g1/g1CollectorState.hpp"
34 #include "gc/g1/g1OopClosures.inline.hpp"
35 #include "gc/g1/g1RemSet.hpp"
36 #include "gc/g1/g1StringDedup.hpp"
37 #include "gc/g1/heapRegion.inline.hpp"
38 #include "gc/g1/heapRegionManager.inline.hpp"
39 #include "gc/g1/heapRegionRemSet.hpp"
40 #include "gc/g1/heapRegionSet.inline.hpp"
41 #include "gc/g1/suspendibleThreadSet.hpp"
42 #include "gc/shared/gcId.hpp"
43 #include "gc/shared/gcTimer.hpp"
44 #include "gc/shared/gcTrace.hpp"
45 #include "gc/shared/gcTraceTime.hpp"
46 #include "gc/shared/genOopClosures.inline.hpp"
47 #include "gc/shared/referencePolicy.hpp"
48 #include "gc/shared/strongRootsScope.hpp"
49 #include "gc/shared/taskqueue.inline.hpp"
50 #include "gc/shared/vmGCOperations.hpp"
51 #include "logging/log.hpp"
52 #include "memory/allocation.hpp"
53 #include "memory/resourceArea.hpp"
54 #include "oops/oop.inline.hpp"
55 #include "runtime/atomic.inline.hpp"
56 #include "runtime/handles.inline.hpp"
57 #include "runtime/java.hpp"
58 #include "runtime/prefetch.inline.hpp"
59 #include "services/memTracker.hpp"
60
61 // Concurrent marking bit map wrapper
62
63 CMBitMapRO::CMBitMapRO(int shifter) :
64 _bm(),
65 _shifter(shifter) {
66 _bmStartWord = 0;
67 _bmWordSize = 0;
68 }
69
70 HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
71 const HeapWord* limit) const {
261 rs.release();
262 return false;
263 }
264 assert(_virtual_space.committed_size() == rs.size(),
265 "Didn't reserve backing store for all of ConcurrentMark stack?");
266 _base = (oop*) _virtual_space.low();
267 setEmpty();
268 _capacity = (jint) capacity;
269 _saved_index = -1;
270 _should_expand = false;
271 return true;
272 }
273
274 void CMMarkStack::expand() {
275 // Called, during remark, if we've overflown the marking stack during marking.
276 assert(isEmpty(), "stack should been emptied while handling overflow");
277 assert(_capacity <= (jint) MarkStackSizeMax, "stack bigger than permitted");
278 // Clear expansion flag
279 _should_expand = false;
280 if (_capacity == (jint) MarkStackSizeMax) {
281 log_trace(gc)("(benign) Can't expand marking stack capacity, at max size limit");
282 return;
283 }
284 // Double capacity if possible
285 jint new_capacity = MIN2(_capacity*2, (jint) MarkStackSizeMax);
286 // Do not give up existing stack until we have managed to
287 // get the double capacity that we desired.
288 ReservedSpace rs(ReservedSpace::allocation_align_size_up(new_capacity *
289 sizeof(oop)));
290 if (rs.is_reserved()) {
291 // Release the backing store associated with old stack
292 _virtual_space.release();
293 // Reinitialize virtual space for new stack
294 if (!_virtual_space.initialize(rs, rs.size())) {
295 fatal("Not enough swap for expanded marking stack capacity");
296 }
297 _base = (oop*)(_virtual_space.low());
298 _index = 0;
299 _capacity = new_capacity;
300 } else {
301 // Failed to double capacity, continue;
302 log_trace(gc)("(benign) Failed to expand marking stack capacity from " SIZE_FORMAT "K to " SIZE_FORMAT "K",
303 _capacity / K, new_capacity / K);
304 }
305 }
306
307 void CMMarkStack::set_should_expand() {
308 // If we're resetting the marking state because of an
309 // marking stack overflow, record that we should, if
310 // possible, expand the stack.
311 _should_expand = _cm->has_overflown();
312 }
313
314 CMMarkStack::~CMMarkStack() {
315 if (_base != NULL) {
316 _base = NULL;
317 _virtual_space.release();
318 }
319 }
320
321 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
322 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
323 jint start = _index;
324 jint next_index = start + n;
553 // Calculate the number of parallel marking threads by scaling
554 // the number of parallel GC threads.
555 uint marking_thread_num = scale_parallel_threads(ParallelGCThreads);
556 FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num);
557 _sleep_factor = 0.0;
558 _marking_task_overhead = 1.0;
559 }
560
561 assert(ConcGCThreads > 0, "Should have been set");
562 _parallel_marking_threads = ConcGCThreads;
563 _max_parallel_marking_threads = _parallel_marking_threads;
564
565 if (parallel_marking_threads() > 1) {
566 _cleanup_task_overhead = 1.0;
567 } else {
568 _cleanup_task_overhead = marking_task_overhead();
569 }
570 _cleanup_sleep_factor =
571 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
572
573 _parallel_workers = new WorkGang("G1 Marker",
574 _max_parallel_marking_threads, false, true);
575 if (_parallel_workers == NULL) {
576 vm_exit_during_initialization("Failed necessary allocation.");
577 } else {
578 _parallel_workers->initialize_workers();
579 }
580
581 if (FLAG_IS_DEFAULT(MarkStackSize)) {
582 size_t mark_stack_size =
583 MIN2(MarkStackSizeMax,
584 MAX2(MarkStackSize, (size_t) (parallel_marking_threads() * TASKQUEUE_SIZE)));
585 // Verify that the calculated value for MarkStackSize is in range.
586 // It would be nice to use the private utility routine from Arguments.
587 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
588 warning("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): "
589 "must be between 1 and " SIZE_FORMAT,
590 mark_stack_size, MarkStackSizeMax);
591 return;
592 }
891 return;
892 }
893
894 // If we're executing the concurrent phase of marking, reset the marking
895 // state; otherwise the marking state is reset after reference processing,
896 // during the remark pause.
897 // If we reset here as a result of an overflow during the remark we will
898 // see assertion failures from any subsequent set_concurrency_and_phase()
899 // calls.
900 if (concurrent()) {
901 // let the task associated with with worker 0 do this
902 if (worker_id == 0) {
903 // task 0 is responsible for clearing the global data structures
904 // We should be here because of an overflow. During STW we should
905 // not clear the overflow flag since we rely on it being true when
906 // we exit this method to abort the pause and restart concurrent
907 // marking.
908 reset_marking_state(true /* clear_overflow */);
909 force_overflow()->update();
910
911 log_info(gc)("Concurrent Mark reset for overflow");
912 }
913 }
914
915 // after this, each task should reset its own data structures then
916 // then go into the second barrier
917 }
918
919 void ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
920 SuspendibleThreadSetLeaver sts_leave(concurrent());
921 _second_overflow_barrier_sync.enter();
922
923 // at this point everything should be re-initialized and ready to go
924 }
925
926 #ifndef PRODUCT
927 void ForceOverflowSettings::init() {
928 _num_remaining = G1ConcMarkForceOverflow;
929 _force = false;
930 update();
931 }
1053 ConcurrentMark* _cm;
1054
1055 public:
1056 CMRootRegionScanTask(ConcurrentMark* cm) :
1057 AbstractGangTask("Root Region Scan"), _cm(cm) { }
1058
1059 void work(uint worker_id) {
1060 assert(Thread::current()->is_ConcurrentGC_thread(),
1061 "this should only be done by a conc GC thread");
1062
1063 CMRootRegions* root_regions = _cm->root_regions();
1064 HeapRegion* hr = root_regions->claim_next();
1065 while (hr != NULL) {
1066 _cm->scanRootRegion(hr, worker_id);
1067 hr = root_regions->claim_next();
1068 }
1069 }
1070 };
1071
1072 void ConcurrentMark::scanRootRegions() {
1073 jlong scan_start = os::elapsed_counter();
1074
1075 // Start of concurrent marking.
1076 ClassLoaderDataGraph::clear_claimed_marks();
1077
1078 // scan_in_progress() will have been set to true only if there was
1079 // at least one root region to scan. So, if it's false, we
1080 // should not attempt to do any further work.
1081 if (root_regions()->scan_in_progress()) {
1082 log_info(gc)("Concurrent Root Region Scan (%.3fs)", TimeHelper::counter_to_seconds(scan_start));
1083
1084 _parallel_marking_threads = calc_parallel_marking_threads();
1085 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1086 "Maximum number of marking threads exceeded");
1087 uint active_workers = MAX2(1U, parallel_marking_threads());
1088
1089 CMRootRegionScanTask task(this);
1090 _parallel_workers->set_active_workers(active_workers);
1091 _parallel_workers->run_task(&task);
1092
1093 double end_time_ms = os::elapsed_counter();
1094 log_info(gc)("Concurrent Root Region Scan (%.3fs, %.3fs) %.3fms",
1095 TimeHelper::counter_to_seconds(scan_start),
1096 TimeHelper::counter_to_seconds(end_time_ms),
1097 TimeHelper::counter_to_millis(end_time_ms - scan_start));
1098
1099 // It's possible that has_aborted() is true here without actually
1100 // aborting the survivor scan earlier. This is OK as it's
1101 // mainly used for sanity checking.
1102 root_regions()->scan_finished();
1103 }
1104 }
1105
1106 void ConcurrentMark::markFromRoots() {
1107 // we might be tempted to assert that:
1108 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1109 // "inconsistent argument?");
1110 // However that wouldn't be right, because it's possible that
1111 // a safepoint is indeed in progress as a younger generation
1112 // stop-the-world GC happens even as we mark in this generation.
1113
1114 _restart_for_overflow = false;
1115 force_overflow_conc()->init();
1116
1117 // _g1h has _n_par_threads
1118 _parallel_marking_threads = calc_parallel_marking_threads();
1119 assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1120 "Maximum number of marking threads exceeded");
1121
1122 uint active_workers = MAX2(1U, parallel_marking_threads());
1123 assert(active_workers > 0, "Should have been set");
1124
1125 // Parallel task terminator is set in "set_concurrency_and_phase()"
1126 set_concurrency_and_phase(active_workers, true /* concurrent */);
1127
1128 CMConcurrentMarkingTask markingTask(this, cmThread());
1129 _parallel_workers->set_active_workers(active_workers);
1130 _parallel_workers->run_task(&markingTask);
1131 print_stats();
1132 }
1133
1134 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1135 // world is stopped at this checkpoint
1136 assert(SafepointSynchronize::is_at_safepoint(),
1137 "world should be stopped");
1138
1139 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1140
1141 // If a full collection has happened, we shouldn't do this.
1142 if (has_aborted()) {
1143 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1144 return;
1145 }
1146
1147 SvcGCMarker sgcm(SvcGCMarker::OTHER);
1148
1149 if (VerifyDuringGC) {
1150 HandleMark hm; // handle scope
1151 g1h->prepare_for_verify();
1152 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)");
1153 }
1154 g1h->check_bitmaps("Remark Start");
1155
1156 G1CollectorPolicy* g1p = g1h->g1_policy();
1157 g1p->record_concurrent_mark_remark_start();
1158
1159 double start = os::elapsedTime();
1160
1161 checkpointRootsFinalWork();
1162
1163 double mark_work_end = os::elapsedTime();
1164
1165 weakRefsWork(clear_all_soft_refs);
1166
1167 if (has_overflown()) {
1168 // Oops. We overflowed. Restart concurrent marking.
1169 _restart_for_overflow = true;
1170 log_develop(gc)("Remark led to restart for overflow.");
1171
1172 // Verify the heap w.r.t. the previous marking bitmap.
1173 if (VerifyDuringGC) {
1174 HandleMark hm; // handle scope
1175 g1h->prepare_for_verify();
1176 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (overflow)");
1177 }
1178
1179 // Clear the marking state because we will be restarting
1180 // marking due to overflowing the global mark stack.
1181 reset_marking_state();
1182 } else {
1183 {
1184 GCTraceTime(Debug, gc) trace("GC aggregate-data", g1h->gc_timer_cm());
1185
1186 // Aggregate the per-task counting data that we have accumulated
1187 // while marking.
1188 aggregate_count_data();
1189 }
1190
1191 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1192 // We're done with marking.
1193 // This is the end of the marking cycle, we're expected all
1194 // threads to have SATB queues with active set to true.
1195 satb_mq_set.set_active_all_threads(false, /* new active value */
1196 true /* expected_active */);
1197
1198 if (VerifyDuringGC) {
1199 HandleMark hm; // handle scope
1200 g1h->prepare_for_verify();
1201 Universe::verify(VerifyOption_G1UseNextMarking, "During GC (after)");
1202 }
1203 g1h->check_bitmaps("Remark End");
1204 assert(!restart_for_overflow(), "sanity");
1205 // Completely reset the marking state since marking completed
1206 set_non_marking_state();
1207 }
1208
1209 // Expand the marking stack, if we have to and if we can.
1210 if (_markStack.should_expand()) {
1211 _markStack.expand();
1212 }
1213
1214 // Statistics
1215 double now = os::elapsedTime();
1216 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1217 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1218 _remark_times.add((now - start) * 1000.0);
1219
1220 g1p->record_concurrent_mark_remark_end();
1221
1700
1701 };
1702
1703 void ConcurrentMark::cleanup() {
1704 // world is stopped at this checkpoint
1705 assert(SafepointSynchronize::is_at_safepoint(),
1706 "world should be stopped");
1707 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1708
1709 // If a full collection has happened, we shouldn't do this.
1710 if (has_aborted()) {
1711 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1712 return;
1713 }
1714
1715 g1h->verify_region_sets_optional();
1716
1717 if (VerifyDuringGC) {
1718 HandleMark hm; // handle scope
1719 g1h->prepare_for_verify();
1720 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)");
1721 }
1722 g1h->check_bitmaps("Cleanup Start");
1723
1724 G1CollectorPolicy* g1p = g1h->g1_policy();
1725 g1p->record_concurrent_mark_cleanup_start();
1726
1727 double start = os::elapsedTime();
1728
1729 HeapRegionRemSet::reset_for_cleanup_tasks();
1730
1731 // Do counting once more with the world stopped for good measure.
1732 G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
1733
1734 g1h->workers()->run_task(&g1_par_count_task);
1735
1736 if (VerifyDuringGC) {
1737 // Verify that the counting data accumulated during marking matches
1738 // that calculated by walking the marking bitmap.
1739
1740 // Bitmaps to hold expected values
1742 BitMap expected_card_bm(_card_bm.size(), true);
1743
1744 G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
1745 &_region_bm,
1746 &_card_bm,
1747 &expected_region_bm,
1748 &expected_card_bm);
1749
1750 g1h->workers()->run_task(&g1_par_verify_task);
1751
1752 guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
1753 }
1754
1755 size_t start_used_bytes = g1h->used();
1756 g1h->collector_state()->set_mark_in_progress(false);
1757
1758 double count_end = os::elapsedTime();
1759 double this_final_counting_time = (count_end - start);
1760 _total_counting_time += this_final_counting_time;
1761
1762 if (Log<LOG_TAGS(gc, liveness)>::is_trace()) {
1763 G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1764 _g1h->heap_region_iterate(&cl);
1765 }
1766
1767 // Install newly created mark bitMap as "prev".
1768 swapMarkBitMaps();
1769
1770 g1h->reset_gc_time_stamp();
1771
1772 uint n_workers = _g1h->workers()->active_workers();
1773
1774 // Note end of marking in all heap regions.
1775 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1776 g1h->workers()->run_task(&g1_par_note_end_task);
1777 g1h->check_gc_time_stamps();
1778
1779 if (!cleanup_list_is_empty()) {
1780 // The cleanup list is not empty, so we'll have to process it
1781 // concurrently. Notify anyone else that might be wanting free
1782 // regions that there will be more free regions coming soon.
1783 g1h->set_free_regions_coming();
1786 // call below, since it affects the metric by which we sort the heap
1787 // regions.
1788 if (G1ScrubRemSets) {
1789 double rs_scrub_start = os::elapsedTime();
1790 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers);
1791 g1h->workers()->run_task(&g1_par_scrub_rs_task);
1792
1793 double rs_scrub_end = os::elapsedTime();
1794 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
1795 _total_rs_scrub_time += this_rs_scrub_time;
1796 }
1797
1798 // this will also free any regions totally full of garbage objects,
1799 // and sort the regions.
1800 g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1801
1802 // Statistics.
1803 double end = os::elapsedTime();
1804 _cleanup_times.add((end - start) * 1000.0);
1805
1806 // Clean up will have freed any regions completely full of garbage.
1807 // Update the soft reference policy with the new heap occupancy.
1808 Universe::update_heap_info_at_gc();
1809
1810 if (VerifyDuringGC) {
1811 HandleMark hm; // handle scope
1812 g1h->prepare_for_verify();
1813 Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (after)");
1814 }
1815
1816 g1h->check_bitmaps("Cleanup End");
1817
1818 g1h->verify_region_sets_optional();
1819
1820 // We need to make this be a "collection" so any collection pause that
1821 // races with it goes around and waits for completeCleanup to finish.
1822 g1h->increment_total_collections();
1823
1824 // Clean out dead classes and update Metaspace sizes.
1825 if (ClassUnloadingWithConcurrentMark) {
1826 ClassLoaderDataGraph::purge();
1827 }
1828 MetaspaceGC::compute_new_size();
1829
1830 // We reclaimed old regions so we should calculate the sizes to make
1831 // sure we update the old gen/space data.
1832 g1h->g1mm()->update_sizes();
1833 g1h->allocation_context_stats().update_after_mark();
1834
1835 g1h->trace_heap_after_concurrent_cycle();
1836 }
1837
1838 void ConcurrentMark::completeCleanup() {
1839 if (has_aborted()) return;
1840
1841 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1842
1843 _cleanup_list.verify_optional();
1844 FreeRegionList tmp_free_list("Tmp Free List");
1845
1846 log_develop(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "
1847 "cleanup list has %u entries",
1848 _cleanup_list.length());
1849
1850 // No one else should be accessing the _cleanup_list at this point,
1851 // so it is not necessary to take any locks
1852 while (!_cleanup_list.is_empty()) {
1853 HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */);
1854 assert(hr != NULL, "Got NULL from a non-empty list");
1855 hr->par_clear();
1856 tmp_free_list.add_ordered(hr);
1857
1858 // Instead of adding one region at a time to the secondary_free_list,
1859 // we accumulate them in the local list and move them a few at a
1860 // time. This also cuts down on the number of notify_all() calls
1861 // we do during this process. We'll also append the local list when
1862 // _cleanup_list is empty (which means we just removed the last
1863 // region from the _cleanup_list).
1864 if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1865 _cleanup_list.is_empty()) {
1866 log_develop(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "
1867 "appending %u entries to the secondary_free_list, "
1868 "cleanup list still has %u entries",
1869 tmp_free_list.length(),
1870 _cleanup_list.length());
1871
1872 {
1873 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1874 g1h->secondary_free_list_add(&tmp_free_list);
1875 SecondaryFreeList_lock->notify_all();
1876 }
1877 #ifndef PRODUCT
1878 if (G1StressConcRegionFreeing) {
1879 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1880 os::sleep(Thread::current(), (jlong) 1, false);
1881 }
1882 }
1883 #endif
1884 }
1885 }
1886 assert(tmp_free_list.is_empty(), "post-condition");
1887 }
1888
1889 // Supporting Object and Oop closures for reference discovery
1890 // and processing in during marking
2107 // Skip processing the discovered references if we have
2108 // overflown the global marking stack. Reference objects
2109 // only get discovered once so it is OK to not
2110 // de-populate the discovered reference lists. We could have,
2111 // but the only benefit would be that, when marking restarts,
2112 // less reference objects are discovered.
2113 return;
2114 }
2115
2116 ResourceMark rm;
2117 HandleMark hm;
2118
2119 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2120
2121 // Is alive closure.
2122 G1CMIsAliveClosure g1_is_alive(g1h);
2123
2124 // Inner scope to exclude the cleaning of the string and symbol
2125 // tables from the displayed time.
2126 {
2127 GCTraceTime(Debug, gc) trace("GC ref-proc", g1h->gc_timer_cm());
2128
2129 ReferenceProcessor* rp = g1h->ref_processor_cm();
2130
2131 // See the comment in G1CollectedHeap::ref_processing_init()
2132 // about how reference processing currently works in G1.
2133
2134 // Set the soft reference policy
2135 rp->setup_policy(clear_all_soft_refs);
2136 assert(_markStack.isEmpty(), "mark stack should be empty");
2137
2138 // Instances of the 'Keep Alive' and 'Complete GC' closures used
2139 // in serial reference processing. Note these closures are also
2140 // used for serially processing (by the the current thread) the
2141 // JNI references during parallel reference processing.
2142 //
2143 // These closures do not need to synchronize with the worker
2144 // threads involved in parallel reference processing as these
2145 // instances are executed serially by the current thread (e.g.
2146 // reference processing is not multi-threaded and is thus
2147 // performed by the current thread instead of a gang worker).
2197 set_has_overflown();
2198 }
2199
2200 assert(rp->num_q() == active_workers, "why not");
2201
2202 rp->enqueue_discovered_references(executor);
2203
2204 rp->verify_no_references_recorded();
2205 assert(!rp->discovery_enabled(), "Post condition");
2206 }
2207
2208 if (has_overflown()) {
2209 // We can not trust g1_is_alive if the marking stack overflowed
2210 return;
2211 }
2212
2213 assert(_markStack.isEmpty(), "Marking should have completed");
2214
2215 // Unload Klasses, String, Symbols, Code Cache, etc.
2216 {
2217 GCTraceTime(Debug, gc) trace("Unloading", g1h->gc_timer_cm());
2218
2219 if (ClassUnloadingWithConcurrentMark) {
2220 bool purged_classes;
2221
2222 {
2223 GCTraceTime(Trace, gc) trace("System Dictionary Unloading", g1h->gc_timer_cm());
2224 purged_classes = SystemDictionary::do_unloading(&g1_is_alive, false /* Defer klass cleaning */);
2225 }
2226
2227 {
2228 GCTraceTime(Trace, gc) trace("Parallel Unloading", g1h->gc_timer_cm());
2229 weakRefsWorkParallelPart(&g1_is_alive, purged_classes);
2230 }
2231 }
2232
2233 if (G1StringDedup::is_enabled()) {
2234 GCTraceTime(Trace, gc) trace("String Deduplication Unlink", g1h->gc_timer_cm());
2235 G1StringDedup::unlink(&g1_is_alive);
2236 }
2237 }
2238 }
2239
2240 void ConcurrentMark::swapMarkBitMaps() {
2241 CMBitMapRO* temp = _prevMarkBitMap;
2242 _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap;
2243 _nextMarkBitMap = (CMBitMap*) temp;
2244 }
2245
2246 // Closure for marking entries in SATB buffers.
2247 class CMSATBBufferClosure : public SATBBufferClosure {
2248 private:
2249 CMTask* _task;
2250 G1CollectedHeap* _g1h;
2251
2252 // This is very similar to CMTask::deal_with_reference, but with
2253 // more relaxed requirements for the argument, so this must be more
2254 // circumspect about treating the argument as an object.
2335 true /* do_termination */,
2336 false /* is_serial */);
2337 } while (task->has_aborted() && !_cm->has_overflown());
2338 // If we overflow, then we do not want to restart. We instead
2339 // want to abort remark and do concurrent marking again.
2340 task->record_end_time();
2341 }
2342 }
2343
2344 CMRemarkTask(ConcurrentMark* cm, uint active_workers) :
2345 AbstractGangTask("Par Remark"), _cm(cm) {
2346 _cm->terminator()->reset_for_reuse(active_workers);
2347 }
2348 };
2349
2350 void ConcurrentMark::checkpointRootsFinalWork() {
2351 ResourceMark rm;
2352 HandleMark hm;
2353 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2354
2355 GCTraceTime(Debug, gc) trace("Finalize Marking", g1h->gc_timer_cm());
2356
2357 g1h->ensure_parsability(false);
2358
2359 // this is remark, so we'll use up all active threads
2360 uint active_workers = g1h->workers()->active_workers();
2361 set_concurrency_and_phase(active_workers, false /* concurrent */);
2362 // Leave _parallel_marking_threads at it's
2363 // value originally calculated in the ConcurrentMark
2364 // constructor and pass values of the active workers
2365 // through the gang in the task.
2366
2367 {
2368 StrongRootsScope srs(active_workers);
2369
2370 CMRemarkTask remarkTask(this, active_workers);
2371 // We will start all available threads, even if we decide that the
2372 // active_workers will be fewer. The extra ones will just bail out
2373 // immediately.
2374 g1h->workers()->run_task(&remarkTask);
2375 }
2652 // Clear the global region bitmap - it will be filled as part
2653 // of the final counting task.
2654 _region_bm.clear();
2655
2656 uint max_regions = _g1h->max_regions();
2657 assert(_max_worker_id > 0, "uninitialized");
2658
2659 for (uint i = 0; i < _max_worker_id; i += 1) {
2660 BitMap* task_card_bm = count_card_bitmap_for(i);
2661 size_t* marked_bytes_array = count_marked_bytes_array_for(i);
2662
2663 assert(task_card_bm->size() == _card_bm.size(), "size mismatch");
2664 assert(marked_bytes_array != NULL, "uninitialized");
2665
2666 memset(marked_bytes_array, 0, (size_t) max_regions * sizeof(size_t));
2667 task_card_bm->clear();
2668 }
2669 }
2670
2671 void ConcurrentMark::print_stats() {
2672 LogHandle(gc, stats) log;
2673 if (!log.is_debug()) {
2674 return;
2675 }
2676 log.debug("---------------------------------------------------------------------");
2677 for (size_t i = 0; i < _active_tasks; ++i) {
2678 _tasks[i]->print_stats();
2679 log.debug("---------------------------------------------------------------------");
2680 }
2681 }
2682
2683 // abandon current marking iteration due to a Full GC
2684 void ConcurrentMark::abort() {
2685 if (!cmThread()->during_cycle() || _has_aborted) {
2686 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
2687 return;
2688 }
2689
2690 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
2691 // concurrent bitmap clearing.
2692 _nextMarkBitMap->clearAll();
2693
2694 // Note we cannot clear the previous marking bitmap here
2695 // since VerifyDuringGC verifies the objects marked during
2696 // a full GC against the previous bitmap.
2697
2698 // Clear the liveness counting data
2699 clear_all_count_data();
2703 _tasks[i]->clear_region_fields();
2704 }
2705 _first_overflow_barrier_sync.abort();
2706 _second_overflow_barrier_sync.abort();
2707 _has_aborted = true;
2708
2709 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2710 satb_mq_set.abandon_partial_marking();
2711 // This can be called either during or outside marking, we'll read
2712 // the expected_active value from the SATB queue set.
2713 satb_mq_set.set_active_all_threads(
2714 false, /* new active value */
2715 satb_mq_set.is_active() /* expected_active */);
2716
2717 _g1h->trace_heap_after_concurrent_cycle();
2718 _g1h->register_concurrent_cycle_end();
2719 }
2720
2721 static void print_ms_time_info(const char* prefix, const char* name,
2722 NumberSeq& ns) {
2723 log_trace(gc, marking, stats, exit)("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2724 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2725 if (ns.num() > 0) {
2726 log_trace(gc, marking, stats, exit)("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2727 prefix, ns.sd(), ns.maximum());
2728 }
2729 }
2730
2731 void ConcurrentMark::print_summary_info() {
2732 LogHandle(gc, marking, stats, exit) log;
2733 if (!log.is_trace()) {
2734 return;
2735 }
2736
2737 log.trace(" Concurrent marking:");
2738 print_ms_time_info(" ", "init marks", _init_times);
2739 print_ms_time_info(" ", "remarks", _remark_times);
2740 {
2741 print_ms_time_info(" ", "final marks", _remark_mark_times);
2742 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2743
2744 }
2745 print_ms_time_info(" ", "cleanups", _cleanup_times);
2746 log.trace(" Final counting total time = %8.2f s (avg = %8.2f ms).",
2747 _total_counting_time, (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2748 if (G1ScrubRemSets) {
2749 log.trace(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
2750 _total_rs_scrub_time, (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2751 }
2752 log.trace(" Total stop_world time = %8.2f s.",
2753 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2754 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2755 cmThread()->vtime_accum(), cmThread()->vtime_mark_accum());
2756 }
2757
2758 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2759 _parallel_workers->print_worker_threads_on(st);
2760 }
2761
2762 void ConcurrentMark::print_on_error(outputStream* st) const {
2763 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2764 p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2765 _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2766 _nextMarkBitMap->print_on_error(st, " Next Bits: ");
2767 }
2768
2769 // We take a break if someone is trying to stop the world.
2770 bool ConcurrentMark::do_yield_check(uint worker_id) {
2771 if (SuspendibleThreadSet::should_yield()) {
2772 if (worker_id == 0) {
2773 _g1h->g1_policy()->record_concurrent_pause();
2774 }
2775 SuspendibleThreadSet::yield();
3115
3116 // This keeps claiming and applying the closure to completed buffers
3117 // until we run out of buffers or we need to abort.
3118 while (!has_aborted() &&
3119 satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
3120 regular_clock_call();
3121 }
3122
3123 _draining_satb_buffers = false;
3124
3125 assert(has_aborted() ||
3126 concurrent() ||
3127 satb_mq_set.completed_buffers_num() == 0, "invariant");
3128
3129 // again, this was a potentially expensive operation, decrease the
3130 // limits to get the regular clock call early
3131 decrease_limits();
3132 }
3133
3134 void CMTask::print_stats() {
3135 log_debug(gc, stats)("Marking Stats, task = %u, calls = %d",
3136 _worker_id, _calls);
3137 log_debug(gc, stats)(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3138 _elapsed_time_ms, _termination_time_ms);
3139 log_debug(gc, stats)(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3140 _step_times_ms.num(), _step_times_ms.avg(),
3141 _step_times_ms.sd());
3142 log_debug(gc, stats)(" max = %1.2lfms, total = %1.2lfms",
3143 _step_times_ms.maximum(), _step_times_ms.sum());
3144 }
3145
3146 bool ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, oop& obj) {
3147 return _task_queues->steal(worker_id, hash_seed, obj);
3148 }
3149
3150 /*****************************************************************************
3151
3152 The do_marking_step(time_target_ms, ...) method is the building
3153 block of the parallel marking framework. It can be called in parallel
3154 with other invocations of do_marking_step() on different tasks
3155 (but only one per task, obviously) and concurrently with the
3156 mutator threads, or during remark, hence it eliminates the need
3157 for two versions of the code. When called during remark, it will
3158 pick up from where the task left off during the concurrent marking
3159 phase. Interestingly, tasks are also claimable during evacuation
3160 pauses too, since do_marking_step() ensures that it aborts before
3161 it needs to yield.
3162
3632 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
3633 #else // _LP64
3634 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s"
3635 #endif // _LP64
3636
3637 // For per-region info
3638 #define G1PPRL_TYPE_FORMAT " %-4s"
3639 #define G1PPRL_TYPE_H_FORMAT " %4s"
3640 #define G1PPRL_BYTE_FORMAT " " SIZE_FORMAT_W(9)
3641 #define G1PPRL_BYTE_H_FORMAT " %9s"
3642 #define G1PPRL_DOUBLE_FORMAT " %14.1f"
3643 #define G1PPRL_DOUBLE_H_FORMAT " %14s"
3644
3645 // For summary info
3646 #define G1PPRL_SUM_ADDR_FORMAT(tag) " " tag ":" G1PPRL_ADDR_BASE_FORMAT
3647 #define G1PPRL_SUM_BYTE_FORMAT(tag) " " tag ": " SIZE_FORMAT
3648 #define G1PPRL_SUM_MB_FORMAT(tag) " " tag ": %1.2f MB"
3649 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
3650
3651 G1PrintRegionLivenessInfoClosure::
3652 G1PrintRegionLivenessInfoClosure(const char* phase_name)
3653 : _total_used_bytes(0), _total_capacity_bytes(0),
3654 _total_prev_live_bytes(0), _total_next_live_bytes(0),
3655 _hum_used_bytes(0), _hum_capacity_bytes(0),
3656 _hum_prev_live_bytes(0), _hum_next_live_bytes(0),
3657 _total_remset_bytes(0), _total_strong_code_roots_bytes(0) {
3658 G1CollectedHeap* g1h = G1CollectedHeap::heap();
3659 MemRegion g1_reserved = g1h->g1_reserved();
3660 double now = os::elapsedTime();
3661
3662 // Print the header of the output.
3663 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
3664 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" HEAP"
3665 G1PPRL_SUM_ADDR_FORMAT("reserved")
3666 G1PPRL_SUM_BYTE_FORMAT("region-size"),
3667 p2i(g1_reserved.start()), p2i(g1_reserved.end()),
3668 HeapRegion::GrainBytes);
3669 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
3670 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3671 G1PPRL_TYPE_H_FORMAT
3672 G1PPRL_ADDR_BASE_H_FORMAT
3673 G1PPRL_BYTE_H_FORMAT
3674 G1PPRL_BYTE_H_FORMAT
3675 G1PPRL_BYTE_H_FORMAT
3676 G1PPRL_DOUBLE_H_FORMAT
3677 G1PPRL_BYTE_H_FORMAT
3678 G1PPRL_BYTE_H_FORMAT,
3679 "type", "address-range",
3680 "used", "prev-live", "next-live", "gc-eff",
3681 "remset", "code-roots");
3682 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3683 G1PPRL_TYPE_H_FORMAT
3684 G1PPRL_ADDR_BASE_H_FORMAT
3685 G1PPRL_BYTE_H_FORMAT
3686 G1PPRL_BYTE_H_FORMAT
3687 G1PPRL_BYTE_H_FORMAT
3688 G1PPRL_DOUBLE_H_FORMAT
3689 G1PPRL_BYTE_H_FORMAT
3690 G1PPRL_BYTE_H_FORMAT,
3691 "", "",
3692 "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)",
3693 "(bytes)", "(bytes)");
3694 }
3695
3696 // It takes as a parameter a reference to one of the _hum_* fields, it
3697 // deduces the corresponding value for a region in a humongous region
3698 // series (either the region size, or what's left if the _hum_* field
3699 // is < the region size), and updates the _hum_* field accordingly.
3700 size_t G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* hum_bytes) {
3701 size_t bytes = 0;
3702 // The > 0 check is to deal with the prev and next live bytes which
3744 _hum_used_bytes = used_bytes;
3745 _hum_prev_live_bytes = prev_live_bytes;
3746 _hum_next_live_bytes = next_live_bytes;
3747 get_hum_bytes(&used_bytes, &capacity_bytes,
3748 &prev_live_bytes, &next_live_bytes);
3749 end = bottom + HeapRegion::GrainWords;
3750 } else if (r->is_continues_humongous()) {
3751 get_hum_bytes(&used_bytes, &capacity_bytes,
3752 &prev_live_bytes, &next_live_bytes);
3753 assert(end == bottom + HeapRegion::GrainWords, "invariant");
3754 }
3755
3756 _total_used_bytes += used_bytes;
3757 _total_capacity_bytes += capacity_bytes;
3758 _total_prev_live_bytes += prev_live_bytes;
3759 _total_next_live_bytes += next_live_bytes;
3760 _total_remset_bytes += remset_bytes;
3761 _total_strong_code_roots_bytes += strong_code_roots_bytes;
3762
3763 // Print a line for this particular region.
3764 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3765 G1PPRL_TYPE_FORMAT
3766 G1PPRL_ADDR_BASE_FORMAT
3767 G1PPRL_BYTE_FORMAT
3768 G1PPRL_BYTE_FORMAT
3769 G1PPRL_BYTE_FORMAT
3770 G1PPRL_DOUBLE_FORMAT
3771 G1PPRL_BYTE_FORMAT
3772 G1PPRL_BYTE_FORMAT,
3773 type, p2i(bottom), p2i(end),
3774 used_bytes, prev_live_bytes, next_live_bytes, gc_eff,
3775 remset_bytes, strong_code_roots_bytes);
3776
3777 return false;
3778 }
3779
3780 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
3781 // add static memory usages to remembered set sizes
3782 _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
3783 // Print the footer of the output.
3784 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
3785 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3786 " SUMMARY"
3787 G1PPRL_SUM_MB_FORMAT("capacity")
3788 G1PPRL_SUM_MB_PERC_FORMAT("used")
3789 G1PPRL_SUM_MB_PERC_FORMAT("prev-live")
3790 G1PPRL_SUM_MB_PERC_FORMAT("next-live")
3791 G1PPRL_SUM_MB_FORMAT("remset")
3792 G1PPRL_SUM_MB_FORMAT("code-roots"),
3793 bytes_to_mb(_total_capacity_bytes),
3794 bytes_to_mb(_total_used_bytes),
3795 perc(_total_used_bytes, _total_capacity_bytes),
3796 bytes_to_mb(_total_prev_live_bytes),
3797 perc(_total_prev_live_bytes, _total_capacity_bytes),
3798 bytes_to_mb(_total_next_live_bytes),
3799 perc(_total_next_live_bytes, _total_capacity_bytes),
3800 bytes_to_mb(_total_remset_bytes),
3801 bytes_to_mb(_total_strong_code_roots_bytes));
3802 }
|