< prev index next >

src/share/vm/gc/g1/concurrentMark.cpp

Print this page




  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/metadataOnStackMark.hpp"
  27 #include "classfile/symbolTable.hpp"
  28 #include "code/codeCache.hpp"
  29 #include "gc/g1/concurrentMark.inline.hpp"
  30 #include "gc/g1/concurrentMarkThread.inline.hpp"
  31 #include "gc/g1/g1CollectedHeap.inline.hpp"
  32 #include "gc/g1/g1CollectorPolicy.hpp"
  33 #include "gc/g1/g1CollectorState.hpp"
  34 #include "gc/g1/g1ErgoVerbose.hpp"
  35 #include "gc/g1/g1Log.hpp"
  36 #include "gc/g1/g1OopClosures.inline.hpp"
  37 #include "gc/g1/g1RemSet.hpp"
  38 #include "gc/g1/g1StringDedup.hpp"
  39 #include "gc/g1/heapRegion.inline.hpp"
  40 #include "gc/g1/heapRegionManager.inline.hpp"
  41 #include "gc/g1/heapRegionRemSet.hpp"
  42 #include "gc/g1/heapRegionSet.inline.hpp"
  43 #include "gc/g1/suspendibleThreadSet.hpp"
  44 #include "gc/shared/gcId.hpp"
  45 #include "gc/shared/gcTimer.hpp"
  46 #include "gc/shared/gcTrace.hpp"
  47 #include "gc/shared/gcTraceTime.hpp"
  48 #include "gc/shared/genOopClosures.inline.hpp"
  49 #include "gc/shared/referencePolicy.hpp"
  50 #include "gc/shared/strongRootsScope.hpp"
  51 #include "gc/shared/taskqueue.inline.hpp"
  52 #include "gc/shared/vmGCOperations.hpp"

  53 #include "memory/allocation.hpp"
  54 #include "memory/resourceArea.hpp"
  55 #include "oops/oop.inline.hpp"
  56 #include "runtime/atomic.inline.hpp"
  57 #include "runtime/handles.inline.hpp"
  58 #include "runtime/java.hpp"
  59 #include "runtime/prefetch.inline.hpp"
  60 #include "services/memTracker.hpp"
  61 
  62 // Concurrent marking bit map wrapper
  63 
  64 CMBitMapRO::CMBitMapRO(int shifter) :
  65   _bm(),
  66   _shifter(shifter) {
  67   _bmStartWord = 0;
  68   _bmWordSize = 0;
  69 }
  70 
  71 HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
  72                                                const HeapWord* limit) const {


 262     rs.release();
 263     return false;
 264   }
 265   assert(_virtual_space.committed_size() == rs.size(),
 266          "Didn't reserve backing store for all of ConcurrentMark stack?");
 267   _base = (oop*) _virtual_space.low();
 268   setEmpty();
 269   _capacity = (jint) capacity;
 270   _saved_index = -1;
 271   _should_expand = false;
 272   return true;
 273 }
 274 
 275 void CMMarkStack::expand() {
 276   // Called, during remark, if we've overflown the marking stack during marking.
 277   assert(isEmpty(), "stack should been emptied while handling overflow");
 278   assert(_capacity <= (jint) MarkStackSizeMax, "stack bigger than permitted");
 279   // Clear expansion flag
 280   _should_expand = false;
 281   if (_capacity == (jint) MarkStackSizeMax) {
 282     if (PrintGCDetails && Verbose) {
 283       gclog_or_tty->print_cr(" (benign) Can't expand marking stack capacity, at max size limit");
 284     }
 285     return;
 286   }
 287   // Double capacity if possible
 288   jint new_capacity = MIN2(_capacity*2, (jint) MarkStackSizeMax);
 289   // Do not give up existing stack until we have managed to
 290   // get the double capacity that we desired.
 291   ReservedSpace rs(ReservedSpace::allocation_align_size_up(new_capacity *
 292                                                            sizeof(oop)));
 293   if (rs.is_reserved()) {
 294     // Release the backing store associated with old stack
 295     _virtual_space.release();
 296     // Reinitialize virtual space for new stack
 297     if (!_virtual_space.initialize(rs, rs.size())) {
 298       fatal("Not enough swap for expanded marking stack capacity");
 299     }
 300     _base = (oop*)(_virtual_space.low());
 301     _index = 0;
 302     _capacity = new_capacity;
 303   } else {
 304     if (PrintGCDetails && Verbose) {
 305       // Failed to double capacity, continue;
 306       gclog_or_tty->print(" (benign) Failed to expand marking stack capacity from "
 307                           SIZE_FORMAT "K to " SIZE_FORMAT "K",
 308                           _capacity / K, new_capacity / K);
 309     }
 310   }
 311 }
 312 
 313 void CMMarkStack::set_should_expand() {
 314   // If we're resetting the marking state because of an
 315   // marking stack overflow, record that we should, if
 316   // possible, expand the stack.
 317   _should_expand = _cm->has_overflown();
 318 }
 319 
 320 CMMarkStack::~CMMarkStack() {
 321   if (_base != NULL) {
 322     _base = NULL;
 323     _virtual_space.release();
 324   }
 325 }
 326 
 327 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
 328   MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
 329   jint start = _index;
 330   jint next_index = start + n;


 559     // Calculate the number of parallel marking threads by scaling
 560     // the number of parallel GC threads.
 561     uint marking_thread_num = scale_parallel_threads(ParallelGCThreads);
 562     FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num);
 563     _sleep_factor             = 0.0;
 564     _marking_task_overhead    = 1.0;
 565   }
 566 
 567   assert(ConcGCThreads > 0, "Should have been set");
 568   _parallel_marking_threads = ConcGCThreads;
 569   _max_parallel_marking_threads = _parallel_marking_threads;
 570 
 571   if (parallel_marking_threads() > 1) {
 572     _cleanup_task_overhead = 1.0;
 573   } else {
 574     _cleanup_task_overhead = marking_task_overhead();
 575   }
 576   _cleanup_sleep_factor =
 577                    (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
 578 
 579 #if 0
 580   gclog_or_tty->print_cr("Marking Threads          %d", parallel_marking_threads());
 581   gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead());
 582   gclog_or_tty->print_cr("CM Sleep Factor          %1.4lf", sleep_factor());
 583   gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead());
 584   gclog_or_tty->print_cr("CL Sleep Factor          %1.4lf", cleanup_sleep_factor());
 585 #endif
 586 
 587   _parallel_workers = new WorkGang("G1 Marker",
 588        _max_parallel_marking_threads, false, true);
 589   if (_parallel_workers == NULL) {
 590     vm_exit_during_initialization("Failed necessary allocation.");
 591   } else {
 592     _parallel_workers->initialize_workers();
 593   }
 594 
 595   if (FLAG_IS_DEFAULT(MarkStackSize)) {
 596     size_t mark_stack_size =
 597       MIN2(MarkStackSizeMax,
 598           MAX2(MarkStackSize, (size_t) (parallel_marking_threads() * TASKQUEUE_SIZE)));
 599     // Verify that the calculated value for MarkStackSize is in range.
 600     // It would be nice to use the private utility routine from Arguments.
 601     if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
 602       warning("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): "
 603               "must be between 1 and " SIZE_FORMAT,
 604               mark_stack_size, MarkStackSizeMax);
 605       return;
 606     }


 905     return;
 906   }
 907 
 908   // If we're executing the concurrent phase of marking, reset the marking
 909   // state; otherwise the marking state is reset after reference processing,
 910   // during the remark pause.
 911   // If we reset here as a result of an overflow during the remark we will
 912   // see assertion failures from any subsequent set_concurrency_and_phase()
 913   // calls.
 914   if (concurrent()) {
 915     // let the task associated with with worker 0 do this
 916     if (worker_id == 0) {
 917       // task 0 is responsible for clearing the global data structures
 918       // We should be here because of an overflow. During STW we should
 919       // not clear the overflow flag since we rely on it being true when
 920       // we exit this method to abort the pause and restart concurrent
 921       // marking.
 922       reset_marking_state(true /* clear_overflow */);
 923       force_overflow()->update();
 924 
 925       if (G1Log::fine()) {
 926         gclog_or_tty->gclog_stamp();
 927         gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]");
 928       }
 929     }
 930   }
 931 
 932   // after this, each task should reset its own data structures then
 933   // then go into the second barrier
 934 }
 935 
 936 void ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
 937   SuspendibleThreadSetLeaver sts_leave(concurrent());
 938   _second_overflow_barrier_sync.enter();
 939 
 940   // at this point everything should be re-initialized and ready to go
 941 }
 942 
 943 #ifndef PRODUCT
 944 void ForceOverflowSettings::init() {
 945   _num_remaining = G1ConcMarkForceOverflow;
 946   _force = false;
 947   update();
 948 }


1070   ConcurrentMark* _cm;
1071 
1072 public:
1073   CMRootRegionScanTask(ConcurrentMark* cm) :
1074     AbstractGangTask("Root Region Scan"), _cm(cm) { }
1075 
1076   void work(uint worker_id) {
1077     assert(Thread::current()->is_ConcurrentGC_thread(),
1078            "this should only be done by a conc GC thread");
1079 
1080     CMRootRegions* root_regions = _cm->root_regions();
1081     HeapRegion* hr = root_regions->claim_next();
1082     while (hr != NULL) {
1083       _cm->scanRootRegion(hr, worker_id);
1084       hr = root_regions->claim_next();
1085     }
1086   }
1087 };
1088 
1089 void ConcurrentMark::scanRootRegions() {
1090   double scan_start = os::elapsedTime();
1091 
1092   // Start of concurrent marking.
1093   ClassLoaderDataGraph::clear_claimed_marks();
1094 
1095   // scan_in_progress() will have been set to true only if there was
1096   // at least one root region to scan. So, if it's false, we
1097   // should not attempt to do any further work.
1098   if (root_regions()->scan_in_progress()) {
1099     if (G1Log::fine()) {
1100       gclog_or_tty->gclog_stamp();
1101       gclog_or_tty->print_cr("[GC concurrent-root-region-scan-start]");
1102     }
1103 
1104     _parallel_marking_threads = calc_parallel_marking_threads();
1105     assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1106            "Maximum number of marking threads exceeded");
1107     uint active_workers = MAX2(1U, parallel_marking_threads());
1108 
1109     CMRootRegionScanTask task(this);
1110     _parallel_workers->set_active_workers(active_workers);
1111     _parallel_workers->run_task(&task);
1112 
1113     if (G1Log::fine()) {
1114       gclog_or_tty->gclog_stamp();
1115       gclog_or_tty->print_cr("[GC concurrent-root-region-scan-end, %1.7lf secs]", os::elapsedTime() - scan_start);
1116     }

1117 
1118     // It's possible that has_aborted() is true here without actually
1119     // aborting the survivor scan earlier. This is OK as it's
1120     // mainly used for sanity checking.
1121     root_regions()->scan_finished();
1122   }
1123 }
1124 
1125 void ConcurrentMark::markFromRoots() {
1126   // we might be tempted to assert that:
1127   // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1128   //        "inconsistent argument?");
1129   // However that wouldn't be right, because it's possible that
1130   // a safepoint is indeed in progress as a younger generation
1131   // stop-the-world GC happens even as we mark in this generation.
1132 
1133   _restart_for_overflow = false;
1134   force_overflow_conc()->init();
1135 
1136   // _g1h has _n_par_threads
1137   _parallel_marking_threads = calc_parallel_marking_threads();
1138   assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1139     "Maximum number of marking threads exceeded");
1140 
1141   uint active_workers = MAX2(1U, parallel_marking_threads());
1142   assert(active_workers > 0, "Should have been set");
1143 
1144   // Parallel task terminator is set in "set_concurrency_and_phase()"
1145   set_concurrency_and_phase(active_workers, true /* concurrent */);
1146 
1147   CMConcurrentMarkingTask markingTask(this, cmThread());
1148   _parallel_workers->set_active_workers(active_workers);
1149   _parallel_workers->run_task(&markingTask);
1150   print_stats();
1151 }
1152 
1153 // Helper class to get rid of some boilerplate code.
1154 class G1CMTraceTime : public StackObj {
1155   GCTraceTimeImpl _gc_trace_time;
1156   static bool doit_and_prepend(bool doit) {
1157     if (doit) {
1158       gclog_or_tty->put(' ');
1159     }
1160     return doit;
1161   }
1162 
1163  public:
1164   G1CMTraceTime(const char* title, bool doit)
1165     : _gc_trace_time(title, doit_and_prepend(doit), false, G1CollectedHeap::heap()->gc_timer_cm()) {
1166   }
1167 };
1168 
1169 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1170   // world is stopped at this checkpoint
1171   assert(SafepointSynchronize::is_at_safepoint(),
1172          "world should be stopped");
1173 
1174   G1CollectedHeap* g1h = G1CollectedHeap::heap();
1175 
1176   // If a full collection has happened, we shouldn't do this.
1177   if (has_aborted()) {
1178     g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1179     return;
1180   }
1181 
1182   SvcGCMarker sgcm(SvcGCMarker::OTHER);
1183 
1184   if (VerifyDuringGC) {
1185     HandleMark hm;  // handle scope
1186     g1h->prepare_for_verify();
1187     Universe::verify(VerifyOption_G1UsePrevMarking,
1188                      " VerifyDuringGC:(before)");
1189   }
1190   g1h->check_bitmaps("Remark Start");
1191 
1192   G1CollectorPolicy* g1p = g1h->g1_policy();
1193   g1p->record_concurrent_mark_remark_start();
1194 
1195   double start = os::elapsedTime();
1196 
1197   checkpointRootsFinalWork();
1198 
1199   double mark_work_end = os::elapsedTime();
1200 
1201   weakRefsWork(clear_all_soft_refs);
1202 
1203   if (has_overflown()) {
1204     // Oops.  We overflowed.  Restart concurrent marking.
1205     _restart_for_overflow = true;
1206     if (G1TraceMarkStackOverflow) {
1207       gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
1208     }
1209 
1210     // Verify the heap w.r.t. the previous marking bitmap.
1211     if (VerifyDuringGC) {
1212       HandleMark hm;  // handle scope
1213       g1h->prepare_for_verify();
1214       Universe::verify(VerifyOption_G1UsePrevMarking,
1215                        " VerifyDuringGC:(overflow)");
1216     }
1217 
1218     // Clear the marking state because we will be restarting
1219     // marking due to overflowing the global mark stack.
1220     reset_marking_state();
1221   } else {
1222     {
1223       G1CMTraceTime trace("GC aggregate-data", G1Log::finer());
1224 
1225       // Aggregate the per-task counting data that we have accumulated
1226       // while marking.
1227       aggregate_count_data();
1228     }
1229 
1230     SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1231     // We're done with marking.
1232     // This is the end of  the marking cycle, we're expected all
1233     // threads to have SATB queues with active set to true.
1234     satb_mq_set.set_active_all_threads(false, /* new active value */
1235                                        true /* expected_active */);
1236 
1237     if (VerifyDuringGC) {
1238       HandleMark hm;  // handle scope
1239       g1h->prepare_for_verify();
1240       Universe::verify(VerifyOption_G1UseNextMarking,
1241                        " VerifyDuringGC:(after)");
1242     }
1243     g1h->check_bitmaps("Remark End");
1244     assert(!restart_for_overflow(), "sanity");
1245     // Completely reset the marking state since marking completed
1246     set_non_marking_state();
1247   }
1248 
1249   // Expand the marking stack, if we have to and if we can.
1250   if (_markStack.should_expand()) {
1251     _markStack.expand();
1252   }
1253 
1254   // Statistics
1255   double now = os::elapsedTime();
1256   _remark_mark_times.add((mark_work_end - start) * 1000.0);
1257   _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1258   _remark_times.add((now - start) * 1000.0);
1259 
1260   g1p->record_concurrent_mark_remark_end();
1261 


1740 
1741 };
1742 
1743 void ConcurrentMark::cleanup() {
1744   // world is stopped at this checkpoint
1745   assert(SafepointSynchronize::is_at_safepoint(),
1746          "world should be stopped");
1747   G1CollectedHeap* g1h = G1CollectedHeap::heap();
1748 
1749   // If a full collection has happened, we shouldn't do this.
1750   if (has_aborted()) {
1751     g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1752     return;
1753   }
1754 
1755   g1h->verify_region_sets_optional();
1756 
1757   if (VerifyDuringGC) {
1758     HandleMark hm;  // handle scope
1759     g1h->prepare_for_verify();
1760     Universe::verify(VerifyOption_G1UsePrevMarking,
1761                      " VerifyDuringGC:(before)");
1762   }
1763   g1h->check_bitmaps("Cleanup Start");
1764 
1765   G1CollectorPolicy* g1p = g1h->g1_policy();
1766   g1p->record_concurrent_mark_cleanup_start();
1767 
1768   double start = os::elapsedTime();
1769 
1770   HeapRegionRemSet::reset_for_cleanup_tasks();
1771 
1772   // Do counting once more with the world stopped for good measure.
1773   G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
1774 
1775   g1h->workers()->run_task(&g1_par_count_task);
1776 
1777   if (VerifyDuringGC) {
1778     // Verify that the counting data accumulated during marking matches
1779     // that calculated by walking the marking bitmap.
1780 
1781     // Bitmaps to hold expected values


1783     BitMap expected_card_bm(_card_bm.size(), true);
1784 
1785     G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
1786                                                  &_region_bm,
1787                                                  &_card_bm,
1788                                                  &expected_region_bm,
1789                                                  &expected_card_bm);
1790 
1791     g1h->workers()->run_task(&g1_par_verify_task);
1792 
1793     guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
1794   }
1795 
1796   size_t start_used_bytes = g1h->used();
1797   g1h->collector_state()->set_mark_in_progress(false);
1798 
1799   double count_end = os::elapsedTime();
1800   double this_final_counting_time = (count_end - start);
1801   _total_counting_time += this_final_counting_time;
1802 
1803   if (G1PrintRegionLivenessInfo) {
1804     G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking");
1805     _g1h->heap_region_iterate(&cl);
1806   }
1807 
1808   // Install newly created mark bitMap as "prev".
1809   swapMarkBitMaps();
1810 
1811   g1h->reset_gc_time_stamp();
1812 
1813   uint n_workers = _g1h->workers()->active_workers();
1814 
1815   // Note end of marking in all heap regions.
1816   G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1817   g1h->workers()->run_task(&g1_par_note_end_task);
1818   g1h->check_gc_time_stamps();
1819 
1820   if (!cleanup_list_is_empty()) {
1821     // The cleanup list is not empty, so we'll have to process it
1822     // concurrently. Notify anyone else that might be wanting free
1823     // regions that there will be more free regions coming soon.
1824     g1h->set_free_regions_coming();


1827   // call below, since it affects the metric by which we sort the heap
1828   // regions.
1829   if (G1ScrubRemSets) {
1830     double rs_scrub_start = os::elapsedTime();
1831     G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers);
1832     g1h->workers()->run_task(&g1_par_scrub_rs_task);
1833 
1834     double rs_scrub_end = os::elapsedTime();
1835     double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
1836     _total_rs_scrub_time += this_rs_scrub_time;
1837   }
1838 
1839   // this will also free any regions totally full of garbage objects,
1840   // and sort the regions.
1841   g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1842 
1843   // Statistics.
1844   double end = os::elapsedTime();
1845   _cleanup_times.add((end - start) * 1000.0);
1846 
1847   if (G1Log::fine()) {
1848     g1h->g1_policy()->print_heap_transition(start_used_bytes);
1849   }
1850 
1851   // Clean up will have freed any regions completely full of garbage.
1852   // Update the soft reference policy with the new heap occupancy.
1853   Universe::update_heap_info_at_gc();
1854 
1855   if (VerifyDuringGC) {
1856     HandleMark hm;  // handle scope
1857     g1h->prepare_for_verify();
1858     Universe::verify(VerifyOption_G1UsePrevMarking,
1859                      " VerifyDuringGC:(after)");
1860   }
1861 
1862   g1h->check_bitmaps("Cleanup End");
1863 
1864   g1h->verify_region_sets_optional();
1865 
1866   // We need to make this be a "collection" so any collection pause that
1867   // races with it goes around and waits for completeCleanup to finish.
1868   g1h->increment_total_collections();
1869 
1870   // Clean out dead classes and update Metaspace sizes.
1871   if (ClassUnloadingWithConcurrentMark) {
1872     ClassLoaderDataGraph::purge();
1873   }
1874   MetaspaceGC::compute_new_size();
1875 
1876   // We reclaimed old regions so we should calculate the sizes to make
1877   // sure we update the old gen/space data.
1878   g1h->g1mm()->update_sizes();
1879   g1h->allocation_context_stats().update_after_mark();
1880 
1881   g1h->trace_heap_after_concurrent_cycle();
1882 }
1883 
1884 void ConcurrentMark::completeCleanup() {
1885   if (has_aborted()) return;
1886 
1887   G1CollectedHeap* g1h = G1CollectedHeap::heap();
1888 
1889   _cleanup_list.verify_optional();
1890   FreeRegionList tmp_free_list("Tmp Free List");
1891 
1892   if (G1ConcRegionFreeingVerbose) {
1893     gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1894                            "cleanup list has %u entries",
1895                            _cleanup_list.length());
1896   }
1897 
1898   // No one else should be accessing the _cleanup_list at this point,
1899   // so it is not necessary to take any locks
1900   while (!_cleanup_list.is_empty()) {
1901     HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */);
1902     assert(hr != NULL, "Got NULL from a non-empty list");
1903     hr->par_clear();
1904     tmp_free_list.add_ordered(hr);
1905 
1906     // Instead of adding one region at a time to the secondary_free_list,
1907     // we accumulate them in the local list and move them a few at a
1908     // time. This also cuts down on the number of notify_all() calls
1909     // we do during this process. We'll also append the local list when
1910     // _cleanup_list is empty (which means we just removed the last
1911     // region from the _cleanup_list).
1912     if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1913         _cleanup_list.is_empty()) {
1914       if (G1ConcRegionFreeingVerbose) {
1915         gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1916                                "appending %u entries to the secondary_free_list, "
1917                                "cleanup list still has %u entries",
1918                                tmp_free_list.length(),
1919                                _cleanup_list.length());
1920       }
1921 
1922       {
1923         MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1924         g1h->secondary_free_list_add(&tmp_free_list);
1925         SecondaryFreeList_lock->notify_all();
1926       }
1927 #ifndef PRODUCT
1928       if (G1StressConcRegionFreeing) {
1929         for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1930           os::sleep(Thread::current(), (jlong) 1, false);
1931         }
1932       }
1933 #endif
1934     }
1935   }
1936   assert(tmp_free_list.is_empty(), "post-condition");
1937 }
1938 
1939 // Supporting Object and Oop closures for reference discovery
1940 // and processing in during marking


2157     // Skip processing the discovered references if we have
2158     // overflown the global marking stack. Reference objects
2159     // only get discovered once so it is OK to not
2160     // de-populate the discovered reference lists. We could have,
2161     // but the only benefit would be that, when marking restarts,
2162     // less reference objects are discovered.
2163     return;
2164   }
2165 
2166   ResourceMark rm;
2167   HandleMark   hm;
2168 
2169   G1CollectedHeap* g1h = G1CollectedHeap::heap();
2170 
2171   // Is alive closure.
2172   G1CMIsAliveClosure g1_is_alive(g1h);
2173 
2174   // Inner scope to exclude the cleaning of the string and symbol
2175   // tables from the displayed time.
2176   {
2177     G1CMTraceTime t("GC ref-proc", G1Log::finer());
2178 
2179     ReferenceProcessor* rp = g1h->ref_processor_cm();
2180 
2181     // See the comment in G1CollectedHeap::ref_processing_init()
2182     // about how reference processing currently works in G1.
2183 
2184     // Set the soft reference policy
2185     rp->setup_policy(clear_all_soft_refs);
2186     assert(_markStack.isEmpty(), "mark stack should be empty");
2187 
2188     // Instances of the 'Keep Alive' and 'Complete GC' closures used
2189     // in serial reference processing. Note these closures are also
2190     // used for serially processing (by the the current thread) the
2191     // JNI references during parallel reference processing.
2192     //
2193     // These closures do not need to synchronize with the worker
2194     // threads involved in parallel reference processing as these
2195     // instances are executed serially by the current thread (e.g.
2196     // reference processing is not multi-threaded and is thus
2197     // performed by the current thread instead of a gang worker).


2247       set_has_overflown();
2248     }
2249 
2250     assert(rp->num_q() == active_workers, "why not");
2251 
2252     rp->enqueue_discovered_references(executor);
2253 
2254     rp->verify_no_references_recorded();
2255     assert(!rp->discovery_enabled(), "Post condition");
2256   }
2257 
2258   if (has_overflown()) {
2259     // We can not trust g1_is_alive if the marking stack overflowed
2260     return;
2261   }
2262 
2263   assert(_markStack.isEmpty(), "Marking should have completed");
2264 
2265   // Unload Klasses, String, Symbols, Code Cache, etc.
2266   {
2267     G1CMTraceTime trace("Unloading", G1Log::finer());
2268 
2269     if (ClassUnloadingWithConcurrentMark) {
2270       bool purged_classes;
2271 
2272       {
2273         G1CMTraceTime trace("System Dictionary Unloading", G1Log::finest());
2274         purged_classes = SystemDictionary::do_unloading(&g1_is_alive, false /* Defer klass cleaning */);
2275       }
2276 
2277       {
2278         G1CMTraceTime trace("Parallel Unloading", G1Log::finest());
2279         weakRefsWorkParallelPart(&g1_is_alive, purged_classes);
2280       }
2281     }
2282 
2283     if (G1StringDedup::is_enabled()) {
2284       G1CMTraceTime trace("String Deduplication Unlink", G1Log::finest());
2285       G1StringDedup::unlink(&g1_is_alive);
2286     }
2287   }
2288 }
2289 
2290 void ConcurrentMark::swapMarkBitMaps() {
2291   CMBitMapRO* temp = _prevMarkBitMap;
2292   _prevMarkBitMap  = (CMBitMapRO*)_nextMarkBitMap;
2293   _nextMarkBitMap  = (CMBitMap*)  temp;
2294 }
2295 
2296 // Closure for marking entries in SATB buffers.
2297 class CMSATBBufferClosure : public SATBBufferClosure {
2298 private:
2299   CMTask* _task;
2300   G1CollectedHeap* _g1h;
2301 
2302   // This is very similar to CMTask::deal_with_reference, but with
2303   // more relaxed requirements for the argument, so this must be more
2304   // circumspect about treating the argument as an object.


2385                               true         /* do_termination       */,
2386                               false        /* is_serial            */);
2387       } while (task->has_aborted() && !_cm->has_overflown());
2388       // If we overflow, then we do not want to restart. We instead
2389       // want to abort remark and do concurrent marking again.
2390       task->record_end_time();
2391     }
2392   }
2393 
2394   CMRemarkTask(ConcurrentMark* cm, uint active_workers) :
2395     AbstractGangTask("Par Remark"), _cm(cm) {
2396     _cm->terminator()->reset_for_reuse(active_workers);
2397   }
2398 };
2399 
2400 void ConcurrentMark::checkpointRootsFinalWork() {
2401   ResourceMark rm;
2402   HandleMark   hm;
2403   G1CollectedHeap* g1h = G1CollectedHeap::heap();
2404 
2405   G1CMTraceTime trace("Finalize Marking", G1Log::finer());
2406 
2407   g1h->ensure_parsability(false);
2408 
2409   // this is remark, so we'll use up all active threads
2410   uint active_workers = g1h->workers()->active_workers();
2411   set_concurrency_and_phase(active_workers, false /* concurrent */);
2412   // Leave _parallel_marking_threads at it's
2413   // value originally calculated in the ConcurrentMark
2414   // constructor and pass values of the active workers
2415   // through the gang in the task.
2416 
2417   {
2418     StrongRootsScope srs(active_workers);
2419 
2420     CMRemarkTask remarkTask(this, active_workers);
2421     // We will start all available threads, even if we decide that the
2422     // active_workers will be fewer. The extra ones will just bail out
2423     // immediately.
2424     g1h->workers()->run_task(&remarkTask);
2425   }


2702   // Clear the global region bitmap - it will be filled as part
2703   // of the final counting task.
2704   _region_bm.clear();
2705 
2706   uint max_regions = _g1h->max_regions();
2707   assert(_max_worker_id > 0, "uninitialized");
2708 
2709   for (uint i = 0; i < _max_worker_id; i += 1) {
2710     BitMap* task_card_bm = count_card_bitmap_for(i);
2711     size_t* marked_bytes_array = count_marked_bytes_array_for(i);
2712 
2713     assert(task_card_bm->size() == _card_bm.size(), "size mismatch");
2714     assert(marked_bytes_array != NULL, "uninitialized");
2715 
2716     memset(marked_bytes_array, 0, (size_t) max_regions * sizeof(size_t));
2717     task_card_bm->clear();
2718   }
2719 }
2720 
2721 void ConcurrentMark::print_stats() {
2722   if (G1MarkingVerboseLevel > 0) {
2723     gclog_or_tty->print_cr("---------------------------------------------------------------------");



2724     for (size_t i = 0; i < _active_tasks; ++i) {
2725       _tasks[i]->print_stats();
2726       gclog_or_tty->print_cr("---------------------------------------------------------------------");
2727     }
2728   }
2729 }
2730 
2731 // abandon current marking iteration due to a Full GC
2732 void ConcurrentMark::abort() {
2733   if (!cmThread()->during_cycle() || _has_aborted) {
2734     // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
2735     return;
2736   }
2737 
2738   // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
2739   // concurrent bitmap clearing.
2740   _nextMarkBitMap->clearAll();
2741 
2742   // Note we cannot clear the previous marking bitmap here
2743   // since VerifyDuringGC verifies the objects marked during
2744   // a full GC against the previous bitmap.
2745 
2746   // Clear the liveness counting data
2747   clear_all_count_data();


2751     _tasks[i]->clear_region_fields();
2752   }
2753   _first_overflow_barrier_sync.abort();
2754   _second_overflow_barrier_sync.abort();
2755   _has_aborted = true;
2756 
2757   SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2758   satb_mq_set.abandon_partial_marking();
2759   // This can be called either during or outside marking, we'll read
2760   // the expected_active value from the SATB queue set.
2761   satb_mq_set.set_active_all_threads(
2762                                  false, /* new active value */
2763                                  satb_mq_set.is_active() /* expected_active */);
2764 
2765   _g1h->trace_heap_after_concurrent_cycle();
2766   _g1h->register_concurrent_cycle_end();
2767 }
2768 
2769 static void print_ms_time_info(const char* prefix, const char* name,
2770                                NumberSeq& ns) {
2771   gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2772                          prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2773   if (ns.num() > 0) {
2774     gclog_or_tty->print_cr("%s         [std. dev = %8.2f ms, max = %8.2f ms]",
2775                            prefix, ns.sd(), ns.maximum());
2776   }
2777 }
2778 
2779 void ConcurrentMark::print_summary_info() {
2780   gclog_or_tty->print_cr(" Concurrent marking:");





2781   print_ms_time_info("  ", "init marks", _init_times);
2782   print_ms_time_info("  ", "remarks", _remark_times);
2783   {
2784     print_ms_time_info("     ", "final marks", _remark_mark_times);
2785     print_ms_time_info("     ", "weak refs", _remark_weak_ref_times);
2786 
2787   }
2788   print_ms_time_info("  ", "cleanups", _cleanup_times);
2789   gclog_or_tty->print_cr("    Final counting total time = %8.2f s (avg = %8.2f ms).",
2790                          _total_counting_time,
2791                          (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 /
2792                           (double)_cleanup_times.num()
2793                          : 0.0));
2794   if (G1ScrubRemSets) {
2795     gclog_or_tty->print_cr("    RS scrub total time = %8.2f s (avg = %8.2f ms).",
2796                            _total_rs_scrub_time,
2797                            (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 /
2798                             (double)_cleanup_times.num()
2799                            : 0.0));
2800   }
2801   gclog_or_tty->print_cr("  Total stop_world time = %8.2f s.",
2802                          (_init_times.sum() + _remark_times.sum() +
2803                           _cleanup_times.sum())/1000.0);
2804   gclog_or_tty->print_cr("  Total concurrent time = %8.2f s "
2805                 "(%8.2f s marking).",
2806                 cmThread()->vtime_accum(),
2807                 cmThread()->vtime_mark_accum());
2808 }
2809 
2810 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2811   _parallel_workers->print_worker_threads_on(st);
2812 }
2813 
2814 void ConcurrentMark::print_on_error(outputStream* st) const {
2815   st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2816       p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2817   _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2818   _nextMarkBitMap->print_on_error(st, " Next Bits: ");
2819 }
2820 
2821 // We take a break if someone is trying to stop the world.
2822 bool ConcurrentMark::do_yield_check(uint worker_id) {
2823   if (SuspendibleThreadSet::should_yield()) {
2824     if (worker_id == 0) {
2825       _g1h->g1_policy()->record_concurrent_pause();
2826     }
2827     SuspendibleThreadSet::yield();


3167 
3168   // This keeps claiming and applying the closure to completed buffers
3169   // until we run out of buffers or we need to abort.
3170   while (!has_aborted() &&
3171          satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
3172     regular_clock_call();
3173   }
3174 
3175   _draining_satb_buffers = false;
3176 
3177   assert(has_aborted() ||
3178          concurrent() ||
3179          satb_mq_set.completed_buffers_num() == 0, "invariant");
3180 
3181   // again, this was a potentially expensive operation, decrease the
3182   // limits to get the regular clock call early
3183   decrease_limits();
3184 }
3185 
3186 void CMTask::print_stats() {
3187   gclog_or_tty->print_cr("Marking Stats, task = %u, calls = %d",
3188                          _worker_id, _calls);
3189   gclog_or_tty->print_cr("  Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3190                          _elapsed_time_ms, _termination_time_ms);
3191   gclog_or_tty->print_cr("  Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3192                          _step_times_ms.num(), _step_times_ms.avg(),
3193                          _step_times_ms.sd());
3194   gclog_or_tty->print_cr("                    max = %1.2lfms, total = %1.2lfms",
3195                          _step_times_ms.maximum(), _step_times_ms.sum());
3196 }
3197 
3198 bool ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, oop& obj) {
3199   return _task_queues->steal(worker_id, hash_seed, obj);
3200 }
3201 
3202 /*****************************************************************************
3203 
3204     The do_marking_step(time_target_ms, ...) method is the building
3205     block of the parallel marking framework. It can be called in parallel
3206     with other invocations of do_marking_step() on different tasks
3207     (but only one per task, obviously) and concurrently with the
3208     mutator threads, or during remark, hence it eliminates the need
3209     for two versions of the code. When called during remark, it will
3210     pick up from where the task left off during the concurrent marking
3211     phase. Interestingly, tasks are also claimable during evacuation
3212     pauses too, since do_marking_step() ensures that it aborts before
3213     it needs to yield.
3214 


3684 #define G1PPRL_ADDR_BASE_H_FORMAT  " %37s"
3685 #else // _LP64
3686 #define G1PPRL_ADDR_BASE_H_FORMAT  " %21s"
3687 #endif // _LP64
3688 
3689 // For per-region info
3690 #define G1PPRL_TYPE_FORMAT            "   %-4s"
3691 #define G1PPRL_TYPE_H_FORMAT          "   %4s"
3692 #define G1PPRL_BYTE_FORMAT            "  " SIZE_FORMAT_W(9)
3693 #define G1PPRL_BYTE_H_FORMAT          "  %9s"
3694 #define G1PPRL_DOUBLE_FORMAT          "  %14.1f"
3695 #define G1PPRL_DOUBLE_H_FORMAT        "  %14s"
3696 
3697 // For summary info
3698 #define G1PPRL_SUM_ADDR_FORMAT(tag)    "  " tag ":" G1PPRL_ADDR_BASE_FORMAT
3699 #define G1PPRL_SUM_BYTE_FORMAT(tag)    "  " tag ": " SIZE_FORMAT
3700 #define G1PPRL_SUM_MB_FORMAT(tag)      "  " tag ": %1.2f MB"
3701 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
3702 
3703 G1PrintRegionLivenessInfoClosure::
3704 G1PrintRegionLivenessInfoClosure(outputStream* out, const char* phase_name)
3705   : _out(out),
3706     _total_used_bytes(0), _total_capacity_bytes(0),
3707     _total_prev_live_bytes(0), _total_next_live_bytes(0),
3708     _hum_used_bytes(0), _hum_capacity_bytes(0),
3709     _hum_prev_live_bytes(0), _hum_next_live_bytes(0),
3710     _total_remset_bytes(0), _total_strong_code_roots_bytes(0) {
3711   G1CollectedHeap* g1h = G1CollectedHeap::heap();
3712   MemRegion g1_reserved = g1h->g1_reserved();
3713   double now = os::elapsedTime();
3714 
3715   // Print the header of the output.
3716   _out->cr();
3717   _out->print_cr(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
3718   _out->print_cr(G1PPRL_LINE_PREFIX" HEAP"
3719                  G1PPRL_SUM_ADDR_FORMAT("reserved")
3720                  G1PPRL_SUM_BYTE_FORMAT("region-size"),
3721                  p2i(g1_reserved.start()), p2i(g1_reserved.end()),
3722                  HeapRegion::GrainBytes);
3723   _out->print_cr(G1PPRL_LINE_PREFIX);
3724   _out->print_cr(G1PPRL_LINE_PREFIX
3725                 G1PPRL_TYPE_H_FORMAT
3726                 G1PPRL_ADDR_BASE_H_FORMAT
3727                 G1PPRL_BYTE_H_FORMAT
3728                 G1PPRL_BYTE_H_FORMAT
3729                 G1PPRL_BYTE_H_FORMAT
3730                 G1PPRL_DOUBLE_H_FORMAT
3731                 G1PPRL_BYTE_H_FORMAT
3732                 G1PPRL_BYTE_H_FORMAT,
3733                 "type", "address-range",
3734                 "used", "prev-live", "next-live", "gc-eff",
3735                 "remset", "code-roots");
3736   _out->print_cr(G1PPRL_LINE_PREFIX
3737                 G1PPRL_TYPE_H_FORMAT
3738                 G1PPRL_ADDR_BASE_H_FORMAT
3739                 G1PPRL_BYTE_H_FORMAT
3740                 G1PPRL_BYTE_H_FORMAT
3741                 G1PPRL_BYTE_H_FORMAT
3742                 G1PPRL_DOUBLE_H_FORMAT
3743                 G1PPRL_BYTE_H_FORMAT
3744                 G1PPRL_BYTE_H_FORMAT,
3745                 "", "",
3746                 "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)",
3747                 "(bytes)", "(bytes)");
3748 }
3749 
3750 // It takes as a parameter a reference to one of the _hum_* fields, it
3751 // deduces the corresponding value for a region in a humongous region
3752 // series (either the region size, or what's left if the _hum_* field
3753 // is < the region size), and updates the _hum_* field accordingly.
3754 size_t G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* hum_bytes) {
3755   size_t bytes = 0;
3756   // The > 0 check is to deal with the prev and next live bytes which


3798     _hum_used_bytes      = used_bytes;
3799     _hum_prev_live_bytes = prev_live_bytes;
3800     _hum_next_live_bytes = next_live_bytes;
3801     get_hum_bytes(&used_bytes, &capacity_bytes,
3802                   &prev_live_bytes, &next_live_bytes);
3803     end = bottom + HeapRegion::GrainWords;
3804   } else if (r->is_continues_humongous()) {
3805     get_hum_bytes(&used_bytes, &capacity_bytes,
3806                   &prev_live_bytes, &next_live_bytes);
3807     assert(end == bottom + HeapRegion::GrainWords, "invariant");
3808   }
3809 
3810   _total_used_bytes      += used_bytes;
3811   _total_capacity_bytes  += capacity_bytes;
3812   _total_prev_live_bytes += prev_live_bytes;
3813   _total_next_live_bytes += next_live_bytes;
3814   _total_remset_bytes    += remset_bytes;
3815   _total_strong_code_roots_bytes += strong_code_roots_bytes;
3816 
3817   // Print a line for this particular region.
3818   _out->print_cr(G1PPRL_LINE_PREFIX
3819                  G1PPRL_TYPE_FORMAT
3820                  G1PPRL_ADDR_BASE_FORMAT
3821                  G1PPRL_BYTE_FORMAT
3822                  G1PPRL_BYTE_FORMAT
3823                  G1PPRL_BYTE_FORMAT
3824                  G1PPRL_DOUBLE_FORMAT
3825                  G1PPRL_BYTE_FORMAT
3826                  G1PPRL_BYTE_FORMAT,
3827                  type, p2i(bottom), p2i(end),
3828                  used_bytes, prev_live_bytes, next_live_bytes, gc_eff,
3829                  remset_bytes, strong_code_roots_bytes);
3830 
3831   return false;
3832 }
3833 
3834 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
3835   // add static memory usages to remembered set sizes
3836   _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
3837   // Print the footer of the output.
3838   _out->print_cr(G1PPRL_LINE_PREFIX);
3839   _out->print_cr(G1PPRL_LINE_PREFIX
3840                  " SUMMARY"
3841                  G1PPRL_SUM_MB_FORMAT("capacity")
3842                  G1PPRL_SUM_MB_PERC_FORMAT("used")
3843                  G1PPRL_SUM_MB_PERC_FORMAT("prev-live")
3844                  G1PPRL_SUM_MB_PERC_FORMAT("next-live")
3845                  G1PPRL_SUM_MB_FORMAT("remset")
3846                  G1PPRL_SUM_MB_FORMAT("code-roots"),
3847                  bytes_to_mb(_total_capacity_bytes),
3848                  bytes_to_mb(_total_used_bytes),
3849                  perc(_total_used_bytes, _total_capacity_bytes),
3850                  bytes_to_mb(_total_prev_live_bytes),
3851                  perc(_total_prev_live_bytes, _total_capacity_bytes),
3852                  bytes_to_mb(_total_next_live_bytes),
3853                  perc(_total_next_live_bytes, _total_capacity_bytes),
3854                  bytes_to_mb(_total_remset_bytes),
3855                  bytes_to_mb(_total_strong_code_roots_bytes));
3856   _out->cr();
3857 }


  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/metadataOnStackMark.hpp"
  27 #include "classfile/symbolTable.hpp"
  28 #include "code/codeCache.hpp"
  29 #include "gc/g1/concurrentMark.inline.hpp"
  30 #include "gc/g1/concurrentMarkThread.inline.hpp"
  31 #include "gc/g1/g1CollectedHeap.inline.hpp"
  32 #include "gc/g1/g1CollectorPolicy.hpp"
  33 #include "gc/g1/g1CollectorState.hpp"


  34 #include "gc/g1/g1OopClosures.inline.hpp"
  35 #include "gc/g1/g1RemSet.hpp"
  36 #include "gc/g1/g1StringDedup.hpp"
  37 #include "gc/g1/heapRegion.inline.hpp"
  38 #include "gc/g1/heapRegionManager.inline.hpp"
  39 #include "gc/g1/heapRegionRemSet.hpp"
  40 #include "gc/g1/heapRegionSet.inline.hpp"
  41 #include "gc/g1/suspendibleThreadSet.hpp"
  42 #include "gc/shared/gcId.hpp"
  43 #include "gc/shared/gcTimer.hpp"
  44 #include "gc/shared/gcTrace.hpp"
  45 #include "gc/shared/gcTraceTime.hpp"
  46 #include "gc/shared/genOopClosures.inline.hpp"
  47 #include "gc/shared/referencePolicy.hpp"
  48 #include "gc/shared/strongRootsScope.hpp"
  49 #include "gc/shared/taskqueue.inline.hpp"
  50 #include "gc/shared/vmGCOperations.hpp"
  51 #include "logging/log.hpp"
  52 #include "memory/allocation.hpp"
  53 #include "memory/resourceArea.hpp"
  54 #include "oops/oop.inline.hpp"
  55 #include "runtime/atomic.inline.hpp"
  56 #include "runtime/handles.inline.hpp"
  57 #include "runtime/java.hpp"
  58 #include "runtime/prefetch.inline.hpp"
  59 #include "services/memTracker.hpp"
  60 
  61 // Concurrent marking bit map wrapper
  62 
  63 CMBitMapRO::CMBitMapRO(int shifter) :
  64   _bm(),
  65   _shifter(shifter) {
  66   _bmStartWord = 0;
  67   _bmWordSize = 0;
  68 }
  69 
  70 HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
  71                                                const HeapWord* limit) const {


 261     rs.release();
 262     return false;
 263   }
 264   assert(_virtual_space.committed_size() == rs.size(),
 265          "Didn't reserve backing store for all of ConcurrentMark stack?");
 266   _base = (oop*) _virtual_space.low();
 267   setEmpty();
 268   _capacity = (jint) capacity;
 269   _saved_index = -1;
 270   _should_expand = false;
 271   return true;
 272 }
 273 
 274 void CMMarkStack::expand() {
 275   // Called, during remark, if we've overflown the marking stack during marking.
 276   assert(isEmpty(), "stack should been emptied while handling overflow");
 277   assert(_capacity <= (jint) MarkStackSizeMax, "stack bigger than permitted");
 278   // Clear expansion flag
 279   _should_expand = false;
 280   if (_capacity == (jint) MarkStackSizeMax) {
 281     log_trace(gc)("(benign) Can't expand marking stack capacity, at max size limit");


 282     return;
 283   }
 284   // Double capacity if possible
 285   jint new_capacity = MIN2(_capacity*2, (jint) MarkStackSizeMax);
 286   // Do not give up existing stack until we have managed to
 287   // get the double capacity that we desired.
 288   ReservedSpace rs(ReservedSpace::allocation_align_size_up(new_capacity *
 289                                                            sizeof(oop)));
 290   if (rs.is_reserved()) {
 291     // Release the backing store associated with old stack
 292     _virtual_space.release();
 293     // Reinitialize virtual space for new stack
 294     if (!_virtual_space.initialize(rs, rs.size())) {
 295       fatal("Not enough swap for expanded marking stack capacity");
 296     }
 297     _base = (oop*)(_virtual_space.low());
 298     _index = 0;
 299     _capacity = new_capacity;
 300   } else {

 301     // Failed to double capacity, continue;
 302     log_trace(gc)("(benign) Failed to expand marking stack capacity from " SIZE_FORMAT "K to " SIZE_FORMAT "K",

 303                   _capacity / K, new_capacity / K);
 304   }

 305 }
 306 
 307 void CMMarkStack::set_should_expand() {
 308   // If we're resetting the marking state because of an
 309   // marking stack overflow, record that we should, if
 310   // possible, expand the stack.
 311   _should_expand = _cm->has_overflown();
 312 }
 313 
 314 CMMarkStack::~CMMarkStack() {
 315   if (_base != NULL) {
 316     _base = NULL;
 317     _virtual_space.release();
 318   }
 319 }
 320 
 321 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
 322   MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
 323   jint start = _index;
 324   jint next_index = start + n;


 553     // Calculate the number of parallel marking threads by scaling
 554     // the number of parallel GC threads.
 555     uint marking_thread_num = scale_parallel_threads(ParallelGCThreads);
 556     FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num);
 557     _sleep_factor             = 0.0;
 558     _marking_task_overhead    = 1.0;
 559   }
 560 
 561   assert(ConcGCThreads > 0, "Should have been set");
 562   _parallel_marking_threads = ConcGCThreads;
 563   _max_parallel_marking_threads = _parallel_marking_threads;
 564 
 565   if (parallel_marking_threads() > 1) {
 566     _cleanup_task_overhead = 1.0;
 567   } else {
 568     _cleanup_task_overhead = marking_task_overhead();
 569   }
 570   _cleanup_sleep_factor =
 571                    (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
 572 








 573   _parallel_workers = new WorkGang("G1 Marker",
 574        _max_parallel_marking_threads, false, true);
 575   if (_parallel_workers == NULL) {
 576     vm_exit_during_initialization("Failed necessary allocation.");
 577   } else {
 578     _parallel_workers->initialize_workers();
 579   }
 580 
 581   if (FLAG_IS_DEFAULT(MarkStackSize)) {
 582     size_t mark_stack_size =
 583       MIN2(MarkStackSizeMax,
 584           MAX2(MarkStackSize, (size_t) (parallel_marking_threads() * TASKQUEUE_SIZE)));
 585     // Verify that the calculated value for MarkStackSize is in range.
 586     // It would be nice to use the private utility routine from Arguments.
 587     if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
 588       warning("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): "
 589               "must be between 1 and " SIZE_FORMAT,
 590               mark_stack_size, MarkStackSizeMax);
 591       return;
 592     }


 891     return;
 892   }
 893 
 894   // If we're executing the concurrent phase of marking, reset the marking
 895   // state; otherwise the marking state is reset after reference processing,
 896   // during the remark pause.
 897   // If we reset here as a result of an overflow during the remark we will
 898   // see assertion failures from any subsequent set_concurrency_and_phase()
 899   // calls.
 900   if (concurrent()) {
 901     // let the task associated with with worker 0 do this
 902     if (worker_id == 0) {
 903       // task 0 is responsible for clearing the global data structures
 904       // We should be here because of an overflow. During STW we should
 905       // not clear the overflow flag since we rely on it being true when
 906       // we exit this method to abort the pause and restart concurrent
 907       // marking.
 908       reset_marking_state(true /* clear_overflow */);
 909       force_overflow()->update();
 910 
 911       log_info(gc)("Concurrent Mark reset for overflow");



 912     }
 913   }
 914 
 915   // after this, each task should reset its own data structures then
 916   // then go into the second barrier
 917 }
 918 
 919 void ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
 920   SuspendibleThreadSetLeaver sts_leave(concurrent());
 921   _second_overflow_barrier_sync.enter();
 922 
 923   // at this point everything should be re-initialized and ready to go
 924 }
 925 
 926 #ifndef PRODUCT
 927 void ForceOverflowSettings::init() {
 928   _num_remaining = G1ConcMarkForceOverflow;
 929   _force = false;
 930   update();
 931 }


1053   ConcurrentMark* _cm;
1054 
1055 public:
1056   CMRootRegionScanTask(ConcurrentMark* cm) :
1057     AbstractGangTask("Root Region Scan"), _cm(cm) { }
1058 
1059   void work(uint worker_id) {
1060     assert(Thread::current()->is_ConcurrentGC_thread(),
1061            "this should only be done by a conc GC thread");
1062 
1063     CMRootRegions* root_regions = _cm->root_regions();
1064     HeapRegion* hr = root_regions->claim_next();
1065     while (hr != NULL) {
1066       _cm->scanRootRegion(hr, worker_id);
1067       hr = root_regions->claim_next();
1068     }
1069   }
1070 };
1071 
1072 void ConcurrentMark::scanRootRegions() {
1073   jlong scan_start = os::elapsed_counter();
1074 
1075   // Start of concurrent marking.
1076   ClassLoaderDataGraph::clear_claimed_marks();
1077 
1078   // scan_in_progress() will have been set to true only if there was
1079   // at least one root region to scan. So, if it's false, we
1080   // should not attempt to do any further work.
1081   if (root_regions()->scan_in_progress()) {
1082     log_info(gc)("Concurrent Root Region Scan (%.3fs)", TimeHelper::counter_to_seconds(scan_start));



1083 
1084     _parallel_marking_threads = calc_parallel_marking_threads();
1085     assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1086            "Maximum number of marking threads exceeded");
1087     uint active_workers = MAX2(1U, parallel_marking_threads());
1088 
1089     CMRootRegionScanTask task(this);
1090     _parallel_workers->set_active_workers(active_workers);
1091     _parallel_workers->run_task(&task);
1092 
1093     double end_time_ms = os::elapsed_counter();
1094     log_info(gc)("Concurrent Root Region Scan (%.3fs, %.3fs) %.3fms",
1095                  TimeHelper::counter_to_seconds(scan_start),
1096                  TimeHelper::counter_to_seconds(end_time_ms),
1097                  TimeHelper::counter_to_millis(end_time_ms - scan_start));
1098 
1099     // It's possible that has_aborted() is true here without actually
1100     // aborting the survivor scan earlier. This is OK as it's
1101     // mainly used for sanity checking.
1102     root_regions()->scan_finished();
1103   }
1104 }
1105 
1106 void ConcurrentMark::markFromRoots() {
1107   // we might be tempted to assert that:
1108   // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1109   //        "inconsistent argument?");
1110   // However that wouldn't be right, because it's possible that
1111   // a safepoint is indeed in progress as a younger generation
1112   // stop-the-world GC happens even as we mark in this generation.
1113 
1114   _restart_for_overflow = false;
1115   force_overflow_conc()->init();
1116 
1117   // _g1h has _n_par_threads
1118   _parallel_marking_threads = calc_parallel_marking_threads();
1119   assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1120     "Maximum number of marking threads exceeded");
1121 
1122   uint active_workers = MAX2(1U, parallel_marking_threads());
1123   assert(active_workers > 0, "Should have been set");
1124 
1125   // Parallel task terminator is set in "set_concurrency_and_phase()"
1126   set_concurrency_and_phase(active_workers, true /* concurrent */);
1127 
1128   CMConcurrentMarkingTask markingTask(this, cmThread());
1129   _parallel_workers->set_active_workers(active_workers);
1130   _parallel_workers->run_task(&markingTask);
1131   print_stats();
1132 }
1133 
















1134 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1135   // world is stopped at this checkpoint
1136   assert(SafepointSynchronize::is_at_safepoint(),
1137          "world should be stopped");
1138 
1139   G1CollectedHeap* g1h = G1CollectedHeap::heap();
1140 
1141   // If a full collection has happened, we shouldn't do this.
1142   if (has_aborted()) {
1143     g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1144     return;
1145   }
1146 
1147   SvcGCMarker sgcm(SvcGCMarker::OTHER);
1148 
1149   if (VerifyDuringGC) {
1150     HandleMark hm;  // handle scope
1151     g1h->prepare_for_verify();
1152     Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)");

1153   }
1154   g1h->check_bitmaps("Remark Start");
1155 
1156   G1CollectorPolicy* g1p = g1h->g1_policy();
1157   g1p->record_concurrent_mark_remark_start();
1158 
1159   double start = os::elapsedTime();
1160 
1161   checkpointRootsFinalWork();
1162 
1163   double mark_work_end = os::elapsedTime();
1164 
1165   weakRefsWork(clear_all_soft_refs);
1166 
1167   if (has_overflown()) {
1168     // Oops.  We overflowed.  Restart concurrent marking.
1169     _restart_for_overflow = true;
1170     log_develop(gc)("Remark led to restart for overflow.");


1171 
1172     // Verify the heap w.r.t. the previous marking bitmap.
1173     if (VerifyDuringGC) {
1174       HandleMark hm;  // handle scope
1175       g1h->prepare_for_verify();
1176       Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (overflow)");

1177     }
1178 
1179     // Clear the marking state because we will be restarting
1180     // marking due to overflowing the global mark stack.
1181     reset_marking_state();
1182   } else {
1183     {
1184       GCTraceTime(Debug, gc) trace("GC aggregate-data", g1h->gc_timer_cm());
1185 
1186       // Aggregate the per-task counting data that we have accumulated
1187       // while marking.
1188       aggregate_count_data();
1189     }
1190 
1191     SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1192     // We're done with marking.
1193     // This is the end of  the marking cycle, we're expected all
1194     // threads to have SATB queues with active set to true.
1195     satb_mq_set.set_active_all_threads(false, /* new active value */
1196                                        true /* expected_active */);
1197 
1198     if (VerifyDuringGC) {
1199       HandleMark hm;  // handle scope
1200       g1h->prepare_for_verify();
1201       Universe::verify(VerifyOption_G1UseNextMarking, "During GC (after)");

1202     }
1203     g1h->check_bitmaps("Remark End");
1204     assert(!restart_for_overflow(), "sanity");
1205     // Completely reset the marking state since marking completed
1206     set_non_marking_state();
1207   }
1208 
1209   // Expand the marking stack, if we have to and if we can.
1210   if (_markStack.should_expand()) {
1211     _markStack.expand();
1212   }
1213 
1214   // Statistics
1215   double now = os::elapsedTime();
1216   _remark_mark_times.add((mark_work_end - start) * 1000.0);
1217   _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1218   _remark_times.add((now - start) * 1000.0);
1219 
1220   g1p->record_concurrent_mark_remark_end();
1221 


1700 
1701 };
1702 
1703 void ConcurrentMark::cleanup() {
1704   // world is stopped at this checkpoint
1705   assert(SafepointSynchronize::is_at_safepoint(),
1706          "world should be stopped");
1707   G1CollectedHeap* g1h = G1CollectedHeap::heap();
1708 
1709   // If a full collection has happened, we shouldn't do this.
1710   if (has_aborted()) {
1711     g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1712     return;
1713   }
1714 
1715   g1h->verify_region_sets_optional();
1716 
1717   if (VerifyDuringGC) {
1718     HandleMark hm;  // handle scope
1719     g1h->prepare_for_verify();
1720     Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (before)");

1721   }
1722   g1h->check_bitmaps("Cleanup Start");
1723 
1724   G1CollectorPolicy* g1p = g1h->g1_policy();
1725   g1p->record_concurrent_mark_cleanup_start();
1726 
1727   double start = os::elapsedTime();
1728 
1729   HeapRegionRemSet::reset_for_cleanup_tasks();
1730 
1731   // Do counting once more with the world stopped for good measure.
1732   G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
1733 
1734   g1h->workers()->run_task(&g1_par_count_task);
1735 
1736   if (VerifyDuringGC) {
1737     // Verify that the counting data accumulated during marking matches
1738     // that calculated by walking the marking bitmap.
1739 
1740     // Bitmaps to hold expected values


1742     BitMap expected_card_bm(_card_bm.size(), true);
1743 
1744     G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
1745                                                  &_region_bm,
1746                                                  &_card_bm,
1747                                                  &expected_region_bm,
1748                                                  &expected_card_bm);
1749 
1750     g1h->workers()->run_task(&g1_par_verify_task);
1751 
1752     guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
1753   }
1754 
1755   size_t start_used_bytes = g1h->used();
1756   g1h->collector_state()->set_mark_in_progress(false);
1757 
1758   double count_end = os::elapsedTime();
1759   double this_final_counting_time = (count_end - start);
1760   _total_counting_time += this_final_counting_time;
1761 
1762   if (Log<LOG_TAGS(gc, liveness)>::is_trace()) {
1763     G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1764     _g1h->heap_region_iterate(&cl);
1765   }
1766 
1767   // Install newly created mark bitMap as "prev".
1768   swapMarkBitMaps();
1769 
1770   g1h->reset_gc_time_stamp();
1771 
1772   uint n_workers = _g1h->workers()->active_workers();
1773 
1774   // Note end of marking in all heap regions.
1775   G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1776   g1h->workers()->run_task(&g1_par_note_end_task);
1777   g1h->check_gc_time_stamps();
1778 
1779   if (!cleanup_list_is_empty()) {
1780     // The cleanup list is not empty, so we'll have to process it
1781     // concurrently. Notify anyone else that might be wanting free
1782     // regions that there will be more free regions coming soon.
1783     g1h->set_free_regions_coming();


1786   // call below, since it affects the metric by which we sort the heap
1787   // regions.
1788   if (G1ScrubRemSets) {
1789     double rs_scrub_start = os::elapsedTime();
1790     G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers);
1791     g1h->workers()->run_task(&g1_par_scrub_rs_task);
1792 
1793     double rs_scrub_end = os::elapsedTime();
1794     double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
1795     _total_rs_scrub_time += this_rs_scrub_time;
1796   }
1797 
1798   // this will also free any regions totally full of garbage objects,
1799   // and sort the regions.
1800   g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1801 
1802   // Statistics.
1803   double end = os::elapsedTime();
1804   _cleanup_times.add((end - start) * 1000.0);
1805 




1806   // Clean up will have freed any regions completely full of garbage.
1807   // Update the soft reference policy with the new heap occupancy.
1808   Universe::update_heap_info_at_gc();
1809 
1810   if (VerifyDuringGC) {
1811     HandleMark hm;  // handle scope
1812     g1h->prepare_for_verify();
1813     Universe::verify(VerifyOption_G1UsePrevMarking, "During GC (after)");

1814   }
1815 
1816   g1h->check_bitmaps("Cleanup End");
1817 
1818   g1h->verify_region_sets_optional();
1819 
1820   // We need to make this be a "collection" so any collection pause that
1821   // races with it goes around and waits for completeCleanup to finish.
1822   g1h->increment_total_collections();
1823 
1824   // Clean out dead classes and update Metaspace sizes.
1825   if (ClassUnloadingWithConcurrentMark) {
1826     ClassLoaderDataGraph::purge();
1827   }
1828   MetaspaceGC::compute_new_size();
1829 
1830   // We reclaimed old regions so we should calculate the sizes to make
1831   // sure we update the old gen/space data.
1832   g1h->g1mm()->update_sizes();
1833   g1h->allocation_context_stats().update_after_mark();
1834 
1835   g1h->trace_heap_after_concurrent_cycle();
1836 }
1837 
1838 void ConcurrentMark::completeCleanup() {
1839   if (has_aborted()) return;
1840 
1841   G1CollectedHeap* g1h = G1CollectedHeap::heap();
1842 
1843   _cleanup_list.verify_optional();
1844   FreeRegionList tmp_free_list("Tmp Free List");
1845 
1846   log_develop(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "

1847                             "cleanup list has %u entries",
1848                             _cleanup_list.length());

1849 
1850   // No one else should be accessing the _cleanup_list at this point,
1851   // so it is not necessary to take any locks
1852   while (!_cleanup_list.is_empty()) {
1853     HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */);
1854     assert(hr != NULL, "Got NULL from a non-empty list");
1855     hr->par_clear();
1856     tmp_free_list.add_ordered(hr);
1857 
1858     // Instead of adding one region at a time to the secondary_free_list,
1859     // we accumulate them in the local list and move them a few at a
1860     // time. This also cuts down on the number of notify_all() calls
1861     // we do during this process. We'll also append the local list when
1862     // _cleanup_list is empty (which means we just removed the last
1863     // region from the _cleanup_list).
1864     if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1865         _cleanup_list.is_empty()) {
1866       log_develop(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "

1867                                 "appending %u entries to the secondary_free_list, "
1868                                 "cleanup list still has %u entries",
1869                                 tmp_free_list.length(),
1870                                 _cleanup_list.length());

1871 
1872       {
1873         MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1874         g1h->secondary_free_list_add(&tmp_free_list);
1875         SecondaryFreeList_lock->notify_all();
1876       }
1877 #ifndef PRODUCT
1878       if (G1StressConcRegionFreeing) {
1879         for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1880           os::sleep(Thread::current(), (jlong) 1, false);
1881         }
1882       }
1883 #endif
1884     }
1885   }
1886   assert(tmp_free_list.is_empty(), "post-condition");
1887 }
1888 
1889 // Supporting Object and Oop closures for reference discovery
1890 // and processing in during marking


2107     // Skip processing the discovered references if we have
2108     // overflown the global marking stack. Reference objects
2109     // only get discovered once so it is OK to not
2110     // de-populate the discovered reference lists. We could have,
2111     // but the only benefit would be that, when marking restarts,
2112     // less reference objects are discovered.
2113     return;
2114   }
2115 
2116   ResourceMark rm;
2117   HandleMark   hm;
2118 
2119   G1CollectedHeap* g1h = G1CollectedHeap::heap();
2120 
2121   // Is alive closure.
2122   G1CMIsAliveClosure g1_is_alive(g1h);
2123 
2124   // Inner scope to exclude the cleaning of the string and symbol
2125   // tables from the displayed time.
2126   {
2127     GCTraceTime(Debug, gc) trace("GC ref-proc", g1h->gc_timer_cm());
2128 
2129     ReferenceProcessor* rp = g1h->ref_processor_cm();
2130 
2131     // See the comment in G1CollectedHeap::ref_processing_init()
2132     // about how reference processing currently works in G1.
2133 
2134     // Set the soft reference policy
2135     rp->setup_policy(clear_all_soft_refs);
2136     assert(_markStack.isEmpty(), "mark stack should be empty");
2137 
2138     // Instances of the 'Keep Alive' and 'Complete GC' closures used
2139     // in serial reference processing. Note these closures are also
2140     // used for serially processing (by the the current thread) the
2141     // JNI references during parallel reference processing.
2142     //
2143     // These closures do not need to synchronize with the worker
2144     // threads involved in parallel reference processing as these
2145     // instances are executed serially by the current thread (e.g.
2146     // reference processing is not multi-threaded and is thus
2147     // performed by the current thread instead of a gang worker).


2197       set_has_overflown();
2198     }
2199 
2200     assert(rp->num_q() == active_workers, "why not");
2201 
2202     rp->enqueue_discovered_references(executor);
2203 
2204     rp->verify_no_references_recorded();
2205     assert(!rp->discovery_enabled(), "Post condition");
2206   }
2207 
2208   if (has_overflown()) {
2209     // We can not trust g1_is_alive if the marking stack overflowed
2210     return;
2211   }
2212 
2213   assert(_markStack.isEmpty(), "Marking should have completed");
2214 
2215   // Unload Klasses, String, Symbols, Code Cache, etc.
2216   {
2217     GCTraceTime(Debug, gc) trace("Unloading", g1h->gc_timer_cm());
2218 
2219     if (ClassUnloadingWithConcurrentMark) {
2220       bool purged_classes;
2221 
2222       {
2223         GCTraceTime(Trace, gc) trace("System Dictionary Unloading", g1h->gc_timer_cm());
2224         purged_classes = SystemDictionary::do_unloading(&g1_is_alive, false /* Defer klass cleaning */);
2225       }
2226 
2227       {
2228         GCTraceTime(Trace, gc) trace("Parallel Unloading", g1h->gc_timer_cm());
2229         weakRefsWorkParallelPart(&g1_is_alive, purged_classes);
2230       }
2231     }
2232 
2233     if (G1StringDedup::is_enabled()) {
2234       GCTraceTime(Trace, gc) trace("String Deduplication Unlink", g1h->gc_timer_cm());
2235       G1StringDedup::unlink(&g1_is_alive);
2236     }
2237   }
2238 }
2239 
2240 void ConcurrentMark::swapMarkBitMaps() {
2241   CMBitMapRO* temp = _prevMarkBitMap;
2242   _prevMarkBitMap  = (CMBitMapRO*)_nextMarkBitMap;
2243   _nextMarkBitMap  = (CMBitMap*)  temp;
2244 }
2245 
2246 // Closure for marking entries in SATB buffers.
2247 class CMSATBBufferClosure : public SATBBufferClosure {
2248 private:
2249   CMTask* _task;
2250   G1CollectedHeap* _g1h;
2251 
2252   // This is very similar to CMTask::deal_with_reference, but with
2253   // more relaxed requirements for the argument, so this must be more
2254   // circumspect about treating the argument as an object.


2335                               true         /* do_termination       */,
2336                               false        /* is_serial            */);
2337       } while (task->has_aborted() && !_cm->has_overflown());
2338       // If we overflow, then we do not want to restart. We instead
2339       // want to abort remark and do concurrent marking again.
2340       task->record_end_time();
2341     }
2342   }
2343 
2344   CMRemarkTask(ConcurrentMark* cm, uint active_workers) :
2345     AbstractGangTask("Par Remark"), _cm(cm) {
2346     _cm->terminator()->reset_for_reuse(active_workers);
2347   }
2348 };
2349 
2350 void ConcurrentMark::checkpointRootsFinalWork() {
2351   ResourceMark rm;
2352   HandleMark   hm;
2353   G1CollectedHeap* g1h = G1CollectedHeap::heap();
2354 
2355   GCTraceTime(Debug, gc) trace("Finalize Marking", g1h->gc_timer_cm());
2356 
2357   g1h->ensure_parsability(false);
2358 
2359   // this is remark, so we'll use up all active threads
2360   uint active_workers = g1h->workers()->active_workers();
2361   set_concurrency_and_phase(active_workers, false /* concurrent */);
2362   // Leave _parallel_marking_threads at it's
2363   // value originally calculated in the ConcurrentMark
2364   // constructor and pass values of the active workers
2365   // through the gang in the task.
2366 
2367   {
2368     StrongRootsScope srs(active_workers);
2369 
2370     CMRemarkTask remarkTask(this, active_workers);
2371     // We will start all available threads, even if we decide that the
2372     // active_workers will be fewer. The extra ones will just bail out
2373     // immediately.
2374     g1h->workers()->run_task(&remarkTask);
2375   }


2652   // Clear the global region bitmap - it will be filled as part
2653   // of the final counting task.
2654   _region_bm.clear();
2655 
2656   uint max_regions = _g1h->max_regions();
2657   assert(_max_worker_id > 0, "uninitialized");
2658 
2659   for (uint i = 0; i < _max_worker_id; i += 1) {
2660     BitMap* task_card_bm = count_card_bitmap_for(i);
2661     size_t* marked_bytes_array = count_marked_bytes_array_for(i);
2662 
2663     assert(task_card_bm->size() == _card_bm.size(), "size mismatch");
2664     assert(marked_bytes_array != NULL, "uninitialized");
2665 
2666     memset(marked_bytes_array, 0, (size_t) max_regions * sizeof(size_t));
2667     task_card_bm->clear();
2668   }
2669 }
2670 
2671 void ConcurrentMark::print_stats() {
2672   LogHandle(gc, stats) log;
2673   if (!log.is_debug()) {
2674     return;
2675   }
2676   log.debug("---------------------------------------------------------------------");
2677   for (size_t i = 0; i < _active_tasks; ++i) {
2678     _tasks[i]->print_stats();
2679     log.debug("---------------------------------------------------------------------");

2680   }
2681 }
2682 
2683 // abandon current marking iteration due to a Full GC
2684 void ConcurrentMark::abort() {
2685   if (!cmThread()->during_cycle() || _has_aborted) {
2686     // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
2687     return;
2688   }
2689 
2690   // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
2691   // concurrent bitmap clearing.
2692   _nextMarkBitMap->clearAll();
2693 
2694   // Note we cannot clear the previous marking bitmap here
2695   // since VerifyDuringGC verifies the objects marked during
2696   // a full GC against the previous bitmap.
2697 
2698   // Clear the liveness counting data
2699   clear_all_count_data();


2703     _tasks[i]->clear_region_fields();
2704   }
2705   _first_overflow_barrier_sync.abort();
2706   _second_overflow_barrier_sync.abort();
2707   _has_aborted = true;
2708 
2709   SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2710   satb_mq_set.abandon_partial_marking();
2711   // This can be called either during or outside marking, we'll read
2712   // the expected_active value from the SATB queue set.
2713   satb_mq_set.set_active_all_threads(
2714                                  false, /* new active value */
2715                                  satb_mq_set.is_active() /* expected_active */);
2716 
2717   _g1h->trace_heap_after_concurrent_cycle();
2718   _g1h->register_concurrent_cycle_end();
2719 }
2720 
2721 static void print_ms_time_info(const char* prefix, const char* name,
2722                                NumberSeq& ns) {
2723   log_trace(gc, marking, stats, exit)("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2724                                       prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2725   if (ns.num() > 0) {
2726     log_trace(gc, marking, stats, exit)("%s         [std. dev = %8.2f ms, max = %8.2f ms]",
2727                                         prefix, ns.sd(), ns.maximum());
2728   }
2729 }
2730 
2731 void ConcurrentMark::print_summary_info() {
2732   LogHandle(gc, marking, stats, exit) log;
2733   if (!log.is_trace()) {
2734     return;
2735   }
2736 
2737   log.trace(" Concurrent marking:");
2738   print_ms_time_info("  ", "init marks", _init_times);
2739   print_ms_time_info("  ", "remarks", _remark_times);
2740   {
2741     print_ms_time_info("     ", "final marks", _remark_mark_times);
2742     print_ms_time_info("     ", "weak refs", _remark_weak_ref_times);
2743 
2744   }
2745   print_ms_time_info("  ", "cleanups", _cleanup_times);
2746   log.trace("    Final counting total time = %8.2f s (avg = %8.2f ms).",
2747             _total_counting_time, (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / (double)_cleanup_times.num() : 0.0));



2748   if (G1ScrubRemSets) {
2749     log.trace("    RS scrub total time = %8.2f s (avg = %8.2f ms).",
2750               _total_rs_scrub_time, (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2751   }
2752   log.trace("  Total stop_world time = %8.2f s.",
2753             (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2754   log.trace("  Total concurrent time = %8.2f s (%8.2f s marking).",
2755             cmThread()->vtime_accum(), cmThread()->vtime_mark_accum());






2756 }
2757 
2758 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2759   _parallel_workers->print_worker_threads_on(st);
2760 }
2761 
2762 void ConcurrentMark::print_on_error(outputStream* st) const {
2763   st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2764       p2i(_prevMarkBitMap), p2i(_nextMarkBitMap));
2765   _prevMarkBitMap->print_on_error(st, " Prev Bits: ");
2766   _nextMarkBitMap->print_on_error(st, " Next Bits: ");
2767 }
2768 
2769 // We take a break if someone is trying to stop the world.
2770 bool ConcurrentMark::do_yield_check(uint worker_id) {
2771   if (SuspendibleThreadSet::should_yield()) {
2772     if (worker_id == 0) {
2773       _g1h->g1_policy()->record_concurrent_pause();
2774     }
2775     SuspendibleThreadSet::yield();


3115 
3116   // This keeps claiming and applying the closure to completed buffers
3117   // until we run out of buffers or we need to abort.
3118   while (!has_aborted() &&
3119          satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
3120     regular_clock_call();
3121   }
3122 
3123   _draining_satb_buffers = false;
3124 
3125   assert(has_aborted() ||
3126          concurrent() ||
3127          satb_mq_set.completed_buffers_num() == 0, "invariant");
3128 
3129   // again, this was a potentially expensive operation, decrease the
3130   // limits to get the regular clock call early
3131   decrease_limits();
3132 }
3133 
3134 void CMTask::print_stats() {
3135   log_debug(gc, stats)("Marking Stats, task = %u, calls = %d",
3136                        _worker_id, _calls);
3137   log_debug(gc, stats)("  Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3138                        _elapsed_time_ms, _termination_time_ms);
3139   log_debug(gc, stats)("  Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3140                        _step_times_ms.num(), _step_times_ms.avg(),
3141                        _step_times_ms.sd());
3142   log_debug(gc, stats)("                    max = %1.2lfms, total = %1.2lfms",
3143                        _step_times_ms.maximum(), _step_times_ms.sum());
3144 }
3145 
3146 bool ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, oop& obj) {
3147   return _task_queues->steal(worker_id, hash_seed, obj);
3148 }
3149 
3150 /*****************************************************************************
3151 
3152     The do_marking_step(time_target_ms, ...) method is the building
3153     block of the parallel marking framework. It can be called in parallel
3154     with other invocations of do_marking_step() on different tasks
3155     (but only one per task, obviously) and concurrently with the
3156     mutator threads, or during remark, hence it eliminates the need
3157     for two versions of the code. When called during remark, it will
3158     pick up from where the task left off during the concurrent marking
3159     phase. Interestingly, tasks are also claimable during evacuation
3160     pauses too, since do_marking_step() ensures that it aborts before
3161     it needs to yield.
3162 


3632 #define G1PPRL_ADDR_BASE_H_FORMAT  " %37s"
3633 #else // _LP64
3634 #define G1PPRL_ADDR_BASE_H_FORMAT  " %21s"
3635 #endif // _LP64
3636 
3637 // For per-region info
3638 #define G1PPRL_TYPE_FORMAT            "   %-4s"
3639 #define G1PPRL_TYPE_H_FORMAT          "   %4s"
3640 #define G1PPRL_BYTE_FORMAT            "  " SIZE_FORMAT_W(9)
3641 #define G1PPRL_BYTE_H_FORMAT          "  %9s"
3642 #define G1PPRL_DOUBLE_FORMAT          "  %14.1f"
3643 #define G1PPRL_DOUBLE_H_FORMAT        "  %14s"
3644 
3645 // For summary info
3646 #define G1PPRL_SUM_ADDR_FORMAT(tag)    "  " tag ":" G1PPRL_ADDR_BASE_FORMAT
3647 #define G1PPRL_SUM_BYTE_FORMAT(tag)    "  " tag ": " SIZE_FORMAT
3648 #define G1PPRL_SUM_MB_FORMAT(tag)      "  " tag ": %1.2f MB"
3649 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
3650 
3651 G1PrintRegionLivenessInfoClosure::
3652 G1PrintRegionLivenessInfoClosure(const char* phase_name)
3653   : _total_used_bytes(0), _total_capacity_bytes(0),

3654     _total_prev_live_bytes(0), _total_next_live_bytes(0),
3655     _hum_used_bytes(0), _hum_capacity_bytes(0),
3656     _hum_prev_live_bytes(0), _hum_next_live_bytes(0),
3657     _total_remset_bytes(0), _total_strong_code_roots_bytes(0) {
3658   G1CollectedHeap* g1h = G1CollectedHeap::heap();
3659   MemRegion g1_reserved = g1h->g1_reserved();
3660   double now = os::elapsedTime();
3661 
3662   // Print the header of the output.
3663   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
3664   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" HEAP"

3665                           G1PPRL_SUM_ADDR_FORMAT("reserved")
3666                           G1PPRL_SUM_BYTE_FORMAT("region-size"),
3667                           p2i(g1_reserved.start()), p2i(g1_reserved.end()),
3668                           HeapRegion::GrainBytes);
3669   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
3670   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3671                           G1PPRL_TYPE_H_FORMAT
3672                           G1PPRL_ADDR_BASE_H_FORMAT
3673                           G1PPRL_BYTE_H_FORMAT
3674                           G1PPRL_BYTE_H_FORMAT
3675                           G1PPRL_BYTE_H_FORMAT
3676                           G1PPRL_DOUBLE_H_FORMAT
3677                           G1PPRL_BYTE_H_FORMAT
3678                           G1PPRL_BYTE_H_FORMAT,
3679                           "type", "address-range",
3680                           "used", "prev-live", "next-live", "gc-eff",
3681                           "remset", "code-roots");
3682   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3683                           G1PPRL_TYPE_H_FORMAT
3684                           G1PPRL_ADDR_BASE_H_FORMAT
3685                           G1PPRL_BYTE_H_FORMAT
3686                           G1PPRL_BYTE_H_FORMAT
3687                           G1PPRL_BYTE_H_FORMAT
3688                           G1PPRL_DOUBLE_H_FORMAT
3689                           G1PPRL_BYTE_H_FORMAT
3690                           G1PPRL_BYTE_H_FORMAT,
3691                           "", "",
3692                           "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)",
3693                           "(bytes)", "(bytes)");
3694 }
3695 
3696 // It takes as a parameter a reference to one of the _hum_* fields, it
3697 // deduces the corresponding value for a region in a humongous region
3698 // series (either the region size, or what's left if the _hum_* field
3699 // is < the region size), and updates the _hum_* field accordingly.
3700 size_t G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* hum_bytes) {
3701   size_t bytes = 0;
3702   // The > 0 check is to deal with the prev and next live bytes which


3744     _hum_used_bytes      = used_bytes;
3745     _hum_prev_live_bytes = prev_live_bytes;
3746     _hum_next_live_bytes = next_live_bytes;
3747     get_hum_bytes(&used_bytes, &capacity_bytes,
3748                   &prev_live_bytes, &next_live_bytes);
3749     end = bottom + HeapRegion::GrainWords;
3750   } else if (r->is_continues_humongous()) {
3751     get_hum_bytes(&used_bytes, &capacity_bytes,
3752                   &prev_live_bytes, &next_live_bytes);
3753     assert(end == bottom + HeapRegion::GrainWords, "invariant");
3754   }
3755 
3756   _total_used_bytes      += used_bytes;
3757   _total_capacity_bytes  += capacity_bytes;
3758   _total_prev_live_bytes += prev_live_bytes;
3759   _total_next_live_bytes += next_live_bytes;
3760   _total_remset_bytes    += remset_bytes;
3761   _total_strong_code_roots_bytes += strong_code_roots_bytes;
3762 
3763   // Print a line for this particular region.
3764   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3765                           G1PPRL_TYPE_FORMAT
3766                           G1PPRL_ADDR_BASE_FORMAT
3767                           G1PPRL_BYTE_FORMAT
3768                           G1PPRL_BYTE_FORMAT
3769                           G1PPRL_BYTE_FORMAT
3770                           G1PPRL_DOUBLE_FORMAT
3771                           G1PPRL_BYTE_FORMAT
3772                           G1PPRL_BYTE_FORMAT,
3773                           type, p2i(bottom), p2i(end),
3774                           used_bytes, prev_live_bytes, next_live_bytes, gc_eff,
3775                           remset_bytes, strong_code_roots_bytes);
3776 
3777   return false;
3778 }
3779 
3780 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
3781   // add static memory usages to remembered set sizes
3782   _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
3783   // Print the footer of the output.
3784   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
3785   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
3786                          " SUMMARY"
3787                          G1PPRL_SUM_MB_FORMAT("capacity")
3788                          G1PPRL_SUM_MB_PERC_FORMAT("used")
3789                          G1PPRL_SUM_MB_PERC_FORMAT("prev-live")
3790                          G1PPRL_SUM_MB_PERC_FORMAT("next-live")
3791                          G1PPRL_SUM_MB_FORMAT("remset")
3792                          G1PPRL_SUM_MB_FORMAT("code-roots"),
3793                          bytes_to_mb(_total_capacity_bytes),
3794                          bytes_to_mb(_total_used_bytes),
3795                          perc(_total_used_bytes, _total_capacity_bytes),
3796                          bytes_to_mb(_total_prev_live_bytes),
3797                          perc(_total_prev_live_bytes, _total_capacity_bytes),
3798                          bytes_to_mb(_total_next_live_bytes),
3799                          perc(_total_next_live_bytes, _total_capacity_bytes),
3800                          bytes_to_mb(_total_remset_bytes),
3801                          bytes_to_mb(_total_strong_code_roots_bytes));

3802 }
< prev index next >