1923 g1h->verify_region_sets_optional();
1924
1925 if (VerifyDuringGC) {
1926 HandleMark hm; // handle scope
1927 g1h->prepare_for_verify();
1928 Universe::verify(VerifyOption_G1UsePrevMarking,
1929 " VerifyDuringGC:(before)");
1930 }
1931 g1h->check_bitmaps("Cleanup Start");
1932
1933 G1CollectorPolicy* g1p = g1h->g1_policy();
1934 g1p->record_concurrent_mark_cleanup_start();
1935
1936 double start = os::elapsedTime();
1937
1938 HeapRegionRemSet::reset_for_cleanup_tasks();
1939
1940 // Do counting once more with the world stopped for good measure.
1941 G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
1942
1943 g1h->set_par_threads();
1944 uint n_workers = _g1h->workers()->active_workers();
1945 g1h->workers()->run_task(&g1_par_count_task);
1946 // Done with the parallel phase so reset to 0.
1947 g1h->set_par_threads(0);
1948
1949 if (VerifyDuringGC) {
1950 // Verify that the counting data accumulated during marking matches
1951 // that calculated by walking the marking bitmap.
1952
1953 // Bitmaps to hold expected values
1954 BitMap expected_region_bm(_region_bm.size(), true);
1955 BitMap expected_card_bm(_card_bm.size(), true);
1956
1957 G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
1958 &_region_bm,
1959 &_card_bm,
1960 &expected_region_bm,
1961 &expected_card_bm);
1962
1963 g1h->set_par_threads((int)n_workers);
1964 g1h->workers()->run_task(&g1_par_verify_task);
1965 // Done with the parallel phase so reset to 0.
1966 g1h->set_par_threads(0);
1967
1968 guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
1969 }
1970
1971 size_t start_used_bytes = g1h->used();
1972 g1h->set_marking_complete();
1973
1974 double count_end = os::elapsedTime();
1975 double this_final_counting_time = (count_end - start);
1976 _total_counting_time += this_final_counting_time;
1977
1978 if (G1PrintRegionLivenessInfo) {
1979 G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking");
1980 _g1h->heap_region_iterate(&cl);
1981 }
1982
1983 // Install newly created mark bitMap as "prev".
1984 swapMarkBitMaps();
1985
1986 g1h->reset_gc_time_stamp();
1987
1988 // Note end of marking in all heap regions.
1989 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1990 g1h->set_par_threads((int)n_workers);
1991 g1h->workers()->run_task(&g1_par_note_end_task);
1992 g1h->set_par_threads(0);
1993 g1h->check_gc_time_stamps();
1994
1995 if (!cleanup_list_is_empty()) {
1996 // The cleanup list is not empty, so we'll have to process it
1997 // concurrently. Notify anyone else that might be wanting free
1998 // regions that there will be more free regions coming soon.
1999 g1h->set_free_regions_coming();
2000 }
2001
2002 // call below, since it affects the metric by which we sort the heap
2003 // regions.
2004 if (G1ScrubRemSets) {
2005 double rs_scrub_start = os::elapsedTime();
2006 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers);
2007 g1h->set_par_threads((int)n_workers);
2008 g1h->workers()->run_task(&g1_par_scrub_rs_task);
2009 g1h->set_par_threads(0);
2010
2011 double rs_scrub_end = os::elapsedTime();
2012 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
2013 _total_rs_scrub_time += this_rs_scrub_time;
2014 }
2015
2016 // this will also free any regions totally full of garbage objects,
2017 // and sort the regions.
2018 g1h->g1_policy()->record_concurrent_mark_cleanup_end((int)n_workers);
2019
2020 // Statistics.
2021 double end = os::elapsedTime();
2022 _cleanup_times.add((end - start) * 1000.0);
2023
2024 if (G1Log::fine()) {
2025 g1h->g1_policy()->print_heap_transition(start_used_bytes);
2026 }
2027
2028 // Clean up will have freed any regions completely full of garbage.
2029 // Update the soft reference policy with the new heap occupancy.
2290 CMTask* task = _cm->task(worker_id);
2291 G1CMIsAliveClosure g1_is_alive(_g1h);
2292 G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */);
2293 G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */);
2294
2295 _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain);
2296 }
2297 };
2298
2299 void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) {
2300 assert(_workers != NULL, "Need parallel worker threads.");
2301 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
2302
2303 G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm);
2304
2305 // We need to reset the concurrency level before each
2306 // proxy task execution, so that the termination protocol
2307 // and overflow handling in CMTask::do_marking_step() knows
2308 // how many workers to wait for.
2309 _cm->set_concurrency(_active_workers);
2310 _g1h->set_par_threads(_active_workers);
2311 _workers->run_task(&proc_task_proxy);
2312 _g1h->set_par_threads(0);
2313 }
2314
2315 class G1CMRefEnqueueTaskProxy: public AbstractGangTask {
2316 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
2317 EnqueueTask& _enq_task;
2318
2319 public:
2320 G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) :
2321 AbstractGangTask("Enqueue reference objects in parallel"),
2322 _enq_task(enq_task) { }
2323
2324 virtual void work(uint worker_id) {
2325 _enq_task.work(worker_id);
2326 }
2327 };
2328
2329 void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
2330 assert(_workers != NULL, "Need parallel worker threads.");
2331 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
2332
2333 G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task);
2334
2335 // Not strictly necessary but...
2336 //
2337 // We need to reset the concurrency level before each
2338 // proxy task execution, so that the termination protocol
2339 // and overflow handling in CMTask::do_marking_step() knows
2340 // how many workers to wait for.
2341 _cm->set_concurrency(_active_workers);
2342 _g1h->set_par_threads(_active_workers);
2343 _workers->run_task(&enq_task_proxy);
2344 _g1h->set_par_threads(0);
2345 }
2346
2347 void ConcurrentMark::weakRefsWorkParallelPart(BoolObjectClosure* is_alive, bool purged_classes) {
2348 G1CollectedHeap::heap()->parallel_cleaning(is_alive, true, true, purged_classes);
2349 }
2350
2351 void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {
2352 if (has_overflown()) {
2353 // Skip processing the discovered references if we have
2354 // overflown the global marking stack. Reference objects
2355 // only get discovered once so it is OK to not
2356 // de-populate the discovered reference lists. We could have,
2357 // but the only benefit would be that, when marking restarts,
2358 // less reference objects are discovered.
2359 return;
2360 }
2361
2362 ResourceMark rm;
2363 HandleMark hm;
2364
2606 // this is remark, so we'll use up all active threads
2607 uint active_workers = g1h->workers()->active_workers();
2608 if (active_workers == 0) {
2609 assert(active_workers > 0, "Should have been set earlier");
2610 active_workers = (uint) ParallelGCThreads;
2611 g1h->workers()->set_active_workers(active_workers);
2612 }
2613 set_concurrency_and_phase(active_workers, false /* concurrent */);
2614 // Leave _parallel_marking_threads at it's
2615 // value originally calculated in the ConcurrentMark
2616 // constructor and pass values of the active workers
2617 // through the gang in the task.
2618
2619 {
2620 StrongRootsScope srs(active_workers);
2621
2622 CMRemarkTask remarkTask(this, active_workers);
2623 // We will start all available threads, even if we decide that the
2624 // active_workers will be fewer. The extra ones will just bail out
2625 // immediately.
2626 g1h->set_par_threads(active_workers);
2627 g1h->workers()->run_task(&remarkTask);
2628 g1h->set_par_threads(0);
2629 }
2630
2631 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2632 guarantee(has_overflown() ||
2633 satb_mq_set.completed_buffers_num() == 0,
2634 err_msg("Invariant: has_overflown = %s, num buffers = %d",
2635 BOOL_TO_STR(has_overflown()),
2636 satb_mq_set.completed_buffers_num()));
2637
2638 print_stats();
2639 }
2640
2641 void ConcurrentMark::clearRangePrevBitmap(MemRegion mr) {
2642 // Note we are overriding the read-only view of the prev map here, via
2643 // the cast.
2644 ((CMBitMap*)_prevMarkBitMap)->clearRange(mr);
2645 }
2646
2647 void ConcurrentMark::clearRangeNextBitmap(MemRegion mr) {
2648 _nextMarkBitMap->clearRange(mr);
2982 _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm),
2983 _max_worker_id(max_worker_id),
2984 _active_workers(n_workers),
2985 _hrclaimer(_active_workers) {
2986 }
2987
2988 void work(uint worker_id) {
2989 AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_worker_id);
2990
2991 _g1h->heap_region_par_iterate(&cl, worker_id, &_hrclaimer);
2992 }
2993 };
2994
2995
2996 void ConcurrentMark::aggregate_count_data() {
2997 uint n_workers = _g1h->workers()->active_workers();
2998
2999 G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm,
3000 _max_worker_id, n_workers);
3001
3002 _g1h->set_par_threads(n_workers);
3003 _g1h->workers()->run_task(&g1_par_agg_task);
3004 _g1h->set_par_threads(0);
3005 }
3006
3007 // Clear the per-worker arrays used to store the per-region counting data
3008 void ConcurrentMark::clear_all_count_data() {
3009 // Clear the global card bitmap - it will be filled during
3010 // liveness count aggregation (during remark) and the
3011 // final counting task.
3012 _card_bm.clear();
3013
3014 // Clear the global region bitmap - it will be filled as part
3015 // of the final counting task.
3016 _region_bm.clear();
3017
3018 uint max_regions = _g1h->max_regions();
3019 assert(_max_worker_id > 0, "uninitialized");
3020
3021 for (uint i = 0; i < _max_worker_id; i += 1) {
3022 BitMap* task_card_bm = count_card_bitmap_for(i);
3023 size_t* marked_bytes_array = count_marked_bytes_array_for(i);
3024
|
1923 g1h->verify_region_sets_optional();
1924
1925 if (VerifyDuringGC) {
1926 HandleMark hm; // handle scope
1927 g1h->prepare_for_verify();
1928 Universe::verify(VerifyOption_G1UsePrevMarking,
1929 " VerifyDuringGC:(before)");
1930 }
1931 g1h->check_bitmaps("Cleanup Start");
1932
1933 G1CollectorPolicy* g1p = g1h->g1_policy();
1934 g1p->record_concurrent_mark_cleanup_start();
1935
1936 double start = os::elapsedTime();
1937
1938 HeapRegionRemSet::reset_for_cleanup_tasks();
1939
1940 // Do counting once more with the world stopped for good measure.
1941 G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm);
1942
1943 g1h->workers()->run_task(&g1_par_count_task);
1944
1945 if (VerifyDuringGC) {
1946 // Verify that the counting data accumulated during marking matches
1947 // that calculated by walking the marking bitmap.
1948
1949 // Bitmaps to hold expected values
1950 BitMap expected_region_bm(_region_bm.size(), true);
1951 BitMap expected_card_bm(_card_bm.size(), true);
1952
1953 G1ParVerifyFinalCountTask g1_par_verify_task(g1h,
1954 &_region_bm,
1955 &_card_bm,
1956 &expected_region_bm,
1957 &expected_card_bm);
1958
1959 g1h->workers()->run_task(&g1_par_verify_task);
1960
1961 guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures");
1962 }
1963
1964 size_t start_used_bytes = g1h->used();
1965 g1h->set_marking_complete();
1966
1967 double count_end = os::elapsedTime();
1968 double this_final_counting_time = (count_end - start);
1969 _total_counting_time += this_final_counting_time;
1970
1971 if (G1PrintRegionLivenessInfo) {
1972 G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking");
1973 _g1h->heap_region_iterate(&cl);
1974 }
1975
1976 // Install newly created mark bitMap as "prev".
1977 swapMarkBitMaps();
1978
1979 g1h->reset_gc_time_stamp();
1980
1981 uint n_workers = _g1h->workers()->active_workers();
1982
1983 // Note end of marking in all heap regions.
1984 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1985 g1h->workers()->run_task(&g1_par_note_end_task);
1986 g1h->check_gc_time_stamps();
1987
1988 if (!cleanup_list_is_empty()) {
1989 // The cleanup list is not empty, so we'll have to process it
1990 // concurrently. Notify anyone else that might be wanting free
1991 // regions that there will be more free regions coming soon.
1992 g1h->set_free_regions_coming();
1993 }
1994
1995 // call below, since it affects the metric by which we sort the heap
1996 // regions.
1997 if (G1ScrubRemSets) {
1998 double rs_scrub_start = os::elapsedTime();
1999 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers);
2000 g1h->workers()->run_task(&g1_par_scrub_rs_task);
2001
2002 double rs_scrub_end = os::elapsedTime();
2003 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
2004 _total_rs_scrub_time += this_rs_scrub_time;
2005 }
2006
2007 // this will also free any regions totally full of garbage objects,
2008 // and sort the regions.
2009 g1h->g1_policy()->record_concurrent_mark_cleanup_end((int)n_workers);
2010
2011 // Statistics.
2012 double end = os::elapsedTime();
2013 _cleanup_times.add((end - start) * 1000.0);
2014
2015 if (G1Log::fine()) {
2016 g1h->g1_policy()->print_heap_transition(start_used_bytes);
2017 }
2018
2019 // Clean up will have freed any regions completely full of garbage.
2020 // Update the soft reference policy with the new heap occupancy.
2281 CMTask* task = _cm->task(worker_id);
2282 G1CMIsAliveClosure g1_is_alive(_g1h);
2283 G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */);
2284 G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */);
2285
2286 _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain);
2287 }
2288 };
2289
2290 void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) {
2291 assert(_workers != NULL, "Need parallel worker threads.");
2292 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
2293
2294 G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm);
2295
2296 // We need to reset the concurrency level before each
2297 // proxy task execution, so that the termination protocol
2298 // and overflow handling in CMTask::do_marking_step() knows
2299 // how many workers to wait for.
2300 _cm->set_concurrency(_active_workers);
2301 _workers->run_task(&proc_task_proxy);
2302 }
2303
2304 class G1CMRefEnqueueTaskProxy: public AbstractGangTask {
2305 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
2306 EnqueueTask& _enq_task;
2307
2308 public:
2309 G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) :
2310 AbstractGangTask("Enqueue reference objects in parallel"),
2311 _enq_task(enq_task) { }
2312
2313 virtual void work(uint worker_id) {
2314 _enq_task.work(worker_id);
2315 }
2316 };
2317
2318 void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
2319 assert(_workers != NULL, "Need parallel worker threads.");
2320 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
2321
2322 G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task);
2323
2324 // Not strictly necessary but...
2325 //
2326 // We need to reset the concurrency level before each
2327 // proxy task execution, so that the termination protocol
2328 // and overflow handling in CMTask::do_marking_step() knows
2329 // how many workers to wait for.
2330 _cm->set_concurrency(_active_workers);
2331 _workers->run_task(&enq_task_proxy);
2332 }
2333
2334 void ConcurrentMark::weakRefsWorkParallelPart(BoolObjectClosure* is_alive, bool purged_classes) {
2335 G1CollectedHeap::heap()->parallel_cleaning(is_alive, true, true, purged_classes);
2336 }
2337
2338 void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {
2339 if (has_overflown()) {
2340 // Skip processing the discovered references if we have
2341 // overflown the global marking stack. Reference objects
2342 // only get discovered once so it is OK to not
2343 // de-populate the discovered reference lists. We could have,
2344 // but the only benefit would be that, when marking restarts,
2345 // less reference objects are discovered.
2346 return;
2347 }
2348
2349 ResourceMark rm;
2350 HandleMark hm;
2351
2593 // this is remark, so we'll use up all active threads
2594 uint active_workers = g1h->workers()->active_workers();
2595 if (active_workers == 0) {
2596 assert(active_workers > 0, "Should have been set earlier");
2597 active_workers = (uint) ParallelGCThreads;
2598 g1h->workers()->set_active_workers(active_workers);
2599 }
2600 set_concurrency_and_phase(active_workers, false /* concurrent */);
2601 // Leave _parallel_marking_threads at it's
2602 // value originally calculated in the ConcurrentMark
2603 // constructor and pass values of the active workers
2604 // through the gang in the task.
2605
2606 {
2607 StrongRootsScope srs(active_workers);
2608
2609 CMRemarkTask remarkTask(this, active_workers);
2610 // We will start all available threads, even if we decide that the
2611 // active_workers will be fewer. The extra ones will just bail out
2612 // immediately.
2613 g1h->workers()->run_task(&remarkTask);
2614 }
2615
2616 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2617 guarantee(has_overflown() ||
2618 satb_mq_set.completed_buffers_num() == 0,
2619 err_msg("Invariant: has_overflown = %s, num buffers = %d",
2620 BOOL_TO_STR(has_overflown()),
2621 satb_mq_set.completed_buffers_num()));
2622
2623 print_stats();
2624 }
2625
2626 void ConcurrentMark::clearRangePrevBitmap(MemRegion mr) {
2627 // Note we are overriding the read-only view of the prev map here, via
2628 // the cast.
2629 ((CMBitMap*)_prevMarkBitMap)->clearRange(mr);
2630 }
2631
2632 void ConcurrentMark::clearRangeNextBitmap(MemRegion mr) {
2633 _nextMarkBitMap->clearRange(mr);
2967 _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm),
2968 _max_worker_id(max_worker_id),
2969 _active_workers(n_workers),
2970 _hrclaimer(_active_workers) {
2971 }
2972
2973 void work(uint worker_id) {
2974 AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_worker_id);
2975
2976 _g1h->heap_region_par_iterate(&cl, worker_id, &_hrclaimer);
2977 }
2978 };
2979
2980
2981 void ConcurrentMark::aggregate_count_data() {
2982 uint n_workers = _g1h->workers()->active_workers();
2983
2984 G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm,
2985 _max_worker_id, n_workers);
2986
2987 _g1h->workers()->run_task(&g1_par_agg_task);
2988 }
2989
2990 // Clear the per-worker arrays used to store the per-region counting data
2991 void ConcurrentMark::clear_all_count_data() {
2992 // Clear the global card bitmap - it will be filled during
2993 // liveness count aggregation (during remark) and the
2994 // final counting task.
2995 _card_bm.clear();
2996
2997 // Clear the global region bitmap - it will be filled as part
2998 // of the final counting task.
2999 _region_bm.clear();
3000
3001 uint max_regions = _g1h->max_regions();
3002 assert(_max_worker_id > 0, "uninitialized");
3003
3004 for (uint i = 0; i < _max_worker_id; i += 1) {
3005 BitMap* task_card_bm = count_card_bitmap_for(i);
3006 size_t* marked_bytes_array = count_marked_bytes_array_for(i);
3007
|