src/share/vm/gc_implementation/g1/concurrentMark.cpp

Print this page
rev 4278 : [mq]: 8009536-fix
rev 4279 : 8009940: G1: assert(_finger == _heap_end) failed, concurrentMark.cpp:809
Summary: Skip reference processing if the global marking stack overflows during remark. Do not call set_phase() from within parallel reference processing; use reset_for_reuse() instead. CMTask-0 should reset the marking state only during the concurrent phase of the marking cycle; if an overflow occurs at any stage during the remark, the marking state will be reset after reference processing.


 767   set_concurrent_marking_in_progress();
 768 }
 769 
 770 
 771 void ConcurrentMark::reset_marking_state(bool clear_overflow) {
 772   _markStack.set_should_expand();
 773   _markStack.setEmpty();        // Also clears the _markStack overflow flag
 774   if (clear_overflow) {
 775     clear_has_overflown();
 776   } else {
 777     assert(has_overflown(), "pre-condition");
 778   }
 779   _finger = _heap_start;
 780 
 781   for (uint i = 0; i < _max_worker_id; ++i) {
 782     CMTaskQueue* queue = _task_queues->queue(i);
 783     queue->set_empty();
 784   }
 785 }
 786 
 787 void ConcurrentMark::set_phase(uint active_tasks, bool concurrent) {
 788   assert(active_tasks <= _max_worker_id, "we should not have more");
 789 
 790   _active_tasks = active_tasks;
 791   // Need to update the three data structures below according to the
 792   // number of active threads for this phase.
 793   _terminator   = ParallelTaskTerminator((int) active_tasks, _task_queues);
 794   _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
 795   _second_overflow_barrier_sync.set_n_workers((int) active_tasks);




 796 
 797   _concurrent = concurrent;
 798   // We propagate this to all tasks, not just the active ones.
 799   for (uint i = 0; i < _max_worker_id; ++i)
 800     _tasks[i]->set_concurrent(concurrent);
 801 
 802   if (concurrent) {
 803     set_concurrent_marking_in_progress();
 804   } else {
 805     // We currently assume that the concurrent flag has been set to
 806     // false before we start remark. At this point we should also be
 807     // in a STW phase.
 808     assert(!concurrent_marking_in_progress(), "invariant");
 809     assert(_finger == _heap_end, "only way to get here");


 810     update_g1_committed(true);
 811   }
 812 }
 813 
 814 void ConcurrentMark::set_non_marking_state() {
 815   // We set the global marking state to some default values when we're
 816   // not doing marking.
 817   reset_marking_state();
 818   _active_tasks = 0;
 819   clear_concurrent_marking_in_progress();
 820 }
 821 
 822 ConcurrentMark::~ConcurrentMark() {
 823   // The ConcurrentMark instance is never freed.
 824   ShouldNotReachHere();
 825 }
 826 
 827 void ConcurrentMark::clearNextBitmap() {
 828   G1CollectedHeap* g1h = G1CollectedHeap::heap();
 829   G1CollectorPolicy* g1p = g1h->g1_policy();


 957 
 958 void ConcurrentMark::enter_first_sync_barrier(uint worker_id) {
 959   if (verbose_low()) {
 960     gclog_or_tty->print_cr("[%u] entering first barrier", worker_id);
 961   }
 962 
 963   if (concurrent()) {
 964     ConcurrentGCThread::stsLeave();
 965   }
 966   _first_overflow_barrier_sync.enter();
 967   if (concurrent()) {
 968     ConcurrentGCThread::stsJoin();
 969   }
 970   // at this point everyone should have synced up and not be doing any
 971   // more work
 972 
 973   if (verbose_low()) {
 974     gclog_or_tty->print_cr("[%u] leaving first barrier", worker_id);
 975   }
 976 







 977   // let the task associated with with worker 0 do this
 978   if (worker_id == 0) {
 979     // task 0 is responsible for clearing the global data structures
 980     // We should be here because of an overflow. During STW we should
 981     // not clear the overflow flag since we rely on it being true when
 982     // we exit this method to abort the pause and restart concurent
 983     // marking.
 984     reset_marking_state(concurrent() /* clear_overflow */);
 985     force_overflow()->update();
 986 
 987     if (G1Log::fine()) {
 988       gclog_or_tty->date_stamp(PrintGCDateStamps);
 989       gclog_or_tty->stamp(PrintGCTimeStamps);
 990       gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]");
 991     }
 992   }

 993 
 994   // after this, each task should reset its own data structures then
 995   // then go into the second barrier
 996 }
 997 
 998 void ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
 999   if (verbose_low()) {
1000     gclog_or_tty->print_cr("[%u] entering second barrier", worker_id);
1001   }
1002 
1003   if (concurrent()) {
1004     ConcurrentGCThread::stsLeave();
1005   }
1006   _second_overflow_barrier_sync.enter();
1007   if (concurrent()) {
1008     ConcurrentGCThread::stsJoin();
1009   }
1010   // at this point everything should be re-initialised and ready to go
1011 
1012   if (verbose_low()) {
1013     gclog_or_tty->print_cr("[%u] leaving second barrier", worker_id);
1014   }
1015 }
1016 
1017 #ifndef PRODUCT
1018 void ForceOverflowSettings::init() {
1019   _num_remaining = G1ConcMarkForceOverflow;
1020   _force = false;
1021   update();
1022 }
1023 
1024 void ForceOverflowSettings::update() {
1025   if (_num_remaining > 0) {
1026     _num_remaining -= 1;
1027     _force = true;
1028   } else {
1029     _force = false;
1030   }


1206 }
1207 
1208 void ConcurrentMark::markFromRoots() {
1209   // we might be tempted to assert that:
1210   // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1211   //        "inconsistent argument?");
1212   // However that wouldn't be right, because it's possible that
1213   // a safepoint is indeed in progress as a younger generation
1214   // stop-the-world GC happens even as we mark in this generation.
1215 
1216   _restart_for_overflow = false;
1217   force_overflow_conc()->init();
1218 
1219   // _g1h has _n_par_threads
1220   _parallel_marking_threads = calc_parallel_marking_threads();
1221   assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1222     "Maximum number of marking threads exceeded");
1223 
1224   uint active_workers = MAX2(1U, parallel_marking_threads());
1225 
1226   // Parallel task terminator is set in "set_phase()"
1227   set_phase(active_workers, true /* concurrent */);
1228 
1229   CMConcurrentMarkingTask markingTask(this, cmThread());
1230   if (use_parallel_marking_threads()) {
1231     _parallel_workers->set_active_workers((int)active_workers);
1232     // Don't set _n_par_threads because it affects MT in proceess_strong_roots()
1233     // and the decisions on that MT processing is made elsewhere.
1234     assert(_parallel_workers->active_workers() > 0, "Should have been set");
1235     _parallel_workers->run_task(&markingTask);
1236   } else {
1237     markingTask.work(0);
1238   }
1239   print_stats();
1240 }
1241 
1242 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1243   // world is stopped at this checkpoint
1244   assert(SafepointSynchronize::is_at_safepoint(),
1245          "world should be stopped");
1246 
1247   G1CollectedHeap* g1h = G1CollectedHeap::heap();


2344     ReferenceProcessor* rp = _g1h->ref_processor_cm();
2345     assert(rp->processing_is_mt(), "shouldn't be here otherwise");
2346   }
2347 
2348   virtual void work(uint worker_id) {
2349     CMTask* task = _cm->task(worker_id);
2350     G1CMIsAliveClosure g1_is_alive(_g1h);
2351     G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */);
2352     G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */);
2353 
2354     _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain);
2355   }
2356 };
2357 
2358 void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) {
2359   assert(_workers != NULL, "Need parallel worker threads.");
2360   assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
2361 
2362   G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm);
2363 
2364   // We need to reset the phase for each task execution so that
2365   // the termination protocol of CMTask::do_marking_step works.
2366   _cm->set_phase(_active_workers, false /* concurrent */);


2367   _g1h->set_par_threads(_active_workers);
2368   _workers->run_task(&proc_task_proxy);
2369   _g1h->set_par_threads(0);
2370 }
2371 
2372 class G1CMRefEnqueueTaskProxy: public AbstractGangTask {
2373   typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
2374   EnqueueTask& _enq_task;
2375 
2376 public:
2377   G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) :
2378     AbstractGangTask("Enqueue reference objects in parallel"),
2379     _enq_task(enq_task) { }
2380 
2381   virtual void work(uint worker_id) {
2382     _enq_task.work(worker_id);
2383   }
2384 };
2385 
2386 void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
2387   assert(_workers != NULL, "Need parallel worker threads.");
2388   assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
2389 
2390   G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task);
2391 







2392   _g1h->set_par_threads(_active_workers);
2393   _workers->run_task(&enq_task_proxy);
2394   _g1h->set_par_threads(0);
2395 }
2396 
2397 void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {










2398   ResourceMark rm;
2399   HandleMark   hm;
2400 
2401   G1CollectedHeap* g1h = G1CollectedHeap::heap();
2402 
2403   // Is alive closure.
2404   G1CMIsAliveClosure g1_is_alive(g1h);
2405 
2406   // Inner scope to exclude the cleaning of the string and symbol
2407   // tables from the displayed time.
2408   {
2409     if (G1Log::finer()) {
2410       gclog_or_tty->put(' ');
2411     }
2412     TraceTime t("GC ref-proc", G1Log::finer(), false, gclog_or_tty);
2413 
2414     ReferenceProcessor* rp = g1h->ref_processor_cm();
2415 
2416     // See the comment in G1CollectedHeap::ref_processing_init()
2417     // about how reference processing currently works in G1.


2433     //
2434     // The gang tasks involved in parallel reference procssing create
2435     // their own instances of these closures, which do their own
2436     // synchronization among themselves.
2437     G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */);
2438     G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */);
2439 
2440     // We need at least one active thread. If reference processing
2441     // is not multi-threaded we use the current (VMThread) thread,
2442     // otherwise we use the work gang from the G1CollectedHeap and
2443     // we utilize all the worker threads we can.
2444     bool processing_is_mt = rp->processing_is_mt() && g1h->workers() != NULL;
2445     uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U);
2446     active_workers = MAX2(MIN2(active_workers, _max_worker_id), 1U);
2447 
2448     // Parallel processing task executor.
2449     G1CMRefProcTaskExecutor par_task_executor(g1h, this,
2450                                               g1h->workers(), active_workers);
2451     AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL);
2452 




2453     // Set the degree of MT processing here.  If the discovery was done MT,
2454     // the number of threads involved during discovery could differ from
2455     // the number of active workers.  This is OK as long as the discovered
2456     // Reference lists are balanced (see balance_all_queues() and balance_queues()).
2457     rp->set_active_mt_degree(active_workers);
2458 
2459     // Process the weak references.
2460     rp->process_discovered_references(&g1_is_alive,
2461                                       &g1_keep_alive,
2462                                       &g1_drain_mark_stack,
2463                                       executor);
2464 
2465     // The do_oop work routines of the keep_alive and drain_marking_stack
2466     // oop closures will set the has_overflown flag if we overflow the
2467     // global marking stack.
2468 
2469     assert(_markStack.overflow() || _markStack.isEmpty(),
2470             "mark stack should be empty (unless it overflowed)");
2471 
2472     if (_markStack.overflow()) {


2523     _cm->terminator()->reset_for_reuse(active_workers);
2524   }
2525 };
2526 
2527 void ConcurrentMark::checkpointRootsFinalWork() {
2528   ResourceMark rm;
2529   HandleMark   hm;
2530   G1CollectedHeap* g1h = G1CollectedHeap::heap();
2531 
2532   g1h->ensure_parsability(false);
2533 
2534   if (G1CollectedHeap::use_parallel_gc_threads()) {
2535     G1CollectedHeap::StrongRootsScope srs(g1h);
2536     // this is remark, so we'll use up all active threads
2537     uint active_workers = g1h->workers()->active_workers();
2538     if (active_workers == 0) {
2539       assert(active_workers > 0, "Should have been set earlier");
2540       active_workers = (uint) ParallelGCThreads;
2541       g1h->workers()->set_active_workers(active_workers);
2542     }
2543     set_phase(active_workers, false /* concurrent */);
2544     // Leave _parallel_marking_threads at it's
2545     // value originally calculated in the ConcurrentMark
2546     // constructor and pass values of the active workers
2547     // through the gang in the task.
2548 
2549     CMRemarkTask remarkTask(this, active_workers, false /* is_serial */);
2550     // We will start all available threads, even if we decide that the
2551     // active_workers will be fewer. The extra ones will just bail out
2552     // immediately.
2553     g1h->set_par_threads(active_workers);
2554     g1h->workers()->run_task(&remarkTask);
2555     g1h->set_par_threads(0);
2556   } else {
2557     G1CollectedHeap::StrongRootsScope srs(g1h);
2558     uint active_workers = 1;
2559     set_phase(active_workers, false /* concurrent */);
2560 
2561     // Note - if there's no work gang then the VMThread will be
2562     // the thread to execute the remark - serially. We have
2563     // to pass true for the is_serial parameter so that
2564     // CMTask::do_marking_step() doesn't enter the sync
2565     // barriers in the event of an overflow. Doing so will
2566     // cause an assert that the current thread is not a
2567     // concurrent GC thread.
2568     CMRemarkTask remarkTask(this, active_workers, true /* is_serial*/);
2569     remarkTask.work(0);
2570   }
2571   SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2572   guarantee(satb_mq_set.completed_buffers_num() == 0, "invariant");
2573 
2574   print_stats();
2575 
2576 #if VERIFY_OBJS_PROCESSED
2577   if (_scan_obj_cl.objs_processed != ThreadLocalObjQueue::objs_enqueued) {
2578     gclog_or_tty->print_cr("Processed = %d, enqueued = %d.",
2579                            _scan_obj_cl.objs_processed,


3930       phase, tasks attempt to keep the global mark stack at a small
3931       length but not totally empty, so that entries are available for
3932       popping by other tasks. Only when there is no more work, tasks
3933       will totally drain the global mark stack.
3934 
3935       (4) SATB Buffer Queue. This is where completed SATB buffers are
3936       made available. Buffers are regularly removed from this queue
3937       and scanned for roots, so that the queue doesn't get too
3938       long. During remark, all completed buffers are processed, as
3939       well as the filled in parts of any uncompleted buffers.
3940 
3941     The do_marking_step() method tries to abort when the time target
3942     has been reached. There are a few other cases when the
3943     do_marking_step() method also aborts:
3944 
3945       (1) When the marking phase has been aborted (after a Full GC).
3946 
3947       (2) When a global overflow (on the global stack) has been
3948       triggered. Before the task aborts, it will actually sync up with
3949       the other tasks to ensure that all the marking data structures
3950       (local queues, stacks, fingers etc.)  are re-initialised so that
3951       when do_marking_step() completes, the marking phase can
3952       immediately restart.
3953 
3954       (3) When enough completed SATB buffers are available. The
3955       do_marking_step() method only tries to drain SATB buffers right
3956       at the beginning. So, if enough buffers are available, the
3957       marking step aborts and the SATB buffers are processed at
3958       the beginning of the next invocation.
3959 
3960       (4) To yield. when we have to yield then we abort and yield
3961       right at the end of do_marking_step(). This saves us from a lot
3962       of hassle as, by yielding we might allow a Full GC. If this
3963       happens then objects will be compacted underneath our feet, the
3964       heap might shrink, etc. We save checking for this by just
3965       aborting and doing the yield right at the end.
3966 
3967     From the above it follows that the do_marking_step() method should
3968     be called in a loop (or, otherwise, regularly) until it completes.
3969 
3970     If a marking step completes without its has_aborted() flag being


4377       if (!is_serial) {
4378         // We only need to enter the sync barrier if being called
4379         // from a parallel context
4380         _cm->enter_first_sync_barrier(_worker_id);
4381         
4382         // When we exit this sync barrier we know that all tasks have
4383         // stopped doing marking work. So, it's now safe to
4384         // re-initialise our data structures. At the end of this method,
4385         // task 0 will clear the global data structures.
4386       }
4387 
4388       statsOnly( ++_aborted_overflow );
4389 
4390       // We clear the local state of this task...
4391       clear_region_fields();
4392 
4393       if (!is_serial) {
4394         // ...and enter the second barrier.
4395         _cm->enter_second_sync_barrier(_worker_id);
4396       }
4397       // At this point everything has bee re-initialised and we're

4398       // ready to restart.
4399     }
4400 
4401     if (_cm->verbose_low()) {
4402       gclog_or_tty->print_cr("[%u] <<<<<<<<<< ABORTING, target = %1.2lfms, "
4403                              "elapsed = %1.2lfms <<<<<<<<<<",
4404                              _worker_id, _time_target_ms, elapsed_time_ms);
4405       if (_cm->has_aborted()) {
4406         gclog_or_tty->print_cr("[%u] ========== MARKING ABORTED ==========",
4407                                _worker_id);
4408       }
4409     }
4410   } else {
4411     if (_cm->verbose_low()) {
4412       gclog_or_tty->print_cr("[%u] <<<<<<<<<< FINISHED, target = %1.2lfms, "
4413                              "elapsed = %1.2lfms <<<<<<<<<<",
4414                              _worker_id, _time_target_ms, elapsed_time_ms);
4415     }
4416   }
4417 




 767   set_concurrent_marking_in_progress();
 768 }
 769 
 770 
 771 void ConcurrentMark::reset_marking_state(bool clear_overflow) {
 772   _markStack.set_should_expand();
 773   _markStack.setEmpty();        // Also clears the _markStack overflow flag
 774   if (clear_overflow) {
 775     clear_has_overflown();
 776   } else {
 777     assert(has_overflown(), "pre-condition");
 778   }
 779   _finger = _heap_start;
 780 
 781   for (uint i = 0; i < _max_worker_id; ++i) {
 782     CMTaskQueue* queue = _task_queues->queue(i);
 783     queue->set_empty();
 784   }
 785 }
 786 
 787 void ConcurrentMark::set_concurrency(uint active_tasks) {
 788   assert(active_tasks <= _max_worker_id, "we should not have more");
 789 
 790   _active_tasks = active_tasks;
 791   // Need to update the three data structures below according to the
 792   // number of active threads for this phase.
 793   _terminator   = ParallelTaskTerminator((int) active_tasks, _task_queues);
 794   _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
 795   _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
 796 }
 797 
 798 void ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurrent) {
 799   set_concurrency(active_tasks);
 800 
 801   _concurrent = concurrent;
 802   // We propagate this to all tasks, not just the active ones.
 803   for (uint i = 0; i < _max_worker_id; ++i)
 804     _tasks[i]->set_concurrent(concurrent);
 805 
 806   if (concurrent) {
 807     set_concurrent_marking_in_progress();
 808   } else {
 809     // We currently assume that the concurrent flag has been set to
 810     // false before we start remark. At this point we should also be
 811     // in a STW phase.
 812     assert(!concurrent_marking_in_progress(), "invariant");
 813     assert(_finger == _heap_end,
 814            err_msg("only way to get here: _finger: "PTR_FORMAT", _heap_end: "PTR_FORMAT,
 815                    _finger, _heap_end));
 816     update_g1_committed(true);
 817   }
 818 }
 819 
 820 void ConcurrentMark::set_non_marking_state() {
 821   // We set the global marking state to some default values when we're
 822   // not doing marking.
 823   reset_marking_state();
 824   _active_tasks = 0;
 825   clear_concurrent_marking_in_progress();
 826 }
 827 
 828 ConcurrentMark::~ConcurrentMark() {
 829   // The ConcurrentMark instance is never freed.
 830   ShouldNotReachHere();
 831 }
 832 
 833 void ConcurrentMark::clearNextBitmap() {
 834   G1CollectedHeap* g1h = G1CollectedHeap::heap();
 835   G1CollectorPolicy* g1p = g1h->g1_policy();


 963 
 964 void ConcurrentMark::enter_first_sync_barrier(uint worker_id) {
 965   if (verbose_low()) {
 966     gclog_or_tty->print_cr("[%u] entering first barrier", worker_id);
 967   }
 968 
 969   if (concurrent()) {
 970     ConcurrentGCThread::stsLeave();
 971   }
 972   _first_overflow_barrier_sync.enter();
 973   if (concurrent()) {
 974     ConcurrentGCThread::stsJoin();
 975   }
 976   // at this point everyone should have synced up and not be doing any
 977   // more work
 978 
 979   if (verbose_low()) {
 980     gclog_or_tty->print_cr("[%u] leaving first barrier", worker_id);
 981   }
 982 
 983   // If we're executing the concurrent phase of marking, reset the marking
 984   // state; otherwise the marking state is reset after reference processing,
 985   // during the remark pause.
 986   // If we reset here as a result of an overflow during the remark we will
 987   // see assertion failures from any subsequent set_concurrency_and_phase()
 988   // calls.
 989   if (concurrent()) {
 990     // let the task associated with with worker 0 do this
 991     if (worker_id == 0) {
 992       // task 0 is responsible for clearing the global data structures
 993       // We should be here because of an overflow. During STW we should
 994       // not clear the overflow flag since we rely on it being true when
 995       // we exit this method to abort the pause and restart concurent
 996       // marking.
 997       reset_marking_state(true /* clear_overflow */);
 998       force_overflow()->update();
 999 
1000       if (G1Log::fine()) {
1001         gclog_or_tty->date_stamp(PrintGCDateStamps);
1002         gclog_or_tty->stamp(PrintGCTimeStamps);
1003         gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]");
1004       }
1005     }
1006   }
1007 
1008   // after this, each task should reset its own data structures then
1009   // then go into the second barrier
1010 }
1011 
1012 void ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
1013   if (verbose_low()) {
1014     gclog_or_tty->print_cr("[%u] entering second barrier", worker_id);
1015   }
1016 
1017   if (concurrent()) {
1018     ConcurrentGCThread::stsLeave();
1019   }
1020   _second_overflow_barrier_sync.enter();
1021   if (concurrent()) {
1022     ConcurrentGCThread::stsJoin();
1023   }
1024   // at this point everything should be re-initialized and ready to go
1025 
1026   if (verbose_low()) {
1027     gclog_or_tty->print_cr("[%u] leaving second barrier", worker_id);
1028   }
1029 }
1030 
1031 #ifndef PRODUCT
1032 void ForceOverflowSettings::init() {
1033   _num_remaining = G1ConcMarkForceOverflow;
1034   _force = false;
1035   update();
1036 }
1037 
1038 void ForceOverflowSettings::update() {
1039   if (_num_remaining > 0) {
1040     _num_remaining -= 1;
1041     _force = true;
1042   } else {
1043     _force = false;
1044   }


1220 }
1221 
1222 void ConcurrentMark::markFromRoots() {
1223   // we might be tempted to assert that:
1224   // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1225   //        "inconsistent argument?");
1226   // However that wouldn't be right, because it's possible that
1227   // a safepoint is indeed in progress as a younger generation
1228   // stop-the-world GC happens even as we mark in this generation.
1229 
1230   _restart_for_overflow = false;
1231   force_overflow_conc()->init();
1232 
1233   // _g1h has _n_par_threads
1234   _parallel_marking_threads = calc_parallel_marking_threads();
1235   assert(parallel_marking_threads() <= max_parallel_marking_threads(),
1236     "Maximum number of marking threads exceeded");
1237 
1238   uint active_workers = MAX2(1U, parallel_marking_threads());
1239 
1240   // Parallel task terminator is set in "set_concurrency_and_phase()"
1241   set_concurrency_and_phase(active_workers, true /* concurrent */);
1242 
1243   CMConcurrentMarkingTask markingTask(this, cmThread());
1244   if (use_parallel_marking_threads()) {
1245     _parallel_workers->set_active_workers((int)active_workers);
1246     // Don't set _n_par_threads because it affects MT in proceess_strong_roots()
1247     // and the decisions on that MT processing is made elsewhere.
1248     assert(_parallel_workers->active_workers() > 0, "Should have been set");
1249     _parallel_workers->run_task(&markingTask);
1250   } else {
1251     markingTask.work(0);
1252   }
1253   print_stats();
1254 }
1255 
1256 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1257   // world is stopped at this checkpoint
1258   assert(SafepointSynchronize::is_at_safepoint(),
1259          "world should be stopped");
1260 
1261   G1CollectedHeap* g1h = G1CollectedHeap::heap();


2358     ReferenceProcessor* rp = _g1h->ref_processor_cm();
2359     assert(rp->processing_is_mt(), "shouldn't be here otherwise");
2360   }
2361 
2362   virtual void work(uint worker_id) {
2363     CMTask* task = _cm->task(worker_id);
2364     G1CMIsAliveClosure g1_is_alive(_g1h);
2365     G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */);
2366     G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */);
2367 
2368     _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain);
2369   }
2370 };
2371 
2372 void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) {
2373   assert(_workers != NULL, "Need parallel worker threads.");
2374   assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
2375 
2376   G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm);
2377 
2378   // We need to reset the concurrency level before each
2379   // proxy task execution, so that the termination protocol
2380   // and overflow handling in CMTask::do_marking_step() knows
2381   // how many workers to wait for.
2382   _cm->set_concurrency(_active_workers);
2383   _g1h->set_par_threads(_active_workers);
2384   _workers->run_task(&proc_task_proxy);
2385   _g1h->set_par_threads(0);
2386 }
2387 
2388 class G1CMRefEnqueueTaskProxy: public AbstractGangTask {
2389   typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
2390   EnqueueTask& _enq_task;
2391 
2392 public:
2393   G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) :
2394     AbstractGangTask("Enqueue reference objects in parallel"),
2395     _enq_task(enq_task) { }
2396 
2397   virtual void work(uint worker_id) {
2398     _enq_task.work(worker_id);
2399   }
2400 };
2401 
2402 void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
2403   assert(_workers != NULL, "Need parallel worker threads.");
2404   assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
2405 
2406   G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task);
2407 
2408   // Not strictly necessary but...
2409   //
2410   // We need to reset the concurrency level before each
2411   // proxy task execution, so that the termination protocol
2412   // and overflow handling in CMTask::do_marking_step() knows
2413   // how many workers to wait for.
2414   _cm->set_concurrency(_active_workers);
2415   _g1h->set_par_threads(_active_workers);
2416   _workers->run_task(&enq_task_proxy);
2417   _g1h->set_par_threads(0);
2418 }
2419 
2420 void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {
2421   if (has_overflown()) {
2422     // Skip processing the discovered references if we have
2423     // overflown the global marking stack. Reference objects
2424     // only get discovered once so it is OK to not
2425     // de-populate the discovered reference lists. We could have,
2426     // but the only benefit would be that, when marking restarts,
2427     // less reference objects are discovered.
2428     return;
2429   }
2430 
2431   ResourceMark rm;
2432   HandleMark   hm;
2433 
2434   G1CollectedHeap* g1h = G1CollectedHeap::heap();
2435 
2436   // Is alive closure.
2437   G1CMIsAliveClosure g1_is_alive(g1h);
2438 
2439   // Inner scope to exclude the cleaning of the string and symbol
2440   // tables from the displayed time.
2441   {
2442     if (G1Log::finer()) {
2443       gclog_or_tty->put(' ');
2444     }
2445     TraceTime t("GC ref-proc", G1Log::finer(), false, gclog_or_tty);
2446 
2447     ReferenceProcessor* rp = g1h->ref_processor_cm();
2448 
2449     // See the comment in G1CollectedHeap::ref_processing_init()
2450     // about how reference processing currently works in G1.


2466     //
2467     // The gang tasks involved in parallel reference procssing create
2468     // their own instances of these closures, which do their own
2469     // synchronization among themselves.
2470     G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */);
2471     G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */);
2472 
2473     // We need at least one active thread. If reference processing
2474     // is not multi-threaded we use the current (VMThread) thread,
2475     // otherwise we use the work gang from the G1CollectedHeap and
2476     // we utilize all the worker threads we can.
2477     bool processing_is_mt = rp->processing_is_mt() && g1h->workers() != NULL;
2478     uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U);
2479     active_workers = MAX2(MIN2(active_workers, _max_worker_id), 1U);
2480 
2481     // Parallel processing task executor.
2482     G1CMRefProcTaskExecutor par_task_executor(g1h, this,
2483                                               g1h->workers(), active_workers);
2484     AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL);
2485 
2486     // Set the concurrency level. The phase was already set prior to
2487     // executing the remark task.
2488     set_concurrency(active_workers);
2489 
2490     // Set the degree of MT processing here.  If the discovery was done MT,
2491     // the number of threads involved during discovery could differ from
2492     // the number of active workers.  This is OK as long as the discovered
2493     // Reference lists are balanced (see balance_all_queues() and balance_queues()).
2494     rp->set_active_mt_degree(active_workers);
2495 
2496     // Process the weak references.
2497     rp->process_discovered_references(&g1_is_alive,
2498                                       &g1_keep_alive,
2499                                       &g1_drain_mark_stack,
2500                                       executor);
2501 
2502     // The do_oop work routines of the keep_alive and drain_marking_stack
2503     // oop closures will set the has_overflown flag if we overflow the
2504     // global marking stack.
2505 
2506     assert(_markStack.overflow() || _markStack.isEmpty(),
2507             "mark stack should be empty (unless it overflowed)");
2508 
2509     if (_markStack.overflow()) {


2560     _cm->terminator()->reset_for_reuse(active_workers);
2561   }
2562 };
2563 
2564 void ConcurrentMark::checkpointRootsFinalWork() {
2565   ResourceMark rm;
2566   HandleMark   hm;
2567   G1CollectedHeap* g1h = G1CollectedHeap::heap();
2568 
2569   g1h->ensure_parsability(false);
2570 
2571   if (G1CollectedHeap::use_parallel_gc_threads()) {
2572     G1CollectedHeap::StrongRootsScope srs(g1h);
2573     // this is remark, so we'll use up all active threads
2574     uint active_workers = g1h->workers()->active_workers();
2575     if (active_workers == 0) {
2576       assert(active_workers > 0, "Should have been set earlier");
2577       active_workers = (uint) ParallelGCThreads;
2578       g1h->workers()->set_active_workers(active_workers);
2579     }
2580     set_concurrency_and_phase(active_workers, false /* concurrent */);
2581     // Leave _parallel_marking_threads at it's
2582     // value originally calculated in the ConcurrentMark
2583     // constructor and pass values of the active workers
2584     // through the gang in the task.
2585 
2586     CMRemarkTask remarkTask(this, active_workers, false /* is_serial */);
2587     // We will start all available threads, even if we decide that the
2588     // active_workers will be fewer. The extra ones will just bail out
2589     // immediately.
2590     g1h->set_par_threads(active_workers);
2591     g1h->workers()->run_task(&remarkTask);
2592     g1h->set_par_threads(0);
2593   } else {
2594     G1CollectedHeap::StrongRootsScope srs(g1h);
2595     uint active_workers = 1;
2596     set_concurrency_and_phase(active_workers, false /* concurrent */);
2597 
2598     // Note - if there's no work gang then the VMThread will be
2599     // the thread to execute the remark - serially. We have
2600     // to pass true for the is_serial parameter so that
2601     // CMTask::do_marking_step() doesn't enter the sync
2602     // barriers in the event of an overflow. Doing so will
2603     // cause an assert that the current thread is not a
2604     // concurrent GC thread.
2605     CMRemarkTask remarkTask(this, active_workers, true /* is_serial*/);
2606     remarkTask.work(0);
2607   }
2608   SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2609   guarantee(satb_mq_set.completed_buffers_num() == 0, "invariant");
2610 
2611   print_stats();
2612 
2613 #if VERIFY_OBJS_PROCESSED
2614   if (_scan_obj_cl.objs_processed != ThreadLocalObjQueue::objs_enqueued) {
2615     gclog_or_tty->print_cr("Processed = %d, enqueued = %d.",
2616                            _scan_obj_cl.objs_processed,


3967       phase, tasks attempt to keep the global mark stack at a small
3968       length but not totally empty, so that entries are available for
3969       popping by other tasks. Only when there is no more work, tasks
3970       will totally drain the global mark stack.
3971 
3972       (4) SATB Buffer Queue. This is where completed SATB buffers are
3973       made available. Buffers are regularly removed from this queue
3974       and scanned for roots, so that the queue doesn't get too
3975       long. During remark, all completed buffers are processed, as
3976       well as the filled in parts of any uncompleted buffers.
3977 
3978     The do_marking_step() method tries to abort when the time target
3979     has been reached. There are a few other cases when the
3980     do_marking_step() method also aborts:
3981 
3982       (1) When the marking phase has been aborted (after a Full GC).
3983 
3984       (2) When a global overflow (on the global stack) has been
3985       triggered. Before the task aborts, it will actually sync up with
3986       the other tasks to ensure that all the marking data structures
3987       (local queues, stacks, fingers etc.)  are re-initialized so that
3988       when do_marking_step() completes, the marking phase can
3989       immediately restart.
3990 
3991       (3) When enough completed SATB buffers are available. The
3992       do_marking_step() method only tries to drain SATB buffers right
3993       at the beginning. So, if enough buffers are available, the
3994       marking step aborts and the SATB buffers are processed at
3995       the beginning of the next invocation.
3996 
3997       (4) To yield. when we have to yield then we abort and yield
3998       right at the end of do_marking_step(). This saves us from a lot
3999       of hassle as, by yielding we might allow a Full GC. If this
4000       happens then objects will be compacted underneath our feet, the
4001       heap might shrink, etc. We save checking for this by just
4002       aborting and doing the yield right at the end.
4003 
4004     From the above it follows that the do_marking_step() method should
4005     be called in a loop (or, otherwise, regularly) until it completes.
4006 
4007     If a marking step completes without its has_aborted() flag being


4414       if (!is_serial) {
4415         // We only need to enter the sync barrier if being called
4416         // from a parallel context
4417         _cm->enter_first_sync_barrier(_worker_id);
4418         
4419         // When we exit this sync barrier we know that all tasks have
4420         // stopped doing marking work. So, it's now safe to
4421         // re-initialise our data structures. At the end of this method,
4422         // task 0 will clear the global data structures.
4423       }
4424 
4425       statsOnly( ++_aborted_overflow );
4426 
4427       // We clear the local state of this task...
4428       clear_region_fields();
4429 
4430       if (!is_serial) {
4431         // ...and enter the second barrier.
4432         _cm->enter_second_sync_barrier(_worker_id);
4433       }
4434       // At this point, if we're during the concurrent phase of
4435       // marking, everything has been re-initialized and we're
4436       // ready to restart.
4437     }
4438 
4439     if (_cm->verbose_low()) {
4440       gclog_or_tty->print_cr("[%u] <<<<<<<<<< ABORTING, target = %1.2lfms, "
4441                              "elapsed = %1.2lfms <<<<<<<<<<",
4442                              _worker_id, _time_target_ms, elapsed_time_ms);
4443       if (_cm->has_aborted()) {
4444         gclog_or_tty->print_cr("[%u] ========== MARKING ABORTED ==========",
4445                                _worker_id);
4446       }
4447     }
4448   } else {
4449     if (_cm->verbose_low()) {
4450       gclog_or_tty->print_cr("[%u] <<<<<<<<<< FINISHED, target = %1.2lfms, "
4451                              "elapsed = %1.2lfms <<<<<<<<<<",
4452                              _worker_id, _time_target_ms, elapsed_time_ms);
4453     }
4454   }
4455