2410 } else { 2411 warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant", 2412 CMSRemarkVerifyVariant); 2413 } 2414 if (!silent) gclog_or_tty->print(" done] "); 2415 return true; 2416 } 2417 2418 void CMSCollector::verify_after_remark_work_1() { 2419 ResourceMark rm; 2420 HandleMark hm; 2421 GenCollectedHeap* gch = GenCollectedHeap::heap(); 2422 2423 // Get a clear set of claim bits for the roots processing to work with. 2424 ClassLoaderDataGraph::clear_claimed_marks(); 2425 2426 // Mark from roots one level into CMS 2427 MarkRefsIntoClosure notOlder(_span, verification_mark_bm()); 2428 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. 2429 2430 gch->gen_process_roots(_cmsGen->level(), 2431 true, // younger gens are roots 2432 true, // activate StrongRootsScope 2433 GenCollectedHeap::ScanningOption(roots_scanning_options()), 2434 should_unload_classes(), 2435 ¬Older, 2436 NULL, 2437 NULL); // SSS: Provide correct closure 2438 2439 // Now mark from the roots 2440 MarkFromRootsClosure markFromRootsClosure(this, _span, 2441 verification_mark_bm(), verification_mark_stack(), 2442 false /* don't yield */, true /* verifying */); 2443 assert(_restart_addr == NULL, "Expected pre-condition"); 2444 verification_mark_bm()->iterate(&markFromRootsClosure); 2445 while (_restart_addr != NULL) { 2446 // Deal with stack overflow: by restarting at the indicated 2447 // address. 2448 HeapWord* ra = _restart_addr; 2449 markFromRootsClosure.reset(ra); 2450 _restart_addr = NULL; 2451 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end()); 2452 } 2453 assert(verification_mark_stack()->isEmpty(), "Should have been drained"); 2454 verify_work_stacks_empty(); 2455 2456 // Marking completed -- now verify that each bit marked in 2457 // verification_mark_bm() is also marked in markBitMap(); flag all 2478 void do_klass(Klass* k) { 2479 k->oops_do(&_oop_closure); 2480 } 2481 }; 2482 2483 void CMSCollector::verify_after_remark_work_2() { 2484 ResourceMark rm; 2485 HandleMark hm; 2486 GenCollectedHeap* gch = GenCollectedHeap::heap(); 2487 2488 // Get a clear set of claim bits for the roots processing to work with. 2489 ClassLoaderDataGraph::clear_claimed_marks(); 2490 2491 // Mark from roots one level into CMS 2492 MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(), 2493 markBitMap()); 2494 CLDToOopClosure cld_closure(¬Older, true); 2495 2496 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. 2497 2498 gch->gen_process_roots(_cmsGen->level(), 2499 true, // younger gens are roots 2500 true, // activate StrongRootsScope 2501 GenCollectedHeap::ScanningOption(roots_scanning_options()), 2502 should_unload_classes(), 2503 ¬Older, 2504 NULL, 2505 &cld_closure); 2506 2507 // Now mark from the roots 2508 MarkFromRootsVerifyClosure markFromRootsClosure(this, _span, 2509 verification_mark_bm(), markBitMap(), verification_mark_stack()); 2510 assert(_restart_addr == NULL, "Expected pre-condition"); 2511 verification_mark_bm()->iterate(&markFromRootsClosure); 2512 while (_restart_addr != NULL) { 2513 // Deal with stack overflow: by restarting at the indicated 2514 // address. 2515 HeapWord* ra = _restart_addr; 2516 markFromRootsClosure.reset(ra); 2517 _restart_addr = NULL; 2518 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end()); 2519 } 2520 assert(verification_mark_stack()->isEmpty(), "Should have been drained"); 2521 verify_work_stacks_empty(); 2522 2523 VerifyKlassOopsKlassClosure verify_klass_oops(verification_mark_bm()); 2524 ClassLoaderDataGraph::classes_do(&verify_klass_oops); 2525 2895 // CMS work 2896 2897 // The common parts of CMSParInitialMarkTask and CMSParRemarkTask. 2898 class CMSParMarkTask : public AbstractGangTask { 2899 protected: 2900 CMSCollector* _collector; 2901 uint _n_workers; 2902 CMSParMarkTask(const char* name, CMSCollector* collector, uint n_workers) : 2903 AbstractGangTask(name), 2904 _collector(collector), 2905 _n_workers(n_workers) {} 2906 // Work method in support of parallel rescan ... of young gen spaces 2907 void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl, 2908 ContiguousSpace* space, 2909 HeapWord** chunk_array, size_t chunk_top); 2910 void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl); 2911 }; 2912 2913 // Parallel initial mark task 2914 class CMSParInitialMarkTask: public CMSParMarkTask { 2915 public: 2916 CMSParInitialMarkTask(CMSCollector* collector, uint n_workers) : 2917 CMSParMarkTask("Scan roots and young gen for initial mark in parallel", 2918 collector, n_workers) {} 2919 void work(uint worker_id); 2920 }; 2921 2922 // Checkpoint the roots into this generation from outside 2923 // this generation. [Note this initial checkpoint need only 2924 // be approximate -- we'll do a catch up phase subsequently.] 2925 void CMSCollector::checkpointRootsInitial() { 2926 assert(_collectorState == InitialMarking, "Wrong collector state"); 2927 check_correct_thread_executing(); 2928 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause()); 2929 2930 save_heap_summary(); 2931 report_heap_summary(GCWhen::BeforeGC); 2932 2933 ReferenceProcessor* rp = ref_processor(); 2934 assert(_restart_addr == NULL, "Control point invariant"); 2935 { 2936 // acquire locks for subsequent manipulations 2937 MutexLockerEx x(bitMapLock(), 2938 Mutex::_no_safepoint_check_flag); 2986 2987 // Need to remember all newly created CLDs, 2988 // so that we can guarantee that the remark finds them. 2989 ClassLoaderDataGraph::remember_new_clds(true); 2990 2991 // Whenever a CLD is found, it will be claimed before proceeding to mark 2992 // the klasses. The claimed marks need to be cleared before marking starts. 2993 ClassLoaderDataGraph::clear_claimed_marks(); 2994 2995 if (CMSPrintEdenSurvivorChunks) { 2996 print_eden_and_survivor_chunk_arrays(); 2997 } 2998 2999 { 3000 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;) 3001 if (CMSParallelInitialMarkEnabled) { 3002 // The parallel version. 3003 FlexibleWorkGang* workers = gch->workers(); 3004 assert(workers != NULL, "Need parallel worker threads."); 3005 uint n_workers = workers->active_workers(); 3006 CMSParInitialMarkTask tsk(this, n_workers); 3007 gch->set_par_threads(n_workers); 3008 initialize_sequential_subtasks_for_young_gen_rescan(n_workers); 3009 if (n_workers > 1) { 3010 StrongRootsScope srs; 3011 workers->run_task(&tsk); 3012 } else { 3013 StrongRootsScope srs; 3014 tsk.work(0); 3015 } 3016 gch->set_par_threads(0); 3017 } else { 3018 // The serial version. 3019 CLDToOopClosure cld_closure(¬Older, true); 3020 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. 3021 gch->gen_process_roots(_cmsGen->level(), 3022 true, // younger gens are roots 3023 true, // activate StrongRootsScope 3024 GenCollectedHeap::ScanningOption(roots_scanning_options()), 3025 should_unload_classes(), 3026 ¬Older, 3027 NULL, 3028 &cld_closure); 3029 } 3030 } 3031 3032 // Clear mod-union table; it will be dirtied in the prologue of 3033 // CMS generation per each younger generation collection. 3034 3035 assert(_modUnionTable.isAllClear(), 3036 "Was cleared in most recent final checkpoint phase" 3037 " or no bits are set in the gc_prologue before the start of the next " 3038 "subsequent marking phase."); 3039 3040 assert(_ct->klass_rem_set()->mod_union_is_clear(), "Must be"); 3041 3042 // Save the end of the used_region of the constituent generations 3043 // to be used to limit the extent of sweep in each generation. 4434 GenCollectedHeap* gch = GenCollectedHeap::heap(); 4435 Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap)); 4436 4437 // ---------- young gen roots -------------- 4438 { 4439 work_on_young_gen_roots(worker_id, &par_mri_cl); 4440 _timer.stop(); 4441 if (PrintCMSStatistics != 0) { 4442 gclog_or_tty->print_cr( 4443 "Finished young gen initial mark scan work in %dth thread: %3.3f sec", 4444 worker_id, _timer.seconds()); 4445 } 4446 } 4447 4448 // ---------- remaining roots -------------- 4449 _timer.reset(); 4450 _timer.start(); 4451 4452 CLDToOopClosure cld_closure(&par_mri_cl, true); 4453 4454 gch->gen_process_roots(_collector->_cmsGen->level(), 4455 false, // yg was scanned above 4456 false, // this is parallel code 4457 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()), 4458 _collector->should_unload_classes(), 4459 &par_mri_cl, 4460 NULL, 4461 &cld_closure); 4462 assert(_collector->should_unload_classes() 4463 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), 4464 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); 4465 _timer.stop(); 4466 if (PrintCMSStatistics != 0) { 4467 gclog_or_tty->print_cr( 4468 "Finished remaining root initial mark scan work in %dth thread: %3.3f sec", 4469 worker_id, _timer.seconds()); 4470 } 4471 } 4472 4473 // Parallel remark task 4474 class CMSParRemarkTask: public CMSParMarkTask { 4475 CompactibleFreeListSpace* _cms_space; 4476 4477 // The per-thread work queues, available here for stealing. 4478 OopTaskQueueSet* _task_queues; 4479 ParallelTaskTerminator _term; 4480 4481 public: 4482 // A value of 0 passed to n_workers will cause the number of 4483 // workers to be taken from the active workers in the work gang. 4484 CMSParRemarkTask(CMSCollector* collector, 4485 CompactibleFreeListSpace* cms_space, 4486 uint n_workers, FlexibleWorkGang* workers, 4487 OopTaskQueueSet* task_queues): 4488 CMSParMarkTask("Rescan roots and grey objects in parallel", 4489 collector, n_workers), 4490 _cms_space(cms_space), 4491 _task_queues(task_queues), 4492 _term(n_workers, task_queues) { } 4493 4494 OopTaskQueueSet* task_queues() { return _task_queues; } 4495 4496 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); } 4497 4498 ParallelTaskTerminator* terminator() { return &_term; } 4499 uint n_workers() { return _n_workers; } 4500 4501 void work(uint worker_id); 4502 4503 private: 4504 // ... of dirty cards in old space 4505 void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i, 4506 Par_MarkRefsIntoAndScanClosure* cl); 4507 4508 // ... work stealing for the above 4509 void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed); 4510 }; 4511 4512 class RemarkKlassClosure : public KlassClosure { 4570 work_queue(worker_id)); 4571 4572 // Rescan young gen roots first since these are likely 4573 // coarsely partitioned and may, on that account, constitute 4574 // the critical path; thus, it's best to start off that 4575 // work first. 4576 // ---------- young gen roots -------------- 4577 { 4578 work_on_young_gen_roots(worker_id, &par_mrias_cl); 4579 _timer.stop(); 4580 if (PrintCMSStatistics != 0) { 4581 gclog_or_tty->print_cr( 4582 "Finished young gen rescan work in %dth thread: %3.3f sec", 4583 worker_id, _timer.seconds()); 4584 } 4585 } 4586 4587 // ---------- remaining roots -------------- 4588 _timer.reset(); 4589 _timer.start(); 4590 gch->gen_process_roots(_collector->_cmsGen->level(), 4591 false, // yg was scanned above 4592 false, // this is parallel code 4593 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()), 4594 _collector->should_unload_classes(), 4595 &par_mrias_cl, 4596 NULL, 4597 NULL); // The dirty klasses will be handled below 4598 4599 assert(_collector->should_unload_classes() 4600 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), 4601 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); 4602 _timer.stop(); 4603 if (PrintCMSStatistics != 0) { 4604 gclog_or_tty->print_cr( 4605 "Finished remaining root rescan work in %dth thread: %3.3f sec", 4606 worker_id, _timer.seconds()); 4607 } 4608 4609 // ---------- unhandled CLD scanning ---------- 4610 if (worker_id == 0) { // Single threaded at the moment. 4611 _timer.reset(); 4612 _timer.start(); 5050 assert(pst->valid(), "Error"); 5051 } 5052 } 5053 5054 // Parallel version of remark 5055 void CMSCollector::do_remark_parallel() { 5056 GenCollectedHeap* gch = GenCollectedHeap::heap(); 5057 FlexibleWorkGang* workers = gch->workers(); 5058 assert(workers != NULL, "Need parallel worker threads."); 5059 // Choose to use the number of GC workers most recently set 5060 // into "active_workers". If active_workers is not set, set it 5061 // to ParallelGCThreads. 5062 uint n_workers = workers->active_workers(); 5063 if (n_workers == 0) { 5064 assert(n_workers > 0, "Should have been set during scavenge"); 5065 n_workers = ParallelGCThreads; 5066 workers->set_active_workers(n_workers); 5067 } 5068 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace(); 5069 5070 CMSParRemarkTask tsk(this, 5071 cms_space, 5072 n_workers, workers, task_queues()); 5073 5074 // Set up for parallel process_roots work. 5075 gch->set_par_threads(n_workers); 5076 // We won't be iterating over the cards in the card table updating 5077 // the younger_gen cards, so we shouldn't call the following else 5078 // the verification code as well as subsequent younger_refs_iterate 5079 // code would get confused. XXX 5080 // gch->rem_set()->prepare_for_younger_refs_iterate(true); // parallel 5081 5082 // The young gen rescan work will not be done as part of 5083 // process_roots (which currently doesn't know how to 5084 // parallelize such a scan), but rather will be broken up into 5085 // a set of parallel tasks (via the sampling that the [abortable] 5086 // preclean phase did of eden, plus the [two] tasks of 5087 // scanning the [two] survivor spaces. Further fine-grain 5088 // parallelization of the scanning of the survivor spaces 5089 // themselves, and of precleaning of the younger gen itself 5090 // is deferred to the future. 5091 initialize_sequential_subtasks_for_young_gen_rescan(n_workers); 5092 5093 // The dirty card rescan work is broken up into a "sequence" 5094 // of parallel tasks (per constituent space) that are dynamically 5095 // claimed by the parallel threads. 5096 cms_space->initialize_sequential_subtasks_for_rescan(n_workers); 5097 5098 // It turns out that even when we're using 1 thread, doing the work in a 5099 // separate thread causes wide variance in run times. We can't help this 5100 // in the multi-threaded case, but we special-case n=1 here to get 5101 // repeatable measurements of the 1-thread overhead of the parallel code. 5102 if (n_workers > 1) { 5103 // Make refs discovery MT-safe, if it isn't already: it may not 5104 // necessarily be so, since it's possible that we are doing 5105 // ST marking. 5106 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), true); 5107 StrongRootsScope srs; 5108 workers->run_task(&tsk); 5109 } else { 5110 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false); 5111 StrongRootsScope srs; 5112 tsk.work(0); 5113 } 5114 5115 gch->set_par_threads(0); // 0 ==> non-parallel. 5116 // restore, single-threaded for now, any preserved marks 5117 // as a result of work_q overflow 5118 restore_preserved_marks_if_any(); 5119 } 5120 5121 // Non-parallel version of remark 5122 void CMSCollector::do_remark_non_parallel() { 5123 ResourceMark rm; 5124 HandleMark hm; 5125 GenCollectedHeap* gch = GenCollectedHeap::heap(); 5126 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false); 5127 5128 MarkRefsIntoAndScanClosure 5129 mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */, 5130 &_markStack, this, 5131 false /* should_yield */, false /* not precleaning */); 5132 MarkFromDirtyCardsClosure 5133 markFromDirtyCardsClosure(this, _span, 5134 NULL, // space is set further below 5135 &_markBitMap, &_markStack, &mrias_cl); 5159 _modUnionTable.dirty_range_iterate_clear(cms_span, 5160 &markFromDirtyCardsClosure); 5161 verify_work_stacks_empty(); 5162 if (PrintCMSStatistics != 0) { 5163 gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ", 5164 markFromDirtyCardsClosure.num_dirty_cards()); 5165 } 5166 } 5167 } 5168 if (VerifyDuringGC && 5169 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) { 5170 HandleMark hm; // Discard invalid handles created during verification 5171 Universe::verify(); 5172 } 5173 { 5174 GCTraceTime t("root rescan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); 5175 5176 verify_work_stacks_empty(); 5177 5178 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. 5179 StrongRootsScope srs; 5180 5181 gch->gen_process_roots(_cmsGen->level(), 5182 true, // younger gens as roots 5183 false, // use the local StrongRootsScope 5184 GenCollectedHeap::ScanningOption(roots_scanning_options()), 5185 should_unload_classes(), 5186 &mrias_cl, 5187 NULL, 5188 NULL); // The dirty klasses will be handled below 5189 5190 assert(should_unload_classes() 5191 || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), 5192 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); 5193 } 5194 5195 { 5196 GCTraceTime t("visit unhandled CLDs", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); 5197 5198 verify_work_stacks_empty(); 5199 5200 // Scan all class loader data objects that might have been introduced 5201 // during concurrent marking. 5202 ResourceMark rm; 5203 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds(); | 2410 } else { 2411 warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant", 2412 CMSRemarkVerifyVariant); 2413 } 2414 if (!silent) gclog_or_tty->print(" done] "); 2415 return true; 2416 } 2417 2418 void CMSCollector::verify_after_remark_work_1() { 2419 ResourceMark rm; 2420 HandleMark hm; 2421 GenCollectedHeap* gch = GenCollectedHeap::heap(); 2422 2423 // Get a clear set of claim bits for the roots processing to work with. 2424 ClassLoaderDataGraph::clear_claimed_marks(); 2425 2426 // Mark from roots one level into CMS 2427 MarkRefsIntoClosure notOlder(_span, verification_mark_bm()); 2428 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. 2429 2430 { 2431 StrongRootsScope srs(1); 2432 2433 gch->gen_process_roots(&srs, 2434 _cmsGen->level(), 2435 true, // younger gens are roots 2436 GenCollectedHeap::ScanningOption(roots_scanning_options()), 2437 should_unload_classes(), 2438 ¬Older, 2439 NULL, 2440 NULL); 2441 } 2442 2443 // Now mark from the roots 2444 MarkFromRootsClosure markFromRootsClosure(this, _span, 2445 verification_mark_bm(), verification_mark_stack(), 2446 false /* don't yield */, true /* verifying */); 2447 assert(_restart_addr == NULL, "Expected pre-condition"); 2448 verification_mark_bm()->iterate(&markFromRootsClosure); 2449 while (_restart_addr != NULL) { 2450 // Deal with stack overflow: by restarting at the indicated 2451 // address. 2452 HeapWord* ra = _restart_addr; 2453 markFromRootsClosure.reset(ra); 2454 _restart_addr = NULL; 2455 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end()); 2456 } 2457 assert(verification_mark_stack()->isEmpty(), "Should have been drained"); 2458 verify_work_stacks_empty(); 2459 2460 // Marking completed -- now verify that each bit marked in 2461 // verification_mark_bm() is also marked in markBitMap(); flag all 2482 void do_klass(Klass* k) { 2483 k->oops_do(&_oop_closure); 2484 } 2485 }; 2486 2487 void CMSCollector::verify_after_remark_work_2() { 2488 ResourceMark rm; 2489 HandleMark hm; 2490 GenCollectedHeap* gch = GenCollectedHeap::heap(); 2491 2492 // Get a clear set of claim bits for the roots processing to work with. 2493 ClassLoaderDataGraph::clear_claimed_marks(); 2494 2495 // Mark from roots one level into CMS 2496 MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(), 2497 markBitMap()); 2498 CLDToOopClosure cld_closure(¬Older, true); 2499 2500 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. 2501 2502 { 2503 StrongRootsScope srs(1); 2504 2505 gch->gen_process_roots(&srs, 2506 _cmsGen->level(), 2507 true, // younger gens are roots 2508 GenCollectedHeap::ScanningOption(roots_scanning_options()), 2509 should_unload_classes(), 2510 ¬Older, 2511 NULL, 2512 &cld_closure); 2513 } 2514 2515 // Now mark from the roots 2516 MarkFromRootsVerifyClosure markFromRootsClosure(this, _span, 2517 verification_mark_bm(), markBitMap(), verification_mark_stack()); 2518 assert(_restart_addr == NULL, "Expected pre-condition"); 2519 verification_mark_bm()->iterate(&markFromRootsClosure); 2520 while (_restart_addr != NULL) { 2521 // Deal with stack overflow: by restarting at the indicated 2522 // address. 2523 HeapWord* ra = _restart_addr; 2524 markFromRootsClosure.reset(ra); 2525 _restart_addr = NULL; 2526 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end()); 2527 } 2528 assert(verification_mark_stack()->isEmpty(), "Should have been drained"); 2529 verify_work_stacks_empty(); 2530 2531 VerifyKlassOopsKlassClosure verify_klass_oops(verification_mark_bm()); 2532 ClassLoaderDataGraph::classes_do(&verify_klass_oops); 2533 2903 // CMS work 2904 2905 // The common parts of CMSParInitialMarkTask and CMSParRemarkTask. 2906 class CMSParMarkTask : public AbstractGangTask { 2907 protected: 2908 CMSCollector* _collector; 2909 uint _n_workers; 2910 CMSParMarkTask(const char* name, CMSCollector* collector, uint n_workers) : 2911 AbstractGangTask(name), 2912 _collector(collector), 2913 _n_workers(n_workers) {} 2914 // Work method in support of parallel rescan ... of young gen spaces 2915 void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl, 2916 ContiguousSpace* space, 2917 HeapWord** chunk_array, size_t chunk_top); 2918 void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl); 2919 }; 2920 2921 // Parallel initial mark task 2922 class CMSParInitialMarkTask: public CMSParMarkTask { 2923 StrongRootsScope* _strong_roots_scope; 2924 public: 2925 CMSParInitialMarkTask(CMSCollector* collector, StrongRootsScope* strong_roots_scope, uint n_workers) : 2926 CMSParMarkTask("Scan roots and young gen for initial mark in parallel", collector, n_workers), 2927 _strong_roots_scope(strong_roots_scope) {} 2928 void work(uint worker_id); 2929 }; 2930 2931 // Checkpoint the roots into this generation from outside 2932 // this generation. [Note this initial checkpoint need only 2933 // be approximate -- we'll do a catch up phase subsequently.] 2934 void CMSCollector::checkpointRootsInitial() { 2935 assert(_collectorState == InitialMarking, "Wrong collector state"); 2936 check_correct_thread_executing(); 2937 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause()); 2938 2939 save_heap_summary(); 2940 report_heap_summary(GCWhen::BeforeGC); 2941 2942 ReferenceProcessor* rp = ref_processor(); 2943 assert(_restart_addr == NULL, "Control point invariant"); 2944 { 2945 // acquire locks for subsequent manipulations 2946 MutexLockerEx x(bitMapLock(), 2947 Mutex::_no_safepoint_check_flag); 2995 2996 // Need to remember all newly created CLDs, 2997 // so that we can guarantee that the remark finds them. 2998 ClassLoaderDataGraph::remember_new_clds(true); 2999 3000 // Whenever a CLD is found, it will be claimed before proceeding to mark 3001 // the klasses. The claimed marks need to be cleared before marking starts. 3002 ClassLoaderDataGraph::clear_claimed_marks(); 3003 3004 if (CMSPrintEdenSurvivorChunks) { 3005 print_eden_and_survivor_chunk_arrays(); 3006 } 3007 3008 { 3009 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;) 3010 if (CMSParallelInitialMarkEnabled) { 3011 // The parallel version. 3012 FlexibleWorkGang* workers = gch->workers(); 3013 assert(workers != NULL, "Need parallel worker threads."); 3014 uint n_workers = workers->active_workers(); 3015 3016 StrongRootsScope srs(n_workers); 3017 3018 CMSParInitialMarkTask tsk(this, &srs, n_workers); 3019 initialize_sequential_subtasks_for_young_gen_rescan(n_workers); 3020 if (n_workers > 1) { 3021 workers->run_task(&tsk); 3022 } else { 3023 tsk.work(0); 3024 } 3025 } else { 3026 // The serial version. 3027 CLDToOopClosure cld_closure(¬Older, true); 3028 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. 3029 3030 StrongRootsScope srs(1); 3031 3032 gch->gen_process_roots(&srs, 3033 _cmsGen->level(), 3034 true, // younger gens are roots 3035 GenCollectedHeap::ScanningOption(roots_scanning_options()), 3036 should_unload_classes(), 3037 ¬Older, 3038 NULL, 3039 &cld_closure); 3040 } 3041 } 3042 3043 // Clear mod-union table; it will be dirtied in the prologue of 3044 // CMS generation per each younger generation collection. 3045 3046 assert(_modUnionTable.isAllClear(), 3047 "Was cleared in most recent final checkpoint phase" 3048 " or no bits are set in the gc_prologue before the start of the next " 3049 "subsequent marking phase."); 3050 3051 assert(_ct->klass_rem_set()->mod_union_is_clear(), "Must be"); 3052 3053 // Save the end of the used_region of the constituent generations 3054 // to be used to limit the extent of sweep in each generation. 4445 GenCollectedHeap* gch = GenCollectedHeap::heap(); 4446 Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap)); 4447 4448 // ---------- young gen roots -------------- 4449 { 4450 work_on_young_gen_roots(worker_id, &par_mri_cl); 4451 _timer.stop(); 4452 if (PrintCMSStatistics != 0) { 4453 gclog_or_tty->print_cr( 4454 "Finished young gen initial mark scan work in %dth thread: %3.3f sec", 4455 worker_id, _timer.seconds()); 4456 } 4457 } 4458 4459 // ---------- remaining roots -------------- 4460 _timer.reset(); 4461 _timer.start(); 4462 4463 CLDToOopClosure cld_closure(&par_mri_cl, true); 4464 4465 gch->gen_process_roots(_strong_roots_scope, 4466 _collector->_cmsGen->level(), 4467 false, // yg was scanned above 4468 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()), 4469 _collector->should_unload_classes(), 4470 &par_mri_cl, 4471 NULL, 4472 &cld_closure); 4473 assert(_collector->should_unload_classes() 4474 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), 4475 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); 4476 _timer.stop(); 4477 if (PrintCMSStatistics != 0) { 4478 gclog_or_tty->print_cr( 4479 "Finished remaining root initial mark scan work in %dth thread: %3.3f sec", 4480 worker_id, _timer.seconds()); 4481 } 4482 } 4483 4484 // Parallel remark task 4485 class CMSParRemarkTask: public CMSParMarkTask { 4486 CompactibleFreeListSpace* _cms_space; 4487 4488 // The per-thread work queues, available here for stealing. 4489 OopTaskQueueSet* _task_queues; 4490 ParallelTaskTerminator _term; 4491 StrongRootsScope* _strong_roots_scope; 4492 4493 public: 4494 // A value of 0 passed to n_workers will cause the number of 4495 // workers to be taken from the active workers in the work gang. 4496 CMSParRemarkTask(CMSCollector* collector, 4497 CompactibleFreeListSpace* cms_space, 4498 uint n_workers, FlexibleWorkGang* workers, 4499 OopTaskQueueSet* task_queues, 4500 StrongRootsScope* strong_roots_scope): 4501 CMSParMarkTask("Rescan roots and grey objects in parallel", 4502 collector, n_workers), 4503 _cms_space(cms_space), 4504 _task_queues(task_queues), 4505 _term(n_workers, task_queues), 4506 _strong_roots_scope(strong_roots_scope) { } 4507 4508 OopTaskQueueSet* task_queues() { return _task_queues; } 4509 4510 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); } 4511 4512 ParallelTaskTerminator* terminator() { return &_term; } 4513 uint n_workers() { return _n_workers; } 4514 4515 void work(uint worker_id); 4516 4517 private: 4518 // ... of dirty cards in old space 4519 void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i, 4520 Par_MarkRefsIntoAndScanClosure* cl); 4521 4522 // ... work stealing for the above 4523 void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed); 4524 }; 4525 4526 class RemarkKlassClosure : public KlassClosure { 4584 work_queue(worker_id)); 4585 4586 // Rescan young gen roots first since these are likely 4587 // coarsely partitioned and may, on that account, constitute 4588 // the critical path; thus, it's best to start off that 4589 // work first. 4590 // ---------- young gen roots -------------- 4591 { 4592 work_on_young_gen_roots(worker_id, &par_mrias_cl); 4593 _timer.stop(); 4594 if (PrintCMSStatistics != 0) { 4595 gclog_or_tty->print_cr( 4596 "Finished young gen rescan work in %dth thread: %3.3f sec", 4597 worker_id, _timer.seconds()); 4598 } 4599 } 4600 4601 // ---------- remaining roots -------------- 4602 _timer.reset(); 4603 _timer.start(); 4604 gch->gen_process_roots(_strong_roots_scope, 4605 _collector->_cmsGen->level(), 4606 false, // yg was scanned above 4607 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()), 4608 _collector->should_unload_classes(), 4609 &par_mrias_cl, 4610 NULL, 4611 NULL); // The dirty klasses will be handled below 4612 4613 assert(_collector->should_unload_classes() 4614 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), 4615 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); 4616 _timer.stop(); 4617 if (PrintCMSStatistics != 0) { 4618 gclog_or_tty->print_cr( 4619 "Finished remaining root rescan work in %dth thread: %3.3f sec", 4620 worker_id, _timer.seconds()); 4621 } 4622 4623 // ---------- unhandled CLD scanning ---------- 4624 if (worker_id == 0) { // Single threaded at the moment. 4625 _timer.reset(); 4626 _timer.start(); 5064 assert(pst->valid(), "Error"); 5065 } 5066 } 5067 5068 // Parallel version of remark 5069 void CMSCollector::do_remark_parallel() { 5070 GenCollectedHeap* gch = GenCollectedHeap::heap(); 5071 FlexibleWorkGang* workers = gch->workers(); 5072 assert(workers != NULL, "Need parallel worker threads."); 5073 // Choose to use the number of GC workers most recently set 5074 // into "active_workers". If active_workers is not set, set it 5075 // to ParallelGCThreads. 5076 uint n_workers = workers->active_workers(); 5077 if (n_workers == 0) { 5078 assert(n_workers > 0, "Should have been set during scavenge"); 5079 n_workers = ParallelGCThreads; 5080 workers->set_active_workers(n_workers); 5081 } 5082 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace(); 5083 5084 StrongRootsScope srs(n_workers); 5085 5086 CMSParRemarkTask tsk(this, cms_space, n_workers, workers, task_queues(), &srs); 5087 5088 // We won't be iterating over the cards in the card table updating 5089 // the younger_gen cards, so we shouldn't call the following else 5090 // the verification code as well as subsequent younger_refs_iterate 5091 // code would get confused. XXX 5092 // gch->rem_set()->prepare_for_younger_refs_iterate(true); // parallel 5093 5094 // The young gen rescan work will not be done as part of 5095 // process_roots (which currently doesn't know how to 5096 // parallelize such a scan), but rather will be broken up into 5097 // a set of parallel tasks (via the sampling that the [abortable] 5098 // preclean phase did of eden, plus the [two] tasks of 5099 // scanning the [two] survivor spaces. Further fine-grain 5100 // parallelization of the scanning of the survivor spaces 5101 // themselves, and of precleaning of the younger gen itself 5102 // is deferred to the future. 5103 initialize_sequential_subtasks_for_young_gen_rescan(n_workers); 5104 5105 // The dirty card rescan work is broken up into a "sequence" 5106 // of parallel tasks (per constituent space) that are dynamically 5107 // claimed by the parallel threads. 5108 cms_space->initialize_sequential_subtasks_for_rescan(n_workers); 5109 5110 // It turns out that even when we're using 1 thread, doing the work in a 5111 // separate thread causes wide variance in run times. We can't help this 5112 // in the multi-threaded case, but we special-case n=1 here to get 5113 // repeatable measurements of the 1-thread overhead of the parallel code. 5114 if (n_workers > 1) { 5115 // Make refs discovery MT-safe, if it isn't already: it may not 5116 // necessarily be so, since it's possible that we are doing 5117 // ST marking. 5118 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), true); 5119 workers->run_task(&tsk); 5120 } else { 5121 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false); 5122 tsk.work(0); 5123 } 5124 5125 // restore, single-threaded for now, any preserved marks 5126 // as a result of work_q overflow 5127 restore_preserved_marks_if_any(); 5128 } 5129 5130 // Non-parallel version of remark 5131 void CMSCollector::do_remark_non_parallel() { 5132 ResourceMark rm; 5133 HandleMark hm; 5134 GenCollectedHeap* gch = GenCollectedHeap::heap(); 5135 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false); 5136 5137 MarkRefsIntoAndScanClosure 5138 mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */, 5139 &_markStack, this, 5140 false /* should_yield */, false /* not precleaning */); 5141 MarkFromDirtyCardsClosure 5142 markFromDirtyCardsClosure(this, _span, 5143 NULL, // space is set further below 5144 &_markBitMap, &_markStack, &mrias_cl); 5168 _modUnionTable.dirty_range_iterate_clear(cms_span, 5169 &markFromDirtyCardsClosure); 5170 verify_work_stacks_empty(); 5171 if (PrintCMSStatistics != 0) { 5172 gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ", 5173 markFromDirtyCardsClosure.num_dirty_cards()); 5174 } 5175 } 5176 } 5177 if (VerifyDuringGC && 5178 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) { 5179 HandleMark hm; // Discard invalid handles created during verification 5180 Universe::verify(); 5181 } 5182 { 5183 GCTraceTime t("root rescan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); 5184 5185 verify_work_stacks_empty(); 5186 5187 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. 5188 StrongRootsScope srs(1); 5189 5190 gch->gen_process_roots(&srs, 5191 _cmsGen->level(), 5192 true, // younger gens as roots 5193 GenCollectedHeap::ScanningOption(roots_scanning_options()), 5194 should_unload_classes(), 5195 &mrias_cl, 5196 NULL, 5197 NULL); // The dirty klasses will be handled below 5198 5199 assert(should_unload_classes() 5200 || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), 5201 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); 5202 } 5203 5204 { 5205 GCTraceTime t("visit unhandled CLDs", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); 5206 5207 verify_work_stacks_empty(); 5208 5209 // Scan all class loader data objects that might have been introduced 5210 // during concurrent marking. 5211 ResourceMark rm; 5212 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds(); |