1 /*
2 * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/classLoaderData.hpp"
27 #include "classfile/stringTable.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "code/codeCache.hpp"
30 #include "gc_implementation/shared/adaptiveSizePolicy.hpp"
31 #include "gc_implementation/concurrentMarkSweep/cmsCollectorPolicy.hpp"
32 #include "gc_implementation/concurrentMarkSweep/cmsOopClosures.inline.hpp"
33 #include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp"
34 #include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.inline.hpp"
35 #include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp"
36 #include "gc_implementation/concurrentMarkSweep/vmCMSOperations.hpp"
37 #include "gc_implementation/parNew/parNewGeneration.hpp"
38 #include "gc_implementation/shared/collectorCounters.hpp"
39 #include "gc_implementation/shared/gcTimer.hpp"
40 #include "gc_implementation/shared/gcTrace.hpp"
41 #include "gc_implementation/shared/gcTraceTime.hpp"
42 #include "gc_implementation/shared/isGCActiveMark.hpp"
43 #include "gc_interface/collectedHeap.inline.hpp"
44 #include "memory/allocation.hpp"
45 #include "memory/cardGeneration.inline.hpp"
46 #include "memory/cardTableRS.hpp"
47 #include "memory/collectorPolicy.hpp"
48 #include "memory/gcLocker.inline.hpp"
49 #include "memory/genCollectedHeap.hpp"
50 #include "memory/genMarkSweep.hpp"
51 #include "memory/genOopClosures.inline.hpp"
52 #include "memory/iterator.inline.hpp"
53 #include "memory/padded.hpp"
54 #include "memory/referencePolicy.hpp"
55 #include "memory/resourceArea.hpp"
56 #include "memory/strongRootsScope.hpp"
57 #include "memory/tenuredGeneration.hpp"
58 #include "oops/oop.inline.hpp"
59 #include "prims/jvmtiExport.hpp"
60 #include "runtime/atomic.inline.hpp"
61 #include "runtime/globals_extension.hpp"
62 #include "runtime/handles.inline.hpp"
63 #include "runtime/java.hpp"
64 #include "runtime/orderAccess.inline.hpp"
65 #include "runtime/vmThread.hpp"
66 #include "services/memoryService.hpp"
67 #include "services/runtimeService.hpp"
68 #include "utilities/stack.inline.hpp"
69 #include "utilities/taskqueue.inline.hpp"
70
71 // statics
72 CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;
73 bool CMSCollector::_full_gc_requested = false;
74 GCCause::Cause CMSCollector::_full_gc_cause = GCCause::_no_gc;
75
76 //////////////////////////////////////////////////////////////////
77 // In support of CMS/VM thread synchronization
78 //////////////////////////////////////////////////////////////////
79 // We split use of the CGC_lock into 2 "levels".
80 // The low-level locking is of the usual CGC_lock monitor. We introduce
81 // a higher level "token" (hereafter "CMS token") built on top of the
82 // low level monitor (hereafter "CGC lock").
83 // The token-passing protocol gives priority to the VM thread. The
84 // CMS-lock doesn't provide any fairness guarantees, but clients
85 // should ensure that it is only held for very short, bounded
86 // durations.
87 //
88 // When either of the CMS thread or the VM thread is involved in
89 // collection operations during which it does not want the other
90 // thread to interfere, it obtains the CMS token.
91 //
92 // If either thread tries to get the token while the other has
93 // it, that thread waits. However, if the VM thread and CMS thread
94 // both want the token, then the VM thread gets priority while the
95 // CMS thread waits. This ensures, for instance, that the "concurrent"
96 // phases of the CMS thread's work do not block out the VM thread
97 // for long periods of time as the CMS thread continues to hog
98 // the token. (See bug 4616232).
99 //
100 // The baton-passing functions are, however, controlled by the
101 // flags _foregroundGCShouldWait and _foregroundGCIsActive,
102 // and here the low-level CMS lock, not the high level token,
103 // ensures mutual exclusion.
104 //
105 // Two important conditions that we have to satisfy:
106 // 1. if a thread does a low-level wait on the CMS lock, then it
107 // relinquishes the CMS token if it were holding that token
108 // when it acquired the low-level CMS lock.
109 // 2. any low-level notifications on the low-level lock
110 // should only be sent when a thread has relinquished the token.
111 //
112 // In the absence of either property, we'd have potential deadlock.
113 //
114 // We protect each of the CMS (concurrent and sequential) phases
115 // with the CMS _token_, not the CMS _lock_.
116 //
117 // The only code protected by CMS lock is the token acquisition code
118 // itself, see ConcurrentMarkSweepThread::[de]synchronize(), and the
119 // baton-passing code.
120 //
121 // Unfortunately, i couldn't come up with a good abstraction to factor and
122 // hide the naked CGC_lock manipulation in the baton-passing code
123 // further below. That's something we should try to do. Also, the proof
124 // of correctness of this 2-level locking scheme is far from obvious,
125 // and potentially quite slippery. We have an uneasy suspicion, for instance,
126 // that there may be a theoretical possibility of delay/starvation in the
127 // low-level lock/wait/notify scheme used for the baton-passing because of
128 // potential interference with the priority scheme embodied in the
129 // CMS-token-passing protocol. See related comments at a CGC_lock->wait()
130 // invocation further below and marked with "XXX 20011219YSR".
131 // Indeed, as we note elsewhere, this may become yet more slippery
132 // in the presence of multiple CMS and/or multiple VM threads. XXX
133
134 class CMSTokenSync: public StackObj {
135 private:
136 bool _is_cms_thread;
137 public:
138 CMSTokenSync(bool is_cms_thread):
139 _is_cms_thread(is_cms_thread) {
140 assert(is_cms_thread == Thread::current()->is_ConcurrentGC_thread(),
141 "Incorrect argument to constructor");
142 ConcurrentMarkSweepThread::synchronize(_is_cms_thread);
143 }
144
145 ~CMSTokenSync() {
146 assert(_is_cms_thread ?
147 ConcurrentMarkSweepThread::cms_thread_has_cms_token() :
148 ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
149 "Incorrect state");
150 ConcurrentMarkSweepThread::desynchronize(_is_cms_thread);
151 }
152 };
153
154 // Convenience class that does a CMSTokenSync, and then acquires
155 // upto three locks.
156 class CMSTokenSyncWithLocks: public CMSTokenSync {
157 private:
158 // Note: locks are acquired in textual declaration order
159 // and released in the opposite order
160 MutexLockerEx _locker1, _locker2, _locker3;
161 public:
162 CMSTokenSyncWithLocks(bool is_cms_thread, Mutex* mutex1,
163 Mutex* mutex2 = NULL, Mutex* mutex3 = NULL):
164 CMSTokenSync(is_cms_thread),
165 _locker1(mutex1, Mutex::_no_safepoint_check_flag),
166 _locker2(mutex2, Mutex::_no_safepoint_check_flag),
167 _locker3(mutex3, Mutex::_no_safepoint_check_flag)
168 { }
169 };
170
171
172 //////////////////////////////////////////////////////////////////
173 // Concurrent Mark-Sweep Generation /////////////////////////////
174 //////////////////////////////////////////////////////////////////
175
176 NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;)
177
178 // This struct contains per-thread things necessary to support parallel
179 // young-gen collection.
180 class CMSParGCThreadState: public CHeapObj<mtGC> {
181 public:
182 CFLS_LAB lab;
183 PromotionInfo promo;
184
185 // Constructor.
186 CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) {
187 promo.setSpace(cfls);
188 }
189 };
190
191 ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration(
192 ReservedSpace rs, size_t initial_byte_size, int level,
193 CardTableRS* ct, bool use_adaptive_freelists,
194 FreeBlockDictionary<FreeChunk>::DictionaryChoice dictionaryChoice) :
195 CardGeneration(rs, initial_byte_size, level, ct),
196 _dilatation_factor(((double)MinChunkSize)/((double)(CollectedHeap::min_fill_size()))),
197 _did_compact(false)
198 {
199 HeapWord* bottom = (HeapWord*) _virtual_space.low();
200 HeapWord* end = (HeapWord*) _virtual_space.high();
201
202 _direct_allocated_words = 0;
203 NOT_PRODUCT(
204 _numObjectsPromoted = 0;
205 _numWordsPromoted = 0;
206 _numObjectsAllocated = 0;
207 _numWordsAllocated = 0;
208 )
209
210 _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end),
211 use_adaptive_freelists,
212 dictionaryChoice);
213 NOT_PRODUCT(debug_cms_space = _cmsSpace;)
214 _cmsSpace->_gen = this;
215
216 _gc_stats = new CMSGCStats();
217
218 // Verify the assumption that FreeChunk::_prev and OopDesc::_klass
219 // offsets match. The ability to tell free chunks from objects
220 // depends on this property.
221 debug_only(
222 FreeChunk* junk = NULL;
223 assert(UseCompressedClassPointers ||
224 junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
225 "Offset of FreeChunk::_prev within FreeChunk must match"
226 " that of OopDesc::_klass within OopDesc");
227 )
228
229 typedef CMSParGCThreadState* CMSParGCThreadStatePtr;
230 _par_gc_thread_states = NEW_C_HEAP_ARRAY(CMSParGCThreadStatePtr, ParallelGCThreads, mtGC);
231 for (uint i = 0; i < ParallelGCThreads; i++) {
232 _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace());
233 }
234
235 _incremental_collection_failed = false;
236 // The "dilatation_factor" is the expansion that can occur on
237 // account of the fact that the minimum object size in the CMS
238 // generation may be larger than that in, say, a contiguous young
239 // generation.
240 // Ideally, in the calculation below, we'd compute the dilatation
241 // factor as: MinChunkSize/(promoting_gen's min object size)
242 // Since we do not have such a general query interface for the
243 // promoting generation, we'll instead just use the minimum
244 // object size (which today is a header's worth of space);
245 // note that all arithmetic is in units of HeapWords.
246 assert(MinChunkSize >= CollectedHeap::min_fill_size(), "just checking");
247 assert(_dilatation_factor >= 1.0, "from previous assert");
248 }
249
250
251 // The field "_initiating_occupancy" represents the occupancy percentage
252 // at which we trigger a new collection cycle. Unless explicitly specified
253 // via CMSInitiatingOccupancyFraction (argument "io" below), it
254 // is calculated by:
255 //
256 // Let "f" be MinHeapFreeRatio in
257 //
258 // _initiating_occupancy = 100-f +
259 // f * (CMSTriggerRatio/100)
260 // where CMSTriggerRatio is the argument "tr" below.
261 //
262 // That is, if we assume the heap is at its desired maximum occupancy at the
263 // end of a collection, we let CMSTriggerRatio of the (purported) free
264 // space be allocated before initiating a new collection cycle.
265 //
266 void ConcurrentMarkSweepGeneration::init_initiating_occupancy(intx io, uintx tr) {
267 assert(io <= 100 && tr <= 100, "Check the arguments");
268 if (io >= 0) {
269 _initiating_occupancy = (double)io / 100.0;
270 } else {
271 _initiating_occupancy = ((100 - MinHeapFreeRatio) +
272 (double)(tr * MinHeapFreeRatio) / 100.0)
273 / 100.0;
274 }
275 }
276
277 void ConcurrentMarkSweepGeneration::ref_processor_init() {
278 assert(collector() != NULL, "no collector");
279 collector()->ref_processor_init();
280 }
281
282 void CMSCollector::ref_processor_init() {
283 if (_ref_processor == NULL) {
284 // Allocate and initialize a reference processor
285 _ref_processor =
286 new ReferenceProcessor(_span, // span
287 (ParallelGCThreads > 1) && ParallelRefProcEnabled, // mt processing
288 (int) ParallelGCThreads, // mt processing degree
289 _cmsGen->refs_discovery_is_mt(), // mt discovery
290 (int) MAX2(ConcGCThreads, ParallelGCThreads), // mt discovery degree
291 _cmsGen->refs_discovery_is_atomic(), // discovery is not atomic
292 &_is_alive_closure); // closure for liveness info
293 // Initialize the _ref_processor field of CMSGen
294 _cmsGen->set_ref_processor(_ref_processor);
295
296 }
297 }
298
299 AdaptiveSizePolicy* CMSCollector::size_policy() {
300 GenCollectedHeap* gch = GenCollectedHeap::heap();
301 return gch->gen_policy()->size_policy();
302 }
303
304 void ConcurrentMarkSweepGeneration::initialize_performance_counters() {
305
306 const char* gen_name = "old";
307 GenCollectorPolicy* gcp = (GenCollectorPolicy*) GenCollectedHeap::heap()->collector_policy();
308
309 // Generation Counters - generation 1, 1 subspace
310 _gen_counters = new GenerationCounters(gen_name, 1, 1,
311 gcp->min_old_size(), gcp->max_old_size(), &_virtual_space);
312
313 _space_counters = new GSpaceCounters(gen_name, 0,
314 _virtual_space.reserved_size(),
315 this, _gen_counters);
316 }
317
318 CMSStats::CMSStats(ConcurrentMarkSweepGeneration* cms_gen, unsigned int alpha):
319 _cms_gen(cms_gen)
320 {
321 assert(alpha <= 100, "bad value");
322 _saved_alpha = alpha;
323
324 // Initialize the alphas to the bootstrap value of 100.
325 _gc0_alpha = _cms_alpha = 100;
326
327 _cms_begin_time.update();
328 _cms_end_time.update();
329
330 _gc0_duration = 0.0;
331 _gc0_period = 0.0;
332 _gc0_promoted = 0;
333
334 _cms_duration = 0.0;
335 _cms_period = 0.0;
336 _cms_allocated = 0;
337
338 _cms_used_at_gc0_begin = 0;
339 _cms_used_at_gc0_end = 0;
340 _allow_duty_cycle_reduction = false;
341 _valid_bits = 0;
342 }
343
344 double CMSStats::cms_free_adjustment_factor(size_t free) const {
345 // TBD: CR 6909490
346 return 1.0;
347 }
348
349 void CMSStats::adjust_cms_free_adjustment_factor(bool fail, size_t free) {
350 }
351
352 // If promotion failure handling is on use
353 // the padded average size of the promotion for each
354 // young generation collection.
355 double CMSStats::time_until_cms_gen_full() const {
356 size_t cms_free = _cms_gen->cmsSpace()->free();
357 GenCollectedHeap* gch = GenCollectedHeap::heap();
358 size_t expected_promotion = MIN2(gch->young_gen()->capacity(),
359 (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average());
360 if (cms_free > expected_promotion) {
361 // Start a cms collection if there isn't enough space to promote
362 // for the next minor collection. Use the padded average as
363 // a safety factor.
364 cms_free -= expected_promotion;
365
366 // Adjust by the safety factor.
367 double cms_free_dbl = (double)cms_free;
368 double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor)/100.0;
369 // Apply a further correction factor which tries to adjust
370 // for recent occurance of concurrent mode failures.
371 cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free);
372 cms_free_dbl = cms_free_dbl * cms_adjustment;
373
374 if (PrintGCDetails && Verbose) {
375 gclog_or_tty->print_cr("CMSStats::time_until_cms_gen_full: cms_free "
376 SIZE_FORMAT " expected_promotion " SIZE_FORMAT,
377 cms_free, expected_promotion);
378 gclog_or_tty->print_cr(" cms_free_dbl %f cms_consumption_rate %f",
379 cms_free_dbl, cms_consumption_rate() + 1.0);
380 }
381 // Add 1 in case the consumption rate goes to zero.
382 return cms_free_dbl / (cms_consumption_rate() + 1.0);
383 }
384 return 0.0;
385 }
386
387 // Compare the duration of the cms collection to the
388 // time remaining before the cms generation is empty.
389 // Note that the time from the start of the cms collection
390 // to the start of the cms sweep (less than the total
391 // duration of the cms collection) can be used. This
392 // has been tried and some applications experienced
393 // promotion failures early in execution. This was
394 // possibly because the averages were not accurate
395 // enough at the beginning.
396 double CMSStats::time_until_cms_start() const {
397 // We add "gc0_period" to the "work" calculation
398 // below because this query is done (mostly) at the
399 // end of a scavenge, so we need to conservatively
400 // account for that much possible delay
401 // in the query so as to avoid concurrent mode failures
402 // due to starting the collection just a wee bit too
403 // late.
404 double work = cms_duration() + gc0_period();
405 double deadline = time_until_cms_gen_full();
406 // If a concurrent mode failure occurred recently, we want to be
407 // more conservative and halve our expected time_until_cms_gen_full()
408 if (work > deadline) {
409 if (Verbose && PrintGCDetails) {
410 gclog_or_tty->print(
411 " CMSCollector: collect because of anticipated promotion "
412 "before full %3.7f + %3.7f > %3.7f ", cms_duration(),
413 gc0_period(), time_until_cms_gen_full());
414 }
415 return 0.0;
416 }
417 return work - deadline;
418 }
419
420 #ifndef PRODUCT
421 void CMSStats::print_on(outputStream *st) const {
422 st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);
423 st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,
424 gc0_duration(), gc0_period(), gc0_promoted());
425 st->print(",cms_dur=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,
426 cms_duration(), cms_period(), cms_allocated());
427 st->print(",cms_since_beg=%g,cms_since_end=%g",
428 cms_time_since_begin(), cms_time_since_end());
429 st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,
430 _cms_used_at_gc0_begin, _cms_used_at_gc0_end);
431
432 if (valid()) {
433 st->print(",promo_rate=%g,cms_alloc_rate=%g",
434 promotion_rate(), cms_allocation_rate());
435 st->print(",cms_consumption_rate=%g,time_until_full=%g",
436 cms_consumption_rate(), time_until_cms_gen_full());
437 }
438 st->print(" ");
439 }
440 #endif // #ifndef PRODUCT
441
442 CMSCollector::CollectorState CMSCollector::_collectorState =
443 CMSCollector::Idling;
444 bool CMSCollector::_foregroundGCIsActive = false;
445 bool CMSCollector::_foregroundGCShouldWait = false;
446
447 CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
448 CardTableRS* ct,
449 ConcurrentMarkSweepPolicy* cp):
450 _cmsGen(cmsGen),
451 _ct(ct),
452 _ref_processor(NULL), // will be set later
453 _conc_workers(NULL), // may be set later
454 _abort_preclean(false),
455 _start_sampling(false),
456 _between_prologue_and_epilogue(false),
457 _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
458 _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize),
459 -1 /* lock-free */, "No_lock" /* dummy */),
460 _modUnionClosure(&_modUnionTable),
461 _modUnionClosurePar(&_modUnionTable),
462 // Adjust my span to cover old (cms) gen
463 _span(cmsGen->reserved()),
464 // Construct the is_alive_closure with _span & markBitMap
465 _is_alive_closure(_span, &_markBitMap),
466 _restart_addr(NULL),
467 _overflow_list(NULL),
468 _stats(cmsGen),
469 _eden_chunk_lock(new Mutex(Mutex::leaf + 1, "CMS_eden_chunk_lock", true,
470 //verify that this lock should be acquired with safepoint check.
471 Monitor::_safepoint_check_sometimes)),
472 _eden_chunk_array(NULL), // may be set in ctor body
473 _eden_chunk_capacity(0), // -- ditto --
474 _eden_chunk_index(0), // -- ditto --
475 _survivor_plab_array(NULL), // -- ditto --
476 _survivor_chunk_array(NULL), // -- ditto --
477 _survivor_chunk_capacity(0), // -- ditto --
478 _survivor_chunk_index(0), // -- ditto --
479 _ser_pmc_preclean_ovflw(0),
480 _ser_kac_preclean_ovflw(0),
481 _ser_pmc_remark_ovflw(0),
482 _par_pmc_remark_ovflw(0),
483 _ser_kac_ovflw(0),
484 _par_kac_ovflw(0),
485 #ifndef PRODUCT
486 _num_par_pushes(0),
487 #endif
488 _collection_count_start(0),
489 _verifying(false),
490 _verification_mark_bm(0, Mutex::leaf + 1, "CMS_verification_mark_bm_lock"),
491 _completed_initialization(false),
492 _collector_policy(cp),
493 _should_unload_classes(CMSClassUnloadingEnabled),
494 _concurrent_cycles_since_last_unload(0),
495 _roots_scanning_options(GenCollectedHeap::SO_None),
496 _inter_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding),
497 _intra_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding),
498 _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) CMSTracer()),
499 _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
500 _cms_start_registered(false)
501 {
502 if (ExplicitGCInvokesConcurrentAndUnloadsClasses) {
503 ExplicitGCInvokesConcurrent = true;
504 }
505 // Now expand the span and allocate the collection support structures
506 // (MUT, marking bit map etc.) to cover both generations subject to
507 // collection.
508
509 // For use by dirty card to oop closures.
510 _cmsGen->cmsSpace()->set_collector(this);
511
512 // Allocate MUT and marking bit map
513 {
514 MutexLockerEx x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag);
515 if (!_markBitMap.allocate(_span)) {
516 warning("Failed to allocate CMS Bit Map");
517 return;
518 }
519 assert(_markBitMap.covers(_span), "_markBitMap inconsistency?");
520 }
521 {
522 _modUnionTable.allocate(_span);
523 assert(_modUnionTable.covers(_span), "_modUnionTable inconsistency?");
524 }
525
526 if (!_markStack.allocate(MarkStackSize)) {
527 warning("Failed to allocate CMS Marking Stack");
528 return;
529 }
530
531 // Support for multi-threaded concurrent phases
532 if (CMSConcurrentMTEnabled) {
533 if (FLAG_IS_DEFAULT(ConcGCThreads)) {
534 // just for now
535 FLAG_SET_DEFAULT(ConcGCThreads, (ParallelGCThreads + 3)/4);
536 }
537 if (ConcGCThreads > 1) {
538 _conc_workers = new YieldingFlexibleWorkGang("CMS Thread",
539 ConcGCThreads, true);
540 if (_conc_workers == NULL) {
541 warning("GC/CMS: _conc_workers allocation failure: "
542 "forcing -CMSConcurrentMTEnabled");
543 CMSConcurrentMTEnabled = false;
544 } else {
545 _conc_workers->initialize_workers();
546 }
547 } else {
548 CMSConcurrentMTEnabled = false;
549 }
550 }
551 if (!CMSConcurrentMTEnabled) {
552 ConcGCThreads = 0;
553 } else {
554 // Turn off CMSCleanOnEnter optimization temporarily for
555 // the MT case where it's not fixed yet; see 6178663.
556 CMSCleanOnEnter = false;
557 }
558 assert((_conc_workers != NULL) == (ConcGCThreads > 1),
559 "Inconsistency");
560
561 // Parallel task queues; these are shared for the
562 // concurrent and stop-world phases of CMS, but
563 // are not shared with parallel scavenge (ParNew).
564 {
565 uint i;
566 uint num_queues = (uint) MAX2(ParallelGCThreads, ConcGCThreads);
567
568 if ((CMSParallelRemarkEnabled || CMSConcurrentMTEnabled
569 || ParallelRefProcEnabled)
570 && num_queues > 0) {
571 _task_queues = new OopTaskQueueSet(num_queues);
572 if (_task_queues == NULL) {
573 warning("task_queues allocation failure.");
574 return;
575 }
576 _hash_seed = NEW_C_HEAP_ARRAY(int, num_queues, mtGC);
577 typedef Padded<OopTaskQueue> PaddedOopTaskQueue;
578 for (i = 0; i < num_queues; i++) {
579 PaddedOopTaskQueue *q = new PaddedOopTaskQueue();
580 if (q == NULL) {
581 warning("work_queue allocation failure.");
582 return;
583 }
584 _task_queues->register_queue(i, q);
585 }
586 for (i = 0; i < num_queues; i++) {
587 _task_queues->queue(i)->initialize();
588 _hash_seed[i] = 17; // copied from ParNew
589 }
590 }
591 }
592
593 _cmsGen ->init_initiating_occupancy(CMSInitiatingOccupancyFraction, CMSTriggerRatio);
594
595 // Clip CMSBootstrapOccupancy between 0 and 100.
596 _bootstrap_occupancy = ((double)CMSBootstrapOccupancy)/(double)100;
597
598 // Now tell CMS generations the identity of their collector
599 ConcurrentMarkSweepGeneration::set_collector(this);
600
601 // Create & start a CMS thread for this CMS collector
602 _cmsThread = ConcurrentMarkSweepThread::start(this);
603 assert(cmsThread() != NULL, "CMS Thread should have been created");
604 assert(cmsThread()->collector() == this,
605 "CMS Thread should refer to this gen");
606 assert(CGC_lock != NULL, "Where's the CGC_lock?");
607
608 // Support for parallelizing young gen rescan
609 GenCollectedHeap* gch = GenCollectedHeap::heap();
610 assert(gch->young_gen()->kind() == Generation::ParNew, "CMS can only be used with ParNew");
611 _young_gen = (ParNewGeneration*)gch->young_gen();
612 if (gch->supports_inline_contig_alloc()) {
613 _top_addr = gch->top_addr();
614 _end_addr = gch->end_addr();
615 assert(_young_gen != NULL, "no _young_gen");
616 _eden_chunk_index = 0;
617 _eden_chunk_capacity = (_young_gen->max_capacity()+CMSSamplingGrain)/CMSSamplingGrain;
618 _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity, mtGC);
619 }
620
621 // Support for parallelizing survivor space rescan
622 if ((CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) || CMSParallelInitialMarkEnabled) {
623 const size_t max_plab_samples =
624 ((DefNewGeneration*)_young_gen)->max_survivor_size() / plab_sample_minimum_size();
625
626 _survivor_plab_array = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads, mtGC);
627 _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, 2*max_plab_samples, mtGC);
628 _cursor = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads, mtGC);
629 _survivor_chunk_capacity = 2*max_plab_samples;
630 for (uint i = 0; i < ParallelGCThreads; i++) {
631 HeapWord** vec = NEW_C_HEAP_ARRAY(HeapWord*, max_plab_samples, mtGC);
632 ChunkArray* cur = ::new (&_survivor_plab_array[i]) ChunkArray(vec, max_plab_samples);
633 assert(cur->end() == 0, "Should be 0");
634 assert(cur->array() == vec, "Should be vec");
635 assert(cur->capacity() == max_plab_samples, "Error");
636 }
637 }
638
639 NOT_PRODUCT(_overflow_counter = CMSMarkStackOverflowInterval;)
640 _gc_counters = new CollectorCounters("CMS", 1);
641 _completed_initialization = true;
642 _inter_sweep_timer.start(); // start of time
643 }
644
645 size_t CMSCollector::plab_sample_minimum_size() {
646 // The default value of MinTLABSize is 2k, but there is
647 // no way to get the default value if the flag has been overridden.
648 return MAX2(ThreadLocalAllocBuffer::min_size() * HeapWordSize, 2 * K);
649 }
650
651 const char* ConcurrentMarkSweepGeneration::name() const {
652 return "concurrent mark-sweep generation";
653 }
654 void ConcurrentMarkSweepGeneration::update_counters() {
655 if (UsePerfData) {
656 _space_counters->update_all();
657 _gen_counters->update_all();
658 }
659 }
660
661 // this is an optimized version of update_counters(). it takes the
662 // used value as a parameter rather than computing it.
663 //
664 void ConcurrentMarkSweepGeneration::update_counters(size_t used) {
665 if (UsePerfData) {
666 _space_counters->update_used(used);
667 _space_counters->update_capacity();
668 _gen_counters->update_all();
669 }
670 }
671
672 void ConcurrentMarkSweepGeneration::print() const {
673 Generation::print();
674 cmsSpace()->print();
675 }
676
677 #ifndef PRODUCT
678 void ConcurrentMarkSweepGeneration::print_statistics() {
679 cmsSpace()->printFLCensus(0);
680 }
681 #endif
682
683 void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) {
684 GenCollectedHeap* gch = GenCollectedHeap::heap();
685 if (PrintGCDetails) {
686 if (Verbose) {
687 gclog_or_tty->print("[%d %s-%s: "SIZE_FORMAT"("SIZE_FORMAT")]",
688 level(), short_name(), s, used(), capacity());
689 } else {
690 gclog_or_tty->print("[%d %s-%s: "SIZE_FORMAT"K("SIZE_FORMAT"K)]",
691 level(), short_name(), s, used() / K, capacity() / K);
692 }
693 }
694 if (Verbose) {
695 gclog_or_tty->print(" "SIZE_FORMAT"("SIZE_FORMAT")",
696 gch->used(), gch->capacity());
697 } else {
698 gclog_or_tty->print(" "SIZE_FORMAT"K("SIZE_FORMAT"K)",
699 gch->used() / K, gch->capacity() / K);
700 }
701 }
702
703 size_t
704 ConcurrentMarkSweepGeneration::contiguous_available() const {
705 // dld proposes an improvement in precision here. If the committed
706 // part of the space ends in a free block we should add that to
707 // uncommitted size in the calculation below. Will make this
708 // change later, staying with the approximation below for the
709 // time being. -- ysr.
710 return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
711 }
712
713 size_t
714 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
715 return _cmsSpace->max_alloc_in_words() * HeapWordSize;
716 }
717
718 size_t ConcurrentMarkSweepGeneration::max_available() const {
719 return free() + _virtual_space.uncommitted_size();
720 }
721
722 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
723 size_t available = max_available();
724 size_t av_promo = (size_t)gc_stats()->avg_promoted()->padded_average();
725 bool res = (available >= av_promo) || (available >= max_promotion_in_bytes);
726 if (Verbose && PrintGCDetails) {
727 gclog_or_tty->print_cr(
728 "CMS: promo attempt is%s safe: available("SIZE_FORMAT") %s av_promo("SIZE_FORMAT"),"
729 "max_promo("SIZE_FORMAT")",
730 res? "":" not", available, res? ">=":"<",
731 av_promo, max_promotion_in_bytes);
732 }
733 return res;
734 }
735
736 // At a promotion failure dump information on block layout in heap
737 // (cms old generation).
738 void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {
739 if (CMSDumpAtPromotionFailure) {
740 cmsSpace()->dump_at_safepoint_with_locks(collector(), gclog_or_tty);
741 }
742 }
743
744 void ConcurrentMarkSweepGeneration::reset_after_compaction() {
745 // Clear the promotion information. These pointers can be adjusted
746 // along with all the other pointers into the heap but
747 // compaction is expected to be a rare event with
748 // a heap using cms so don't do it without seeing the need.
749 for (uint i = 0; i < ParallelGCThreads; i++) {
750 _par_gc_thread_states[i]->promo.reset();
751 }
752 }
753
754 void ConcurrentMarkSweepGeneration::compute_new_size() {
755 assert_locked_or_safepoint(Heap_lock);
756
757 // If incremental collection failed, we just want to expand
758 // to the limit.
759 if (incremental_collection_failed()) {
760 clear_incremental_collection_failed();
761 grow_to_reserved();
762 return;
763 }
764
765 // The heap has been compacted but not reset yet.
766 // Any metric such as free() or used() will be incorrect.
767
768 CardGeneration::compute_new_size();
769
770 // Reset again after a possible resizing
771 if (did_compact()) {
772 cmsSpace()->reset_after_compaction();
773 }
774 }
775
776 void ConcurrentMarkSweepGeneration::compute_new_size_free_list() {
777 assert_locked_or_safepoint(Heap_lock);
778
779 // If incremental collection failed, we just want to expand
780 // to the limit.
781 if (incremental_collection_failed()) {
782 clear_incremental_collection_failed();
783 grow_to_reserved();
784 return;
785 }
786
787 double free_percentage = ((double) free()) / capacity();
788 double desired_free_percentage = (double) MinHeapFreeRatio / 100;
789 double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
790
791 // compute expansion delta needed for reaching desired free percentage
792 if (free_percentage < desired_free_percentage) {
793 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
794 assert(desired_capacity >= capacity(), "invalid expansion size");
795 size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
796 if (PrintGCDetails && Verbose) {
797 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
798 gclog_or_tty->print_cr("\nFrom compute_new_size: ");
799 gclog_or_tty->print_cr(" Free fraction %f", free_percentage);
800 gclog_or_tty->print_cr(" Desired free fraction %f",
801 desired_free_percentage);
802 gclog_or_tty->print_cr(" Maximum free fraction %f",
803 maximum_free_percentage);
804 gclog_or_tty->print_cr(" Capacity "SIZE_FORMAT, capacity()/1000);
805 gclog_or_tty->print_cr(" Desired capacity "SIZE_FORMAT,
806 desired_capacity/1000);
807 int prev_level = level() - 1;
808 if (prev_level >= 0) {
809 size_t prev_size = 0;
810 GenCollectedHeap* gch = GenCollectedHeap::heap();
811 Generation* prev_gen = gch->young_gen();
812 prev_size = prev_gen->capacity();
813 gclog_or_tty->print_cr(" Younger gen size "SIZE_FORMAT,
814 prev_size/1000);
815 }
816 gclog_or_tty->print_cr(" unsafe_max_alloc_nogc "SIZE_FORMAT,
817 unsafe_max_alloc_nogc()/1000);
818 gclog_or_tty->print_cr(" contiguous available "SIZE_FORMAT,
819 contiguous_available()/1000);
820 gclog_or_tty->print_cr(" Expand by "SIZE_FORMAT" (bytes)",
821 expand_bytes);
822 }
823 // safe if expansion fails
824 expand_for_gc_cause(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
825 if (PrintGCDetails && Verbose) {
826 gclog_or_tty->print_cr(" Expanded free fraction %f",
827 ((double) free()) / capacity());
828 }
829 } else {
830 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
831 assert(desired_capacity <= capacity(), "invalid expansion size");
832 size_t shrink_bytes = capacity() - desired_capacity;
833 // Don't shrink unless the delta is greater than the minimum shrink we want
834 if (shrink_bytes >= MinHeapDeltaBytes) {
835 shrink_free_list_by(shrink_bytes);
836 }
837 }
838 }
839
840 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
841 return cmsSpace()->freelistLock();
842 }
843
844 HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size,
845 bool tlab) {
846 CMSSynchronousYieldRequest yr;
847 MutexLockerEx x(freelistLock(),
848 Mutex::_no_safepoint_check_flag);
849 return have_lock_and_allocate(size, tlab);
850 }
851
852 HeapWord* ConcurrentMarkSweepGeneration::have_lock_and_allocate(size_t size,
853 bool tlab /* ignored */) {
854 assert_lock_strong(freelistLock());
855 size_t adjustedSize = CompactibleFreeListSpace::adjustObjectSize(size);
856 HeapWord* res = cmsSpace()->allocate(adjustedSize);
857 // Allocate the object live (grey) if the background collector has
858 // started marking. This is necessary because the marker may
859 // have passed this address and consequently this object will
860 // not otherwise be greyed and would be incorrectly swept up.
861 // Note that if this object contains references, the writing
862 // of those references will dirty the card containing this object
863 // allowing the object to be blackened (and its references scanned)
864 // either during a preclean phase or at the final checkpoint.
865 if (res != NULL) {
866 // We may block here with an uninitialized object with
867 // its mark-bit or P-bits not yet set. Such objects need
868 // to be safely navigable by block_start().
869 assert(oop(res)->klass_or_null() == NULL, "Object should be uninitialized here.");
870 assert(!((FreeChunk*)res)->is_free(), "Error, block will look free but show wrong size");
871 collector()->direct_allocated(res, adjustedSize);
872 _direct_allocated_words += adjustedSize;
873 // allocation counters
874 NOT_PRODUCT(
875 _numObjectsAllocated++;
876 _numWordsAllocated += (int)adjustedSize;
877 )
878 }
879 return res;
880 }
881
882 // In the case of direct allocation by mutators in a generation that
883 // is being concurrently collected, the object must be allocated
884 // live (grey) if the background collector has started marking.
885 // This is necessary because the marker may
886 // have passed this address and consequently this object will
887 // not otherwise be greyed and would be incorrectly swept up.
888 // Note that if this object contains references, the writing
889 // of those references will dirty the card containing this object
890 // allowing the object to be blackened (and its references scanned)
891 // either during a preclean phase or at the final checkpoint.
892 void CMSCollector::direct_allocated(HeapWord* start, size_t size) {
893 assert(_markBitMap.covers(start, size), "Out of bounds");
894 if (_collectorState >= Marking) {
895 MutexLockerEx y(_markBitMap.lock(),
896 Mutex::_no_safepoint_check_flag);
897 // [see comments preceding SweepClosure::do_blk() below for details]
898 //
899 // Can the P-bits be deleted now? JJJ
900 //
901 // 1. need to mark the object as live so it isn't collected
902 // 2. need to mark the 2nd bit to indicate the object may be uninitialized
903 // 3. need to mark the end of the object so marking, precleaning or sweeping
904 // can skip over uninitialized or unparsable objects. An allocated
905 // object is considered uninitialized for our purposes as long as
906 // its klass word is NULL. All old gen objects are parsable
907 // as soon as they are initialized.)
908 _markBitMap.mark(start); // object is live
909 _markBitMap.mark(start + 1); // object is potentially uninitialized?
910 _markBitMap.mark(start + size - 1);
911 // mark end of object
912 }
913 // check that oop looks uninitialized
914 assert(oop(start)->klass_or_null() == NULL, "_klass should be NULL");
915 }
916
917 void CMSCollector::promoted(bool par, HeapWord* start,
918 bool is_obj_array, size_t obj_size) {
919 assert(_markBitMap.covers(start), "Out of bounds");
920 // See comment in direct_allocated() about when objects should
921 // be allocated live.
922 if (_collectorState >= Marking) {
923 // we already hold the marking bit map lock, taken in
924 // the prologue
925 if (par) {
926 _markBitMap.par_mark(start);
927 } else {
928 _markBitMap.mark(start);
929 }
930 // We don't need to mark the object as uninitialized (as
931 // in direct_allocated above) because this is being done with the
932 // world stopped and the object will be initialized by the
933 // time the marking, precleaning or sweeping get to look at it.
934 // But see the code for copying objects into the CMS generation,
935 // where we need to ensure that concurrent readers of the
936 // block offset table are able to safely navigate a block that
937 // is in flux from being free to being allocated (and in
938 // transition while being copied into) and subsequently
939 // becoming a bona-fide object when the copy/promotion is complete.
940 assert(SafepointSynchronize::is_at_safepoint(),
941 "expect promotion only at safepoints");
942
943 if (_collectorState < Sweeping) {
944 // Mark the appropriate cards in the modUnionTable, so that
945 // this object gets scanned before the sweep. If this is
946 // not done, CMS generation references in the object might
947 // not get marked.
948 // For the case of arrays, which are otherwise precisely
949 // marked, we need to dirty the entire array, not just its head.
950 if (is_obj_array) {
951 // The [par_]mark_range() method expects mr.end() below to
952 // be aligned to the granularity of a bit's representation
953 // in the heap. In the case of the MUT below, that's a
954 // card size.
955 MemRegion mr(start,
956 (HeapWord*)round_to((intptr_t)(start + obj_size),
957 CardTableModRefBS::card_size /* bytes */));
958 if (par) {
959 _modUnionTable.par_mark_range(mr);
960 } else {
961 _modUnionTable.mark_range(mr);
962 }
963 } else { // not an obj array; we can just mark the head
964 if (par) {
965 _modUnionTable.par_mark(start);
966 } else {
967 _modUnionTable.mark(start);
968 }
969 }
970 }
971 }
972 }
973
974 oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) {
975 assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");
976 // allocate, copy and if necessary update promoinfo --
977 // delegate to underlying space.
978 assert_lock_strong(freelistLock());
979
980 #ifndef PRODUCT
981 if (GenCollectedHeap::heap()->promotion_should_fail()) {
982 return NULL;
983 }
984 #endif // #ifndef PRODUCT
985
986 oop res = _cmsSpace->promote(obj, obj_size);
987 if (res == NULL) {
988 // expand and retry
989 size_t s = _cmsSpace->expansionSpaceRequired(obj_size); // HeapWords
990 expand_for_gc_cause(s*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_promotion);
991 // Since this is the old generation, we don't try to promote
992 // into a more senior generation.
993 res = _cmsSpace->promote(obj, obj_size);
994 }
995 if (res != NULL) {
996 // See comment in allocate() about when objects should
997 // be allocated live.
998 assert(obj->is_oop(), "Will dereference klass pointer below");
999 collector()->promoted(false, // Not parallel
1000 (HeapWord*)res, obj->is_objArray(), obj_size);
1001 // promotion counters
1002 NOT_PRODUCT(
1003 _numObjectsPromoted++;
1004 _numWordsPromoted +=
1005 (int)(CompactibleFreeListSpace::adjustObjectSize(obj->size()));
1006 )
1007 }
1008 return res;
1009 }
1010
1011
1012 // IMPORTANT: Notes on object size recognition in CMS.
1013 // ---------------------------------------------------
1014 // A block of storage in the CMS generation is always in
1015 // one of three states. A free block (FREE), an allocated
1016 // object (OBJECT) whose size() method reports the correct size,
1017 // and an intermediate state (TRANSIENT) in which its size cannot
1018 // be accurately determined.
1019 // STATE IDENTIFICATION: (32 bit and 64 bit w/o COOPS)
1020 // -----------------------------------------------------
1021 // FREE: klass_word & 1 == 1; mark_word holds block size
1022 //
1023 // OBJECT: klass_word installed; klass_word != 0 && klass_word & 1 == 0;
1024 // obj->size() computes correct size
1025 //
1026 // TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT
1027 //
1028 // STATE IDENTIFICATION: (64 bit+COOPS)
1029 // ------------------------------------
1030 // FREE: mark_word & CMS_FREE_BIT == 1; mark_word & ~CMS_FREE_BIT gives block_size
1031 //
1032 // OBJECT: klass_word installed; klass_word != 0;
1033 // obj->size() computes correct size
1034 //
1035 // TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT
1036 //
1037 //
1038 // STATE TRANSITION DIAGRAM
1039 //
1040 // mut / parnew mut / parnew
1041 // FREE --------------------> TRANSIENT ---------------------> OBJECT --|
1042 // ^ |
1043 // |------------------------ DEAD <------------------------------------|
1044 // sweep mut
1045 //
1046 // While a block is in TRANSIENT state its size cannot be determined
1047 // so readers will either need to come back later or stall until
1048 // the size can be determined. Note that for the case of direct
1049 // allocation, P-bits, when available, may be used to determine the
1050 // size of an object that may not yet have been initialized.
1051
1052 // Things to support parallel young-gen collection.
1053 oop
1054 ConcurrentMarkSweepGeneration::par_promote(int thread_num,
1055 oop old, markOop m,
1056 size_t word_sz) {
1057 #ifndef PRODUCT
1058 if (GenCollectedHeap::heap()->promotion_should_fail()) {
1059 return NULL;
1060 }
1061 #endif // #ifndef PRODUCT
1062
1063 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1064 PromotionInfo* promoInfo = &ps->promo;
1065 // if we are tracking promotions, then first ensure space for
1066 // promotion (including spooling space for saving header if necessary).
1067 // then allocate and copy, then track promoted info if needed.
1068 // When tracking (see PromotionInfo::track()), the mark word may
1069 // be displaced and in this case restoration of the mark word
1070 // occurs in the (oop_since_save_marks_)iterate phase.
1071 if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {
1072 // Out of space for allocating spooling buffers;
1073 // try expanding and allocating spooling buffers.
1074 if (!expand_and_ensure_spooling_space(promoInfo)) {
1075 return NULL;
1076 }
1077 }
1078 assert(promoInfo->has_spooling_space(), "Control point invariant");
1079 const size_t alloc_sz = CompactibleFreeListSpace::adjustObjectSize(word_sz);
1080 HeapWord* obj_ptr = ps->lab.alloc(alloc_sz);
1081 if (obj_ptr == NULL) {
1082 obj_ptr = expand_and_par_lab_allocate(ps, alloc_sz);
1083 if (obj_ptr == NULL) {
1084 return NULL;
1085 }
1086 }
1087 oop obj = oop(obj_ptr);
1088 OrderAccess::storestore();
1089 assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
1090 assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");
1091 // IMPORTANT: See note on object initialization for CMS above.
1092 // Otherwise, copy the object. Here we must be careful to insert the
1093 // klass pointer last, since this marks the block as an allocated object.
1094 // Except with compressed oops it's the mark word.
1095 HeapWord* old_ptr = (HeapWord*)old;
1096 // Restore the mark word copied above.
1097 obj->set_mark(m);
1098 assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
1099 assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");
1100 OrderAccess::storestore();
1101
1102 if (UseCompressedClassPointers) {
1103 // Copy gap missed by (aligned) header size calculation below
1104 obj->set_klass_gap(old->klass_gap());
1105 }
1106 if (word_sz > (size_t)oopDesc::header_size()) {
1107 Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),
1108 obj_ptr + oopDesc::header_size(),
1109 word_sz - oopDesc::header_size());
1110 }
1111
1112 // Now we can track the promoted object, if necessary. We take care
1113 // to delay the transition from uninitialized to full object
1114 // (i.e., insertion of klass pointer) until after, so that it
1115 // atomically becomes a promoted object.
1116 if (promoInfo->tracking()) {
1117 promoInfo->track((PromotedObject*)obj, old->klass());
1118 }
1119 assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
1120 assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");
1121 assert(old->is_oop(), "Will use and dereference old klass ptr below");
1122
1123 // Finally, install the klass pointer (this should be volatile).
1124 OrderAccess::storestore();
1125 obj->set_klass(old->klass());
1126 // We should now be able to calculate the right size for this object
1127 assert(obj->is_oop() && obj->size() == (int)word_sz, "Error, incorrect size computed for promoted object");
1128
1129 collector()->promoted(true, // parallel
1130 obj_ptr, old->is_objArray(), word_sz);
1131
1132 NOT_PRODUCT(
1133 Atomic::inc_ptr(&_numObjectsPromoted);
1134 Atomic::add_ptr(alloc_sz, &_numWordsPromoted);
1135 )
1136
1137 return obj;
1138 }
1139
1140 void
1141 ConcurrentMarkSweepGeneration::
1142 par_promote_alloc_done(int thread_num) {
1143 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1144 ps->lab.retire(thread_num);
1145 }
1146
1147 void
1148 ConcurrentMarkSweepGeneration::
1149 par_oop_since_save_marks_iterate_done(int thread_num) {
1150 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1151 ParScanWithoutBarrierClosure* dummy_cl = NULL;
1152 ps->promo.promoted_oops_iterate_nv(dummy_cl);
1153 }
1154
1155 bool ConcurrentMarkSweepGeneration::should_collect(bool full,
1156 size_t size,
1157 bool tlab)
1158 {
1159 // We allow a STW collection only if a full
1160 // collection was requested.
1161 return full || should_allocate(size, tlab); // FIX ME !!!
1162 // This and promotion failure handling are connected at the
1163 // hip and should be fixed by untying them.
1164 }
1165
1166 bool CMSCollector::shouldConcurrentCollect() {
1167 if (_full_gc_requested) {
1168 if (Verbose && PrintGCDetails) {
1169 gclog_or_tty->print_cr("CMSCollector: collect because of explicit "
1170 " gc request (or gc_locker)");
1171 }
1172 return true;
1173 }
1174
1175 FreelistLocker x(this);
1176 // ------------------------------------------------------------------
1177 // Print out lots of information which affects the initiation of
1178 // a collection.
1179 if (PrintCMSInitiationStatistics && stats().valid()) {
1180 gclog_or_tty->print("CMSCollector shouldConcurrentCollect: ");
1181 gclog_or_tty->stamp();
1182 gclog_or_tty->cr();
1183 stats().print_on(gclog_or_tty);
1184 gclog_or_tty->print_cr("time_until_cms_gen_full %3.7f",
1185 stats().time_until_cms_gen_full());
1186 gclog_or_tty->print_cr("free="SIZE_FORMAT, _cmsGen->free());
1187 gclog_or_tty->print_cr("contiguous_available="SIZE_FORMAT,
1188 _cmsGen->contiguous_available());
1189 gclog_or_tty->print_cr("promotion_rate=%g", stats().promotion_rate());
1190 gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate());
1191 gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy());
1192 gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy());
1193 gclog_or_tty->print_cr("cms_time_since_begin=%3.7f", stats().cms_time_since_begin());
1194 gclog_or_tty->print_cr("cms_time_since_end=%3.7f", stats().cms_time_since_end());
1195 gclog_or_tty->print_cr("metadata initialized %d",
1196 MetaspaceGC::should_concurrent_collect());
1197 }
1198 // ------------------------------------------------------------------
1199
1200 // If the estimated time to complete a cms collection (cms_duration())
1201 // is less than the estimated time remaining until the cms generation
1202 // is full, start a collection.
1203 if (!UseCMSInitiatingOccupancyOnly) {
1204 if (stats().valid()) {
1205 if (stats().time_until_cms_start() == 0.0) {
1206 return true;
1207 }
1208 } else {
1209 // We want to conservatively collect somewhat early in order
1210 // to try and "bootstrap" our CMS/promotion statistics;
1211 // this branch will not fire after the first successful CMS
1212 // collection because the stats should then be valid.
1213 if (_cmsGen->occupancy() >= _bootstrap_occupancy) {
1214 if (Verbose && PrintGCDetails) {
1215 gclog_or_tty->print_cr(
1216 " CMSCollector: collect for bootstrapping statistics:"
1217 " occupancy = %f, boot occupancy = %f", _cmsGen->occupancy(),
1218 _bootstrap_occupancy);
1219 }
1220 return true;
1221 }
1222 }
1223 }
1224
1225 // Otherwise, we start a collection cycle if
1226 // old gen want a collection cycle started. Each may use
1227 // an appropriate criterion for making this decision.
1228 // XXX We need to make sure that the gen expansion
1229 // criterion dovetails well with this. XXX NEED TO FIX THIS
1230 if (_cmsGen->should_concurrent_collect()) {
1231 if (Verbose && PrintGCDetails) {
1232 gclog_or_tty->print_cr("CMS old gen initiated");
1233 }
1234 return true;
1235 }
1236
1237 // We start a collection if we believe an incremental collection may fail;
1238 // this is not likely to be productive in practice because it's probably too
1239 // late anyway.
1240 GenCollectedHeap* gch = GenCollectedHeap::heap();
1241 assert(gch->collector_policy()->is_generation_policy(),
1242 "You may want to check the correctness of the following");
1243 if (gch->incremental_collection_will_fail(true /* consult_young */)) {
1244 if (Verbose && PrintGCDetails) {
1245 gclog_or_tty->print("CMSCollector: collect because incremental collection will fail ");
1246 }
1247 return true;
1248 }
1249
1250 if (MetaspaceGC::should_concurrent_collect()) {
1251 if (Verbose && PrintGCDetails) {
1252 gclog_or_tty->print("CMSCollector: collect for metadata allocation ");
1253 }
1254 return true;
1255 }
1256
1257 // CMSTriggerInterval starts a CMS cycle if enough time has passed.
1258 if (CMSTriggerInterval >= 0) {
1259 if (CMSTriggerInterval == 0) {
1260 // Trigger always
1261 return true;
1262 }
1263
1264 // Check the CMS time since begin (we do not check the stats validity
1265 // as we want to be able to trigger the first CMS cycle as well)
1266 if (stats().cms_time_since_begin() >= (CMSTriggerInterval / ((double) MILLIUNITS))) {
1267 if (Verbose && PrintGCDetails) {
1268 if (stats().valid()) {
1269 gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (time since last begin %3.7f secs)",
1270 stats().cms_time_since_begin());
1271 } else {
1272 gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (first collection)");
1273 }
1274 }
1275 return true;
1276 }
1277 }
1278
1279 return false;
1280 }
1281
1282 void CMSCollector::set_did_compact(bool v) { _cmsGen->set_did_compact(v); }
1283
1284 // Clear _expansion_cause fields of constituent generations
1285 void CMSCollector::clear_expansion_cause() {
1286 _cmsGen->clear_expansion_cause();
1287 }
1288
1289 // We should be conservative in starting a collection cycle. To
1290 // start too eagerly runs the risk of collecting too often in the
1291 // extreme. To collect too rarely falls back on full collections,
1292 // which works, even if not optimum in terms of concurrent work.
1293 // As a work around for too eagerly collecting, use the flag
1294 // UseCMSInitiatingOccupancyOnly. This also has the advantage of
1295 // giving the user an easily understandable way of controlling the
1296 // collections.
1297 // We want to start a new collection cycle if any of the following
1298 // conditions hold:
1299 // . our current occupancy exceeds the configured initiating occupancy
1300 // for this generation, or
1301 // . we recently needed to expand this space and have not, since that
1302 // expansion, done a collection of this generation, or
1303 // . the underlying space believes that it may be a good idea to initiate
1304 // a concurrent collection (this may be based on criteria such as the
1305 // following: the space uses linear allocation and linear allocation is
1306 // going to fail, or there is believed to be excessive fragmentation in
1307 // the generation, etc... or ...
1308 // [.(currently done by CMSCollector::shouldConcurrentCollect() only for
1309 // the case of the old generation; see CR 6543076):
1310 // we may be approaching a point at which allocation requests may fail because
1311 // we will be out of sufficient free space given allocation rate estimates.]
1312 bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const {
1313
1314 assert_lock_strong(freelistLock());
1315 if (occupancy() > initiating_occupancy()) {
1316 if (PrintGCDetails && Verbose) {
1317 gclog_or_tty->print(" %s: collect because of occupancy %f / %f ",
1318 short_name(), occupancy(), initiating_occupancy());
1319 }
1320 return true;
1321 }
1322 if (UseCMSInitiatingOccupancyOnly) {
1323 return false;
1324 }
1325 if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) {
1326 if (PrintGCDetails && Verbose) {
1327 gclog_or_tty->print(" %s: collect because expanded for allocation ",
1328 short_name());
1329 }
1330 return true;
1331 }
1332 if (_cmsSpace->should_concurrent_collect()) {
1333 if (PrintGCDetails && Verbose) {
1334 gclog_or_tty->print(" %s: collect because cmsSpace says so ",
1335 short_name());
1336 }
1337 return true;
1338 }
1339 return false;
1340 }
1341
1342 void ConcurrentMarkSweepGeneration::collect(bool full,
1343 bool clear_all_soft_refs,
1344 size_t size,
1345 bool tlab)
1346 {
1347 collector()->collect(full, clear_all_soft_refs, size, tlab);
1348 }
1349
1350 void CMSCollector::collect(bool full,
1351 bool clear_all_soft_refs,
1352 size_t size,
1353 bool tlab)
1354 {
1355 // The following "if" branch is present for defensive reasons.
1356 // In the current uses of this interface, it can be replaced with:
1357 // assert(!GC_locker.is_active(), "Can't be called otherwise");
1358 // But I am not placing that assert here to allow future
1359 // generality in invoking this interface.
1360 if (GC_locker::is_active()) {
1361 // A consistency test for GC_locker
1362 assert(GC_locker::needs_gc(), "Should have been set already");
1363 // Skip this foreground collection, instead
1364 // expanding the heap if necessary.
1365 // Need the free list locks for the call to free() in compute_new_size()
1366 compute_new_size();
1367 return;
1368 }
1369 acquire_control_and_collect(full, clear_all_soft_refs);
1370 }
1371
1372 void CMSCollector::request_full_gc(unsigned int full_gc_count, GCCause::Cause cause) {
1373 GenCollectedHeap* gch = GenCollectedHeap::heap();
1374 unsigned int gc_count = gch->total_full_collections();
1375 if (gc_count == full_gc_count) {
1376 MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag);
1377 _full_gc_requested = true;
1378 _full_gc_cause = cause;
1379 CGC_lock->notify(); // nudge CMS thread
1380 } else {
1381 assert(gc_count > full_gc_count, "Error: causal loop");
1382 }
1383 }
1384
1385 bool CMSCollector::is_external_interruption() {
1386 GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause();
1387 return GCCause::is_user_requested_gc(cause) ||
1388 GCCause::is_serviceability_requested_gc(cause);
1389 }
1390
1391 void CMSCollector::report_concurrent_mode_interruption() {
1392 if (is_external_interruption()) {
1393 if (PrintGCDetails) {
1394 gclog_or_tty->print(" (concurrent mode interrupted)");
1395 }
1396 } else {
1397 if (PrintGCDetails) {
1398 gclog_or_tty->print(" (concurrent mode failure)");
1399 }
1400 _gc_tracer_cm->report_concurrent_mode_failure();
1401 }
1402 }
1403
1404
1405 // The foreground and background collectors need to coordinate in order
1406 // to make sure that they do not mutually interfere with CMS collections.
1407 // When a background collection is active,
1408 // the foreground collector may need to take over (preempt) and
1409 // synchronously complete an ongoing collection. Depending on the
1410 // frequency of the background collections and the heap usage
1411 // of the application, this preemption can be seldom or frequent.
1412 // There are only certain
1413 // points in the background collection that the "collection-baton"
1414 // can be passed to the foreground collector.
1415 //
1416 // The foreground collector will wait for the baton before
1417 // starting any part of the collection. The foreground collector
1418 // will only wait at one location.
1419 //
1420 // The background collector will yield the baton before starting a new
1421 // phase of the collection (e.g., before initial marking, marking from roots,
1422 // precleaning, final re-mark, sweep etc.) This is normally done at the head
1423 // of the loop which switches the phases. The background collector does some
1424 // of the phases (initial mark, final re-mark) with the world stopped.
1425 // Because of locking involved in stopping the world,
1426 // the foreground collector should not block waiting for the background
1427 // collector when it is doing a stop-the-world phase. The background
1428 // collector will yield the baton at an additional point just before
1429 // it enters a stop-the-world phase. Once the world is stopped, the
1430 // background collector checks the phase of the collection. If the
1431 // phase has not changed, it proceeds with the collection. If the
1432 // phase has changed, it skips that phase of the collection. See
1433 // the comments on the use of the Heap_lock in collect_in_background().
1434 //
1435 // Variable used in baton passing.
1436 // _foregroundGCIsActive - Set to true by the foreground collector when
1437 // it wants the baton. The foreground clears it when it has finished
1438 // the collection.
1439 // _foregroundGCShouldWait - Set to true by the background collector
1440 // when it is running. The foreground collector waits while
1441 // _foregroundGCShouldWait is true.
1442 // CGC_lock - monitor used to protect access to the above variables
1443 // and to notify the foreground and background collectors.
1444 // _collectorState - current state of the CMS collection.
1445 //
1446 // The foreground collector
1447 // acquires the CGC_lock
1448 // sets _foregroundGCIsActive
1449 // waits on the CGC_lock for _foregroundGCShouldWait to be false
1450 // various locks acquired in preparation for the collection
1451 // are released so as not to block the background collector
1452 // that is in the midst of a collection
1453 // proceeds with the collection
1454 // clears _foregroundGCIsActive
1455 // returns
1456 //
1457 // The background collector in a loop iterating on the phases of the
1458 // collection
1459 // acquires the CGC_lock
1460 // sets _foregroundGCShouldWait
1461 // if _foregroundGCIsActive is set
1462 // clears _foregroundGCShouldWait, notifies _CGC_lock
1463 // waits on _CGC_lock for _foregroundGCIsActive to become false
1464 // and exits the loop.
1465 // otherwise
1466 // proceed with that phase of the collection
1467 // if the phase is a stop-the-world phase,
1468 // yield the baton once more just before enqueueing
1469 // the stop-world CMS operation (executed by the VM thread).
1470 // returns after all phases of the collection are done
1471 //
1472
1473 void CMSCollector::acquire_control_and_collect(bool full,
1474 bool clear_all_soft_refs) {
1475 assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
1476 assert(!Thread::current()->is_ConcurrentGC_thread(),
1477 "shouldn't try to acquire control from self!");
1478
1479 // Start the protocol for acquiring control of the
1480 // collection from the background collector (aka CMS thread).
1481 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1482 "VM thread should have CMS token");
1483 // Remember the possibly interrupted state of an ongoing
1484 // concurrent collection
1485 CollectorState first_state = _collectorState;
1486
1487 // Signal to a possibly ongoing concurrent collection that
1488 // we want to do a foreground collection.
1489 _foregroundGCIsActive = true;
1490
1491 // release locks and wait for a notify from the background collector
1492 // releasing the locks in only necessary for phases which
1493 // do yields to improve the granularity of the collection.
1494 assert_lock_strong(bitMapLock());
1495 // We need to lock the Free list lock for the space that we are
1496 // currently collecting.
1497 assert(haveFreelistLocks(), "Must be holding free list locks");
1498 bitMapLock()->unlock();
1499 releaseFreelistLocks();
1500 {
1501 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1502 if (_foregroundGCShouldWait) {
1503 // We are going to be waiting for action for the CMS thread;
1504 // it had better not be gone (for instance at shutdown)!
1505 assert(ConcurrentMarkSweepThread::cmst() != NULL,
1506 "CMS thread must be running");
1507 // Wait here until the background collector gives us the go-ahead
1508 ConcurrentMarkSweepThread::clear_CMS_flag(
1509 ConcurrentMarkSweepThread::CMS_vm_has_token); // release token
1510 // Get a possibly blocked CMS thread going:
1511 // Note that we set _foregroundGCIsActive true above,
1512 // without protection of the CGC_lock.
1513 CGC_lock->notify();
1514 assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(),
1515 "Possible deadlock");
1516 while (_foregroundGCShouldWait) {
1517 // wait for notification
1518 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1519 // Possibility of delay/starvation here, since CMS token does
1520 // not know to give priority to VM thread? Actually, i think
1521 // there wouldn't be any delay/starvation, but the proof of
1522 // that "fact" (?) appears non-trivial. XXX 20011219YSR
1523 }
1524 ConcurrentMarkSweepThread::set_CMS_flag(
1525 ConcurrentMarkSweepThread::CMS_vm_has_token);
1526 }
1527 }
1528 // The CMS_token is already held. Get back the other locks.
1529 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1530 "VM thread should have CMS token");
1531 getFreelistLocks();
1532 bitMapLock()->lock_without_safepoint_check();
1533 if (TraceCMSState) {
1534 gclog_or_tty->print_cr("CMS foreground collector has asked for control "
1535 INTPTR_FORMAT " with first state %d", p2i(Thread::current()), first_state);
1536 gclog_or_tty->print_cr(" gets control with state %d", _collectorState);
1537 }
1538
1539 // Inform cms gen if this was due to partial collection failing.
1540 // The CMS gen may use this fact to determine its expansion policy.
1541 GenCollectedHeap* gch = GenCollectedHeap::heap();
1542 if (gch->incremental_collection_will_fail(false /* don't consult_young */)) {
1543 assert(!_cmsGen->incremental_collection_failed(),
1544 "Should have been noticed, reacted to and cleared");
1545 _cmsGen->set_incremental_collection_failed();
1546 }
1547
1548 if (first_state > Idling) {
1549 report_concurrent_mode_interruption();
1550 }
1551
1552 set_did_compact(true);
1553
1554 // If the collection is being acquired from the background
1555 // collector, there may be references on the discovered
1556 // references lists. Abandon those references, since some
1557 // of them may have become unreachable after concurrent
1558 // discovery; the STW compacting collector will redo discovery
1559 // more precisely, without being subject to floating garbage.
1560 // Leaving otherwise unreachable references in the discovered
1561 // lists would require special handling.
1562 ref_processor()->disable_discovery();
1563 ref_processor()->abandon_partial_discovery();
1564 ref_processor()->verify_no_references_recorded();
1565
1566 if (first_state > Idling) {
1567 save_heap_summary();
1568 }
1569
1570 do_compaction_work(clear_all_soft_refs);
1571
1572 // Has the GC time limit been exceeded?
1573 size_t max_eden_size = _young_gen->max_capacity() -
1574 _young_gen->to()->capacity() -
1575 _young_gen->from()->capacity();
1576 GCCause::Cause gc_cause = gch->gc_cause();
1577 size_policy()->check_gc_overhead_limit(_young_gen->used(),
1578 _young_gen->eden()->used(),
1579 _cmsGen->max_capacity(),
1580 max_eden_size,
1581 full,
1582 gc_cause,
1583 gch->collector_policy());
1584
1585 // Reset the expansion cause, now that we just completed
1586 // a collection cycle.
1587 clear_expansion_cause();
1588 _foregroundGCIsActive = false;
1589 return;
1590 }
1591
1592 // Resize the tenured generation
1593 // after obtaining the free list locks for the
1594 // two generations.
1595 void CMSCollector::compute_new_size() {
1596 assert_locked_or_safepoint(Heap_lock);
1597 FreelistLocker z(this);
1598 MetaspaceGC::compute_new_size();
1599 _cmsGen->compute_new_size_free_list();
1600 }
1601
1602 // A work method used by the foreground collector to do
1603 // a mark-sweep-compact.
1604 void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
1605 GenCollectedHeap* gch = GenCollectedHeap::heap();
1606
1607 STWGCTimer* gc_timer = GenMarkSweep::gc_timer();
1608 gc_timer->register_gc_start();
1609
1610 SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer();
1611 gc_tracer->report_gc_start(gch->gc_cause(), gc_timer->gc_start());
1612
1613 GCTraceTime t("CMS:MSC ", PrintGCDetails && Verbose, true, NULL, gc_tracer->gc_id());
1614
1615 // Temporarily widen the span of the weak reference processing to
1616 // the entire heap.
1617 MemRegion new_span(GenCollectedHeap::heap()->reserved_region());
1618 ReferenceProcessorSpanMutator rp_mut_span(ref_processor(), new_span);
1619 // Temporarily, clear the "is_alive_non_header" field of the
1620 // reference processor.
1621 ReferenceProcessorIsAliveMutator rp_mut_closure(ref_processor(), NULL);
1622 // Temporarily make reference _processing_ single threaded (non-MT).
1623 ReferenceProcessorMTProcMutator rp_mut_mt_processing(ref_processor(), false);
1624 // Temporarily make refs discovery atomic
1625 ReferenceProcessorAtomicMutator rp_mut_atomic(ref_processor(), true);
1626 // Temporarily make reference _discovery_ single threaded (non-MT)
1627 ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false);
1628
1629 ref_processor()->set_enqueuing_is_done(false);
1630 ref_processor()->enable_discovery();
1631 ref_processor()->setup_policy(clear_all_soft_refs);
1632 // If an asynchronous collection finishes, the _modUnionTable is
1633 // all clear. If we are assuming the collection from an asynchronous
1634 // collection, clear the _modUnionTable.
1635 assert(_collectorState != Idling || _modUnionTable.isAllClear(),
1636 "_modUnionTable should be clear if the baton was not passed");
1637 _modUnionTable.clear_all();
1638 assert(_collectorState != Idling || _ct->klass_rem_set()->mod_union_is_clear(),
1639 "mod union for klasses should be clear if the baton was passed");
1640 _ct->klass_rem_set()->clear_mod_union();
1641
1642 // We must adjust the allocation statistics being maintained
1643 // in the free list space. We do so by reading and clearing
1644 // the sweep timer and updating the block flux rate estimates below.
1645 assert(!_intra_sweep_timer.is_active(), "_intra_sweep_timer should be inactive");
1646 if (_inter_sweep_timer.is_active()) {
1647 _inter_sweep_timer.stop();
1648 // Note that we do not use this sample to update the _inter_sweep_estimate.
1649 _cmsGen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()),
1650 _inter_sweep_estimate.padded_average(),
1651 _intra_sweep_estimate.padded_average());
1652 }
1653
1654 GenMarkSweep::invoke_at_safepoint(_cmsGen->level(),
1655 ref_processor(), clear_all_soft_refs);
1656 #ifdef ASSERT
1657 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
1658 size_t free_size = cms_space->free();
1659 assert(free_size ==
1660 pointer_delta(cms_space->end(), cms_space->compaction_top())
1661 * HeapWordSize,
1662 "All the free space should be compacted into one chunk at top");
1663 assert(cms_space->dictionary()->total_chunk_size(
1664 debug_only(cms_space->freelistLock())) == 0 ||
1665 cms_space->totalSizeInIndexedFreeLists() == 0,
1666 "All the free space should be in a single chunk");
1667 size_t num = cms_space->totalCount();
1668 assert((free_size == 0 && num == 0) ||
1669 (free_size > 0 && (num == 1 || num == 2)),
1670 "There should be at most 2 free chunks after compaction");
1671 #endif // ASSERT
1672 _collectorState = Resetting;
1673 assert(_restart_addr == NULL,
1674 "Should have been NULL'd before baton was passed");
1675 reset(false /* == !concurrent */);
1676 _cmsGen->reset_after_compaction();
1677 _concurrent_cycles_since_last_unload = 0;
1678
1679 // Clear any data recorded in the PLAB chunk arrays.
1680 if (_survivor_plab_array != NULL) {
1681 reset_survivor_plab_arrays();
1682 }
1683
1684 // Adjust the per-size allocation stats for the next epoch.
1685 _cmsGen->cmsSpace()->endSweepFLCensus(sweep_count() /* fake */);
1686 // Restart the "inter sweep timer" for the next epoch.
1687 _inter_sweep_timer.reset();
1688 _inter_sweep_timer.start();
1689
1690 gc_timer->register_gc_end();
1691
1692 gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions());
1693
1694 // For a mark-sweep-compact, compute_new_size() will be called
1695 // in the heap's do_collection() method.
1696 }
1697
1698 void CMSCollector::print_eden_and_survivor_chunk_arrays() {
1699 ContiguousSpace* eden_space = _young_gen->eden();
1700 ContiguousSpace* from_space = _young_gen->from();
1701 ContiguousSpace* to_space = _young_gen->to();
1702 // Eden
1703 if (_eden_chunk_array != NULL) {
1704 gclog_or_tty->print_cr("eden " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1705 p2i(eden_space->bottom()), p2i(eden_space->top()),
1706 p2i(eden_space->end()), eden_space->capacity());
1707 gclog_or_tty->print_cr("_eden_chunk_index=" SIZE_FORMAT ", "
1708 "_eden_chunk_capacity=" SIZE_FORMAT,
1709 _eden_chunk_index, _eden_chunk_capacity);
1710 for (size_t i = 0; i < _eden_chunk_index; i++) {
1711 gclog_or_tty->print_cr("_eden_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT,
1712 i, p2i(_eden_chunk_array[i]));
1713 }
1714 }
1715 // Survivor
1716 if (_survivor_chunk_array != NULL) {
1717 gclog_or_tty->print_cr("survivor " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")",
1718 p2i(from_space->bottom()), p2i(from_space->top()),
1719 p2i(from_space->end()), from_space->capacity());
1720 gclog_or_tty->print_cr("_survivor_chunk_index=" SIZE_FORMAT ", "
1721 "_survivor_chunk_capacity=" SIZE_FORMAT,
1722 _survivor_chunk_index, _survivor_chunk_capacity);
1723 for (size_t i = 0; i < _survivor_chunk_index; i++) {
1724 gclog_or_tty->print_cr("_survivor_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT,
1725 i, p2i(_survivor_chunk_array[i]));
1726 }
1727 }
1728 }
1729
1730 void CMSCollector::getFreelistLocks() const {
1731 // Get locks for all free lists in all generations that this
1732 // collector is responsible for
1733 _cmsGen->freelistLock()->lock_without_safepoint_check();
1734 }
1735
1736 void CMSCollector::releaseFreelistLocks() const {
1737 // Release locks for all free lists in all generations that this
1738 // collector is responsible for
1739 _cmsGen->freelistLock()->unlock();
1740 }
1741
1742 bool CMSCollector::haveFreelistLocks() const {
1743 // Check locks for all free lists in all generations that this
1744 // collector is responsible for
1745 assert_lock_strong(_cmsGen->freelistLock());
1746 PRODUCT_ONLY(ShouldNotReachHere());
1747 return true;
1748 }
1749
1750 // A utility class that is used by the CMS collector to
1751 // temporarily "release" the foreground collector from its
1752 // usual obligation to wait for the background collector to
1753 // complete an ongoing phase before proceeding.
1754 class ReleaseForegroundGC: public StackObj {
1755 private:
1756 CMSCollector* _c;
1757 public:
1758 ReleaseForegroundGC(CMSCollector* c) : _c(c) {
1759 assert(_c->_foregroundGCShouldWait, "Else should not need to call");
1760 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1761 // allow a potentially blocked foreground collector to proceed
1762 _c->_foregroundGCShouldWait = false;
1763 if (_c->_foregroundGCIsActive) {
1764 CGC_lock->notify();
1765 }
1766 assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1767 "Possible deadlock");
1768 }
1769
1770 ~ReleaseForegroundGC() {
1771 assert(!_c->_foregroundGCShouldWait, "Usage protocol violation?");
1772 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1773 _c->_foregroundGCShouldWait = true;
1774 }
1775 };
1776
1777 void CMSCollector::collect_in_background(GCCause::Cause cause) {
1778 assert(Thread::current()->is_ConcurrentGC_thread(),
1779 "A CMS asynchronous collection is only allowed on a CMS thread.");
1780
1781 GenCollectedHeap* gch = GenCollectedHeap::heap();
1782 {
1783 bool safepoint_check = Mutex::_no_safepoint_check_flag;
1784 MutexLockerEx hl(Heap_lock, safepoint_check);
1785 FreelistLocker fll(this);
1786 MutexLockerEx x(CGC_lock, safepoint_check);
1787 if (_foregroundGCIsActive || !UseAsyncConcMarkSweepGC) {
1788 // The foreground collector is active or we're
1789 // not using asynchronous collections. Skip this
1790 // background collection.
1791 assert(!_foregroundGCShouldWait, "Should be clear");
1792 return;
1793 } else {
1794 assert(_collectorState == Idling, "Should be idling before start.");
1795 _collectorState = InitialMarking;
1796 register_gc_start(cause);
1797 // Reset the expansion cause, now that we are about to begin
1798 // a new cycle.
1799 clear_expansion_cause();
1800
1801 // Clear the MetaspaceGC flag since a concurrent collection
1802 // is starting but also clear it after the collection.
1803 MetaspaceGC::set_should_concurrent_collect(false);
1804 }
1805 // Decide if we want to enable class unloading as part of the
1806 // ensuing concurrent GC cycle.
1807 update_should_unload_classes();
1808 _full_gc_requested = false; // acks all outstanding full gc requests
1809 _full_gc_cause = GCCause::_no_gc;
1810 // Signal that we are about to start a collection
1811 gch->increment_total_full_collections(); // ... starting a collection cycle
1812 _collection_count_start = gch->total_full_collections();
1813 }
1814
1815 // Used for PrintGC
1816 size_t prev_used;
1817 if (PrintGC && Verbose) {
1818 prev_used = _cmsGen->used();
1819 }
1820
1821 // The change of the collection state is normally done at this level;
1822 // the exceptions are phases that are executed while the world is
1823 // stopped. For those phases the change of state is done while the
1824 // world is stopped. For baton passing purposes this allows the
1825 // background collector to finish the phase and change state atomically.
1826 // The foreground collector cannot wait on a phase that is done
1827 // while the world is stopped because the foreground collector already
1828 // has the world stopped and would deadlock.
1829 while (_collectorState != Idling) {
1830 if (TraceCMSState) {
1831 gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
1832 p2i(Thread::current()), _collectorState);
1833 }
1834 // The foreground collector
1835 // holds the Heap_lock throughout its collection.
1836 // holds the CMS token (but not the lock)
1837 // except while it is waiting for the background collector to yield.
1838 //
1839 // The foreground collector should be blocked (not for long)
1840 // if the background collector is about to start a phase
1841 // executed with world stopped. If the background
1842 // collector has already started such a phase, the
1843 // foreground collector is blocked waiting for the
1844 // Heap_lock. The stop-world phases (InitialMarking and FinalMarking)
1845 // are executed in the VM thread.
1846 //
1847 // The locking order is
1848 // PendingListLock (PLL) -- if applicable (FinalMarking)
1849 // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue())
1850 // CMS token (claimed in
1851 // stop_world_and_do() -->
1852 // safepoint_synchronize() -->
1853 // CMSThread::synchronize())
1854
1855 {
1856 // Check if the FG collector wants us to yield.
1857 CMSTokenSync x(true); // is cms thread
1858 if (waitForForegroundGC()) {
1859 // We yielded to a foreground GC, nothing more to be
1860 // done this round.
1861 assert(_foregroundGCShouldWait == false, "We set it to false in "
1862 "waitForForegroundGC()");
1863 if (TraceCMSState) {
1864 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1865 " exiting collection CMS state %d",
1866 p2i(Thread::current()), _collectorState);
1867 }
1868 return;
1869 } else {
1870 // The background collector can run but check to see if the
1871 // foreground collector has done a collection while the
1872 // background collector was waiting to get the CGC_lock
1873 // above. If yes, break so that _foregroundGCShouldWait
1874 // is cleared before returning.
1875 if (_collectorState == Idling) {
1876 break;
1877 }
1878 }
1879 }
1880
1881 assert(_foregroundGCShouldWait, "Foreground collector, if active, "
1882 "should be waiting");
1883
1884 switch (_collectorState) {
1885 case InitialMarking:
1886 {
1887 ReleaseForegroundGC x(this);
1888 stats().record_cms_begin();
1889 VM_CMS_Initial_Mark initial_mark_op(this);
1890 VMThread::execute(&initial_mark_op);
1891 }
1892 // The collector state may be any legal state at this point
1893 // since the background collector may have yielded to the
1894 // foreground collector.
1895 break;
1896 case Marking:
1897 // initial marking in checkpointRootsInitialWork has been completed
1898 if (markFromRoots()) { // we were successful
1899 assert(_collectorState == Precleaning, "Collector state should "
1900 "have changed");
1901 } else {
1902 assert(_foregroundGCIsActive, "Internal state inconsistency");
1903 }
1904 break;
1905 case Precleaning:
1906 // marking from roots in markFromRoots has been completed
1907 preclean();
1908 assert(_collectorState == AbortablePreclean ||
1909 _collectorState == FinalMarking,
1910 "Collector state should have changed");
1911 break;
1912 case AbortablePreclean:
1913 abortable_preclean();
1914 assert(_collectorState == FinalMarking, "Collector state should "
1915 "have changed");
1916 break;
1917 case FinalMarking:
1918 {
1919 ReleaseForegroundGC x(this);
1920
1921 VM_CMS_Final_Remark final_remark_op(this);
1922 VMThread::execute(&final_remark_op);
1923 }
1924 assert(_foregroundGCShouldWait, "block post-condition");
1925 break;
1926 case Sweeping:
1927 // final marking in checkpointRootsFinal has been completed
1928 sweep();
1929 assert(_collectorState == Resizing, "Collector state change "
1930 "to Resizing must be done under the free_list_lock");
1931
1932 case Resizing: {
1933 // Sweeping has been completed...
1934 // At this point the background collection has completed.
1935 // Don't move the call to compute_new_size() down
1936 // into code that might be executed if the background
1937 // collection was preempted.
1938 {
1939 ReleaseForegroundGC x(this); // unblock FG collection
1940 MutexLockerEx y(Heap_lock, Mutex::_no_safepoint_check_flag);
1941 CMSTokenSync z(true); // not strictly needed.
1942 if (_collectorState == Resizing) {
1943 compute_new_size();
1944 save_heap_summary();
1945 _collectorState = Resetting;
1946 } else {
1947 assert(_collectorState == Idling, "The state should only change"
1948 " because the foreground collector has finished the collection");
1949 }
1950 }
1951 break;
1952 }
1953 case Resetting:
1954 // CMS heap resizing has been completed
1955 reset(true);
1956 assert(_collectorState == Idling, "Collector state should "
1957 "have changed");
1958
1959 MetaspaceGC::set_should_concurrent_collect(false);
1960
1961 stats().record_cms_end();
1962 // Don't move the concurrent_phases_end() and compute_new_size()
1963 // calls to here because a preempted background collection
1964 // has it's state set to "Resetting".
1965 break;
1966 case Idling:
1967 default:
1968 ShouldNotReachHere();
1969 break;
1970 }
1971 if (TraceCMSState) {
1972 gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
1973 p2i(Thread::current()), _collectorState);
1974 }
1975 assert(_foregroundGCShouldWait, "block post-condition");
1976 }
1977
1978 // Should this be in gc_epilogue?
1979 collector_policy()->counters()->update_counters();
1980
1981 {
1982 // Clear _foregroundGCShouldWait and, in the event that the
1983 // foreground collector is waiting, notify it, before
1984 // returning.
1985 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1986 _foregroundGCShouldWait = false;
1987 if (_foregroundGCIsActive) {
1988 CGC_lock->notify();
1989 }
1990 assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
1991 "Possible deadlock");
1992 }
1993 if (TraceCMSState) {
1994 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
1995 " exiting collection CMS state %d",
1996 p2i(Thread::current()), _collectorState);
1997 }
1998 if (PrintGC && Verbose) {
1999 _cmsGen->print_heap_change(prev_used);
2000 }
2001 }
2002
2003 void CMSCollector::register_gc_start(GCCause::Cause cause) {
2004 _cms_start_registered = true;
2005 _gc_timer_cm->register_gc_start();
2006 _gc_tracer_cm->report_gc_start(cause, _gc_timer_cm->gc_start());
2007 }
2008
2009 void CMSCollector::register_gc_end() {
2010 if (_cms_start_registered) {
2011 report_heap_summary(GCWhen::AfterGC);
2012
2013 _gc_timer_cm->register_gc_end();
2014 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
2015 _cms_start_registered = false;
2016 }
2017 }
2018
2019 void CMSCollector::save_heap_summary() {
2020 GenCollectedHeap* gch = GenCollectedHeap::heap();
2021 _last_heap_summary = gch->create_heap_summary();
2022 _last_metaspace_summary = gch->create_metaspace_summary();
2023 }
2024
2025 void CMSCollector::report_heap_summary(GCWhen::Type when) {
2026 _gc_tracer_cm->report_gc_heap_summary(when, _last_heap_summary);
2027 _gc_tracer_cm->report_metaspace_summary(when, _last_metaspace_summary);
2028 }
2029
2030 bool CMSCollector::waitForForegroundGC() {
2031 bool res = false;
2032 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
2033 "CMS thread should have CMS token");
2034 // Block the foreground collector until the
2035 // background collectors decides whether to
2036 // yield.
2037 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2038 _foregroundGCShouldWait = true;
2039 if (_foregroundGCIsActive) {
2040 // The background collector yields to the
2041 // foreground collector and returns a value
2042 // indicating that it has yielded. The foreground
2043 // collector can proceed.
2044 res = true;
2045 _foregroundGCShouldWait = false;
2046 ConcurrentMarkSweepThread::clear_CMS_flag(
2047 ConcurrentMarkSweepThread::CMS_cms_has_token);
2048 ConcurrentMarkSweepThread::set_CMS_flag(
2049 ConcurrentMarkSweepThread::CMS_cms_wants_token);
2050 // Get a possibly blocked foreground thread going
2051 CGC_lock->notify();
2052 if (TraceCMSState) {
2053 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d",
2054 p2i(Thread::current()), _collectorState);
2055 }
2056 while (_foregroundGCIsActive) {
2057 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
2058 }
2059 ConcurrentMarkSweepThread::set_CMS_flag(
2060 ConcurrentMarkSweepThread::CMS_cms_has_token);
2061 ConcurrentMarkSweepThread::clear_CMS_flag(
2062 ConcurrentMarkSweepThread::CMS_cms_wants_token);
2063 }
2064 if (TraceCMSState) {
2065 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d",
2066 p2i(Thread::current()), _collectorState);
2067 }
2068 return res;
2069 }
2070
2071 // Because of the need to lock the free lists and other structures in
2072 // the collector, common to all the generations that the collector is
2073 // collecting, we need the gc_prologues of individual CMS generations
2074 // delegate to their collector. It may have been simpler had the
2075 // current infrastructure allowed one to call a prologue on a
2076 // collector. In the absence of that we have the generation's
2077 // prologue delegate to the collector, which delegates back
2078 // some "local" work to a worker method in the individual generations
2079 // that it's responsible for collecting, while itself doing any
2080 // work common to all generations it's responsible for. A similar
2081 // comment applies to the gc_epilogue()'s.
2082 // The role of the variable _between_prologue_and_epilogue is to
2083 // enforce the invocation protocol.
2084 void CMSCollector::gc_prologue(bool full) {
2085 // Call gc_prologue_work() for the CMSGen
2086 // we are responsible for.
2087
2088 // The following locking discipline assumes that we are only called
2089 // when the world is stopped.
2090 assert(SafepointSynchronize::is_at_safepoint(), "world is stopped assumption");
2091
2092 // The CMSCollector prologue must call the gc_prologues for the
2093 // "generations" that it's responsible
2094 // for.
2095
2096 assert( Thread::current()->is_VM_thread()
2097 || ( CMSScavengeBeforeRemark
2098 && Thread::current()->is_ConcurrentGC_thread()),
2099 "Incorrect thread type for prologue execution");
2100
2101 if (_between_prologue_and_epilogue) {
2102 // We have already been invoked; this is a gc_prologue delegation
2103 // from yet another CMS generation that we are responsible for, just
2104 // ignore it since all relevant work has already been done.
2105 return;
2106 }
2107
2108 // set a bit saying prologue has been called; cleared in epilogue
2109 _between_prologue_and_epilogue = true;
2110 // Claim locks for common data structures, then call gc_prologue_work()
2111 // for each CMSGen.
2112
2113 getFreelistLocks(); // gets free list locks on constituent spaces
2114 bitMapLock()->lock_without_safepoint_check();
2115
2116 // Should call gc_prologue_work() for all cms gens we are responsible for
2117 bool duringMarking = _collectorState >= Marking
2118 && _collectorState < Sweeping;
2119
2120 // The young collections clear the modified oops state, which tells if
2121 // there are any modified oops in the class. The remark phase also needs
2122 // that information. Tell the young collection to save the union of all
2123 // modified klasses.
2124 if (duringMarking) {
2125 _ct->klass_rem_set()->set_accumulate_modified_oops(true);
2126 }
2127
2128 bool registerClosure = duringMarking;
2129
2130 _cmsGen->gc_prologue_work(full, registerClosure, &_modUnionClosurePar);
2131
2132 if (!full) {
2133 stats().record_gc0_begin();
2134 }
2135 }
2136
2137 void ConcurrentMarkSweepGeneration::gc_prologue(bool full) {
2138
2139 _capacity_at_prologue = capacity();
2140 _used_at_prologue = used();
2141
2142 // Delegate to CMScollector which knows how to coordinate between
2143 // this and any other CMS generations that it is responsible for
2144 // collecting.
2145 collector()->gc_prologue(full);
2146 }
2147
2148 // This is a "private" interface for use by this generation's CMSCollector.
2149 // Not to be called directly by any other entity (for instance,
2150 // GenCollectedHeap, which calls the "public" gc_prologue method above).
2151 void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
2152 bool registerClosure, ModUnionClosure* modUnionClosure) {
2153 assert(!incremental_collection_failed(), "Shouldn't be set yet");
2154 assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,
2155 "Should be NULL");
2156 if (registerClosure) {
2157 cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure);
2158 }
2159 cmsSpace()->gc_prologue();
2160 // Clear stat counters
2161 NOT_PRODUCT(
2162 assert(_numObjectsPromoted == 0, "check");
2163 assert(_numWordsPromoted == 0, "check");
2164 if (Verbose && PrintGC) {
2165 gclog_or_tty->print("Allocated "SIZE_FORMAT" objects, "
2166 SIZE_FORMAT" bytes concurrently",
2167 _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord));
2168 }
2169 _numObjectsAllocated = 0;
2170 _numWordsAllocated = 0;
2171 )
2172 }
2173
2174 void CMSCollector::gc_epilogue(bool full) {
2175 // The following locking discipline assumes that we are only called
2176 // when the world is stopped.
2177 assert(SafepointSynchronize::is_at_safepoint(),
2178 "world is stopped assumption");
2179
2180 // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks
2181 // if linear allocation blocks need to be appropriately marked to allow the
2182 // the blocks to be parsable. We also check here whether we need to nudge the
2183 // CMS collector thread to start a new cycle (if it's not already active).
2184 assert( Thread::current()->is_VM_thread()
2185 || ( CMSScavengeBeforeRemark
2186 && Thread::current()->is_ConcurrentGC_thread()),
2187 "Incorrect thread type for epilogue execution");
2188
2189 if (!_between_prologue_and_epilogue) {
2190 // We have already been invoked; this is a gc_epilogue delegation
2191 // from yet another CMS generation that we are responsible for, just
2192 // ignore it since all relevant work has already been done.
2193 return;
2194 }
2195 assert(haveFreelistLocks(), "must have freelist locks");
2196 assert_lock_strong(bitMapLock());
2197
2198 _ct->klass_rem_set()->set_accumulate_modified_oops(false);
2199
2200 _cmsGen->gc_epilogue_work(full);
2201
2202 if (_collectorState == AbortablePreclean || _collectorState == Precleaning) {
2203 // in case sampling was not already enabled, enable it
2204 _start_sampling = true;
2205 }
2206 // reset _eden_chunk_array so sampling starts afresh
2207 _eden_chunk_index = 0;
2208
2209 size_t cms_used = _cmsGen->cmsSpace()->used();
2210
2211 // update performance counters - this uses a special version of
2212 // update_counters() that allows the utilization to be passed as a
2213 // parameter, avoiding multiple calls to used().
2214 //
2215 _cmsGen->update_counters(cms_used);
2216
2217 bitMapLock()->unlock();
2218 releaseFreelistLocks();
2219
2220 if (!CleanChunkPoolAsync) {
2221 Chunk::clean_chunk_pool();
2222 }
2223
2224 set_did_compact(false);
2225 _between_prologue_and_epilogue = false; // ready for next cycle
2226 }
2227
2228 void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
2229 collector()->gc_epilogue(full);
2230
2231 // Also reset promotion tracking in par gc thread states.
2232 for (uint i = 0; i < ParallelGCThreads; i++) {
2233 _par_gc_thread_states[i]->promo.stopTrackingPromotions(i);
2234 }
2235 }
2236
2237 void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) {
2238 assert(!incremental_collection_failed(), "Should have been cleared");
2239 cmsSpace()->setPreconsumptionDirtyCardClosure(NULL);
2240 cmsSpace()->gc_epilogue();
2241 // Print stat counters
2242 NOT_PRODUCT(
2243 assert(_numObjectsAllocated == 0, "check");
2244 assert(_numWordsAllocated == 0, "check");
2245 if (Verbose && PrintGC) {
2246 gclog_or_tty->print("Promoted "SIZE_FORMAT" objects, "
2247 SIZE_FORMAT" bytes",
2248 _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord));
2249 }
2250 _numObjectsPromoted = 0;
2251 _numWordsPromoted = 0;
2252 )
2253
2254 if (PrintGC && Verbose) {
2255 // Call down the chain in contiguous_available needs the freelistLock
2256 // so print this out before releasing the freeListLock.
2257 gclog_or_tty->print(" Contiguous available "SIZE_FORMAT" bytes ",
2258 contiguous_available());
2259 }
2260 }
2261
2262 #ifndef PRODUCT
2263 bool CMSCollector::have_cms_token() {
2264 Thread* thr = Thread::current();
2265 if (thr->is_VM_thread()) {
2266 return ConcurrentMarkSweepThread::vm_thread_has_cms_token();
2267 } else if (thr->is_ConcurrentGC_thread()) {
2268 return ConcurrentMarkSweepThread::cms_thread_has_cms_token();
2269 } else if (thr->is_GC_task_thread()) {
2270 return ConcurrentMarkSweepThread::vm_thread_has_cms_token() &&
2271 ParGCRareEvent_lock->owned_by_self();
2272 }
2273 return false;
2274 }
2275 #endif
2276
2277 // Check reachability of the given heap address in CMS generation,
2278 // treating all other generations as roots.
2279 bool CMSCollector::is_cms_reachable(HeapWord* addr) {
2280 // We could "guarantee" below, rather than assert, but I'll
2281 // leave these as "asserts" so that an adventurous debugger
2282 // could try this in the product build provided some subset of
2283 // the conditions were met, provided they were interested in the
2284 // results and knew that the computation below wouldn't interfere
2285 // with other concurrent computations mutating the structures
2286 // being read or written.
2287 assert(SafepointSynchronize::is_at_safepoint(),
2288 "Else mutations in object graph will make answer suspect");
2289 assert(have_cms_token(), "Should hold cms token");
2290 assert(haveFreelistLocks(), "must hold free list locks");
2291 assert_lock_strong(bitMapLock());
2292
2293 // Clear the marking bit map array before starting, but, just
2294 // for kicks, first report if the given address is already marked
2295 gclog_or_tty->print_cr("Start: Address " PTR_FORMAT " is%s marked", p2i(addr),
2296 _markBitMap.isMarked(addr) ? "" : " not");
2297
2298 if (verify_after_remark()) {
2299 MutexLockerEx x(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2300 bool result = verification_mark_bm()->isMarked(addr);
2301 gclog_or_tty->print_cr("TransitiveMark: Address " PTR_FORMAT " %s marked", p2i(addr),
2302 result ? "IS" : "is NOT");
2303 return result;
2304 } else {
2305 gclog_or_tty->print_cr("Could not compute result");
2306 return false;
2307 }
2308 }
2309
2310
2311 void
2312 CMSCollector::print_on_error(outputStream* st) {
2313 CMSCollector* collector = ConcurrentMarkSweepGeneration::_collector;
2314 if (collector != NULL) {
2315 CMSBitMap* bitmap = &collector->_markBitMap;
2316 st->print_cr("Marking Bits: (CMSBitMap*) " PTR_FORMAT, p2i(bitmap));
2317 bitmap->print_on_error(st, " Bits: ");
2318
2319 st->cr();
2320
2321 CMSBitMap* mut_bitmap = &collector->_modUnionTable;
2322 st->print_cr("Mod Union Table: (CMSBitMap*) " PTR_FORMAT, p2i(mut_bitmap));
2323 mut_bitmap->print_on_error(st, " Bits: ");
2324 }
2325 }
2326
2327 ////////////////////////////////////////////////////////
2328 // CMS Verification Support
2329 ////////////////////////////////////////////////////////
2330 // Following the remark phase, the following invariant
2331 // should hold -- each object in the CMS heap which is
2332 // marked in markBitMap() should be marked in the verification_mark_bm().
2333
2334 class VerifyMarkedClosure: public BitMapClosure {
2335 CMSBitMap* _marks;
2336 bool _failed;
2337
2338 public:
2339 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2340
2341 bool do_bit(size_t offset) {
2342 HeapWord* addr = _marks->offsetToHeapWord(offset);
2343 if (!_marks->isMarked(addr)) {
2344 oop(addr)->print_on(gclog_or_tty);
2345 gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", p2i(addr));
2346 _failed = true;
2347 }
2348 return true;
2349 }
2350
2351 bool failed() { return _failed; }
2352 };
2353
2354 bool CMSCollector::verify_after_remark(bool silent) {
2355 if (!silent) gclog_or_tty->print(" [Verifying CMS Marking... ");
2356 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2357 static bool init = false;
2358
2359 assert(SafepointSynchronize::is_at_safepoint(),
2360 "Else mutations in object graph will make answer suspect");
2361 assert(have_cms_token(),
2362 "Else there may be mutual interference in use of "
2363 " verification data structures");
2364 assert(_collectorState > Marking && _collectorState <= Sweeping,
2365 "Else marking info checked here may be obsolete");
2366 assert(haveFreelistLocks(), "must hold free list locks");
2367 assert_lock_strong(bitMapLock());
2368
2369
2370 // Allocate marking bit map if not already allocated
2371 if (!init) { // first time
2372 if (!verification_mark_bm()->allocate(_span)) {
2373 return false;
2374 }
2375 init = true;
2376 }
2377
2378 assert(verification_mark_stack()->isEmpty(), "Should be empty");
2379
2380 // Turn off refs discovery -- so we will be tracing through refs.
2381 // This is as intended, because by this time
2382 // GC must already have cleared any refs that need to be cleared,
2383 // and traced those that need to be marked; moreover,
2384 // the marking done here is not going to interfere in any
2385 // way with the marking information used by GC.
2386 NoRefDiscovery no_discovery(ref_processor());
2387
2388 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
2389
2390 // Clear any marks from a previous round
2391 verification_mark_bm()->clear_all();
2392 assert(verification_mark_stack()->isEmpty(), "markStack should be empty");
2393 verify_work_stacks_empty();
2394
2395 GenCollectedHeap* gch = GenCollectedHeap::heap();
2396 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2397 // Update the saved marks which may affect the root scans.
2398 gch->save_marks();
2399
2400 if (CMSRemarkVerifyVariant == 1) {
2401 // In this first variant of verification, we complete
2402 // all marking, then check if the new marks-vector is
2403 // a subset of the CMS marks-vector.
2404 verify_after_remark_work_1();
2405 } else if (CMSRemarkVerifyVariant == 2) {
2406 // In this second variant of verification, we flag an error
2407 // (i.e. an object reachable in the new marks-vector not reachable
2408 // in the CMS marks-vector) immediately, also indicating the
2409 // identify of an object (A) that references the unmarked object (B) --
2410 // presumably, a mutation to A failed to be picked up by preclean/remark?
2411 verify_after_remark_work_2();
2412 } else {
2413 warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant",
2414 CMSRemarkVerifyVariant);
2415 }
2416 if (!silent) gclog_or_tty->print(" done] ");
2417 return true;
2418 }
2419
2420 void CMSCollector::verify_after_remark_work_1() {
2421 ResourceMark rm;
2422 HandleMark hm;
2423 GenCollectedHeap* gch = GenCollectedHeap::heap();
2424
2425 // Get a clear set of claim bits for the roots processing to work with.
2426 ClassLoaderDataGraph::clear_claimed_marks();
2427
2428 // Mark from roots one level into CMS
2429 MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
2430 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2431
2432 gch->gen_process_roots(_cmsGen->level(),
2433 true, // younger gens are roots
2434 true, // activate StrongRootsScope
2435 GenCollectedHeap::ScanningOption(roots_scanning_options()),
2436 should_unload_classes(),
2437 ¬Older,
2438 NULL,
2439 NULL); // SSS: Provide correct closure
2440
2441 // Now mark from the roots
2442 MarkFromRootsClosure markFromRootsClosure(this, _span,
2443 verification_mark_bm(), verification_mark_stack(),
2444 false /* don't yield */, true /* verifying */);
2445 assert(_restart_addr == NULL, "Expected pre-condition");
2446 verification_mark_bm()->iterate(&markFromRootsClosure);
2447 while (_restart_addr != NULL) {
2448 // Deal with stack overflow: by restarting at the indicated
2449 // address.
2450 HeapWord* ra = _restart_addr;
2451 markFromRootsClosure.reset(ra);
2452 _restart_addr = NULL;
2453 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2454 }
2455 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2456 verify_work_stacks_empty();
2457
2458 // Marking completed -- now verify that each bit marked in
2459 // verification_mark_bm() is also marked in markBitMap(); flag all
2460 // errors by printing corresponding objects.
2461 VerifyMarkedClosure vcl(markBitMap());
2462 verification_mark_bm()->iterate(&vcl);
2463 if (vcl.failed()) {
2464 gclog_or_tty->print("Verification failed");
2465 gch->print_on(gclog_or_tty);
2466 fatal("CMS: failed marking verification after remark");
2467 }
2468 }
2469
2470 class VerifyKlassOopsKlassClosure : public KlassClosure {
2471 class VerifyKlassOopsClosure : public OopClosure {
2472 CMSBitMap* _bitmap;
2473 public:
2474 VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { }
2475 void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); }
2476 void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2477 } _oop_closure;
2478 public:
2479 VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {}
2480 void do_klass(Klass* k) {
2481 k->oops_do(&_oop_closure);
2482 }
2483 };
2484
2485 void CMSCollector::verify_after_remark_work_2() {
2486 ResourceMark rm;
2487 HandleMark hm;
2488 GenCollectedHeap* gch = GenCollectedHeap::heap();
2489
2490 // Get a clear set of claim bits for the roots processing to work with.
2491 ClassLoaderDataGraph::clear_claimed_marks();
2492
2493 // Mark from roots one level into CMS
2494 MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(),
2495 markBitMap());
2496 CLDToOopClosure cld_closure(¬Older, true);
2497
2498 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2499
2500 gch->gen_process_roots(_cmsGen->level(),
2501 true, // younger gens are roots
2502 true, // activate StrongRootsScope
2503 GenCollectedHeap::ScanningOption(roots_scanning_options()),
2504 should_unload_classes(),
2505 ¬Older,
2506 NULL,
2507 &cld_closure);
2508
2509 // Now mark from the roots
2510 MarkFromRootsVerifyClosure markFromRootsClosure(this, _span,
2511 verification_mark_bm(), markBitMap(), verification_mark_stack());
2512 assert(_restart_addr == NULL, "Expected pre-condition");
2513 verification_mark_bm()->iterate(&markFromRootsClosure);
2514 while (_restart_addr != NULL) {
2515 // Deal with stack overflow: by restarting at the indicated
2516 // address.
2517 HeapWord* ra = _restart_addr;
2518 markFromRootsClosure.reset(ra);
2519 _restart_addr = NULL;
2520 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2521 }
2522 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2523 verify_work_stacks_empty();
2524
2525 VerifyKlassOopsKlassClosure verify_klass_oops(verification_mark_bm());
2526 ClassLoaderDataGraph::classes_do(&verify_klass_oops);
2527
2528 // Marking completed -- now verify that each bit marked in
2529 // verification_mark_bm() is also marked in markBitMap(); flag all
2530 // errors by printing corresponding objects.
2531 VerifyMarkedClosure vcl(markBitMap());
2532 verification_mark_bm()->iterate(&vcl);
2533 assert(!vcl.failed(), "Else verification above should not have succeeded");
2534 }
2535
2536 void ConcurrentMarkSweepGeneration::save_marks() {
2537 // delegate to CMS space
2538 cmsSpace()->save_marks();
2539 for (uint i = 0; i < ParallelGCThreads; i++) {
2540 _par_gc_thread_states[i]->promo.startTrackingPromotions();
2541 }
2542 }
2543
2544 bool ConcurrentMarkSweepGeneration::no_allocs_since_save_marks() {
2545 return cmsSpace()->no_allocs_since_save_marks();
2546 }
2547
2548 #define CMS_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \
2549 \
2550 void ConcurrentMarkSweepGeneration:: \
2551 oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \
2552 cl->set_generation(this); \
2553 cmsSpace()->oop_since_save_marks_iterate##nv_suffix(cl); \
2554 cl->reset_generation(); \
2555 save_marks(); \
2556 }
2557
2558 ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DEFN)
2559
2560 void
2561 ConcurrentMarkSweepGeneration::oop_iterate(ExtendedOopClosure* cl) {
2562 if (freelistLock()->owned_by_self()) {
2563 Generation::oop_iterate(cl);
2564 } else {
2565 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
2566 Generation::oop_iterate(cl);
2567 }
2568 }
2569
2570 void
2571 ConcurrentMarkSweepGeneration::object_iterate(ObjectClosure* cl) {
2572 if (freelistLock()->owned_by_self()) {
2573 Generation::object_iterate(cl);
2574 } else {
2575 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
2576 Generation::object_iterate(cl);
2577 }
2578 }
2579
2580 void
2581 ConcurrentMarkSweepGeneration::safe_object_iterate(ObjectClosure* cl) {
2582 if (freelistLock()->owned_by_self()) {
2583 Generation::safe_object_iterate(cl);
2584 } else {
2585 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
2586 Generation::safe_object_iterate(cl);
2587 }
2588 }
2589
2590 void
2591 ConcurrentMarkSweepGeneration::post_compact() {
2592 }
2593
2594 void
2595 ConcurrentMarkSweepGeneration::prepare_for_verify() {
2596 // Fix the linear allocation blocks to look like free blocks.
2597
2598 // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those
2599 // are not called when the heap is verified during universe initialization and
2600 // at vm shutdown.
2601 if (freelistLock()->owned_by_self()) {
2602 cmsSpace()->prepare_for_verify();
2603 } else {
2604 MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag);
2605 cmsSpace()->prepare_for_verify();
2606 }
2607 }
2608
2609 void
2610 ConcurrentMarkSweepGeneration::verify() {
2611 // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those
2612 // are not called when the heap is verified during universe initialization and
2613 // at vm shutdown.
2614 if (freelistLock()->owned_by_self()) {
2615 cmsSpace()->verify();
2616 } else {
2617 MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag);
2618 cmsSpace()->verify();
2619 }
2620 }
2621
2622 void CMSCollector::verify() {
2623 _cmsGen->verify();
2624 }
2625
2626 #ifndef PRODUCT
2627 bool CMSCollector::overflow_list_is_empty() const {
2628 assert(_num_par_pushes >= 0, "Inconsistency");
2629 if (_overflow_list == NULL) {
2630 assert(_num_par_pushes == 0, "Inconsistency");
2631 }
2632 return _overflow_list == NULL;
2633 }
2634
2635 // The methods verify_work_stacks_empty() and verify_overflow_empty()
2636 // merely consolidate assertion checks that appear to occur together frequently.
2637 void CMSCollector::verify_work_stacks_empty() const {
2638 assert(_markStack.isEmpty(), "Marking stack should be empty");
2639 assert(overflow_list_is_empty(), "Overflow list should be empty");
2640 }
2641
2642 void CMSCollector::verify_overflow_empty() const {
2643 assert(overflow_list_is_empty(), "Overflow list should be empty");
2644 assert(no_preserved_marks(), "No preserved marks");
2645 }
2646 #endif // PRODUCT
2647
2648 // Decide if we want to enable class unloading as part of the
2649 // ensuing concurrent GC cycle. We will collect and
2650 // unload classes if it's the case that:
2651 // (1) an explicit gc request has been made and the flag
2652 // ExplicitGCInvokesConcurrentAndUnloadsClasses is set, OR
2653 // (2) (a) class unloading is enabled at the command line, and
2654 // (b) old gen is getting really full
2655 // NOTE: Provided there is no change in the state of the heap between
2656 // calls to this method, it should have idempotent results. Moreover,
2657 // its results should be monotonically increasing (i.e. going from 0 to 1,
2658 // but not 1 to 0) between successive calls between which the heap was
2659 // not collected. For the implementation below, it must thus rely on
2660 // the property that concurrent_cycles_since_last_unload()
2661 // will not decrease unless a collection cycle happened and that
2662 // _cmsGen->is_too_full() are
2663 // themselves also monotonic in that sense. See check_monotonicity()
2664 // below.
2665 void CMSCollector::update_should_unload_classes() {
2666 _should_unload_classes = false;
2667 // Condition 1 above
2668 if (_full_gc_requested && ExplicitGCInvokesConcurrentAndUnloadsClasses) {
2669 _should_unload_classes = true;
2670 } else if (CMSClassUnloadingEnabled) { // Condition 2.a above
2671 // Disjuncts 2.b.(i,ii,iii) above
2672 _should_unload_classes = (concurrent_cycles_since_last_unload() >=
2673 CMSClassUnloadingMaxInterval)
2674 || _cmsGen->is_too_full();
2675 }
2676 }
2677
2678 bool ConcurrentMarkSweepGeneration::is_too_full() const {
2679 bool res = should_concurrent_collect();
2680 res = res && (occupancy() > (double)CMSIsTooFullPercentage/100.0);
2681 return res;
2682 }
2683
2684 void CMSCollector::setup_cms_unloading_and_verification_state() {
2685 const bool should_verify = VerifyBeforeGC || VerifyAfterGC || VerifyDuringGC
2686 || VerifyBeforeExit;
2687 const int rso = GenCollectedHeap::SO_AllCodeCache;
2688
2689 // We set the proper root for this CMS cycle here.
2690 if (should_unload_classes()) { // Should unload classes this cycle
2691 remove_root_scanning_option(rso); // Shrink the root set appropriately
2692 set_verifying(should_verify); // Set verification state for this cycle
2693 return; // Nothing else needs to be done at this time
2694 }
2695
2696 // Not unloading classes this cycle
2697 assert(!should_unload_classes(), "Inconsistency!");
2698
2699 if ((!verifying() || unloaded_classes_last_cycle()) && should_verify) {
2700 // Include symbols, strings and code cache elements to prevent their resurrection.
2701 add_root_scanning_option(rso);
2702 set_verifying(true);
2703 } else if (verifying() && !should_verify) {
2704 // We were verifying, but some verification flags got disabled.
2705 set_verifying(false);
2706 // Exclude symbols, strings and code cache elements from root scanning to
2707 // reduce IM and RM pauses.
2708 remove_root_scanning_option(rso);
2709 }
2710 }
2711
2712
2713 #ifndef PRODUCT
2714 HeapWord* CMSCollector::block_start(const void* p) const {
2715 const HeapWord* addr = (HeapWord*)p;
2716 if (_span.contains(p)) {
2717 if (_cmsGen->cmsSpace()->is_in_reserved(addr)) {
2718 return _cmsGen->cmsSpace()->block_start(p);
2719 }
2720 }
2721 return NULL;
2722 }
2723 #endif
2724
2725 HeapWord*
2726 ConcurrentMarkSweepGeneration::expand_and_allocate(size_t word_size,
2727 bool tlab,
2728 bool parallel) {
2729 CMSSynchronousYieldRequest yr;
2730 assert(!tlab, "Can't deal with TLAB allocation");
2731 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
2732 expand_for_gc_cause(word_size*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_allocation);
2733 if (GCExpandToAllocateDelayMillis > 0) {
2734 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2735 }
2736 return have_lock_and_allocate(word_size, tlab);
2737 }
2738
2739 void ConcurrentMarkSweepGeneration::expand_for_gc_cause(
2740 size_t bytes,
2741 size_t expand_bytes,
2742 CMSExpansionCause::Cause cause)
2743 {
2744
2745 bool success = expand(bytes, expand_bytes);
2746
2747 // remember why we expanded; this information is used
2748 // by shouldConcurrentCollect() when making decisions on whether to start
2749 // a new CMS cycle.
2750 if (success) {
2751 set_expansion_cause(cause);
2752 if (PrintGCDetails && Verbose) {
2753 gclog_or_tty->print_cr("Expanded CMS gen for %s",
2754 CMSExpansionCause::to_string(cause));
2755 }
2756 }
2757 }
2758
2759 HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) {
2760 HeapWord* res = NULL;
2761 MutexLocker x(ParGCRareEvent_lock);
2762 while (true) {
2763 // Expansion by some other thread might make alloc OK now:
2764 res = ps->lab.alloc(word_sz);
2765 if (res != NULL) return res;
2766 // If there's not enough expansion space available, give up.
2767 if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) {
2768 return NULL;
2769 }
2770 // Otherwise, we try expansion.
2771 expand_for_gc_cause(word_sz*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_allocate_par_lab);
2772 // Now go around the loop and try alloc again;
2773 // A competing par_promote might beat us to the expansion space,
2774 // so we may go around the loop again if promotion fails again.
2775 if (GCExpandToAllocateDelayMillis > 0) {
2776 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2777 }
2778 }
2779 }
2780
2781
2782 bool ConcurrentMarkSweepGeneration::expand_and_ensure_spooling_space(
2783 PromotionInfo* promo) {
2784 MutexLocker x(ParGCRareEvent_lock);
2785 size_t refill_size_bytes = promo->refillSize() * HeapWordSize;
2786 while (true) {
2787 // Expansion by some other thread might make alloc OK now:
2788 if (promo->ensure_spooling_space()) {
2789 assert(promo->has_spooling_space(),
2790 "Post-condition of successful ensure_spooling_space()");
2791 return true;
2792 }
2793 // If there's not enough expansion space available, give up.
2794 if (_virtual_space.uncommitted_size() < refill_size_bytes) {
2795 return false;
2796 }
2797 // Otherwise, we try expansion.
2798 expand_for_gc_cause(refill_size_bytes, MinHeapDeltaBytes, CMSExpansionCause::_allocate_par_spooling_space);
2799 // Now go around the loop and try alloc again;
2800 // A competing allocation might beat us to the expansion space,
2801 // so we may go around the loop again if allocation fails again.
2802 if (GCExpandToAllocateDelayMillis > 0) {
2803 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
2804 }
2805 }
2806 }
2807
2808 void ConcurrentMarkSweepGeneration::shrink(size_t bytes) {
2809 // Only shrink if a compaction was done so that all the free space
2810 // in the generation is in a contiguous block at the end.
2811 if (did_compact()) {
2812 CardGeneration::shrink(bytes);
2813 }
2814 }
2815
2816 void ConcurrentMarkSweepGeneration::assert_correct_size_change_locking() {
2817 assert_locked_or_safepoint(Heap_lock);
2818 }
2819
2820 void ConcurrentMarkSweepGeneration::shrink_free_list_by(size_t bytes) {
2821 assert_locked_or_safepoint(Heap_lock);
2822 assert_lock_strong(freelistLock());
2823 if (PrintGCDetails && Verbose) {
2824 warning("Shrinking of CMS not yet implemented");
2825 }
2826 return;
2827 }
2828
2829
2830 // Simple ctor/dtor wrapper for accounting & timer chores around concurrent
2831 // phases.
2832 class CMSPhaseAccounting: public StackObj {
2833 public:
2834 CMSPhaseAccounting(CMSCollector *collector,
2835 const char *phase,
2836 const GCId gc_id,
2837 bool print_cr = true);
2838 ~CMSPhaseAccounting();
2839
2840 private:
2841 CMSCollector *_collector;
2842 const char *_phase;
2843 elapsedTimer _wallclock;
2844 bool _print_cr;
2845 const GCId _gc_id;
2846
2847 public:
2848 // Not MT-safe; so do not pass around these StackObj's
2849 // where they may be accessed by other threads.
2850 jlong wallclock_millis() {
2851 assert(_wallclock.is_active(), "Wall clock should not stop");
2852 _wallclock.stop(); // to record time
2853 jlong ret = _wallclock.milliseconds();
2854 _wallclock.start(); // restart
2855 return ret;
2856 }
2857 };
2858
2859 CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector,
2860 const char *phase,
2861 const GCId gc_id,
2862 bool print_cr) :
2863 _collector(collector), _phase(phase), _print_cr(print_cr), _gc_id(gc_id) {
2864
2865 if (PrintCMSStatistics != 0) {
2866 _collector->resetYields();
2867 }
2868 if (PrintGCDetails) {
2869 gclog_or_tty->gclog_stamp(_gc_id);
2870 gclog_or_tty->print_cr("[%s-concurrent-%s-start]",
2871 _collector->cmsGen()->short_name(), _phase);
2872 }
2873 _collector->resetTimer();
2874 _wallclock.start();
2875 _collector->startTimer();
2876 }
2877
2878 CMSPhaseAccounting::~CMSPhaseAccounting() {
2879 assert(_wallclock.is_active(), "Wall clock should not have stopped");
2880 _collector->stopTimer();
2881 _wallclock.stop();
2882 if (PrintGCDetails) {
2883 gclog_or_tty->gclog_stamp(_gc_id);
2884 gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]",
2885 _collector->cmsGen()->short_name(),
2886 _phase, _collector->timerValue(), _wallclock.seconds());
2887 if (_print_cr) {
2888 gclog_or_tty->cr();
2889 }
2890 if (PrintCMSStatistics != 0) {
2891 gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase,
2892 _collector->yields());
2893 }
2894 }
2895 }
2896
2897 // CMS work
2898
2899 // The common parts of CMSParInitialMarkTask and CMSParRemarkTask.
2900 class CMSParMarkTask : public AbstractGangTask {
2901 protected:
2902 CMSCollector* _collector;
2903 int _n_workers;
2904 CMSParMarkTask(const char* name, CMSCollector* collector, int n_workers) :
2905 AbstractGangTask(name),
2906 _collector(collector),
2907 _n_workers(n_workers) {}
2908 // Work method in support of parallel rescan ... of young gen spaces
2909 void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl,
2910 ContiguousSpace* space,
2911 HeapWord** chunk_array, size_t chunk_top);
2912 void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl);
2913 };
2914
2915 // Parallel initial mark task
2916 class CMSParInitialMarkTask: public CMSParMarkTask {
2917 public:
2918 CMSParInitialMarkTask(CMSCollector* collector, int n_workers) :
2919 CMSParMarkTask("Scan roots and young gen for initial mark in parallel",
2920 collector, n_workers) {}
2921 void work(uint worker_id);
2922 };
2923
2924 // Checkpoint the roots into this generation from outside
2925 // this generation. [Note this initial checkpoint need only
2926 // be approximate -- we'll do a catch up phase subsequently.]
2927 void CMSCollector::checkpointRootsInitial() {
2928 assert(_collectorState == InitialMarking, "Wrong collector state");
2929 check_correct_thread_executing();
2930 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
2931
2932 save_heap_summary();
2933 report_heap_summary(GCWhen::BeforeGC);
2934
2935 ReferenceProcessor* rp = ref_processor();
2936 assert(_restart_addr == NULL, "Control point invariant");
2937 {
2938 // acquire locks for subsequent manipulations
2939 MutexLockerEx x(bitMapLock(),
2940 Mutex::_no_safepoint_check_flag);
2941 checkpointRootsInitialWork();
2942 // enable ("weak") refs discovery
2943 rp->enable_discovery();
2944 _collectorState = Marking;
2945 }
2946 }
2947
2948 void CMSCollector::checkpointRootsInitialWork() {
2949 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
2950 assert(_collectorState == InitialMarking, "just checking");
2951
2952 // If there has not been a GC[n-1] since last GC[n] cycle completed,
2953 // precede our marking with a collection of all
2954 // younger generations to keep floating garbage to a minimum.
2955 // XXX: we won't do this for now -- it's an optimization to be done later.
2956
2957 // already have locks
2958 assert_lock_strong(bitMapLock());
2959 assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
2960
2961 // Setup the verification and class unloading state for this
2962 // CMS collection cycle.
2963 setup_cms_unloading_and_verification_state();
2964
2965 NOT_PRODUCT(GCTraceTime t("\ncheckpointRootsInitialWork",
2966 PrintGCDetails && Verbose, true, _gc_timer_cm, _gc_tracer_cm->gc_id());)
2967
2968 // Reset all the PLAB chunk arrays if necessary.
2969 if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
2970 reset_survivor_plab_arrays();
2971 }
2972
2973 ResourceMark rm;
2974 HandleMark hm;
2975
2976 MarkRefsIntoClosure notOlder(_span, &_markBitMap);
2977 GenCollectedHeap* gch = GenCollectedHeap::heap();
2978
2979 verify_work_stacks_empty();
2980 verify_overflow_empty();
2981
2982 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2983 // Update the saved marks which may affect the root scans.
2984 gch->save_marks();
2985
2986 // weak reference processing has not started yet.
2987 ref_processor()->set_enqueuing_is_done(false);
2988
2989 // Need to remember all newly created CLDs,
2990 // so that we can guarantee that the remark finds them.
2991 ClassLoaderDataGraph::remember_new_clds(true);
2992
2993 // Whenever a CLD is found, it will be claimed before proceeding to mark
2994 // the klasses. The claimed marks need to be cleared before marking starts.
2995 ClassLoaderDataGraph::clear_claimed_marks();
2996
2997 if (CMSPrintEdenSurvivorChunks) {
2998 print_eden_and_survivor_chunk_arrays();
2999 }
3000
3001 {
3002 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
3003 if (CMSParallelInitialMarkEnabled) {
3004 // The parallel version.
3005 FlexibleWorkGang* workers = gch->workers();
3006 assert(workers != NULL, "Need parallel worker threads.");
3007 int n_workers = workers->active_workers();
3008 CMSParInitialMarkTask tsk(this, n_workers);
3009 gch->set_par_threads(n_workers);
3010 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
3011 if (n_workers > 1) {
3012 StrongRootsScope srs;
3013 workers->run_task(&tsk);
3014 } else {
3015 StrongRootsScope srs;
3016 tsk.work(0);
3017 }
3018 gch->set_par_threads(0);
3019 } else {
3020 // The serial version.
3021 CLDToOopClosure cld_closure(¬Older, true);
3022 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
3023 gch->gen_process_roots(_cmsGen->level(),
3024 true, // younger gens are roots
3025 true, // activate StrongRootsScope
3026 GenCollectedHeap::ScanningOption(roots_scanning_options()),
3027 should_unload_classes(),
3028 ¬Older,
3029 NULL,
3030 &cld_closure);
3031 }
3032 }
3033
3034 // Clear mod-union table; it will be dirtied in the prologue of
3035 // CMS generation per each younger generation collection.
3036
3037 assert(_modUnionTable.isAllClear(),
3038 "Was cleared in most recent final checkpoint phase"
3039 " or no bits are set in the gc_prologue before the start of the next "
3040 "subsequent marking phase.");
3041
3042 assert(_ct->klass_rem_set()->mod_union_is_clear(), "Must be");
3043
3044 // Save the end of the used_region of the constituent generations
3045 // to be used to limit the extent of sweep in each generation.
3046 save_sweep_limits();
3047 verify_overflow_empty();
3048 }
3049
3050 bool CMSCollector::markFromRoots() {
3051 // we might be tempted to assert that:
3052 // assert(!SafepointSynchronize::is_at_safepoint(),
3053 // "inconsistent argument?");
3054 // However that wouldn't be right, because it's possible that
3055 // a safepoint is indeed in progress as a younger generation
3056 // stop-the-world GC happens even as we mark in this generation.
3057 assert(_collectorState == Marking, "inconsistent state?");
3058 check_correct_thread_executing();
3059 verify_overflow_empty();
3060
3061 // Weak ref discovery note: We may be discovering weak
3062 // refs in this generation concurrent (but interleaved) with
3063 // weak ref discovery by a younger generation collector.
3064
3065 CMSTokenSyncWithLocks ts(true, bitMapLock());
3066 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3067 CMSPhaseAccounting pa(this, "mark", _gc_tracer_cm->gc_id(), !PrintGCDetails);
3068 bool res = markFromRootsWork();
3069 if (res) {
3070 _collectorState = Precleaning;
3071 } else { // We failed and a foreground collection wants to take over
3072 assert(_foregroundGCIsActive, "internal state inconsistency");
3073 assert(_restart_addr == NULL, "foreground will restart from scratch");
3074 if (PrintGCDetails) {
3075 gclog_or_tty->print_cr("bailing out to foreground collection");
3076 }
3077 }
3078 verify_overflow_empty();
3079 return res;
3080 }
3081
3082 bool CMSCollector::markFromRootsWork() {
3083 // iterate over marked bits in bit map, doing a full scan and mark
3084 // from these roots using the following algorithm:
3085 // . if oop is to the right of the current scan pointer,
3086 // mark corresponding bit (we'll process it later)
3087 // . else (oop is to left of current scan pointer)
3088 // push oop on marking stack
3089 // . drain the marking stack
3090
3091 // Note that when we do a marking step we need to hold the
3092 // bit map lock -- recall that direct allocation (by mutators)
3093 // and promotion (by younger generation collectors) is also
3094 // marking the bit map. [the so-called allocate live policy.]
3095 // Because the implementation of bit map marking is not
3096 // robust wrt simultaneous marking of bits in the same word,
3097 // we need to make sure that there is no such interference
3098 // between concurrent such updates.
3099
3100 // already have locks
3101 assert_lock_strong(bitMapLock());
3102
3103 verify_work_stacks_empty();
3104 verify_overflow_empty();
3105 bool result = false;
3106 if (CMSConcurrentMTEnabled && ConcGCThreads > 0) {
3107 result = do_marking_mt();
3108 } else {
3109 result = do_marking_st();
3110 }
3111 return result;
3112 }
3113
3114 // Forward decl
3115 class CMSConcMarkingTask;
3116
3117 class CMSConcMarkingTerminator: public ParallelTaskTerminator {
3118 CMSCollector* _collector;
3119 CMSConcMarkingTask* _task;
3120 public:
3121 virtual void yield();
3122
3123 // "n_threads" is the number of threads to be terminated.
3124 // "queue_set" is a set of work queues of other threads.
3125 // "collector" is the CMS collector associated with this task terminator.
3126 // "yield" indicates whether we need the gang as a whole to yield.
3127 CMSConcMarkingTerminator(int n_threads, TaskQueueSetSuper* queue_set, CMSCollector* collector) :
3128 ParallelTaskTerminator(n_threads, queue_set),
3129 _collector(collector) { }
3130
3131 void set_task(CMSConcMarkingTask* task) {
3132 _task = task;
3133 }
3134 };
3135
3136 class CMSConcMarkingTerminatorTerminator: public TerminatorTerminator {
3137 CMSConcMarkingTask* _task;
3138 public:
3139 bool should_exit_termination();
3140 void set_task(CMSConcMarkingTask* task) {
3141 _task = task;
3142 }
3143 };
3144
3145 // MT Concurrent Marking Task
3146 class CMSConcMarkingTask: public YieldingFlexibleGangTask {
3147 CMSCollector* _collector;
3148 int _n_workers; // requested/desired # workers
3149 bool _result;
3150 CompactibleFreeListSpace* _cms_space;
3151 char _pad_front[64]; // padding to ...
3152 HeapWord* _global_finger; // ... avoid sharing cache line
3153 char _pad_back[64];
3154 HeapWord* _restart_addr;
3155
3156 // Exposed here for yielding support
3157 Mutex* const _bit_map_lock;
3158
3159 // The per thread work queues, available here for stealing
3160 OopTaskQueueSet* _task_queues;
3161
3162 // Termination (and yielding) support
3163 CMSConcMarkingTerminator _term;
3164 CMSConcMarkingTerminatorTerminator _term_term;
3165
3166 public:
3167 CMSConcMarkingTask(CMSCollector* collector,
3168 CompactibleFreeListSpace* cms_space,
3169 YieldingFlexibleWorkGang* workers,
3170 OopTaskQueueSet* task_queues):
3171 YieldingFlexibleGangTask("Concurrent marking done multi-threaded"),
3172 _collector(collector),
3173 _cms_space(cms_space),
3174 _n_workers(0), _result(true),
3175 _task_queues(task_queues),
3176 _term(_n_workers, task_queues, _collector),
3177 _bit_map_lock(collector->bitMapLock())
3178 {
3179 _requested_size = _n_workers;
3180 _term.set_task(this);
3181 _term_term.set_task(this);
3182 _restart_addr = _global_finger = _cms_space->bottom();
3183 }
3184
3185
3186 OopTaskQueueSet* task_queues() { return _task_queues; }
3187
3188 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
3189
3190 HeapWord** global_finger_addr() { return &_global_finger; }
3191
3192 CMSConcMarkingTerminator* terminator() { return &_term; }
3193
3194 virtual void set_for_termination(int active_workers) {
3195 terminator()->reset_for_reuse(active_workers);
3196 }
3197
3198 void work(uint worker_id);
3199 bool should_yield() {
3200 return ConcurrentMarkSweepThread::should_yield()
3201 && !_collector->foregroundGCIsActive();
3202 }
3203
3204 virtual void coordinator_yield(); // stuff done by coordinator
3205 bool result() { return _result; }
3206
3207 void reset(HeapWord* ra) {
3208 assert(_global_finger >= _cms_space->end(), "Postcondition of ::work(i)");
3209 _restart_addr = _global_finger = ra;
3210 _term.reset_for_reuse();
3211 }
3212
3213 static bool get_work_from_overflow_stack(CMSMarkStack* ovflw_stk,
3214 OopTaskQueue* work_q);
3215
3216 private:
3217 void do_scan_and_mark(int i, CompactibleFreeListSpace* sp);
3218 void do_work_steal(int i);
3219 void bump_global_finger(HeapWord* f);
3220 };
3221
3222 bool CMSConcMarkingTerminatorTerminator::should_exit_termination() {
3223 assert(_task != NULL, "Error");
3224 return _task->yielding();
3225 // Note that we do not need the disjunct || _task->should_yield() above
3226 // because we want terminating threads to yield only if the task
3227 // is already in the midst of yielding, which happens only after at least one
3228 // thread has yielded.
3229 }
3230
3231 void CMSConcMarkingTerminator::yield() {
3232 if (_task->should_yield()) {
3233 _task->yield();
3234 } else {
3235 ParallelTaskTerminator::yield();
3236 }
3237 }
3238
3239 ////////////////////////////////////////////////////////////////
3240 // Concurrent Marking Algorithm Sketch
3241 ////////////////////////////////////////////////////////////////
3242 // Until all tasks exhausted (both spaces):
3243 // -- claim next available chunk
3244 // -- bump global finger via CAS
3245 // -- find first object that starts in this chunk
3246 // and start scanning bitmap from that position
3247 // -- scan marked objects for oops
3248 // -- CAS-mark target, and if successful:
3249 // . if target oop is above global finger (volatile read)
3250 // nothing to do
3251 // . if target oop is in chunk and above local finger
3252 // then nothing to do
3253 // . else push on work-queue
3254 // -- Deal with possible overflow issues:
3255 // . local work-queue overflow causes stuff to be pushed on
3256 // global (common) overflow queue
3257 // . always first empty local work queue
3258 // . then get a batch of oops from global work queue if any
3259 // . then do work stealing
3260 // -- When all tasks claimed (both spaces)
3261 // and local work queue empty,
3262 // then in a loop do:
3263 // . check global overflow stack; steal a batch of oops and trace
3264 // . try to steal from other threads oif GOS is empty
3265 // . if neither is available, offer termination
3266 // -- Terminate and return result
3267 //
3268 void CMSConcMarkingTask::work(uint worker_id) {
3269 elapsedTimer _timer;
3270 ResourceMark rm;
3271 HandleMark hm;
3272
3273 DEBUG_ONLY(_collector->verify_overflow_empty();)
3274
3275 // Before we begin work, our work queue should be empty
3276 assert(work_queue(worker_id)->size() == 0, "Expected to be empty");
3277 // Scan the bitmap covering _cms_space, tracing through grey objects.
3278 _timer.start();
3279 do_scan_and_mark(worker_id, _cms_space);
3280 _timer.stop();
3281 if (PrintCMSStatistics != 0) {
3282 gclog_or_tty->print_cr("Finished cms space scanning in %dth thread: %3.3f sec",
3283 worker_id, _timer.seconds());
3284 // XXX: need xxx/xxx type of notation, two timers
3285 }
3286
3287 // ... do work stealing
3288 _timer.reset();
3289 _timer.start();
3290 do_work_steal(worker_id);
3291 _timer.stop();
3292 if (PrintCMSStatistics != 0) {
3293 gclog_or_tty->print_cr("Finished work stealing in %dth thread: %3.3f sec",
3294 worker_id, _timer.seconds());
3295 // XXX: need xxx/xxx type of notation, two timers
3296 }
3297 assert(_collector->_markStack.isEmpty(), "Should have been emptied");
3298 assert(work_queue(worker_id)->size() == 0, "Should have been emptied");
3299 // Note that under the current task protocol, the
3300 // following assertion is true even of the spaces
3301 // expanded since the completion of the concurrent
3302 // marking. XXX This will likely change under a strict
3303 // ABORT semantics.
3304 // After perm removal the comparison was changed to
3305 // greater than or equal to from strictly greater than.
3306 // Before perm removal the highest address sweep would
3307 // have been at the end of perm gen but now is at the
3308 // end of the tenured gen.
3309 assert(_global_finger >= _cms_space->end(),
3310 "All tasks have been completed");
3311 DEBUG_ONLY(_collector->verify_overflow_empty();)
3312 }
3313
3314 void CMSConcMarkingTask::bump_global_finger(HeapWord* f) {
3315 HeapWord* read = _global_finger;
3316 HeapWord* cur = read;
3317 while (f > read) {
3318 cur = read;
3319 read = (HeapWord*) Atomic::cmpxchg_ptr(f, &_global_finger, cur);
3320 if (cur == read) {
3321 // our cas succeeded
3322 assert(_global_finger >= f, "protocol consistency");
3323 break;
3324 }
3325 }
3326 }
3327
3328 // This is really inefficient, and should be redone by
3329 // using (not yet available) block-read and -write interfaces to the
3330 // stack and the work_queue. XXX FIX ME !!!
3331 bool CMSConcMarkingTask::get_work_from_overflow_stack(CMSMarkStack* ovflw_stk,
3332 OopTaskQueue* work_q) {
3333 // Fast lock-free check
3334 if (ovflw_stk->length() == 0) {
3335 return false;
3336 }
3337 assert(work_q->size() == 0, "Shouldn't steal");
3338 MutexLockerEx ml(ovflw_stk->par_lock(),
3339 Mutex::_no_safepoint_check_flag);
3340 // Grab up to 1/4 the size of the work queue
3341 size_t num = MIN2((size_t)(work_q->max_elems() - work_q->size())/4,
3342 (size_t)ParGCDesiredObjsFromOverflowList);
3343 num = MIN2(num, ovflw_stk->length());
3344 for (int i = (int) num; i > 0; i--) {
3345 oop cur = ovflw_stk->pop();
3346 assert(cur != NULL, "Counted wrong?");
3347 work_q->push(cur);
3348 }
3349 return num > 0;
3350 }
3351
3352 void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
3353 SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
3354 int n_tasks = pst->n_tasks();
3355 // We allow that there may be no tasks to do here because
3356 // we are restarting after a stack overflow.
3357 assert(pst->valid() || n_tasks == 0, "Uninitialized use?");
3358 uint nth_task = 0;
3359
3360 HeapWord* aligned_start = sp->bottom();
3361 if (sp->used_region().contains(_restart_addr)) {
3362 // Align down to a card boundary for the start of 0th task
3363 // for this space.
3364 aligned_start =
3365 (HeapWord*)align_size_down((uintptr_t)_restart_addr,
3366 CardTableModRefBS::card_size);
3367 }
3368
3369 size_t chunk_size = sp->marking_task_size();
3370 while (!pst->is_task_claimed(/* reference */ nth_task)) {
3371 // Having claimed the nth task in this space,
3372 // compute the chunk that it corresponds to:
3373 MemRegion span = MemRegion(aligned_start + nth_task*chunk_size,
3374 aligned_start + (nth_task+1)*chunk_size);
3375 // Try and bump the global finger via a CAS;
3376 // note that we need to do the global finger bump
3377 // _before_ taking the intersection below, because
3378 // the task corresponding to that region will be
3379 // deemed done even if the used_region() expands
3380 // because of allocation -- as it almost certainly will
3381 // during start-up while the threads yield in the
3382 // closure below.
3383 HeapWord* finger = span.end();
3384 bump_global_finger(finger); // atomically
3385 // There are null tasks here corresponding to chunks
3386 // beyond the "top" address of the space.
3387 span = span.intersection(sp->used_region());
3388 if (!span.is_empty()) { // Non-null task
3389 HeapWord* prev_obj;
3390 assert(!span.contains(_restart_addr) || nth_task == 0,
3391 "Inconsistency");
3392 if (nth_task == 0) {
3393 // For the 0th task, we'll not need to compute a block_start.
3394 if (span.contains(_restart_addr)) {
3395 // In the case of a restart because of stack overflow,
3396 // we might additionally skip a chunk prefix.
3397 prev_obj = _restart_addr;
3398 } else {
3399 prev_obj = span.start();
3400 }
3401 } else {
3402 // We want to skip the first object because
3403 // the protocol is to scan any object in its entirety
3404 // that _starts_ in this span; a fortiori, any
3405 // object starting in an earlier span is scanned
3406 // as part of an earlier claimed task.
3407 // Below we use the "careful" version of block_start
3408 // so we do not try to navigate uninitialized objects.
3409 prev_obj = sp->block_start_careful(span.start());
3410 // Below we use a variant of block_size that uses the
3411 // Printezis bits to avoid waiting for allocated
3412 // objects to become initialized/parsable.
3413 while (prev_obj < span.start()) {
3414 size_t sz = sp->block_size_no_stall(prev_obj, _collector);
3415 if (sz > 0) {
3416 prev_obj += sz;
3417 } else {
3418 // In this case we may end up doing a bit of redundant
3419 // scanning, but that appears unavoidable, short of
3420 // locking the free list locks; see bug 6324141.
3421 break;
3422 }
3423 }
3424 }
3425 if (prev_obj < span.end()) {
3426 MemRegion my_span = MemRegion(prev_obj, span.end());
3427 // Do the marking work within a non-empty span --
3428 // the last argument to the constructor indicates whether the
3429 // iteration should be incremental with periodic yields.
3430 Par_MarkFromRootsClosure cl(this, _collector, my_span,
3431 &_collector->_markBitMap,
3432 work_queue(i),
3433 &_collector->_markStack);
3434 _collector->_markBitMap.iterate(&cl, my_span.start(), my_span.end());
3435 } // else nothing to do for this task
3436 } // else nothing to do for this task
3437 }
3438 // We'd be tempted to assert here that since there are no
3439 // more tasks left to claim in this space, the global_finger
3440 // must exceed space->top() and a fortiori space->end(). However,
3441 // that would not quite be correct because the bumping of
3442 // global_finger occurs strictly after the claiming of a task,
3443 // so by the time we reach here the global finger may not yet
3444 // have been bumped up by the thread that claimed the last
3445 // task.
3446 pst->all_tasks_completed();
3447 }
3448
3449 class Par_ConcMarkingClosure: public MetadataAwareOopClosure {
3450 private:
3451 CMSCollector* _collector;
3452 CMSConcMarkingTask* _task;
3453 MemRegion _span;
3454 CMSBitMap* _bit_map;
3455 CMSMarkStack* _overflow_stack;
3456 OopTaskQueue* _work_queue;
3457 protected:
3458 DO_OOP_WORK_DEFN
3459 public:
3460 Par_ConcMarkingClosure(CMSCollector* collector, CMSConcMarkingTask* task, OopTaskQueue* work_queue,
3461 CMSBitMap* bit_map, CMSMarkStack* overflow_stack):
3462 MetadataAwareOopClosure(collector->ref_processor()),
3463 _collector(collector),
3464 _task(task),
3465 _span(collector->_span),
3466 _work_queue(work_queue),
3467 _bit_map(bit_map),
3468 _overflow_stack(overflow_stack)
3469 { }
3470 virtual void do_oop(oop* p);
3471 virtual void do_oop(narrowOop* p);
3472
3473 void trim_queue(size_t max);
3474 void handle_stack_overflow(HeapWord* lost);
3475 void do_yield_check() {
3476 if (_task->should_yield()) {
3477 _task->yield();
3478 }
3479 }
3480 };
3481
3482 // Grey object scanning during work stealing phase --
3483 // the salient assumption here is that any references
3484 // that are in these stolen objects being scanned must
3485 // already have been initialized (else they would not have
3486 // been published), so we do not need to check for
3487 // uninitialized objects before pushing here.
3488 void Par_ConcMarkingClosure::do_oop(oop obj) {
3489 assert(obj->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj)));
3490 HeapWord* addr = (HeapWord*)obj;
3491 // Check if oop points into the CMS generation
3492 // and is not marked
3493 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
3494 // a white object ...
3495 // If we manage to "claim" the object, by being the
3496 // first thread to mark it, then we push it on our
3497 // marking stack
3498 if (_bit_map->par_mark(addr)) { // ... now grey
3499 // push on work queue (grey set)
3500 bool simulate_overflow = false;
3501 NOT_PRODUCT(
3502 if (CMSMarkStackOverflowALot &&
3503 _collector->simulate_overflow()) {
3504 // simulate a stack overflow
3505 simulate_overflow = true;
3506 }
3507 )
3508 if (simulate_overflow ||
3509 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
3510 // stack overflow
3511 if (PrintCMSStatistics != 0) {
3512 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
3513 SIZE_FORMAT, _overflow_stack->capacity());
3514 }
3515 // We cannot assert that the overflow stack is full because
3516 // it may have been emptied since.
3517 assert(simulate_overflow ||
3518 _work_queue->size() == _work_queue->max_elems(),
3519 "Else push should have succeeded");
3520 handle_stack_overflow(addr);
3521 }
3522 } // Else, some other thread got there first
3523 do_yield_check();
3524 }
3525 }
3526
3527 void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
3528 void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
3529
3530 void Par_ConcMarkingClosure::trim_queue(size_t max) {
3531 while (_work_queue->size() > max) {
3532 oop new_oop;
3533 if (_work_queue->pop_local(new_oop)) {
3534 assert(new_oop->is_oop(), "Should be an oop");
3535 assert(_bit_map->isMarked((HeapWord*)new_oop), "Grey object");
3536 assert(_span.contains((HeapWord*)new_oop), "Not in span");
3537 new_oop->oop_iterate(this); // do_oop() above
3538 do_yield_check();
3539 }
3540 }
3541 }
3542
3543 // Upon stack overflow, we discard (part of) the stack,
3544 // remembering the least address amongst those discarded
3545 // in CMSCollector's _restart_address.
3546 void Par_ConcMarkingClosure::handle_stack_overflow(HeapWord* lost) {
3547 // We need to do this under a mutex to prevent other
3548 // workers from interfering with the work done below.
3549 MutexLockerEx ml(_overflow_stack->par_lock(),
3550 Mutex::_no_safepoint_check_flag);
3551 // Remember the least grey address discarded
3552 HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost);
3553 _collector->lower_restart_addr(ra);
3554 _overflow_stack->reset(); // discard stack contents
3555 _overflow_stack->expand(); // expand the stack if possible
3556 }
3557
3558
3559 void CMSConcMarkingTask::do_work_steal(int i) {
3560 OopTaskQueue* work_q = work_queue(i);
3561 oop obj_to_scan;
3562 CMSBitMap* bm = &(_collector->_markBitMap);
3563 CMSMarkStack* ovflw = &(_collector->_markStack);
3564 int* seed = _collector->hash_seed(i);
3565 Par_ConcMarkingClosure cl(_collector, this, work_q, bm, ovflw);
3566 while (true) {
3567 cl.trim_queue(0);
3568 assert(work_q->size() == 0, "Should have been emptied above");
3569 if (get_work_from_overflow_stack(ovflw, work_q)) {
3570 // Can't assert below because the work obtained from the
3571 // overflow stack may already have been stolen from us.
3572 // assert(work_q->size() > 0, "Work from overflow stack");
3573 continue;
3574 } else if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
3575 assert(obj_to_scan->is_oop(), "Should be an oop");
3576 assert(bm->isMarked((HeapWord*)obj_to_scan), "Grey object");
3577 obj_to_scan->oop_iterate(&cl);
3578 } else if (terminator()->offer_termination(&_term_term)) {
3579 assert(work_q->size() == 0, "Impossible!");
3580 break;
3581 } else if (yielding() || should_yield()) {
3582 yield();
3583 }
3584 }
3585 }
3586
3587 // This is run by the CMS (coordinator) thread.
3588 void CMSConcMarkingTask::coordinator_yield() {
3589 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
3590 "CMS thread should hold CMS token");
3591 // First give up the locks, then yield, then re-lock
3592 // We should probably use a constructor/destructor idiom to
3593 // do this unlock/lock or modify the MutexUnlocker class to
3594 // serve our purpose. XXX
3595 assert_lock_strong(_bit_map_lock);
3596 _bit_map_lock->unlock();
3597 ConcurrentMarkSweepThread::desynchronize(true);
3598 _collector->stopTimer();
3599 if (PrintCMSStatistics != 0) {
3600 _collector->incrementYields();
3601 }
3602
3603 // It is possible for whichever thread initiated the yield request
3604 // not to get a chance to wake up and take the bitmap lock between
3605 // this thread releasing it and reacquiring it. So, while the
3606 // should_yield() flag is on, let's sleep for a bit to give the
3607 // other thread a chance to wake up. The limit imposed on the number
3608 // of iterations is defensive, to avoid any unforseen circumstances
3609 // putting us into an infinite loop. Since it's always been this
3610 // (coordinator_yield()) method that was observed to cause the
3611 // problem, we are using a parameter (CMSCoordinatorYieldSleepCount)
3612 // which is by default non-zero. For the other seven methods that
3613 // also perform the yield operation, as are using a different
3614 // parameter (CMSYieldSleepCount) which is by default zero. This way we
3615 // can enable the sleeping for those methods too, if necessary.
3616 // See 6442774.
3617 //
3618 // We really need to reconsider the synchronization between the GC
3619 // thread and the yield-requesting threads in the future and we
3620 // should really use wait/notify, which is the recommended
3621 // way of doing this type of interaction. Additionally, we should
3622 // consolidate the eight methods that do the yield operation and they
3623 // are almost identical into one for better maintainability and
3624 // readability. See 6445193.
3625 //
3626 // Tony 2006.06.29
3627 for (unsigned i = 0; i < CMSCoordinatorYieldSleepCount &&
3628 ConcurrentMarkSweepThread::should_yield() &&
3629 !CMSCollector::foregroundGCIsActive(); ++i) {
3630 os::sleep(Thread::current(), 1, false);
3631 }
3632
3633 ConcurrentMarkSweepThread::synchronize(true);
3634 _bit_map_lock->lock_without_safepoint_check();
3635 _collector->startTimer();
3636 }
3637
3638 bool CMSCollector::do_marking_mt() {
3639 assert(ConcGCThreads > 0 && conc_workers() != NULL, "precondition");
3640 int num_workers = AdaptiveSizePolicy::calc_active_conc_workers(
3641 conc_workers()->total_workers(),
3642 conc_workers()->active_workers(),
3643 Threads::number_of_non_daemon_threads());
3644 conc_workers()->set_active_workers(num_workers);
3645
3646 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
3647
3648 CMSConcMarkingTask tsk(this,
3649 cms_space,
3650 conc_workers(),
3651 task_queues());
3652
3653 // Since the actual number of workers we get may be different
3654 // from the number we requested above, do we need to do anything different
3655 // below? In particular, may be we need to subclass the SequantialSubTasksDone
3656 // class?? XXX
3657 cms_space ->initialize_sequential_subtasks_for_marking(num_workers);
3658
3659 // Refs discovery is already non-atomic.
3660 assert(!ref_processor()->discovery_is_atomic(), "Should be non-atomic");
3661 assert(ref_processor()->discovery_is_mt(), "Discovery should be MT");
3662 conc_workers()->start_task(&tsk);
3663 while (tsk.yielded()) {
3664 tsk.coordinator_yield();
3665 conc_workers()->continue_task(&tsk);
3666 }
3667 // If the task was aborted, _restart_addr will be non-NULL
3668 assert(tsk.completed() || _restart_addr != NULL, "Inconsistency");
3669 while (_restart_addr != NULL) {
3670 // XXX For now we do not make use of ABORTED state and have not
3671 // yet implemented the right abort semantics (even in the original
3672 // single-threaded CMS case). That needs some more investigation
3673 // and is deferred for now; see CR# TBF. 07252005YSR. XXX
3674 assert(!CMSAbortSemantics || tsk.aborted(), "Inconsistency");
3675 // If _restart_addr is non-NULL, a marking stack overflow
3676 // occurred; we need to do a fresh marking iteration from the
3677 // indicated restart address.
3678 if (_foregroundGCIsActive) {
3679 // We may be running into repeated stack overflows, having
3680 // reached the limit of the stack size, while making very
3681 // slow forward progress. It may be best to bail out and
3682 // let the foreground collector do its job.
3683 // Clear _restart_addr, so that foreground GC
3684 // works from scratch. This avoids the headache of
3685 // a "rescan" which would otherwise be needed because
3686 // of the dirty mod union table & card table.
3687 _restart_addr = NULL;
3688 return false;
3689 }
3690 // Adjust the task to restart from _restart_addr
3691 tsk.reset(_restart_addr);
3692 cms_space ->initialize_sequential_subtasks_for_marking(num_workers,
3693 _restart_addr);
3694 _restart_addr = NULL;
3695 // Get the workers going again
3696 conc_workers()->start_task(&tsk);
3697 while (tsk.yielded()) {
3698 tsk.coordinator_yield();
3699 conc_workers()->continue_task(&tsk);
3700 }
3701 }
3702 assert(tsk.completed(), "Inconsistency");
3703 assert(tsk.result() == true, "Inconsistency");
3704 return true;
3705 }
3706
3707 bool CMSCollector::do_marking_st() {
3708 ResourceMark rm;
3709 HandleMark hm;
3710
3711 // Temporarily make refs discovery single threaded (non-MT)
3712 ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false);
3713 MarkFromRootsClosure markFromRootsClosure(this, _span, &_markBitMap,
3714 &_markStack, CMSYield);
3715 // the last argument to iterate indicates whether the iteration
3716 // should be incremental with periodic yields.
3717 _markBitMap.iterate(&markFromRootsClosure);
3718 // If _restart_addr is non-NULL, a marking stack overflow
3719 // occurred; we need to do a fresh iteration from the
3720 // indicated restart address.
3721 while (_restart_addr != NULL) {
3722 if (_foregroundGCIsActive) {
3723 // We may be running into repeated stack overflows, having
3724 // reached the limit of the stack size, while making very
3725 // slow forward progress. It may be best to bail out and
3726 // let the foreground collector do its job.
3727 // Clear _restart_addr, so that foreground GC
3728 // works from scratch. This avoids the headache of
3729 // a "rescan" which would otherwise be needed because
3730 // of the dirty mod union table & card table.
3731 _restart_addr = NULL;
3732 return false; // indicating failure to complete marking
3733 }
3734 // Deal with stack overflow:
3735 // we restart marking from _restart_addr
3736 HeapWord* ra = _restart_addr;
3737 markFromRootsClosure.reset(ra);
3738 _restart_addr = NULL;
3739 _markBitMap.iterate(&markFromRootsClosure, ra, _span.end());
3740 }
3741 return true;
3742 }
3743
3744 void CMSCollector::preclean() {
3745 check_correct_thread_executing();
3746 assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
3747 verify_work_stacks_empty();
3748 verify_overflow_empty();
3749 _abort_preclean = false;
3750 if (CMSPrecleaningEnabled) {
3751 if (!CMSEdenChunksRecordAlways) {
3752 _eden_chunk_index = 0;
3753 }
3754 size_t used = get_eden_used();
3755 size_t capacity = get_eden_capacity();
3756 // Don't start sampling unless we will get sufficiently
3757 // many samples.
3758 if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
3759 * CMSScheduleRemarkEdenPenetration)) {
3760 _start_sampling = true;
3761 } else {
3762 _start_sampling = false;
3763 }
3764 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3765 CMSPhaseAccounting pa(this, "preclean", _gc_tracer_cm->gc_id(), !PrintGCDetails);
3766 preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
3767 }
3768 CMSTokenSync x(true); // is cms thread
3769 if (CMSPrecleaningEnabled) {
3770 sample_eden();
3771 _collectorState = AbortablePreclean;
3772 } else {
3773 _collectorState = FinalMarking;
3774 }
3775 verify_work_stacks_empty();
3776 verify_overflow_empty();
3777 }
3778
3779 // Try and schedule the remark such that young gen
3780 // occupancy is CMSScheduleRemarkEdenPenetration %.
3781 void CMSCollector::abortable_preclean() {
3782 check_correct_thread_executing();
3783 assert(CMSPrecleaningEnabled, "Inconsistent control state");
3784 assert(_collectorState == AbortablePreclean, "Inconsistent control state");
3785
3786 // If Eden's current occupancy is below this threshold,
3787 // immediately schedule the remark; else preclean
3788 // past the next scavenge in an effort to
3789 // schedule the pause as described above. By choosing
3790 // CMSScheduleRemarkEdenSizeThreshold >= max eden size
3791 // we will never do an actual abortable preclean cycle.
3792 if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) {
3793 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3794 CMSPhaseAccounting pa(this, "abortable-preclean", _gc_tracer_cm->gc_id(), !PrintGCDetails);
3795 // We need more smarts in the abortable preclean
3796 // loop below to deal with cases where allocation
3797 // in young gen is very very slow, and our precleaning
3798 // is running a losing race against a horde of
3799 // mutators intent on flooding us with CMS updates
3800 // (dirty cards).
3801 // One, admittedly dumb, strategy is to give up
3802 // after a certain number of abortable precleaning loops
3803 // or after a certain maximum time. We want to make
3804 // this smarter in the next iteration.
3805 // XXX FIX ME!!! YSR
3806 size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0;
3807 while (!(should_abort_preclean() ||
3808 ConcurrentMarkSweepThread::should_terminate())) {
3809 workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2);
3810 cumworkdone += workdone;
3811 loops++;
3812 // Voluntarily terminate abortable preclean phase if we have
3813 // been at it for too long.
3814 if ((CMSMaxAbortablePrecleanLoops != 0) &&
3815 loops >= CMSMaxAbortablePrecleanLoops) {
3816 if (PrintGCDetails) {
3817 gclog_or_tty->print(" CMS: abort preclean due to loops ");
3818 }
3819 break;
3820 }
3821 if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) {
3822 if (PrintGCDetails) {
3823 gclog_or_tty->print(" CMS: abort preclean due to time ");
3824 }
3825 break;
3826 }
3827 // If we are doing little work each iteration, we should
3828 // take a short break.
3829 if (workdone < CMSAbortablePrecleanMinWorkPerIteration) {
3830 // Sleep for some time, waiting for work to accumulate
3831 stopTimer();
3832 cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis);
3833 startTimer();
3834 waited++;
3835 }
3836 }
3837 if (PrintCMSStatistics > 0) {
3838 gclog_or_tty->print(" [" SIZE_FORMAT " iterations, " SIZE_FORMAT " waits, " SIZE_FORMAT " cards)] ",
3839 loops, waited, cumworkdone);
3840 }
3841 }
3842 CMSTokenSync x(true); // is cms thread
3843 if (_collectorState != Idling) {
3844 assert(_collectorState == AbortablePreclean,
3845 "Spontaneous state transition?");
3846 _collectorState = FinalMarking;
3847 } // Else, a foreground collection completed this CMS cycle.
3848 return;
3849 }
3850
3851 // Respond to an Eden sampling opportunity
3852 void CMSCollector::sample_eden() {
3853 // Make sure a young gc cannot sneak in between our
3854 // reading and recording of a sample.
3855 assert(Thread::current()->is_ConcurrentGC_thread(),
3856 "Only the cms thread may collect Eden samples");
3857 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
3858 "Should collect samples while holding CMS token");
3859 if (!_start_sampling) {
3860 return;
3861 }
3862 // When CMSEdenChunksRecordAlways is true, the eden chunk array
3863 // is populated by the young generation.
3864 if (_eden_chunk_array != NULL && !CMSEdenChunksRecordAlways) {
3865 if (_eden_chunk_index < _eden_chunk_capacity) {
3866 _eden_chunk_array[_eden_chunk_index] = *_top_addr; // take sample
3867 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
3868 "Unexpected state of Eden");
3869 // We'd like to check that what we just sampled is an oop-start address;
3870 // however, we cannot do that here since the object may not yet have been
3871 // initialized. So we'll instead do the check when we _use_ this sample
3872 // later.
3873 if (_eden_chunk_index == 0 ||
3874 (pointer_delta(_eden_chunk_array[_eden_chunk_index],
3875 _eden_chunk_array[_eden_chunk_index-1])
3876 >= CMSSamplingGrain)) {
3877 _eden_chunk_index++; // commit sample
3878 }
3879 }
3880 }
3881 if ((_collectorState == AbortablePreclean) && !_abort_preclean) {
3882 size_t used = get_eden_used();
3883 size_t capacity = get_eden_capacity();
3884 assert(used <= capacity, "Unexpected state of Eden");
3885 if (used > (capacity/100 * CMSScheduleRemarkEdenPenetration)) {
3886 _abort_preclean = true;
3887 }
3888 }
3889 }
3890
3891
3892 size_t CMSCollector::preclean_work(bool clean_refs, bool clean_survivor) {
3893 assert(_collectorState == Precleaning ||
3894 _collectorState == AbortablePreclean, "incorrect state");
3895 ResourceMark rm;
3896 HandleMark hm;
3897
3898 // Precleaning is currently not MT but the reference processor
3899 // may be set for MT. Disable it temporarily here.
3900 ReferenceProcessor* rp = ref_processor();
3901 ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(rp, false);
3902
3903 // Do one pass of scrubbing the discovered reference lists
3904 // to remove any reference objects with strongly-reachable
3905 // referents.
3906 if (clean_refs) {
3907 CMSPrecleanRefsYieldClosure yield_cl(this);
3908 assert(rp->span().equals(_span), "Spans should be equal");
3909 CMSKeepAliveClosure keep_alive(this, _span, &_markBitMap,
3910 &_markStack, true /* preclean */);
3911 CMSDrainMarkingStackClosure complete_trace(this,
3912 _span, &_markBitMap, &_markStack,
3913 &keep_alive, true /* preclean */);
3914
3915 // We don't want this step to interfere with a young
3916 // collection because we don't want to take CPU
3917 // or memory bandwidth away from the young GC threads
3918 // (which may be as many as there are CPUs).
3919 // Note that we don't need to protect ourselves from
3920 // interference with mutators because they can't
3921 // manipulate the discovered reference lists nor affect
3922 // the computed reachability of the referents, the
3923 // only properties manipulated by the precleaning
3924 // of these reference lists.
3925 stopTimer();
3926 CMSTokenSyncWithLocks x(true /* is cms thread */,
3927 bitMapLock());
3928 startTimer();
3929 sample_eden();
3930
3931 // The following will yield to allow foreground
3932 // collection to proceed promptly. XXX YSR:
3933 // The code in this method may need further
3934 // tweaking for better performance and some restructuring
3935 // for cleaner interfaces.
3936 GCTimer *gc_timer = NULL; // Currently not tracing concurrent phases
3937 rp->preclean_discovered_references(
3938 rp->is_alive_non_header(), &keep_alive, &complete_trace, &yield_cl,
3939 gc_timer, _gc_tracer_cm->gc_id());
3940 }
3941
3942 if (clean_survivor) { // preclean the active survivor space(s)
3943 PushAndMarkClosure pam_cl(this, _span, ref_processor(),
3944 &_markBitMap, &_modUnionTable,
3945 &_markStack, true /* precleaning phase */);
3946 stopTimer();
3947 CMSTokenSyncWithLocks ts(true /* is cms thread */,
3948 bitMapLock());
3949 startTimer();
3950 unsigned int before_count =
3951 GenCollectedHeap::heap()->total_collections();
3952 SurvivorSpacePrecleanClosure
3953 sss_cl(this, _span, &_markBitMap, &_markStack,
3954 &pam_cl, before_count, CMSYield);
3955 _young_gen->from()->object_iterate_careful(&sss_cl);
3956 _young_gen->to()->object_iterate_careful(&sss_cl);
3957 }
3958 MarkRefsIntoAndScanClosure
3959 mrias_cl(_span, ref_processor(), &_markBitMap, &_modUnionTable,
3960 &_markStack, this, CMSYield,
3961 true /* precleaning phase */);
3962 // CAUTION: The following closure has persistent state that may need to
3963 // be reset upon a decrease in the sequence of addresses it
3964 // processes.
3965 ScanMarkedObjectsAgainCarefullyClosure
3966 smoac_cl(this, _span,
3967 &_markBitMap, &_markStack, &mrias_cl, CMSYield);
3968
3969 // Preclean dirty cards in ModUnionTable and CardTable using
3970 // appropriate convergence criterion;
3971 // repeat CMSPrecleanIter times unless we find that
3972 // we are losing.
3973 assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large");
3974 assert(CMSPrecleanNumerator < CMSPrecleanDenominator,
3975 "Bad convergence multiplier");
3976 assert(CMSPrecleanThreshold >= 100,
3977 "Unreasonably low CMSPrecleanThreshold");
3978
3979 size_t numIter, cumNumCards, lastNumCards, curNumCards;
3980 for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0;
3981 numIter < CMSPrecleanIter;
3982 numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) {
3983 curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl);
3984 if (Verbose && PrintGCDetails) {
3985 gclog_or_tty->print(" (modUnionTable: " SIZE_FORMAT " cards)", curNumCards);
3986 }
3987 // Either there are very few dirty cards, so re-mark
3988 // pause will be small anyway, or our pre-cleaning isn't
3989 // that much faster than the rate at which cards are being
3990 // dirtied, so we might as well stop and re-mark since
3991 // precleaning won't improve our re-mark time by much.
3992 if (curNumCards <= CMSPrecleanThreshold ||
3993 (numIter > 0 &&
3994 (curNumCards * CMSPrecleanDenominator >
3995 lastNumCards * CMSPrecleanNumerator))) {
3996 numIter++;
3997 cumNumCards += curNumCards;
3998 break;
3999 }
4000 }
4001
4002 preclean_klasses(&mrias_cl, _cmsGen->freelistLock());
4003
4004 curNumCards = preclean_card_table(_cmsGen, &smoac_cl);
4005 cumNumCards += curNumCards;
4006 if (PrintGCDetails && PrintCMSStatistics != 0) {
4007 gclog_or_tty->print_cr(" (cardTable: " SIZE_FORMAT " cards, re-scanned " SIZE_FORMAT " cards, " SIZE_FORMAT " iterations)",
4008 curNumCards, cumNumCards, numIter);
4009 }
4010 return cumNumCards; // as a measure of useful work done
4011 }
4012
4013 // PRECLEANING NOTES:
4014 // Precleaning involves:
4015 // . reading the bits of the modUnionTable and clearing the set bits.
4016 // . For the cards corresponding to the set bits, we scan the
4017 // objects on those cards. This means we need the free_list_lock
4018 // so that we can safely iterate over the CMS space when scanning
4019 // for oops.
4020 // . When we scan the objects, we'll be both reading and setting
4021 // marks in the marking bit map, so we'll need the marking bit map.
4022 // . For protecting _collector_state transitions, we take the CGC_lock.
4023 // Note that any races in the reading of of card table entries by the
4024 // CMS thread on the one hand and the clearing of those entries by the
4025 // VM thread or the setting of those entries by the mutator threads on the
4026 // other are quite benign. However, for efficiency it makes sense to keep
4027 // the VM thread from racing with the CMS thread while the latter is
4028 // dirty card info to the modUnionTable. We therefore also use the
4029 // CGC_lock to protect the reading of the card table and the mod union
4030 // table by the CM thread.
4031 // . We run concurrently with mutator updates, so scanning
4032 // needs to be done carefully -- we should not try to scan
4033 // potentially uninitialized objects.
4034 //
4035 // Locking strategy: While holding the CGC_lock, we scan over and
4036 // reset a maximal dirty range of the mod union / card tables, then lock
4037 // the free_list_lock and bitmap lock to do a full marking, then
4038 // release these locks; and repeat the cycle. This allows for a
4039 // certain amount of fairness in the sharing of these locks between
4040 // the CMS collector on the one hand, and the VM thread and the
4041 // mutators on the other.
4042
4043 // NOTE: preclean_mod_union_table() and preclean_card_table()
4044 // further below are largely identical; if you need to modify
4045 // one of these methods, please check the other method too.
4046
4047 size_t CMSCollector::preclean_mod_union_table(
4048 ConcurrentMarkSweepGeneration* gen,
4049 ScanMarkedObjectsAgainCarefullyClosure* cl) {
4050 verify_work_stacks_empty();
4051 verify_overflow_empty();
4052
4053 // strategy: starting with the first card, accumulate contiguous
4054 // ranges of dirty cards; clear these cards, then scan the region
4055 // covered by these cards.
4056
4057 // Since all of the MUT is committed ahead, we can just use
4058 // that, in case the generations expand while we are precleaning.
4059 // It might also be fine to just use the committed part of the
4060 // generation, but we might potentially miss cards when the
4061 // generation is rapidly expanding while we are in the midst
4062 // of precleaning.
4063 HeapWord* startAddr = gen->reserved().start();
4064 HeapWord* endAddr = gen->reserved().end();
4065
4066 cl->setFreelistLock(gen->freelistLock()); // needed for yielding
4067
4068 size_t numDirtyCards, cumNumDirtyCards;
4069 HeapWord *nextAddr, *lastAddr;
4070 for (cumNumDirtyCards = numDirtyCards = 0,
4071 nextAddr = lastAddr = startAddr;
4072 nextAddr < endAddr;
4073 nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
4074
4075 ResourceMark rm;
4076 HandleMark hm;
4077
4078 MemRegion dirtyRegion;
4079 {
4080 stopTimer();
4081 // Potential yield point
4082 CMSTokenSync ts(true);
4083 startTimer();
4084 sample_eden();
4085 // Get dirty region starting at nextOffset (inclusive),
4086 // simultaneously clearing it.
4087 dirtyRegion =
4088 _modUnionTable.getAndClearMarkedRegion(nextAddr, endAddr);
4089 assert(dirtyRegion.start() >= nextAddr,
4090 "returned region inconsistent?");
4091 }
4092 // Remember where the next search should begin.
4093 // The returned region (if non-empty) is a right open interval,
4094 // so lastOffset is obtained from the right end of that
4095 // interval.
4096 lastAddr = dirtyRegion.end();
4097 // Should do something more transparent and less hacky XXX
4098 numDirtyCards =
4099 _modUnionTable.heapWordDiffToOffsetDiff(dirtyRegion.word_size());
4100
4101 // We'll scan the cards in the dirty region (with periodic
4102 // yields for foreground GC as needed).
4103 if (!dirtyRegion.is_empty()) {
4104 assert(numDirtyCards > 0, "consistency check");
4105 HeapWord* stop_point = NULL;
4106 stopTimer();
4107 // Potential yield point
4108 CMSTokenSyncWithLocks ts(true, gen->freelistLock(),
4109 bitMapLock());
4110 startTimer();
4111 {
4112 verify_work_stacks_empty();
4113 verify_overflow_empty();
4114 sample_eden();
4115 stop_point =
4116 gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
4117 }
4118 if (stop_point != NULL) {
4119 // The careful iteration stopped early either because it found an
4120 // uninitialized object, or because we were in the midst of an
4121 // "abortable preclean", which should now be aborted. Redirty
4122 // the bits corresponding to the partially-scanned or unscanned
4123 // cards. We'll either restart at the next block boundary or
4124 // abort the preclean.
4125 assert((_collectorState == AbortablePreclean && should_abort_preclean()),
4126 "Should only be AbortablePreclean.");
4127 _modUnionTable.mark_range(MemRegion(stop_point, dirtyRegion.end()));
4128 if (should_abort_preclean()) {
4129 break; // out of preclean loop
4130 } else {
4131 // Compute the next address at which preclean should pick up;
4132 // might need bitMapLock in order to read P-bits.
4133 lastAddr = next_card_start_after_block(stop_point);
4134 }
4135 }
4136 } else {
4137 assert(lastAddr == endAddr, "consistency check");
4138 assert(numDirtyCards == 0, "consistency check");
4139 break;
4140 }
4141 }
4142 verify_work_stacks_empty();
4143 verify_overflow_empty();
4144 return cumNumDirtyCards;
4145 }
4146
4147 // NOTE: preclean_mod_union_table() above and preclean_card_table()
4148 // below are largely identical; if you need to modify
4149 // one of these methods, please check the other method too.
4150
4151 size_t CMSCollector::preclean_card_table(ConcurrentMarkSweepGeneration* gen,
4152 ScanMarkedObjectsAgainCarefullyClosure* cl) {
4153 // strategy: it's similar to precleamModUnionTable above, in that
4154 // we accumulate contiguous ranges of dirty cards, mark these cards
4155 // precleaned, then scan the region covered by these cards.
4156 HeapWord* endAddr = (HeapWord*)(gen->_virtual_space.high());
4157 HeapWord* startAddr = (HeapWord*)(gen->_virtual_space.low());
4158
4159 cl->setFreelistLock(gen->freelistLock()); // needed for yielding
4160
4161 size_t numDirtyCards, cumNumDirtyCards;
4162 HeapWord *lastAddr, *nextAddr;
4163
4164 for (cumNumDirtyCards = numDirtyCards = 0,
4165 nextAddr = lastAddr = startAddr;
4166 nextAddr < endAddr;
4167 nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
4168
4169 ResourceMark rm;
4170 HandleMark hm;
4171
4172 MemRegion dirtyRegion;
4173 {
4174 // See comments in "Precleaning notes" above on why we
4175 // do this locking. XXX Could the locking overheads be
4176 // too high when dirty cards are sparse? [I don't think so.]
4177 stopTimer();
4178 CMSTokenSync x(true); // is cms thread
4179 startTimer();
4180 sample_eden();
4181 // Get and clear dirty region from card table
4182 dirtyRegion = _ct->ct_bs()->dirty_card_range_after_reset(
4183 MemRegion(nextAddr, endAddr),
4184 true,
4185 CardTableModRefBS::precleaned_card_val());
4186
4187 assert(dirtyRegion.start() >= nextAddr,
4188 "returned region inconsistent?");
4189 }
4190 lastAddr = dirtyRegion.end();
4191 numDirtyCards =
4192 dirtyRegion.word_size()/CardTableModRefBS::card_size_in_words;
4193
4194 if (!dirtyRegion.is_empty()) {
4195 stopTimer();
4196 CMSTokenSyncWithLocks ts(true, gen->freelistLock(), bitMapLock());
4197 startTimer();
4198 sample_eden();
4199 verify_work_stacks_empty();
4200 verify_overflow_empty();
4201 HeapWord* stop_point =
4202 gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
4203 if (stop_point != NULL) {
4204 assert((_collectorState == AbortablePreclean && should_abort_preclean()),
4205 "Should only be AbortablePreclean.");
4206 _ct->ct_bs()->invalidate(MemRegion(stop_point, dirtyRegion.end()));
4207 if (should_abort_preclean()) {
4208 break; // out of preclean loop
4209 } else {
4210 // Compute the next address at which preclean should pick up.
4211 lastAddr = next_card_start_after_block(stop_point);
4212 }
4213 }
4214 } else {
4215 break;
4216 }
4217 }
4218 verify_work_stacks_empty();
4219 verify_overflow_empty();
4220 return cumNumDirtyCards;
4221 }
4222
4223 class PrecleanKlassClosure : public KlassClosure {
4224 KlassToOopClosure _cm_klass_closure;
4225 public:
4226 PrecleanKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {}
4227 void do_klass(Klass* k) {
4228 if (k->has_accumulated_modified_oops()) {
4229 k->clear_accumulated_modified_oops();
4230
4231 _cm_klass_closure.do_klass(k);
4232 }
4233 }
4234 };
4235
4236 // The freelist lock is needed to prevent asserts, is it really needed?
4237 void CMSCollector::preclean_klasses(MarkRefsIntoAndScanClosure* cl, Mutex* freelistLock) {
4238
4239 cl->set_freelistLock(freelistLock);
4240
4241 CMSTokenSyncWithLocks ts(true, freelistLock, bitMapLock());
4242
4243 // SSS: Add equivalent to ScanMarkedObjectsAgainCarefullyClosure::do_yield_check and should_abort_preclean?
4244 // SSS: We should probably check if precleaning should be aborted, at suitable intervals?
4245 PrecleanKlassClosure preclean_klass_closure(cl);
4246 ClassLoaderDataGraph::classes_do(&preclean_klass_closure);
4247
4248 verify_work_stacks_empty();
4249 verify_overflow_empty();
4250 }
4251
4252 void CMSCollector::checkpointRootsFinal() {
4253 assert(_collectorState == FinalMarking, "incorrect state transition?");
4254 check_correct_thread_executing();
4255 // world is stopped at this checkpoint
4256 assert(SafepointSynchronize::is_at_safepoint(),
4257 "world should be stopped");
4258 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
4259
4260 verify_work_stacks_empty();
4261 verify_overflow_empty();
4262
4263 if (PrintGCDetails) {
4264 gclog_or_tty->print("[YG occupancy: "SIZE_FORMAT" K ("SIZE_FORMAT" K)]",
4265 _young_gen->used() / K,
4266 _young_gen->capacity() / K);
4267 }
4268 {
4269 if (CMSScavengeBeforeRemark) {
4270 GenCollectedHeap* gch = GenCollectedHeap::heap();
4271 // Temporarily set flag to false, GCH->do_collection will
4272 // expect it to be false and set to true
4273 FlagSetting fl(gch->_is_gc_active, false);
4274 NOT_PRODUCT(GCTraceTime t("Scavenge-Before-Remark",
4275 PrintGCDetails && Verbose, true, _gc_timer_cm, _gc_tracer_cm->gc_id());)
4276 int level = _cmsGen->level() - 1;
4277 if (level >= 0) {
4278 gch->do_collection(true, // full (i.e. force, see below)
4279 false, // !clear_all_soft_refs
4280 0, // size
4281 false, // is_tlab
4282 level // max_level
4283 );
4284 }
4285 }
4286 FreelistLocker x(this);
4287 MutexLockerEx y(bitMapLock(),
4288 Mutex::_no_safepoint_check_flag);
4289 checkpointRootsFinalWork();
4290 }
4291 verify_work_stacks_empty();
4292 verify_overflow_empty();
4293 }
4294
4295 void CMSCollector::checkpointRootsFinalWork() {
4296 NOT_PRODUCT(GCTraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());)
4297
4298 assert(haveFreelistLocks(), "must have free list locks");
4299 assert_lock_strong(bitMapLock());
4300
4301 ResourceMark rm;
4302 HandleMark hm;
4303
4304 GenCollectedHeap* gch = GenCollectedHeap::heap();
4305
4306 if (should_unload_classes()) {
4307 CodeCache::gc_prologue();
4308 }
4309 assert(haveFreelistLocks(), "must have free list locks");
4310 assert_lock_strong(bitMapLock());
4311
4312 // We might assume that we need not fill TLAB's when
4313 // CMSScavengeBeforeRemark is set, because we may have just done
4314 // a scavenge which would have filled all TLAB's -- and besides
4315 // Eden would be empty. This however may not always be the case --
4316 // for instance although we asked for a scavenge, it may not have
4317 // happened because of a JNI critical section. We probably need
4318 // a policy for deciding whether we can in that case wait until
4319 // the critical section releases and then do the remark following
4320 // the scavenge, and skip it here. In the absence of that policy,
4321 // or of an indication of whether the scavenge did indeed occur,
4322 // we cannot rely on TLAB's having been filled and must do
4323 // so here just in case a scavenge did not happen.
4324 gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
4325 // Update the saved marks which may affect the root scans.
4326 gch->save_marks();
4327
4328 if (CMSPrintEdenSurvivorChunks) {
4329 print_eden_and_survivor_chunk_arrays();
4330 }
4331
4332 {
4333 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
4334
4335 // Note on the role of the mod union table:
4336 // Since the marker in "markFromRoots" marks concurrently with
4337 // mutators, it is possible for some reachable objects not to have been
4338 // scanned. For instance, an only reference to an object A was
4339 // placed in object B after the marker scanned B. Unless B is rescanned,
4340 // A would be collected. Such updates to references in marked objects
4341 // are detected via the mod union table which is the set of all cards
4342 // dirtied since the first checkpoint in this GC cycle and prior to
4343 // the most recent young generation GC, minus those cleaned up by the
4344 // concurrent precleaning.
4345 if (CMSParallelRemarkEnabled) {
4346 GCTraceTime t("Rescan (parallel) ", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
4347 do_remark_parallel();
4348 } else {
4349 GCTraceTime t("Rescan (non-parallel) ", PrintGCDetails, false,
4350 _gc_timer_cm, _gc_tracer_cm->gc_id());
4351 do_remark_non_parallel();
4352 }
4353 }
4354 verify_work_stacks_empty();
4355 verify_overflow_empty();
4356
4357 {
4358 NOT_PRODUCT(GCTraceTime ts("refProcessingWork", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());)
4359 refProcessingWork();
4360 }
4361 verify_work_stacks_empty();
4362 verify_overflow_empty();
4363
4364 if (should_unload_classes()) {
4365 CodeCache::gc_epilogue();
4366 }
4367 JvmtiExport::gc_epilogue();
4368
4369 // If we encountered any (marking stack / work queue) overflow
4370 // events during the current CMS cycle, take appropriate
4371 // remedial measures, where possible, so as to try and avoid
4372 // recurrence of that condition.
4373 assert(_markStack.isEmpty(), "No grey objects");
4374 size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
4375 _ser_kac_ovflw + _ser_kac_preclean_ovflw;
4376 if (ser_ovflw > 0) {
4377 if (PrintCMSStatistics != 0) {
4378 gclog_or_tty->print_cr("Marking stack overflow (benign) "
4379 "(pmc_pc="SIZE_FORMAT", pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT
4380 ", kac_preclean="SIZE_FORMAT")",
4381 _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw,
4382 _ser_kac_ovflw, _ser_kac_preclean_ovflw);
4383 }
4384 _markStack.expand();
4385 _ser_pmc_remark_ovflw = 0;
4386 _ser_pmc_preclean_ovflw = 0;
4387 _ser_kac_preclean_ovflw = 0;
4388 _ser_kac_ovflw = 0;
4389 }
4390 if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
4391 if (PrintCMSStatistics != 0) {
4392 gclog_or_tty->print_cr("Work queue overflow (benign) "
4393 "(pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT")",
4394 _par_pmc_remark_ovflw, _par_kac_ovflw);
4395 }
4396 _par_pmc_remark_ovflw = 0;
4397 _par_kac_ovflw = 0;
4398 }
4399 if (PrintCMSStatistics != 0) {
4400 if (_markStack._hit_limit > 0) {
4401 gclog_or_tty->print_cr(" (benign) Hit max stack size limit ("SIZE_FORMAT")",
4402 _markStack._hit_limit);
4403 }
4404 if (_markStack._failed_double > 0) {
4405 gclog_or_tty->print_cr(" (benign) Failed stack doubling ("SIZE_FORMAT"),"
4406 " current capacity "SIZE_FORMAT,
4407 _markStack._failed_double,
4408 _markStack.capacity());
4409 }
4410 }
4411 _markStack._hit_limit = 0;
4412 _markStack._failed_double = 0;
4413
4414 if ((VerifyAfterGC || VerifyDuringGC) &&
4415 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
4416 verify_after_remark();
4417 }
4418
4419 _gc_tracer_cm->report_object_count_after_gc(&_is_alive_closure);
4420
4421 // Change under the freelistLocks.
4422 _collectorState = Sweeping;
4423 // Call isAllClear() under bitMapLock
4424 assert(_modUnionTable.isAllClear(),
4425 "Should be clear by end of the final marking");
4426 assert(_ct->klass_rem_set()->mod_union_is_clear(),
4427 "Should be clear by end of the final marking");
4428 }
4429
4430 void CMSParInitialMarkTask::work(uint worker_id) {
4431 elapsedTimer _timer;
4432 ResourceMark rm;
4433 HandleMark hm;
4434
4435 // ---------- scan from roots --------------
4436 _timer.start();
4437 GenCollectedHeap* gch = GenCollectedHeap::heap();
4438 Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap));
4439
4440 // ---------- young gen roots --------------
4441 {
4442 work_on_young_gen_roots(worker_id, &par_mri_cl);
4443 _timer.stop();
4444 if (PrintCMSStatistics != 0) {
4445 gclog_or_tty->print_cr(
4446 "Finished young gen initial mark scan work in %dth thread: %3.3f sec",
4447 worker_id, _timer.seconds());
4448 }
4449 }
4450
4451 // ---------- remaining roots --------------
4452 _timer.reset();
4453 _timer.start();
4454
4455 CLDToOopClosure cld_closure(&par_mri_cl, true);
4456
4457 gch->gen_process_roots(_collector->_cmsGen->level(),
4458 false, // yg was scanned above
4459 false, // this is parallel code
4460 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4461 _collector->should_unload_classes(),
4462 &par_mri_cl,
4463 NULL,
4464 &cld_closure);
4465 assert(_collector->should_unload_classes()
4466 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4467 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4468 _timer.stop();
4469 if (PrintCMSStatistics != 0) {
4470 gclog_or_tty->print_cr(
4471 "Finished remaining root initial mark scan work in %dth thread: %3.3f sec",
4472 worker_id, _timer.seconds());
4473 }
4474 }
4475
4476 // Parallel remark task
4477 class CMSParRemarkTask: public CMSParMarkTask {
4478 CompactibleFreeListSpace* _cms_space;
4479
4480 // The per-thread work queues, available here for stealing.
4481 OopTaskQueueSet* _task_queues;
4482 ParallelTaskTerminator _term;
4483
4484 public:
4485 // A value of 0 passed to n_workers will cause the number of
4486 // workers to be taken from the active workers in the work gang.
4487 CMSParRemarkTask(CMSCollector* collector,
4488 CompactibleFreeListSpace* cms_space,
4489 int n_workers, FlexibleWorkGang* workers,
4490 OopTaskQueueSet* task_queues):
4491 CMSParMarkTask("Rescan roots and grey objects in parallel",
4492 collector, n_workers),
4493 _cms_space(cms_space),
4494 _task_queues(task_queues),
4495 _term(n_workers, task_queues) { }
4496
4497 OopTaskQueueSet* task_queues() { return _task_queues; }
4498
4499 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
4500
4501 ParallelTaskTerminator* terminator() { return &_term; }
4502 int n_workers() { return _n_workers; }
4503
4504 void work(uint worker_id);
4505
4506 private:
4507 // ... of dirty cards in old space
4508 void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i,
4509 Par_MarkRefsIntoAndScanClosure* cl);
4510
4511 // ... work stealing for the above
4512 void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed);
4513 };
4514
4515 class RemarkKlassClosure : public KlassClosure {
4516 KlassToOopClosure _cm_klass_closure;
4517 public:
4518 RemarkKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {}
4519 void do_klass(Klass* k) {
4520 // Check if we have modified any oops in the Klass during the concurrent marking.
4521 if (k->has_accumulated_modified_oops()) {
4522 k->clear_accumulated_modified_oops();
4523
4524 // We could have transfered the current modified marks to the accumulated marks,
4525 // like we do with the Card Table to Mod Union Table. But it's not really necessary.
4526 } else if (k->has_modified_oops()) {
4527 // Don't clear anything, this info is needed by the next young collection.
4528 } else {
4529 // No modified oops in the Klass.
4530 return;
4531 }
4532
4533 // The klass has modified fields, need to scan the klass.
4534 _cm_klass_closure.do_klass(k);
4535 }
4536 };
4537
4538 void CMSParMarkTask::work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl) {
4539 ParNewGeneration* young_gen = _collector->_young_gen;
4540 ContiguousSpace* eden_space = young_gen->eden();
4541 ContiguousSpace* from_space = young_gen->from();
4542 ContiguousSpace* to_space = young_gen->to();
4543
4544 HeapWord** eca = _collector->_eden_chunk_array;
4545 size_t ect = _collector->_eden_chunk_index;
4546 HeapWord** sca = _collector->_survivor_chunk_array;
4547 size_t sct = _collector->_survivor_chunk_index;
4548
4549 assert(ect <= _collector->_eden_chunk_capacity, "out of bounds");
4550 assert(sct <= _collector->_survivor_chunk_capacity, "out of bounds");
4551
4552 do_young_space_rescan(worker_id, cl, to_space, NULL, 0);
4553 do_young_space_rescan(worker_id, cl, from_space, sca, sct);
4554 do_young_space_rescan(worker_id, cl, eden_space, eca, ect);
4555 }
4556
4557 // work_queue(i) is passed to the closure
4558 // Par_MarkRefsIntoAndScanClosure. The "i" parameter
4559 // also is passed to do_dirty_card_rescan_tasks() and to
4560 // do_work_steal() to select the i-th task_queue.
4561
4562 void CMSParRemarkTask::work(uint worker_id) {
4563 elapsedTimer _timer;
4564 ResourceMark rm;
4565 HandleMark hm;
4566
4567 // ---------- rescan from roots --------------
4568 _timer.start();
4569 GenCollectedHeap* gch = GenCollectedHeap::heap();
4570 Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
4571 _collector->_span, _collector->ref_processor(),
4572 &(_collector->_markBitMap),
4573 work_queue(worker_id));
4574
4575 // Rescan young gen roots first since these are likely
4576 // coarsely partitioned and may, on that account, constitute
4577 // the critical path; thus, it's best to start off that
4578 // work first.
4579 // ---------- young gen roots --------------
4580 {
4581 work_on_young_gen_roots(worker_id, &par_mrias_cl);
4582 _timer.stop();
4583 if (PrintCMSStatistics != 0) {
4584 gclog_or_tty->print_cr(
4585 "Finished young gen rescan work in %dth thread: %3.3f sec",
4586 worker_id, _timer.seconds());
4587 }
4588 }
4589
4590 // ---------- remaining roots --------------
4591 _timer.reset();
4592 _timer.start();
4593 gch->gen_process_roots(_collector->_cmsGen->level(),
4594 false, // yg was scanned above
4595 false, // this is parallel code
4596 GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
4597 _collector->should_unload_classes(),
4598 &par_mrias_cl,
4599 NULL,
4600 NULL); // The dirty klasses will be handled below
4601
4602 assert(_collector->should_unload_classes()
4603 || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
4604 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
4605 _timer.stop();
4606 if (PrintCMSStatistics != 0) {
4607 gclog_or_tty->print_cr(
4608 "Finished remaining root rescan work in %dth thread: %3.3f sec",
4609 worker_id, _timer.seconds());
4610 }
4611
4612 // ---------- unhandled CLD scanning ----------
4613 if (worker_id == 0) { // Single threaded at the moment.
4614 _timer.reset();
4615 _timer.start();
4616
4617 // Scan all new class loader data objects and new dependencies that were
4618 // introduced during concurrent marking.
4619 ResourceMark rm;
4620 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
4621 for (int i = 0; i < array->length(); i++) {
4622 par_mrias_cl.do_class_loader_data(array->at(i));
4623 }
4624
4625 // We don't need to keep track of new CLDs anymore.
4626 ClassLoaderDataGraph::remember_new_clds(false);
4627
4628 _timer.stop();
4629 if (PrintCMSStatistics != 0) {
4630 gclog_or_tty->print_cr(
4631 "Finished unhandled CLD scanning work in %dth thread: %3.3f sec",
4632 worker_id, _timer.seconds());
4633 }
4634 }
4635
4636 // ---------- dirty klass scanning ----------
4637 if (worker_id == 0) { // Single threaded at the moment.
4638 _timer.reset();
4639 _timer.start();
4640
4641 // Scan all classes that was dirtied during the concurrent marking phase.
4642 RemarkKlassClosure remark_klass_closure(&par_mrias_cl);
4643 ClassLoaderDataGraph::classes_do(&remark_klass_closure);
4644
4645 _timer.stop();
4646 if (PrintCMSStatistics != 0) {
4647 gclog_or_tty->print_cr(
4648 "Finished dirty klass scanning work in %dth thread: %3.3f sec",
4649 worker_id, _timer.seconds());
4650 }
4651 }
4652
4653 // We might have added oops to ClassLoaderData::_handles during the
4654 // concurrent marking phase. These oops point to newly allocated objects
4655 // that are guaranteed to be kept alive. Either by the direct allocation
4656 // code, or when the young collector processes the roots. Hence,
4657 // we don't have to revisit the _handles block during the remark phase.
4658
4659 // ---------- rescan dirty cards ------------
4660 _timer.reset();
4661 _timer.start();
4662
4663 // Do the rescan tasks for each of the two spaces
4664 // (cms_space) in turn.
4665 // "worker_id" is passed to select the task_queue for "worker_id"
4666 do_dirty_card_rescan_tasks(_cms_space, worker_id, &par_mrias_cl);
4667 _timer.stop();
4668 if (PrintCMSStatistics != 0) {
4669 gclog_or_tty->print_cr(
4670 "Finished dirty card rescan work in %dth thread: %3.3f sec",
4671 worker_id, _timer.seconds());
4672 }
4673
4674 // ---------- steal work from other threads ...
4675 // ---------- ... and drain overflow list.
4676 _timer.reset();
4677 _timer.start();
4678 do_work_steal(worker_id, &par_mrias_cl, _collector->hash_seed(worker_id));
4679 _timer.stop();
4680 if (PrintCMSStatistics != 0) {
4681 gclog_or_tty->print_cr(
4682 "Finished work stealing in %dth thread: %3.3f sec",
4683 worker_id, _timer.seconds());
4684 }
4685 }
4686
4687 // Note that parameter "i" is not used.
4688 void
4689 CMSParMarkTask::do_young_space_rescan(uint worker_id,
4690 OopsInGenClosure* cl, ContiguousSpace* space,
4691 HeapWord** chunk_array, size_t chunk_top) {
4692 // Until all tasks completed:
4693 // . claim an unclaimed task
4694 // . compute region boundaries corresponding to task claimed
4695 // using chunk_array
4696 // . par_oop_iterate(cl) over that region
4697
4698 ResourceMark rm;
4699 HandleMark hm;
4700
4701 SequentialSubTasksDone* pst = space->par_seq_tasks();
4702
4703 uint nth_task = 0;
4704 uint n_tasks = pst->n_tasks();
4705
4706 if (n_tasks > 0) {
4707 assert(pst->valid(), "Uninitialized use?");
4708 HeapWord *start, *end;
4709 while (!pst->is_task_claimed(/* reference */ nth_task)) {
4710 // We claimed task # nth_task; compute its boundaries.
4711 if (chunk_top == 0) { // no samples were taken
4712 assert(nth_task == 0 && n_tasks == 1, "Can have only 1 eden task");
4713 start = space->bottom();
4714 end = space->top();
4715 } else if (nth_task == 0) {
4716 start = space->bottom();
4717 end = chunk_array[nth_task];
4718 } else if (nth_task < (uint)chunk_top) {
4719 assert(nth_task >= 1, "Control point invariant");
4720 start = chunk_array[nth_task - 1];
4721 end = chunk_array[nth_task];
4722 } else {
4723 assert(nth_task == (uint)chunk_top, "Control point invariant");
4724 start = chunk_array[chunk_top - 1];
4725 end = space->top();
4726 }
4727 MemRegion mr(start, end);
4728 // Verify that mr is in space
4729 assert(mr.is_empty() || space->used_region().contains(mr),
4730 "Should be in space");
4731 // Verify that "start" is an object boundary
4732 assert(mr.is_empty() || oop(mr.start())->is_oop(),
4733 "Should be an oop");
4734 space->par_oop_iterate(mr, cl);
4735 }
4736 pst->all_tasks_completed();
4737 }
4738 }
4739
4740 void
4741 CMSParRemarkTask::do_dirty_card_rescan_tasks(
4742 CompactibleFreeListSpace* sp, int i,
4743 Par_MarkRefsIntoAndScanClosure* cl) {
4744 // Until all tasks completed:
4745 // . claim an unclaimed task
4746 // . compute region boundaries corresponding to task claimed
4747 // . transfer dirty bits ct->mut for that region
4748 // . apply rescanclosure to dirty mut bits for that region
4749
4750 ResourceMark rm;
4751 HandleMark hm;
4752
4753 OopTaskQueue* work_q = work_queue(i);
4754 ModUnionClosure modUnionClosure(&(_collector->_modUnionTable));
4755 // CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION!
4756 // CAUTION: This closure has state that persists across calls to
4757 // the work method dirty_range_iterate_clear() in that it has
4758 // embedded in it a (subtype of) UpwardsObjectClosure. The
4759 // use of that state in the embedded UpwardsObjectClosure instance
4760 // assumes that the cards are always iterated (even if in parallel
4761 // by several threads) in monotonically increasing order per each
4762 // thread. This is true of the implementation below which picks
4763 // card ranges (chunks) in monotonically increasing order globally
4764 // and, a-fortiori, in monotonically increasing order per thread
4765 // (the latter order being a subsequence of the former).
4766 // If the work code below is ever reorganized into a more chaotic
4767 // work-partitioning form than the current "sequential tasks"
4768 // paradigm, the use of that persistent state will have to be
4769 // revisited and modified appropriately. See also related
4770 // bug 4756801 work on which should examine this code to make
4771 // sure that the changes there do not run counter to the
4772 // assumptions made here and necessary for correctness and
4773 // efficiency. Note also that this code might yield inefficient
4774 // behavior in the case of very large objects that span one or
4775 // more work chunks. Such objects would potentially be scanned
4776 // several times redundantly. Work on 4756801 should try and
4777 // address that performance anomaly if at all possible. XXX
4778 MemRegion full_span = _collector->_span;
4779 CMSBitMap* bm = &(_collector->_markBitMap); // shared
4780 MarkFromDirtyCardsClosure
4781 greyRescanClosure(_collector, full_span, // entire span of interest
4782 sp, bm, work_q, cl);
4783
4784 SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
4785 assert(pst->valid(), "Uninitialized use?");
4786 uint nth_task = 0;
4787 const int alignment = CardTableModRefBS::card_size * BitsPerWord;
4788 MemRegion span = sp->used_region();
4789 HeapWord* start_addr = span.start();
4790 HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(),
4791 alignment);
4792 const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units
4793 assert((HeapWord*)round_to((intptr_t)start_addr, alignment) ==
4794 start_addr, "Check alignment");
4795 assert((size_t)round_to((intptr_t)chunk_size, alignment) ==
4796 chunk_size, "Check alignment");
4797
4798 while (!pst->is_task_claimed(/* reference */ nth_task)) {
4799 // Having claimed the nth_task, compute corresponding mem-region,
4800 // which is a-fortiori aligned correctly (i.e. at a MUT boundary).
4801 // The alignment restriction ensures that we do not need any
4802 // synchronization with other gang-workers while setting or
4803 // clearing bits in thus chunk of the MUT.
4804 MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size,
4805 start_addr + (nth_task+1)*chunk_size);
4806 // The last chunk's end might be way beyond end of the
4807 // used region. In that case pull back appropriately.
4808 if (this_span.end() > end_addr) {
4809 this_span.set_end(end_addr);
4810 assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)");
4811 }
4812 // Iterate over the dirty cards covering this chunk, marking them
4813 // precleaned, and setting the corresponding bits in the mod union
4814 // table. Since we have been careful to partition at Card and MUT-word
4815 // boundaries no synchronization is needed between parallel threads.
4816 _collector->_ct->ct_bs()->dirty_card_iterate(this_span,
4817 &modUnionClosure);
4818
4819 // Having transferred these marks into the modUnionTable,
4820 // rescan the marked objects on the dirty cards in the modUnionTable.
4821 // Even if this is at a synchronous collection, the initial marking
4822 // may have been done during an asynchronous collection so there
4823 // may be dirty bits in the mod-union table.
4824 _collector->_modUnionTable.dirty_range_iterate_clear(
4825 this_span, &greyRescanClosure);
4826 _collector->_modUnionTable.verifyNoOneBitsInRange(
4827 this_span.start(),
4828 this_span.end());
4829 }
4830 pst->all_tasks_completed(); // declare that i am done
4831 }
4832
4833 // . see if we can share work_queues with ParNew? XXX
4834 void
4835 CMSParRemarkTask::do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl,
4836 int* seed) {
4837 OopTaskQueue* work_q = work_queue(i);
4838 NOT_PRODUCT(int num_steals = 0;)
4839 oop obj_to_scan;
4840 CMSBitMap* bm = &(_collector->_markBitMap);
4841
4842 while (true) {
4843 // Completely finish any left over work from (an) earlier round(s)
4844 cl->trim_queue(0);
4845 size_t num_from_overflow_list = MIN2((size_t)(work_q->max_elems() - work_q->size())/4,
4846 (size_t)ParGCDesiredObjsFromOverflowList);
4847 // Now check if there's any work in the overflow list
4848 // Passing ParallelGCThreads as the third parameter, no_of_gc_threads,
4849 // only affects the number of attempts made to get work from the
4850 // overflow list and does not affect the number of workers. Just
4851 // pass ParallelGCThreads so this behavior is unchanged.
4852 if (_collector->par_take_from_overflow_list(num_from_overflow_list,
4853 work_q,
4854 ParallelGCThreads)) {
4855 // found something in global overflow list;
4856 // not yet ready to go stealing work from others.
4857 // We'd like to assert(work_q->size() != 0, ...)
4858 // because we just took work from the overflow list,
4859 // but of course we can't since all of that could have
4860 // been already stolen from us.
4861 // "He giveth and He taketh away."
4862 continue;
4863 }
4864 // Verify that we have no work before we resort to stealing
4865 assert(work_q->size() == 0, "Have work, shouldn't steal");
4866 // Try to steal from other queues that have work
4867 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
4868 NOT_PRODUCT(num_steals++;)
4869 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
4870 assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
4871 // Do scanning work
4872 obj_to_scan->oop_iterate(cl);
4873 // Loop around, finish this work, and try to steal some more
4874 } else if (terminator()->offer_termination()) {
4875 break; // nirvana from the infinite cycle
4876 }
4877 }
4878 NOT_PRODUCT(
4879 if (PrintCMSStatistics != 0) {
4880 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
4881 }
4882 )
4883 assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
4884 "Else our work is not yet done");
4885 }
4886
4887 // Record object boundaries in _eden_chunk_array by sampling the eden
4888 // top in the slow-path eden object allocation code path and record
4889 // the boundaries, if CMSEdenChunksRecordAlways is true. If
4890 // CMSEdenChunksRecordAlways is false, we use the other asynchronous
4891 // sampling in sample_eden() that activates during the part of the
4892 // preclean phase.
4893 void CMSCollector::sample_eden_chunk() {
4894 if (CMSEdenChunksRecordAlways && _eden_chunk_array != NULL) {
4895 if (_eden_chunk_lock->try_lock()) {
4896 // Record a sample. This is the critical section. The contents
4897 // of the _eden_chunk_array have to be non-decreasing in the
4898 // address order.
4899 _eden_chunk_array[_eden_chunk_index] = *_top_addr;
4900 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
4901 "Unexpected state of Eden");
4902 if (_eden_chunk_index == 0 ||
4903 ((_eden_chunk_array[_eden_chunk_index] > _eden_chunk_array[_eden_chunk_index-1]) &&
4904 (pointer_delta(_eden_chunk_array[_eden_chunk_index],
4905 _eden_chunk_array[_eden_chunk_index-1]) >= CMSSamplingGrain))) {
4906 _eden_chunk_index++; // commit sample
4907 }
4908 _eden_chunk_lock->unlock();
4909 }
4910 }
4911 }
4912
4913 // Return a thread-local PLAB recording array, as appropriate.
4914 void* CMSCollector::get_data_recorder(int thr_num) {
4915 if (_survivor_plab_array != NULL &&
4916 (CMSPLABRecordAlways ||
4917 (_collectorState > Marking && _collectorState < FinalMarking))) {
4918 assert(thr_num < (int)ParallelGCThreads, "thr_num is out of bounds");
4919 ChunkArray* ca = &_survivor_plab_array[thr_num];
4920 ca->reset(); // clear it so that fresh data is recorded
4921 return (void*) ca;
4922 } else {
4923 return NULL;
4924 }
4925 }
4926
4927 // Reset all the thread-local PLAB recording arrays
4928 void CMSCollector::reset_survivor_plab_arrays() {
4929 for (uint i = 0; i < ParallelGCThreads; i++) {
4930 _survivor_plab_array[i].reset();
4931 }
4932 }
4933
4934 // Merge the per-thread plab arrays into the global survivor chunk
4935 // array which will provide the partitioning of the survivor space
4936 // for CMS initial scan and rescan.
4937 void CMSCollector::merge_survivor_plab_arrays(ContiguousSpace* surv,
4938 int no_of_gc_threads) {
4939 assert(_survivor_plab_array != NULL, "Error");
4940 assert(_survivor_chunk_array != NULL, "Error");
4941 assert(_collectorState == FinalMarking ||
4942 (CMSParallelInitialMarkEnabled && _collectorState == InitialMarking), "Error");
4943 for (int j = 0; j < no_of_gc_threads; j++) {
4944 _cursor[j] = 0;
4945 }
4946 HeapWord* top = surv->top();
4947 size_t i;
4948 for (i = 0; i < _survivor_chunk_capacity; i++) { // all sca entries
4949 HeapWord* min_val = top; // Higher than any PLAB address
4950 uint min_tid = 0; // position of min_val this round
4951 for (int j = 0; j < no_of_gc_threads; j++) {
4952 ChunkArray* cur_sca = &_survivor_plab_array[j];
4953 if (_cursor[j] == cur_sca->end()) {
4954 continue;
4955 }
4956 assert(_cursor[j] < cur_sca->end(), "ctl pt invariant");
4957 HeapWord* cur_val = cur_sca->nth(_cursor[j]);
4958 assert(surv->used_region().contains(cur_val), "Out of bounds value");
4959 if (cur_val < min_val) {
4960 min_tid = j;
4961 min_val = cur_val;
4962 } else {
4963 assert(cur_val < top, "All recorded addresses should be less");
4964 }
4965 }
4966 // At this point min_val and min_tid are respectively
4967 // the least address in _survivor_plab_array[j]->nth(_cursor[j])
4968 // and the thread (j) that witnesses that address.
4969 // We record this address in the _survivor_chunk_array[i]
4970 // and increment _cursor[min_tid] prior to the next round i.
4971 if (min_val == top) {
4972 break;
4973 }
4974 _survivor_chunk_array[i] = min_val;
4975 _cursor[min_tid]++;
4976 }
4977 // We are all done; record the size of the _survivor_chunk_array
4978 _survivor_chunk_index = i; // exclusive: [0, i)
4979 if (PrintCMSStatistics > 0) {
4980 gclog_or_tty->print(" (Survivor:" SIZE_FORMAT "chunks) ", i);
4981 }
4982 // Verify that we used up all the recorded entries
4983 #ifdef ASSERT
4984 size_t total = 0;
4985 for (int j = 0; j < no_of_gc_threads; j++) {
4986 assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
4987 total += _cursor[j];
4988 }
4989 assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
4990 // Check that the merged array is in sorted order
4991 if (total > 0) {
4992 for (size_t i = 0; i < total - 1; i++) {
4993 if (PrintCMSStatistics > 0) {
4994 gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
4995 i, p2i(_survivor_chunk_array[i]));
4996 }
4997 assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
4998 "Not sorted");
4999 }
5000 }
5001 #endif // ASSERT
5002 }
5003
5004 // Set up the space's par_seq_tasks structure for work claiming
5005 // for parallel initial scan and rescan of young gen.
5006 // See ParRescanTask where this is currently used.
5007 void
5008 CMSCollector::
5009 initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
5010 assert(n_threads > 0, "Unexpected n_threads argument");
5011
5012 // Eden space
5013 if (!_young_gen->eden()->is_empty()) {
5014 SequentialSubTasksDone* pst = _young_gen->eden()->par_seq_tasks();
5015 assert(!pst->valid(), "Clobbering existing data?");
5016 // Each valid entry in [0, _eden_chunk_index) represents a task.
5017 size_t n_tasks = _eden_chunk_index + 1;
5018 assert(n_tasks == 1 || _eden_chunk_array != NULL, "Error");
5019 // Sets the condition for completion of the subtask (how many threads
5020 // need to finish in order to be done).
5021 pst->set_n_threads(n_threads);
5022 pst->set_n_tasks((int)n_tasks);
5023 }
5024
5025 // Merge the survivor plab arrays into _survivor_chunk_array
5026 if (_survivor_plab_array != NULL) {
5027 merge_survivor_plab_arrays(_young_gen->from(), n_threads);
5028 } else {
5029 assert(_survivor_chunk_index == 0, "Error");
5030 }
5031
5032 // To space
5033 {
5034 SequentialSubTasksDone* pst = _young_gen->to()->par_seq_tasks();
5035 assert(!pst->valid(), "Clobbering existing data?");
5036 // Sets the condition for completion of the subtask (how many threads
5037 // need to finish in order to be done).
5038 pst->set_n_threads(n_threads);
5039 pst->set_n_tasks(1);
5040 assert(pst->valid(), "Error");
5041 }
5042
5043 // From space
5044 {
5045 SequentialSubTasksDone* pst = _young_gen->from()->par_seq_tasks();
5046 assert(!pst->valid(), "Clobbering existing data?");
5047 size_t n_tasks = _survivor_chunk_index + 1;
5048 assert(n_tasks == 1 || _survivor_chunk_array != NULL, "Error");
5049 // Sets the condition for completion of the subtask (how many threads
5050 // need to finish in order to be done).
5051 pst->set_n_threads(n_threads);
5052 pst->set_n_tasks((int)n_tasks);
5053 assert(pst->valid(), "Error");
5054 }
5055 }
5056
5057 // Parallel version of remark
5058 void CMSCollector::do_remark_parallel() {
5059 GenCollectedHeap* gch = GenCollectedHeap::heap();
5060 FlexibleWorkGang* workers = gch->workers();
5061 assert(workers != NULL, "Need parallel worker threads.");
5062 // Choose to use the number of GC workers most recently set
5063 // into "active_workers". If active_workers is not set, set it
5064 // to ParallelGCThreads.
5065 int n_workers = workers->active_workers();
5066 if (n_workers == 0) {
5067 assert(n_workers > 0, "Should have been set during scavenge");
5068 n_workers = ParallelGCThreads;
5069 workers->set_active_workers(n_workers);
5070 }
5071 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
5072
5073 CMSParRemarkTask tsk(this,
5074 cms_space,
5075 n_workers, workers, task_queues());
5076
5077 // Set up for parallel process_roots work.
5078 gch->set_par_threads(n_workers);
5079 // We won't be iterating over the cards in the card table updating
5080 // the younger_gen cards, so we shouldn't call the following else
5081 // the verification code as well as subsequent younger_refs_iterate
5082 // code would get confused. XXX
5083 // gch->rem_set()->prepare_for_younger_refs_iterate(true); // parallel
5084
5085 // The young gen rescan work will not be done as part of
5086 // process_roots (which currently doesn't know how to
5087 // parallelize such a scan), but rather will be broken up into
5088 // a set of parallel tasks (via the sampling that the [abortable]
5089 // preclean phase did of eden, plus the [two] tasks of
5090 // scanning the [two] survivor spaces. Further fine-grain
5091 // parallelization of the scanning of the survivor spaces
5092 // themselves, and of precleaning of the younger gen itself
5093 // is deferred to the future.
5094 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
5095
5096 // The dirty card rescan work is broken up into a "sequence"
5097 // of parallel tasks (per constituent space) that are dynamically
5098 // claimed by the parallel threads.
5099 cms_space->initialize_sequential_subtasks_for_rescan(n_workers);
5100
5101 // It turns out that even when we're using 1 thread, doing the work in a
5102 // separate thread causes wide variance in run times. We can't help this
5103 // in the multi-threaded case, but we special-case n=1 here to get
5104 // repeatable measurements of the 1-thread overhead of the parallel code.
5105 if (n_workers > 1) {
5106 // Make refs discovery MT-safe, if it isn't already: it may not
5107 // necessarily be so, since it's possible that we are doing
5108 // ST marking.
5109 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), true);
5110 StrongRootsScope srs;
5111 workers->run_task(&tsk);
5112 } else {
5113 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
5114 StrongRootsScope srs;
5115 tsk.work(0);
5116 }
5117
5118 gch->set_par_threads(0); // 0 ==> non-parallel.
5119 // restore, single-threaded for now, any preserved marks
5120 // as a result of work_q overflow
5121 restore_preserved_marks_if_any();
5122 }
5123
5124 // Non-parallel version of remark
5125 void CMSCollector::do_remark_non_parallel() {
5126 ResourceMark rm;
5127 HandleMark hm;
5128 GenCollectedHeap* gch = GenCollectedHeap::heap();
5129 ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false);
5130
5131 MarkRefsIntoAndScanClosure
5132 mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */,
5133 &_markStack, this,
5134 false /* should_yield */, false /* not precleaning */);
5135 MarkFromDirtyCardsClosure
5136 markFromDirtyCardsClosure(this, _span,
5137 NULL, // space is set further below
5138 &_markBitMap, &_markStack, &mrias_cl);
5139 {
5140 GCTraceTime t("grey object rescan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5141 // Iterate over the dirty cards, setting the corresponding bits in the
5142 // mod union table.
5143 {
5144 ModUnionClosure modUnionClosure(&_modUnionTable);
5145 _ct->ct_bs()->dirty_card_iterate(
5146 _cmsGen->used_region(),
5147 &modUnionClosure);
5148 }
5149 // Having transferred these marks into the modUnionTable, we just need
5150 // to rescan the marked objects on the dirty cards in the modUnionTable.
5151 // The initial marking may have been done during an asynchronous
5152 // collection so there may be dirty bits in the mod-union table.
5153 const int alignment =
5154 CardTableModRefBS::card_size * BitsPerWord;
5155 {
5156 // ... First handle dirty cards in CMS gen
5157 markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
5158 MemRegion ur = _cmsGen->used_region();
5159 HeapWord* lb = ur.start();
5160 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
5161 MemRegion cms_span(lb, ub);
5162 _modUnionTable.dirty_range_iterate_clear(cms_span,
5163 &markFromDirtyCardsClosure);
5164 verify_work_stacks_empty();
5165 if (PrintCMSStatistics != 0) {
5166 gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ",
5167 markFromDirtyCardsClosure.num_dirty_cards());
5168 }
5169 }
5170 }
5171 if (VerifyDuringGC &&
5172 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
5173 HandleMark hm; // Discard invalid handles created during verification
5174 Universe::verify();
5175 }
5176 {
5177 GCTraceTime t("root rescan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5178
5179 verify_work_stacks_empty();
5180
5181 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
5182 StrongRootsScope srs;
5183
5184 gch->gen_process_roots(_cmsGen->level(),
5185 true, // younger gens as roots
5186 false, // use the local StrongRootsScope
5187 GenCollectedHeap::ScanningOption(roots_scanning_options()),
5188 should_unload_classes(),
5189 &mrias_cl,
5190 NULL,
5191 NULL); // The dirty klasses will be handled below
5192
5193 assert(should_unload_classes()
5194 || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache),
5195 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
5196 }
5197
5198 {
5199 GCTraceTime t("visit unhandled CLDs", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5200
5201 verify_work_stacks_empty();
5202
5203 // Scan all class loader data objects that might have been introduced
5204 // during concurrent marking.
5205 ResourceMark rm;
5206 GrowableArray<ClassLoaderData*>* array = ClassLoaderDataGraph::new_clds();
5207 for (int i = 0; i < array->length(); i++) {
5208 mrias_cl.do_class_loader_data(array->at(i));
5209 }
5210
5211 // We don't need to keep track of new CLDs anymore.
5212 ClassLoaderDataGraph::remember_new_clds(false);
5213
5214 verify_work_stacks_empty();
5215 }
5216
5217 {
5218 GCTraceTime t("dirty klass scan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5219
5220 verify_work_stacks_empty();
5221
5222 RemarkKlassClosure remark_klass_closure(&mrias_cl);
5223 ClassLoaderDataGraph::classes_do(&remark_klass_closure);
5224
5225 verify_work_stacks_empty();
5226 }
5227
5228 // We might have added oops to ClassLoaderData::_handles during the
5229 // concurrent marking phase. These oops point to newly allocated objects
5230 // that are guaranteed to be kept alive. Either by the direct allocation
5231 // code, or when the young collector processes the roots. Hence,
5232 // we don't have to revisit the _handles block during the remark phase.
5233
5234 verify_work_stacks_empty();
5235 // Restore evacuated mark words, if any, used for overflow list links
5236 if (!CMSOverflowEarlyRestoration) {
5237 restore_preserved_marks_if_any();
5238 }
5239 verify_overflow_empty();
5240 }
5241
5242 ////////////////////////////////////////////////////////
5243 // Parallel Reference Processing Task Proxy Class
5244 ////////////////////////////////////////////////////////
5245 class CMSRefProcTaskProxy: public AbstractGangTaskWOopQueues {
5246 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
5247 CMSCollector* _collector;
5248 CMSBitMap* _mark_bit_map;
5249 const MemRegion _span;
5250 ProcessTask& _task;
5251
5252 public:
5253 CMSRefProcTaskProxy(ProcessTask& task,
5254 CMSCollector* collector,
5255 const MemRegion& span,
5256 CMSBitMap* mark_bit_map,
5257 AbstractWorkGang* workers,
5258 OopTaskQueueSet* task_queues):
5259 // XXX Should superclass AGTWOQ also know about AWG since it knows
5260 // about the task_queues used by the AWG? Then it could initialize
5261 // the terminator() object. See 6984287. The set_for_termination()
5262 // below is a temporary band-aid for the regression in 6984287.
5263 AbstractGangTaskWOopQueues("Process referents by policy in parallel",
5264 task_queues),
5265 _task(task),
5266 _collector(collector), _span(span), _mark_bit_map(mark_bit_map)
5267 {
5268 assert(_collector->_span.equals(_span) && !_span.is_empty(),
5269 "Inconsistency in _span");
5270 set_for_termination(workers->active_workers());
5271 }
5272
5273 OopTaskQueueSet* task_queues() { return queues(); }
5274
5275 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
5276
5277 void do_work_steal(int i,
5278 CMSParDrainMarkingStackClosure* drain,
5279 CMSParKeepAliveClosure* keep_alive,
5280 int* seed);
5281
5282 virtual void work(uint worker_id);
5283 };
5284
5285 void CMSRefProcTaskProxy::work(uint worker_id) {
5286 ResourceMark rm;
5287 HandleMark hm;
5288 assert(_collector->_span.equals(_span), "Inconsistency in _span");
5289 CMSParKeepAliveClosure par_keep_alive(_collector, _span,
5290 _mark_bit_map,
5291 work_queue(worker_id));
5292 CMSParDrainMarkingStackClosure par_drain_stack(_collector, _span,
5293 _mark_bit_map,
5294 work_queue(worker_id));
5295 CMSIsAliveClosure is_alive_closure(_span, _mark_bit_map);
5296 _task.work(worker_id, is_alive_closure, par_keep_alive, par_drain_stack);
5297 if (_task.marks_oops_alive()) {
5298 do_work_steal(worker_id, &par_drain_stack, &par_keep_alive,
5299 _collector->hash_seed(worker_id));
5300 }
5301 assert(work_queue(worker_id)->size() == 0, "work_queue should be empty");
5302 assert(_collector->_overflow_list == NULL, "non-empty _overflow_list");
5303 }
5304
5305 class CMSRefEnqueueTaskProxy: public AbstractGangTask {
5306 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
5307 EnqueueTask& _task;
5308
5309 public:
5310 CMSRefEnqueueTaskProxy(EnqueueTask& task)
5311 : AbstractGangTask("Enqueue reference objects in parallel"),
5312 _task(task)
5313 { }
5314
5315 virtual void work(uint worker_id)
5316 {
5317 _task.work(worker_id);
5318 }
5319 };
5320
5321 CMSParKeepAliveClosure::CMSParKeepAliveClosure(CMSCollector* collector,
5322 MemRegion span, CMSBitMap* bit_map, OopTaskQueue* work_queue):
5323 _span(span),
5324 _bit_map(bit_map),
5325 _work_queue(work_queue),
5326 _mark_and_push(collector, span, bit_map, work_queue),
5327 _low_water_mark(MIN2((uint)(work_queue->max_elems()/4),
5328 (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads)))
5329 { }
5330
5331 // . see if we can share work_queues with ParNew? XXX
5332 void CMSRefProcTaskProxy::do_work_steal(int i,
5333 CMSParDrainMarkingStackClosure* drain,
5334 CMSParKeepAliveClosure* keep_alive,
5335 int* seed) {
5336 OopTaskQueue* work_q = work_queue(i);
5337 NOT_PRODUCT(int num_steals = 0;)
5338 oop obj_to_scan;
5339
5340 while (true) {
5341 // Completely finish any left over work from (an) earlier round(s)
5342 drain->trim_queue(0);
5343 size_t num_from_overflow_list = MIN2((size_t)(work_q->max_elems() - work_q->size())/4,
5344 (size_t)ParGCDesiredObjsFromOverflowList);
5345 // Now check if there's any work in the overflow list
5346 // Passing ParallelGCThreads as the third parameter, no_of_gc_threads,
5347 // only affects the number of attempts made to get work from the
5348 // overflow list and does not affect the number of workers. Just
5349 // pass ParallelGCThreads so this behavior is unchanged.
5350 if (_collector->par_take_from_overflow_list(num_from_overflow_list,
5351 work_q,
5352 ParallelGCThreads)) {
5353 // Found something in global overflow list;
5354 // not yet ready to go stealing work from others.
5355 // We'd like to assert(work_q->size() != 0, ...)
5356 // because we just took work from the overflow list,
5357 // but of course we can't, since all of that might have
5358 // been already stolen from us.
5359 continue;
5360 }
5361 // Verify that we have no work before we resort to stealing
5362 assert(work_q->size() == 0, "Have work, shouldn't steal");
5363 // Try to steal from other queues that have work
5364 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5365 NOT_PRODUCT(num_steals++;)
5366 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5367 assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5368 // Do scanning work
5369 obj_to_scan->oop_iterate(keep_alive);
5370 // Loop around, finish this work, and try to steal some more
5371 } else if (terminator()->offer_termination()) {
5372 break; // nirvana from the infinite cycle
5373 }
5374 }
5375 NOT_PRODUCT(
5376 if (PrintCMSStatistics != 0) {
5377 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
5378 }
5379 )
5380 }
5381
5382 void CMSRefProcTaskExecutor::execute(ProcessTask& task)
5383 {
5384 GenCollectedHeap* gch = GenCollectedHeap::heap();
5385 FlexibleWorkGang* workers = gch->workers();
5386 assert(workers != NULL, "Need parallel worker threads.");
5387 CMSRefProcTaskProxy rp_task(task, &_collector,
5388 _collector.ref_processor()->span(),
5389 _collector.markBitMap(),
5390 workers, _collector.task_queues());
5391 workers->run_task(&rp_task);
5392 }
5393
5394 void CMSRefProcTaskExecutor::execute(EnqueueTask& task)
5395 {
5396
5397 GenCollectedHeap* gch = GenCollectedHeap::heap();
5398 FlexibleWorkGang* workers = gch->workers();
5399 assert(workers != NULL, "Need parallel worker threads.");
5400 CMSRefEnqueueTaskProxy enq_task(task);
5401 workers->run_task(&enq_task);
5402 }
5403
5404 void CMSCollector::refProcessingWork() {
5405 ResourceMark rm;
5406 HandleMark hm;
5407
5408 ReferenceProcessor* rp = ref_processor();
5409 assert(rp->span().equals(_span), "Spans should be equal");
5410 assert(!rp->enqueuing_is_done(), "Enqueuing should not be complete");
5411 // Process weak references.
5412 rp->setup_policy(false);
5413 verify_work_stacks_empty();
5414
5415 CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap,
5416 &_markStack, false /* !preclean */);
5417 CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this,
5418 _span, &_markBitMap, &_markStack,
5419 &cmsKeepAliveClosure, false /* !preclean */);
5420 {
5421 GCTraceTime t("weak refs processing", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5422
5423 ReferenceProcessorStats stats;
5424 if (rp->processing_is_mt()) {
5425 // Set the degree of MT here. If the discovery is done MT, there
5426 // may have been a different number of threads doing the discovery
5427 // and a different number of discovered lists may have Ref objects.
5428 // That is OK as long as the Reference lists are balanced (see
5429 // balance_all_queues() and balance_queues()).
5430 GenCollectedHeap* gch = GenCollectedHeap::heap();
5431 int active_workers = ParallelGCThreads;
5432 FlexibleWorkGang* workers = gch->workers();
5433 if (workers != NULL) {
5434 active_workers = workers->active_workers();
5435 // The expectation is that active_workers will have already
5436 // been set to a reasonable value. If it has not been set,
5437 // investigate.
5438 assert(active_workers > 0, "Should have been set during scavenge");
5439 }
5440 rp->set_active_mt_degree(active_workers);
5441 CMSRefProcTaskExecutor task_executor(*this);
5442 stats = rp->process_discovered_references(&_is_alive_closure,
5443 &cmsKeepAliveClosure,
5444 &cmsDrainMarkingStackClosure,
5445 &task_executor,
5446 _gc_timer_cm,
5447 _gc_tracer_cm->gc_id());
5448 } else {
5449 stats = rp->process_discovered_references(&_is_alive_closure,
5450 &cmsKeepAliveClosure,
5451 &cmsDrainMarkingStackClosure,
5452 NULL,
5453 _gc_timer_cm,
5454 _gc_tracer_cm->gc_id());
5455 }
5456 _gc_tracer_cm->report_gc_reference_stats(stats);
5457
5458 }
5459
5460 // This is the point where the entire marking should have completed.
5461 verify_work_stacks_empty();
5462
5463 if (should_unload_classes()) {
5464 {
5465 GCTraceTime t("class unloading", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5466
5467 // Unload classes and purge the SystemDictionary.
5468 bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure);
5469
5470 // Unload nmethods.
5471 CodeCache::do_unloading(&_is_alive_closure, purged_class);
5472
5473 // Prune dead klasses from subklass/sibling/implementor lists.
5474 Klass::clean_weak_klass_links(&_is_alive_closure);
5475 }
5476
5477 {
5478 GCTraceTime t("scrub symbol table", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5479 // Clean up unreferenced symbols in symbol table.
5480 SymbolTable::unlink();
5481 }
5482
5483 {
5484 GCTraceTime t("scrub string table", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
5485 // Delete entries for dead interned strings.
5486 StringTable::unlink(&_is_alive_closure);
5487 }
5488 }
5489
5490
5491 // Restore any preserved marks as a result of mark stack or
5492 // work queue overflow
5493 restore_preserved_marks_if_any(); // done single-threaded for now
5494
5495 rp->set_enqueuing_is_done(true);
5496 if (rp->processing_is_mt()) {
5497 rp->balance_all_queues();
5498 CMSRefProcTaskExecutor task_executor(*this);
5499 rp->enqueue_discovered_references(&task_executor);
5500 } else {
5501 rp->enqueue_discovered_references(NULL);
5502 }
5503 rp->verify_no_references_recorded();
5504 assert(!rp->discovery_enabled(), "should have been disabled");
5505 }
5506
5507 #ifndef PRODUCT
5508 void CMSCollector::check_correct_thread_executing() {
5509 Thread* t = Thread::current();
5510 // Only the VM thread or the CMS thread should be here.
5511 assert(t->is_ConcurrentGC_thread() || t->is_VM_thread(),
5512 "Unexpected thread type");
5513 // If this is the vm thread, the foreground process
5514 // should not be waiting. Note that _foregroundGCIsActive is
5515 // true while the foreground collector is waiting.
5516 if (_foregroundGCShouldWait) {
5517 // We cannot be the VM thread
5518 assert(t->is_ConcurrentGC_thread(),
5519 "Should be CMS thread");
5520 } else {
5521 // We can be the CMS thread only if we are in a stop-world
5522 // phase of CMS collection.
5523 if (t->is_ConcurrentGC_thread()) {
5524 assert(_collectorState == InitialMarking ||
5525 _collectorState == FinalMarking,
5526 "Should be a stop-world phase");
5527 // The CMS thread should be holding the CMS_token.
5528 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
5529 "Potential interference with concurrently "
5530 "executing VM thread");
5531 }
5532 }
5533 }
5534 #endif
5535
5536 void CMSCollector::sweep() {
5537 assert(_collectorState == Sweeping, "just checking");
5538 check_correct_thread_executing();
5539 verify_work_stacks_empty();
5540 verify_overflow_empty();
5541 increment_sweep_count();
5542 TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause());
5543
5544 _inter_sweep_timer.stop();
5545 _inter_sweep_estimate.sample(_inter_sweep_timer.seconds());
5546
5547 assert(!_intra_sweep_timer.is_active(), "Should not be active");
5548 _intra_sweep_timer.reset();
5549 _intra_sweep_timer.start();
5550 {
5551 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
5552 CMSPhaseAccounting pa(this, "sweep", _gc_tracer_cm->gc_id(), !PrintGCDetails);
5553 // First sweep the old gen
5554 {
5555 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
5556 bitMapLock());
5557 sweepWork(_cmsGen);
5558 }
5559
5560 // Update Universe::_heap_*_at_gc figures.
5561 // We need all the free list locks to make the abstract state
5562 // transition from Sweeping to Resetting. See detailed note
5563 // further below.
5564 {
5565 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock());
5566 // Update heap occupancy information which is used as
5567 // input to soft ref clearing policy at the next gc.
5568 Universe::update_heap_info_at_gc();
5569 _collectorState = Resizing;
5570 }
5571 }
5572 verify_work_stacks_empty();
5573 verify_overflow_empty();
5574
5575 if (should_unload_classes()) {
5576 // Delay purge to the beginning of the next safepoint. Metaspace::contains
5577 // requires that the virtual spaces are stable and not deleted.
5578 ClassLoaderDataGraph::set_should_purge(true);
5579 }
5580
5581 _intra_sweep_timer.stop();
5582 _intra_sweep_estimate.sample(_intra_sweep_timer.seconds());
5583
5584 _inter_sweep_timer.reset();
5585 _inter_sweep_timer.start();
5586
5587 // We need to use a monotonically non-decreasing time in ms
5588 // or we will see time-warp warnings and os::javaTimeMillis()
5589 // does not guarantee monotonicity.
5590 jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC;
5591 update_time_of_last_gc(now);
5592
5593 // NOTE on abstract state transitions:
5594 // Mutators allocate-live and/or mark the mod-union table dirty
5595 // based on the state of the collection. The former is done in
5596 // the interval [Marking, Sweeping] and the latter in the interval
5597 // [Marking, Sweeping). Thus the transitions into the Marking state
5598 // and out of the Sweeping state must be synchronously visible
5599 // globally to the mutators.
5600 // The transition into the Marking state happens with the world
5601 // stopped so the mutators will globally see it. Sweeping is
5602 // done asynchronously by the background collector so the transition
5603 // from the Sweeping state to the Resizing state must be done
5604 // under the freelistLock (as is the check for whether to
5605 // allocate-live and whether to dirty the mod-union table).
5606 assert(_collectorState == Resizing, "Change of collector state to"
5607 " Resizing must be done under the freelistLocks (plural)");
5608
5609 // Now that sweeping has been completed, we clear
5610 // the incremental_collection_failed flag,
5611 // thus inviting a younger gen collection to promote into
5612 // this generation. If such a promotion may still fail,
5613 // the flag will be set again when a young collection is
5614 // attempted.
5615 GenCollectedHeap* gch = GenCollectedHeap::heap();
5616 gch->clear_incremental_collection_failed(); // Worth retrying as fresh space may have been freed up
5617 gch->update_full_collections_completed(_collection_count_start);
5618 }
5619
5620 // FIX ME!!! Looks like this belongs in CFLSpace, with
5621 // CMSGen merely delegating to it.
5622 void ConcurrentMarkSweepGeneration::setNearLargestChunk() {
5623 double nearLargestPercent = FLSLargestBlockCoalesceProximity;
5624 HeapWord* minAddr = _cmsSpace->bottom();
5625 HeapWord* largestAddr =
5626 (HeapWord*) _cmsSpace->dictionary()->find_largest_dict();
5627 if (largestAddr == NULL) {
5628 // The dictionary appears to be empty. In this case
5629 // try to coalesce at the end of the heap.
5630 largestAddr = _cmsSpace->end();
5631 }
5632 size_t largestOffset = pointer_delta(largestAddr, minAddr);
5633 size_t nearLargestOffset =
5634 (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
5635 if (PrintFLSStatistics != 0) {
5636 gclog_or_tty->print_cr(
5637 "CMS: Large Block: " PTR_FORMAT ";"
5638 " Proximity: " PTR_FORMAT " -> " PTR_FORMAT,
5639 p2i(largestAddr),
5640 p2i(_cmsSpace->nearLargestChunk()), p2i(minAddr + nearLargestOffset));
5641 }
5642 _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
5643 }
5644
5645 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
5646 return addr >= _cmsSpace->nearLargestChunk();
5647 }
5648
5649 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
5650 return _cmsSpace->find_chunk_at_end();
5651 }
5652
5653 void ConcurrentMarkSweepGeneration::update_gc_stats(int current_level,
5654 bool full) {
5655 // The next lower level has been collected. Gather any statistics
5656 // that are of interest at this point.
5657 if (!full && (current_level + 1) == level()) {
5658 // Gather statistics on the young generation collection.
5659 collector()->stats().record_gc0_end(used());
5660 }
5661 }
5662
5663 void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* gen) {
5664 // We iterate over the space(s) underlying this generation,
5665 // checking the mark bit map to see if the bits corresponding
5666 // to specific blocks are marked or not. Blocks that are
5667 // marked are live and are not swept up. All remaining blocks
5668 // are swept up, with coalescing on-the-fly as we sweep up
5669 // contiguous free and/or garbage blocks:
5670 // We need to ensure that the sweeper synchronizes with allocators
5671 // and stop-the-world collectors. In particular, the following
5672 // locks are used:
5673 // . CMS token: if this is held, a stop the world collection cannot occur
5674 // . freelistLock: if this is held no allocation can occur from this
5675 // generation by another thread
5676 // . bitMapLock: if this is held, no other thread can access or update
5677 //
5678
5679 // Note that we need to hold the freelistLock if we use
5680 // block iterate below; else the iterator might go awry if
5681 // a mutator (or promotion) causes block contents to change
5682 // (for instance if the allocator divvies up a block).
5683 // If we hold the free list lock, for all practical purposes
5684 // young generation GC's can't occur (they'll usually need to
5685 // promote), so we might as well prevent all young generation
5686 // GC's while we do a sweeping step. For the same reason, we might
5687 // as well take the bit map lock for the entire duration
5688
5689 // check that we hold the requisite locks
5690 assert(have_cms_token(), "Should hold cms token");
5691 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), "Should possess CMS token to sweep");
5692 assert_lock_strong(gen->freelistLock());
5693 assert_lock_strong(bitMapLock());
5694
5695 assert(!_inter_sweep_timer.is_active(), "Was switched off in an outer context");
5696 assert(_intra_sweep_timer.is_active(), "Was switched on in an outer context");
5697 gen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()),
5698 _inter_sweep_estimate.padded_average(),
5699 _intra_sweep_estimate.padded_average());
5700 gen->setNearLargestChunk();
5701
5702 {
5703 SweepClosure sweepClosure(this, gen, &_markBitMap, CMSYield);
5704 gen->cmsSpace()->blk_iterate_careful(&sweepClosure);
5705 // We need to free-up/coalesce garbage/blocks from a
5706 // co-terminal free run. This is done in the SweepClosure
5707 // destructor; so, do not remove this scope, else the
5708 // end-of-sweep-census below will be off by a little bit.
5709 }
5710 gen->cmsSpace()->sweep_completed();
5711 gen->cmsSpace()->endSweepFLCensus(sweep_count());
5712 if (should_unload_classes()) { // unloaded classes this cycle,
5713 _concurrent_cycles_since_last_unload = 0; // ... reset count
5714 } else { // did not unload classes,
5715 _concurrent_cycles_since_last_unload++; // ... increment count
5716 }
5717 }
5718
5719 // Reset CMS data structures (for now just the marking bit map)
5720 // preparatory for the next cycle.
5721 void CMSCollector::reset(bool concurrent) {
5722 if (concurrent) {
5723 CMSTokenSyncWithLocks ts(true, bitMapLock());
5724
5725 // If the state is not "Resetting", the foreground thread
5726 // has done a collection and the resetting.
5727 if (_collectorState != Resetting) {
5728 assert(_collectorState == Idling, "The state should only change"
5729 " because the foreground collector has finished the collection");
5730 return;
5731 }
5732
5733 // Clear the mark bitmap (no grey objects to start with)
5734 // for the next cycle.
5735 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
5736 CMSPhaseAccounting cmspa(this, "reset", _gc_tracer_cm->gc_id(), !PrintGCDetails);
5737
5738 HeapWord* curAddr = _markBitMap.startWord();
5739 while (curAddr < _markBitMap.endWord()) {
5740 size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr);
5741 MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining));
5742 _markBitMap.clear_large_range(chunk);
5743 if (ConcurrentMarkSweepThread::should_yield() &&
5744 !foregroundGCIsActive() &&
5745 CMSYield) {
5746 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
5747 "CMS thread should hold CMS token");
5748 assert_lock_strong(bitMapLock());
5749 bitMapLock()->unlock();
5750 ConcurrentMarkSweepThread::desynchronize(true);
5751 stopTimer();
5752 if (PrintCMSStatistics != 0) {
5753 incrementYields();
5754 }
5755
5756 // See the comment in coordinator_yield()
5757 for (unsigned i = 0; i < CMSYieldSleepCount &&
5758 ConcurrentMarkSweepThread::should_yield() &&
5759 !CMSCollector::foregroundGCIsActive(); ++i) {
5760 os::sleep(Thread::current(), 1, false);
5761 }
5762
5763 ConcurrentMarkSweepThread::synchronize(true);
5764 bitMapLock()->lock_without_safepoint_check();
5765 startTimer();
5766 }
5767 curAddr = chunk.end();
5768 }
5769 // A successful mostly concurrent collection has been done.
5770 // Because only the full (i.e., concurrent mode failure) collections
5771 // are being measured for gc overhead limits, clean the "near" flag
5772 // and count.
5773 size_policy()->reset_gc_overhead_limit_count();
5774 _collectorState = Idling;
5775 } else {
5776 // already have the lock
5777 assert(_collectorState == Resetting, "just checking");
5778 assert_lock_strong(bitMapLock());
5779 _markBitMap.clear_all();
5780 _collectorState = Idling;
5781 }
5782
5783 register_gc_end();
5784 }
5785
5786 void CMSCollector::do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause) {
5787 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
5788 GCTraceTime t(GCCauseString("GC", gc_cause), PrintGC, !PrintGCDetails, NULL, _gc_tracer_cm->gc_id());
5789 TraceCollectorStats tcs(counters());
5790
5791 switch (op) {
5792 case CMS_op_checkpointRootsInitial: {
5793 SvcGCMarker sgcm(SvcGCMarker::OTHER);
5794 checkpointRootsInitial();
5795 if (PrintGC) {
5796 _cmsGen->printOccupancy("initial-mark");
5797 }
5798 break;
5799 }
5800 case CMS_op_checkpointRootsFinal: {
5801 SvcGCMarker sgcm(SvcGCMarker::OTHER);
5802 checkpointRootsFinal();
5803 if (PrintGC) {
5804 _cmsGen->printOccupancy("remark");
5805 }
5806 break;
5807 }
5808 default:
5809 fatal("No such CMS_op");
5810 }
5811 }
5812
5813 #ifndef PRODUCT
5814 size_t const CMSCollector::skip_header_HeapWords() {
5815 return FreeChunk::header_size();
5816 }
5817
5818 // Try and collect here conditions that should hold when
5819 // CMS thread is exiting. The idea is that the foreground GC
5820 // thread should not be blocked if it wants to terminate
5821 // the CMS thread and yet continue to run the VM for a while
5822 // after that.
5823 void CMSCollector::verify_ok_to_terminate() const {
5824 assert(Thread::current()->is_ConcurrentGC_thread(),
5825 "should be called by CMS thread");
5826 assert(!_foregroundGCShouldWait, "should be false");
5827 // We could check here that all the various low-level locks
5828 // are not held by the CMS thread, but that is overkill; see
5829 // also CMSThread::verify_ok_to_terminate() where the CGC_lock
5830 // is checked.
5831 }
5832 #endif
5833
5834 size_t CMSCollector::block_size_using_printezis_bits(HeapWord* addr) const {
5835 assert(_markBitMap.isMarked(addr) && _markBitMap.isMarked(addr + 1),
5836 "missing Printezis mark?");
5837 HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
5838 size_t size = pointer_delta(nextOneAddr + 1, addr);
5839 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
5840 "alignment problem");
5841 assert(size >= 3, "Necessary for Printezis marks to work");
5842 return size;
5843 }
5844
5845 // A variant of the above (block_size_using_printezis_bits()) except
5846 // that we return 0 if the P-bits are not yet set.
5847 size_t CMSCollector::block_size_if_printezis_bits(HeapWord* addr) const {
5848 if (_markBitMap.isMarked(addr + 1)) {
5849 assert(_markBitMap.isMarked(addr), "P-bit can be set only for marked objects");
5850 HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
5851 size_t size = pointer_delta(nextOneAddr + 1, addr);
5852 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
5853 "alignment problem");
5854 assert(size >= 3, "Necessary for Printezis marks to work");
5855 return size;
5856 }
5857 return 0;
5858 }
5859
5860 HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
5861 size_t sz = 0;
5862 oop p = (oop)addr;
5863 if (p->klass_or_null() != NULL) {
5864 sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
5865 } else {
5866 sz = block_size_using_printezis_bits(addr);
5867 }
5868 assert(sz > 0, "size must be nonzero");
5869 HeapWord* next_block = addr + sz;
5870 HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block,
5871 CardTableModRefBS::card_size);
5872 assert(round_down((uintptr_t)addr, CardTableModRefBS::card_size) <
5873 round_down((uintptr_t)next_card, CardTableModRefBS::card_size),
5874 "must be different cards");
5875 return next_card;
5876 }
5877
5878
5879 // CMS Bit Map Wrapper /////////////////////////////////////////
5880
5881 // Construct a CMS bit map infrastructure, but don't create the
5882 // bit vector itself. That is done by a separate call CMSBitMap::allocate()
5883 // further below.
5884 CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
5885 _bm(),
5886 _shifter(shifter),
5887 _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true,
5888 Monitor::_safepoint_check_sometimes) : NULL)
5889 {
5890 _bmStartWord = 0;
5891 _bmWordSize = 0;
5892 }
5893
5894 bool CMSBitMap::allocate(MemRegion mr) {
5895 _bmStartWord = mr.start();
5896 _bmWordSize = mr.word_size();
5897 ReservedSpace brs(ReservedSpace::allocation_align_size_up(
5898 (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1));
5899 if (!brs.is_reserved()) {
5900 warning("CMS bit map allocation failure");
5901 return false;
5902 }
5903 // For now we'll just commit all of the bit map up front.
5904 // Later on we'll try to be more parsimonious with swap.
5905 if (!_virtual_space.initialize(brs, brs.size())) {
5906 warning("CMS bit map backing store failure");
5907 return false;
5908 }
5909 assert(_virtual_space.committed_size() == brs.size(),
5910 "didn't reserve backing store for all of CMS bit map?");
5911 _bm.set_map((BitMap::bm_word_t*)_virtual_space.low());
5912 assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
5913 _bmWordSize, "inconsistency in bit map sizing");
5914 _bm.set_size(_bmWordSize >> _shifter);
5915
5916 // bm.clear(); // can we rely on getting zero'd memory? verify below
5917 assert(isAllClear(),
5918 "Expected zero'd memory from ReservedSpace constructor");
5919 assert(_bm.size() == heapWordDiffToOffsetDiff(sizeInWords()),
5920 "consistency check");
5921 return true;
5922 }
5923
5924 void CMSBitMap::dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl) {
5925 HeapWord *next_addr, *end_addr, *last_addr;
5926 assert_locked();
5927 assert(covers(mr), "out-of-range error");
5928 // XXX assert that start and end are appropriately aligned
5929 for (next_addr = mr.start(), end_addr = mr.end();
5930 next_addr < end_addr; next_addr = last_addr) {
5931 MemRegion dirty_region = getAndClearMarkedRegion(next_addr, end_addr);
5932 last_addr = dirty_region.end();
5933 if (!dirty_region.is_empty()) {
5934 cl->do_MemRegion(dirty_region);
5935 } else {
5936 assert(last_addr == end_addr, "program logic");
5937 return;
5938 }
5939 }
5940 }
5941
5942 void CMSBitMap::print_on_error(outputStream* st, const char* prefix) const {
5943 _bm.print_on_error(st, prefix);
5944 }
5945
5946 #ifndef PRODUCT
5947 void CMSBitMap::assert_locked() const {
5948 CMSLockVerifier::assert_locked(lock());
5949 }
5950
5951 bool CMSBitMap::covers(MemRegion mr) const {
5952 // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
5953 assert((size_t)_bm.size() == (_bmWordSize >> _shifter),
5954 "size inconsistency");
5955 return (mr.start() >= _bmStartWord) &&
5956 (mr.end() <= endWord());
5957 }
5958
5959 bool CMSBitMap::covers(HeapWord* start, size_t size) const {
5960 return (start >= _bmStartWord && (start + size) <= endWord());
5961 }
5962
5963 void CMSBitMap::verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) {
5964 // verify that there are no 1 bits in the interval [left, right)
5965 FalseBitMapClosure falseBitMapClosure;
5966 iterate(&falseBitMapClosure, left, right);
5967 }
5968
5969 void CMSBitMap::region_invariant(MemRegion mr)
5970 {
5971 assert_locked();
5972 // mr = mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
5973 assert(!mr.is_empty(), "unexpected empty region");
5974 assert(covers(mr), "mr should be covered by bit map");
5975 // convert address range into offset range
5976 size_t start_ofs = heapWordToOffset(mr.start());
5977 // Make sure that end() is appropriately aligned
5978 assert(mr.end() == (HeapWord*)round_to((intptr_t)mr.end(),
5979 (1 << (_shifter+LogHeapWordSize))),
5980 "Misaligned mr.end()");
5981 size_t end_ofs = heapWordToOffset(mr.end());
5982 assert(end_ofs > start_ofs, "Should mark at least one bit");
5983 }
5984
5985 #endif
5986
5987 bool CMSMarkStack::allocate(size_t size) {
5988 // allocate a stack of the requisite depth
5989 ReservedSpace rs(ReservedSpace::allocation_align_size_up(
5990 size * sizeof(oop)));
5991 if (!rs.is_reserved()) {
5992 warning("CMSMarkStack allocation failure");
5993 return false;
5994 }
5995 if (!_virtual_space.initialize(rs, rs.size())) {
5996 warning("CMSMarkStack backing store failure");
5997 return false;
5998 }
5999 assert(_virtual_space.committed_size() == rs.size(),
6000 "didn't reserve backing store for all of CMS stack?");
6001 _base = (oop*)(_virtual_space.low());
6002 _index = 0;
6003 _capacity = size;
6004 NOT_PRODUCT(_max_depth = 0);
6005 return true;
6006 }
6007
6008 // XXX FIX ME !!! In the MT case we come in here holding a
6009 // leaf lock. For printing we need to take a further lock
6010 // which has lower rank. We need to recalibrate the two
6011 // lock-ranks involved in order to be able to print the
6012 // messages below. (Or defer the printing to the caller.
6013 // For now we take the expedient path of just disabling the
6014 // messages for the problematic case.)
6015 void CMSMarkStack::expand() {
6016 assert(_capacity <= MarkStackSizeMax, "stack bigger than permitted");
6017 if (_capacity == MarkStackSizeMax) {
6018 if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
6019 // We print a warning message only once per CMS cycle.
6020 gclog_or_tty->print_cr(" (benign) Hit CMSMarkStack max size limit");
6021 }
6022 return;
6023 }
6024 // Double capacity if possible
6025 size_t new_capacity = MIN2(_capacity*2, MarkStackSizeMax);
6026 // Do not give up existing stack until we have managed to
6027 // get the double capacity that we desired.
6028 ReservedSpace rs(ReservedSpace::allocation_align_size_up(
6029 new_capacity * sizeof(oop)));
6030 if (rs.is_reserved()) {
6031 // Release the backing store associated with old stack
6032 _virtual_space.release();
6033 // Reinitialize virtual space for new stack
6034 if (!_virtual_space.initialize(rs, rs.size())) {
6035 fatal("Not enough swap for expanded marking stack");
6036 }
6037 _base = (oop*)(_virtual_space.low());
6038 _index = 0;
6039 _capacity = new_capacity;
6040 } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
6041 // Failed to double capacity, continue;
6042 // we print a detail message only once per CMS cycle.
6043 gclog_or_tty->print(" (benign) Failed to expand marking stack from "SIZE_FORMAT"K to "
6044 SIZE_FORMAT"K",
6045 _capacity / K, new_capacity / K);
6046 }
6047 }
6048
6049
6050 // Closures
6051 // XXX: there seems to be a lot of code duplication here;
6052 // should refactor and consolidate common code.
6053
6054 // This closure is used to mark refs into the CMS generation in
6055 // the CMS bit map. Called at the first checkpoint. This closure
6056 // assumes that we do not need to re-mark dirty cards; if the CMS
6057 // generation on which this is used is not an oldest
6058 // generation then this will lose younger_gen cards!
6059
6060 MarkRefsIntoClosure::MarkRefsIntoClosure(
6061 MemRegion span, CMSBitMap* bitMap):
6062 _span(span),
6063 _bitMap(bitMap)
6064 {
6065 assert(_ref_processor == NULL, "deliberately left NULL");
6066 assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
6067 }
6068
6069 void MarkRefsIntoClosure::do_oop(oop obj) {
6070 // if p points into _span, then mark corresponding bit in _markBitMap
6071 assert(obj->is_oop(), "expected an oop");
6072 HeapWord* addr = (HeapWord*)obj;
6073 if (_span.contains(addr)) {
6074 // this should be made more efficient
6075 _bitMap->mark(addr);
6076 }
6077 }
6078
6079 void MarkRefsIntoClosure::do_oop(oop* p) { MarkRefsIntoClosure::do_oop_work(p); }
6080 void MarkRefsIntoClosure::do_oop(narrowOop* p) { MarkRefsIntoClosure::do_oop_work(p); }
6081
6082 Par_MarkRefsIntoClosure::Par_MarkRefsIntoClosure(
6083 MemRegion span, CMSBitMap* bitMap):
6084 _span(span),
6085 _bitMap(bitMap)
6086 {
6087 assert(_ref_processor == NULL, "deliberately left NULL");
6088 assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
6089 }
6090
6091 void Par_MarkRefsIntoClosure::do_oop(oop obj) {
6092 // if p points into _span, then mark corresponding bit in _markBitMap
6093 assert(obj->is_oop(), "expected an oop");
6094 HeapWord* addr = (HeapWord*)obj;
6095 if (_span.contains(addr)) {
6096 // this should be made more efficient
6097 _bitMap->par_mark(addr);
6098 }
6099 }
6100
6101 void Par_MarkRefsIntoClosure::do_oop(oop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); }
6102 void Par_MarkRefsIntoClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); }
6103
6104 // A variant of the above, used for CMS marking verification.
6105 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
6106 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
6107 _span(span),
6108 _verification_bm(verification_bm),
6109 _cms_bm(cms_bm)
6110 {
6111 assert(_ref_processor == NULL, "deliberately left NULL");
6112 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
6113 }
6114
6115 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
6116 // if p points into _span, then mark corresponding bit in _markBitMap
6117 assert(obj->is_oop(), "expected an oop");
6118 HeapWord* addr = (HeapWord*)obj;
6119 if (_span.contains(addr)) {
6120 _verification_bm->mark(addr);
6121 if (!_cms_bm->isMarked(addr)) {
6122 oop(addr)->print();
6123 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", p2i(addr));
6124 fatal("... aborting");
6125 }
6126 }
6127 }
6128
6129 void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
6130 void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
6131
6132 //////////////////////////////////////////////////
6133 // MarkRefsIntoAndScanClosure
6134 //////////////////////////////////////////////////
6135
6136 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
6137 ReferenceProcessor* rp,
6138 CMSBitMap* bit_map,
6139 CMSBitMap* mod_union_table,
6140 CMSMarkStack* mark_stack,
6141 CMSCollector* collector,
6142 bool should_yield,
6143 bool concurrent_precleaning):
6144 _collector(collector),
6145 _span(span),
6146 _bit_map(bit_map),
6147 _mark_stack(mark_stack),
6148 _pushAndMarkClosure(collector, span, rp, bit_map, mod_union_table,
6149 mark_stack, concurrent_precleaning),
6150 _yield(should_yield),
6151 _concurrent_precleaning(concurrent_precleaning),
6152 _freelistLock(NULL)
6153 {
6154 _ref_processor = rp;
6155 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
6156 }
6157
6158 // This closure is used to mark refs into the CMS generation at the
6159 // second (final) checkpoint, and to scan and transitively follow
6160 // the unmarked oops. It is also used during the concurrent precleaning
6161 // phase while scanning objects on dirty cards in the CMS generation.
6162 // The marks are made in the marking bit map and the marking stack is
6163 // used for keeping the (newly) grey objects during the scan.
6164 // The parallel version (Par_...) appears further below.
6165 void MarkRefsIntoAndScanClosure::do_oop(oop obj) {
6166 if (obj != NULL) {
6167 assert(obj->is_oop(), "expected an oop");
6168 HeapWord* addr = (HeapWord*)obj;
6169 assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)");
6170 assert(_collector->overflow_list_is_empty(),
6171 "overflow list should be empty");
6172 if (_span.contains(addr) &&
6173 !_bit_map->isMarked(addr)) {
6174 // mark bit map (object is now grey)
6175 _bit_map->mark(addr);
6176 // push on marking stack (stack should be empty), and drain the
6177 // stack by applying this closure to the oops in the oops popped
6178 // from the stack (i.e. blacken the grey objects)
6179 bool res = _mark_stack->push(obj);
6180 assert(res, "Should have space to push on empty stack");
6181 do {
6182 oop new_oop = _mark_stack->pop();
6183 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
6184 assert(_bit_map->isMarked((HeapWord*)new_oop),
6185 "only grey objects on this stack");
6186 // iterate over the oops in this oop, marking and pushing
6187 // the ones in CMS heap (i.e. in _span).
6188 new_oop->oop_iterate(&_pushAndMarkClosure);
6189 // check if it's time to yield
6190 do_yield_check();
6191 } while (!_mark_stack->isEmpty() ||
6192 (!_concurrent_precleaning && take_from_overflow_list()));
6193 // if marking stack is empty, and we are not doing this
6194 // during precleaning, then check the overflow list
6195 }
6196 assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
6197 assert(_collector->overflow_list_is_empty(),
6198 "overflow list was drained above");
6199 // We could restore evacuated mark words, if any, used for
6200 // overflow list links here because the overflow list is
6201 // provably empty here. That would reduce the maximum
6202 // size requirements for preserved_{oop,mark}_stack.
6203 // But we'll just postpone it until we are all done
6204 // so we can just stream through.
6205 if (!_concurrent_precleaning && CMSOverflowEarlyRestoration) {
6206 _collector->restore_preserved_marks_if_any();
6207 assert(_collector->no_preserved_marks(), "No preserved marks");
6208 }
6209 assert(!CMSOverflowEarlyRestoration || _collector->no_preserved_marks(),
6210 "All preserved marks should have been restored above");
6211 }
6212 }
6213
6214 void MarkRefsIntoAndScanClosure::do_oop(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
6215 void MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
6216
6217 void MarkRefsIntoAndScanClosure::do_yield_work() {
6218 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6219 "CMS thread should hold CMS token");
6220 assert_lock_strong(_freelistLock);
6221 assert_lock_strong(_bit_map->lock());
6222 // relinquish the free_list_lock and bitMaplock()
6223 _bit_map->lock()->unlock();
6224 _freelistLock->unlock();
6225 ConcurrentMarkSweepThread::desynchronize(true);
6226 _collector->stopTimer();
6227 if (PrintCMSStatistics != 0) {
6228 _collector->incrementYields();
6229 }
6230
6231 // See the comment in coordinator_yield()
6232 for (unsigned i = 0;
6233 i < CMSYieldSleepCount &&
6234 ConcurrentMarkSweepThread::should_yield() &&
6235 !CMSCollector::foregroundGCIsActive();
6236 ++i) {
6237 os::sleep(Thread::current(), 1, false);
6238 }
6239
6240 ConcurrentMarkSweepThread::synchronize(true);
6241 _freelistLock->lock_without_safepoint_check();
6242 _bit_map->lock()->lock_without_safepoint_check();
6243 _collector->startTimer();
6244 }
6245
6246 ///////////////////////////////////////////////////////////
6247 // Par_MarkRefsIntoAndScanClosure: a parallel version of
6248 // MarkRefsIntoAndScanClosure
6249 ///////////////////////////////////////////////////////////
6250 Par_MarkRefsIntoAndScanClosure::Par_MarkRefsIntoAndScanClosure(
6251 CMSCollector* collector, MemRegion span, ReferenceProcessor* rp,
6252 CMSBitMap* bit_map, OopTaskQueue* work_queue):
6253 _span(span),
6254 _bit_map(bit_map),
6255 _work_queue(work_queue),
6256 _low_water_mark(MIN2((uint)(work_queue->max_elems()/4),
6257 (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads))),
6258 _par_pushAndMarkClosure(collector, span, rp, bit_map, work_queue)
6259 {
6260 _ref_processor = rp;
6261 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
6262 }
6263
6264 // This closure is used to mark refs into the CMS generation at the
6265 // second (final) checkpoint, and to scan and transitively follow
6266 // the unmarked oops. The marks are made in the marking bit map and
6267 // the work_queue is used for keeping the (newly) grey objects during
6268 // the scan phase whence they are also available for stealing by parallel
6269 // threads. Since the marking bit map is shared, updates are
6270 // synchronized (via CAS).
6271 void Par_MarkRefsIntoAndScanClosure::do_oop(oop obj) {
6272 if (obj != NULL) {
6273 // Ignore mark word because this could be an already marked oop
6274 // that may be chained at the end of the overflow list.
6275 assert(obj->is_oop(true), "expected an oop");
6276 HeapWord* addr = (HeapWord*)obj;
6277 if (_span.contains(addr) &&
6278 !_bit_map->isMarked(addr)) {
6279 // mark bit map (object will become grey):
6280 // It is possible for several threads to be
6281 // trying to "claim" this object concurrently;
6282 // the unique thread that succeeds in marking the
6283 // object first will do the subsequent push on
6284 // to the work queue (or overflow list).
6285 if (_bit_map->par_mark(addr)) {
6286 // push on work_queue (which may not be empty), and trim the
6287 // queue to an appropriate length by applying this closure to
6288 // the oops in the oops popped from the stack (i.e. blacken the
6289 // grey objects)
6290 bool res = _work_queue->push(obj);
6291 assert(res, "Low water mark should be less than capacity?");
6292 trim_queue(_low_water_mark);
6293 } // Else, another thread claimed the object
6294 }
6295 }
6296 }
6297
6298 void Par_MarkRefsIntoAndScanClosure::do_oop(oop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
6299 void Par_MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
6300
6301 // This closure is used to rescan the marked objects on the dirty cards
6302 // in the mod union table and the card table proper.
6303 size_t ScanMarkedObjectsAgainCarefullyClosure::do_object_careful_m(
6304 oop p, MemRegion mr) {
6305
6306 size_t size = 0;
6307 HeapWord* addr = (HeapWord*)p;
6308 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6309 assert(_span.contains(addr), "we are scanning the CMS generation");
6310 // check if it's time to yield
6311 if (do_yield_check()) {
6312 // We yielded for some foreground stop-world work,
6313 // and we have been asked to abort this ongoing preclean cycle.
6314 return 0;
6315 }
6316 if (_bitMap->isMarked(addr)) {
6317 // it's marked; is it potentially uninitialized?
6318 if (p->klass_or_null() != NULL) {
6319 // an initialized object; ignore mark word in verification below
6320 // since we are running concurrent with mutators
6321 assert(p->is_oop(true), "should be an oop");
6322 if (p->is_objArray()) {
6323 // objArrays are precisely marked; restrict scanning
6324 // to dirty cards only.
6325 size = CompactibleFreeListSpace::adjustObjectSize(
6326 p->oop_iterate(_scanningClosure, mr));
6327 } else {
6328 // A non-array may have been imprecisely marked; we need
6329 // to scan object in its entirety.
6330 size = CompactibleFreeListSpace::adjustObjectSize(
6331 p->oop_iterate(_scanningClosure));
6332 }
6333 #ifdef ASSERT
6334 size_t direct_size =
6335 CompactibleFreeListSpace::adjustObjectSize(p->size());
6336 assert(size == direct_size, "Inconsistency in size");
6337 assert(size >= 3, "Necessary for Printezis marks to work");
6338 if (!_bitMap->isMarked(addr+1)) {
6339 _bitMap->verifyNoOneBitsInRange(addr+2, addr+size);
6340 } else {
6341 _bitMap->verifyNoOneBitsInRange(addr+2, addr+size-1);
6342 assert(_bitMap->isMarked(addr+size-1),
6343 "inconsistent Printezis mark");
6344 }
6345 #endif // ASSERT
6346 } else {
6347 // An uninitialized object.
6348 assert(_bitMap->isMarked(addr+1), "missing Printezis mark?");
6349 HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
6350 size = pointer_delta(nextOneAddr + 1, addr);
6351 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6352 "alignment problem");
6353 // Note that pre-cleaning needn't redirty the card. OopDesc::set_klass()
6354 // will dirty the card when the klass pointer is installed in the
6355 // object (signaling the completion of initialization).
6356 }
6357 } else {
6358 // Either a not yet marked object or an uninitialized object
6359 if (p->klass_or_null() == NULL) {
6360 // An uninitialized object, skip to the next card, since
6361 // we may not be able to read its P-bits yet.
6362 assert(size == 0, "Initial value");
6363 } else {
6364 // An object not (yet) reached by marking: we merely need to
6365 // compute its size so as to go look at the next block.
6366 assert(p->is_oop(true), "should be an oop");
6367 size = CompactibleFreeListSpace::adjustObjectSize(p->size());
6368 }
6369 }
6370 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6371 return size;
6372 }
6373
6374 void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
6375 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6376 "CMS thread should hold CMS token");
6377 assert_lock_strong(_freelistLock);
6378 assert_lock_strong(_bitMap->lock());
6379 // relinquish the free_list_lock and bitMaplock()
6380 _bitMap->lock()->unlock();
6381 _freelistLock->unlock();
6382 ConcurrentMarkSweepThread::desynchronize(true);
6383 _collector->stopTimer();
6384 if (PrintCMSStatistics != 0) {
6385 _collector->incrementYields();
6386 }
6387
6388 // See the comment in coordinator_yield()
6389 for (unsigned i = 0; i < CMSYieldSleepCount &&
6390 ConcurrentMarkSweepThread::should_yield() &&
6391 !CMSCollector::foregroundGCIsActive(); ++i) {
6392 os::sleep(Thread::current(), 1, false);
6393 }
6394
6395 ConcurrentMarkSweepThread::synchronize(true);
6396 _freelistLock->lock_without_safepoint_check();
6397 _bitMap->lock()->lock_without_safepoint_check();
6398 _collector->startTimer();
6399 }
6400
6401
6402 //////////////////////////////////////////////////////////////////
6403 // SurvivorSpacePrecleanClosure
6404 //////////////////////////////////////////////////////////////////
6405 // This (single-threaded) closure is used to preclean the oops in
6406 // the survivor spaces.
6407 size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) {
6408
6409 HeapWord* addr = (HeapWord*)p;
6410 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6411 assert(!_span.contains(addr), "we are scanning the survivor spaces");
6412 assert(p->klass_or_null() != NULL, "object should be initialized");
6413 // an initialized object; ignore mark word in verification below
6414 // since we are running concurrent with mutators
6415 assert(p->is_oop(true), "should be an oop");
6416 // Note that we do not yield while we iterate over
6417 // the interior oops of p, pushing the relevant ones
6418 // on our marking stack.
6419 size_t size = p->oop_iterate(_scanning_closure);
6420 do_yield_check();
6421 // Observe that below, we do not abandon the preclean
6422 // phase as soon as we should; rather we empty the
6423 // marking stack before returning. This is to satisfy
6424 // some existing assertions. In general, it may be a
6425 // good idea to abort immediately and complete the marking
6426 // from the grey objects at a later time.
6427 while (!_mark_stack->isEmpty()) {
6428 oop new_oop = _mark_stack->pop();
6429 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
6430 assert(_bit_map->isMarked((HeapWord*)new_oop),
6431 "only grey objects on this stack");
6432 // iterate over the oops in this oop, marking and pushing
6433 // the ones in CMS heap (i.e. in _span).
6434 new_oop->oop_iterate(_scanning_closure);
6435 // check if it's time to yield
6436 do_yield_check();
6437 }
6438 unsigned int after_count =
6439 GenCollectedHeap::heap()->total_collections();
6440 bool abort = (_before_count != after_count) ||
6441 _collector->should_abort_preclean();
6442 return abort ? 0 : size;
6443 }
6444
6445 void SurvivorSpacePrecleanClosure::do_yield_work() {
6446 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6447 "CMS thread should hold CMS token");
6448 assert_lock_strong(_bit_map->lock());
6449 // Relinquish the bit map lock
6450 _bit_map->lock()->unlock();
6451 ConcurrentMarkSweepThread::desynchronize(true);
6452 _collector->stopTimer();
6453 if (PrintCMSStatistics != 0) {
6454 _collector->incrementYields();
6455 }
6456
6457 // See the comment in coordinator_yield()
6458 for (unsigned i = 0; i < CMSYieldSleepCount &&
6459 ConcurrentMarkSweepThread::should_yield() &&
6460 !CMSCollector::foregroundGCIsActive(); ++i) {
6461 os::sleep(Thread::current(), 1, false);
6462 }
6463
6464 ConcurrentMarkSweepThread::synchronize(true);
6465 _bit_map->lock()->lock_without_safepoint_check();
6466 _collector->startTimer();
6467 }
6468
6469 // This closure is used to rescan the marked objects on the dirty cards
6470 // in the mod union table and the card table proper. In the parallel
6471 // case, although the bitMap is shared, we do a single read so the
6472 // isMarked() query is "safe".
6473 bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) {
6474 // Ignore mark word because we are running concurrent with mutators
6475 assert(p->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(p)));
6476 HeapWord* addr = (HeapWord*)p;
6477 assert(_span.contains(addr), "we are scanning the CMS generation");
6478 bool is_obj_array = false;
6479 #ifdef ASSERT
6480 if (!_parallel) {
6481 assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)");
6482 assert(_collector->overflow_list_is_empty(),
6483 "overflow list should be empty");
6484
6485 }
6486 #endif // ASSERT
6487 if (_bit_map->isMarked(addr)) {
6488 // Obj arrays are precisely marked, non-arrays are not;
6489 // so we scan objArrays precisely and non-arrays in their
6490 // entirety.
6491 if (p->is_objArray()) {
6492 is_obj_array = true;
6493 if (_parallel) {
6494 p->oop_iterate(_par_scan_closure, mr);
6495 } else {
6496 p->oop_iterate(_scan_closure, mr);
6497 }
6498 } else {
6499 if (_parallel) {
6500 p->oop_iterate(_par_scan_closure);
6501 } else {
6502 p->oop_iterate(_scan_closure);
6503 }
6504 }
6505 }
6506 #ifdef ASSERT
6507 if (!_parallel) {
6508 assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
6509 assert(_collector->overflow_list_is_empty(),
6510 "overflow list should be empty");
6511
6512 }
6513 #endif // ASSERT
6514 return is_obj_array;
6515 }
6516
6517 MarkFromRootsClosure::MarkFromRootsClosure(CMSCollector* collector,
6518 MemRegion span,
6519 CMSBitMap* bitMap, CMSMarkStack* markStack,
6520 bool should_yield, bool verifying):
6521 _collector(collector),
6522 _span(span),
6523 _bitMap(bitMap),
6524 _mut(&collector->_modUnionTable),
6525 _markStack(markStack),
6526 _yield(should_yield),
6527 _skipBits(0)
6528 {
6529 assert(_markStack->isEmpty(), "stack should be empty");
6530 _finger = _bitMap->startWord();
6531 _threshold = _finger;
6532 assert(_collector->_restart_addr == NULL, "Sanity check");
6533 assert(_span.contains(_finger), "Out of bounds _finger?");
6534 DEBUG_ONLY(_verifying = verifying;)
6535 }
6536
6537 void MarkFromRootsClosure::reset(HeapWord* addr) {
6538 assert(_markStack->isEmpty(), "would cause duplicates on stack");
6539 assert(_span.contains(addr), "Out of bounds _finger?");
6540 _finger = addr;
6541 _threshold = (HeapWord*)round_to(
6542 (intptr_t)_finger, CardTableModRefBS::card_size);
6543 }
6544
6545 // Should revisit to see if this should be restructured for
6546 // greater efficiency.
6547 bool MarkFromRootsClosure::do_bit(size_t offset) {
6548 if (_skipBits > 0) {
6549 _skipBits--;
6550 return true;
6551 }
6552 // convert offset into a HeapWord*
6553 HeapWord* addr = _bitMap->startWord() + offset;
6554 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
6555 "address out of range");
6556 assert(_bitMap->isMarked(addr), "tautology");
6557 if (_bitMap->isMarked(addr+1)) {
6558 // this is an allocated but not yet initialized object
6559 assert(_skipBits == 0, "tautology");
6560 _skipBits = 2; // skip next two marked bits ("Printezis-marks")
6561 oop p = oop(addr);
6562 if (p->klass_or_null() == NULL) {
6563 DEBUG_ONLY(if (!_verifying) {)
6564 // We re-dirty the cards on which this object lies and increase
6565 // the _threshold so that we'll come back to scan this object
6566 // during the preclean or remark phase. (CMSCleanOnEnter)
6567 if (CMSCleanOnEnter) {
6568 size_t sz = _collector->block_size_using_printezis_bits(addr);
6569 HeapWord* end_card_addr = (HeapWord*)round_to(
6570 (intptr_t)(addr+sz), CardTableModRefBS::card_size);
6571 MemRegion redirty_range = MemRegion(addr, end_card_addr);
6572 assert(!redirty_range.is_empty(), "Arithmetical tautology");
6573 // Bump _threshold to end_card_addr; note that
6574 // _threshold cannot possibly exceed end_card_addr, anyhow.
6575 // This prevents future clearing of the card as the scan proceeds
6576 // to the right.
6577 assert(_threshold <= end_card_addr,
6578 "Because we are just scanning into this object");
6579 if (_threshold < end_card_addr) {
6580 _threshold = end_card_addr;
6581 }
6582 if (p->klass_or_null() != NULL) {
6583 // Redirty the range of cards...
6584 _mut->mark_range(redirty_range);
6585 } // ...else the setting of klass will dirty the card anyway.
6586 }
6587 DEBUG_ONLY(})
6588 return true;
6589 }
6590 }
6591 scanOopsInOop(addr);
6592 return true;
6593 }
6594
6595 // We take a break if we've been at this for a while,
6596 // so as to avoid monopolizing the locks involved.
6597 void MarkFromRootsClosure::do_yield_work() {
6598 // First give up the locks, then yield, then re-lock
6599 // We should probably use a constructor/destructor idiom to
6600 // do this unlock/lock or modify the MutexUnlocker class to
6601 // serve our purpose. XXX
6602 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6603 "CMS thread should hold CMS token");
6604 assert_lock_strong(_bitMap->lock());
6605 _bitMap->lock()->unlock();
6606 ConcurrentMarkSweepThread::desynchronize(true);
6607 _collector->stopTimer();
6608 if (PrintCMSStatistics != 0) {
6609 _collector->incrementYields();
6610 }
6611
6612 // See the comment in coordinator_yield()
6613 for (unsigned i = 0; i < CMSYieldSleepCount &&
6614 ConcurrentMarkSweepThread::should_yield() &&
6615 !CMSCollector::foregroundGCIsActive(); ++i) {
6616 os::sleep(Thread::current(), 1, false);
6617 }
6618
6619 ConcurrentMarkSweepThread::synchronize(true);
6620 _bitMap->lock()->lock_without_safepoint_check();
6621 _collector->startTimer();
6622 }
6623
6624 void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) {
6625 assert(_bitMap->isMarked(ptr), "expected bit to be set");
6626 assert(_markStack->isEmpty(),
6627 "should drain stack to limit stack usage");
6628 // convert ptr to an oop preparatory to scanning
6629 oop obj = oop(ptr);
6630 // Ignore mark word in verification below, since we
6631 // may be running concurrent with mutators.
6632 assert(obj->is_oop(true), "should be an oop");
6633 assert(_finger <= ptr, "_finger runneth ahead");
6634 // advance the finger to right end of this object
6635 _finger = ptr + obj->size();
6636 assert(_finger > ptr, "we just incremented it above");
6637 // On large heaps, it may take us some time to get through
6638 // the marking phase. During
6639 // this time it's possible that a lot of mutations have
6640 // accumulated in the card table and the mod union table --
6641 // these mutation records are redundant until we have
6642 // actually traced into the corresponding card.
6643 // Here, we check whether advancing the finger would make
6644 // us cross into a new card, and if so clear corresponding
6645 // cards in the MUT (preclean them in the card-table in the
6646 // future).
6647
6648 DEBUG_ONLY(if (!_verifying) {)
6649 // The clean-on-enter optimization is disabled by default,
6650 // until we fix 6178663.
6651 if (CMSCleanOnEnter && (_finger > _threshold)) {
6652 // [_threshold, _finger) represents the interval
6653 // of cards to be cleared in MUT (or precleaned in card table).
6654 // The set of cards to be cleared is all those that overlap
6655 // with the interval [_threshold, _finger); note that
6656 // _threshold is always kept card-aligned but _finger isn't
6657 // always card-aligned.
6658 HeapWord* old_threshold = _threshold;
6659 assert(old_threshold == (HeapWord*)round_to(
6660 (intptr_t)old_threshold, CardTableModRefBS::card_size),
6661 "_threshold should always be card-aligned");
6662 _threshold = (HeapWord*)round_to(
6663 (intptr_t)_finger, CardTableModRefBS::card_size);
6664 MemRegion mr(old_threshold, _threshold);
6665 assert(!mr.is_empty(), "Control point invariant");
6666 assert(_span.contains(mr), "Should clear within span");
6667 _mut->clear_range(mr);
6668 }
6669 DEBUG_ONLY(})
6670 // Note: the finger doesn't advance while we drain
6671 // the stack below.
6672 PushOrMarkClosure pushOrMarkClosure(_collector,
6673 _span, _bitMap, _markStack,
6674 _finger, this);
6675 bool res = _markStack->push(obj);
6676 assert(res, "Empty non-zero size stack should have space for single push");
6677 while (!_markStack->isEmpty()) {
6678 oop new_oop = _markStack->pop();
6679 // Skip verifying header mark word below because we are
6680 // running concurrent with mutators.
6681 assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
6682 // now scan this oop's oops
6683 new_oop->oop_iterate(&pushOrMarkClosure);
6684 do_yield_check();
6685 }
6686 assert(_markStack->isEmpty(), "tautology, emphasizing post-condition");
6687 }
6688
6689 Par_MarkFromRootsClosure::Par_MarkFromRootsClosure(CMSConcMarkingTask* task,
6690 CMSCollector* collector, MemRegion span,
6691 CMSBitMap* bit_map,
6692 OopTaskQueue* work_queue,
6693 CMSMarkStack* overflow_stack):
6694 _collector(collector),
6695 _whole_span(collector->_span),
6696 _span(span),
6697 _bit_map(bit_map),
6698 _mut(&collector->_modUnionTable),
6699 _work_queue(work_queue),
6700 _overflow_stack(overflow_stack),
6701 _skip_bits(0),
6702 _task(task)
6703 {
6704 assert(_work_queue->size() == 0, "work_queue should be empty");
6705 _finger = span.start();
6706 _threshold = _finger; // XXX Defer clear-on-enter optimization for now
6707 assert(_span.contains(_finger), "Out of bounds _finger?");
6708 }
6709
6710 // Should revisit to see if this should be restructured for
6711 // greater efficiency.
6712 bool Par_MarkFromRootsClosure::do_bit(size_t offset) {
6713 if (_skip_bits > 0) {
6714 _skip_bits--;
6715 return true;
6716 }
6717 // convert offset into a HeapWord*
6718 HeapWord* addr = _bit_map->startWord() + offset;
6719 assert(_bit_map->endWord() && addr < _bit_map->endWord(),
6720 "address out of range");
6721 assert(_bit_map->isMarked(addr), "tautology");
6722 if (_bit_map->isMarked(addr+1)) {
6723 // this is an allocated object that might not yet be initialized
6724 assert(_skip_bits == 0, "tautology");
6725 _skip_bits = 2; // skip next two marked bits ("Printezis-marks")
6726 oop p = oop(addr);
6727 if (p->klass_or_null() == NULL) {
6728 // in the case of Clean-on-Enter optimization, redirty card
6729 // and avoid clearing card by increasing the threshold.
6730 return true;
6731 }
6732 }
6733 scan_oops_in_oop(addr);
6734 return true;
6735 }
6736
6737 void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) {
6738 assert(_bit_map->isMarked(ptr), "expected bit to be set");
6739 // Should we assert that our work queue is empty or
6740 // below some drain limit?
6741 assert(_work_queue->size() == 0,
6742 "should drain stack to limit stack usage");
6743 // convert ptr to an oop preparatory to scanning
6744 oop obj = oop(ptr);
6745 // Ignore mark word in verification below, since we
6746 // may be running concurrent with mutators.
6747 assert(obj->is_oop(true), "should be an oop");
6748 assert(_finger <= ptr, "_finger runneth ahead");
6749 // advance the finger to right end of this object
6750 _finger = ptr + obj->size();
6751 assert(_finger > ptr, "we just incremented it above");
6752 // On large heaps, it may take us some time to get through
6753 // the marking phase. During
6754 // this time it's possible that a lot of mutations have
6755 // accumulated in the card table and the mod union table --
6756 // these mutation records are redundant until we have
6757 // actually traced into the corresponding card.
6758 // Here, we check whether advancing the finger would make
6759 // us cross into a new card, and if so clear corresponding
6760 // cards in the MUT (preclean them in the card-table in the
6761 // future).
6762
6763 // The clean-on-enter optimization is disabled by default,
6764 // until we fix 6178663.
6765 if (CMSCleanOnEnter && (_finger > _threshold)) {
6766 // [_threshold, _finger) represents the interval
6767 // of cards to be cleared in MUT (or precleaned in card table).
6768 // The set of cards to be cleared is all those that overlap
6769 // with the interval [_threshold, _finger); note that
6770 // _threshold is always kept card-aligned but _finger isn't
6771 // always card-aligned.
6772 HeapWord* old_threshold = _threshold;
6773 assert(old_threshold == (HeapWord*)round_to(
6774 (intptr_t)old_threshold, CardTableModRefBS::card_size),
6775 "_threshold should always be card-aligned");
6776 _threshold = (HeapWord*)round_to(
6777 (intptr_t)_finger, CardTableModRefBS::card_size);
6778 MemRegion mr(old_threshold, _threshold);
6779 assert(!mr.is_empty(), "Control point invariant");
6780 assert(_span.contains(mr), "Should clear within span"); // _whole_span ??
6781 _mut->clear_range(mr);
6782 }
6783
6784 // Note: the local finger doesn't advance while we drain
6785 // the stack below, but the global finger sure can and will.
6786 HeapWord** gfa = _task->global_finger_addr();
6787 Par_PushOrMarkClosure pushOrMarkClosure(_collector,
6788 _span, _bit_map,
6789 _work_queue,
6790 _overflow_stack,
6791 _finger,
6792 gfa, this);
6793 bool res = _work_queue->push(obj); // overflow could occur here
6794 assert(res, "Will hold once we use workqueues");
6795 while (true) {
6796 oop new_oop;
6797 if (!_work_queue->pop_local(new_oop)) {
6798 // We emptied our work_queue; check if there's stuff that can
6799 // be gotten from the overflow stack.
6800 if (CMSConcMarkingTask::get_work_from_overflow_stack(
6801 _overflow_stack, _work_queue)) {
6802 do_yield_check();
6803 continue;
6804 } else { // done
6805 break;
6806 }
6807 }
6808 // Skip verifying header mark word below because we are
6809 // running concurrent with mutators.
6810 assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
6811 // now scan this oop's oops
6812 new_oop->oop_iterate(&pushOrMarkClosure);
6813 do_yield_check();
6814 }
6815 assert(_work_queue->size() == 0, "tautology, emphasizing post-condition");
6816 }
6817
6818 // Yield in response to a request from VM Thread or
6819 // from mutators.
6820 void Par_MarkFromRootsClosure::do_yield_work() {
6821 assert(_task != NULL, "sanity");
6822 _task->yield();
6823 }
6824
6825 // A variant of the above used for verifying CMS marking work.
6826 MarkFromRootsVerifyClosure::MarkFromRootsVerifyClosure(CMSCollector* collector,
6827 MemRegion span,
6828 CMSBitMap* verification_bm, CMSBitMap* cms_bm,
6829 CMSMarkStack* mark_stack):
6830 _collector(collector),
6831 _span(span),
6832 _verification_bm(verification_bm),
6833 _cms_bm(cms_bm),
6834 _mark_stack(mark_stack),
6835 _pam_verify_closure(collector, span, verification_bm, cms_bm,
6836 mark_stack)
6837 {
6838 assert(_mark_stack->isEmpty(), "stack should be empty");
6839 _finger = _verification_bm->startWord();
6840 assert(_collector->_restart_addr == NULL, "Sanity check");
6841 assert(_span.contains(_finger), "Out of bounds _finger?");
6842 }
6843
6844 void MarkFromRootsVerifyClosure::reset(HeapWord* addr) {
6845 assert(_mark_stack->isEmpty(), "would cause duplicates on stack");
6846 assert(_span.contains(addr), "Out of bounds _finger?");
6847 _finger = addr;
6848 }
6849
6850 // Should revisit to see if this should be restructured for
6851 // greater efficiency.
6852 bool MarkFromRootsVerifyClosure::do_bit(size_t offset) {
6853 // convert offset into a HeapWord*
6854 HeapWord* addr = _verification_bm->startWord() + offset;
6855 assert(_verification_bm->endWord() && addr < _verification_bm->endWord(),
6856 "address out of range");
6857 assert(_verification_bm->isMarked(addr), "tautology");
6858 assert(_cms_bm->isMarked(addr), "tautology");
6859
6860 assert(_mark_stack->isEmpty(),
6861 "should drain stack to limit stack usage");
6862 // convert addr to an oop preparatory to scanning
6863 oop obj = oop(addr);
6864 assert(obj->is_oop(), "should be an oop");
6865 assert(_finger <= addr, "_finger runneth ahead");
6866 // advance the finger to right end of this object
6867 _finger = addr + obj->size();
6868 assert(_finger > addr, "we just incremented it above");
6869 // Note: the finger doesn't advance while we drain
6870 // the stack below.
6871 bool res = _mark_stack->push(obj);
6872 assert(res, "Empty non-zero size stack should have space for single push");
6873 while (!_mark_stack->isEmpty()) {
6874 oop new_oop = _mark_stack->pop();
6875 assert(new_oop->is_oop(), "Oops! expected to pop an oop");
6876 // now scan this oop's oops
6877 new_oop->oop_iterate(&_pam_verify_closure);
6878 }
6879 assert(_mark_stack->isEmpty(), "tautology, emphasizing post-condition");
6880 return true;
6881 }
6882
6883 PushAndMarkVerifyClosure::PushAndMarkVerifyClosure(
6884 CMSCollector* collector, MemRegion span,
6885 CMSBitMap* verification_bm, CMSBitMap* cms_bm,
6886 CMSMarkStack* mark_stack):
6887 MetadataAwareOopClosure(collector->ref_processor()),
6888 _collector(collector),
6889 _span(span),
6890 _verification_bm(verification_bm),
6891 _cms_bm(cms_bm),
6892 _mark_stack(mark_stack)
6893 { }
6894
6895 void PushAndMarkVerifyClosure::do_oop(oop* p) { PushAndMarkVerifyClosure::do_oop_work(p); }
6896 void PushAndMarkVerifyClosure::do_oop(narrowOop* p) { PushAndMarkVerifyClosure::do_oop_work(p); }
6897
6898 // Upon stack overflow, we discard (part of) the stack,
6899 // remembering the least address amongst those discarded
6900 // in CMSCollector's _restart_address.
6901 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
6902 // Remember the least grey address discarded
6903 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
6904 _collector->lower_restart_addr(ra);
6905 _mark_stack->reset(); // discard stack contents
6906 _mark_stack->expand(); // expand the stack if possible
6907 }
6908
6909 void PushAndMarkVerifyClosure::do_oop(oop obj) {
6910 assert(obj->is_oop_or_null(), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj)));
6911 HeapWord* addr = (HeapWord*)obj;
6912 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
6913 // Oop lies in _span and isn't yet grey or black
6914 _verification_bm->mark(addr); // now grey
6915 if (!_cms_bm->isMarked(addr)) {
6916 oop(addr)->print();
6917 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)",
6918 p2i(addr));
6919 fatal("... aborting");
6920 }
6921
6922 if (!_mark_stack->push(obj)) { // stack overflow
6923 if (PrintCMSStatistics != 0) {
6924 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
6925 SIZE_FORMAT, _mark_stack->capacity());
6926 }
6927 assert(_mark_stack->isFull(), "Else push should have succeeded");
6928 handle_stack_overflow(addr);
6929 }
6930 // anything including and to the right of _finger
6931 // will be scanned as we iterate over the remainder of the
6932 // bit map
6933 }
6934 }
6935
6936 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
6937 MemRegion span,
6938 CMSBitMap* bitMap, CMSMarkStack* markStack,
6939 HeapWord* finger, MarkFromRootsClosure* parent) :
6940 MetadataAwareOopClosure(collector->ref_processor()),
6941 _collector(collector),
6942 _span(span),
6943 _bitMap(bitMap),
6944 _markStack(markStack),
6945 _finger(finger),
6946 _parent(parent)
6947 { }
6948
6949 Par_PushOrMarkClosure::Par_PushOrMarkClosure(CMSCollector* collector,
6950 MemRegion span,
6951 CMSBitMap* bit_map,
6952 OopTaskQueue* work_queue,
6953 CMSMarkStack* overflow_stack,
6954 HeapWord* finger,
6955 HeapWord** global_finger_addr,
6956 Par_MarkFromRootsClosure* parent) :
6957 MetadataAwareOopClosure(collector->ref_processor()),
6958 _collector(collector),
6959 _whole_span(collector->_span),
6960 _span(span),
6961 _bit_map(bit_map),
6962 _work_queue(work_queue),
6963 _overflow_stack(overflow_stack),
6964 _finger(finger),
6965 _global_finger_addr(global_finger_addr),
6966 _parent(parent)
6967 { }
6968
6969 // Assumes thread-safe access by callers, who are
6970 // responsible for mutual exclusion.
6971 void CMSCollector::lower_restart_addr(HeapWord* low) {
6972 assert(_span.contains(low), "Out of bounds addr");
6973 if (_restart_addr == NULL) {
6974 _restart_addr = low;
6975 } else {
6976 _restart_addr = MIN2(_restart_addr, low);
6977 }
6978 }
6979
6980 // Upon stack overflow, we discard (part of) the stack,
6981 // remembering the least address amongst those discarded
6982 // in CMSCollector's _restart_address.
6983 void PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
6984 // Remember the least grey address discarded
6985 HeapWord* ra = (HeapWord*)_markStack->least_value(lost);
6986 _collector->lower_restart_addr(ra);
6987 _markStack->reset(); // discard stack contents
6988 _markStack->expand(); // expand the stack if possible
6989 }
6990
6991 // Upon stack overflow, we discard (part of) the stack,
6992 // remembering the least address amongst those discarded
6993 // in CMSCollector's _restart_address.
6994 void Par_PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
6995 // We need to do this under a mutex to prevent other
6996 // workers from interfering with the work done below.
6997 MutexLockerEx ml(_overflow_stack->par_lock(),
6998 Mutex::_no_safepoint_check_flag);
6999 // Remember the least grey address discarded
7000 HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost);
7001 _collector->lower_restart_addr(ra);
7002 _overflow_stack->reset(); // discard stack contents
7003 _overflow_stack->expand(); // expand the stack if possible
7004 }
7005
7006 void PushOrMarkClosure::do_oop(oop obj) {
7007 // Ignore mark word because we are running concurrent with mutators.
7008 assert(obj->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj)));
7009 HeapWord* addr = (HeapWord*)obj;
7010 if (_span.contains(addr) && !_bitMap->isMarked(addr)) {
7011 // Oop lies in _span and isn't yet grey or black
7012 _bitMap->mark(addr); // now grey
7013 if (addr < _finger) {
7014 // the bit map iteration has already either passed, or
7015 // sampled, this bit in the bit map; we'll need to
7016 // use the marking stack to scan this oop's oops.
7017 bool simulate_overflow = false;
7018 NOT_PRODUCT(
7019 if (CMSMarkStackOverflowALot &&
7020 _collector->simulate_overflow()) {
7021 // simulate a stack overflow
7022 simulate_overflow = true;
7023 }
7024 )
7025 if (simulate_overflow || !_markStack->push(obj)) { // stack overflow
7026 if (PrintCMSStatistics != 0) {
7027 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
7028 SIZE_FORMAT, _markStack->capacity());
7029 }
7030 assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded");
7031 handle_stack_overflow(addr);
7032 }
7033 }
7034 // anything including and to the right of _finger
7035 // will be scanned as we iterate over the remainder of the
7036 // bit map
7037 do_yield_check();
7038 }
7039 }
7040
7041 void PushOrMarkClosure::do_oop(oop* p) { PushOrMarkClosure::do_oop_work(p); }
7042 void PushOrMarkClosure::do_oop(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); }
7043
7044 void Par_PushOrMarkClosure::do_oop(oop obj) {
7045 // Ignore mark word because we are running concurrent with mutators.
7046 assert(obj->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj)));
7047 HeapWord* addr = (HeapWord*)obj;
7048 if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) {
7049 // Oop lies in _span and isn't yet grey or black
7050 // We read the global_finger (volatile read) strictly after marking oop
7051 bool res = _bit_map->par_mark(addr); // now grey
7052 volatile HeapWord** gfa = (volatile HeapWord**)_global_finger_addr;
7053 // Should we push this marked oop on our stack?
7054 // -- if someone else marked it, nothing to do
7055 // -- if target oop is above global finger nothing to do
7056 // -- if target oop is in chunk and above local finger
7057 // then nothing to do
7058 // -- else push on work queue
7059 if ( !res // someone else marked it, they will deal with it
7060 || (addr >= *gfa) // will be scanned in a later task
7061 || (_span.contains(addr) && addr >= _finger)) { // later in this chunk
7062 return;
7063 }
7064 // the bit map iteration has already either passed, or
7065 // sampled, this bit in the bit map; we'll need to
7066 // use the marking stack to scan this oop's oops.
7067 bool simulate_overflow = false;
7068 NOT_PRODUCT(
7069 if (CMSMarkStackOverflowALot &&
7070 _collector->simulate_overflow()) {
7071 // simulate a stack overflow
7072 simulate_overflow = true;
7073 }
7074 )
7075 if (simulate_overflow ||
7076 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
7077 // stack overflow
7078 if (PrintCMSStatistics != 0) {
7079 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
7080 SIZE_FORMAT, _overflow_stack->capacity());
7081 }
7082 // We cannot assert that the overflow stack is full because
7083 // it may have been emptied since.
7084 assert(simulate_overflow ||
7085 _work_queue->size() == _work_queue->max_elems(),
7086 "Else push should have succeeded");
7087 handle_stack_overflow(addr);
7088 }
7089 do_yield_check();
7090 }
7091 }
7092
7093 void Par_PushOrMarkClosure::do_oop(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
7094 void Par_PushOrMarkClosure::do_oop(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
7095
7096 PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector,
7097 MemRegion span,
7098 ReferenceProcessor* rp,
7099 CMSBitMap* bit_map,
7100 CMSBitMap* mod_union_table,
7101 CMSMarkStack* mark_stack,
7102 bool concurrent_precleaning):
7103 MetadataAwareOopClosure(rp),
7104 _collector(collector),
7105 _span(span),
7106 _bit_map(bit_map),
7107 _mod_union_table(mod_union_table),
7108 _mark_stack(mark_stack),
7109 _concurrent_precleaning(concurrent_precleaning)
7110 {
7111 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
7112 }
7113
7114 // Grey object rescan during pre-cleaning and second checkpoint phases --
7115 // the non-parallel version (the parallel version appears further below.)
7116 void PushAndMarkClosure::do_oop(oop obj) {
7117 // Ignore mark word verification. If during concurrent precleaning,
7118 // the object monitor may be locked. If during the checkpoint
7119 // phases, the object may already have been reached by a different
7120 // path and may be at the end of the global overflow list (so
7121 // the mark word may be NULL).
7122 assert(obj->is_oop_or_null(true /* ignore mark word */),
7123 err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj)));
7124 HeapWord* addr = (HeapWord*)obj;
7125 // Check if oop points into the CMS generation
7126 // and is not marked
7127 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
7128 // a white object ...
7129 _bit_map->mark(addr); // ... now grey
7130 // push on the marking stack (grey set)
7131 bool simulate_overflow = false;
7132 NOT_PRODUCT(
7133 if (CMSMarkStackOverflowALot &&
7134 _collector->simulate_overflow()) {
7135 // simulate a stack overflow
7136 simulate_overflow = true;
7137 }
7138 )
7139 if (simulate_overflow || !_mark_stack->push(obj)) {
7140 if (_concurrent_precleaning) {
7141 // During precleaning we can just dirty the appropriate card(s)
7142 // in the mod union table, thus ensuring that the object remains
7143 // in the grey set and continue. In the case of object arrays
7144 // we need to dirty all of the cards that the object spans,
7145 // since the rescan of object arrays will be limited to the
7146 // dirty cards.
7147 // Note that no one can be interfering with us in this action
7148 // of dirtying the mod union table, so no locking or atomics
7149 // are required.
7150 if (obj->is_objArray()) {
7151 size_t sz = obj->size();
7152 HeapWord* end_card_addr = (HeapWord*)round_to(
7153 (intptr_t)(addr+sz), CardTableModRefBS::card_size);
7154 MemRegion redirty_range = MemRegion(addr, end_card_addr);
7155 assert(!redirty_range.is_empty(), "Arithmetical tautology");
7156 _mod_union_table->mark_range(redirty_range);
7157 } else {
7158 _mod_union_table->mark(addr);
7159 }
7160 _collector->_ser_pmc_preclean_ovflw++;
7161 } else {
7162 // During the remark phase, we need to remember this oop
7163 // in the overflow list.
7164 _collector->push_on_overflow_list(obj);
7165 _collector->_ser_pmc_remark_ovflw++;
7166 }
7167 }
7168 }
7169 }
7170
7171 Par_PushAndMarkClosure::Par_PushAndMarkClosure(CMSCollector* collector,
7172 MemRegion span,
7173 ReferenceProcessor* rp,
7174 CMSBitMap* bit_map,
7175 OopTaskQueue* work_queue):
7176 MetadataAwareOopClosure(rp),
7177 _collector(collector),
7178 _span(span),
7179 _bit_map(bit_map),
7180 _work_queue(work_queue)
7181 {
7182 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
7183 }
7184
7185 void PushAndMarkClosure::do_oop(oop* p) { PushAndMarkClosure::do_oop_work(p); }
7186 void PushAndMarkClosure::do_oop(narrowOop* p) { PushAndMarkClosure::do_oop_work(p); }
7187
7188 // Grey object rescan during second checkpoint phase --
7189 // the parallel version.
7190 void Par_PushAndMarkClosure::do_oop(oop obj) {
7191 // In the assert below, we ignore the mark word because
7192 // this oop may point to an already visited object that is
7193 // on the overflow stack (in which case the mark word has
7194 // been hijacked for chaining into the overflow stack --
7195 // if this is the last object in the overflow stack then
7196 // its mark word will be NULL). Because this object may
7197 // have been subsequently popped off the global overflow
7198 // stack, and the mark word possibly restored to the prototypical
7199 // value, by the time we get to examined this failing assert in
7200 // the debugger, is_oop_or_null(false) may subsequently start
7201 // to hold.
7202 assert(obj->is_oop_or_null(true),
7203 err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj)));
7204 HeapWord* addr = (HeapWord*)obj;
7205 // Check if oop points into the CMS generation
7206 // and is not marked
7207 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
7208 // a white object ...
7209 // If we manage to "claim" the object, by being the
7210 // first thread to mark it, then we push it on our
7211 // marking stack
7212 if (_bit_map->par_mark(addr)) { // ... now grey
7213 // push on work queue (grey set)
7214 bool simulate_overflow = false;
7215 NOT_PRODUCT(
7216 if (CMSMarkStackOverflowALot &&
7217 _collector->par_simulate_overflow()) {
7218 // simulate a stack overflow
7219 simulate_overflow = true;
7220 }
7221 )
7222 if (simulate_overflow || !_work_queue->push(obj)) {
7223 _collector->par_push_on_overflow_list(obj);
7224 _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS
7225 }
7226 } // Else, some other thread got there first
7227 }
7228 }
7229
7230 void Par_PushAndMarkClosure::do_oop(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
7231 void Par_PushAndMarkClosure::do_oop(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
7232
7233 void CMSPrecleanRefsYieldClosure::do_yield_work() {
7234 Mutex* bml = _collector->bitMapLock();
7235 assert_lock_strong(bml);
7236 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7237 "CMS thread should hold CMS token");
7238
7239 bml->unlock();
7240 ConcurrentMarkSweepThread::desynchronize(true);
7241
7242 _collector->stopTimer();
7243 if (PrintCMSStatistics != 0) {
7244 _collector->incrementYields();
7245 }
7246
7247 // See the comment in coordinator_yield()
7248 for (unsigned i = 0; i < CMSYieldSleepCount &&
7249 ConcurrentMarkSweepThread::should_yield() &&
7250 !CMSCollector::foregroundGCIsActive(); ++i) {
7251 os::sleep(Thread::current(), 1, false);
7252 }
7253
7254 ConcurrentMarkSweepThread::synchronize(true);
7255 bml->lock();
7256
7257 _collector->startTimer();
7258 }
7259
7260 bool CMSPrecleanRefsYieldClosure::should_return() {
7261 if (ConcurrentMarkSweepThread::should_yield()) {
7262 do_yield_work();
7263 }
7264 return _collector->foregroundGCIsActive();
7265 }
7266
7267 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
7268 assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
7269 "mr should be aligned to start at a card boundary");
7270 // We'd like to assert:
7271 // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
7272 // "mr should be a range of cards");
7273 // However, that would be too strong in one case -- the last
7274 // partition ends at _unallocated_block which, in general, can be
7275 // an arbitrary boundary, not necessarily card aligned.
7276 if (PrintCMSStatistics != 0) {
7277 _num_dirty_cards +=
7278 mr.word_size()/CardTableModRefBS::card_size_in_words;
7279 }
7280 _space->object_iterate_mem(mr, &_scan_cl);
7281 }
7282
7283 SweepClosure::SweepClosure(CMSCollector* collector,
7284 ConcurrentMarkSweepGeneration* g,
7285 CMSBitMap* bitMap, bool should_yield) :
7286 _collector(collector),
7287 _g(g),
7288 _sp(g->cmsSpace()),
7289 _limit(_sp->sweep_limit()),
7290 _freelistLock(_sp->freelistLock()),
7291 _bitMap(bitMap),
7292 _yield(should_yield),
7293 _inFreeRange(false), // No free range at beginning of sweep
7294 _freeRangeInFreeLists(false), // No free range at beginning of sweep
7295 _lastFreeRangeCoalesced(false),
7296 _freeFinger(g->used_region().start())
7297 {
7298 NOT_PRODUCT(
7299 _numObjectsFreed = 0;
7300 _numWordsFreed = 0;
7301 _numObjectsLive = 0;
7302 _numWordsLive = 0;
7303 _numObjectsAlreadyFree = 0;
7304 _numWordsAlreadyFree = 0;
7305 _last_fc = NULL;
7306
7307 _sp->initializeIndexedFreeListArrayReturnedBytes();
7308 _sp->dictionary()->initialize_dict_returned_bytes();
7309 )
7310 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7311 "sweep _limit out of bounds");
7312 if (CMSTraceSweeper) {
7313 gclog_or_tty->print_cr("\n====================\nStarting new sweep with limit " PTR_FORMAT,
7314 p2i(_limit));
7315 }
7316 }
7317
7318 void SweepClosure::print_on(outputStream* st) const {
7319 tty->print_cr("_sp = [" PTR_FORMAT "," PTR_FORMAT ")",
7320 p2i(_sp->bottom()), p2i(_sp->end()));
7321 tty->print_cr("_limit = " PTR_FORMAT, p2i(_limit));
7322 tty->print_cr("_freeFinger = " PTR_FORMAT, p2i(_freeFinger));
7323 NOT_PRODUCT(tty->print_cr("_last_fc = " PTR_FORMAT, p2i(_last_fc));)
7324 tty->print_cr("_inFreeRange = %d, _freeRangeInFreeLists = %d, _lastFreeRangeCoalesced = %d",
7325 _inFreeRange, _freeRangeInFreeLists, _lastFreeRangeCoalesced);
7326 }
7327
7328 #ifndef PRODUCT
7329 // Assertion checking only: no useful work in product mode --
7330 // however, if any of the flags below become product flags,
7331 // you may need to review this code to see if it needs to be
7332 // enabled in product mode.
7333 SweepClosure::~SweepClosure() {
7334 assert_lock_strong(_freelistLock);
7335 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7336 "sweep _limit out of bounds");
7337 if (inFreeRange()) {
7338 warning("inFreeRange() should have been reset; dumping state of SweepClosure");
7339 print();
7340 ShouldNotReachHere();
7341 }
7342 if (Verbose && PrintGC) {
7343 gclog_or_tty->print("Collected "SIZE_FORMAT" objects, " SIZE_FORMAT " bytes",
7344 _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
7345 gclog_or_tty->print_cr("\nLive "SIZE_FORMAT" objects, "
7346 SIZE_FORMAT" bytes "
7347 "Already free "SIZE_FORMAT" objects, "SIZE_FORMAT" bytes",
7348 _numObjectsLive, _numWordsLive*sizeof(HeapWord),
7349 _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
7350 size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree)
7351 * sizeof(HeapWord);
7352 gclog_or_tty->print_cr("Total sweep: "SIZE_FORMAT" bytes", totalBytes);
7353
7354 if (PrintCMSStatistics && CMSVerifyReturnedBytes) {
7355 size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
7356 size_t dict_returned_bytes = _sp->dictionary()->sum_dict_returned_bytes();
7357 size_t returned_bytes = indexListReturnedBytes + dict_returned_bytes;
7358 gclog_or_tty->print("Returned "SIZE_FORMAT" bytes", returned_bytes);
7359 gclog_or_tty->print(" Indexed List Returned "SIZE_FORMAT" bytes",
7360 indexListReturnedBytes);
7361 gclog_or_tty->print_cr(" Dictionary Returned "SIZE_FORMAT" bytes",
7362 dict_returned_bytes);
7363 }
7364 }
7365 if (CMSTraceSweeper) {
7366 gclog_or_tty->print_cr("end of sweep with _limit = " PTR_FORMAT "\n================",
7367 p2i(_limit));
7368 }
7369 }
7370 #endif // PRODUCT
7371
7372 void SweepClosure::initialize_free_range(HeapWord* freeFinger,
7373 bool freeRangeInFreeLists) {
7374 if (CMSTraceSweeper) {
7375 gclog_or_tty->print("---- Start free range at " PTR_FORMAT " with free block (%d)\n",
7376 p2i(freeFinger), freeRangeInFreeLists);
7377 }
7378 assert(!inFreeRange(), "Trampling existing free range");
7379 set_inFreeRange(true);
7380 set_lastFreeRangeCoalesced(false);
7381
7382 set_freeFinger(freeFinger);
7383 set_freeRangeInFreeLists(freeRangeInFreeLists);
7384 if (CMSTestInFreeList) {
7385 if (freeRangeInFreeLists) {
7386 FreeChunk* fc = (FreeChunk*) freeFinger;
7387 assert(fc->is_free(), "A chunk on the free list should be free.");
7388 assert(fc->size() > 0, "Free range should have a size");
7389 assert(_sp->verify_chunk_in_free_list(fc), "Chunk is not in free lists");
7390 }
7391 }
7392 }
7393
7394 // Note that the sweeper runs concurrently with mutators. Thus,
7395 // it is possible for direct allocation in this generation to happen
7396 // in the middle of the sweep. Note that the sweeper also coalesces
7397 // contiguous free blocks. Thus, unless the sweeper and the allocator
7398 // synchronize appropriately freshly allocated blocks may get swept up.
7399 // This is accomplished by the sweeper locking the free lists while
7400 // it is sweeping. Thus blocks that are determined to be free are
7401 // indeed free. There is however one additional complication:
7402 // blocks that have been allocated since the final checkpoint and
7403 // mark, will not have been marked and so would be treated as
7404 // unreachable and swept up. To prevent this, the allocator marks
7405 // the bit map when allocating during the sweep phase. This leads,
7406 // however, to a further complication -- objects may have been allocated
7407 // but not yet initialized -- in the sense that the header isn't yet
7408 // installed. The sweeper can not then determine the size of the block
7409 // in order to skip over it. To deal with this case, we use a technique
7410 // (due to Printezis) to encode such uninitialized block sizes in the
7411 // bit map. Since the bit map uses a bit per every HeapWord, but the
7412 // CMS generation has a minimum object size of 3 HeapWords, it follows
7413 // that "normal marks" won't be adjacent in the bit map (there will
7414 // always be at least two 0 bits between successive 1 bits). We make use
7415 // of these "unused" bits to represent uninitialized blocks -- the bit
7416 // corresponding to the start of the uninitialized object and the next
7417 // bit are both set. Finally, a 1 bit marks the end of the object that
7418 // started with the two consecutive 1 bits to indicate its potentially
7419 // uninitialized state.
7420
7421 size_t SweepClosure::do_blk_careful(HeapWord* addr) {
7422 FreeChunk* fc = (FreeChunk*)addr;
7423 size_t res;
7424
7425 // Check if we are done sweeping. Below we check "addr >= _limit" rather
7426 // than "addr == _limit" because although _limit was a block boundary when
7427 // we started the sweep, it may no longer be one because heap expansion
7428 // may have caused us to coalesce the block ending at the address _limit
7429 // with a newly expanded chunk (this happens when _limit was set to the
7430 // previous _end of the space), so we may have stepped past _limit:
7431 // see the following Zeno-like trail of CRs 6977970, 7008136, 7042740.
7432 if (addr >= _limit) { // we have swept up to or past the limit: finish up
7433 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7434 "sweep _limit out of bounds");
7435 assert(addr < _sp->end(), "addr out of bounds");
7436 // Flush any free range we might be holding as a single
7437 // coalesced chunk to the appropriate free list.
7438 if (inFreeRange()) {
7439 assert(freeFinger() >= _sp->bottom() && freeFinger() < _limit,
7440 err_msg("freeFinger() " PTR_FORMAT" is out-of-bounds", p2i(freeFinger())));
7441 flush_cur_free_chunk(freeFinger(),
7442 pointer_delta(addr, freeFinger()));
7443 if (CMSTraceSweeper) {
7444 gclog_or_tty->print("Sweep: last chunk: ");
7445 gclog_or_tty->print("put_free_blk " PTR_FORMAT " ("SIZE_FORMAT") "
7446 "[coalesced:%d]\n",
7447 p2i(freeFinger()), pointer_delta(addr, freeFinger()),
7448 lastFreeRangeCoalesced() ? 1 : 0);
7449 }
7450 }
7451
7452 // help the iterator loop finish
7453 return pointer_delta(_sp->end(), addr);
7454 }
7455
7456 assert(addr < _limit, "sweep invariant");
7457 // check if we should yield
7458 do_yield_check(addr);
7459 if (fc->is_free()) {
7460 // Chunk that is already free
7461 res = fc->size();
7462 do_already_free_chunk(fc);
7463 debug_only(_sp->verifyFreeLists());
7464 // If we flush the chunk at hand in lookahead_and_flush()
7465 // and it's coalesced with a preceding chunk, then the
7466 // process of "mangling" the payload of the coalesced block
7467 // will cause erasure of the size information from the
7468 // (erstwhile) header of all the coalesced blocks but the
7469 // first, so the first disjunct in the assert will not hold
7470 // in that specific case (in which case the second disjunct
7471 // will hold).
7472 assert(res == fc->size() || ((HeapWord*)fc) + res >= _limit,
7473 "Otherwise the size info doesn't change at this step");
7474 NOT_PRODUCT(
7475 _numObjectsAlreadyFree++;
7476 _numWordsAlreadyFree += res;
7477 )
7478 NOT_PRODUCT(_last_fc = fc;)
7479 } else if (!_bitMap->isMarked(addr)) {
7480 // Chunk is fresh garbage
7481 res = do_garbage_chunk(fc);
7482 debug_only(_sp->verifyFreeLists());
7483 NOT_PRODUCT(
7484 _numObjectsFreed++;
7485 _numWordsFreed += res;
7486 )
7487 } else {
7488 // Chunk that is alive.
7489 res = do_live_chunk(fc);
7490 debug_only(_sp->verifyFreeLists());
7491 NOT_PRODUCT(
7492 _numObjectsLive++;
7493 _numWordsLive += res;
7494 )
7495 }
7496 return res;
7497 }
7498
7499 // For the smart allocation, record following
7500 // split deaths - a free chunk is removed from its free list because
7501 // it is being split into two or more chunks.
7502 // split birth - a free chunk is being added to its free list because
7503 // a larger free chunk has been split and resulted in this free chunk.
7504 // coal death - a free chunk is being removed from its free list because
7505 // it is being coalesced into a large free chunk.
7506 // coal birth - a free chunk is being added to its free list because
7507 // it was created when two or more free chunks where coalesced into
7508 // this free chunk.
7509 //
7510 // These statistics are used to determine the desired number of free
7511 // chunks of a given size. The desired number is chosen to be relative
7512 // to the end of a CMS sweep. The desired number at the end of a sweep
7513 // is the
7514 // count-at-end-of-previous-sweep (an amount that was enough)
7515 // - count-at-beginning-of-current-sweep (the excess)
7516 // + split-births (gains in this size during interval)
7517 // - split-deaths (demands on this size during interval)
7518 // where the interval is from the end of one sweep to the end of the
7519 // next.
7520 //
7521 // When sweeping the sweeper maintains an accumulated chunk which is
7522 // the chunk that is made up of chunks that have been coalesced. That
7523 // will be termed the left-hand chunk. A new chunk of garbage that
7524 // is being considered for coalescing will be referred to as the
7525 // right-hand chunk.
7526 //
7527 // When making a decision on whether to coalesce a right-hand chunk with
7528 // the current left-hand chunk, the current count vs. the desired count
7529 // of the left-hand chunk is considered. Also if the right-hand chunk
7530 // is near the large chunk at the end of the heap (see
7531 // ConcurrentMarkSweepGeneration::isNearLargestChunk()), then the
7532 // left-hand chunk is coalesced.
7533 //
7534 // When making a decision about whether to split a chunk, the desired count
7535 // vs. the current count of the candidate to be split is also considered.
7536 // If the candidate is underpopulated (currently fewer chunks than desired)
7537 // a chunk of an overpopulated (currently more chunks than desired) size may
7538 // be chosen. The "hint" associated with a free list, if non-null, points
7539 // to a free list which may be overpopulated.
7540 //
7541
7542 void SweepClosure::do_already_free_chunk(FreeChunk* fc) {
7543 const size_t size = fc->size();
7544 // Chunks that cannot be coalesced are not in the
7545 // free lists.
7546 if (CMSTestInFreeList && !fc->cantCoalesce()) {
7547 assert(_sp->verify_chunk_in_free_list(fc),
7548 "free chunk should be in free lists");
7549 }
7550 // a chunk that is already free, should not have been
7551 // marked in the bit map
7552 HeapWord* const addr = (HeapWord*) fc;
7553 assert(!_bitMap->isMarked(addr), "free chunk should be unmarked");
7554 // Verify that the bit map has no bits marked between
7555 // addr and purported end of this block.
7556 _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
7557
7558 // Some chunks cannot be coalesced under any circumstances.
7559 // See the definition of cantCoalesce().
7560 if (!fc->cantCoalesce()) {
7561 // This chunk can potentially be coalesced.
7562 if (_sp->adaptive_freelists()) {
7563 // All the work is done in
7564 do_post_free_or_garbage_chunk(fc, size);
7565 } else { // Not adaptive free lists
7566 // this is a free chunk that can potentially be coalesced by the sweeper;
7567 if (!inFreeRange()) {
7568 // if the next chunk is a free block that can't be coalesced
7569 // it doesn't make sense to remove this chunk from the free lists
7570 FreeChunk* nextChunk = (FreeChunk*)(addr + size);
7571 assert((HeapWord*)nextChunk <= _sp->end(), "Chunk size out of bounds?");
7572 if ((HeapWord*)nextChunk < _sp->end() && // There is another free chunk to the right ...
7573 nextChunk->is_free() && // ... which is free...
7574 nextChunk->cantCoalesce()) { // ... but can't be coalesced
7575 // nothing to do
7576 } else {
7577 // Potentially the start of a new free range:
7578 // Don't eagerly remove it from the free lists.
7579 // No need to remove it if it will just be put
7580 // back again. (Also from a pragmatic point of view
7581 // if it is a free block in a region that is beyond
7582 // any allocated blocks, an assertion will fail)
7583 // Remember the start of a free run.
7584 initialize_free_range(addr, true);
7585 // end - can coalesce with next chunk
7586 }
7587 } else {
7588 // the midst of a free range, we are coalescing
7589 print_free_block_coalesced(fc);
7590 if (CMSTraceSweeper) {
7591 gclog_or_tty->print(" -- pick up free block " PTR_FORMAT " (" SIZE_FORMAT ")\n", p2i(fc), size);
7592 }
7593 // remove it from the free lists
7594 _sp->removeFreeChunkFromFreeLists(fc);
7595 set_lastFreeRangeCoalesced(true);
7596 // If the chunk is being coalesced and the current free range is
7597 // in the free lists, remove the current free range so that it
7598 // will be returned to the free lists in its entirety - all
7599 // the coalesced pieces included.
7600 if (freeRangeInFreeLists()) {
7601 FreeChunk* ffc = (FreeChunk*) freeFinger();
7602 assert(ffc->size() == pointer_delta(addr, freeFinger()),
7603 "Size of free range is inconsistent with chunk size.");
7604 if (CMSTestInFreeList) {
7605 assert(_sp->verify_chunk_in_free_list(ffc),
7606 "free range is not in free lists");
7607 }
7608 _sp->removeFreeChunkFromFreeLists(ffc);
7609 set_freeRangeInFreeLists(false);
7610 }
7611 }
7612 }
7613 // Note that if the chunk is not coalescable (the else arm
7614 // below), we unconditionally flush, without needing to do
7615 // a "lookahead," as we do below.
7616 if (inFreeRange()) lookahead_and_flush(fc, size);
7617 } else {
7618 // Code path common to both original and adaptive free lists.
7619
7620 // cant coalesce with previous block; this should be treated
7621 // as the end of a free run if any
7622 if (inFreeRange()) {
7623 // we kicked some butt; time to pick up the garbage
7624 assert(freeFinger() < addr, "freeFinger points too high");
7625 flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger()));
7626 }
7627 // else, nothing to do, just continue
7628 }
7629 }
7630
7631 size_t SweepClosure::do_garbage_chunk(FreeChunk* fc) {
7632 // This is a chunk of garbage. It is not in any free list.
7633 // Add it to a free list or let it possibly be coalesced into
7634 // a larger chunk.
7635 HeapWord* const addr = (HeapWord*) fc;
7636 const size_t size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
7637
7638 if (_sp->adaptive_freelists()) {
7639 // Verify that the bit map has no bits marked between
7640 // addr and purported end of just dead object.
7641 _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
7642
7643 do_post_free_or_garbage_chunk(fc, size);
7644 } else {
7645 if (!inFreeRange()) {
7646 // start of a new free range
7647 assert(size > 0, "A free range should have a size");
7648 initialize_free_range(addr, false);
7649 } else {
7650 // this will be swept up when we hit the end of the
7651 // free range
7652 if (CMSTraceSweeper) {
7653 gclog_or_tty->print(" -- pick up garbage " PTR_FORMAT " (" SIZE_FORMAT ")\n", p2i(fc), size);
7654 }
7655 // If the chunk is being coalesced and the current free range is
7656 // in the free lists, remove the current free range so that it
7657 // will be returned to the free lists in its entirety - all
7658 // the coalesced pieces included.
7659 if (freeRangeInFreeLists()) {
7660 FreeChunk* ffc = (FreeChunk*)freeFinger();
7661 assert(ffc->size() == pointer_delta(addr, freeFinger()),
7662 "Size of free range is inconsistent with chunk size.");
7663 if (CMSTestInFreeList) {
7664 assert(_sp->verify_chunk_in_free_list(ffc),
7665 "free range is not in free lists");
7666 }
7667 _sp->removeFreeChunkFromFreeLists(ffc);
7668 set_freeRangeInFreeLists(false);
7669 }
7670 set_lastFreeRangeCoalesced(true);
7671 }
7672 // this will be swept up when we hit the end of the free range
7673
7674 // Verify that the bit map has no bits marked between
7675 // addr and purported end of just dead object.
7676 _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
7677 }
7678 assert(_limit >= addr + size,
7679 "A freshly garbage chunk can't possibly straddle over _limit");
7680 if (inFreeRange()) lookahead_and_flush(fc, size);
7681 return size;
7682 }
7683
7684 size_t SweepClosure::do_live_chunk(FreeChunk* fc) {
7685 HeapWord* addr = (HeapWord*) fc;
7686 // The sweeper has just found a live object. Return any accumulated
7687 // left hand chunk to the free lists.
7688 if (inFreeRange()) {
7689 assert(freeFinger() < addr, "freeFinger points too high");
7690 flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger()));
7691 }
7692
7693 // This object is live: we'd normally expect this to be
7694 // an oop, and like to assert the following:
7695 // assert(oop(addr)->is_oop(), "live block should be an oop");
7696 // However, as we commented above, this may be an object whose
7697 // header hasn't yet been initialized.
7698 size_t size;
7699 assert(_bitMap->isMarked(addr), "Tautology for this control point");
7700 if (_bitMap->isMarked(addr + 1)) {
7701 // Determine the size from the bit map, rather than trying to
7702 // compute it from the object header.
7703 HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
7704 size = pointer_delta(nextOneAddr + 1, addr);
7705 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
7706 "alignment problem");
7707
7708 #ifdef ASSERT
7709 if (oop(addr)->klass_or_null() != NULL) {
7710 // Ignore mark word because we are running concurrent with mutators
7711 assert(oop(addr)->is_oop(true), "live block should be an oop");
7712 assert(size ==
7713 CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()),
7714 "P-mark and computed size do not agree");
7715 }
7716 #endif
7717
7718 } else {
7719 // This should be an initialized object that's alive.
7720 assert(oop(addr)->klass_or_null() != NULL,
7721 "Should be an initialized object");
7722 // Ignore mark word because we are running concurrent with mutators
7723 assert(oop(addr)->is_oop(true), "live block should be an oop");
7724 // Verify that the bit map has no bits marked between
7725 // addr and purported end of this block.
7726 size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
7727 assert(size >= 3, "Necessary for Printezis marks to work");
7728 assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
7729 DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
7730 }
7731 return size;
7732 }
7733
7734 void SweepClosure::do_post_free_or_garbage_chunk(FreeChunk* fc,
7735 size_t chunkSize) {
7736 // do_post_free_or_garbage_chunk() should only be called in the case
7737 // of the adaptive free list allocator.
7738 const bool fcInFreeLists = fc->is_free();
7739 assert(_sp->adaptive_freelists(), "Should only be used in this case.");
7740 assert((HeapWord*)fc <= _limit, "sweep invariant");
7741 if (CMSTestInFreeList && fcInFreeLists) {
7742 assert(_sp->verify_chunk_in_free_list(fc), "free chunk is not in free lists");
7743 }
7744
7745 if (CMSTraceSweeper) {
7746 gclog_or_tty->print_cr(" -- pick up another chunk at " PTR_FORMAT " (" SIZE_FORMAT ")", p2i(fc), chunkSize);
7747 }
7748
7749 HeapWord* const fc_addr = (HeapWord*) fc;
7750
7751 bool coalesce;
7752 const size_t left = pointer_delta(fc_addr, freeFinger());
7753 const size_t right = chunkSize;
7754 switch (FLSCoalescePolicy) {
7755 // numeric value forms a coalition aggressiveness metric
7756 case 0: { // never coalesce
7757 coalesce = false;
7758 break;
7759 }
7760 case 1: { // coalesce if left & right chunks on overpopulated lists
7761 coalesce = _sp->coalOverPopulated(left) &&
7762 _sp->coalOverPopulated(right);
7763 break;
7764 }
7765 case 2: { // coalesce if left chunk on overpopulated list (default)
7766 coalesce = _sp->coalOverPopulated(left);
7767 break;
7768 }
7769 case 3: { // coalesce if left OR right chunk on overpopulated list
7770 coalesce = _sp->coalOverPopulated(left) ||
7771 _sp->coalOverPopulated(right);
7772 break;
7773 }
7774 case 4: { // always coalesce
7775 coalesce = true;
7776 break;
7777 }
7778 default:
7779 ShouldNotReachHere();
7780 }
7781
7782 // Should the current free range be coalesced?
7783 // If the chunk is in a free range and either we decided to coalesce above
7784 // or the chunk is near the large block at the end of the heap
7785 // (isNearLargestChunk() returns true), then coalesce this chunk.
7786 const bool doCoalesce = inFreeRange()
7787 && (coalesce || _g->isNearLargestChunk(fc_addr));
7788 if (doCoalesce) {
7789 // Coalesce the current free range on the left with the new
7790 // chunk on the right. If either is on a free list,
7791 // it must be removed from the list and stashed in the closure.
7792 if (freeRangeInFreeLists()) {
7793 FreeChunk* const ffc = (FreeChunk*)freeFinger();
7794 assert(ffc->size() == pointer_delta(fc_addr, freeFinger()),
7795 "Size of free range is inconsistent with chunk size.");
7796 if (CMSTestInFreeList) {
7797 assert(_sp->verify_chunk_in_free_list(ffc),
7798 "Chunk is not in free lists");
7799 }
7800 _sp->coalDeath(ffc->size());
7801 _sp->removeFreeChunkFromFreeLists(ffc);
7802 set_freeRangeInFreeLists(false);
7803 }
7804 if (fcInFreeLists) {
7805 _sp->coalDeath(chunkSize);
7806 assert(fc->size() == chunkSize,
7807 "The chunk has the wrong size or is not in the free lists");
7808 _sp->removeFreeChunkFromFreeLists(fc);
7809 }
7810 set_lastFreeRangeCoalesced(true);
7811 print_free_block_coalesced(fc);
7812 } else { // not in a free range and/or should not coalesce
7813 // Return the current free range and start a new one.
7814 if (inFreeRange()) {
7815 // In a free range but cannot coalesce with the right hand chunk.
7816 // Put the current free range into the free lists.
7817 flush_cur_free_chunk(freeFinger(),
7818 pointer_delta(fc_addr, freeFinger()));
7819 }
7820 // Set up for new free range. Pass along whether the right hand
7821 // chunk is in the free lists.
7822 initialize_free_range((HeapWord*)fc, fcInFreeLists);
7823 }
7824 }
7825
7826 // Lookahead flush:
7827 // If we are tracking a free range, and this is the last chunk that
7828 // we'll look at because its end crosses past _limit, we'll preemptively
7829 // flush it along with any free range we may be holding on to. Note that
7830 // this can be the case only for an already free or freshly garbage
7831 // chunk. If this block is an object, it can never straddle
7832 // over _limit. The "straddling" occurs when _limit is set at
7833 // the previous end of the space when this cycle started, and
7834 // a subsequent heap expansion caused the previously co-terminal
7835 // free block to be coalesced with the newly expanded portion,
7836 // thus rendering _limit a non-block-boundary making it dangerous
7837 // for the sweeper to step over and examine.
7838 void SweepClosure::lookahead_and_flush(FreeChunk* fc, size_t chunk_size) {
7839 assert(inFreeRange(), "Should only be called if currently in a free range.");
7840 HeapWord* const eob = ((HeapWord*)fc) + chunk_size;
7841 assert(_sp->used_region().contains(eob - 1),
7842 err_msg("eob = " PTR_FORMAT " eob-1 = " PTR_FORMAT " _limit = " PTR_FORMAT
7843 " out of bounds wrt _sp = [" PTR_FORMAT "," PTR_FORMAT ")"
7844 " when examining fc = " PTR_FORMAT "(" SIZE_FORMAT ")",
7845 p2i(eob), p2i(eob-1), p2i(_limit), p2i(_sp->bottom()), p2i(_sp->end()), p2i(fc), chunk_size));
7846 if (eob >= _limit) {
7847 assert(eob == _limit || fc->is_free(), "Only a free chunk should allow us to cross over the limit");
7848 if (CMSTraceSweeper) {
7849 gclog_or_tty->print_cr("_limit " PTR_FORMAT " reached or crossed by block "
7850 "[" PTR_FORMAT "," PTR_FORMAT ") in space "
7851 "[" PTR_FORMAT "," PTR_FORMAT ")",
7852 p2i(_limit), p2i(fc), p2i(eob), p2i(_sp->bottom()), p2i(_sp->end()));
7853 }
7854 // Return the storage we are tracking back into the free lists.
7855 if (CMSTraceSweeper) {
7856 gclog_or_tty->print_cr("Flushing ... ");
7857 }
7858 assert(freeFinger() < eob, "Error");
7859 flush_cur_free_chunk( freeFinger(), pointer_delta(eob, freeFinger()));
7860 }
7861 }
7862
7863 void SweepClosure::flush_cur_free_chunk(HeapWord* chunk, size_t size) {
7864 assert(inFreeRange(), "Should only be called if currently in a free range.");
7865 assert(size > 0,
7866 "A zero sized chunk cannot be added to the free lists.");
7867 if (!freeRangeInFreeLists()) {
7868 if (CMSTestInFreeList) {
7869 FreeChunk* fc = (FreeChunk*) chunk;
7870 fc->set_size(size);
7871 assert(!_sp->verify_chunk_in_free_list(fc),
7872 "chunk should not be in free lists yet");
7873 }
7874 if (CMSTraceSweeper) {
7875 gclog_or_tty->print_cr(" -- add free block " PTR_FORMAT " (" SIZE_FORMAT ") to free lists",
7876 p2i(chunk), size);
7877 }
7878 // A new free range is going to be starting. The current
7879 // free range has not been added to the free lists yet or
7880 // was removed so add it back.
7881 // If the current free range was coalesced, then the death
7882 // of the free range was recorded. Record a birth now.
7883 if (lastFreeRangeCoalesced()) {
7884 _sp->coalBirth(size);
7885 }
7886 _sp->addChunkAndRepairOffsetTable(chunk, size,
7887 lastFreeRangeCoalesced());
7888 } else if (CMSTraceSweeper) {
7889 gclog_or_tty->print_cr("Already in free list: nothing to flush");
7890 }
7891 set_inFreeRange(false);
7892 set_freeRangeInFreeLists(false);
7893 }
7894
7895 // We take a break if we've been at this for a while,
7896 // so as to avoid monopolizing the locks involved.
7897 void SweepClosure::do_yield_work(HeapWord* addr) {
7898 // Return current free chunk being used for coalescing (if any)
7899 // to the appropriate freelist. After yielding, the next
7900 // free block encountered will start a coalescing range of
7901 // free blocks. If the next free block is adjacent to the
7902 // chunk just flushed, they will need to wait for the next
7903 // sweep to be coalesced.
7904 if (inFreeRange()) {
7905 flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger()));
7906 }
7907
7908 // First give up the locks, then yield, then re-lock.
7909 // We should probably use a constructor/destructor idiom to
7910 // do this unlock/lock or modify the MutexUnlocker class to
7911 // serve our purpose. XXX
7912 assert_lock_strong(_bitMap->lock());
7913 assert_lock_strong(_freelistLock);
7914 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7915 "CMS thread should hold CMS token");
7916 _bitMap->lock()->unlock();
7917 _freelistLock->unlock();
7918 ConcurrentMarkSweepThread::desynchronize(true);
7919 _collector->stopTimer();
7920 if (PrintCMSStatistics != 0) {
7921 _collector->incrementYields();
7922 }
7923
7924 // See the comment in coordinator_yield()
7925 for (unsigned i = 0; i < CMSYieldSleepCount &&
7926 ConcurrentMarkSweepThread::should_yield() &&
7927 !CMSCollector::foregroundGCIsActive(); ++i) {
7928 os::sleep(Thread::current(), 1, false);
7929 }
7930
7931 ConcurrentMarkSweepThread::synchronize(true);
7932 _freelistLock->lock();
7933 _bitMap->lock()->lock_without_safepoint_check();
7934 _collector->startTimer();
7935 }
7936
7937 #ifndef PRODUCT
7938 // This is actually very useful in a product build if it can
7939 // be called from the debugger. Compile it into the product
7940 // as needed.
7941 bool debug_verify_chunk_in_free_list(FreeChunk* fc) {
7942 return debug_cms_space->verify_chunk_in_free_list(fc);
7943 }
7944 #endif
7945
7946 void SweepClosure::print_free_block_coalesced(FreeChunk* fc) const {
7947 if (CMSTraceSweeper) {
7948 gclog_or_tty->print_cr("Sweep:coal_free_blk " PTR_FORMAT " (" SIZE_FORMAT ")",
7949 p2i(fc), fc->size());
7950 }
7951 }
7952
7953 // CMSIsAliveClosure
7954 bool CMSIsAliveClosure::do_object_b(oop obj) {
7955 HeapWord* addr = (HeapWord*)obj;
7956 return addr != NULL &&
7957 (!_span.contains(addr) || _bit_map->isMarked(addr));
7958 }
7959
7960
7961 CMSKeepAliveClosure::CMSKeepAliveClosure( CMSCollector* collector,
7962 MemRegion span,
7963 CMSBitMap* bit_map, CMSMarkStack* mark_stack,
7964 bool cpc):
7965 _collector(collector),
7966 _span(span),
7967 _bit_map(bit_map),
7968 _mark_stack(mark_stack),
7969 _concurrent_precleaning(cpc) {
7970 assert(!_span.is_empty(), "Empty span could spell trouble");
7971 }
7972
7973
7974 // CMSKeepAliveClosure: the serial version
7975 void CMSKeepAliveClosure::do_oop(oop obj) {
7976 HeapWord* addr = (HeapWord*)obj;
7977 if (_span.contains(addr) &&
7978 !_bit_map->isMarked(addr)) {
7979 _bit_map->mark(addr);
7980 bool simulate_overflow = false;
7981 NOT_PRODUCT(
7982 if (CMSMarkStackOverflowALot &&
7983 _collector->simulate_overflow()) {
7984 // simulate a stack overflow
7985 simulate_overflow = true;
7986 }
7987 )
7988 if (simulate_overflow || !_mark_stack->push(obj)) {
7989 if (_concurrent_precleaning) {
7990 // We dirty the overflown object and let the remark
7991 // phase deal with it.
7992 assert(_collector->overflow_list_is_empty(), "Error");
7993 // In the case of object arrays, we need to dirty all of
7994 // the cards that the object spans. No locking or atomics
7995 // are needed since no one else can be mutating the mod union
7996 // table.
7997 if (obj->is_objArray()) {
7998 size_t sz = obj->size();
7999 HeapWord* end_card_addr =
8000 (HeapWord*)round_to((intptr_t)(addr+sz), CardTableModRefBS::card_size);
8001 MemRegion redirty_range = MemRegion(addr, end_card_addr);
8002 assert(!redirty_range.is_empty(), "Arithmetical tautology");
8003 _collector->_modUnionTable.mark_range(redirty_range);
8004 } else {
8005 _collector->_modUnionTable.mark(addr);
8006 }
8007 _collector->_ser_kac_preclean_ovflw++;
8008 } else {
8009 _collector->push_on_overflow_list(obj);
8010 _collector->_ser_kac_ovflw++;
8011 }
8012 }
8013 }
8014 }
8015
8016 void CMSKeepAliveClosure::do_oop(oop* p) { CMSKeepAliveClosure::do_oop_work(p); }
8017 void CMSKeepAliveClosure::do_oop(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); }
8018
8019 // CMSParKeepAliveClosure: a parallel version of the above.
8020 // The work queues are private to each closure (thread),
8021 // but (may be) available for stealing by other threads.
8022 void CMSParKeepAliveClosure::do_oop(oop obj) {
8023 HeapWord* addr = (HeapWord*)obj;
8024 if (_span.contains(addr) &&
8025 !_bit_map->isMarked(addr)) {
8026 // In general, during recursive tracing, several threads
8027 // may be concurrently getting here; the first one to
8028 // "tag" it, claims it.
8029 if (_bit_map->par_mark(addr)) {
8030 bool res = _work_queue->push(obj);
8031 assert(res, "Low water mark should be much less than capacity");
8032 // Do a recursive trim in the hope that this will keep
8033 // stack usage lower, but leave some oops for potential stealers
8034 trim_queue(_low_water_mark);
8035 } // Else, another thread got there first
8036 }
8037 }
8038
8039 void CMSParKeepAliveClosure::do_oop(oop* p) { CMSParKeepAliveClosure::do_oop_work(p); }
8040 void CMSParKeepAliveClosure::do_oop(narrowOop* p) { CMSParKeepAliveClosure::do_oop_work(p); }
8041
8042 void CMSParKeepAliveClosure::trim_queue(uint max) {
8043 while (_work_queue->size() > max) {
8044 oop new_oop;
8045 if (_work_queue->pop_local(new_oop)) {
8046 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
8047 assert(_bit_map->isMarked((HeapWord*)new_oop),
8048 "no white objects on this stack!");
8049 assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop");
8050 // iterate over the oops in this oop, marking and pushing
8051 // the ones in CMS heap (i.e. in _span).
8052 new_oop->oop_iterate(&_mark_and_push);
8053 }
8054 }
8055 }
8056
8057 CMSInnerParMarkAndPushClosure::CMSInnerParMarkAndPushClosure(
8058 CMSCollector* collector,
8059 MemRegion span, CMSBitMap* bit_map,
8060 OopTaskQueue* work_queue):
8061 _collector(collector),
8062 _span(span),
8063 _bit_map(bit_map),
8064 _work_queue(work_queue) { }
8065
8066 void CMSInnerParMarkAndPushClosure::do_oop(oop obj) {
8067 HeapWord* addr = (HeapWord*)obj;
8068 if (_span.contains(addr) &&
8069 !_bit_map->isMarked(addr)) {
8070 if (_bit_map->par_mark(addr)) {
8071 bool simulate_overflow = false;
8072 NOT_PRODUCT(
8073 if (CMSMarkStackOverflowALot &&
8074 _collector->par_simulate_overflow()) {
8075 // simulate a stack overflow
8076 simulate_overflow = true;
8077 }
8078 )
8079 if (simulate_overflow || !_work_queue->push(obj)) {
8080 _collector->par_push_on_overflow_list(obj);
8081 _collector->_par_kac_ovflw++;
8082 }
8083 } // Else another thread got there already
8084 }
8085 }
8086
8087 void CMSInnerParMarkAndPushClosure::do_oop(oop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); }
8088 void CMSInnerParMarkAndPushClosure::do_oop(narrowOop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); }
8089
8090 //////////////////////////////////////////////////////////////////
8091 // CMSExpansionCause /////////////////////////////
8092 //////////////////////////////////////////////////////////////////
8093 const char* CMSExpansionCause::to_string(CMSExpansionCause::Cause cause) {
8094 switch (cause) {
8095 case _no_expansion:
8096 return "No expansion";
8097 case _satisfy_free_ratio:
8098 return "Free ratio";
8099 case _satisfy_promotion:
8100 return "Satisfy promotion";
8101 case _satisfy_allocation:
8102 return "allocation";
8103 case _allocate_par_lab:
8104 return "Par LAB";
8105 case _allocate_par_spooling_space:
8106 return "Par Spooling Space";
8107 case _adaptive_size_policy:
8108 return "Ergonomics";
8109 default:
8110 return "unknown";
8111 }
8112 }
8113
8114 void CMSDrainMarkingStackClosure::do_void() {
8115 // the max number to take from overflow list at a time
8116 const size_t num = _mark_stack->capacity()/4;
8117 assert(!_concurrent_precleaning || _collector->overflow_list_is_empty(),
8118 "Overflow list should be NULL during concurrent phases");
8119 while (!_mark_stack->isEmpty() ||
8120 // if stack is empty, check the overflow list
8121 _collector->take_from_overflow_list(num, _mark_stack)) {
8122 oop obj = _mark_stack->pop();
8123 HeapWord* addr = (HeapWord*)obj;
8124 assert(_span.contains(addr), "Should be within span");
8125 assert(_bit_map->isMarked(addr), "Should be marked");
8126 assert(obj->is_oop(), "Should be an oop");
8127 obj->oop_iterate(_keep_alive);
8128 }
8129 }
8130
8131 void CMSParDrainMarkingStackClosure::do_void() {
8132 // drain queue
8133 trim_queue(0);
8134 }
8135
8136 // Trim our work_queue so its length is below max at return
8137 void CMSParDrainMarkingStackClosure::trim_queue(uint max) {
8138 while (_work_queue->size() > max) {
8139 oop new_oop;
8140 if (_work_queue->pop_local(new_oop)) {
8141 assert(new_oop->is_oop(), "Expected an oop");
8142 assert(_bit_map->isMarked((HeapWord*)new_oop),
8143 "no white objects on this stack!");
8144 assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop");
8145 // iterate over the oops in this oop, marking and pushing
8146 // the ones in CMS heap (i.e. in _span).
8147 new_oop->oop_iterate(&_mark_and_push);
8148 }
8149 }
8150 }
8151
8152 ////////////////////////////////////////////////////////////////////
8153 // Support for Marking Stack Overflow list handling and related code
8154 ////////////////////////////////////////////////////////////////////
8155 // Much of the following code is similar in shape and spirit to the
8156 // code used in ParNewGC. We should try and share that code
8157 // as much as possible in the future.
8158
8159 #ifndef PRODUCT
8160 // Debugging support for CMSStackOverflowALot
8161
8162 // It's OK to call this multi-threaded; the worst thing
8163 // that can happen is that we'll get a bunch of closely
8164 // spaced simulated overflows, but that's OK, in fact
8165 // probably good as it would exercise the overflow code
8166 // under contention.
8167 bool CMSCollector::simulate_overflow() {
8168 if (_overflow_counter-- <= 0) { // just being defensive
8169 _overflow_counter = CMSMarkStackOverflowInterval;
8170 return true;
8171 } else {
8172 return false;
8173 }
8174 }
8175
8176 bool CMSCollector::par_simulate_overflow() {
8177 return simulate_overflow();
8178 }
8179 #endif
8180
8181 // Single-threaded
8182 bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) {
8183 assert(stack->isEmpty(), "Expected precondition");
8184 assert(stack->capacity() > num, "Shouldn't bite more than can chew");
8185 size_t i = num;
8186 oop cur = _overflow_list;
8187 const markOop proto = markOopDesc::prototype();
8188 NOT_PRODUCT(ssize_t n = 0;)
8189 for (oop next; i > 0 && cur != NULL; cur = next, i--) {
8190 next = oop(cur->mark());
8191 cur->set_mark(proto); // until proven otherwise
8192 assert(cur->is_oop(), "Should be an oop");
8193 bool res = stack->push(cur);
8194 assert(res, "Bit off more than can chew?");
8195 NOT_PRODUCT(n++;)
8196 }
8197 _overflow_list = cur;
8198 #ifndef PRODUCT
8199 assert(_num_par_pushes >= n, "Too many pops?");
8200 _num_par_pushes -=n;
8201 #endif
8202 return !stack->isEmpty();
8203 }
8204
8205 #define BUSY (cast_to_oop<intptr_t>(0x1aff1aff))
8206 // (MT-safe) Get a prefix of at most "num" from the list.
8207 // The overflow list is chained through the mark word of
8208 // each object in the list. We fetch the entire list,
8209 // break off a prefix of the right size and return the
8210 // remainder. If other threads try to take objects from
8211 // the overflow list at that time, they will wait for
8212 // some time to see if data becomes available. If (and
8213 // only if) another thread places one or more object(s)
8214 // on the global list before we have returned the suffix
8215 // to the global list, we will walk down our local list
8216 // to find its end and append the global list to
8217 // our suffix before returning it. This suffix walk can
8218 // prove to be expensive (quadratic in the amount of traffic)
8219 // when there are many objects in the overflow list and
8220 // there is much producer-consumer contention on the list.
8221 // *NOTE*: The overflow list manipulation code here and
8222 // in ParNewGeneration:: are very similar in shape,
8223 // except that in the ParNew case we use the old (from/eden)
8224 // copy of the object to thread the list via its klass word.
8225 // Because of the common code, if you make any changes in
8226 // the code below, please check the ParNew version to see if
8227 // similar changes might be needed.
8228 // CR 6797058 has been filed to consolidate the common code.
8229 bool CMSCollector::par_take_from_overflow_list(size_t num,
8230 OopTaskQueue* work_q,
8231 int no_of_gc_threads) {
8232 assert(work_q->size() == 0, "First empty local work queue");
8233 assert(num < work_q->max_elems(), "Can't bite more than we can chew");
8234 if (_overflow_list == NULL) {
8235 return false;
8236 }
8237 // Grab the entire list; we'll put back a suffix
8238 oop prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list));
8239 Thread* tid = Thread::current();
8240 // Before "no_of_gc_threads" was introduced CMSOverflowSpinCount was
8241 // set to ParallelGCThreads.
8242 size_t CMSOverflowSpinCount = (size_t) no_of_gc_threads; // was ParallelGCThreads;
8243 size_t sleep_time_millis = MAX2((size_t)1, num/100);
8244 // If the list is busy, we spin for a short while,
8245 // sleeping between attempts to get the list.
8246 for (size_t spin = 0; prefix == BUSY && spin < CMSOverflowSpinCount; spin++) {
8247 os::sleep(tid, sleep_time_millis, false);
8248 if (_overflow_list == NULL) {
8249 // Nothing left to take
8250 return false;
8251 } else if (_overflow_list != BUSY) {
8252 // Try and grab the prefix
8253 prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list));
8254 }
8255 }
8256 // If the list was found to be empty, or we spun long
8257 // enough, we give up and return empty-handed. If we leave
8258 // the list in the BUSY state below, it must be the case that
8259 // some other thread holds the overflow list and will set it
8260 // to a non-BUSY state in the future.
8261 if (prefix == NULL || prefix == BUSY) {
8262 // Nothing to take or waited long enough
8263 if (prefix == NULL) {
8264 // Write back the NULL in case we overwrote it with BUSY above
8265 // and it is still the same value.
8266 (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY);
8267 }
8268 return false;
8269 }
8270 assert(prefix != NULL && prefix != BUSY, "Error");
8271 size_t i = num;
8272 oop cur = prefix;
8273 // Walk down the first "num" objects, unless we reach the end.
8274 for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--);
8275 if (cur->mark() == NULL) {
8276 // We have "num" or fewer elements in the list, so there
8277 // is nothing to return to the global list.
8278 // Write back the NULL in lieu of the BUSY we wrote
8279 // above, if it is still the same value.
8280 if (_overflow_list == BUSY) {
8281 (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY);
8282 }
8283 } else {
8284 // Chop off the suffix and return it to the global list.
8285 assert(cur->mark() != BUSY, "Error");
8286 oop suffix_head = cur->mark(); // suffix will be put back on global list
8287 cur->set_mark(NULL); // break off suffix
8288 // It's possible that the list is still in the empty(busy) state
8289 // we left it in a short while ago; in that case we may be
8290 // able to place back the suffix without incurring the cost
8291 // of a walk down the list.
8292 oop observed_overflow_list = _overflow_list;
8293 oop cur_overflow_list = observed_overflow_list;
8294 bool attached = false;
8295 while (observed_overflow_list == BUSY || observed_overflow_list == NULL) {
8296 observed_overflow_list =
8297 (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur_overflow_list);
8298 if (cur_overflow_list == observed_overflow_list) {
8299 attached = true;
8300 break;
8301 } else cur_overflow_list = observed_overflow_list;
8302 }
8303 if (!attached) {
8304 // Too bad, someone else sneaked in (at least) an element; we'll need
8305 // to do a splice. Find tail of suffix so we can prepend suffix to global
8306 // list.
8307 for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark()));
8308 oop suffix_tail = cur;
8309 assert(suffix_tail != NULL && suffix_tail->mark() == NULL,
8310 "Tautology");
8311 observed_overflow_list = _overflow_list;
8312 do {
8313 cur_overflow_list = observed_overflow_list;
8314 if (cur_overflow_list != BUSY) {
8315 // Do the splice ...
8316 suffix_tail->set_mark(markOop(cur_overflow_list));
8317 } else { // cur_overflow_list == BUSY
8318 suffix_tail->set_mark(NULL);
8319 }
8320 // ... and try to place spliced list back on overflow_list ...
8321 observed_overflow_list =
8322 (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur_overflow_list);
8323 } while (cur_overflow_list != observed_overflow_list);
8324 // ... until we have succeeded in doing so.
8325 }
8326 }
8327
8328 // Push the prefix elements on work_q
8329 assert(prefix != NULL, "control point invariant");
8330 const markOop proto = markOopDesc::prototype();
8331 oop next;
8332 NOT_PRODUCT(ssize_t n = 0;)
8333 for (cur = prefix; cur != NULL; cur = next) {
8334 next = oop(cur->mark());
8335 cur->set_mark(proto); // until proven otherwise
8336 assert(cur->is_oop(), "Should be an oop");
8337 bool res = work_q->push(cur);
8338 assert(res, "Bit off more than we can chew?");
8339 NOT_PRODUCT(n++;)
8340 }
8341 #ifndef PRODUCT
8342 assert(_num_par_pushes >= n, "Too many pops?");
8343 Atomic::add_ptr(-(intptr_t)n, &_num_par_pushes);
8344 #endif
8345 return true;
8346 }
8347
8348 // Single-threaded
8349 void CMSCollector::push_on_overflow_list(oop p) {
8350 NOT_PRODUCT(_num_par_pushes++;)
8351 assert(p->is_oop(), "Not an oop");
8352 preserve_mark_if_necessary(p);
8353 p->set_mark((markOop)_overflow_list);
8354 _overflow_list = p;
8355 }
8356
8357 // Multi-threaded; use CAS to prepend to overflow list
8358 void CMSCollector::par_push_on_overflow_list(oop p) {
8359 NOT_PRODUCT(Atomic::inc_ptr(&_num_par_pushes);)
8360 assert(p->is_oop(), "Not an oop");
8361 par_preserve_mark_if_necessary(p);
8362 oop observed_overflow_list = _overflow_list;
8363 oop cur_overflow_list;
8364 do {
8365 cur_overflow_list = observed_overflow_list;
8366 if (cur_overflow_list != BUSY) {
8367 p->set_mark(markOop(cur_overflow_list));
8368 } else {
8369 p->set_mark(NULL);
8370 }
8371 observed_overflow_list =
8372 (oop) Atomic::cmpxchg_ptr(p, &_overflow_list, cur_overflow_list);
8373 } while (cur_overflow_list != observed_overflow_list);
8374 }
8375 #undef BUSY
8376
8377 // Single threaded
8378 // General Note on GrowableArray: pushes may silently fail
8379 // because we are (temporarily) out of C-heap for expanding
8380 // the stack. The problem is quite ubiquitous and affects
8381 // a lot of code in the JVM. The prudent thing for GrowableArray
8382 // to do (for now) is to exit with an error. However, that may
8383 // be too draconian in some cases because the caller may be
8384 // able to recover without much harm. For such cases, we
8385 // should probably introduce a "soft_push" method which returns
8386 // an indication of success or failure with the assumption that
8387 // the caller may be able to recover from a failure; code in
8388 // the VM can then be changed, incrementally, to deal with such
8389 // failures where possible, thus, incrementally hardening the VM
8390 // in such low resource situations.
8391 void CMSCollector::preserve_mark_work(oop p, markOop m) {
8392 _preserved_oop_stack.push(p);
8393 _preserved_mark_stack.push(m);
8394 assert(m == p->mark(), "Mark word changed");
8395 assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
8396 "bijection");
8397 }
8398
8399 // Single threaded
8400 void CMSCollector::preserve_mark_if_necessary(oop p) {
8401 markOop m = p->mark();
8402 if (m->must_be_preserved(p)) {
8403 preserve_mark_work(p, m);
8404 }
8405 }
8406
8407 void CMSCollector::par_preserve_mark_if_necessary(oop p) {
8408 markOop m = p->mark();
8409 if (m->must_be_preserved(p)) {
8410 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
8411 // Even though we read the mark word without holding
8412 // the lock, we are assured that it will not change
8413 // because we "own" this oop, so no other thread can
8414 // be trying to push it on the overflow list; see
8415 // the assertion in preserve_mark_work() that checks
8416 // that m == p->mark().
8417 preserve_mark_work(p, m);
8418 }
8419 }
8420
8421 // We should be able to do this multi-threaded,
8422 // a chunk of stack being a task (this is
8423 // correct because each oop only ever appears
8424 // once in the overflow list. However, it's
8425 // not very easy to completely overlap this with
8426 // other operations, so will generally not be done
8427 // until all work's been completed. Because we
8428 // expect the preserved oop stack (set) to be small,
8429 // it's probably fine to do this single-threaded.
8430 // We can explore cleverer concurrent/overlapped/parallel
8431 // processing of preserved marks if we feel the
8432 // need for this in the future. Stack overflow should
8433 // be so rare in practice and, when it happens, its
8434 // effect on performance so great that this will
8435 // likely just be in the noise anyway.
8436 void CMSCollector::restore_preserved_marks_if_any() {
8437 assert(SafepointSynchronize::is_at_safepoint(),
8438 "world should be stopped");
8439 assert(Thread::current()->is_ConcurrentGC_thread() ||
8440 Thread::current()->is_VM_thread(),
8441 "should be single-threaded");
8442 assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
8443 "bijection");
8444
8445 while (!_preserved_oop_stack.is_empty()) {
8446 oop p = _preserved_oop_stack.pop();
8447 assert(p->is_oop(), "Should be an oop");
8448 assert(_span.contains(p), "oop should be in _span");
8449 assert(p->mark() == markOopDesc::prototype(),
8450 "Set when taken from overflow list");
8451 markOop m = _preserved_mark_stack.pop();
8452 p->set_mark(m);
8453 }
8454 assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(),
8455 "stacks were cleared above");
8456 }
8457
8458 #ifndef PRODUCT
8459 bool CMSCollector::no_preserved_marks() const {
8460 return _preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty();
8461 }
8462 #endif
8463
8464 // Transfer some number of overflown objects to usual marking
8465 // stack. Return true if some objects were transferred.
8466 bool MarkRefsIntoAndScanClosure::take_from_overflow_list() {
8467 size_t num = MIN2((size_t)(_mark_stack->capacity() - _mark_stack->length())/4,
8468 (size_t)ParGCDesiredObjsFromOverflowList);
8469
8470 bool res = _collector->take_from_overflow_list(num, _mark_stack);
8471 assert(_collector->overflow_list_is_empty() || res,
8472 "If list is not empty, we should have taken something");
8473 assert(!res || !_mark_stack->isEmpty(),
8474 "If we took something, it should now be on our stack");
8475 return res;
8476 }
8477
8478 size_t MarkDeadObjectsClosure::do_blk(HeapWord* addr) {
8479 size_t res = _sp->block_size_no_stall(addr, _collector);
8480 if (_sp->block_is_obj(addr)) {
8481 if (_live_bit_map->isMarked(addr)) {
8482 // It can't have been dead in a previous cycle
8483 guarantee(!_dead_bit_map->isMarked(addr), "No resurrection!");
8484 } else {
8485 _dead_bit_map->mark(addr); // mark the dead object
8486 }
8487 }
8488 // Could be 0, if the block size could not be computed without stalling.
8489 return res;
8490 }
8491
8492 TraceCMSMemoryManagerStats::TraceCMSMemoryManagerStats(CMSCollector::CollectorState phase, GCCause::Cause cause): TraceMemoryManagerStats() {
8493
8494 switch (phase) {
8495 case CMSCollector::InitialMarking:
8496 initialize(true /* fullGC */ ,
8497 cause /* cause of the GC */,
8498 true /* recordGCBeginTime */,
8499 true /* recordPreGCUsage */,
8500 false /* recordPeakUsage */,
8501 false /* recordPostGCusage */,
8502 true /* recordAccumulatedGCTime */,
8503 false /* recordGCEndTime */,
8504 false /* countCollection */ );
8505 break;
8506
8507 case CMSCollector::FinalMarking:
8508 initialize(true /* fullGC */ ,
8509 cause /* cause of the GC */,
8510 false /* recordGCBeginTime */,
8511 false /* recordPreGCUsage */,
8512 false /* recordPeakUsage */,
8513 false /* recordPostGCusage */,
8514 true /* recordAccumulatedGCTime */,
8515 false /* recordGCEndTime */,
8516 false /* countCollection */ );
8517 break;
8518
8519 case CMSCollector::Sweeping:
8520 initialize(true /* fullGC */ ,
8521 cause /* cause of the GC */,
8522 false /* recordGCBeginTime */,
8523 false /* recordPreGCUsage */,
8524 true /* recordPeakUsage */,
8525 true /* recordPostGCusage */,
8526 false /* recordAccumulatedGCTime */,
8527 true /* recordGCEndTime */,
8528 true /* countCollection */ );
8529 break;
8530
8531 default:
8532 ShouldNotReachHere();
8533 }
8534 }