1 /*
2 * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/symbolTable.hpp"
27 #include "classfile/systemDictionary.hpp"
28 #include "code/codeCache.hpp"
29 #include "gc_implementation/concurrentMarkSweep/cmsAdaptiveSizePolicy.hpp"
30 #include "gc_implementation/concurrentMarkSweep/cmsCollectorPolicy.hpp"
31 #include "gc_implementation/concurrentMarkSweep/cmsGCAdaptivePolicyCounters.hpp"
32 #include "gc_implementation/concurrentMarkSweep/cmsOopClosures.inline.hpp"
33 #include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp"
34 #include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.inline.hpp"
35 #include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp"
36 #include "gc_implementation/concurrentMarkSweep/vmCMSOperations.hpp"
37 #include "gc_implementation/parNew/parNewGeneration.hpp"
38 #include "gc_implementation/shared/collectorCounters.hpp"
39 #include "gc_implementation/shared/isGCActiveMark.hpp"
40 #include "gc_interface/collectedHeap.inline.hpp"
41 #include "memory/cardTableRS.hpp"
42 #include "memory/collectorPolicy.hpp"
43 #include "memory/gcLocker.inline.hpp"
44 #include "memory/genCollectedHeap.hpp"
45 #include "memory/genMarkSweep.hpp"
46 #include "memory/genOopClosures.inline.hpp"
47 #include "memory/iterator.hpp"
48 #include "memory/referencePolicy.hpp"
49 #include "memory/resourceArea.hpp"
50 #include "oops/oop.inline.hpp"
51 #include "prims/jvmtiExport.hpp"
52 #include "runtime/globals_extension.hpp"
53 #include "runtime/handles.inline.hpp"
54 #include "runtime/java.hpp"
55 #include "runtime/vmThread.hpp"
56 #include "services/memoryService.hpp"
57 #include "services/runtimeService.hpp"
58
59 // statics
60 CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;
61 bool CMSCollector::_full_gc_requested = false;
62
63 //////////////////////////////////////////////////////////////////
64 // In support of CMS/VM thread synchronization
65 //////////////////////////////////////////////////////////////////
66 // We split use of the CGC_lock into 2 "levels".
67 // The low-level locking is of the usual CGC_lock monitor. We introduce
68 // a higher level "token" (hereafter "CMS token") built on top of the
69 // low level monitor (hereafter "CGC lock").
70 // The token-passing protocol gives priority to the VM thread. The
71 // CMS-lock doesn't provide any fairness guarantees, but clients
72 // should ensure that it is only held for very short, bounded
73 // durations.
74 //
75 // When either of the CMS thread or the VM thread is involved in
76 // collection operations during which it does not want the other
77 // thread to interfere, it obtains the CMS token.
78 //
79 // If either thread tries to get the token while the other has
80 // it, that thread waits. However, if the VM thread and CMS thread
81 // both want the token, then the VM thread gets priority while the
82 // CMS thread waits. This ensures, for instance, that the "concurrent"
83 // phases of the CMS thread's work do not block out the VM thread
84 // for long periods of time as the CMS thread continues to hog
85 // the token. (See bug 4616232).
86 //
87 // The baton-passing functions are, however, controlled by the
88 // flags _foregroundGCShouldWait and _foregroundGCIsActive,
89 // and here the low-level CMS lock, not the high level token,
90 // ensures mutual exclusion.
91 //
92 // Two important conditions that we have to satisfy:
93 // 1. if a thread does a low-level wait on the CMS lock, then it
94 // relinquishes the CMS token if it were holding that token
95 // when it acquired the low-level CMS lock.
96 // 2. any low-level notifications on the low-level lock
97 // should only be sent when a thread has relinquished the token.
98 //
99 // In the absence of either property, we'd have potential deadlock.
100 //
101 // We protect each of the CMS (concurrent and sequential) phases
102 // with the CMS _token_, not the CMS _lock_.
103 //
104 // The only code protected by CMS lock is the token acquisition code
105 // itself, see ConcurrentMarkSweepThread::[de]synchronize(), and the
106 // baton-passing code.
107 //
108 // Unfortunately, i couldn't come up with a good abstraction to factor and
109 // hide the naked CGC_lock manipulation in the baton-passing code
110 // further below. That's something we should try to do. Also, the proof
111 // of correctness of this 2-level locking scheme is far from obvious,
112 // and potentially quite slippery. We have an uneasy supsicion, for instance,
113 // that there may be a theoretical possibility of delay/starvation in the
114 // low-level lock/wait/notify scheme used for the baton-passing because of
115 // potential intereference with the priority scheme embodied in the
116 // CMS-token-passing protocol. See related comments at a CGC_lock->wait()
117 // invocation further below and marked with "XXX 20011219YSR".
118 // Indeed, as we note elsewhere, this may become yet more slippery
119 // in the presence of multiple CMS and/or multiple VM threads. XXX
120
121 class CMSTokenSync: public StackObj {
122 private:
123 bool _is_cms_thread;
124 public:
125 CMSTokenSync(bool is_cms_thread):
126 _is_cms_thread(is_cms_thread) {
127 assert(is_cms_thread == Thread::current()->is_ConcurrentGC_thread(),
128 "Incorrect argument to constructor");
129 ConcurrentMarkSweepThread::synchronize(_is_cms_thread);
130 }
131
132 ~CMSTokenSync() {
133 assert(_is_cms_thread ?
134 ConcurrentMarkSweepThread::cms_thread_has_cms_token() :
135 ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
136 "Incorrect state");
137 ConcurrentMarkSweepThread::desynchronize(_is_cms_thread);
138 }
139 };
140
141 // Convenience class that does a CMSTokenSync, and then acquires
142 // upto three locks.
143 class CMSTokenSyncWithLocks: public CMSTokenSync {
144 private:
145 // Note: locks are acquired in textual declaration order
146 // and released in the opposite order
147 MutexLockerEx _locker1, _locker2, _locker3;
148 public:
149 CMSTokenSyncWithLocks(bool is_cms_thread, Mutex* mutex1,
150 Mutex* mutex2 = NULL, Mutex* mutex3 = NULL):
151 CMSTokenSync(is_cms_thread),
152 _locker1(mutex1, Mutex::_no_safepoint_check_flag),
153 _locker2(mutex2, Mutex::_no_safepoint_check_flag),
154 _locker3(mutex3, Mutex::_no_safepoint_check_flag)
155 { }
156 };
157
158
159 // Wrapper class to temporarily disable icms during a foreground cms collection.
160 class ICMSDisabler: public StackObj {
161 public:
162 // The ctor disables icms and wakes up the thread so it notices the change;
163 // the dtor re-enables icms. Note that the CMSCollector methods will check
164 // CMSIncrementalMode.
165 ICMSDisabler() { CMSCollector::disable_icms(); CMSCollector::start_icms(); }
166 ~ICMSDisabler() { CMSCollector::enable_icms(); }
167 };
168
169 //////////////////////////////////////////////////////////////////
170 // Concurrent Mark-Sweep Generation /////////////////////////////
171 //////////////////////////////////////////////////////////////////
172
173 NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;)
174
175 // This struct contains per-thread things necessary to support parallel
176 // young-gen collection.
177 class CMSParGCThreadState: public CHeapObj {
178 public:
179 CFLS_LAB lab;
180 PromotionInfo promo;
181
182 // Constructor.
183 CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) {
184 promo.setSpace(cfls);
185 }
186 };
187
188 ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration(
189 ReservedSpace rs, size_t initial_byte_size, int level,
190 CardTableRS* ct, bool use_adaptive_freelists,
191 FreeBlockDictionary::DictionaryChoice dictionaryChoice) :
192 CardGeneration(rs, initial_byte_size, level, ct),
193 _dilatation_factor(((double)MinChunkSize)/((double)(CollectedHeap::min_fill_size()))),
194 _debug_collection_type(Concurrent_collection_type)
195 {
196 HeapWord* bottom = (HeapWord*) _virtual_space.low();
197 HeapWord* end = (HeapWord*) _virtual_space.high();
198
199 _direct_allocated_words = 0;
200 NOT_PRODUCT(
201 _numObjectsPromoted = 0;
202 _numWordsPromoted = 0;
203 _numObjectsAllocated = 0;
204 _numWordsAllocated = 0;
205 )
206
207 _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end),
208 use_adaptive_freelists,
209 dictionaryChoice);
210 NOT_PRODUCT(debug_cms_space = _cmsSpace;)
211 if (_cmsSpace == NULL) {
212 vm_exit_during_initialization(
213 "CompactibleFreeListSpace allocation failure");
214 }
215 _cmsSpace->_gen = this;
216
217 _gc_stats = new CMSGCStats();
218
219 // Verify the assumption that FreeChunk::_prev and OopDesc::_klass
220 // offsets match. The ability to tell free chunks from objects
221 // depends on this property.
222 debug_only(
223 FreeChunk* junk = NULL;
224 assert(UseCompressedOops ||
225 junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
226 "Offset of FreeChunk::_prev within FreeChunk must match"
227 " that of OopDesc::_klass within OopDesc");
228 )
229 if (CollectedHeap::use_parallel_gc_threads()) {
230 typedef CMSParGCThreadState* CMSParGCThreadStatePtr;
231 _par_gc_thread_states =
232 NEW_C_HEAP_ARRAY(CMSParGCThreadStatePtr, ParallelGCThreads);
233 if (_par_gc_thread_states == NULL) {
234 vm_exit_during_initialization("Could not allocate par gc structs");
235 }
236 for (uint i = 0; i < ParallelGCThreads; i++) {
237 _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace());
238 if (_par_gc_thread_states[i] == NULL) {
239 vm_exit_during_initialization("Could not allocate par gc structs");
240 }
241 }
242 } else {
243 _par_gc_thread_states = NULL;
244 }
245 _incremental_collection_failed = false;
246 // The "dilatation_factor" is the expansion that can occur on
247 // account of the fact that the minimum object size in the CMS
248 // generation may be larger than that in, say, a contiguous young
249 // generation.
250 // Ideally, in the calculation below, we'd compute the dilatation
251 // factor as: MinChunkSize/(promoting_gen's min object size)
252 // Since we do not have such a general query interface for the
253 // promoting generation, we'll instead just use the mimimum
254 // object size (which today is a header's worth of space);
255 // note that all arithmetic is in units of HeapWords.
256 assert(MinChunkSize >= CollectedHeap::min_fill_size(), "just checking");
257 assert(_dilatation_factor >= 1.0, "from previous assert");
258 }
259
260
261 // The field "_initiating_occupancy" represents the occupancy percentage
262 // at which we trigger a new collection cycle. Unless explicitly specified
263 // via CMSInitiating[Perm]OccupancyFraction (argument "io" below), it
264 // is calculated by:
265 //
266 // Let "f" be MinHeapFreeRatio in
267 //
268 // _intiating_occupancy = 100-f +
269 // f * (CMSTrigger[Perm]Ratio/100)
270 // where CMSTrigger[Perm]Ratio is the argument "tr" below.
271 //
272 // That is, if we assume the heap is at its desired maximum occupancy at the
273 // end of a collection, we let CMSTrigger[Perm]Ratio of the (purported) free
274 // space be allocated before initiating a new collection cycle.
275 //
276 void ConcurrentMarkSweepGeneration::init_initiating_occupancy(intx io, intx tr) {
277 assert(io <= 100 && tr >= 0 && tr <= 100, "Check the arguments");
278 if (io >= 0) {
279 _initiating_occupancy = (double)io / 100.0;
280 } else {
281 _initiating_occupancy = ((100 - MinHeapFreeRatio) +
282 (double)(tr * MinHeapFreeRatio) / 100.0)
283 / 100.0;
284 }
285 }
286
287 void ConcurrentMarkSweepGeneration::ref_processor_init() {
288 assert(collector() != NULL, "no collector");
289 collector()->ref_processor_init();
290 }
291
292 void CMSCollector::ref_processor_init() {
293 if (_ref_processor == NULL) {
294 // Allocate and initialize a reference processor
295 _ref_processor = ReferenceProcessor::create_ref_processor(
296 _span, // span
297 _cmsGen->refs_discovery_is_atomic(), // atomic_discovery
298 _cmsGen->refs_discovery_is_mt(), // mt_discovery
299 &_is_alive_closure,
300 ParallelGCThreads,
301 ParallelRefProcEnabled);
302 // Initialize the _ref_processor field of CMSGen
303 _cmsGen->set_ref_processor(_ref_processor);
304
305 // Allocate a dummy ref processor for perm gen.
306 ReferenceProcessor* rp2 = new ReferenceProcessor();
307 if (rp2 == NULL) {
308 vm_exit_during_initialization("Could not allocate ReferenceProcessor object");
309 }
310 _permGen->set_ref_processor(rp2);
311 }
312 }
313
314 CMSAdaptiveSizePolicy* CMSCollector::size_policy() {
315 GenCollectedHeap* gch = GenCollectedHeap::heap();
316 assert(gch->kind() == CollectedHeap::GenCollectedHeap,
317 "Wrong type of heap");
318 CMSAdaptiveSizePolicy* sp = (CMSAdaptiveSizePolicy*)
319 gch->gen_policy()->size_policy();
320 assert(sp->is_gc_cms_adaptive_size_policy(),
321 "Wrong type of size policy");
322 return sp;
323 }
324
325 CMSGCAdaptivePolicyCounters* CMSCollector::gc_adaptive_policy_counters() {
326 CMSGCAdaptivePolicyCounters* results =
327 (CMSGCAdaptivePolicyCounters*) collector_policy()->counters();
328 assert(
329 results->kind() == GCPolicyCounters::CMSGCAdaptivePolicyCountersKind,
330 "Wrong gc policy counter kind");
331 return results;
332 }
333
334
335 void ConcurrentMarkSweepGeneration::initialize_performance_counters() {
336
337 const char* gen_name = "old";
338
339 // Generation Counters - generation 1, 1 subspace
340 _gen_counters = new GenerationCounters(gen_name, 1, 1, &_virtual_space);
341
342 _space_counters = new GSpaceCounters(gen_name, 0,
343 _virtual_space.reserved_size(),
344 this, _gen_counters);
345 }
346
347 CMSStats::CMSStats(ConcurrentMarkSweepGeneration* cms_gen, unsigned int alpha):
348 _cms_gen(cms_gen)
349 {
350 assert(alpha <= 100, "bad value");
351 _saved_alpha = alpha;
352
353 // Initialize the alphas to the bootstrap value of 100.
354 _gc0_alpha = _cms_alpha = 100;
355
356 _cms_begin_time.update();
357 _cms_end_time.update();
358
359 _gc0_duration = 0.0;
360 _gc0_period = 0.0;
361 _gc0_promoted = 0;
362
363 _cms_duration = 0.0;
364 _cms_period = 0.0;
365 _cms_allocated = 0;
366
367 _cms_used_at_gc0_begin = 0;
368 _cms_used_at_gc0_end = 0;
369 _allow_duty_cycle_reduction = false;
370 _valid_bits = 0;
371 _icms_duty_cycle = CMSIncrementalDutyCycle;
372 }
373
374 double CMSStats::cms_free_adjustment_factor(size_t free) const {
375 // TBD: CR 6909490
376 return 1.0;
377 }
378
379 void CMSStats::adjust_cms_free_adjustment_factor(bool fail, size_t free) {
380 }
381
382 // If promotion failure handling is on use
383 // the padded average size of the promotion for each
384 // young generation collection.
385 double CMSStats::time_until_cms_gen_full() const {
386 size_t cms_free = _cms_gen->cmsSpace()->free();
387 GenCollectedHeap* gch = GenCollectedHeap::heap();
388 size_t expected_promotion = gch->get_gen(0)->capacity();
389 if (HandlePromotionFailure) {
390 expected_promotion = MIN2(
391 (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average(),
392 expected_promotion);
393 }
394 if (cms_free > expected_promotion) {
395 // Start a cms collection if there isn't enough space to promote
396 // for the next minor collection. Use the padded average as
397 // a safety factor.
398 cms_free -= expected_promotion;
399
400 // Adjust by the safety factor.
401 double cms_free_dbl = (double)cms_free;
402 double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor)/100.0;
403 // Apply a further correction factor which tries to adjust
404 // for recent occurance of concurrent mode failures.
405 cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free);
406 cms_free_dbl = cms_free_dbl * cms_adjustment;
407
408 if (PrintGCDetails && Verbose) {
409 gclog_or_tty->print_cr("CMSStats::time_until_cms_gen_full: cms_free "
410 SIZE_FORMAT " expected_promotion " SIZE_FORMAT,
411 cms_free, expected_promotion);
412 gclog_or_tty->print_cr(" cms_free_dbl %f cms_consumption_rate %f",
413 cms_free_dbl, cms_consumption_rate() + 1.0);
414 }
415 // Add 1 in case the consumption rate goes to zero.
416 return cms_free_dbl / (cms_consumption_rate() + 1.0);
417 }
418 return 0.0;
419 }
420
421 // Compare the duration of the cms collection to the
422 // time remaining before the cms generation is empty.
423 // Note that the time from the start of the cms collection
424 // to the start of the cms sweep (less than the total
425 // duration of the cms collection) can be used. This
426 // has been tried and some applications experienced
427 // promotion failures early in execution. This was
428 // possibly because the averages were not accurate
429 // enough at the beginning.
430 double CMSStats::time_until_cms_start() const {
431 // We add "gc0_period" to the "work" calculation
432 // below because this query is done (mostly) at the
433 // end of a scavenge, so we need to conservatively
434 // account for that much possible delay
435 // in the query so as to avoid concurrent mode failures
436 // due to starting the collection just a wee bit too
437 // late.
438 double work = cms_duration() + gc0_period();
439 double deadline = time_until_cms_gen_full();
440 // If a concurrent mode failure occurred recently, we want to be
441 // more conservative and halve our expected time_until_cms_gen_full()
442 if (work > deadline) {
443 if (Verbose && PrintGCDetails) {
444 gclog_or_tty->print(
445 " CMSCollector: collect because of anticipated promotion "
446 "before full %3.7f + %3.7f > %3.7f ", cms_duration(),
447 gc0_period(), time_until_cms_gen_full());
448 }
449 return 0.0;
450 }
451 return work - deadline;
452 }
453
454 // Return a duty cycle based on old_duty_cycle and new_duty_cycle, limiting the
455 // amount of change to prevent wild oscillation.
456 unsigned int CMSStats::icms_damped_duty_cycle(unsigned int old_duty_cycle,
457 unsigned int new_duty_cycle) {
458 assert(old_duty_cycle <= 100, "bad input value");
459 assert(new_duty_cycle <= 100, "bad input value");
460
461 // Note: use subtraction with caution since it may underflow (values are
462 // unsigned). Addition is safe since we're in the range 0-100.
463 unsigned int damped_duty_cycle = new_duty_cycle;
464 if (new_duty_cycle < old_duty_cycle) {
465 const unsigned int largest_delta = MAX2(old_duty_cycle / 4, 5U);
466 if (new_duty_cycle + largest_delta < old_duty_cycle) {
467 damped_duty_cycle = old_duty_cycle - largest_delta;
468 }
469 } else if (new_duty_cycle > old_duty_cycle) {
470 const unsigned int largest_delta = MAX2(old_duty_cycle / 4, 15U);
471 if (new_duty_cycle > old_duty_cycle + largest_delta) {
472 damped_duty_cycle = MIN2(old_duty_cycle + largest_delta, 100U);
473 }
474 }
475 assert(damped_duty_cycle <= 100, "invalid duty cycle computed");
476
477 if (CMSTraceIncrementalPacing) {
478 gclog_or_tty->print(" [icms_damped_duty_cycle(%d,%d) = %d] ",
479 old_duty_cycle, new_duty_cycle, damped_duty_cycle);
480 }
481 return damped_duty_cycle;
482 }
483
484 unsigned int CMSStats::icms_update_duty_cycle_impl() {
485 assert(CMSIncrementalPacing && valid(),
486 "should be handled in icms_update_duty_cycle()");
487
488 double cms_time_so_far = cms_timer().seconds();
489 double scaled_duration = cms_duration_per_mb() * _cms_used_at_gc0_end / M;
490 double scaled_duration_remaining = fabsd(scaled_duration - cms_time_so_far);
491
492 // Avoid division by 0.
493 double time_until_full = MAX2(time_until_cms_gen_full(), 0.01);
494 double duty_cycle_dbl = 100.0 * scaled_duration_remaining / time_until_full;
495
496 unsigned int new_duty_cycle = MIN2((unsigned int)duty_cycle_dbl, 100U);
497 if (new_duty_cycle > _icms_duty_cycle) {
498 // Avoid very small duty cycles (1 or 2); 0 is allowed.
499 if (new_duty_cycle > 2) {
500 _icms_duty_cycle = icms_damped_duty_cycle(_icms_duty_cycle,
501 new_duty_cycle);
502 }
503 } else if (_allow_duty_cycle_reduction) {
504 // The duty cycle is reduced only once per cms cycle (see record_cms_end()).
505 new_duty_cycle = icms_damped_duty_cycle(_icms_duty_cycle, new_duty_cycle);
506 // Respect the minimum duty cycle.
507 unsigned int min_duty_cycle = (unsigned int)CMSIncrementalDutyCycleMin;
508 _icms_duty_cycle = MAX2(new_duty_cycle, min_duty_cycle);
509 }
510
511 if (PrintGCDetails || CMSTraceIncrementalPacing) {
512 gclog_or_tty->print(" icms_dc=%d ", _icms_duty_cycle);
513 }
514
515 _allow_duty_cycle_reduction = false;
516 return _icms_duty_cycle;
517 }
518
519 #ifndef PRODUCT
520 void CMSStats::print_on(outputStream *st) const {
521 st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);
522 st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,
523 gc0_duration(), gc0_period(), gc0_promoted());
524 st->print(",cms_dur=%g,cms_dur_per_mb=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,
525 cms_duration(), cms_duration_per_mb(),
526 cms_period(), cms_allocated());
527 st->print(",cms_since_beg=%g,cms_since_end=%g",
528 cms_time_since_begin(), cms_time_since_end());
529 st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,
530 _cms_used_at_gc0_begin, _cms_used_at_gc0_end);
531 if (CMSIncrementalMode) {
532 st->print(",dc=%d", icms_duty_cycle());
533 }
534
535 if (valid()) {
536 st->print(",promo_rate=%g,cms_alloc_rate=%g",
537 promotion_rate(), cms_allocation_rate());
538 st->print(",cms_consumption_rate=%g,time_until_full=%g",
539 cms_consumption_rate(), time_until_cms_gen_full());
540 }
541 st->print(" ");
542 }
543 #endif // #ifndef PRODUCT
544
545 CMSCollector::CollectorState CMSCollector::_collectorState =
546 CMSCollector::Idling;
547 bool CMSCollector::_foregroundGCIsActive = false;
548 bool CMSCollector::_foregroundGCShouldWait = false;
549
550 CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
551 ConcurrentMarkSweepGeneration* permGen,
552 CardTableRS* ct,
553 ConcurrentMarkSweepPolicy* cp):
554 _cmsGen(cmsGen),
555 _permGen(permGen),
556 _ct(ct),
557 _ref_processor(NULL), // will be set later
558 _conc_workers(NULL), // may be set later
559 _abort_preclean(false),
560 _start_sampling(false),
561 _between_prologue_and_epilogue(false),
562 _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),
563 _perm_gen_verify_bit_map(0, -1 /* no mutex */, "No_lock"),
564 _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize),
565 -1 /* lock-free */, "No_lock" /* dummy */),
566 _modUnionClosure(&_modUnionTable),
567 _modUnionClosurePar(&_modUnionTable),
568 // Adjust my span to cover old (cms) gen and perm gen
569 _span(cmsGen->reserved()._union(permGen->reserved())),
570 // Construct the is_alive_closure with _span & markBitMap
571 _is_alive_closure(_span, &_markBitMap),
572 _restart_addr(NULL),
573 _overflow_list(NULL),
574 _stats(cmsGen),
575 _eden_chunk_array(NULL), // may be set in ctor body
576 _eden_chunk_capacity(0), // -- ditto --
577 _eden_chunk_index(0), // -- ditto --
578 _survivor_plab_array(NULL), // -- ditto --
579 _survivor_chunk_array(NULL), // -- ditto --
580 _survivor_chunk_capacity(0), // -- ditto --
581 _survivor_chunk_index(0), // -- ditto --
582 _ser_pmc_preclean_ovflw(0),
583 _ser_kac_preclean_ovflw(0),
584 _ser_pmc_remark_ovflw(0),
585 _par_pmc_remark_ovflw(0),
586 _ser_kac_ovflw(0),
587 _par_kac_ovflw(0),
588 #ifndef PRODUCT
589 _num_par_pushes(0),
590 #endif
591 _collection_count_start(0),
592 _verifying(false),
593 _icms_start_limit(NULL),
594 _icms_stop_limit(NULL),
595 _verification_mark_bm(0, Mutex::leaf + 1, "CMS_verification_mark_bm_lock"),
596 _completed_initialization(false),
597 _collector_policy(cp),
598 _should_unload_classes(false),
599 _concurrent_cycles_since_last_unload(0),
600 _roots_scanning_options(0),
601 _inter_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding),
602 _intra_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding)
603 {
604 if (ExplicitGCInvokesConcurrentAndUnloadsClasses) {
605 ExplicitGCInvokesConcurrent = true;
606 }
607 // Now expand the span and allocate the collection support structures
608 // (MUT, marking bit map etc.) to cover both generations subject to
609 // collection.
610
611 // First check that _permGen is adjacent to _cmsGen and above it.
612 assert( _cmsGen->reserved().word_size() > 0
613 && _permGen->reserved().word_size() > 0,
614 "generations should not be of zero size");
615 assert(_cmsGen->reserved().intersection(_permGen->reserved()).is_empty(),
616 "_cmsGen and _permGen should not overlap");
617 assert(_cmsGen->reserved().end() == _permGen->reserved().start(),
618 "_cmsGen->end() different from _permGen->start()");
619
620 // For use by dirty card to oop closures.
621 _cmsGen->cmsSpace()->set_collector(this);
622 _permGen->cmsSpace()->set_collector(this);
623
624 // Allocate MUT and marking bit map
625 {
626 MutexLockerEx x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag);
627 if (!_markBitMap.allocate(_span)) {
628 warning("Failed to allocate CMS Bit Map");
629 return;
630 }
631 assert(_markBitMap.covers(_span), "_markBitMap inconsistency?");
632 }
633 {
634 _modUnionTable.allocate(_span);
635 assert(_modUnionTable.covers(_span), "_modUnionTable inconsistency?");
636 }
637
638 if (!_markStack.allocate(MarkStackSize)) {
639 warning("Failed to allocate CMS Marking Stack");
640 return;
641 }
642 if (!_revisitStack.allocate(CMSRevisitStackSize)) {
643 warning("Failed to allocate CMS Revisit Stack");
644 return;
645 }
646
647 // Support for multi-threaded concurrent phases
648 if (CollectedHeap::use_parallel_gc_threads() && CMSConcurrentMTEnabled) {
649 if (FLAG_IS_DEFAULT(ConcGCThreads)) {
650 // just for now
651 FLAG_SET_DEFAULT(ConcGCThreads, (ParallelGCThreads + 3)/4);
652 }
653 if (ConcGCThreads > 1) {
654 _conc_workers = new YieldingFlexibleWorkGang("Parallel CMS Threads",
655 ConcGCThreads, true);
656 if (_conc_workers == NULL) {
657 warning("GC/CMS: _conc_workers allocation failure: "
658 "forcing -CMSConcurrentMTEnabled");
659 CMSConcurrentMTEnabled = false;
660 } else {
661 _conc_workers->initialize_workers();
662 }
663 } else {
664 CMSConcurrentMTEnabled = false;
665 }
666 }
667 if (!CMSConcurrentMTEnabled) {
668 ConcGCThreads = 0;
669 } else {
670 // Turn off CMSCleanOnEnter optimization temporarily for
671 // the MT case where it's not fixed yet; see 6178663.
672 CMSCleanOnEnter = false;
673 }
674 assert((_conc_workers != NULL) == (ConcGCThreads > 1),
675 "Inconsistency");
676
677 // Parallel task queues; these are shared for the
678 // concurrent and stop-world phases of CMS, but
679 // are not shared with parallel scavenge (ParNew).
680 {
681 uint i;
682 uint num_queues = (uint) MAX2(ParallelGCThreads, ConcGCThreads);
683
684 if ((CMSParallelRemarkEnabled || CMSConcurrentMTEnabled
685 || ParallelRefProcEnabled)
686 && num_queues > 0) {
687 _task_queues = new OopTaskQueueSet(num_queues);
688 if (_task_queues == NULL) {
689 warning("task_queues allocation failure.");
690 return;
691 }
692 _hash_seed = NEW_C_HEAP_ARRAY(int, num_queues);
693 if (_hash_seed == NULL) {
694 warning("_hash_seed array allocation failure");
695 return;
696 }
697
698 typedef Padded<OopTaskQueue> PaddedOopTaskQueue;
699 for (i = 0; i < num_queues; i++) {
700 PaddedOopTaskQueue *q = new PaddedOopTaskQueue();
701 if (q == NULL) {
702 warning("work_queue allocation failure.");
703 return;
704 }
705 _task_queues->register_queue(i, q);
706 }
707 for (i = 0; i < num_queues; i++) {
708 _task_queues->queue(i)->initialize();
709 _hash_seed[i] = 17; // copied from ParNew
710 }
711 }
712 }
713
714 _cmsGen ->init_initiating_occupancy(CMSInitiatingOccupancyFraction, CMSTriggerRatio);
715 _permGen->init_initiating_occupancy(CMSInitiatingPermOccupancyFraction, CMSTriggerPermRatio);
716
717 // Clip CMSBootstrapOccupancy between 0 and 100.
718 _bootstrap_occupancy = ((double)MIN2((uintx)100, MAX2((uintx)0, CMSBootstrapOccupancy)))
719 /(double)100;
720
721 _full_gcs_since_conc_gc = 0;
722
723 // Now tell CMS generations the identity of their collector
724 ConcurrentMarkSweepGeneration::set_collector(this);
725
726 // Create & start a CMS thread for this CMS collector
727 _cmsThread = ConcurrentMarkSweepThread::start(this);
728 assert(cmsThread() != NULL, "CMS Thread should have been created");
729 assert(cmsThread()->collector() == this,
730 "CMS Thread should refer to this gen");
731 assert(CGC_lock != NULL, "Where's the CGC_lock?");
732
733 // Support for parallelizing young gen rescan
734 GenCollectedHeap* gch = GenCollectedHeap::heap();
735 _young_gen = gch->prev_gen(_cmsGen);
736 if (gch->supports_inline_contig_alloc()) {
737 _top_addr = gch->top_addr();
738 _end_addr = gch->end_addr();
739 assert(_young_gen != NULL, "no _young_gen");
740 _eden_chunk_index = 0;
741 _eden_chunk_capacity = (_young_gen->max_capacity()+CMSSamplingGrain)/CMSSamplingGrain;
742 _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity);
743 if (_eden_chunk_array == NULL) {
744 _eden_chunk_capacity = 0;
745 warning("GC/CMS: _eden_chunk_array allocation failure");
746 }
747 }
748 assert(_eden_chunk_array != NULL || _eden_chunk_capacity == 0, "Error");
749
750 // Support for parallelizing survivor space rescan
751 if (CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) {
752 const size_t max_plab_samples =
753 ((DefNewGeneration*)_young_gen)->max_survivor_size()/MinTLABSize;
754
755 _survivor_plab_array = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads);
756 _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, 2*max_plab_samples);
757 _cursor = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads);
758 if (_survivor_plab_array == NULL || _survivor_chunk_array == NULL
759 || _cursor == NULL) {
760 warning("Failed to allocate survivor plab/chunk array");
761 if (_survivor_plab_array != NULL) {
762 FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array);
763 _survivor_plab_array = NULL;
764 }
765 if (_survivor_chunk_array != NULL) {
766 FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array);
767 _survivor_chunk_array = NULL;
768 }
769 if (_cursor != NULL) {
770 FREE_C_HEAP_ARRAY(size_t, _cursor);
771 _cursor = NULL;
772 }
773 } else {
774 _survivor_chunk_capacity = 2*max_plab_samples;
775 for (uint i = 0; i < ParallelGCThreads; i++) {
776 HeapWord** vec = NEW_C_HEAP_ARRAY(HeapWord*, max_plab_samples);
777 if (vec == NULL) {
778 warning("Failed to allocate survivor plab array");
779 for (int j = i; j > 0; j--) {
780 FREE_C_HEAP_ARRAY(HeapWord*, _survivor_plab_array[j-1].array());
781 }
782 FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array);
783 FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array);
784 _survivor_plab_array = NULL;
785 _survivor_chunk_array = NULL;
786 _survivor_chunk_capacity = 0;
787 break;
788 } else {
789 ChunkArray* cur =
790 ::new (&_survivor_plab_array[i]) ChunkArray(vec,
791 max_plab_samples);
792 assert(cur->end() == 0, "Should be 0");
793 assert(cur->array() == vec, "Should be vec");
794 assert(cur->capacity() == max_plab_samples, "Error");
795 }
796 }
797 }
798 }
799 assert( ( _survivor_plab_array != NULL
800 && _survivor_chunk_array != NULL)
801 || ( _survivor_chunk_capacity == 0
802 && _survivor_chunk_index == 0),
803 "Error");
804
805 // Choose what strong roots should be scanned depending on verification options
806 // and perm gen collection mode.
807 if (!CMSClassUnloadingEnabled) {
808 // If class unloading is disabled we want to include all classes into the root set.
809 add_root_scanning_option(SharedHeap::SO_AllClasses);
810 } else {
811 add_root_scanning_option(SharedHeap::SO_SystemClasses);
812 }
813
814 NOT_PRODUCT(_overflow_counter = CMSMarkStackOverflowInterval;)
815 _gc_counters = new CollectorCounters("CMS", 1);
816 _completed_initialization = true;
817 _inter_sweep_timer.start(); // start of time
818 #ifdef SPARC
819 // Issue a stern warning, but allow use for experimentation and debugging.
820 if (VM_Version::is_sun4v() && UseMemSetInBOT) {
821 assert(!FLAG_IS_DEFAULT(UseMemSetInBOT), "Error");
822 warning("Experimental flag -XX:+UseMemSetInBOT is known to cause instability"
823 " on sun4v; please understand that you are using at your own risk!");
824 }
825 #endif
826 }
827
828 const char* ConcurrentMarkSweepGeneration::name() const {
829 return "concurrent mark-sweep generation";
830 }
831 void ConcurrentMarkSweepGeneration::update_counters() {
832 if (UsePerfData) {
833 _space_counters->update_all();
834 _gen_counters->update_all();
835 }
836 }
837
838 // this is an optimized version of update_counters(). it takes the
839 // used value as a parameter rather than computing it.
840 //
841 void ConcurrentMarkSweepGeneration::update_counters(size_t used) {
842 if (UsePerfData) {
843 _space_counters->update_used(used);
844 _space_counters->update_capacity();
845 _gen_counters->update_all();
846 }
847 }
848
849 void ConcurrentMarkSweepGeneration::print() const {
850 Generation::print();
851 cmsSpace()->print();
852 }
853
854 #ifndef PRODUCT
855 void ConcurrentMarkSweepGeneration::print_statistics() {
856 cmsSpace()->printFLCensus(0);
857 }
858 #endif
859
860 void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) {
861 GenCollectedHeap* gch = GenCollectedHeap::heap();
862 if (PrintGCDetails) {
863 if (Verbose) {
864 gclog_or_tty->print(" [%d %s-%s: "SIZE_FORMAT"("SIZE_FORMAT")]",
865 level(), short_name(), s, used(), capacity());
866 } else {
867 gclog_or_tty->print(" [%d %s-%s: "SIZE_FORMAT"K("SIZE_FORMAT"K)]",
868 level(), short_name(), s, used() / K, capacity() / K);
869 }
870 }
871 if (Verbose) {
872 gclog_or_tty->print(" "SIZE_FORMAT"("SIZE_FORMAT")",
873 gch->used(), gch->capacity());
874 } else {
875 gclog_or_tty->print(" "SIZE_FORMAT"K("SIZE_FORMAT"K)",
876 gch->used() / K, gch->capacity() / K);
877 }
878 }
879
880 size_t
881 ConcurrentMarkSweepGeneration::contiguous_available() const {
882 // dld proposes an improvement in precision here. If the committed
883 // part of the space ends in a free block we should add that to
884 // uncommitted size in the calculation below. Will make this
885 // change later, staying with the approximation below for the
886 // time being. -- ysr.
887 return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());
888 }
889
890 size_t
891 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {
892 return _cmsSpace->max_alloc_in_words() * HeapWordSize;
893 }
894
895 size_t ConcurrentMarkSweepGeneration::max_available() const {
896 return free() + _virtual_space.uncommitted_size();
897 }
898
899 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(
900 size_t max_promotion_in_bytes,
901 bool younger_handles_promotion_failure) const {
902
903 // This is the most conservative test. Full promotion is
904 // guaranteed if this is used. The multiplicative factor is to
905 // account for the worst case "dilatation".
906 double adjusted_max_promo_bytes = _dilatation_factor * max_promotion_in_bytes;
907 if (adjusted_max_promo_bytes > (double)max_uintx) { // larger than size_t
908 adjusted_max_promo_bytes = (double)max_uintx;
909 }
910 bool result = (max_contiguous_available() >= (size_t)adjusted_max_promo_bytes);
911
912 if (younger_handles_promotion_failure && !result) {
913 // Full promotion is not guaranteed because fragmentation
914 // of the cms generation can prevent the full promotion.
915 result = (max_available() >= (size_t)adjusted_max_promo_bytes);
916
917 if (!result) {
918 // With promotion failure handling the test for the ability
919 // to support the promotion does not have to be guaranteed.
920 // Use an average of the amount promoted.
921 result = max_available() >= (size_t)
922 gc_stats()->avg_promoted()->padded_average();
923 if (PrintGC && Verbose && result) {
924 gclog_or_tty->print_cr(
925 "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
926 " max_available: " SIZE_FORMAT
927 " avg_promoted: " SIZE_FORMAT,
928 max_available(), (size_t)
929 gc_stats()->avg_promoted()->padded_average());
930 }
931 } else {
932 if (PrintGC && Verbose) {
933 gclog_or_tty->print_cr(
934 "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
935 " max_available: " SIZE_FORMAT
936 " adj_max_promo_bytes: " SIZE_FORMAT,
937 max_available(), (size_t)adjusted_max_promo_bytes);
938 }
939 }
940 } else {
941 if (PrintGC && Verbose) {
942 gclog_or_tty->print_cr(
943 "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
944 " contiguous_available: " SIZE_FORMAT
945 " adj_max_promo_bytes: " SIZE_FORMAT,
946 max_contiguous_available(), (size_t)adjusted_max_promo_bytes);
947 }
948 }
949 return result;
950 }
951
952 // At a promotion failure dump information on block layout in heap
953 // (cms old generation).
954 void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {
955 if (CMSDumpAtPromotionFailure) {
956 cmsSpace()->dump_at_safepoint_with_locks(collector(), gclog_or_tty);
957 }
958 }
959
960 CompactibleSpace*
961 ConcurrentMarkSweepGeneration::first_compaction_space() const {
962 return _cmsSpace;
963 }
964
965 void ConcurrentMarkSweepGeneration::reset_after_compaction() {
966 // Clear the promotion information. These pointers can be adjusted
967 // along with all the other pointers into the heap but
968 // compaction is expected to be a rare event with
969 // a heap using cms so don't do it without seeing the need.
970 if (CollectedHeap::use_parallel_gc_threads()) {
971 for (uint i = 0; i < ParallelGCThreads; i++) {
972 _par_gc_thread_states[i]->promo.reset();
973 }
974 }
975 }
976
977 void ConcurrentMarkSweepGeneration::space_iterate(SpaceClosure* blk, bool usedOnly) {
978 blk->do_space(_cmsSpace);
979 }
980
981 void ConcurrentMarkSweepGeneration::compute_new_size() {
982 assert_locked_or_safepoint(Heap_lock);
983
984 // If incremental collection failed, we just want to expand
985 // to the limit.
986 if (incremental_collection_failed()) {
987 clear_incremental_collection_failed();
988 grow_to_reserved();
989 return;
990 }
991
992 size_t expand_bytes = 0;
993 double free_percentage = ((double) free()) / capacity();
994 double desired_free_percentage = (double) MinHeapFreeRatio / 100;
995 double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
996
997 // compute expansion delta needed for reaching desired free percentage
998 if (free_percentage < desired_free_percentage) {
999 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
1000 assert(desired_capacity >= capacity(), "invalid expansion size");
1001 expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);
1002 }
1003 if (expand_bytes > 0) {
1004 if (PrintGCDetails && Verbose) {
1005 size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));
1006 gclog_or_tty->print_cr("\nFrom compute_new_size: ");
1007 gclog_or_tty->print_cr(" Free fraction %f", free_percentage);
1008 gclog_or_tty->print_cr(" Desired free fraction %f",
1009 desired_free_percentage);
1010 gclog_or_tty->print_cr(" Maximum free fraction %f",
1011 maximum_free_percentage);
1012 gclog_or_tty->print_cr(" Capactiy "SIZE_FORMAT, capacity()/1000);
1013 gclog_or_tty->print_cr(" Desired capacity "SIZE_FORMAT,
1014 desired_capacity/1000);
1015 int prev_level = level() - 1;
1016 if (prev_level >= 0) {
1017 size_t prev_size = 0;
1018 GenCollectedHeap* gch = GenCollectedHeap::heap();
1019 Generation* prev_gen = gch->_gens[prev_level];
1020 prev_size = prev_gen->capacity();
1021 gclog_or_tty->print_cr(" Younger gen size "SIZE_FORMAT,
1022 prev_size/1000);
1023 }
1024 gclog_or_tty->print_cr(" unsafe_max_alloc_nogc "SIZE_FORMAT,
1025 unsafe_max_alloc_nogc()/1000);
1026 gclog_or_tty->print_cr(" contiguous available "SIZE_FORMAT,
1027 contiguous_available()/1000);
1028 gclog_or_tty->print_cr(" Expand by "SIZE_FORMAT" (bytes)",
1029 expand_bytes);
1030 }
1031 // safe if expansion fails
1032 expand(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);
1033 if (PrintGCDetails && Verbose) {
1034 gclog_or_tty->print_cr(" Expanded free fraction %f",
1035 ((double) free()) / capacity());
1036 }
1037 }
1038 }
1039
1040 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {
1041 return cmsSpace()->freelistLock();
1042 }
1043
1044 HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size,
1045 bool tlab) {
1046 CMSSynchronousYieldRequest yr;
1047 MutexLockerEx x(freelistLock(),
1048 Mutex::_no_safepoint_check_flag);
1049 return have_lock_and_allocate(size, tlab);
1050 }
1051
1052 HeapWord* ConcurrentMarkSweepGeneration::have_lock_and_allocate(size_t size,
1053 bool tlab /* ignored */) {
1054 assert_lock_strong(freelistLock());
1055 size_t adjustedSize = CompactibleFreeListSpace::adjustObjectSize(size);
1056 HeapWord* res = cmsSpace()->allocate(adjustedSize);
1057 // Allocate the object live (grey) if the background collector has
1058 // started marking. This is necessary because the marker may
1059 // have passed this address and consequently this object will
1060 // not otherwise be greyed and would be incorrectly swept up.
1061 // Note that if this object contains references, the writing
1062 // of those references will dirty the card containing this object
1063 // allowing the object to be blackened (and its references scanned)
1064 // either during a preclean phase or at the final checkpoint.
1065 if (res != NULL) {
1066 // We may block here with an uninitialized object with
1067 // its mark-bit or P-bits not yet set. Such objects need
1068 // to be safely navigable by block_start().
1069 assert(oop(res)->klass_or_null() == NULL, "Object should be uninitialized here.");
1070 assert(!((FreeChunk*)res)->isFree(), "Error, block will look free but show wrong size");
1071 collector()->direct_allocated(res, adjustedSize);
1072 _direct_allocated_words += adjustedSize;
1073 // allocation counters
1074 NOT_PRODUCT(
1075 _numObjectsAllocated++;
1076 _numWordsAllocated += (int)adjustedSize;
1077 )
1078 }
1079 return res;
1080 }
1081
1082 // In the case of direct allocation by mutators in a generation that
1083 // is being concurrently collected, the object must be allocated
1084 // live (grey) if the background collector has started marking.
1085 // This is necessary because the marker may
1086 // have passed this address and consequently this object will
1087 // not otherwise be greyed and would be incorrectly swept up.
1088 // Note that if this object contains references, the writing
1089 // of those references will dirty the card containing this object
1090 // allowing the object to be blackened (and its references scanned)
1091 // either during a preclean phase or at the final checkpoint.
1092 void CMSCollector::direct_allocated(HeapWord* start, size_t size) {
1093 assert(_markBitMap.covers(start, size), "Out of bounds");
1094 if (_collectorState >= Marking) {
1095 MutexLockerEx y(_markBitMap.lock(),
1096 Mutex::_no_safepoint_check_flag);
1097 // [see comments preceding SweepClosure::do_blk() below for details]
1098 // 1. need to mark the object as live so it isn't collected
1099 // 2. need to mark the 2nd bit to indicate the object may be uninitialized
1100 // 3. need to mark the end of the object so marking, precleaning or sweeping
1101 // can skip over uninitialized or unparsable objects. An allocated
1102 // object is considered uninitialized for our purposes as long as
1103 // its klass word is NULL. (Unparsable objects are those which are
1104 // initialized in the sense just described, but whose sizes can still
1105 // not be correctly determined. Note that the class of unparsable objects
1106 // can only occur in the perm gen. All old gen objects are parsable
1107 // as soon as they are initialized.)
1108 _markBitMap.mark(start); // object is live
1109 _markBitMap.mark(start + 1); // object is potentially uninitialized?
1110 _markBitMap.mark(start + size - 1);
1111 // mark end of object
1112 }
1113 // check that oop looks uninitialized
1114 assert(oop(start)->klass_or_null() == NULL, "_klass should be NULL");
1115 }
1116
1117 void CMSCollector::promoted(bool par, HeapWord* start,
1118 bool is_obj_array, size_t obj_size) {
1119 assert(_markBitMap.covers(start), "Out of bounds");
1120 // See comment in direct_allocated() about when objects should
1121 // be allocated live.
1122 if (_collectorState >= Marking) {
1123 // we already hold the marking bit map lock, taken in
1124 // the prologue
1125 if (par) {
1126 _markBitMap.par_mark(start);
1127 } else {
1128 _markBitMap.mark(start);
1129 }
1130 // We don't need to mark the object as uninitialized (as
1131 // in direct_allocated above) because this is being done with the
1132 // world stopped and the object will be initialized by the
1133 // time the marking, precleaning or sweeping get to look at it.
1134 // But see the code for copying objects into the CMS generation,
1135 // where we need to ensure that concurrent readers of the
1136 // block offset table are able to safely navigate a block that
1137 // is in flux from being free to being allocated (and in
1138 // transition while being copied into) and subsequently
1139 // becoming a bona-fide object when the copy/promotion is complete.
1140 assert(SafepointSynchronize::is_at_safepoint(),
1141 "expect promotion only at safepoints");
1142
1143 if (_collectorState < Sweeping) {
1144 // Mark the appropriate cards in the modUnionTable, so that
1145 // this object gets scanned before the sweep. If this is
1146 // not done, CMS generation references in the object might
1147 // not get marked.
1148 // For the case of arrays, which are otherwise precisely
1149 // marked, we need to dirty the entire array, not just its head.
1150 if (is_obj_array) {
1151 // The [par_]mark_range() method expects mr.end() below to
1152 // be aligned to the granularity of a bit's representation
1153 // in the heap. In the case of the MUT below, that's a
1154 // card size.
1155 MemRegion mr(start,
1156 (HeapWord*)round_to((intptr_t)(start + obj_size),
1157 CardTableModRefBS::card_size /* bytes */));
1158 if (par) {
1159 _modUnionTable.par_mark_range(mr);
1160 } else {
1161 _modUnionTable.mark_range(mr);
1162 }
1163 } else { // not an obj array; we can just mark the head
1164 if (par) {
1165 _modUnionTable.par_mark(start);
1166 } else {
1167 _modUnionTable.mark(start);
1168 }
1169 }
1170 }
1171 }
1172 }
1173
1174 static inline size_t percent_of_space(Space* space, HeapWord* addr)
1175 {
1176 size_t delta = pointer_delta(addr, space->bottom());
1177 return (size_t)(delta * 100.0 / (space->capacity() / HeapWordSize));
1178 }
1179
1180 void CMSCollector::icms_update_allocation_limits()
1181 {
1182 Generation* gen0 = GenCollectedHeap::heap()->get_gen(0);
1183 EdenSpace* eden = gen0->as_DefNewGeneration()->eden();
1184
1185 const unsigned int duty_cycle = stats().icms_update_duty_cycle();
1186 if (CMSTraceIncrementalPacing) {
1187 stats().print();
1188 }
1189
1190 assert(duty_cycle <= 100, "invalid duty cycle");
1191 if (duty_cycle != 0) {
1192 // The duty_cycle is a percentage between 0 and 100; convert to words and
1193 // then compute the offset from the endpoints of the space.
1194 size_t free_words = eden->free() / HeapWordSize;
1195 double free_words_dbl = (double)free_words;
1196 size_t duty_cycle_words = (size_t)(free_words_dbl * duty_cycle / 100.0);
1197 size_t offset_words = (free_words - duty_cycle_words) / 2;
1198
1199 _icms_start_limit = eden->top() + offset_words;
1200 _icms_stop_limit = eden->end() - offset_words;
1201
1202 // The limits may be adjusted (shifted to the right) by
1203 // CMSIncrementalOffset, to allow the application more mutator time after a
1204 // young gen gc (when all mutators were stopped) and before CMS starts and
1205 // takes away one or more cpus.
1206 if (CMSIncrementalOffset != 0) {
1207 double adjustment_dbl = free_words_dbl * CMSIncrementalOffset / 100.0;
1208 size_t adjustment = (size_t)adjustment_dbl;
1209 HeapWord* tmp_stop = _icms_stop_limit + adjustment;
1210 if (tmp_stop > _icms_stop_limit && tmp_stop < eden->end()) {
1211 _icms_start_limit += adjustment;
1212 _icms_stop_limit = tmp_stop;
1213 }
1214 }
1215 }
1216 if (duty_cycle == 0 || (_icms_start_limit == _icms_stop_limit)) {
1217 _icms_start_limit = _icms_stop_limit = eden->end();
1218 }
1219
1220 // Install the new start limit.
1221 eden->set_soft_end(_icms_start_limit);
1222
1223 if (CMSTraceIncrementalMode) {
1224 gclog_or_tty->print(" icms alloc limits: "
1225 PTR_FORMAT "," PTR_FORMAT
1226 " (" SIZE_FORMAT "%%," SIZE_FORMAT "%%) ",
1227 _icms_start_limit, _icms_stop_limit,
1228 percent_of_space(eden, _icms_start_limit),
1229 percent_of_space(eden, _icms_stop_limit));
1230 if (Verbose) {
1231 gclog_or_tty->print("eden: ");
1232 eden->print_on(gclog_or_tty);
1233 }
1234 }
1235 }
1236
1237 // Any changes here should try to maintain the invariant
1238 // that if this method is called with _icms_start_limit
1239 // and _icms_stop_limit both NULL, then it should return NULL
1240 // and not notify the icms thread.
1241 HeapWord*
1242 CMSCollector::allocation_limit_reached(Space* space, HeapWord* top,
1243 size_t word_size)
1244 {
1245 // A start_limit equal to end() means the duty cycle is 0, so treat that as a
1246 // nop.
1247 if (CMSIncrementalMode && _icms_start_limit != space->end()) {
1248 if (top <= _icms_start_limit) {
1249 if (CMSTraceIncrementalMode) {
1250 space->print_on(gclog_or_tty);
1251 gclog_or_tty->stamp();
1252 gclog_or_tty->print_cr(" start limit top=" PTR_FORMAT
1253 ", new limit=" PTR_FORMAT
1254 " (" SIZE_FORMAT "%%)",
1255 top, _icms_stop_limit,
1256 percent_of_space(space, _icms_stop_limit));
1257 }
1258 ConcurrentMarkSweepThread::start_icms();
1259 assert(top < _icms_stop_limit, "Tautology");
1260 if (word_size < pointer_delta(_icms_stop_limit, top)) {
1261 return _icms_stop_limit;
1262 }
1263
1264 // The allocation will cross both the _start and _stop limits, so do the
1265 // stop notification also and return end().
1266 if (CMSTraceIncrementalMode) {
1267 space->print_on(gclog_or_tty);
1268 gclog_or_tty->stamp();
1269 gclog_or_tty->print_cr(" +stop limit top=" PTR_FORMAT
1270 ", new limit=" PTR_FORMAT
1271 " (" SIZE_FORMAT "%%)",
1272 top, space->end(),
1273 percent_of_space(space, space->end()));
1274 }
1275 ConcurrentMarkSweepThread::stop_icms();
1276 return space->end();
1277 }
1278
1279 if (top <= _icms_stop_limit) {
1280 if (CMSTraceIncrementalMode) {
1281 space->print_on(gclog_or_tty);
1282 gclog_or_tty->stamp();
1283 gclog_or_tty->print_cr(" stop limit top=" PTR_FORMAT
1284 ", new limit=" PTR_FORMAT
1285 " (" SIZE_FORMAT "%%)",
1286 top, space->end(),
1287 percent_of_space(space, space->end()));
1288 }
1289 ConcurrentMarkSweepThread::stop_icms();
1290 return space->end();
1291 }
1292
1293 if (CMSTraceIncrementalMode) {
1294 space->print_on(gclog_or_tty);
1295 gclog_or_tty->stamp();
1296 gclog_or_tty->print_cr(" end limit top=" PTR_FORMAT
1297 ", new limit=" PTR_FORMAT,
1298 top, NULL);
1299 }
1300 }
1301
1302 return NULL;
1303 }
1304
1305 oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) {
1306 assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");
1307 // allocate, copy and if necessary update promoinfo --
1308 // delegate to underlying space.
1309 assert_lock_strong(freelistLock());
1310
1311 #ifndef PRODUCT
1312 if (Universe::heap()->promotion_should_fail()) {
1313 return NULL;
1314 }
1315 #endif // #ifndef PRODUCT
1316
1317 oop res = _cmsSpace->promote(obj, obj_size);
1318 if (res == NULL) {
1319 // expand and retry
1320 size_t s = _cmsSpace->expansionSpaceRequired(obj_size); // HeapWords
1321 expand(s*HeapWordSize, MinHeapDeltaBytes,
1322 CMSExpansionCause::_satisfy_promotion);
1323 // Since there's currently no next generation, we don't try to promote
1324 // into a more senior generation.
1325 assert(next_gen() == NULL, "assumption, based upon which no attempt "
1326 "is made to pass on a possibly failing "
1327 "promotion to next generation");
1328 res = _cmsSpace->promote(obj, obj_size);
1329 }
1330 if (res != NULL) {
1331 // See comment in allocate() about when objects should
1332 // be allocated live.
1333 assert(obj->is_oop(), "Will dereference klass pointer below");
1334 collector()->promoted(false, // Not parallel
1335 (HeapWord*)res, obj->is_objArray(), obj_size);
1336 // promotion counters
1337 NOT_PRODUCT(
1338 _numObjectsPromoted++;
1339 _numWordsPromoted +=
1340 (int)(CompactibleFreeListSpace::adjustObjectSize(obj->size()));
1341 )
1342 }
1343 return res;
1344 }
1345
1346
1347 HeapWord*
1348 ConcurrentMarkSweepGeneration::allocation_limit_reached(Space* space,
1349 HeapWord* top,
1350 size_t word_sz)
1351 {
1352 return collector()->allocation_limit_reached(space, top, word_sz);
1353 }
1354
1355 // IMPORTANT: Notes on object size recognition in CMS.
1356 // ---------------------------------------------------
1357 // A block of storage in the CMS generation is always in
1358 // one of three states. A free block (FREE), an allocated
1359 // object (OBJECT) whose size() method reports the correct size,
1360 // and an intermediate state (TRANSIENT) in which its size cannot
1361 // be accurately determined.
1362 // STATE IDENTIFICATION: (32 bit and 64 bit w/o COOPS)
1363 // -----------------------------------------------------
1364 // FREE: klass_word & 1 == 1; mark_word holds block size
1365 //
1366 // OBJECT: klass_word installed; klass_word != 0 && klass_word & 1 == 0;
1367 // obj->size() computes correct size
1368 // [Perm Gen objects needs to be "parsable" before they can be navigated]
1369 //
1370 // TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT
1371 //
1372 // STATE IDENTIFICATION: (64 bit+COOPS)
1373 // ------------------------------------
1374 // FREE: mark_word & CMS_FREE_BIT == 1; mark_word & ~CMS_FREE_BIT gives block_size
1375 //
1376 // OBJECT: klass_word installed; klass_word != 0;
1377 // obj->size() computes correct size
1378 // [Perm Gen comment above continues to hold]
1379 //
1380 // TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT
1381 //
1382 //
1383 // STATE TRANSITION DIAGRAM
1384 //
1385 // mut / parnew mut / parnew
1386 // FREE --------------------> TRANSIENT ---------------------> OBJECT --|
1387 // ^ |
1388 // |------------------------ DEAD <------------------------------------|
1389 // sweep mut
1390 //
1391 // While a block is in TRANSIENT state its size cannot be determined
1392 // so readers will either need to come back later or stall until
1393 // the size can be determined. Note that for the case of direct
1394 // allocation, P-bits, when available, may be used to determine the
1395 // size of an object that may not yet have been initialized.
1396
1397 // Things to support parallel young-gen collection.
1398 oop
1399 ConcurrentMarkSweepGeneration::par_promote(int thread_num,
1400 oop old, markOop m,
1401 size_t word_sz) {
1402 #ifndef PRODUCT
1403 if (Universe::heap()->promotion_should_fail()) {
1404 return NULL;
1405 }
1406 #endif // #ifndef PRODUCT
1407
1408 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1409 PromotionInfo* promoInfo = &ps->promo;
1410 // if we are tracking promotions, then first ensure space for
1411 // promotion (including spooling space for saving header if necessary).
1412 // then allocate and copy, then track promoted info if needed.
1413 // When tracking (see PromotionInfo::track()), the mark word may
1414 // be displaced and in this case restoration of the mark word
1415 // occurs in the (oop_since_save_marks_)iterate phase.
1416 if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {
1417 // Out of space for allocating spooling buffers;
1418 // try expanding and allocating spooling buffers.
1419 if (!expand_and_ensure_spooling_space(promoInfo)) {
1420 return NULL;
1421 }
1422 }
1423 assert(promoInfo->has_spooling_space(), "Control point invariant");
1424 const size_t alloc_sz = CompactibleFreeListSpace::adjustObjectSize(word_sz);
1425 HeapWord* obj_ptr = ps->lab.alloc(alloc_sz);
1426 if (obj_ptr == NULL) {
1427 obj_ptr = expand_and_par_lab_allocate(ps, alloc_sz);
1428 if (obj_ptr == NULL) {
1429 return NULL;
1430 }
1431 }
1432 oop obj = oop(obj_ptr);
1433 OrderAccess::storestore();
1434 assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
1435 assert(!((FreeChunk*)obj_ptr)->isFree(), "Error, block will look free but show wrong size");
1436 // IMPORTANT: See note on object initialization for CMS above.
1437 // Otherwise, copy the object. Here we must be careful to insert the
1438 // klass pointer last, since this marks the block as an allocated object.
1439 // Except with compressed oops it's the mark word.
1440 HeapWord* old_ptr = (HeapWord*)old;
1441 // Restore the mark word copied above.
1442 obj->set_mark(m);
1443 assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
1444 assert(!((FreeChunk*)obj_ptr)->isFree(), "Error, block will look free but show wrong size");
1445 OrderAccess::storestore();
1446
1447 if (UseCompressedOops) {
1448 // Copy gap missed by (aligned) header size calculation below
1449 obj->set_klass_gap(old->klass_gap());
1450 }
1451 if (word_sz > (size_t)oopDesc::header_size()) {
1452 Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),
1453 obj_ptr + oopDesc::header_size(),
1454 word_sz - oopDesc::header_size());
1455 }
1456
1457 // Now we can track the promoted object, if necessary. We take care
1458 // to delay the transition from uninitialized to full object
1459 // (i.e., insertion of klass pointer) until after, so that it
1460 // atomically becomes a promoted object.
1461 if (promoInfo->tracking()) {
1462 promoInfo->track((PromotedObject*)obj, old->klass());
1463 }
1464 assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
1465 assert(!((FreeChunk*)obj_ptr)->isFree(), "Error, block will look free but show wrong size");
1466 assert(old->is_oop(), "Will use and dereference old klass ptr below");
1467
1468 // Finally, install the klass pointer (this should be volatile).
1469 OrderAccess::storestore();
1470 obj->set_klass(old->klass());
1471 // We should now be able to calculate the right size for this object
1472 assert(obj->is_oop() && obj->size() == (int)word_sz, "Error, incorrect size computed for promoted object");
1473
1474 collector()->promoted(true, // parallel
1475 obj_ptr, old->is_objArray(), word_sz);
1476
1477 NOT_PRODUCT(
1478 Atomic::inc_ptr(&_numObjectsPromoted);
1479 Atomic::add_ptr(alloc_sz, &_numWordsPromoted);
1480 )
1481
1482 return obj;
1483 }
1484
1485 void
1486 ConcurrentMarkSweepGeneration::
1487 par_promote_alloc_undo(int thread_num,
1488 HeapWord* obj, size_t word_sz) {
1489 // CMS does not support promotion undo.
1490 ShouldNotReachHere();
1491 }
1492
1493 void
1494 ConcurrentMarkSweepGeneration::
1495 par_promote_alloc_done(int thread_num) {
1496 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1497 ps->lab.retire(thread_num);
1498 }
1499
1500 void
1501 ConcurrentMarkSweepGeneration::
1502 par_oop_since_save_marks_iterate_done(int thread_num) {
1503 CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];
1504 ParScanWithoutBarrierClosure* dummy_cl = NULL;
1505 ps->promo.promoted_oops_iterate_nv(dummy_cl);
1506 }
1507
1508 // XXXPERM
1509 bool ConcurrentMarkSweepGeneration::should_collect(bool full,
1510 size_t size,
1511 bool tlab)
1512 {
1513 // We allow a STW collection only if a full
1514 // collection was requested.
1515 return full || should_allocate(size, tlab); // FIX ME !!!
1516 // This and promotion failure handling are connected at the
1517 // hip and should be fixed by untying them.
1518 }
1519
1520 bool CMSCollector::shouldConcurrentCollect() {
1521 if (_full_gc_requested) {
1522 if (Verbose && PrintGCDetails) {
1523 gclog_or_tty->print_cr("CMSCollector: collect because of explicit "
1524 " gc request (or gc_locker)");
1525 }
1526 return true;
1527 }
1528
1529 // For debugging purposes, change the type of collection.
1530 // If the rotation is not on the concurrent collection
1531 // type, don't start a concurrent collection.
1532 NOT_PRODUCT(
1533 if (RotateCMSCollectionTypes &&
1534 (_cmsGen->debug_collection_type() !=
1535 ConcurrentMarkSweepGeneration::Concurrent_collection_type)) {
1536 assert(_cmsGen->debug_collection_type() !=
1537 ConcurrentMarkSweepGeneration::Unknown_collection_type,
1538 "Bad cms collection type");
1539 return false;
1540 }
1541 )
1542
1543 FreelistLocker x(this);
1544 // ------------------------------------------------------------------
1545 // Print out lots of information which affects the initiation of
1546 // a collection.
1547 if (PrintCMSInitiationStatistics && stats().valid()) {
1548 gclog_or_tty->print("CMSCollector shouldConcurrentCollect: ");
1549 gclog_or_tty->stamp();
1550 gclog_or_tty->print_cr("");
1551 stats().print_on(gclog_or_tty);
1552 gclog_or_tty->print_cr("time_until_cms_gen_full %3.7f",
1553 stats().time_until_cms_gen_full());
1554 gclog_or_tty->print_cr("free="SIZE_FORMAT, _cmsGen->free());
1555 gclog_or_tty->print_cr("contiguous_available="SIZE_FORMAT,
1556 _cmsGen->contiguous_available());
1557 gclog_or_tty->print_cr("promotion_rate=%g", stats().promotion_rate());
1558 gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate());
1559 gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy());
1560 gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy());
1561 gclog_or_tty->print_cr("initiatingPermOccupancy=%3.7f", _permGen->initiating_occupancy());
1562 }
1563 // ------------------------------------------------------------------
1564
1565 // If the estimated time to complete a cms collection (cms_duration())
1566 // is less than the estimated time remaining until the cms generation
1567 // is full, start a collection.
1568 if (!UseCMSInitiatingOccupancyOnly) {
1569 if (stats().valid()) {
1570 if (stats().time_until_cms_start() == 0.0) {
1571 return true;
1572 }
1573 } else {
1574 // We want to conservatively collect somewhat early in order
1575 // to try and "bootstrap" our CMS/promotion statistics;
1576 // this branch will not fire after the first successful CMS
1577 // collection because the stats should then be valid.
1578 if (_cmsGen->occupancy() >= _bootstrap_occupancy) {
1579 if (Verbose && PrintGCDetails) {
1580 gclog_or_tty->print_cr(
1581 " CMSCollector: collect for bootstrapping statistics:"
1582 " occupancy = %f, boot occupancy = %f", _cmsGen->occupancy(),
1583 _bootstrap_occupancy);
1584 }
1585 return true;
1586 }
1587 }
1588 }
1589
1590 // Otherwise, we start a collection cycle if either the perm gen or
1591 // old gen want a collection cycle started. Each may use
1592 // an appropriate criterion for making this decision.
1593 // XXX We need to make sure that the gen expansion
1594 // criterion dovetails well with this. XXX NEED TO FIX THIS
1595 if (_cmsGen->should_concurrent_collect()) {
1596 if (Verbose && PrintGCDetails) {
1597 gclog_or_tty->print_cr("CMS old gen initiated");
1598 }
1599 return true;
1600 }
1601
1602 // We start a collection if we believe an incremental collection may fail;
1603 // this is not likely to be productive in practice because it's probably too
1604 // late anyway.
1605 GenCollectedHeap* gch = GenCollectedHeap::heap();
1606 assert(gch->collector_policy()->is_two_generation_policy(),
1607 "You may want to check the correctness of the following");
1608 if (gch->incremental_collection_will_fail()) {
1609 if (PrintGCDetails && Verbose) {
1610 gclog_or_tty->print("CMSCollector: collect because incremental collection will fail ");
1611 }
1612 return true;
1613 }
1614
1615 if (CMSClassUnloadingEnabled && _permGen->should_concurrent_collect()) {
1616 bool res = update_should_unload_classes();
1617 if (res) {
1618 if (Verbose && PrintGCDetails) {
1619 gclog_or_tty->print_cr("CMS perm gen initiated");
1620 }
1621 return true;
1622 }
1623 }
1624 return false;
1625 }
1626
1627 // Clear _expansion_cause fields of constituent generations
1628 void CMSCollector::clear_expansion_cause() {
1629 _cmsGen->clear_expansion_cause();
1630 _permGen->clear_expansion_cause();
1631 }
1632
1633 // We should be conservative in starting a collection cycle. To
1634 // start too eagerly runs the risk of collecting too often in the
1635 // extreme. To collect too rarely falls back on full collections,
1636 // which works, even if not optimum in terms of concurrent work.
1637 // As a work around for too eagerly collecting, use the flag
1638 // UseCMSInitiatingOccupancyOnly. This also has the advantage of
1639 // giving the user an easily understandable way of controlling the
1640 // collections.
1641 // We want to start a new collection cycle if any of the following
1642 // conditions hold:
1643 // . our current occupancy exceeds the configured initiating occupancy
1644 // for this generation, or
1645 // . we recently needed to expand this space and have not, since that
1646 // expansion, done a collection of this generation, or
1647 // . the underlying space believes that it may be a good idea to initiate
1648 // a concurrent collection (this may be based on criteria such as the
1649 // following: the space uses linear allocation and linear allocation is
1650 // going to fail, or there is believed to be excessive fragmentation in
1651 // the generation, etc... or ...
1652 // [.(currently done by CMSCollector::shouldConcurrentCollect() only for
1653 // the case of the old generation, not the perm generation; see CR 6543076):
1654 // we may be approaching a point at which allocation requests may fail because
1655 // we will be out of sufficient free space given allocation rate estimates.]
1656 bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const {
1657
1658 assert_lock_strong(freelistLock());
1659 if (occupancy() > initiating_occupancy()) {
1660 if (PrintGCDetails && Verbose) {
1661 gclog_or_tty->print(" %s: collect because of occupancy %f / %f ",
1662 short_name(), occupancy(), initiating_occupancy());
1663 }
1664 return true;
1665 }
1666 if (UseCMSInitiatingOccupancyOnly) {
1667 return false;
1668 }
1669 if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) {
1670 if (PrintGCDetails && Verbose) {
1671 gclog_or_tty->print(" %s: collect because expanded for allocation ",
1672 short_name());
1673 }
1674 return true;
1675 }
1676 if (_cmsSpace->should_concurrent_collect()) {
1677 if (PrintGCDetails && Verbose) {
1678 gclog_or_tty->print(" %s: collect because cmsSpace says so ",
1679 short_name());
1680 }
1681 return true;
1682 }
1683 return false;
1684 }
1685
1686 void ConcurrentMarkSweepGeneration::collect(bool full,
1687 bool clear_all_soft_refs,
1688 size_t size,
1689 bool tlab)
1690 {
1691 collector()->collect(full, clear_all_soft_refs, size, tlab);
1692 }
1693
1694 void CMSCollector::collect(bool full,
1695 bool clear_all_soft_refs,
1696 size_t size,
1697 bool tlab)
1698 {
1699 if (!UseCMSCollectionPassing && _collectorState > Idling) {
1700 // For debugging purposes skip the collection if the state
1701 // is not currently idle
1702 if (TraceCMSState) {
1703 gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " skipped full:%d CMS state %d",
1704 Thread::current(), full, _collectorState);
1705 }
1706 return;
1707 }
1708
1709 // The following "if" branch is present for defensive reasons.
1710 // In the current uses of this interface, it can be replaced with:
1711 // assert(!GC_locker.is_active(), "Can't be called otherwise");
1712 // But I am not placing that assert here to allow future
1713 // generality in invoking this interface.
1714 if (GC_locker::is_active()) {
1715 // A consistency test for GC_locker
1716 assert(GC_locker::needs_gc(), "Should have been set already");
1717 // Skip this foreground collection, instead
1718 // expanding the heap if necessary.
1719 // Need the free list locks for the call to free() in compute_new_size()
1720 compute_new_size();
1721 return;
1722 }
1723 acquire_control_and_collect(full, clear_all_soft_refs);
1724 _full_gcs_since_conc_gc++;
1725
1726 }
1727
1728 void CMSCollector::request_full_gc(unsigned int full_gc_count) {
1729 GenCollectedHeap* gch = GenCollectedHeap::heap();
1730 unsigned int gc_count = gch->total_full_collections();
1731 if (gc_count == full_gc_count) {
1732 MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag);
1733 _full_gc_requested = true;
1734 CGC_lock->notify(); // nudge CMS thread
1735 }
1736 }
1737
1738
1739 // The foreground and background collectors need to coordinate in order
1740 // to make sure that they do not mutually interfere with CMS collections.
1741 // When a background collection is active,
1742 // the foreground collector may need to take over (preempt) and
1743 // synchronously complete an ongoing collection. Depending on the
1744 // frequency of the background collections and the heap usage
1745 // of the application, this preemption can be seldom or frequent.
1746 // There are only certain
1747 // points in the background collection that the "collection-baton"
1748 // can be passed to the foreground collector.
1749 //
1750 // The foreground collector will wait for the baton before
1751 // starting any part of the collection. The foreground collector
1752 // will only wait at one location.
1753 //
1754 // The background collector will yield the baton before starting a new
1755 // phase of the collection (e.g., before initial marking, marking from roots,
1756 // precleaning, final re-mark, sweep etc.) This is normally done at the head
1757 // of the loop which switches the phases. The background collector does some
1758 // of the phases (initial mark, final re-mark) with the world stopped.
1759 // Because of locking involved in stopping the world,
1760 // the foreground collector should not block waiting for the background
1761 // collector when it is doing a stop-the-world phase. The background
1762 // collector will yield the baton at an additional point just before
1763 // it enters a stop-the-world phase. Once the world is stopped, the
1764 // background collector checks the phase of the collection. If the
1765 // phase has not changed, it proceeds with the collection. If the
1766 // phase has changed, it skips that phase of the collection. See
1767 // the comments on the use of the Heap_lock in collect_in_background().
1768 //
1769 // Variable used in baton passing.
1770 // _foregroundGCIsActive - Set to true by the foreground collector when
1771 // it wants the baton. The foreground clears it when it has finished
1772 // the collection.
1773 // _foregroundGCShouldWait - Set to true by the background collector
1774 // when it is running. The foreground collector waits while
1775 // _foregroundGCShouldWait is true.
1776 // CGC_lock - monitor used to protect access to the above variables
1777 // and to notify the foreground and background collectors.
1778 // _collectorState - current state of the CMS collection.
1779 //
1780 // The foreground collector
1781 // acquires the CGC_lock
1782 // sets _foregroundGCIsActive
1783 // waits on the CGC_lock for _foregroundGCShouldWait to be false
1784 // various locks acquired in preparation for the collection
1785 // are released so as not to block the background collector
1786 // that is in the midst of a collection
1787 // proceeds with the collection
1788 // clears _foregroundGCIsActive
1789 // returns
1790 //
1791 // The background collector in a loop iterating on the phases of the
1792 // collection
1793 // acquires the CGC_lock
1794 // sets _foregroundGCShouldWait
1795 // if _foregroundGCIsActive is set
1796 // clears _foregroundGCShouldWait, notifies _CGC_lock
1797 // waits on _CGC_lock for _foregroundGCIsActive to become false
1798 // and exits the loop.
1799 // otherwise
1800 // proceed with that phase of the collection
1801 // if the phase is a stop-the-world phase,
1802 // yield the baton once more just before enqueueing
1803 // the stop-world CMS operation (executed by the VM thread).
1804 // returns after all phases of the collection are done
1805 //
1806
1807 void CMSCollector::acquire_control_and_collect(bool full,
1808 bool clear_all_soft_refs) {
1809 assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
1810 assert(!Thread::current()->is_ConcurrentGC_thread(),
1811 "shouldn't try to acquire control from self!");
1812
1813 // Start the protocol for acquiring control of the
1814 // collection from the background collector (aka CMS thread).
1815 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1816 "VM thread should have CMS token");
1817 // Remember the possibly interrupted state of an ongoing
1818 // concurrent collection
1819 CollectorState first_state = _collectorState;
1820
1821 // Signal to a possibly ongoing concurrent collection that
1822 // we want to do a foreground collection.
1823 _foregroundGCIsActive = true;
1824
1825 // Disable incremental mode during a foreground collection.
1826 ICMSDisabler icms_disabler;
1827
1828 // release locks and wait for a notify from the background collector
1829 // releasing the locks in only necessary for phases which
1830 // do yields to improve the granularity of the collection.
1831 assert_lock_strong(bitMapLock());
1832 // We need to lock the Free list lock for the space that we are
1833 // currently collecting.
1834 assert(haveFreelistLocks(), "Must be holding free list locks");
1835 bitMapLock()->unlock();
1836 releaseFreelistLocks();
1837 {
1838 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
1839 if (_foregroundGCShouldWait) {
1840 // We are going to be waiting for action for the CMS thread;
1841 // it had better not be gone (for instance at shutdown)!
1842 assert(ConcurrentMarkSweepThread::cmst() != NULL,
1843 "CMS thread must be running");
1844 // Wait here until the background collector gives us the go-ahead
1845 ConcurrentMarkSweepThread::clear_CMS_flag(
1846 ConcurrentMarkSweepThread::CMS_vm_has_token); // release token
1847 // Get a possibly blocked CMS thread going:
1848 // Note that we set _foregroundGCIsActive true above,
1849 // without protection of the CGC_lock.
1850 CGC_lock->notify();
1851 assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(),
1852 "Possible deadlock");
1853 while (_foregroundGCShouldWait) {
1854 // wait for notification
1855 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
1856 // Possibility of delay/starvation here, since CMS token does
1857 // not know to give priority to VM thread? Actually, i think
1858 // there wouldn't be any delay/starvation, but the proof of
1859 // that "fact" (?) appears non-trivial. XXX 20011219YSR
1860 }
1861 ConcurrentMarkSweepThread::set_CMS_flag(
1862 ConcurrentMarkSweepThread::CMS_vm_has_token);
1863 }
1864 }
1865 // The CMS_token is already held. Get back the other locks.
1866 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
1867 "VM thread should have CMS token");
1868 getFreelistLocks();
1869 bitMapLock()->lock_without_safepoint_check();
1870 if (TraceCMSState) {
1871 gclog_or_tty->print_cr("CMS foreground collector has asked for control "
1872 INTPTR_FORMAT " with first state %d", Thread::current(), first_state);
1873 gclog_or_tty->print_cr(" gets control with state %d", _collectorState);
1874 }
1875
1876 // Check if we need to do a compaction, or if not, whether
1877 // we need to start the mark-sweep from scratch.
1878 bool should_compact = false;
1879 bool should_start_over = false;
1880 decide_foreground_collection_type(clear_all_soft_refs,
1881 &should_compact, &should_start_over);
1882
1883 NOT_PRODUCT(
1884 if (RotateCMSCollectionTypes) {
1885 if (_cmsGen->debug_collection_type() ==
1886 ConcurrentMarkSweepGeneration::MSC_foreground_collection_type) {
1887 should_compact = true;
1888 } else if (_cmsGen->debug_collection_type() ==
1889 ConcurrentMarkSweepGeneration::MS_foreground_collection_type) {
1890 should_compact = false;
1891 }
1892 }
1893 )
1894
1895 if (PrintGCDetails && first_state > Idling) {
1896 GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause();
1897 if (GCCause::is_user_requested_gc(cause) ||
1898 GCCause::is_serviceability_requested_gc(cause)) {
1899 gclog_or_tty->print(" (concurrent mode interrupted)");
1900 } else {
1901 gclog_or_tty->print(" (concurrent mode failure)");
1902 }
1903 }
1904
1905 if (should_compact) {
1906 // If the collection is being acquired from the background
1907 // collector, there may be references on the discovered
1908 // references lists that have NULL referents (being those
1909 // that were concurrently cleared by a mutator) or
1910 // that are no longer active (having been enqueued concurrently
1911 // by the mutator).
1912 // Scrub the list of those references because Mark-Sweep-Compact
1913 // code assumes referents are not NULL and that all discovered
1914 // Reference objects are active.
1915 ref_processor()->clean_up_discovered_references();
1916
1917 do_compaction_work(clear_all_soft_refs);
1918
1919 // Has the GC time limit been exceeded?
1920 DefNewGeneration* young_gen = _young_gen->as_DefNewGeneration();
1921 size_t max_eden_size = young_gen->max_capacity() -
1922 young_gen->to()->capacity() -
1923 young_gen->from()->capacity();
1924 GenCollectedHeap* gch = GenCollectedHeap::heap();
1925 GCCause::Cause gc_cause = gch->gc_cause();
1926 size_policy()->check_gc_overhead_limit(_young_gen->used(),
1927 young_gen->eden()->used(),
1928 _cmsGen->max_capacity(),
1929 max_eden_size,
1930 full,
1931 gc_cause,
1932 gch->collector_policy());
1933 } else {
1934 do_mark_sweep_work(clear_all_soft_refs, first_state,
1935 should_start_over);
1936 }
1937 // Reset the expansion cause, now that we just completed
1938 // a collection cycle.
1939 clear_expansion_cause();
1940 _foregroundGCIsActive = false;
1941 return;
1942 }
1943
1944 // Resize the perm generation and the tenured generation
1945 // after obtaining the free list locks for the
1946 // two generations.
1947 void CMSCollector::compute_new_size() {
1948 assert_locked_or_safepoint(Heap_lock);
1949 FreelistLocker z(this);
1950 _permGen->compute_new_size();
1951 _cmsGen->compute_new_size();
1952 }
1953
1954 // A work method used by foreground collection to determine
1955 // what type of collection (compacting or not, continuing or fresh)
1956 // it should do.
1957 // NOTE: the intent is to make UseCMSCompactAtFullCollection
1958 // and CMSCompactWhenClearAllSoftRefs the default in the future
1959 // and do away with the flags after a suitable period.
1960 void CMSCollector::decide_foreground_collection_type(
1961 bool clear_all_soft_refs, bool* should_compact,
1962 bool* should_start_over) {
1963 // Normally, we'll compact only if the UseCMSCompactAtFullCollection
1964 // flag is set, and we have either requested a System.gc() or
1965 // the number of full gc's since the last concurrent cycle
1966 // has exceeded the threshold set by CMSFullGCsBeforeCompaction,
1967 // or if an incremental collection has failed
1968 GenCollectedHeap* gch = GenCollectedHeap::heap();
1969 assert(gch->collector_policy()->is_two_generation_policy(),
1970 "You may want to check the correctness of the following");
1971 // Inform cms gen if this was due to partial collection failing.
1972 // The CMS gen may use this fact to determine its expansion policy.
1973 if (gch->incremental_collection_will_fail()) {
1974 assert(!_cmsGen->incremental_collection_failed(),
1975 "Should have been noticed, reacted to and cleared");
1976 _cmsGen->set_incremental_collection_failed();
1977 }
1978 *should_compact =
1979 UseCMSCompactAtFullCollection &&
1980 ((_full_gcs_since_conc_gc >= CMSFullGCsBeforeCompaction) ||
1981 GCCause::is_user_requested_gc(gch->gc_cause()) ||
1982 gch->incremental_collection_will_fail());
1983 *should_start_over = false;
1984 if (clear_all_soft_refs && !*should_compact) {
1985 // We are about to do a last ditch collection attempt
1986 // so it would normally make sense to do a compaction
1987 // to reclaim as much space as possible.
1988 if (CMSCompactWhenClearAllSoftRefs) {
1989 // Default: The rationale is that in this case either
1990 // we are past the final marking phase, in which case
1991 // we'd have to start over, or so little has been done
1992 // that there's little point in saving that work. Compaction
1993 // appears to be the sensible choice in either case.
1994 *should_compact = true;
1995 } else {
1996 // We have been asked to clear all soft refs, but not to
1997 // compact. Make sure that we aren't past the final checkpoint
1998 // phase, for that is where we process soft refs. If we are already
1999 // past that phase, we'll need to redo the refs discovery phase and
2000 // if necessary clear soft refs that weren't previously
2001 // cleared. We do so by remembering the phase in which
2002 // we came in, and if we are past the refs processing
2003 // phase, we'll choose to just redo the mark-sweep
2004 // collection from scratch.
2005 if (_collectorState > FinalMarking) {
2006 // We are past the refs processing phase;
2007 // start over and do a fresh synchronous CMS cycle
2008 _collectorState = Resetting; // skip to reset to start new cycle
2009 reset(false /* == !asynch */);
2010 *should_start_over = true;
2011 } // else we can continue a possibly ongoing current cycle
2012 }
2013 }
2014 }
2015
2016 // A work method used by the foreground collector to do
2017 // a mark-sweep-compact.
2018 void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
2019 GenCollectedHeap* gch = GenCollectedHeap::heap();
2020 TraceTime t("CMS:MSC ", PrintGCDetails && Verbose, true, gclog_or_tty);
2021 if (PrintGC && Verbose && !(GCCause::is_user_requested_gc(gch->gc_cause()))) {
2022 gclog_or_tty->print_cr("Compact ConcurrentMarkSweepGeneration after %d "
2023 "collections passed to foreground collector", _full_gcs_since_conc_gc);
2024 }
2025
2026 // Sample collection interval time and reset for collection pause.
2027 if (UseAdaptiveSizePolicy) {
2028 size_policy()->msc_collection_begin();
2029 }
2030
2031 // Temporarily widen the span of the weak reference processing to
2032 // the entire heap.
2033 MemRegion new_span(GenCollectedHeap::heap()->reserved_region());
2034 ReferenceProcessorSpanMutator x(ref_processor(), new_span);
2035
2036 // Temporarily, clear the "is_alive_non_header" field of the
2037 // reference processor.
2038 ReferenceProcessorIsAliveMutator y(ref_processor(), NULL);
2039
2040 // Temporarily make reference _processing_ single threaded (non-MT).
2041 ReferenceProcessorMTProcMutator z(ref_processor(), false);
2042
2043 // Temporarily make refs discovery atomic
2044 ReferenceProcessorAtomicMutator w(ref_processor(), true);
2045
2046 ref_processor()->set_enqueuing_is_done(false);
2047 ref_processor()->enable_discovery();
2048 ref_processor()->setup_policy(clear_all_soft_refs);
2049 // If an asynchronous collection finishes, the _modUnionTable is
2050 // all clear. If we are assuming the collection from an asynchronous
2051 // collection, clear the _modUnionTable.
2052 assert(_collectorState != Idling || _modUnionTable.isAllClear(),
2053 "_modUnionTable should be clear if the baton was not passed");
2054 _modUnionTable.clear_all();
2055
2056 // We must adjust the allocation statistics being maintained
2057 // in the free list space. We do so by reading and clearing
2058 // the sweep timer and updating the block flux rate estimates below.
2059 assert(!_intra_sweep_timer.is_active(), "_intra_sweep_timer should be inactive");
2060 if (_inter_sweep_timer.is_active()) {
2061 _inter_sweep_timer.stop();
2062 // Note that we do not use this sample to update the _inter_sweep_estimate.
2063 _cmsGen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()),
2064 _inter_sweep_estimate.padded_average(),
2065 _intra_sweep_estimate.padded_average());
2066 }
2067
2068 {
2069 TraceCMSMemoryManagerStats();
2070 }
2071 GenMarkSweep::invoke_at_safepoint(_cmsGen->level(),
2072 ref_processor(), clear_all_soft_refs);
2073 #ifdef ASSERT
2074 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
2075 size_t free_size = cms_space->free();
2076 assert(free_size ==
2077 pointer_delta(cms_space->end(), cms_space->compaction_top())
2078 * HeapWordSize,
2079 "All the free space should be compacted into one chunk at top");
2080 assert(cms_space->dictionary()->totalChunkSize(
2081 debug_only(cms_space->freelistLock())) == 0 ||
2082 cms_space->totalSizeInIndexedFreeLists() == 0,
2083 "All the free space should be in a single chunk");
2084 size_t num = cms_space->totalCount();
2085 assert((free_size == 0 && num == 0) ||
2086 (free_size > 0 && (num == 1 || num == 2)),
2087 "There should be at most 2 free chunks after compaction");
2088 #endif // ASSERT
2089 _collectorState = Resetting;
2090 assert(_restart_addr == NULL,
2091 "Should have been NULL'd before baton was passed");
2092 reset(false /* == !asynch */);
2093 _cmsGen->reset_after_compaction();
2094 _concurrent_cycles_since_last_unload = 0;
2095
2096 if (verifying() && !should_unload_classes()) {
2097 perm_gen_verify_bit_map()->clear_all();
2098 }
2099
2100 // Clear any data recorded in the PLAB chunk arrays.
2101 if (_survivor_plab_array != NULL) {
2102 reset_survivor_plab_arrays();
2103 }
2104
2105 // Adjust the per-size allocation stats for the next epoch.
2106 _cmsGen->cmsSpace()->endSweepFLCensus(sweep_count() /* fake */);
2107 // Restart the "inter sweep timer" for the next epoch.
2108 _inter_sweep_timer.reset();
2109 _inter_sweep_timer.start();
2110
2111 // Sample collection pause time and reset for collection interval.
2112 if (UseAdaptiveSizePolicy) {
2113 size_policy()->msc_collection_end(gch->gc_cause());
2114 }
2115
2116 // For a mark-sweep-compact, compute_new_size() will be called
2117 // in the heap's do_collection() method.
2118 }
2119
2120 // A work method used by the foreground collector to do
2121 // a mark-sweep, after taking over from a possibly on-going
2122 // concurrent mark-sweep collection.
2123 void CMSCollector::do_mark_sweep_work(bool clear_all_soft_refs,
2124 CollectorState first_state, bool should_start_over) {
2125 if (PrintGC && Verbose) {
2126 gclog_or_tty->print_cr("Pass concurrent collection to foreground "
2127 "collector with count %d",
2128 _full_gcs_since_conc_gc);
2129 }
2130 switch (_collectorState) {
2131 case Idling:
2132 if (first_state == Idling || should_start_over) {
2133 // The background GC was not active, or should
2134 // restarted from scratch; start the cycle.
2135 _collectorState = InitialMarking;
2136 }
2137 // If first_state was not Idling, then a background GC
2138 // was in progress and has now finished. No need to do it
2139 // again. Leave the state as Idling.
2140 break;
2141 case Precleaning:
2142 // In the foreground case don't do the precleaning since
2143 // it is not done concurrently and there is extra work
2144 // required.
2145 _collectorState = FinalMarking;
2146 }
2147 if (PrintGCDetails &&
2148 (_collectorState > Idling ||
2149 !GCCause::is_user_requested_gc(GenCollectedHeap::heap()->gc_cause()))) {
2150 gclog_or_tty->print(" (concurrent mode failure)");
2151 }
2152 collect_in_foreground(clear_all_soft_refs);
2153
2154 // For a mark-sweep, compute_new_size() will be called
2155 // in the heap's do_collection() method.
2156 }
2157
2158
2159 void CMSCollector::getFreelistLocks() const {
2160 // Get locks for all free lists in all generations that this
2161 // collector is responsible for
2162 _cmsGen->freelistLock()->lock_without_safepoint_check();
2163 _permGen->freelistLock()->lock_without_safepoint_check();
2164 }
2165
2166 void CMSCollector::releaseFreelistLocks() const {
2167 // Release locks for all free lists in all generations that this
2168 // collector is responsible for
2169 _cmsGen->freelistLock()->unlock();
2170 _permGen->freelistLock()->unlock();
2171 }
2172
2173 bool CMSCollector::haveFreelistLocks() const {
2174 // Check locks for all free lists in all generations that this
2175 // collector is responsible for
2176 assert_lock_strong(_cmsGen->freelistLock());
2177 assert_lock_strong(_permGen->freelistLock());
2178 PRODUCT_ONLY(ShouldNotReachHere());
2179 return true;
2180 }
2181
2182 // A utility class that is used by the CMS collector to
2183 // temporarily "release" the foreground collector from its
2184 // usual obligation to wait for the background collector to
2185 // complete an ongoing phase before proceeding.
2186 class ReleaseForegroundGC: public StackObj {
2187 private:
2188 CMSCollector* _c;
2189 public:
2190 ReleaseForegroundGC(CMSCollector* c) : _c(c) {
2191 assert(_c->_foregroundGCShouldWait, "Else should not need to call");
2192 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2193 // allow a potentially blocked foreground collector to proceed
2194 _c->_foregroundGCShouldWait = false;
2195 if (_c->_foregroundGCIsActive) {
2196 CGC_lock->notify();
2197 }
2198 assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
2199 "Possible deadlock");
2200 }
2201
2202 ~ReleaseForegroundGC() {
2203 assert(!_c->_foregroundGCShouldWait, "Usage protocol violation?");
2204 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2205 _c->_foregroundGCShouldWait = true;
2206 }
2207 };
2208
2209 // There are separate collect_in_background and collect_in_foreground because of
2210 // the different locking requirements of the background collector and the
2211 // foreground collector. There was originally an attempt to share
2212 // one "collect" method between the background collector and the foreground
2213 // collector but the if-then-else required made it cleaner to have
2214 // separate methods.
2215 void CMSCollector::collect_in_background(bool clear_all_soft_refs) {
2216 assert(Thread::current()->is_ConcurrentGC_thread(),
2217 "A CMS asynchronous collection is only allowed on a CMS thread.");
2218
2219 GenCollectedHeap* gch = GenCollectedHeap::heap();
2220 {
2221 bool safepoint_check = Mutex::_no_safepoint_check_flag;
2222 MutexLockerEx hl(Heap_lock, safepoint_check);
2223 FreelistLocker fll(this);
2224 MutexLockerEx x(CGC_lock, safepoint_check);
2225 if (_foregroundGCIsActive || !UseAsyncConcMarkSweepGC) {
2226 // The foreground collector is active or we're
2227 // not using asynchronous collections. Skip this
2228 // background collection.
2229 assert(!_foregroundGCShouldWait, "Should be clear");
2230 return;
2231 } else {
2232 assert(_collectorState == Idling, "Should be idling before start.");
2233 _collectorState = InitialMarking;
2234 // Reset the expansion cause, now that we are about to begin
2235 // a new cycle.
2236 clear_expansion_cause();
2237 }
2238 // Decide if we want to enable class unloading as part of the
2239 // ensuing concurrent GC cycle.
2240 update_should_unload_classes();
2241 _full_gc_requested = false; // acks all outstanding full gc requests
2242 // Signal that we are about to start a collection
2243 gch->increment_total_full_collections(); // ... starting a collection cycle
2244 _collection_count_start = gch->total_full_collections();
2245 }
2246
2247 // Used for PrintGC
2248 size_t prev_used;
2249 if (PrintGC && Verbose) {
2250 prev_used = _cmsGen->used(); // XXXPERM
2251 }
2252
2253 // The change of the collection state is normally done at this level;
2254 // the exceptions are phases that are executed while the world is
2255 // stopped. For those phases the change of state is done while the
2256 // world is stopped. For baton passing purposes this allows the
2257 // background collector to finish the phase and change state atomically.
2258 // The foreground collector cannot wait on a phase that is done
2259 // while the world is stopped because the foreground collector already
2260 // has the world stopped and would deadlock.
2261 while (_collectorState != Idling) {
2262 if (TraceCMSState) {
2263 gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
2264 Thread::current(), _collectorState);
2265 }
2266 // The foreground collector
2267 // holds the Heap_lock throughout its collection.
2268 // holds the CMS token (but not the lock)
2269 // except while it is waiting for the background collector to yield.
2270 //
2271 // The foreground collector should be blocked (not for long)
2272 // if the background collector is about to start a phase
2273 // executed with world stopped. If the background
2274 // collector has already started such a phase, the
2275 // foreground collector is blocked waiting for the
2276 // Heap_lock. The stop-world phases (InitialMarking and FinalMarking)
2277 // are executed in the VM thread.
2278 //
2279 // The locking order is
2280 // PendingListLock (PLL) -- if applicable (FinalMarking)
2281 // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue())
2282 // CMS token (claimed in
2283 // stop_world_and_do() -->
2284 // safepoint_synchronize() -->
2285 // CMSThread::synchronize())
2286
2287 {
2288 // Check if the FG collector wants us to yield.
2289 CMSTokenSync x(true); // is cms thread
2290 if (waitForForegroundGC()) {
2291 // We yielded to a foreground GC, nothing more to be
2292 // done this round.
2293 assert(_foregroundGCShouldWait == false, "We set it to false in "
2294 "waitForForegroundGC()");
2295 if (TraceCMSState) {
2296 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
2297 " exiting collection CMS state %d",
2298 Thread::current(), _collectorState);
2299 }
2300 return;
2301 } else {
2302 // The background collector can run but check to see if the
2303 // foreground collector has done a collection while the
2304 // background collector was waiting to get the CGC_lock
2305 // above. If yes, break so that _foregroundGCShouldWait
2306 // is cleared before returning.
2307 if (_collectorState == Idling) {
2308 break;
2309 }
2310 }
2311 }
2312
2313 assert(_foregroundGCShouldWait, "Foreground collector, if active, "
2314 "should be waiting");
2315
2316 switch (_collectorState) {
2317 case InitialMarking:
2318 {
2319 ReleaseForegroundGC x(this);
2320 stats().record_cms_begin();
2321
2322 VM_CMS_Initial_Mark initial_mark_op(this);
2323 VMThread::execute(&initial_mark_op);
2324 }
2325 // The collector state may be any legal state at this point
2326 // since the background collector may have yielded to the
2327 // foreground collector.
2328 break;
2329 case Marking:
2330 // initial marking in checkpointRootsInitialWork has been completed
2331 if (markFromRoots(true)) { // we were successful
2332 assert(_collectorState == Precleaning, "Collector state should "
2333 "have changed");
2334 } else {
2335 assert(_foregroundGCIsActive, "Internal state inconsistency");
2336 }
2337 break;
2338 case Precleaning:
2339 if (UseAdaptiveSizePolicy) {
2340 size_policy()->concurrent_precleaning_begin();
2341 }
2342 // marking from roots in markFromRoots has been completed
2343 preclean();
2344 if (UseAdaptiveSizePolicy) {
2345 size_policy()->concurrent_precleaning_end();
2346 }
2347 assert(_collectorState == AbortablePreclean ||
2348 _collectorState == FinalMarking,
2349 "Collector state should have changed");
2350 break;
2351 case AbortablePreclean:
2352 if (UseAdaptiveSizePolicy) {
2353 size_policy()->concurrent_phases_resume();
2354 }
2355 abortable_preclean();
2356 if (UseAdaptiveSizePolicy) {
2357 size_policy()->concurrent_precleaning_end();
2358 }
2359 assert(_collectorState == FinalMarking, "Collector state should "
2360 "have changed");
2361 break;
2362 case FinalMarking:
2363 {
2364 ReleaseForegroundGC x(this);
2365
2366 VM_CMS_Final_Remark final_remark_op(this);
2367 VMThread::execute(&final_remark_op);
2368 }
2369 assert(_foregroundGCShouldWait, "block post-condition");
2370 break;
2371 case Sweeping:
2372 if (UseAdaptiveSizePolicy) {
2373 size_policy()->concurrent_sweeping_begin();
2374 }
2375 // final marking in checkpointRootsFinal has been completed
2376 sweep(true);
2377 assert(_collectorState == Resizing, "Collector state change "
2378 "to Resizing must be done under the free_list_lock");
2379 _full_gcs_since_conc_gc = 0;
2380
2381 // Stop the timers for adaptive size policy for the concurrent phases
2382 if (UseAdaptiveSizePolicy) {
2383 size_policy()->concurrent_sweeping_end();
2384 size_policy()->concurrent_phases_end(gch->gc_cause(),
2385 gch->prev_gen(_cmsGen)->capacity(),
2386 _cmsGen->free());
2387 }
2388
2389 case Resizing: {
2390 // Sweeping has been completed...
2391 // At this point the background collection has completed.
2392 // Don't move the call to compute_new_size() down
2393 // into code that might be executed if the background
2394 // collection was preempted.
2395 {
2396 ReleaseForegroundGC x(this); // unblock FG collection
2397 MutexLockerEx y(Heap_lock, Mutex::_no_safepoint_check_flag);
2398 CMSTokenSync z(true); // not strictly needed.
2399 if (_collectorState == Resizing) {
2400 compute_new_size();
2401 _collectorState = Resetting;
2402 } else {
2403 assert(_collectorState == Idling, "The state should only change"
2404 " because the foreground collector has finished the collection");
2405 }
2406 }
2407 break;
2408 }
2409 case Resetting:
2410 // CMS heap resizing has been completed
2411 reset(true);
2412 assert(_collectorState == Idling, "Collector state should "
2413 "have changed");
2414 stats().record_cms_end();
2415 // Don't move the concurrent_phases_end() and compute_new_size()
2416 // calls to here because a preempted background collection
2417 // has it's state set to "Resetting".
2418 break;
2419 case Idling:
2420 default:
2421 ShouldNotReachHere();
2422 break;
2423 }
2424 if (TraceCMSState) {
2425 gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
2426 Thread::current(), _collectorState);
2427 }
2428 assert(_foregroundGCShouldWait, "block post-condition");
2429 }
2430
2431 // Should this be in gc_epilogue?
2432 collector_policy()->counters()->update_counters();
2433
2434 {
2435 // Clear _foregroundGCShouldWait and, in the event that the
2436 // foreground collector is waiting, notify it, before
2437 // returning.
2438 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2439 _foregroundGCShouldWait = false;
2440 if (_foregroundGCIsActive) {
2441 CGC_lock->notify();
2442 }
2443 assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
2444 "Possible deadlock");
2445 }
2446 if (TraceCMSState) {
2447 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
2448 " exiting collection CMS state %d",
2449 Thread::current(), _collectorState);
2450 }
2451 if (PrintGC && Verbose) {
2452 _cmsGen->print_heap_change(prev_used);
2453 }
2454 }
2455
2456 void CMSCollector::collect_in_foreground(bool clear_all_soft_refs) {
2457 assert(_foregroundGCIsActive && !_foregroundGCShouldWait,
2458 "Foreground collector should be waiting, not executing");
2459 assert(Thread::current()->is_VM_thread(), "A foreground collection"
2460 "may only be done by the VM Thread with the world stopped");
2461 assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),
2462 "VM thread should have CMS token");
2463
2464 NOT_PRODUCT(TraceTime t("CMS:MS (foreground) ", PrintGCDetails && Verbose,
2465 true, gclog_or_tty);)
2466 if (UseAdaptiveSizePolicy) {
2467 size_policy()->ms_collection_begin();
2468 }
2469 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact);
2470
2471 HandleMark hm; // Discard invalid handles created during verification
2472
2473 if (VerifyBeforeGC &&
2474 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2475 Universe::verify(true);
2476 }
2477
2478 // Snapshot the soft reference policy to be used in this collection cycle.
2479 ref_processor()->setup_policy(clear_all_soft_refs);
2480
2481 bool init_mark_was_synchronous = false; // until proven otherwise
2482 while (_collectorState != Idling) {
2483 if (TraceCMSState) {
2484 gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",
2485 Thread::current(), _collectorState);
2486 }
2487 switch (_collectorState) {
2488 case InitialMarking:
2489 init_mark_was_synchronous = true; // fact to be exploited in re-mark
2490 checkpointRootsInitial(false);
2491 assert(_collectorState == Marking, "Collector state should have changed"
2492 " within checkpointRootsInitial()");
2493 break;
2494 case Marking:
2495 // initial marking in checkpointRootsInitialWork has been completed
2496 if (VerifyDuringGC &&
2497 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2498 gclog_or_tty->print("Verify before initial mark: ");
2499 Universe::verify(true);
2500 }
2501 {
2502 bool res = markFromRoots(false);
2503 assert(res && _collectorState == FinalMarking, "Collector state should "
2504 "have changed");
2505 break;
2506 }
2507 case FinalMarking:
2508 if (VerifyDuringGC &&
2509 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2510 gclog_or_tty->print("Verify before re-mark: ");
2511 Universe::verify(true);
2512 }
2513 checkpointRootsFinal(false, clear_all_soft_refs,
2514 init_mark_was_synchronous);
2515 assert(_collectorState == Sweeping, "Collector state should not "
2516 "have changed within checkpointRootsFinal()");
2517 break;
2518 case Sweeping:
2519 // final marking in checkpointRootsFinal has been completed
2520 if (VerifyDuringGC &&
2521 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2522 gclog_or_tty->print("Verify before sweep: ");
2523 Universe::verify(true);
2524 }
2525 sweep(false);
2526 assert(_collectorState == Resizing, "Incorrect state");
2527 break;
2528 case Resizing: {
2529 // Sweeping has been completed; the actual resize in this case
2530 // is done separately; nothing to be done in this state.
2531 _collectorState = Resetting;
2532 break;
2533 }
2534 case Resetting:
2535 // The heap has been resized.
2536 if (VerifyDuringGC &&
2537 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2538 gclog_or_tty->print("Verify before reset: ");
2539 Universe::verify(true);
2540 }
2541 reset(false);
2542 assert(_collectorState == Idling, "Collector state should "
2543 "have changed");
2544 break;
2545 case Precleaning:
2546 case AbortablePreclean:
2547 // Elide the preclean phase
2548 _collectorState = FinalMarking;
2549 break;
2550 default:
2551 ShouldNotReachHere();
2552 }
2553 if (TraceCMSState) {
2554 gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d",
2555 Thread::current(), _collectorState);
2556 }
2557 }
2558
2559 if (UseAdaptiveSizePolicy) {
2560 GenCollectedHeap* gch = GenCollectedHeap::heap();
2561 size_policy()->ms_collection_end(gch->gc_cause());
2562 }
2563
2564 if (VerifyAfterGC &&
2565 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
2566 Universe::verify(true);
2567 }
2568 if (TraceCMSState) {
2569 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT
2570 " exiting collection CMS state %d",
2571 Thread::current(), _collectorState);
2572 }
2573 }
2574
2575 bool CMSCollector::waitForForegroundGC() {
2576 bool res = false;
2577 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
2578 "CMS thread should have CMS token");
2579 // Block the foreground collector until the
2580 // background collectors decides whether to
2581 // yield.
2582 MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
2583 _foregroundGCShouldWait = true;
2584 if (_foregroundGCIsActive) {
2585 // The background collector yields to the
2586 // foreground collector and returns a value
2587 // indicating that it has yielded. The foreground
2588 // collector can proceed.
2589 res = true;
2590 _foregroundGCShouldWait = false;
2591 ConcurrentMarkSweepThread::clear_CMS_flag(
2592 ConcurrentMarkSweepThread::CMS_cms_has_token);
2593 ConcurrentMarkSweepThread::set_CMS_flag(
2594 ConcurrentMarkSweepThread::CMS_cms_wants_token);
2595 // Get a possibly blocked foreground thread going
2596 CGC_lock->notify();
2597 if (TraceCMSState) {
2598 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d",
2599 Thread::current(), _collectorState);
2600 }
2601 while (_foregroundGCIsActive) {
2602 CGC_lock->wait(Mutex::_no_safepoint_check_flag);
2603 }
2604 ConcurrentMarkSweepThread::set_CMS_flag(
2605 ConcurrentMarkSweepThread::CMS_cms_has_token);
2606 ConcurrentMarkSweepThread::clear_CMS_flag(
2607 ConcurrentMarkSweepThread::CMS_cms_wants_token);
2608 }
2609 if (TraceCMSState) {
2610 gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d",
2611 Thread::current(), _collectorState);
2612 }
2613 return res;
2614 }
2615
2616 // Because of the need to lock the free lists and other structures in
2617 // the collector, common to all the generations that the collector is
2618 // collecting, we need the gc_prologues of individual CMS generations
2619 // delegate to their collector. It may have been simpler had the
2620 // current infrastructure allowed one to call a prologue on a
2621 // collector. In the absence of that we have the generation's
2622 // prologue delegate to the collector, which delegates back
2623 // some "local" work to a worker method in the individual generations
2624 // that it's responsible for collecting, while itself doing any
2625 // work common to all generations it's responsible for. A similar
2626 // comment applies to the gc_epilogue()'s.
2627 // The role of the varaible _between_prologue_and_epilogue is to
2628 // enforce the invocation protocol.
2629 void CMSCollector::gc_prologue(bool full) {
2630 // Call gc_prologue_work() for each CMSGen and PermGen that
2631 // we are responsible for.
2632
2633 // The following locking discipline assumes that we are only called
2634 // when the world is stopped.
2635 assert(SafepointSynchronize::is_at_safepoint(), "world is stopped assumption");
2636
2637 // The CMSCollector prologue must call the gc_prologues for the
2638 // "generations" (including PermGen if any) that it's responsible
2639 // for.
2640
2641 assert( Thread::current()->is_VM_thread()
2642 || ( CMSScavengeBeforeRemark
2643 && Thread::current()->is_ConcurrentGC_thread()),
2644 "Incorrect thread type for prologue execution");
2645
2646 if (_between_prologue_and_epilogue) {
2647 // We have already been invoked; this is a gc_prologue delegation
2648 // from yet another CMS generation that we are responsible for, just
2649 // ignore it since all relevant work has already been done.
2650 return;
2651 }
2652
2653 // set a bit saying prologue has been called; cleared in epilogue
2654 _between_prologue_and_epilogue = true;
2655 // Claim locks for common data structures, then call gc_prologue_work()
2656 // for each CMSGen and PermGen that we are responsible for.
2657
2658 getFreelistLocks(); // gets free list locks on constituent spaces
2659 bitMapLock()->lock_without_safepoint_check();
2660
2661 // Should call gc_prologue_work() for all cms gens we are responsible for
2662 bool registerClosure = _collectorState >= Marking
2663 && _collectorState < Sweeping;
2664 ModUnionClosure* muc = CollectedHeap::use_parallel_gc_threads() ?
2665 &_modUnionClosurePar
2666 : &_modUnionClosure;
2667 _cmsGen->gc_prologue_work(full, registerClosure, muc);
2668 _permGen->gc_prologue_work(full, registerClosure, muc);
2669
2670 if (!full) {
2671 stats().record_gc0_begin();
2672 }
2673 }
2674
2675 void ConcurrentMarkSweepGeneration::gc_prologue(bool full) {
2676 // Delegate to CMScollector which knows how to coordinate between
2677 // this and any other CMS generations that it is responsible for
2678 // collecting.
2679 collector()->gc_prologue(full);
2680 }
2681
2682 // This is a "private" interface for use by this generation's CMSCollector.
2683 // Not to be called directly by any other entity (for instance,
2684 // GenCollectedHeap, which calls the "public" gc_prologue method above).
2685 void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,
2686 bool registerClosure, ModUnionClosure* modUnionClosure) {
2687 assert(!incremental_collection_failed(), "Shouldn't be set yet");
2688 assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL,
2689 "Should be NULL");
2690 if (registerClosure) {
2691 cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure);
2692 }
2693 cmsSpace()->gc_prologue();
2694 // Clear stat counters
2695 NOT_PRODUCT(
2696 assert(_numObjectsPromoted == 0, "check");
2697 assert(_numWordsPromoted == 0, "check");
2698 if (Verbose && PrintGC) {
2699 gclog_or_tty->print("Allocated "SIZE_FORMAT" objects, "
2700 SIZE_FORMAT" bytes concurrently",
2701 _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord));
2702 }
2703 _numObjectsAllocated = 0;
2704 _numWordsAllocated = 0;
2705 )
2706 }
2707
2708 void CMSCollector::gc_epilogue(bool full) {
2709 // The following locking discipline assumes that we are only called
2710 // when the world is stopped.
2711 assert(SafepointSynchronize::is_at_safepoint(),
2712 "world is stopped assumption");
2713
2714 // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks
2715 // if linear allocation blocks need to be appropriately marked to allow the
2716 // the blocks to be parsable. We also check here whether we need to nudge the
2717 // CMS collector thread to start a new cycle (if it's not already active).
2718 assert( Thread::current()->is_VM_thread()
2719 || ( CMSScavengeBeforeRemark
2720 && Thread::current()->is_ConcurrentGC_thread()),
2721 "Incorrect thread type for epilogue execution");
2722
2723 if (!_between_prologue_and_epilogue) {
2724 // We have already been invoked; this is a gc_epilogue delegation
2725 // from yet another CMS generation that we are responsible for, just
2726 // ignore it since all relevant work has already been done.
2727 return;
2728 }
2729 assert(haveFreelistLocks(), "must have freelist locks");
2730 assert_lock_strong(bitMapLock());
2731
2732 _cmsGen->gc_epilogue_work(full);
2733 _permGen->gc_epilogue_work(full);
2734
2735 if (_collectorState == AbortablePreclean || _collectorState == Precleaning) {
2736 // in case sampling was not already enabled, enable it
2737 _start_sampling = true;
2738 }
2739 // reset _eden_chunk_array so sampling starts afresh
2740 _eden_chunk_index = 0;
2741
2742 size_t cms_used = _cmsGen->cmsSpace()->used();
2743 size_t perm_used = _permGen->cmsSpace()->used();
2744
2745 // update performance counters - this uses a special version of
2746 // update_counters() that allows the utilization to be passed as a
2747 // parameter, avoiding multiple calls to used().
2748 //
2749 _cmsGen->update_counters(cms_used);
2750 _permGen->update_counters(perm_used);
2751
2752 if (CMSIncrementalMode) {
2753 icms_update_allocation_limits();
2754 }
2755
2756 bitMapLock()->unlock();
2757 releaseFreelistLocks();
2758
2759 _between_prologue_and_epilogue = false; // ready for next cycle
2760 }
2761
2762 void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) {
2763 collector()->gc_epilogue(full);
2764
2765 // Also reset promotion tracking in par gc thread states.
2766 if (CollectedHeap::use_parallel_gc_threads()) {
2767 for (uint i = 0; i < ParallelGCThreads; i++) {
2768 _par_gc_thread_states[i]->promo.stopTrackingPromotions(i);
2769 }
2770 }
2771 }
2772
2773 void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) {
2774 assert(!incremental_collection_failed(), "Should have been cleared");
2775 cmsSpace()->setPreconsumptionDirtyCardClosure(NULL);
2776 cmsSpace()->gc_epilogue();
2777 // Print stat counters
2778 NOT_PRODUCT(
2779 assert(_numObjectsAllocated == 0, "check");
2780 assert(_numWordsAllocated == 0, "check");
2781 if (Verbose && PrintGC) {
2782 gclog_or_tty->print("Promoted "SIZE_FORMAT" objects, "
2783 SIZE_FORMAT" bytes",
2784 _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord));
2785 }
2786 _numObjectsPromoted = 0;
2787 _numWordsPromoted = 0;
2788 )
2789
2790 if (PrintGC && Verbose) {
2791 // Call down the chain in contiguous_available needs the freelistLock
2792 // so print this out before releasing the freeListLock.
2793 gclog_or_tty->print(" Contiguous available "SIZE_FORMAT" bytes ",
2794 contiguous_available());
2795 }
2796 }
2797
2798 #ifndef PRODUCT
2799 bool CMSCollector::have_cms_token() {
2800 Thread* thr = Thread::current();
2801 if (thr->is_VM_thread()) {
2802 return ConcurrentMarkSweepThread::vm_thread_has_cms_token();
2803 } else if (thr->is_ConcurrentGC_thread()) {
2804 return ConcurrentMarkSweepThread::cms_thread_has_cms_token();
2805 } else if (thr->is_GC_task_thread()) {
2806 return ConcurrentMarkSweepThread::vm_thread_has_cms_token() &&
2807 ParGCRareEvent_lock->owned_by_self();
2808 }
2809 return false;
2810 }
2811 #endif
2812
2813 // Check reachability of the given heap address in CMS generation,
2814 // treating all other generations as roots.
2815 bool CMSCollector::is_cms_reachable(HeapWord* addr) {
2816 // We could "guarantee" below, rather than assert, but i'll
2817 // leave these as "asserts" so that an adventurous debugger
2818 // could try this in the product build provided some subset of
2819 // the conditions were met, provided they were intersted in the
2820 // results and knew that the computation below wouldn't interfere
2821 // with other concurrent computations mutating the structures
2822 // being read or written.
2823 assert(SafepointSynchronize::is_at_safepoint(),
2824 "Else mutations in object graph will make answer suspect");
2825 assert(have_cms_token(), "Should hold cms token");
2826 assert(haveFreelistLocks(), "must hold free list locks");
2827 assert_lock_strong(bitMapLock());
2828
2829 // Clear the marking bit map array before starting, but, just
2830 // for kicks, first report if the given address is already marked
2831 gclog_or_tty->print_cr("Start: Address 0x%x is%s marked", addr,
2832 _markBitMap.isMarked(addr) ? "" : " not");
2833
2834 if (verify_after_remark()) {
2835 MutexLockerEx x(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2836 bool result = verification_mark_bm()->isMarked(addr);
2837 gclog_or_tty->print_cr("TransitiveMark: Address 0x%x %s marked", addr,
2838 result ? "IS" : "is NOT");
2839 return result;
2840 } else {
2841 gclog_or_tty->print_cr("Could not compute result");
2842 return false;
2843 }
2844 }
2845
2846 ////////////////////////////////////////////////////////
2847 // CMS Verification Support
2848 ////////////////////////////////////////////////////////
2849 // Following the remark phase, the following invariant
2850 // should hold -- each object in the CMS heap which is
2851 // marked in markBitMap() should be marked in the verification_mark_bm().
2852
2853 class VerifyMarkedClosure: public BitMapClosure {
2854 CMSBitMap* _marks;
2855 bool _failed;
2856
2857 public:
2858 VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
2859
2860 bool do_bit(size_t offset) {
2861 HeapWord* addr = _marks->offsetToHeapWord(offset);
2862 if (!_marks->isMarked(addr)) {
2863 oop(addr)->print_on(gclog_or_tty);
2864 gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", addr);
2865 _failed = true;
2866 }
2867 return true;
2868 }
2869
2870 bool failed() { return _failed; }
2871 };
2872
2873 bool CMSCollector::verify_after_remark() {
2874 gclog_or_tty->print(" [Verifying CMS Marking... ");
2875 MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag);
2876 static bool init = false;
2877
2878 assert(SafepointSynchronize::is_at_safepoint(),
2879 "Else mutations in object graph will make answer suspect");
2880 assert(have_cms_token(),
2881 "Else there may be mutual interference in use of "
2882 " verification data structures");
2883 assert(_collectorState > Marking && _collectorState <= Sweeping,
2884 "Else marking info checked here may be obsolete");
2885 assert(haveFreelistLocks(), "must hold free list locks");
2886 assert_lock_strong(bitMapLock());
2887
2888
2889 // Allocate marking bit map if not already allocated
2890 if (!init) { // first time
2891 if (!verification_mark_bm()->allocate(_span)) {
2892 return false;
2893 }
2894 init = true;
2895 }
2896
2897 assert(verification_mark_stack()->isEmpty(), "Should be empty");
2898
2899 // Turn off refs discovery -- so we will be tracing through refs.
2900 // This is as intended, because by this time
2901 // GC must already have cleared any refs that need to be cleared,
2902 // and traced those that need to be marked; moreover,
2903 // the marking done here is not going to intefere in any
2904 // way with the marking information used by GC.
2905 NoRefDiscovery no_discovery(ref_processor());
2906
2907 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
2908
2909 // Clear any marks from a previous round
2910 verification_mark_bm()->clear_all();
2911 assert(verification_mark_stack()->isEmpty(), "markStack should be empty");
2912 verify_work_stacks_empty();
2913
2914 GenCollectedHeap* gch = GenCollectedHeap::heap();
2915 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
2916 // Update the saved marks which may affect the root scans.
2917 gch->save_marks();
2918
2919 if (CMSRemarkVerifyVariant == 1) {
2920 // In this first variant of verification, we complete
2921 // all marking, then check if the new marks-verctor is
2922 // a subset of the CMS marks-vector.
2923 verify_after_remark_work_1();
2924 } else if (CMSRemarkVerifyVariant == 2) {
2925 // In this second variant of verification, we flag an error
2926 // (i.e. an object reachable in the new marks-vector not reachable
2927 // in the CMS marks-vector) immediately, also indicating the
2928 // identify of an object (A) that references the unmarked object (B) --
2929 // presumably, a mutation to A failed to be picked up by preclean/remark?
2930 verify_after_remark_work_2();
2931 } else {
2932 warning("Unrecognized value %d for CMSRemarkVerifyVariant",
2933 CMSRemarkVerifyVariant);
2934 }
2935 gclog_or_tty->print(" done] ");
2936 return true;
2937 }
2938
2939 void CMSCollector::verify_after_remark_work_1() {
2940 ResourceMark rm;
2941 HandleMark hm;
2942 GenCollectedHeap* gch = GenCollectedHeap::heap();
2943
2944 // Mark from roots one level into CMS
2945 MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
2946 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2947
2948 gch->gen_process_strong_roots(_cmsGen->level(),
2949 true, // younger gens are roots
2950 true, // activate StrongRootsScope
2951 true, // collecting perm gen
2952 SharedHeap::ScanningOption(roots_scanning_options()),
2953 ¬Older,
2954 true, // walk code active on stacks
2955 NULL);
2956
2957 // Now mark from the roots
2958 assert(_revisitStack.isEmpty(), "Should be empty");
2959 MarkFromRootsClosure markFromRootsClosure(this, _span,
2960 verification_mark_bm(), verification_mark_stack(), &_revisitStack,
2961 false /* don't yield */, true /* verifying */);
2962 assert(_restart_addr == NULL, "Expected pre-condition");
2963 verification_mark_bm()->iterate(&markFromRootsClosure);
2964 while (_restart_addr != NULL) {
2965 // Deal with stack overflow: by restarting at the indicated
2966 // address.
2967 HeapWord* ra = _restart_addr;
2968 markFromRootsClosure.reset(ra);
2969 _restart_addr = NULL;
2970 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
2971 }
2972 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
2973 verify_work_stacks_empty();
2974 // Should reset the revisit stack above, since no class tree
2975 // surgery is forthcoming.
2976 _revisitStack.reset(); // throwing away all contents
2977
2978 // Marking completed -- now verify that each bit marked in
2979 // verification_mark_bm() is also marked in markBitMap(); flag all
2980 // errors by printing corresponding objects.
2981 VerifyMarkedClosure vcl(markBitMap());
2982 verification_mark_bm()->iterate(&vcl);
2983 if (vcl.failed()) {
2984 gclog_or_tty->print("Verification failed");
2985 Universe::heap()->print_on(gclog_or_tty);
2986 fatal("CMS: failed marking verification after remark");
2987 }
2988 }
2989
2990 void CMSCollector::verify_after_remark_work_2() {
2991 ResourceMark rm;
2992 HandleMark hm;
2993 GenCollectedHeap* gch = GenCollectedHeap::heap();
2994
2995 // Mark from roots one level into CMS
2996 MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(),
2997 markBitMap());
2998 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
2999 gch->gen_process_strong_roots(_cmsGen->level(),
3000 true, // younger gens are roots
3001 true, // activate StrongRootsScope
3002 true, // collecting perm gen
3003 SharedHeap::ScanningOption(roots_scanning_options()),
3004 ¬Older,
3005 true, // walk code active on stacks
3006 NULL);
3007
3008 // Now mark from the roots
3009 assert(_revisitStack.isEmpty(), "Should be empty");
3010 MarkFromRootsVerifyClosure markFromRootsClosure(this, _span,
3011 verification_mark_bm(), markBitMap(), verification_mark_stack());
3012 assert(_restart_addr == NULL, "Expected pre-condition");
3013 verification_mark_bm()->iterate(&markFromRootsClosure);
3014 while (_restart_addr != NULL) {
3015 // Deal with stack overflow: by restarting at the indicated
3016 // address.
3017 HeapWord* ra = _restart_addr;
3018 markFromRootsClosure.reset(ra);
3019 _restart_addr = NULL;
3020 verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end());
3021 }
3022 assert(verification_mark_stack()->isEmpty(), "Should have been drained");
3023 verify_work_stacks_empty();
3024 // Should reset the revisit stack above, since no class tree
3025 // surgery is forthcoming.
3026 _revisitStack.reset(); // throwing away all contents
3027
3028 // Marking completed -- now verify that each bit marked in
3029 // verification_mark_bm() is also marked in markBitMap(); flag all
3030 // errors by printing corresponding objects.
3031 VerifyMarkedClosure vcl(markBitMap());
3032 verification_mark_bm()->iterate(&vcl);
3033 assert(!vcl.failed(), "Else verification above should not have succeeded");
3034 }
3035
3036 void ConcurrentMarkSweepGeneration::save_marks() {
3037 // delegate to CMS space
3038 cmsSpace()->save_marks();
3039 for (uint i = 0; i < ParallelGCThreads; i++) {
3040 _par_gc_thread_states[i]->promo.startTrackingPromotions();
3041 }
3042 }
3043
3044 bool ConcurrentMarkSweepGeneration::no_allocs_since_save_marks() {
3045 return cmsSpace()->no_allocs_since_save_marks();
3046 }
3047
3048 #define CMS_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \
3049 \
3050 void ConcurrentMarkSweepGeneration:: \
3051 oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \
3052 cl->set_generation(this); \
3053 cmsSpace()->oop_since_save_marks_iterate##nv_suffix(cl); \
3054 cl->reset_generation(); \
3055 save_marks(); \
3056 }
3057
3058 ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DEFN)
3059
3060 void
3061 ConcurrentMarkSweepGeneration::object_iterate_since_last_GC(ObjectClosure* blk)
3062 {
3063 // Not currently implemented; need to do the following. -- ysr.
3064 // dld -- I think that is used for some sort of allocation profiler. So it
3065 // really means the objects allocated by the mutator since the last
3066 // GC. We could potentially implement this cheaply by recording only
3067 // the direct allocations in a side data structure.
3068 //
3069 // I think we probably ought not to be required to support these
3070 // iterations at any arbitrary point; I think there ought to be some
3071 // call to enable/disable allocation profiling in a generation/space,
3072 // and the iterator ought to return the objects allocated in the
3073 // gen/space since the enable call, or the last iterator call (which
3074 // will probably be at a GC.) That way, for gens like CM&S that would
3075 // require some extra data structure to support this, we only pay the
3076 // cost when it's in use...
3077 cmsSpace()->object_iterate_since_last_GC(blk);
3078 }
3079
3080 void
3081 ConcurrentMarkSweepGeneration::younger_refs_iterate(OopsInGenClosure* cl) {
3082 cl->set_generation(this);
3083 younger_refs_in_space_iterate(_cmsSpace, cl);
3084 cl->reset_generation();
3085 }
3086
3087 void
3088 ConcurrentMarkSweepGeneration::oop_iterate(MemRegion mr, OopClosure* cl) {
3089 if (freelistLock()->owned_by_self()) {
3090 Generation::oop_iterate(mr, cl);
3091 } else {
3092 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
3093 Generation::oop_iterate(mr, cl);
3094 }
3095 }
3096
3097 void
3098 ConcurrentMarkSweepGeneration::oop_iterate(OopClosure* cl) {
3099 if (freelistLock()->owned_by_self()) {
3100 Generation::oop_iterate(cl);
3101 } else {
3102 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
3103 Generation::oop_iterate(cl);
3104 }
3105 }
3106
3107 void
3108 ConcurrentMarkSweepGeneration::object_iterate(ObjectClosure* cl) {
3109 if (freelistLock()->owned_by_self()) {
3110 Generation::object_iterate(cl);
3111 } else {
3112 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
3113 Generation::object_iterate(cl);
3114 }
3115 }
3116
3117 void
3118 ConcurrentMarkSweepGeneration::safe_object_iterate(ObjectClosure* cl) {
3119 if (freelistLock()->owned_by_self()) {
3120 Generation::safe_object_iterate(cl);
3121 } else {
3122 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
3123 Generation::safe_object_iterate(cl);
3124 }
3125 }
3126
3127 void
3128 ConcurrentMarkSweepGeneration::pre_adjust_pointers() {
3129 }
3130
3131 void
3132 ConcurrentMarkSweepGeneration::post_compact() {
3133 }
3134
3135 void
3136 ConcurrentMarkSweepGeneration::prepare_for_verify() {
3137 // Fix the linear allocation blocks to look like free blocks.
3138
3139 // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those
3140 // are not called when the heap is verified during universe initialization and
3141 // at vm shutdown.
3142 if (freelistLock()->owned_by_self()) {
3143 cmsSpace()->prepare_for_verify();
3144 } else {
3145 MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag);
3146 cmsSpace()->prepare_for_verify();
3147 }
3148 }
3149
3150 void
3151 ConcurrentMarkSweepGeneration::verify(bool allow_dirty /* ignored */) {
3152 // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those
3153 // are not called when the heap is verified during universe initialization and
3154 // at vm shutdown.
3155 if (freelistLock()->owned_by_self()) {
3156 cmsSpace()->verify(false /* ignored */);
3157 } else {
3158 MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag);
3159 cmsSpace()->verify(false /* ignored */);
3160 }
3161 }
3162
3163 void CMSCollector::verify(bool allow_dirty /* ignored */) {
3164 _cmsGen->verify(allow_dirty);
3165 _permGen->verify(allow_dirty);
3166 }
3167
3168 #ifndef PRODUCT
3169 bool CMSCollector::overflow_list_is_empty() const {
3170 assert(_num_par_pushes >= 0, "Inconsistency");
3171 if (_overflow_list == NULL) {
3172 assert(_num_par_pushes == 0, "Inconsistency");
3173 }
3174 return _overflow_list == NULL;
3175 }
3176
3177 // The methods verify_work_stacks_empty() and verify_overflow_empty()
3178 // merely consolidate assertion checks that appear to occur together frequently.
3179 void CMSCollector::verify_work_stacks_empty() const {
3180 assert(_markStack.isEmpty(), "Marking stack should be empty");
3181 assert(overflow_list_is_empty(), "Overflow list should be empty");
3182 }
3183
3184 void CMSCollector::verify_overflow_empty() const {
3185 assert(overflow_list_is_empty(), "Overflow list should be empty");
3186 assert(no_preserved_marks(), "No preserved marks");
3187 }
3188 #endif // PRODUCT
3189
3190 // Decide if we want to enable class unloading as part of the
3191 // ensuing concurrent GC cycle. We will collect the perm gen and
3192 // unload classes if it's the case that:
3193 // (1) an explicit gc request has been made and the flag
3194 // ExplicitGCInvokesConcurrentAndUnloadsClasses is set, OR
3195 // (2) (a) class unloading is enabled at the command line, and
3196 // (b) (i) perm gen threshold has been crossed, or
3197 // (ii) old gen is getting really full, or
3198 // (iii) the previous N CMS collections did not collect the
3199 // perm gen
3200 // NOTE: Provided there is no change in the state of the heap between
3201 // calls to this method, it should have idempotent results. Moreover,
3202 // its results should be monotonically increasing (i.e. going from 0 to 1,
3203 // but not 1 to 0) between successive calls between which the heap was
3204 // not collected. For the implementation below, it must thus rely on
3205 // the property that concurrent_cycles_since_last_unload()
3206 // will not decrease unless a collection cycle happened and that
3207 // _permGen->should_concurrent_collect() and _cmsGen->is_too_full() are
3208 // themselves also monotonic in that sense. See check_monotonicity()
3209 // below.
3210 bool CMSCollector::update_should_unload_classes() {
3211 _should_unload_classes = false;
3212 // Condition 1 above
3213 if (_full_gc_requested && ExplicitGCInvokesConcurrentAndUnloadsClasses) {
3214 _should_unload_classes = true;
3215 } else if (CMSClassUnloadingEnabled) { // Condition 2.a above
3216 // Disjuncts 2.b.(i,ii,iii) above
3217 _should_unload_classes = (concurrent_cycles_since_last_unload() >=
3218 CMSClassUnloadingMaxInterval)
3219 || _permGen->should_concurrent_collect()
3220 || _cmsGen->is_too_full();
3221 }
3222 return _should_unload_classes;
3223 }
3224
3225 bool ConcurrentMarkSweepGeneration::is_too_full() const {
3226 bool res = should_concurrent_collect();
3227 res = res && (occupancy() > (double)CMSIsTooFullPercentage/100.0);
3228 return res;
3229 }
3230
3231 void CMSCollector::setup_cms_unloading_and_verification_state() {
3232 const bool should_verify = VerifyBeforeGC || VerifyAfterGC || VerifyDuringGC
3233 || VerifyBeforeExit;
3234 const int rso = SharedHeap::SO_Symbols | SharedHeap::SO_Strings
3235 | SharedHeap::SO_CodeCache;
3236
3237 if (should_unload_classes()) { // Should unload classes this cycle
3238 remove_root_scanning_option(rso); // Shrink the root set appropriately
3239 set_verifying(should_verify); // Set verification state for this cycle
3240 return; // Nothing else needs to be done at this time
3241 }
3242
3243 // Not unloading classes this cycle
3244 assert(!should_unload_classes(), "Inconsitency!");
3245 if ((!verifying() || unloaded_classes_last_cycle()) && should_verify) {
3246 // We were not verifying, or we _were_ unloading classes in the last cycle,
3247 // AND some verification options are enabled this cycle; in this case,
3248 // we must make sure that the deadness map is allocated if not already so,
3249 // and cleared (if already allocated previously --
3250 // CMSBitMap::sizeInBits() is used to determine if it's allocated).
3251 if (perm_gen_verify_bit_map()->sizeInBits() == 0) {
3252 if (!perm_gen_verify_bit_map()->allocate(_permGen->reserved())) {
3253 warning("Failed to allocate permanent generation verification CMS Bit Map;\n"
3254 "permanent generation verification disabled");
3255 return; // Note that we leave verification disabled, so we'll retry this
3256 // allocation next cycle. We _could_ remember this failure
3257 // and skip further attempts and permanently disable verification
3258 // attempts if that is considered more desirable.
3259 }
3260 assert(perm_gen_verify_bit_map()->covers(_permGen->reserved()),
3261 "_perm_gen_ver_bit_map inconsistency?");
3262 } else {
3263 perm_gen_verify_bit_map()->clear_all();
3264 }
3265 // Include symbols, strings and code cache elements to prevent their resurrection.
3266 add_root_scanning_option(rso);
3267 set_verifying(true);
3268 } else if (verifying() && !should_verify) {
3269 // We were verifying, but some verification flags got disabled.
3270 set_verifying(false);
3271 // Exclude symbols, strings and code cache elements from root scanning to
3272 // reduce IM and RM pauses.
3273 remove_root_scanning_option(rso);
3274 }
3275 }
3276
3277
3278 #ifndef PRODUCT
3279 HeapWord* CMSCollector::block_start(const void* p) const {
3280 const HeapWord* addr = (HeapWord*)p;
3281 if (_span.contains(p)) {
3282 if (_cmsGen->cmsSpace()->is_in_reserved(addr)) {
3283 return _cmsGen->cmsSpace()->block_start(p);
3284 } else {
3285 assert(_permGen->cmsSpace()->is_in_reserved(addr),
3286 "Inconsistent _span?");
3287 return _permGen->cmsSpace()->block_start(p);
3288 }
3289 }
3290 return NULL;
3291 }
3292 #endif
3293
3294 HeapWord*
3295 ConcurrentMarkSweepGeneration::expand_and_allocate(size_t word_size,
3296 bool tlab,
3297 bool parallel) {
3298 CMSSynchronousYieldRequest yr;
3299 assert(!tlab, "Can't deal with TLAB allocation");
3300 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag);
3301 expand(word_size*HeapWordSize, MinHeapDeltaBytes,
3302 CMSExpansionCause::_satisfy_allocation);
3303 if (GCExpandToAllocateDelayMillis > 0) {
3304 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
3305 }
3306 return have_lock_and_allocate(word_size, tlab);
3307 }
3308
3309 // YSR: All of this generation expansion/shrinking stuff is an exact copy of
3310 // OneContigSpaceCardGeneration, which makes me wonder if we should move this
3311 // to CardGeneration and share it...
3312 bool ConcurrentMarkSweepGeneration::expand(size_t bytes, size_t expand_bytes) {
3313 return CardGeneration::expand(bytes, expand_bytes);
3314 }
3315
3316 void ConcurrentMarkSweepGeneration::expand(size_t bytes, size_t expand_bytes,
3317 CMSExpansionCause::Cause cause)
3318 {
3319
3320 bool success = expand(bytes, expand_bytes);
3321
3322 // remember why we expanded; this information is used
3323 // by shouldConcurrentCollect() when making decisions on whether to start
3324 // a new CMS cycle.
3325 if (success) {
3326 set_expansion_cause(cause);
3327 if (PrintGCDetails && Verbose) {
3328 gclog_or_tty->print_cr("Expanded CMS gen for %s",
3329 CMSExpansionCause::to_string(cause));
3330 }
3331 }
3332 }
3333
3334 HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) {
3335 HeapWord* res = NULL;
3336 MutexLocker x(ParGCRareEvent_lock);
3337 while (true) {
3338 // Expansion by some other thread might make alloc OK now:
3339 res = ps->lab.alloc(word_sz);
3340 if (res != NULL) return res;
3341 // If there's not enough expansion space available, give up.
3342 if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) {
3343 return NULL;
3344 }
3345 // Otherwise, we try expansion.
3346 expand(word_sz*HeapWordSize, MinHeapDeltaBytes,
3347 CMSExpansionCause::_allocate_par_lab);
3348 // Now go around the loop and try alloc again;
3349 // A competing par_promote might beat us to the expansion space,
3350 // so we may go around the loop again if promotion fails agaion.
3351 if (GCExpandToAllocateDelayMillis > 0) {
3352 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
3353 }
3354 }
3355 }
3356
3357
3358 bool ConcurrentMarkSweepGeneration::expand_and_ensure_spooling_space(
3359 PromotionInfo* promo) {
3360 MutexLocker x(ParGCRareEvent_lock);
3361 size_t refill_size_bytes = promo->refillSize() * HeapWordSize;
3362 while (true) {
3363 // Expansion by some other thread might make alloc OK now:
3364 if (promo->ensure_spooling_space()) {
3365 assert(promo->has_spooling_space(),
3366 "Post-condition of successful ensure_spooling_space()");
3367 return true;
3368 }
3369 // If there's not enough expansion space available, give up.
3370 if (_virtual_space.uncommitted_size() < refill_size_bytes) {
3371 return false;
3372 }
3373 // Otherwise, we try expansion.
3374 expand(refill_size_bytes, MinHeapDeltaBytes,
3375 CMSExpansionCause::_allocate_par_spooling_space);
3376 // Now go around the loop and try alloc again;
3377 // A competing allocation might beat us to the expansion space,
3378 // so we may go around the loop again if allocation fails again.
3379 if (GCExpandToAllocateDelayMillis > 0) {
3380 os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false);
3381 }
3382 }
3383 }
3384
3385
3386
3387 void ConcurrentMarkSweepGeneration::shrink(size_t bytes) {
3388 assert_locked_or_safepoint(Heap_lock);
3389 size_t size = ReservedSpace::page_align_size_down(bytes);
3390 if (size > 0) {
3391 shrink_by(size);
3392 }
3393 }
3394
3395 bool ConcurrentMarkSweepGeneration::grow_by(size_t bytes) {
3396 assert_locked_or_safepoint(Heap_lock);
3397 bool result = _virtual_space.expand_by(bytes);
3398 if (result) {
3399 HeapWord* old_end = _cmsSpace->end();
3400 size_t new_word_size =
3401 heap_word_size(_virtual_space.committed_size());
3402 MemRegion mr(_cmsSpace->bottom(), new_word_size);
3403 _bts->resize(new_word_size); // resize the block offset shared array
3404 Universe::heap()->barrier_set()->resize_covered_region(mr);
3405 // Hmmmm... why doesn't CFLS::set_end verify locking?
3406 // This is quite ugly; FIX ME XXX
3407 _cmsSpace->assert_locked(freelistLock());
3408 _cmsSpace->set_end((HeapWord*)_virtual_space.high());
3409
3410 // update the space and generation capacity counters
3411 if (UsePerfData) {
3412 _space_counters->update_capacity();
3413 _gen_counters->update_all();
3414 }
3415
3416 if (Verbose && PrintGC) {
3417 size_t new_mem_size = _virtual_space.committed_size();
3418 size_t old_mem_size = new_mem_size - bytes;
3419 gclog_or_tty->print_cr("Expanding %s from %ldK by %ldK to %ldK",
3420 name(), old_mem_size/K, bytes/K, new_mem_size/K);
3421 }
3422 }
3423 return result;
3424 }
3425
3426 bool ConcurrentMarkSweepGeneration::grow_to_reserved() {
3427 assert_locked_or_safepoint(Heap_lock);
3428 bool success = true;
3429 const size_t remaining_bytes = _virtual_space.uncommitted_size();
3430 if (remaining_bytes > 0) {
3431 success = grow_by(remaining_bytes);
3432 DEBUG_ONLY(if (!success) warning("grow to reserved failed");)
3433 }
3434 return success;
3435 }
3436
3437 void ConcurrentMarkSweepGeneration::shrink_by(size_t bytes) {
3438 assert_locked_or_safepoint(Heap_lock);
3439 assert_lock_strong(freelistLock());
3440 // XXX Fix when compaction is implemented.
3441 warning("Shrinking of CMS not yet implemented");
3442 return;
3443 }
3444
3445
3446 // Simple ctor/dtor wrapper for accounting & timer chores around concurrent
3447 // phases.
3448 class CMSPhaseAccounting: public StackObj {
3449 public:
3450 CMSPhaseAccounting(CMSCollector *collector,
3451 const char *phase,
3452 bool print_cr = true);
3453 ~CMSPhaseAccounting();
3454
3455 private:
3456 CMSCollector *_collector;
3457 const char *_phase;
3458 elapsedTimer _wallclock;
3459 bool _print_cr;
3460
3461 public:
3462 // Not MT-safe; so do not pass around these StackObj's
3463 // where they may be accessed by other threads.
3464 jlong wallclock_millis() {
3465 assert(_wallclock.is_active(), "Wall clock should not stop");
3466 _wallclock.stop(); // to record time
3467 jlong ret = _wallclock.milliseconds();
3468 _wallclock.start(); // restart
3469 return ret;
3470 }
3471 };
3472
3473 CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector,
3474 const char *phase,
3475 bool print_cr) :
3476 _collector(collector), _phase(phase), _print_cr(print_cr) {
3477
3478 if (PrintCMSStatistics != 0) {
3479 _collector->resetYields();
3480 }
3481 if (PrintGCDetails && PrintGCTimeStamps) {
3482 gclog_or_tty->date_stamp(PrintGCDateStamps);
3483 gclog_or_tty->stamp();
3484 gclog_or_tty->print_cr(": [%s-concurrent-%s-start]",
3485 _collector->cmsGen()->short_name(), _phase);
3486 }
3487 _collector->resetTimer();
3488 _wallclock.start();
3489 _collector->startTimer();
3490 }
3491
3492 CMSPhaseAccounting::~CMSPhaseAccounting() {
3493 assert(_wallclock.is_active(), "Wall clock should not have stopped");
3494 _collector->stopTimer();
3495 _wallclock.stop();
3496 if (PrintGCDetails) {
3497 gclog_or_tty->date_stamp(PrintGCDateStamps);
3498 if (PrintGCTimeStamps) {
3499 gclog_or_tty->stamp();
3500 gclog_or_tty->print(": ");
3501 }
3502 gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]",
3503 _collector->cmsGen()->short_name(),
3504 _phase, _collector->timerValue(), _wallclock.seconds());
3505 if (_print_cr) {
3506 gclog_or_tty->print_cr("");
3507 }
3508 if (PrintCMSStatistics != 0) {
3509 gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase,
3510 _collector->yields());
3511 }
3512 }
3513 }
3514
3515 // CMS work
3516
3517 // Checkpoint the roots into this generation from outside
3518 // this generation. [Note this initial checkpoint need only
3519 // be approximate -- we'll do a catch up phase subsequently.]
3520 void CMSCollector::checkpointRootsInitial(bool asynch) {
3521 assert(_collectorState == InitialMarking, "Wrong collector state");
3522 check_correct_thread_executing();
3523 TraceCMSMemoryManagerStats tms(_collectorState);
3524 ReferenceProcessor* rp = ref_processor();
3525 SpecializationStats::clear();
3526 assert(_restart_addr == NULL, "Control point invariant");
3527 if (asynch) {
3528 // acquire locks for subsequent manipulations
3529 MutexLockerEx x(bitMapLock(),
3530 Mutex::_no_safepoint_check_flag);
3531 checkpointRootsInitialWork(asynch);
3532 rp->verify_no_references_recorded();
3533 rp->enable_discovery(); // enable ("weak") refs discovery
3534 _collectorState = Marking;
3535 } else {
3536 // (Weak) Refs discovery: this is controlled from genCollectedHeap::do_collection
3537 // which recognizes if we are a CMS generation, and doesn't try to turn on
3538 // discovery; verify that they aren't meddling.
3539 assert(!rp->discovery_is_atomic(),
3540 "incorrect setting of discovery predicate");
3541 assert(!rp->discovery_enabled(), "genCollectedHeap shouldn't control "
3542 "ref discovery for this generation kind");
3543 // already have locks
3544 checkpointRootsInitialWork(asynch);
3545 rp->enable_discovery(); // now enable ("weak") refs discovery
3546 _collectorState = Marking;
3547 }
3548 SpecializationStats::print();
3549 }
3550
3551 void CMSCollector::checkpointRootsInitialWork(bool asynch) {
3552 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
3553 assert(_collectorState == InitialMarking, "just checking");
3554
3555 // If there has not been a GC[n-1] since last GC[n] cycle completed,
3556 // precede our marking with a collection of all
3557 // younger generations to keep floating garbage to a minimum.
3558 // XXX: we won't do this for now -- it's an optimization to be done later.
3559
3560 // already have locks
3561 assert_lock_strong(bitMapLock());
3562 assert(_markBitMap.isAllClear(), "was reset at end of previous cycle");
3563
3564 // Setup the verification and class unloading state for this
3565 // CMS collection cycle.
3566 setup_cms_unloading_and_verification_state();
3567
3568 NOT_PRODUCT(TraceTime t("\ncheckpointRootsInitialWork",
3569 PrintGCDetails && Verbose, true, gclog_or_tty);)
3570 if (UseAdaptiveSizePolicy) {
3571 size_policy()->checkpoint_roots_initial_begin();
3572 }
3573
3574 // Reset all the PLAB chunk arrays if necessary.
3575 if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) {
3576 reset_survivor_plab_arrays();
3577 }
3578
3579 ResourceMark rm;
3580 HandleMark hm;
3581
3582 FalseClosure falseClosure;
3583 // In the case of a synchronous collection, we will elide the
3584 // remark step, so it's important to catch all the nmethod oops
3585 // in this step.
3586 // The final 'true' flag to gen_process_strong_roots will ensure this.
3587 // If 'async' is true, we can relax the nmethod tracing.
3588 MarkRefsIntoClosure notOlder(_span, &_markBitMap);
3589 GenCollectedHeap* gch = GenCollectedHeap::heap();
3590
3591 verify_work_stacks_empty();
3592 verify_overflow_empty();
3593
3594 gch->ensure_parsability(false); // fill TLABs, but no need to retire them
3595 // Update the saved marks which may affect the root scans.
3596 gch->save_marks();
3597
3598 // weak reference processing has not started yet.
3599 ref_processor()->set_enqueuing_is_done(false);
3600
3601 {
3602 // This is not needed. DEBUG_ONLY(RememberKlassesChecker imx(true);)
3603 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
3604 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
3605 gch->gen_process_strong_roots(_cmsGen->level(),
3606 true, // younger gens are roots
3607 true, // activate StrongRootsScope
3608 true, // collecting perm gen
3609 SharedHeap::ScanningOption(roots_scanning_options()),
3610 ¬Older,
3611 true, // walk all of code cache if (so & SO_CodeCache)
3612 NULL);
3613 }
3614
3615 // Clear mod-union table; it will be dirtied in the prologue of
3616 // CMS generation per each younger generation collection.
3617
3618 assert(_modUnionTable.isAllClear(),
3619 "Was cleared in most recent final checkpoint phase"
3620 " or no bits are set in the gc_prologue before the start of the next "
3621 "subsequent marking phase.");
3622
3623 // Temporarily disabled, since pre/post-consumption closures don't
3624 // care about precleaned cards
3625 #if 0
3626 {
3627 MemRegion mr = MemRegion((HeapWord*)_virtual_space.low(),
3628 (HeapWord*)_virtual_space.high());
3629 _ct->ct_bs()->preclean_dirty_cards(mr);
3630 }
3631 #endif
3632
3633 // Save the end of the used_region of the constituent generations
3634 // to be used to limit the extent of sweep in each generation.
3635 save_sweep_limits();
3636 if (UseAdaptiveSizePolicy) {
3637 size_policy()->checkpoint_roots_initial_end(gch->gc_cause());
3638 }
3639 verify_overflow_empty();
3640 }
3641
3642 bool CMSCollector::markFromRoots(bool asynch) {
3643 // we might be tempted to assert that:
3644 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
3645 // "inconsistent argument?");
3646 // However that wouldn't be right, because it's possible that
3647 // a safepoint is indeed in progress as a younger generation
3648 // stop-the-world GC happens even as we mark in this generation.
3649 assert(_collectorState == Marking, "inconsistent state?");
3650 check_correct_thread_executing();
3651 verify_overflow_empty();
3652
3653 bool res;
3654 if (asynch) {
3655
3656 // Start the timers for adaptive size policy for the concurrent phases
3657 // Do it here so that the foreground MS can use the concurrent
3658 // timer since a foreground MS might has the sweep done concurrently
3659 // or STW.
3660 if (UseAdaptiveSizePolicy) {
3661 size_policy()->concurrent_marking_begin();
3662 }
3663
3664 // Weak ref discovery note: We may be discovering weak
3665 // refs in this generation concurrent (but interleaved) with
3666 // weak ref discovery by a younger generation collector.
3667
3668 CMSTokenSyncWithLocks ts(true, bitMapLock());
3669 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
3670 CMSPhaseAccounting pa(this, "mark", !PrintGCDetails);
3671 res = markFromRootsWork(asynch);
3672 if (res) {
3673 _collectorState = Precleaning;
3674 } else { // We failed and a foreground collection wants to take over
3675 assert(_foregroundGCIsActive, "internal state inconsistency");
3676 assert(_restart_addr == NULL, "foreground will restart from scratch");
3677 if (PrintGCDetails) {
3678 gclog_or_tty->print_cr("bailing out to foreground collection");
3679 }
3680 }
3681 if (UseAdaptiveSizePolicy) {
3682 size_policy()->concurrent_marking_end();
3683 }
3684 } else {
3685 assert(SafepointSynchronize::is_at_safepoint(),
3686 "inconsistent with asynch == false");
3687 if (UseAdaptiveSizePolicy) {
3688 size_policy()->ms_collection_marking_begin();
3689 }
3690 // already have locks
3691 res = markFromRootsWork(asynch);
3692 _collectorState = FinalMarking;
3693 if (UseAdaptiveSizePolicy) {
3694 GenCollectedHeap* gch = GenCollectedHeap::heap();
3695 size_policy()->ms_collection_marking_end(gch->gc_cause());
3696 }
3697 }
3698 verify_overflow_empty();
3699 return res;
3700 }
3701
3702 bool CMSCollector::markFromRootsWork(bool asynch) {
3703 // iterate over marked bits in bit map, doing a full scan and mark
3704 // from these roots using the following algorithm:
3705 // . if oop is to the right of the current scan pointer,
3706 // mark corresponding bit (we'll process it later)
3707 // . else (oop is to left of current scan pointer)
3708 // push oop on marking stack
3709 // . drain the marking stack
3710
3711 // Note that when we do a marking step we need to hold the
3712 // bit map lock -- recall that direct allocation (by mutators)
3713 // and promotion (by younger generation collectors) is also
3714 // marking the bit map. [the so-called allocate live policy.]
3715 // Because the implementation of bit map marking is not
3716 // robust wrt simultaneous marking of bits in the same word,
3717 // we need to make sure that there is no such interference
3718 // between concurrent such updates.
3719
3720 // already have locks
3721 assert_lock_strong(bitMapLock());
3722
3723 // Clear the revisit stack, just in case there are any
3724 // obsolete contents from a short-circuited previous CMS cycle.
3725 _revisitStack.reset();
3726 verify_work_stacks_empty();
3727 verify_overflow_empty();
3728 assert(_revisitStack.isEmpty(), "tabula rasa");
3729 DEBUG_ONLY(RememberKlassesChecker cmx(should_unload_classes());)
3730 bool result = false;
3731 if (CMSConcurrentMTEnabled && ConcGCThreads > 0) {
3732 result = do_marking_mt(asynch);
3733 } else {
3734 result = do_marking_st(asynch);
3735 }
3736 return result;
3737 }
3738
3739 // Forward decl
3740 class CMSConcMarkingTask;
3741
3742 class CMSConcMarkingTerminator: public ParallelTaskTerminator {
3743 CMSCollector* _collector;
3744 CMSConcMarkingTask* _task;
3745 public:
3746 virtual void yield();
3747
3748 // "n_threads" is the number of threads to be terminated.
3749 // "queue_set" is a set of work queues of other threads.
3750 // "collector" is the CMS collector associated with this task terminator.
3751 // "yield" indicates whether we need the gang as a whole to yield.
3752 CMSConcMarkingTerminator(int n_threads, TaskQueueSetSuper* queue_set, CMSCollector* collector) :
3753 ParallelTaskTerminator(n_threads, queue_set),
3754 _collector(collector) { }
3755
3756 void set_task(CMSConcMarkingTask* task) {
3757 _task = task;
3758 }
3759 };
3760
3761 class CMSConcMarkingTerminatorTerminator: public TerminatorTerminator {
3762 CMSConcMarkingTask* _task;
3763 public:
3764 bool should_exit_termination();
3765 void set_task(CMSConcMarkingTask* task) {
3766 _task = task;
3767 }
3768 };
3769
3770 // MT Concurrent Marking Task
3771 class CMSConcMarkingTask: public YieldingFlexibleGangTask {
3772 CMSCollector* _collector;
3773 int _n_workers; // requested/desired # workers
3774 bool _asynch;
3775 bool _result;
3776 CompactibleFreeListSpace* _cms_space;
3777 CompactibleFreeListSpace* _perm_space;
3778 char _pad_front[64]; // padding to ...
3779 HeapWord* _global_finger; // ... avoid sharing cache line
3780 char _pad_back[64];
3781 HeapWord* _restart_addr;
3782
3783 // Exposed here for yielding support
3784 Mutex* const _bit_map_lock;
3785
3786 // The per thread work queues, available here for stealing
3787 OopTaskQueueSet* _task_queues;
3788
3789 // Termination (and yielding) support
3790 CMSConcMarkingTerminator _term;
3791 CMSConcMarkingTerminatorTerminator _term_term;
3792
3793 public:
3794 CMSConcMarkingTask(CMSCollector* collector,
3795 CompactibleFreeListSpace* cms_space,
3796 CompactibleFreeListSpace* perm_space,
3797 bool asynch,
3798 YieldingFlexibleWorkGang* workers,
3799 OopTaskQueueSet* task_queues):
3800 YieldingFlexibleGangTask("Concurrent marking done multi-threaded"),
3801 _collector(collector),
3802 _cms_space(cms_space),
3803 _perm_space(perm_space),
3804 _asynch(asynch), _n_workers(0), _result(true),
3805 _task_queues(task_queues),
3806 _term(_n_workers, task_queues, _collector),
3807 _bit_map_lock(collector->bitMapLock())
3808 {
3809 _requested_size = _n_workers;
3810 _term.set_task(this);
3811 _term_term.set_task(this);
3812 assert(_cms_space->bottom() < _perm_space->bottom(),
3813 "Finger incorrectly initialized below");
3814 _restart_addr = _global_finger = _cms_space->bottom();
3815 }
3816
3817
3818 OopTaskQueueSet* task_queues() { return _task_queues; }
3819
3820 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
3821
3822 HeapWord** global_finger_addr() { return &_global_finger; }
3823
3824 CMSConcMarkingTerminator* terminator() { return &_term; }
3825
3826 virtual void set_for_termination(int active_workers) {
3827 terminator()->reset_for_reuse(active_workers);
3828 }
3829
3830 void work(int i);
3831 bool should_yield() {
3832 return ConcurrentMarkSweepThread::should_yield()
3833 && !_collector->foregroundGCIsActive()
3834 && _asynch;
3835 }
3836
3837 virtual void coordinator_yield(); // stuff done by coordinator
3838 bool result() { return _result; }
3839
3840 void reset(HeapWord* ra) {
3841 assert(_global_finger >= _cms_space->end(), "Postcondition of ::work(i)");
3842 assert(_global_finger >= _perm_space->end(), "Postcondition of ::work(i)");
3843 assert(ra < _perm_space->end(), "ra too large");
3844 _restart_addr = _global_finger = ra;
3845 _term.reset_for_reuse();
3846 }
3847
3848 static bool get_work_from_overflow_stack(CMSMarkStack* ovflw_stk,
3849 OopTaskQueue* work_q);
3850
3851 private:
3852 void do_scan_and_mark(int i, CompactibleFreeListSpace* sp);
3853 void do_work_steal(int i);
3854 void bump_global_finger(HeapWord* f);
3855 };
3856
3857 bool CMSConcMarkingTerminatorTerminator::should_exit_termination() {
3858 assert(_task != NULL, "Error");
3859 return _task->yielding();
3860 // Note that we do not need the disjunct || _task->should_yield() above
3861 // because we want terminating threads to yield only if the task
3862 // is already in the midst of yielding, which happens only after at least one
3863 // thread has yielded.
3864 }
3865
3866 void CMSConcMarkingTerminator::yield() {
3867 if (_task->should_yield()) {
3868 _task->yield();
3869 } else {
3870 ParallelTaskTerminator::yield();
3871 }
3872 }
3873
3874 ////////////////////////////////////////////////////////////////
3875 // Concurrent Marking Algorithm Sketch
3876 ////////////////////////////////////////////////////////////////
3877 // Until all tasks exhausted (both spaces):
3878 // -- claim next available chunk
3879 // -- bump global finger via CAS
3880 // -- find first object that starts in this chunk
3881 // and start scanning bitmap from that position
3882 // -- scan marked objects for oops
3883 // -- CAS-mark target, and if successful:
3884 // . if target oop is above global finger (volatile read)
3885 // nothing to do
3886 // . if target oop is in chunk and above local finger
3887 // then nothing to do
3888 // . else push on work-queue
3889 // -- Deal with possible overflow issues:
3890 // . local work-queue overflow causes stuff to be pushed on
3891 // global (common) overflow queue
3892 // . always first empty local work queue
3893 // . then get a batch of oops from global work queue if any
3894 // . then do work stealing
3895 // -- When all tasks claimed (both spaces)
3896 // and local work queue empty,
3897 // then in a loop do:
3898 // . check global overflow stack; steal a batch of oops and trace
3899 // . try to steal from other threads oif GOS is empty
3900 // . if neither is available, offer termination
3901 // -- Terminate and return result
3902 //
3903 void CMSConcMarkingTask::work(int i) {
3904 elapsedTimer _timer;
3905 ResourceMark rm;
3906 HandleMark hm;
3907
3908 DEBUG_ONLY(_collector->verify_overflow_empty();)
3909
3910 // Before we begin work, our work queue should be empty
3911 assert(work_queue(i)->size() == 0, "Expected to be empty");
3912 // Scan the bitmap covering _cms_space, tracing through grey objects.
3913 _timer.start();
3914 do_scan_and_mark(i, _cms_space);
3915 _timer.stop();
3916 if (PrintCMSStatistics != 0) {
3917 gclog_or_tty->print_cr("Finished cms space scanning in %dth thread: %3.3f sec",
3918 i, _timer.seconds()); // XXX: need xxx/xxx type of notation, two timers
3919 }
3920
3921 // ... do the same for the _perm_space
3922 _timer.reset();
3923 _timer.start();
3924 do_scan_and_mark(i, _perm_space);
3925 _timer.stop();
3926 if (PrintCMSStatistics != 0) {
3927 gclog_or_tty->print_cr("Finished perm space scanning in %dth thread: %3.3f sec",
3928 i, _timer.seconds()); // XXX: need xxx/xxx type of notation, two timers
3929 }
3930
3931 // ... do work stealing
3932 _timer.reset();
3933 _timer.start();
3934 do_work_steal(i);
3935 _timer.stop();
3936 if (PrintCMSStatistics != 0) {
3937 gclog_or_tty->print_cr("Finished work stealing in %dth thread: %3.3f sec",
3938 i, _timer.seconds()); // XXX: need xxx/xxx type of notation, two timers
3939 }
3940 assert(_collector->_markStack.isEmpty(), "Should have been emptied");
3941 assert(work_queue(i)->size() == 0, "Should have been emptied");
3942 // Note that under the current task protocol, the
3943 // following assertion is true even of the spaces
3944 // expanded since the completion of the concurrent
3945 // marking. XXX This will likely change under a strict
3946 // ABORT semantics.
3947 assert(_global_finger > _cms_space->end() &&
3948 _global_finger >= _perm_space->end(),
3949 "All tasks have been completed");
3950 DEBUG_ONLY(_collector->verify_overflow_empty();)
3951 }
3952
3953 void CMSConcMarkingTask::bump_global_finger(HeapWord* f) {
3954 HeapWord* read = _global_finger;
3955 HeapWord* cur = read;
3956 while (f > read) {
3957 cur = read;
3958 read = (HeapWord*) Atomic::cmpxchg_ptr(f, &_global_finger, cur);
3959 if (cur == read) {
3960 // our cas succeeded
3961 assert(_global_finger >= f, "protocol consistency");
3962 break;
3963 }
3964 }
3965 }
3966
3967 // This is really inefficient, and should be redone by
3968 // using (not yet available) block-read and -write interfaces to the
3969 // stack and the work_queue. XXX FIX ME !!!
3970 bool CMSConcMarkingTask::get_work_from_overflow_stack(CMSMarkStack* ovflw_stk,
3971 OopTaskQueue* work_q) {
3972 // Fast lock-free check
3973 if (ovflw_stk->length() == 0) {
3974 return false;
3975 }
3976 assert(work_q->size() == 0, "Shouldn't steal");
3977 MutexLockerEx ml(ovflw_stk->par_lock(),
3978 Mutex::_no_safepoint_check_flag);
3979 // Grab up to 1/4 the size of the work queue
3980 size_t num = MIN2((size_t)(work_q->max_elems() - work_q->size())/4,
3981 (size_t)ParGCDesiredObjsFromOverflowList);
3982 num = MIN2(num, ovflw_stk->length());
3983 for (int i = (int) num; i > 0; i--) {
3984 oop cur = ovflw_stk->pop();
3985 assert(cur != NULL, "Counted wrong?");
3986 work_q->push(cur);
3987 }
3988 return num > 0;
3989 }
3990
3991 void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
3992 SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
3993 int n_tasks = pst->n_tasks();
3994 // We allow that there may be no tasks to do here because
3995 // we are restarting after a stack overflow.
3996 assert(pst->valid() || n_tasks == 0, "Uninitialized use?");
3997 int nth_task = 0;
3998
3999 HeapWord* aligned_start = sp->bottom();
4000 if (sp->used_region().contains(_restart_addr)) {
4001 // Align down to a card boundary for the start of 0th task
4002 // for this space.
4003 aligned_start =
4004 (HeapWord*)align_size_down((uintptr_t)_restart_addr,
4005 CardTableModRefBS::card_size);
4006 }
4007
4008 size_t chunk_size = sp->marking_task_size();
4009 while (!pst->is_task_claimed(/* reference */ nth_task)) {
4010 // Having claimed the nth task in this space,
4011 // compute the chunk that it corresponds to:
4012 MemRegion span = MemRegion(aligned_start + nth_task*chunk_size,
4013 aligned_start + (nth_task+1)*chunk_size);
4014 // Try and bump the global finger via a CAS;
4015 // note that we need to do the global finger bump
4016 // _before_ taking the intersection below, because
4017 // the task corresponding to that region will be
4018 // deemed done even if the used_region() expands
4019 // because of allocation -- as it almost certainly will
4020 // during start-up while the threads yield in the
4021 // closure below.
4022 HeapWord* finger = span.end();
4023 bump_global_finger(finger); // atomically
4024 // There are null tasks here corresponding to chunks
4025 // beyond the "top" address of the space.
4026 span = span.intersection(sp->used_region());
4027 if (!span.is_empty()) { // Non-null task
4028 HeapWord* prev_obj;
4029 assert(!span.contains(_restart_addr) || nth_task == 0,
4030 "Inconsistency");
4031 if (nth_task == 0) {
4032 // For the 0th task, we'll not need to compute a block_start.
4033 if (span.contains(_restart_addr)) {
4034 // In the case of a restart because of stack overflow,
4035 // we might additionally skip a chunk prefix.
4036 prev_obj = _restart_addr;
4037 } else {
4038 prev_obj = span.start();
4039 }
4040 } else {
4041 // We want to skip the first object because
4042 // the protocol is to scan any object in its entirety
4043 // that _starts_ in this span; a fortiori, any
4044 // object starting in an earlier span is scanned
4045 // as part of an earlier claimed task.
4046 // Below we use the "careful" version of block_start
4047 // so we do not try to navigate uninitialized objects.
4048 prev_obj = sp->block_start_careful(span.start());
4049 // Below we use a variant of block_size that uses the
4050 // Printezis bits to avoid waiting for allocated
4051 // objects to become initialized/parsable.
4052 while (prev_obj < span.start()) {
4053 size_t sz = sp->block_size_no_stall(prev_obj, _collector);
4054 if (sz > 0) {
4055 prev_obj += sz;
4056 } else {
4057 // In this case we may end up doing a bit of redundant
4058 // scanning, but that appears unavoidable, short of
4059 // locking the free list locks; see bug 6324141.
4060 break;
4061 }
4062 }
4063 }
4064 if (prev_obj < span.end()) {
4065 MemRegion my_span = MemRegion(prev_obj, span.end());
4066 // Do the marking work within a non-empty span --
4067 // the last argument to the constructor indicates whether the
4068 // iteration should be incremental with periodic yields.
4069 Par_MarkFromRootsClosure cl(this, _collector, my_span,
4070 &_collector->_markBitMap,
4071 work_queue(i),
4072 &_collector->_markStack,
4073 &_collector->_revisitStack,
4074 _asynch);
4075 _collector->_markBitMap.iterate(&cl, my_span.start(), my_span.end());
4076 } // else nothing to do for this task
4077 } // else nothing to do for this task
4078 }
4079 // We'd be tempted to assert here that since there are no
4080 // more tasks left to claim in this space, the global_finger
4081 // must exceed space->top() and a fortiori space->end(). However,
4082 // that would not quite be correct because the bumping of
4083 // global_finger occurs strictly after the claiming of a task,
4084 // so by the time we reach here the global finger may not yet
4085 // have been bumped up by the thread that claimed the last
4086 // task.
4087 pst->all_tasks_completed();
4088 }
4089
4090 class Par_ConcMarkingClosure: public Par_KlassRememberingOopClosure {
4091 private:
4092 CMSConcMarkingTask* _task;
4093 MemRegion _span;
4094 CMSBitMap* _bit_map;
4095 CMSMarkStack* _overflow_stack;
4096 OopTaskQueue* _work_queue;
4097 protected:
4098 DO_OOP_WORK_DEFN
4099 public:
4100 Par_ConcMarkingClosure(CMSCollector* collector, CMSConcMarkingTask* task, OopTaskQueue* work_queue,
4101 CMSBitMap* bit_map, CMSMarkStack* overflow_stack,
4102 CMSMarkStack* revisit_stack):
4103 Par_KlassRememberingOopClosure(collector, NULL, revisit_stack),
4104 _task(task),
4105 _span(collector->_span),
4106 _work_queue(work_queue),
4107 _bit_map(bit_map),
4108 _overflow_stack(overflow_stack)
4109 { }
4110 virtual void do_oop(oop* p);
4111 virtual void do_oop(narrowOop* p);
4112 void trim_queue(size_t max);
4113 void handle_stack_overflow(HeapWord* lost);
4114 void do_yield_check() {
4115 if (_task->should_yield()) {
4116 _task->yield();
4117 }
4118 }
4119 };
4120
4121 // Grey object scanning during work stealing phase --
4122 // the salient assumption here is that any references
4123 // that are in these stolen objects being scanned must
4124 // already have been initialized (else they would not have
4125 // been published), so we do not need to check for
4126 // uninitialized objects before pushing here.
4127 void Par_ConcMarkingClosure::do_oop(oop obj) {
4128 assert(obj->is_oop_or_null(true), "expected an oop or NULL");
4129 HeapWord* addr = (HeapWord*)obj;
4130 // Check if oop points into the CMS generation
4131 // and is not marked
4132 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
4133 // a white object ...
4134 // If we manage to "claim" the object, by being the
4135 // first thread to mark it, then we push it on our
4136 // marking stack
4137 if (_bit_map->par_mark(addr)) { // ... now grey
4138 // push on work queue (grey set)
4139 bool simulate_overflow = false;
4140 NOT_PRODUCT(
4141 if (CMSMarkStackOverflowALot &&
4142 _collector->simulate_overflow()) {
4143 // simulate a stack overflow
4144 simulate_overflow = true;
4145 }
4146 )
4147 if (simulate_overflow ||
4148 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
4149 // stack overflow
4150 if (PrintCMSStatistics != 0) {
4151 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
4152 SIZE_FORMAT, _overflow_stack->capacity());
4153 }
4154 // We cannot assert that the overflow stack is full because
4155 // it may have been emptied since.
4156 assert(simulate_overflow ||
4157 _work_queue->size() == _work_queue->max_elems(),
4158 "Else push should have succeeded");
4159 handle_stack_overflow(addr);
4160 }
4161 } // Else, some other thread got there first
4162 do_yield_check();
4163 }
4164 }
4165
4166 void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
4167 void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); }
4168
4169 void Par_ConcMarkingClosure::trim_queue(size_t max) {
4170 while (_work_queue->size() > max) {
4171 oop new_oop;
4172 if (_work_queue->pop_local(new_oop)) {
4173 assert(new_oop->is_oop(), "Should be an oop");
4174 assert(_bit_map->isMarked((HeapWord*)new_oop), "Grey object");
4175 assert(_span.contains((HeapWord*)new_oop), "Not in span");
4176 assert(new_oop->is_parsable(), "Should be parsable");
4177 new_oop->oop_iterate(this); // do_oop() above
4178 do_yield_check();
4179 }
4180 }
4181 }
4182
4183 // Upon stack overflow, we discard (part of) the stack,
4184 // remembering the least address amongst those discarded
4185 // in CMSCollector's _restart_address.
4186 void Par_ConcMarkingClosure::handle_stack_overflow(HeapWord* lost) {
4187 // We need to do this under a mutex to prevent other
4188 // workers from interfering with the work done below.
4189 MutexLockerEx ml(_overflow_stack->par_lock(),
4190 Mutex::_no_safepoint_check_flag);
4191 // Remember the least grey address discarded
4192 HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost);
4193 _collector->lower_restart_addr(ra);
4194 _overflow_stack->reset(); // discard stack contents
4195 _overflow_stack->expand(); // expand the stack if possible
4196 }
4197
4198
4199 void CMSConcMarkingTask::do_work_steal(int i) {
4200 OopTaskQueue* work_q = work_queue(i);
4201 oop obj_to_scan;
4202 CMSBitMap* bm = &(_collector->_markBitMap);
4203 CMSMarkStack* ovflw = &(_collector->_markStack);
4204 CMSMarkStack* revisit = &(_collector->_revisitStack);
4205 int* seed = _collector->hash_seed(i);
4206 Par_ConcMarkingClosure cl(_collector, this, work_q, bm, ovflw, revisit);
4207 while (true) {
4208 cl.trim_queue(0);
4209 assert(work_q->size() == 0, "Should have been emptied above");
4210 if (get_work_from_overflow_stack(ovflw, work_q)) {
4211 // Can't assert below because the work obtained from the
4212 // overflow stack may already have been stolen from us.
4213 // assert(work_q->size() > 0, "Work from overflow stack");
4214 continue;
4215 } else if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
4216 assert(obj_to_scan->is_oop(), "Should be an oop");
4217 assert(bm->isMarked((HeapWord*)obj_to_scan), "Grey object");
4218 obj_to_scan->oop_iterate(&cl);
4219 } else if (terminator()->offer_termination(&_term_term)) {
4220 assert(work_q->size() == 0, "Impossible!");
4221 break;
4222 } else if (yielding() || should_yield()) {
4223 yield();
4224 }
4225 }
4226 }
4227
4228 // This is run by the CMS (coordinator) thread.
4229 void CMSConcMarkingTask::coordinator_yield() {
4230 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
4231 "CMS thread should hold CMS token");
4232 DEBUG_ONLY(RememberKlassesChecker mux(false);)
4233 // First give up the locks, then yield, then re-lock
4234 // We should probably use a constructor/destructor idiom to
4235 // do this unlock/lock or modify the MutexUnlocker class to
4236 // serve our purpose. XXX
4237 assert_lock_strong(_bit_map_lock);
4238 _bit_map_lock->unlock();
4239 ConcurrentMarkSweepThread::desynchronize(true);
4240 ConcurrentMarkSweepThread::acknowledge_yield_request();
4241 _collector->stopTimer();
4242 if (PrintCMSStatistics != 0) {
4243 _collector->incrementYields();
4244 }
4245 _collector->icms_wait();
4246
4247 // It is possible for whichever thread initiated the yield request
4248 // not to get a chance to wake up and take the bitmap lock between
4249 // this thread releasing it and reacquiring it. So, while the
4250 // should_yield() flag is on, let's sleep for a bit to give the
4251 // other thread a chance to wake up. The limit imposed on the number
4252 // of iterations is defensive, to avoid any unforseen circumstances
4253 // putting us into an infinite loop. Since it's always been this
4254 // (coordinator_yield()) method that was observed to cause the
4255 // problem, we are using a parameter (CMSCoordinatorYieldSleepCount)
4256 // which is by default non-zero. For the other seven methods that
4257 // also perform the yield operation, as are using a different
4258 // parameter (CMSYieldSleepCount) which is by default zero. This way we
4259 // can enable the sleeping for those methods too, if necessary.
4260 // See 6442774.
4261 //
4262 // We really need to reconsider the synchronization between the GC
4263 // thread and the yield-requesting threads in the future and we
4264 // should really use wait/notify, which is the recommended
4265 // way of doing this type of interaction. Additionally, we should
4266 // consolidate the eight methods that do the yield operation and they
4267 // are almost identical into one for better maintenability and
4268 // readability. See 6445193.
4269 //
4270 // Tony 2006.06.29
4271 for (unsigned i = 0; i < CMSCoordinatorYieldSleepCount &&
4272 ConcurrentMarkSweepThread::should_yield() &&
4273 !CMSCollector::foregroundGCIsActive(); ++i) {
4274 os::sleep(Thread::current(), 1, false);
4275 ConcurrentMarkSweepThread::acknowledge_yield_request();
4276 }
4277
4278 ConcurrentMarkSweepThread::synchronize(true);
4279 _bit_map_lock->lock_without_safepoint_check();
4280 _collector->startTimer();
4281 }
4282
4283 bool CMSCollector::do_marking_mt(bool asynch) {
4284 assert(ConcGCThreads > 0 && conc_workers() != NULL, "precondition");
4285 // In the future this would be determined ergonomically, based
4286 // on #cpu's, # active mutator threads (and load), and mutation rate.
4287 int num_workers = ConcGCThreads;
4288
4289 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
4290 CompactibleFreeListSpace* perm_space = _permGen->cmsSpace();
4291
4292 CMSConcMarkingTask tsk(this,
4293 cms_space,
4294 perm_space,
4295 asynch,
4296 conc_workers(),
4297 task_queues());
4298
4299 // Since the actual number of workers we get may be different
4300 // from the number we requested above, do we need to do anything different
4301 // below? In particular, may be we need to subclass the SequantialSubTasksDone
4302 // class?? XXX
4303 cms_space ->initialize_sequential_subtasks_for_marking(num_workers);
4304 perm_space->initialize_sequential_subtasks_for_marking(num_workers);
4305
4306 // Refs discovery is already non-atomic.
4307 assert(!ref_processor()->discovery_is_atomic(), "Should be non-atomic");
4308 // Mutate the Refs discovery so it is MT during the
4309 // multi-threaded marking phase.
4310 ReferenceProcessorMTMutator mt(ref_processor(), num_workers > 1);
4311 DEBUG_ONLY(RememberKlassesChecker cmx(should_unload_classes());)
4312 conc_workers()->start_task(&tsk);
4313 while (tsk.yielded()) {
4314 tsk.coordinator_yield();
4315 conc_workers()->continue_task(&tsk);
4316 }
4317 // If the task was aborted, _restart_addr will be non-NULL
4318 assert(tsk.completed() || _restart_addr != NULL, "Inconsistency");
4319 while (_restart_addr != NULL) {
4320 // XXX For now we do not make use of ABORTED state and have not
4321 // yet implemented the right abort semantics (even in the original
4322 // single-threaded CMS case). That needs some more investigation
4323 // and is deferred for now; see CR# TBF. 07252005YSR. XXX
4324 assert(!CMSAbortSemantics || tsk.aborted(), "Inconsistency");
4325 // If _restart_addr is non-NULL, a marking stack overflow
4326 // occurred; we need to do a fresh marking iteration from the
4327 // indicated restart address.
4328 if (_foregroundGCIsActive && asynch) {
4329 // We may be running into repeated stack overflows, having
4330 // reached the limit of the stack size, while making very
4331 // slow forward progress. It may be best to bail out and
4332 // let the foreground collector do its job.
4333 // Clear _restart_addr, so that foreground GC
4334 // works from scratch. This avoids the headache of
4335 // a "rescan" which would otherwise be needed because
4336 // of the dirty mod union table & card table.
4337 _restart_addr = NULL;
4338 return false;
4339 }
4340 // Adjust the task to restart from _restart_addr
4341 tsk.reset(_restart_addr);
4342 cms_space ->initialize_sequential_subtasks_for_marking(num_workers,
4343 _restart_addr);
4344 perm_space->initialize_sequential_subtasks_for_marking(num_workers,
4345 _restart_addr);
4346 _restart_addr = NULL;
4347 // Get the workers going again
4348 conc_workers()->start_task(&tsk);
4349 while (tsk.yielded()) {
4350 tsk.coordinator_yield();
4351 conc_workers()->continue_task(&tsk);
4352 }
4353 }
4354 assert(tsk.completed(), "Inconsistency");
4355 assert(tsk.result() == true, "Inconsistency");
4356 return true;
4357 }
4358
4359 bool CMSCollector::do_marking_st(bool asynch) {
4360 ResourceMark rm;
4361 HandleMark hm;
4362
4363 MarkFromRootsClosure markFromRootsClosure(this, _span, &_markBitMap,
4364 &_markStack, &_revisitStack, CMSYield && asynch);
4365 // the last argument to iterate indicates whether the iteration
4366 // should be incremental with periodic yields.
4367 _markBitMap.iterate(&markFromRootsClosure);
4368 // If _restart_addr is non-NULL, a marking stack overflow
4369 // occurred; we need to do a fresh iteration from the
4370 // indicated restart address.
4371 while (_restart_addr != NULL) {
4372 if (_foregroundGCIsActive && asynch) {
4373 // We may be running into repeated stack overflows, having
4374 // reached the limit of the stack size, while making very
4375 // slow forward progress. It may be best to bail out and
4376 // let the foreground collector do its job.
4377 // Clear _restart_addr, so that foreground GC
4378 // works from scratch. This avoids the headache of
4379 // a "rescan" which would otherwise be needed because
4380 // of the dirty mod union table & card table.
4381 _restart_addr = NULL;
4382 return false; // indicating failure to complete marking
4383 }
4384 // Deal with stack overflow:
4385 // we restart marking from _restart_addr
4386 HeapWord* ra = _restart_addr;
4387 markFromRootsClosure.reset(ra);
4388 _restart_addr = NULL;
4389 _markBitMap.iterate(&markFromRootsClosure, ra, _span.end());
4390 }
4391 return true;
4392 }
4393
4394 void CMSCollector::preclean() {
4395 check_correct_thread_executing();
4396 assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread");
4397 verify_work_stacks_empty();
4398 verify_overflow_empty();
4399 _abort_preclean = false;
4400 if (CMSPrecleaningEnabled) {
4401 // Precleaning is currently not MT but the reference processor
4402 // may be set for MT. Disable it temporarily here.
4403 ReferenceProcessor* rp = ref_processor();
4404 ReferenceProcessorMTProcMutator z(rp, false);
4405 _eden_chunk_index = 0;
4406 size_t used = get_eden_used();
4407 size_t capacity = get_eden_capacity();
4408 // Don't start sampling unless we will get sufficiently
4409 // many samples.
4410 if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100)
4411 * CMSScheduleRemarkEdenPenetration)) {
4412 _start_sampling = true;
4413 } else {
4414 _start_sampling = false;
4415 }
4416 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
4417 CMSPhaseAccounting pa(this, "preclean", !PrintGCDetails);
4418 preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1);
4419 }
4420 CMSTokenSync x(true); // is cms thread
4421 if (CMSPrecleaningEnabled) {
4422 sample_eden();
4423 _collectorState = AbortablePreclean;
4424 } else {
4425 _collectorState = FinalMarking;
4426 }
4427 verify_work_stacks_empty();
4428 verify_overflow_empty();
4429 }
4430
4431 // Try and schedule the remark such that young gen
4432 // occupancy is CMSScheduleRemarkEdenPenetration %.
4433 void CMSCollector::abortable_preclean() {
4434 check_correct_thread_executing();
4435 assert(CMSPrecleaningEnabled, "Inconsistent control state");
4436 assert(_collectorState == AbortablePreclean, "Inconsistent control state");
4437
4438 // If Eden's current occupancy is below this threshold,
4439 // immediately schedule the remark; else preclean
4440 // past the next scavenge in an effort to
4441 // schedule the pause as described avove. By choosing
4442 // CMSScheduleRemarkEdenSizeThreshold >= max eden size
4443 // we will never do an actual abortable preclean cycle.
4444 if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) {
4445 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
4446 CMSPhaseAccounting pa(this, "abortable-preclean", !PrintGCDetails);
4447 // We need more smarts in the abortable preclean
4448 // loop below to deal with cases where allocation
4449 // in young gen is very very slow, and our precleaning
4450 // is running a losing race against a horde of
4451 // mutators intent on flooding us with CMS updates
4452 // (dirty cards).
4453 // One, admittedly dumb, strategy is to give up
4454 // after a certain number of abortable precleaning loops
4455 // or after a certain maximum time. We want to make
4456 // this smarter in the next iteration.
4457 // XXX FIX ME!!! YSR
4458 size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0;
4459 while (!(should_abort_preclean() ||
4460 ConcurrentMarkSweepThread::should_terminate())) {
4461 workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2);
4462 cumworkdone += workdone;
4463 loops++;
4464 // Voluntarily terminate abortable preclean phase if we have
4465 // been at it for too long.
4466 if ((CMSMaxAbortablePrecleanLoops != 0) &&
4467 loops >= CMSMaxAbortablePrecleanLoops) {
4468 if (PrintGCDetails) {
4469 gclog_or_tty->print(" CMS: abort preclean due to loops ");
4470 }
4471 break;
4472 }
4473 if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) {
4474 if (PrintGCDetails) {
4475 gclog_or_tty->print(" CMS: abort preclean due to time ");
4476 }
4477 break;
4478 }
4479 // If we are doing little work each iteration, we should
4480 // take a short break.
4481 if (workdone < CMSAbortablePrecleanMinWorkPerIteration) {
4482 // Sleep for some time, waiting for work to accumulate
4483 stopTimer();
4484 cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis);
4485 startTimer();
4486 waited++;
4487 }
4488 }
4489 if (PrintCMSStatistics > 0) {
4490 gclog_or_tty->print(" [%d iterations, %d waits, %d cards)] ",
4491 loops, waited, cumworkdone);
4492 }
4493 }
4494 CMSTokenSync x(true); // is cms thread
4495 if (_collectorState != Idling) {
4496 assert(_collectorState == AbortablePreclean,
4497 "Spontaneous state transition?");
4498 _collectorState = FinalMarking;
4499 } // Else, a foreground collection completed this CMS cycle.
4500 return;
4501 }
4502
4503 // Respond to an Eden sampling opportunity
4504 void CMSCollector::sample_eden() {
4505 // Make sure a young gc cannot sneak in between our
4506 // reading and recording of a sample.
4507 assert(Thread::current()->is_ConcurrentGC_thread(),
4508 "Only the cms thread may collect Eden samples");
4509 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
4510 "Should collect samples while holding CMS token");
4511 if (!_start_sampling) {
4512 return;
4513 }
4514 if (_eden_chunk_array) {
4515 if (_eden_chunk_index < _eden_chunk_capacity) {
4516 _eden_chunk_array[_eden_chunk_index] = *_top_addr; // take sample
4517 assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr,
4518 "Unexpected state of Eden");
4519 // We'd like to check that what we just sampled is an oop-start address;
4520 // however, we cannot do that here since the object may not yet have been
4521 // initialized. So we'll instead do the check when we _use_ this sample
4522 // later.
4523 if (_eden_chunk_index == 0 ||
4524 (pointer_delta(_eden_chunk_array[_eden_chunk_index],
4525 _eden_chunk_array[_eden_chunk_index-1])
4526 >= CMSSamplingGrain)) {
4527 _eden_chunk_index++; // commit sample
4528 }
4529 }
4530 }
4531 if ((_collectorState == AbortablePreclean) && !_abort_preclean) {
4532 size_t used = get_eden_used();
4533 size_t capacity = get_eden_capacity();
4534 assert(used <= capacity, "Unexpected state of Eden");
4535 if (used > (capacity/100 * CMSScheduleRemarkEdenPenetration)) {
4536 _abort_preclean = true;
4537 }
4538 }
4539 }
4540
4541
4542 size_t CMSCollector::preclean_work(bool clean_refs, bool clean_survivor) {
4543 assert(_collectorState == Precleaning ||
4544 _collectorState == AbortablePreclean, "incorrect state");
4545 ResourceMark rm;
4546 HandleMark hm;
4547 // Do one pass of scrubbing the discovered reference lists
4548 // to remove any reference objects with strongly-reachable
4549 // referents.
4550 if (clean_refs) {
4551 ReferenceProcessor* rp = ref_processor();
4552 CMSPrecleanRefsYieldClosure yield_cl(this);
4553 assert(rp->span().equals(_span), "Spans should be equal");
4554 CMSKeepAliveClosure keep_alive(this, _span, &_markBitMap,
4555 &_markStack, &_revisitStack,
4556 true /* preclean */);
4557 CMSDrainMarkingStackClosure complete_trace(this,
4558 _span, &_markBitMap, &_markStack,
4559 &keep_alive, true /* preclean */);
4560
4561 // We don't want this step to interfere with a young
4562 // collection because we don't want to take CPU
4563 // or memory bandwidth away from the young GC threads
4564 // (which may be as many as there are CPUs).
4565 // Note that we don't need to protect ourselves from
4566 // interference with mutators because they can't
4567 // manipulate the discovered reference lists nor affect
4568 // the computed reachability of the referents, the
4569 // only properties manipulated by the precleaning
4570 // of these reference lists.
4571 stopTimer();
4572 CMSTokenSyncWithLocks x(true /* is cms thread */,
4573 bitMapLock());
4574 startTimer();
4575 sample_eden();
4576
4577 // The following will yield to allow foreground
4578 // collection to proceed promptly. XXX YSR:
4579 // The code in this method may need further
4580 // tweaking for better performance and some restructuring
4581 // for cleaner interfaces.
4582 rp->preclean_discovered_references(
4583 rp->is_alive_non_header(), &keep_alive, &complete_trace,
4584 &yield_cl, should_unload_classes());
4585 }
4586
4587 if (clean_survivor) { // preclean the active survivor space(s)
4588 assert(_young_gen->kind() == Generation::DefNew ||
4589 _young_gen->kind() == Generation::ParNew ||
4590 _young_gen->kind() == Generation::ASParNew,
4591 "incorrect type for cast");
4592 DefNewGeneration* dng = (DefNewGeneration*)_young_gen;
4593 PushAndMarkClosure pam_cl(this, _span, ref_processor(),
4594 &_markBitMap, &_modUnionTable,
4595 &_markStack, &_revisitStack,
4596 true /* precleaning phase */);
4597 stopTimer();
4598 CMSTokenSyncWithLocks ts(true /* is cms thread */,
4599 bitMapLock());
4600 startTimer();
4601 unsigned int before_count =
4602 GenCollectedHeap::heap()->total_collections();
4603 SurvivorSpacePrecleanClosure
4604 sss_cl(this, _span, &_markBitMap, &_markStack,
4605 &pam_cl, before_count, CMSYield);
4606 DEBUG_ONLY(RememberKlassesChecker mx(should_unload_classes());)
4607 dng->from()->object_iterate_careful(&sss_cl);
4608 dng->to()->object_iterate_careful(&sss_cl);
4609 }
4610 MarkRefsIntoAndScanClosure
4611 mrias_cl(_span, ref_processor(), &_markBitMap, &_modUnionTable,
4612 &_markStack, &_revisitStack, this, CMSYield,
4613 true /* precleaning phase */);
4614 // CAUTION: The following closure has persistent state that may need to
4615 // be reset upon a decrease in the sequence of addresses it
4616 // processes.
4617 ScanMarkedObjectsAgainCarefullyClosure
4618 smoac_cl(this, _span,
4619 &_markBitMap, &_markStack, &_revisitStack, &mrias_cl, CMSYield);
4620
4621 // Preclean dirty cards in ModUnionTable and CardTable using
4622 // appropriate convergence criterion;
4623 // repeat CMSPrecleanIter times unless we find that
4624 // we are losing.
4625 assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large");
4626 assert(CMSPrecleanNumerator < CMSPrecleanDenominator,
4627 "Bad convergence multiplier");
4628 assert(CMSPrecleanThreshold >= 100,
4629 "Unreasonably low CMSPrecleanThreshold");
4630
4631 size_t numIter, cumNumCards, lastNumCards, curNumCards;
4632 for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0;
4633 numIter < CMSPrecleanIter;
4634 numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) {
4635 curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl);
4636 if (CMSPermGenPrecleaningEnabled) {
4637 curNumCards += preclean_mod_union_table(_permGen, &smoac_cl);
4638 }
4639 if (Verbose && PrintGCDetails) {
4640 gclog_or_tty->print(" (modUnionTable: %d cards)", curNumCards);
4641 }
4642 // Either there are very few dirty cards, so re-mark
4643 // pause will be small anyway, or our pre-cleaning isn't
4644 // that much faster than the rate at which cards are being
4645 // dirtied, so we might as well stop and re-mark since
4646 // precleaning won't improve our re-mark time by much.
4647 if (curNumCards <= CMSPrecleanThreshold ||
4648 (numIter > 0 &&
4649 (curNumCards * CMSPrecleanDenominator >
4650 lastNumCards * CMSPrecleanNumerator))) {
4651 numIter++;
4652 cumNumCards += curNumCards;
4653 break;
4654 }
4655 }
4656 curNumCards = preclean_card_table(_cmsGen, &smoac_cl);
4657 if (CMSPermGenPrecleaningEnabled) {
4658 curNumCards += preclean_card_table(_permGen, &smoac_cl);
4659 }
4660 cumNumCards += curNumCards;
4661 if (PrintGCDetails && PrintCMSStatistics != 0) {
4662 gclog_or_tty->print_cr(" (cardTable: %d cards, re-scanned %d cards, %d iterations)",
4663 curNumCards, cumNumCards, numIter);
4664 }
4665 return cumNumCards; // as a measure of useful work done
4666 }
4667
4668 // PRECLEANING NOTES:
4669 // Precleaning involves:
4670 // . reading the bits of the modUnionTable and clearing the set bits.
4671 // . For the cards corresponding to the set bits, we scan the
4672 // objects on those cards. This means we need the free_list_lock
4673 // so that we can safely iterate over the CMS space when scanning
4674 // for oops.
4675 // . When we scan the objects, we'll be both reading and setting
4676 // marks in the marking bit map, so we'll need the marking bit map.
4677 // . For protecting _collector_state transitions, we take the CGC_lock.
4678 // Note that any races in the reading of of card table entries by the
4679 // CMS thread on the one hand and the clearing of those entries by the
4680 // VM thread or the setting of those entries by the mutator threads on the
4681 // other are quite benign. However, for efficiency it makes sense to keep
4682 // the VM thread from racing with the CMS thread while the latter is
4683 // dirty card info to the modUnionTable. We therefore also use the
4684 // CGC_lock to protect the reading of the card table and the mod union
4685 // table by the CM thread.
4686 // . We run concurrently with mutator updates, so scanning
4687 // needs to be done carefully -- we should not try to scan
4688 // potentially uninitialized objects.
4689 //
4690 // Locking strategy: While holding the CGC_lock, we scan over and
4691 // reset a maximal dirty range of the mod union / card tables, then lock
4692 // the free_list_lock and bitmap lock to do a full marking, then
4693 // release these locks; and repeat the cycle. This allows for a
4694 // certain amount of fairness in the sharing of these locks between
4695 // the CMS collector on the one hand, and the VM thread and the
4696 // mutators on the other.
4697
4698 // NOTE: preclean_mod_union_table() and preclean_card_table()
4699 // further below are largely identical; if you need to modify
4700 // one of these methods, please check the other method too.
4701
4702 size_t CMSCollector::preclean_mod_union_table(
4703 ConcurrentMarkSweepGeneration* gen,
4704 ScanMarkedObjectsAgainCarefullyClosure* cl) {
4705 verify_work_stacks_empty();
4706 verify_overflow_empty();
4707
4708 // Turn off checking for this method but turn it back on
4709 // selectively. There are yield points in this method
4710 // but it is difficult to turn the checking off just around
4711 // the yield points. It is simpler to selectively turn
4712 // it on.
4713 DEBUG_ONLY(RememberKlassesChecker mux(false);)
4714
4715 // strategy: starting with the first card, accumulate contiguous
4716 // ranges of dirty cards; clear these cards, then scan the region
4717 // covered by these cards.
4718
4719 // Since all of the MUT is committed ahead, we can just use
4720 // that, in case the generations expand while we are precleaning.
4721 // It might also be fine to just use the committed part of the
4722 // generation, but we might potentially miss cards when the
4723 // generation is rapidly expanding while we are in the midst
4724 // of precleaning.
4725 HeapWord* startAddr = gen->reserved().start();
4726 HeapWord* endAddr = gen->reserved().end();
4727
4728 cl->setFreelistLock(gen->freelistLock()); // needed for yielding
4729
4730 size_t numDirtyCards, cumNumDirtyCards;
4731 HeapWord *nextAddr, *lastAddr;
4732 for (cumNumDirtyCards = numDirtyCards = 0,
4733 nextAddr = lastAddr = startAddr;
4734 nextAddr < endAddr;
4735 nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
4736
4737 ResourceMark rm;
4738 HandleMark hm;
4739
4740 MemRegion dirtyRegion;
4741 {
4742 stopTimer();
4743 // Potential yield point
4744 CMSTokenSync ts(true);
4745 startTimer();
4746 sample_eden();
4747 // Get dirty region starting at nextOffset (inclusive),
4748 // simultaneously clearing it.
4749 dirtyRegion =
4750 _modUnionTable.getAndClearMarkedRegion(nextAddr, endAddr);
4751 assert(dirtyRegion.start() >= nextAddr,
4752 "returned region inconsistent?");
4753 }
4754 // Remember where the next search should begin.
4755 // The returned region (if non-empty) is a right open interval,
4756 // so lastOffset is obtained from the right end of that
4757 // interval.
4758 lastAddr = dirtyRegion.end();
4759 // Should do something more transparent and less hacky XXX
4760 numDirtyCards =
4761 _modUnionTable.heapWordDiffToOffsetDiff(dirtyRegion.word_size());
4762
4763 // We'll scan the cards in the dirty region (with periodic
4764 // yields for foreground GC as needed).
4765 if (!dirtyRegion.is_empty()) {
4766 assert(numDirtyCards > 0, "consistency check");
4767 HeapWord* stop_point = NULL;
4768 stopTimer();
4769 // Potential yield point
4770 CMSTokenSyncWithLocks ts(true, gen->freelistLock(),
4771 bitMapLock());
4772 startTimer();
4773 {
4774 verify_work_stacks_empty();
4775 verify_overflow_empty();
4776 sample_eden();
4777 DEBUG_ONLY(RememberKlassesChecker mx(should_unload_classes());)
4778 stop_point =
4779 gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
4780 }
4781 if (stop_point != NULL) {
4782 // The careful iteration stopped early either because it found an
4783 // uninitialized object, or because we were in the midst of an
4784 // "abortable preclean", which should now be aborted. Redirty
4785 // the bits corresponding to the partially-scanned or unscanned
4786 // cards. We'll either restart at the next block boundary or
4787 // abort the preclean.
4788 assert((CMSPermGenPrecleaningEnabled && (gen == _permGen)) ||
4789 (_collectorState == AbortablePreclean && should_abort_preclean()),
4790 "Unparsable objects should only be in perm gen.");
4791 _modUnionTable.mark_range(MemRegion(stop_point, dirtyRegion.end()));
4792 if (should_abort_preclean()) {
4793 break; // out of preclean loop
4794 } else {
4795 // Compute the next address at which preclean should pick up;
4796 // might need bitMapLock in order to read P-bits.
4797 lastAddr = next_card_start_after_block(stop_point);
4798 }
4799 }
4800 } else {
4801 assert(lastAddr == endAddr, "consistency check");
4802 assert(numDirtyCards == 0, "consistency check");
4803 break;
4804 }
4805 }
4806 verify_work_stacks_empty();
4807 verify_overflow_empty();
4808 return cumNumDirtyCards;
4809 }
4810
4811 // NOTE: preclean_mod_union_table() above and preclean_card_table()
4812 // below are largely identical; if you need to modify
4813 // one of these methods, please check the other method too.
4814
4815 size_t CMSCollector::preclean_card_table(ConcurrentMarkSweepGeneration* gen,
4816 ScanMarkedObjectsAgainCarefullyClosure* cl) {
4817 // strategy: it's similar to precleamModUnionTable above, in that
4818 // we accumulate contiguous ranges of dirty cards, mark these cards
4819 // precleaned, then scan the region covered by these cards.
4820 HeapWord* endAddr = (HeapWord*)(gen->_virtual_space.high());
4821 HeapWord* startAddr = (HeapWord*)(gen->_virtual_space.low());
4822
4823 cl->setFreelistLock(gen->freelistLock()); // needed for yielding
4824
4825 size_t numDirtyCards, cumNumDirtyCards;
4826 HeapWord *lastAddr, *nextAddr;
4827
4828 for (cumNumDirtyCards = numDirtyCards = 0,
4829 nextAddr = lastAddr = startAddr;
4830 nextAddr < endAddr;
4831 nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) {
4832
4833 ResourceMark rm;
4834 HandleMark hm;
4835
4836 MemRegion dirtyRegion;
4837 {
4838 // See comments in "Precleaning notes" above on why we
4839 // do this locking. XXX Could the locking overheads be
4840 // too high when dirty cards are sparse? [I don't think so.]
4841 stopTimer();
4842 CMSTokenSync x(true); // is cms thread
4843 startTimer();
4844 sample_eden();
4845 // Get and clear dirty region from card table
4846 dirtyRegion = _ct->ct_bs()->dirty_card_range_after_reset(
4847 MemRegion(nextAddr, endAddr),
4848 true,
4849 CardTableModRefBS::precleaned_card_val());
4850
4851 assert(dirtyRegion.start() >= nextAddr,
4852 "returned region inconsistent?");
4853 }
4854 lastAddr = dirtyRegion.end();
4855 numDirtyCards =
4856 dirtyRegion.word_size()/CardTableModRefBS::card_size_in_words;
4857
4858 if (!dirtyRegion.is_empty()) {
4859 stopTimer();
4860 CMSTokenSyncWithLocks ts(true, gen->freelistLock(), bitMapLock());
4861 startTimer();
4862 sample_eden();
4863 verify_work_stacks_empty();
4864 verify_overflow_empty();
4865 DEBUG_ONLY(RememberKlassesChecker mx(should_unload_classes());)
4866 HeapWord* stop_point =
4867 gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl);
4868 if (stop_point != NULL) {
4869 // The careful iteration stopped early because it found an
4870 // uninitialized object. Redirty the bits corresponding to the
4871 // partially-scanned or unscanned cards, and start again at the
4872 // next block boundary.
4873 assert(CMSPermGenPrecleaningEnabled ||
4874 (_collectorState == AbortablePreclean && should_abort_preclean()),
4875 "Unparsable objects should only be in perm gen.");
4876 _ct->ct_bs()->invalidate(MemRegion(stop_point, dirtyRegion.end()));
4877 if (should_abort_preclean()) {
4878 break; // out of preclean loop
4879 } else {
4880 // Compute the next address at which preclean should pick up.
4881 lastAddr = next_card_start_after_block(stop_point);
4882 }
4883 }
4884 } else {
4885 break;
4886 }
4887 }
4888 verify_work_stacks_empty();
4889 verify_overflow_empty();
4890 return cumNumDirtyCards;
4891 }
4892
4893 void CMSCollector::checkpointRootsFinal(bool asynch,
4894 bool clear_all_soft_refs, bool init_mark_was_synchronous) {
4895 assert(_collectorState == FinalMarking, "incorrect state transition?");
4896 check_correct_thread_executing();
4897 // world is stopped at this checkpoint
4898 assert(SafepointSynchronize::is_at_safepoint(),
4899 "world should be stopped");
4900 TraceCMSMemoryManagerStats tms(_collectorState);
4901 verify_work_stacks_empty();
4902 verify_overflow_empty();
4903
4904 SpecializationStats::clear();
4905 if (PrintGCDetails) {
4906 gclog_or_tty->print("[YG occupancy: "SIZE_FORMAT" K ("SIZE_FORMAT" K)]",
4907 _young_gen->used() / K,
4908 _young_gen->capacity() / K);
4909 }
4910 if (asynch) {
4911 if (CMSScavengeBeforeRemark) {
4912 GenCollectedHeap* gch = GenCollectedHeap::heap();
4913 // Temporarily set flag to false, GCH->do_collection will
4914 // expect it to be false and set to true
4915 FlagSetting fl(gch->_is_gc_active, false);
4916 NOT_PRODUCT(TraceTime t("Scavenge-Before-Remark",
4917 PrintGCDetails && Verbose, true, gclog_or_tty);)
4918 int level = _cmsGen->level() - 1;
4919 if (level >= 0) {
4920 gch->do_collection(true, // full (i.e. force, see below)
4921 false, // !clear_all_soft_refs
4922 0, // size
4923 false, // is_tlab
4924 level // max_level
4925 );
4926 }
4927 }
4928 FreelistLocker x(this);
4929 MutexLockerEx y(bitMapLock(),
4930 Mutex::_no_safepoint_check_flag);
4931 assert(!init_mark_was_synchronous, "but that's impossible!");
4932 checkpointRootsFinalWork(asynch, clear_all_soft_refs, false);
4933 } else {
4934 // already have all the locks
4935 checkpointRootsFinalWork(asynch, clear_all_soft_refs,
4936 init_mark_was_synchronous);
4937 }
4938 verify_work_stacks_empty();
4939 verify_overflow_empty();
4940 SpecializationStats::print();
4941 }
4942
4943 void CMSCollector::checkpointRootsFinalWork(bool asynch,
4944 bool clear_all_soft_refs, bool init_mark_was_synchronous) {
4945
4946 NOT_PRODUCT(TraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, gclog_or_tty);)
4947
4948 assert(haveFreelistLocks(), "must have free list locks");
4949 assert_lock_strong(bitMapLock());
4950
4951 if (UseAdaptiveSizePolicy) {
4952 size_policy()->checkpoint_roots_final_begin();
4953 }
4954
4955 ResourceMark rm;
4956 HandleMark hm;
4957
4958 GenCollectedHeap* gch = GenCollectedHeap::heap();
4959
4960 if (should_unload_classes()) {
4961 CodeCache::gc_prologue();
4962 }
4963 assert(haveFreelistLocks(), "must have free list locks");
4964 assert_lock_strong(bitMapLock());
4965
4966 DEBUG_ONLY(RememberKlassesChecker fmx(should_unload_classes());)
4967 if (!init_mark_was_synchronous) {
4968 // We might assume that we need not fill TLAB's when
4969 // CMSScavengeBeforeRemark is set, because we may have just done
4970 // a scavenge which would have filled all TLAB's -- and besides
4971 // Eden would be empty. This however may not always be the case --
4972 // for instance although we asked for a scavenge, it may not have
4973 // happened because of a JNI critical section. We probably need
4974 // a policy for deciding whether we can in that case wait until
4975 // the critical section releases and then do the remark following
4976 // the scavenge, and skip it here. In the absence of that policy,
4977 // or of an indication of whether the scavenge did indeed occur,
4978 // we cannot rely on TLAB's having been filled and must do
4979 // so here just in case a scavenge did not happen.
4980 gch->ensure_parsability(false); // fill TLAB's, but no need to retire them
4981 // Update the saved marks which may affect the root scans.
4982 gch->save_marks();
4983
4984 {
4985 COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;)
4986
4987 // Note on the role of the mod union table:
4988 // Since the marker in "markFromRoots" marks concurrently with
4989 // mutators, it is possible for some reachable objects not to have been
4990 // scanned. For instance, an only reference to an object A was
4991 // placed in object B after the marker scanned B. Unless B is rescanned,
4992 // A would be collected. Such updates to references in marked objects
4993 // are detected via the mod union table which is the set of all cards
4994 // dirtied since the first checkpoint in this GC cycle and prior to
4995 // the most recent young generation GC, minus those cleaned up by the
4996 // concurrent precleaning.
4997 if (CMSParallelRemarkEnabled && CollectedHeap::use_parallel_gc_threads()) {
4998 TraceTime t("Rescan (parallel) ", PrintGCDetails, false, gclog_or_tty);
4999 do_remark_parallel();
5000 } else {
5001 TraceTime t("Rescan (non-parallel) ", PrintGCDetails, false,
5002 gclog_or_tty);
5003 do_remark_non_parallel();
5004 }
5005 }
5006 } else {
5007 assert(!asynch, "Can't have init_mark_was_synchronous in asynch mode");
5008 // The initial mark was stop-world, so there's no rescanning to
5009 // do; go straight on to the next step below.
5010 }
5011 verify_work_stacks_empty();
5012 verify_overflow_empty();
5013
5014 {
5015 NOT_PRODUCT(TraceTime ts("refProcessingWork", PrintGCDetails, false, gclog_or_tty);)
5016 refProcessingWork(asynch, clear_all_soft_refs);
5017 }
5018 verify_work_stacks_empty();
5019 verify_overflow_empty();
5020
5021 if (should_unload_classes()) {
5022 CodeCache::gc_epilogue();
5023 }
5024
5025 // If we encountered any (marking stack / work queue) overflow
5026 // events during the current CMS cycle, take appropriate
5027 // remedial measures, where possible, so as to try and avoid
5028 // recurrence of that condition.
5029 assert(_markStack.isEmpty(), "No grey objects");
5030 size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
5031 _ser_kac_ovflw + _ser_kac_preclean_ovflw;
5032 if (ser_ovflw > 0) {
5033 if (PrintCMSStatistics != 0) {
5034 gclog_or_tty->print_cr("Marking stack overflow (benign) "
5035 "(pmc_pc="SIZE_FORMAT", pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT
5036 ", kac_preclean="SIZE_FORMAT")",
5037 _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw,
5038 _ser_kac_ovflw, _ser_kac_preclean_ovflw);
5039 }
5040 _markStack.expand();
5041 _ser_pmc_remark_ovflw = 0;
5042 _ser_pmc_preclean_ovflw = 0;
5043 _ser_kac_preclean_ovflw = 0;
5044 _ser_kac_ovflw = 0;
5045 }
5046 if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
5047 if (PrintCMSStatistics != 0) {
5048 gclog_or_tty->print_cr("Work queue overflow (benign) "
5049 "(pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT")",
5050 _par_pmc_remark_ovflw, _par_kac_ovflw);
5051 }
5052 _par_pmc_remark_ovflw = 0;
5053 _par_kac_ovflw = 0;
5054 }
5055 if (PrintCMSStatistics != 0) {
5056 if (_markStack._hit_limit > 0) {
5057 gclog_or_tty->print_cr(" (benign) Hit max stack size limit ("SIZE_FORMAT")",
5058 _markStack._hit_limit);
5059 }
5060 if (_markStack._failed_double > 0) {
5061 gclog_or_tty->print_cr(" (benign) Failed stack doubling ("SIZE_FORMAT"),"
5062 " current capacity "SIZE_FORMAT,
5063 _markStack._failed_double,
5064 _markStack.capacity());
5065 }
5066 }
5067 _markStack._hit_limit = 0;
5068 _markStack._failed_double = 0;
5069
5070 // Check that all the klasses have been checked
5071 assert(_revisitStack.isEmpty(), "Not all klasses revisited");
5072
5073 if ((VerifyAfterGC || VerifyDuringGC) &&
5074 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
5075 verify_after_remark();
5076 }
5077
5078 // Change under the freelistLocks.
5079 _collectorState = Sweeping;
5080 // Call isAllClear() under bitMapLock
5081 assert(_modUnionTable.isAllClear(), "Should be clear by end of the"
5082 " final marking");
5083 if (UseAdaptiveSizePolicy) {
5084 size_policy()->checkpoint_roots_final_end(gch->gc_cause());
5085 }
5086 }
5087
5088 // Parallel remark task
5089 class CMSParRemarkTask: public AbstractGangTask {
5090 CMSCollector* _collector;
5091 int _n_workers;
5092 CompactibleFreeListSpace* _cms_space;
5093 CompactibleFreeListSpace* _perm_space;
5094
5095 // The per-thread work queues, available here for stealing.
5096 OopTaskQueueSet* _task_queues;
5097 ParallelTaskTerminator _term;
5098
5099 public:
5100 CMSParRemarkTask(CMSCollector* collector,
5101 CompactibleFreeListSpace* cms_space,
5102 CompactibleFreeListSpace* perm_space,
5103 int n_workers, FlexibleWorkGang* workers,
5104 OopTaskQueueSet* task_queues):
5105 AbstractGangTask("Rescan roots and grey objects in parallel"),
5106 _collector(collector),
5107 _cms_space(cms_space), _perm_space(perm_space),
5108 _n_workers(n_workers),
5109 _task_queues(task_queues),
5110 _term(n_workers, task_queues) { }
5111
5112 OopTaskQueueSet* task_queues() { return _task_queues; }
5113
5114 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
5115
5116 ParallelTaskTerminator* terminator() { return &_term; }
5117 int n_workers() { return _n_workers; }
5118
5119 void work(int i);
5120
5121 private:
5122 // Work method in support of parallel rescan ... of young gen spaces
5123 void do_young_space_rescan(int i, Par_MarkRefsIntoAndScanClosure* cl,
5124 ContiguousSpace* space,
5125 HeapWord** chunk_array, size_t chunk_top);
5126
5127 // ... of dirty cards in old space
5128 void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i,
5129 Par_MarkRefsIntoAndScanClosure* cl);
5130
5131 // ... work stealing for the above
5132 void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed);
5133 };
5134
5135 // work_queue(i) is passed to the closure
5136 // Par_MarkRefsIntoAndScanClosure. The "i" parameter
5137 // also is passed to do_dirty_card_rescan_tasks() and to
5138 // do_work_steal() to select the i-th task_queue.
5139
5140 void CMSParRemarkTask::work(int i) {
5141 elapsedTimer _timer;
5142 ResourceMark rm;
5143 HandleMark hm;
5144
5145 // ---------- rescan from roots --------------
5146 _timer.start();
5147 GenCollectedHeap* gch = GenCollectedHeap::heap();
5148 Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector,
5149 _collector->_span, _collector->ref_processor(),
5150 &(_collector->_markBitMap),
5151 work_queue(i), &(_collector->_revisitStack));
5152
5153 // Rescan young gen roots first since these are likely
5154 // coarsely partitioned and may, on that account, constitute
5155 // the critical path; thus, it's best to start off that
5156 // work first.
5157 // ---------- young gen roots --------------
5158 {
5159 DefNewGeneration* dng = _collector->_young_gen->as_DefNewGeneration();
5160 EdenSpace* eden_space = dng->eden();
5161 ContiguousSpace* from_space = dng->from();
5162 ContiguousSpace* to_space = dng->to();
5163
5164 HeapWord** eca = _collector->_eden_chunk_array;
5165 size_t ect = _collector->_eden_chunk_index;
5166 HeapWord** sca = _collector->_survivor_chunk_array;
5167 size_t sct = _collector->_survivor_chunk_index;
5168
5169 assert(ect <= _collector->_eden_chunk_capacity, "out of bounds");
5170 assert(sct <= _collector->_survivor_chunk_capacity, "out of bounds");
5171
5172 do_young_space_rescan(i, &par_mrias_cl, to_space, NULL, 0);
5173 do_young_space_rescan(i, &par_mrias_cl, from_space, sca, sct);
5174 do_young_space_rescan(i, &par_mrias_cl, eden_space, eca, ect);
5175
5176 _timer.stop();
5177 if (PrintCMSStatistics != 0) {
5178 gclog_or_tty->print_cr(
5179 "Finished young gen rescan work in %dth thread: %3.3f sec",
5180 i, _timer.seconds());
5181 }
5182 }
5183
5184 // ---------- remaining roots --------------
5185 _timer.reset();
5186 _timer.start();
5187 gch->gen_process_strong_roots(_collector->_cmsGen->level(),
5188 false, // yg was scanned above
5189 false, // this is parallel code
5190 true, // collecting perm gen
5191 SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
5192 &par_mrias_cl,
5193 true, // walk all of code cache if (so & SO_CodeCache)
5194 NULL);
5195 assert(_collector->should_unload_classes()
5196 || (_collector->CMSCollector::roots_scanning_options() & SharedHeap::SO_CodeCache),
5197 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
5198 _timer.stop();
5199 if (PrintCMSStatistics != 0) {
5200 gclog_or_tty->print_cr(
5201 "Finished remaining root rescan work in %dth thread: %3.3f sec",
5202 i, _timer.seconds());
5203 }
5204
5205 // ---------- rescan dirty cards ------------
5206 _timer.reset();
5207 _timer.start();
5208
5209 // Do the rescan tasks for each of the two spaces
5210 // (cms_space and perm_space) in turn.
5211 // "i" is passed to select the "i-th" task_queue
5212 do_dirty_card_rescan_tasks(_cms_space, i, &par_mrias_cl);
5213 do_dirty_card_rescan_tasks(_perm_space, i, &par_mrias_cl);
5214 _timer.stop();
5215 if (PrintCMSStatistics != 0) {
5216 gclog_or_tty->print_cr(
5217 "Finished dirty card rescan work in %dth thread: %3.3f sec",
5218 i, _timer.seconds());
5219 }
5220
5221 // ---------- steal work from other threads ...
5222 // ---------- ... and drain overflow list.
5223 _timer.reset();
5224 _timer.start();
5225 do_work_steal(i, &par_mrias_cl, _collector->hash_seed(i));
5226 _timer.stop();
5227 if (PrintCMSStatistics != 0) {
5228 gclog_or_tty->print_cr(
5229 "Finished work stealing in %dth thread: %3.3f sec",
5230 i, _timer.seconds());
5231 }
5232 }
5233
5234 // Note that parameter "i" is not used.
5235 void
5236 CMSParRemarkTask::do_young_space_rescan(int i,
5237 Par_MarkRefsIntoAndScanClosure* cl, ContiguousSpace* space,
5238 HeapWord** chunk_array, size_t chunk_top) {
5239 // Until all tasks completed:
5240 // . claim an unclaimed task
5241 // . compute region boundaries corresponding to task claimed
5242 // using chunk_array
5243 // . par_oop_iterate(cl) over that region
5244
5245 ResourceMark rm;
5246 HandleMark hm;
5247
5248 SequentialSubTasksDone* pst = space->par_seq_tasks();
5249 assert(pst->valid(), "Uninitialized use?");
5250
5251 int nth_task = 0;
5252 int n_tasks = pst->n_tasks();
5253
5254 HeapWord *start, *end;
5255 while (!pst->is_task_claimed(/* reference */ nth_task)) {
5256 // We claimed task # nth_task; compute its boundaries.
5257 if (chunk_top == 0) { // no samples were taken
5258 assert(nth_task == 0 && n_tasks == 1, "Can have only 1 EdenSpace task");
5259 start = space->bottom();
5260 end = space->top();
5261 } else if (nth_task == 0) {
5262 start = space->bottom();
5263 end = chunk_array[nth_task];
5264 } else if (nth_task < (jint)chunk_top) {
5265 assert(nth_task >= 1, "Control point invariant");
5266 start = chunk_array[nth_task - 1];
5267 end = chunk_array[nth_task];
5268 } else {
5269 assert(nth_task == (jint)chunk_top, "Control point invariant");
5270 start = chunk_array[chunk_top - 1];
5271 end = space->top();
5272 }
5273 MemRegion mr(start, end);
5274 // Verify that mr is in space
5275 assert(mr.is_empty() || space->used_region().contains(mr),
5276 "Should be in space");
5277 // Verify that "start" is an object boundary
5278 assert(mr.is_empty() || oop(mr.start())->is_oop(),
5279 "Should be an oop");
5280 space->par_oop_iterate(mr, cl);
5281 }
5282 pst->all_tasks_completed();
5283 }
5284
5285 void
5286 CMSParRemarkTask::do_dirty_card_rescan_tasks(
5287 CompactibleFreeListSpace* sp, int i,
5288 Par_MarkRefsIntoAndScanClosure* cl) {
5289 // Until all tasks completed:
5290 // . claim an unclaimed task
5291 // . compute region boundaries corresponding to task claimed
5292 // . transfer dirty bits ct->mut for that region
5293 // . apply rescanclosure to dirty mut bits for that region
5294
5295 ResourceMark rm;
5296 HandleMark hm;
5297
5298 OopTaskQueue* work_q = work_queue(i);
5299 ModUnionClosure modUnionClosure(&(_collector->_modUnionTable));
5300 // CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION!
5301 // CAUTION: This closure has state that persists across calls to
5302 // the work method dirty_range_iterate_clear() in that it has
5303 // imbedded in it a (subtype of) UpwardsObjectClosure. The
5304 // use of that state in the imbedded UpwardsObjectClosure instance
5305 // assumes that the cards are always iterated (even if in parallel
5306 // by several threads) in monotonically increasing order per each
5307 // thread. This is true of the implementation below which picks
5308 // card ranges (chunks) in monotonically increasing order globally
5309 // and, a-fortiori, in monotonically increasing order per thread
5310 // (the latter order being a subsequence of the former).
5311 // If the work code below is ever reorganized into a more chaotic
5312 // work-partitioning form than the current "sequential tasks"
5313 // paradigm, the use of that persistent state will have to be
5314 // revisited and modified appropriately. See also related
5315 // bug 4756801 work on which should examine this code to make
5316 // sure that the changes there do not run counter to the
5317 // assumptions made here and necessary for correctness and
5318 // efficiency. Note also that this code might yield inefficient
5319 // behaviour in the case of very large objects that span one or
5320 // more work chunks. Such objects would potentially be scanned
5321 // several times redundantly. Work on 4756801 should try and
5322 // address that performance anomaly if at all possible. XXX
5323 MemRegion full_span = _collector->_span;
5324 CMSBitMap* bm = &(_collector->_markBitMap); // shared
5325 CMSMarkStack* rs = &(_collector->_revisitStack); // shared
5326 MarkFromDirtyCardsClosure
5327 greyRescanClosure(_collector, full_span, // entire span of interest
5328 sp, bm, work_q, rs, cl);
5329
5330 SequentialSubTasksDone* pst = sp->conc_par_seq_tasks();
5331 assert(pst->valid(), "Uninitialized use?");
5332 int nth_task = 0;
5333 const int alignment = CardTableModRefBS::card_size * BitsPerWord;
5334 MemRegion span = sp->used_region();
5335 HeapWord* start_addr = span.start();
5336 HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(),
5337 alignment);
5338 const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units
5339 assert((HeapWord*)round_to((intptr_t)start_addr, alignment) ==
5340 start_addr, "Check alignment");
5341 assert((size_t)round_to((intptr_t)chunk_size, alignment) ==
5342 chunk_size, "Check alignment");
5343
5344 while (!pst->is_task_claimed(/* reference */ nth_task)) {
5345 // Having claimed the nth_task, compute corresponding mem-region,
5346 // which is a-fortiori aligned correctly (i.e. at a MUT bopundary).
5347 // The alignment restriction ensures that we do not need any
5348 // synchronization with other gang-workers while setting or
5349 // clearing bits in thus chunk of the MUT.
5350 MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size,
5351 start_addr + (nth_task+1)*chunk_size);
5352 // The last chunk's end might be way beyond end of the
5353 // used region. In that case pull back appropriately.
5354 if (this_span.end() > end_addr) {
5355 this_span.set_end(end_addr);
5356 assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)");
5357 }
5358 // Iterate over the dirty cards covering this chunk, marking them
5359 // precleaned, and setting the corresponding bits in the mod union
5360 // table. Since we have been careful to partition at Card and MUT-word
5361 // boundaries no synchronization is needed between parallel threads.
5362 _collector->_ct->ct_bs()->dirty_card_iterate(this_span,
5363 &modUnionClosure);
5364
5365 // Having transferred these marks into the modUnionTable,
5366 // rescan the marked objects on the dirty cards in the modUnionTable.
5367 // Even if this is at a synchronous collection, the initial marking
5368 // may have been done during an asynchronous collection so there
5369 // may be dirty bits in the mod-union table.
5370 _collector->_modUnionTable.dirty_range_iterate_clear(
5371 this_span, &greyRescanClosure);
5372 _collector->_modUnionTable.verifyNoOneBitsInRange(
5373 this_span.start(),
5374 this_span.end());
5375 }
5376 pst->all_tasks_completed(); // declare that i am done
5377 }
5378
5379 // . see if we can share work_queues with ParNew? XXX
5380 void
5381 CMSParRemarkTask::do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl,
5382 int* seed) {
5383 OopTaskQueue* work_q = work_queue(i);
5384 NOT_PRODUCT(int num_steals = 0;)
5385 oop obj_to_scan;
5386 CMSBitMap* bm = &(_collector->_markBitMap);
5387
5388 while (true) {
5389 // Completely finish any left over work from (an) earlier round(s)
5390 cl->trim_queue(0);
5391 size_t num_from_overflow_list = MIN2((size_t)(work_q->max_elems() - work_q->size())/4,
5392 (size_t)ParGCDesiredObjsFromOverflowList);
5393 // Now check if there's any work in the overflow list
5394 // Passing ParallelGCThreads as the third parameter, no_of_gc_threads,
5395 // only affects the number of attempts made to get work from the
5396 // overflow list and does not affect the number of workers. Just
5397 // pass ParallelGCThreads so this behavior is unchanged.
5398 if (_collector->par_take_from_overflow_list(num_from_overflow_list,
5399 work_q,
5400 ParallelGCThreads)) {
5401 // found something in global overflow list;
5402 // not yet ready to go stealing work from others.
5403 // We'd like to assert(work_q->size() != 0, ...)
5404 // because we just took work from the overflow list,
5405 // but of course we can't since all of that could have
5406 // been already stolen from us.
5407 // "He giveth and He taketh away."
5408 continue;
5409 }
5410 // Verify that we have no work before we resort to stealing
5411 assert(work_q->size() == 0, "Have work, shouldn't steal");
5412 // Try to steal from other queues that have work
5413 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5414 NOT_PRODUCT(num_steals++;)
5415 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5416 assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5417 // Do scanning work
5418 obj_to_scan->oop_iterate(cl);
5419 // Loop around, finish this work, and try to steal some more
5420 } else if (terminator()->offer_termination()) {
5421 break; // nirvana from the infinite cycle
5422 }
5423 }
5424 NOT_PRODUCT(
5425 if (PrintCMSStatistics != 0) {
5426 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
5427 }
5428 )
5429 assert(work_q->size() == 0 && _collector->overflow_list_is_empty(),
5430 "Else our work is not yet done");
5431 }
5432
5433 // Return a thread-local PLAB recording array, as appropriate.
5434 void* CMSCollector::get_data_recorder(int thr_num) {
5435 if (_survivor_plab_array != NULL &&
5436 (CMSPLABRecordAlways ||
5437 (_collectorState > Marking && _collectorState < FinalMarking))) {
5438 assert(thr_num < (int)ParallelGCThreads, "thr_num is out of bounds");
5439 ChunkArray* ca = &_survivor_plab_array[thr_num];
5440 ca->reset(); // clear it so that fresh data is recorded
5441 return (void*) ca;
5442 } else {
5443 return NULL;
5444 }
5445 }
5446
5447 // Reset all the thread-local PLAB recording arrays
5448 void CMSCollector::reset_survivor_plab_arrays() {
5449 for (uint i = 0; i < ParallelGCThreads; i++) {
5450 _survivor_plab_array[i].reset();
5451 }
5452 }
5453
5454 // Merge the per-thread plab arrays into the global survivor chunk
5455 // array which will provide the partitioning of the survivor space
5456 // for CMS rescan.
5457 void CMSCollector::merge_survivor_plab_arrays(ContiguousSpace* surv,
5458 int no_of_gc_threads) {
5459 assert(_survivor_plab_array != NULL, "Error");
5460 assert(_survivor_chunk_array != NULL, "Error");
5461 assert(_collectorState == FinalMarking, "Error");
5462 for (int j = 0; j < no_of_gc_threads; j++) {
5463 _cursor[j] = 0;
5464 }
5465 HeapWord* top = surv->top();
5466 size_t i;
5467 for (i = 0; i < _survivor_chunk_capacity; i++) { // all sca entries
5468 HeapWord* min_val = top; // Higher than any PLAB address
5469 uint min_tid = 0; // position of min_val this round
5470 for (int j = 0; j < no_of_gc_threads; j++) {
5471 ChunkArray* cur_sca = &_survivor_plab_array[j];
5472 if (_cursor[j] == cur_sca->end()) {
5473 continue;
5474 }
5475 assert(_cursor[j] < cur_sca->end(), "ctl pt invariant");
5476 HeapWord* cur_val = cur_sca->nth(_cursor[j]);
5477 assert(surv->used_region().contains(cur_val), "Out of bounds value");
5478 if (cur_val < min_val) {
5479 min_tid = j;
5480 min_val = cur_val;
5481 } else {
5482 assert(cur_val < top, "All recorded addresses should be less");
5483 }
5484 }
5485 // At this point min_val and min_tid are respectively
5486 // the least address in _survivor_plab_array[j]->nth(_cursor[j])
5487 // and the thread (j) that witnesses that address.
5488 // We record this address in the _survivor_chunk_array[i]
5489 // and increment _cursor[min_tid] prior to the next round i.
5490 if (min_val == top) {
5491 break;
5492 }
5493 _survivor_chunk_array[i] = min_val;
5494 _cursor[min_tid]++;
5495 }
5496 // We are all done; record the size of the _survivor_chunk_array
5497 _survivor_chunk_index = i; // exclusive: [0, i)
5498 if (PrintCMSStatistics > 0) {
5499 gclog_or_tty->print(" (Survivor:" SIZE_FORMAT "chunks) ", i);
5500 }
5501 // Verify that we used up all the recorded entries
5502 #ifdef ASSERT
5503 size_t total = 0;
5504 for (int j = 0; j < no_of_gc_threads; j++) {
5505 assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant");
5506 total += _cursor[j];
5507 }
5508 assert(total == _survivor_chunk_index, "Ctl Pt Invariant");
5509 // Check that the merged array is in sorted order
5510 if (total > 0) {
5511 for (size_t i = 0; i < total - 1; i++) {
5512 if (PrintCMSStatistics > 0) {
5513 gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ",
5514 i, _survivor_chunk_array[i]);
5515 }
5516 assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1],
5517 "Not sorted");
5518 }
5519 }
5520 #endif // ASSERT
5521 }
5522
5523 // Set up the space's par_seq_tasks structure for work claiming
5524 // for parallel rescan of young gen.
5525 // See ParRescanTask where this is currently used.
5526 void
5527 CMSCollector::
5528 initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) {
5529 assert(n_threads > 0, "Unexpected n_threads argument");
5530 DefNewGeneration* dng = (DefNewGeneration*)_young_gen;
5531
5532 // Eden space
5533 {
5534 SequentialSubTasksDone* pst = dng->eden()->par_seq_tasks();
5535 assert(!pst->valid(), "Clobbering existing data?");
5536 // Each valid entry in [0, _eden_chunk_index) represents a task.
5537 size_t n_tasks = _eden_chunk_index + 1;
5538 assert(n_tasks == 1 || _eden_chunk_array != NULL, "Error");
5539 // Sets the condition for completion of the subtask (how many threads
5540 // need to finish in order to be done).
5541 pst->set_n_threads(n_threads);
5542 pst->set_n_tasks((int)n_tasks);
5543 }
5544
5545 // Merge the survivor plab arrays into _survivor_chunk_array
5546 if (_survivor_plab_array != NULL) {
5547 merge_survivor_plab_arrays(dng->from(), n_threads);
5548 } else {
5549 assert(_survivor_chunk_index == 0, "Error");
5550 }
5551
5552 // To space
5553 {
5554 SequentialSubTasksDone* pst = dng->to()->par_seq_tasks();
5555 assert(!pst->valid(), "Clobbering existing data?");
5556 // Sets the condition for completion of the subtask (how many threads
5557 // need to finish in order to be done).
5558 pst->set_n_threads(n_threads);
5559 pst->set_n_tasks(1);
5560 assert(pst->valid(), "Error");
5561 }
5562
5563 // From space
5564 {
5565 SequentialSubTasksDone* pst = dng->from()->par_seq_tasks();
5566 assert(!pst->valid(), "Clobbering existing data?");
5567 size_t n_tasks = _survivor_chunk_index + 1;
5568 assert(n_tasks == 1 || _survivor_chunk_array != NULL, "Error");
5569 // Sets the condition for completion of the subtask (how many threads
5570 // need to finish in order to be done).
5571 pst->set_n_threads(n_threads);
5572 pst->set_n_tasks((int)n_tasks);
5573 assert(pst->valid(), "Error");
5574 }
5575 }
5576
5577 // Parallel version of remark
5578 void CMSCollector::do_remark_parallel() {
5579 GenCollectedHeap* gch = GenCollectedHeap::heap();
5580 FlexibleWorkGang* workers = gch->workers();
5581 assert(workers != NULL, "Need parallel worker threads.");
5582 int n_workers = workers->total_workers();
5583 CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();
5584 CompactibleFreeListSpace* perm_space = _permGen->cmsSpace();
5585
5586 CMSParRemarkTask tsk(this,
5587 cms_space, perm_space,
5588 n_workers, workers, task_queues());
5589
5590 // Set up for parallel process_strong_roots work.
5591 gch->set_par_threads(n_workers);
5592 // We won't be iterating over the cards in the card table updating
5593 // the younger_gen cards, so we shouldn't call the following else
5594 // the verification code as well as subsequent younger_refs_iterate
5595 // code would get confused. XXX
5596 // gch->rem_set()->prepare_for_younger_refs_iterate(true); // parallel
5597
5598 // The young gen rescan work will not be done as part of
5599 // process_strong_roots (which currently doesn't knw how to
5600 // parallelize such a scan), but rather will be broken up into
5601 // a set of parallel tasks (via the sampling that the [abortable]
5602 // preclean phase did of EdenSpace, plus the [two] tasks of
5603 // scanning the [two] survivor spaces. Further fine-grain
5604 // parallelization of the scanning of the survivor spaces
5605 // themselves, and of precleaning of the younger gen itself
5606 // is deferred to the future.
5607 initialize_sequential_subtasks_for_young_gen_rescan(n_workers);
5608
5609 // The dirty card rescan work is broken up into a "sequence"
5610 // of parallel tasks (per constituent space) that are dynamically
5611 // claimed by the parallel threads.
5612 cms_space->initialize_sequential_subtasks_for_rescan(n_workers);
5613 perm_space->initialize_sequential_subtasks_for_rescan(n_workers);
5614
5615 // It turns out that even when we're using 1 thread, doing the work in a
5616 // separate thread causes wide variance in run times. We can't help this
5617 // in the multi-threaded case, but we special-case n=1 here to get
5618 // repeatable measurements of the 1-thread overhead of the parallel code.
5619 if (n_workers > 1) {
5620 // Make refs discovery MT-safe
5621 ReferenceProcessorMTMutator mt(ref_processor(), true);
5622 GenCollectedHeap::StrongRootsScope srs(gch);
5623 workers->run_task(&tsk);
5624 } else {
5625 GenCollectedHeap::StrongRootsScope srs(gch);
5626 tsk.work(0);
5627 }
5628 gch->set_par_threads(0); // 0 ==> non-parallel.
5629 // restore, single-threaded for now, any preserved marks
5630 // as a result of work_q overflow
5631 restore_preserved_marks_if_any();
5632 }
5633
5634 // Non-parallel version of remark
5635 void CMSCollector::do_remark_non_parallel() {
5636 ResourceMark rm;
5637 HandleMark hm;
5638 GenCollectedHeap* gch = GenCollectedHeap::heap();
5639 MarkRefsIntoAndScanClosure
5640 mrias_cl(_span, ref_processor(), &_markBitMap, &_modUnionTable,
5641 &_markStack, &_revisitStack, this,
5642 false /* should_yield */, false /* not precleaning */);
5643 MarkFromDirtyCardsClosure
5644 markFromDirtyCardsClosure(this, _span,
5645 NULL, // space is set further below
5646 &_markBitMap, &_markStack, &_revisitStack,
5647 &mrias_cl);
5648 {
5649 TraceTime t("grey object rescan", PrintGCDetails, false, gclog_or_tty);
5650 // Iterate over the dirty cards, setting the corresponding bits in the
5651 // mod union table.
5652 {
5653 ModUnionClosure modUnionClosure(&_modUnionTable);
5654 _ct->ct_bs()->dirty_card_iterate(
5655 _cmsGen->used_region(),
5656 &modUnionClosure);
5657 _ct->ct_bs()->dirty_card_iterate(
5658 _permGen->used_region(),
5659 &modUnionClosure);
5660 }
5661 // Having transferred these marks into the modUnionTable, we just need
5662 // to rescan the marked objects on the dirty cards in the modUnionTable.
5663 // The initial marking may have been done during an asynchronous
5664 // collection so there may be dirty bits in the mod-union table.
5665 const int alignment =
5666 CardTableModRefBS::card_size * BitsPerWord;
5667 {
5668 // ... First handle dirty cards in CMS gen
5669 markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace());
5670 MemRegion ur = _cmsGen->used_region();
5671 HeapWord* lb = ur.start();
5672 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
5673 MemRegion cms_span(lb, ub);
5674 _modUnionTable.dirty_range_iterate_clear(cms_span,
5675 &markFromDirtyCardsClosure);
5676 verify_work_stacks_empty();
5677 if (PrintCMSStatistics != 0) {
5678 gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ",
5679 markFromDirtyCardsClosure.num_dirty_cards());
5680 }
5681 }
5682 {
5683 // .. and then repeat for dirty cards in perm gen
5684 markFromDirtyCardsClosure.set_space(_permGen->cmsSpace());
5685 MemRegion ur = _permGen->used_region();
5686 HeapWord* lb = ur.start();
5687 HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment);
5688 MemRegion perm_span(lb, ub);
5689 _modUnionTable.dirty_range_iterate_clear(perm_span,
5690 &markFromDirtyCardsClosure);
5691 verify_work_stacks_empty();
5692 if (PrintCMSStatistics != 0) {
5693 gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in perm gen) ",
5694 markFromDirtyCardsClosure.num_dirty_cards());
5695 }
5696 }
5697 }
5698 if (VerifyDuringGC &&
5699 GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {
5700 HandleMark hm; // Discard invalid handles created during verification
5701 Universe::verify(true);
5702 }
5703 {
5704 TraceTime t("root rescan", PrintGCDetails, false, gclog_or_tty);
5705
5706 verify_work_stacks_empty();
5707
5708 gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
5709 GenCollectedHeap::StrongRootsScope srs(gch);
5710 gch->gen_process_strong_roots(_cmsGen->level(),
5711 true, // younger gens as roots
5712 false, // use the local StrongRootsScope
5713 true, // collecting perm gen
5714 SharedHeap::ScanningOption(roots_scanning_options()),
5715 &mrias_cl,
5716 true, // walk code active on stacks
5717 NULL);
5718 assert(should_unload_classes()
5719 || (roots_scanning_options() & SharedHeap::SO_CodeCache),
5720 "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
5721 }
5722 verify_work_stacks_empty();
5723 // Restore evacuated mark words, if any, used for overflow list links
5724 if (!CMSOverflowEarlyRestoration) {
5725 restore_preserved_marks_if_any();
5726 }
5727 verify_overflow_empty();
5728 }
5729
5730 ////////////////////////////////////////////////////////
5731 // Parallel Reference Processing Task Proxy Class
5732 ////////////////////////////////////////////////////////
5733 class CMSRefProcTaskProxy: public AbstractGangTaskWOopQueues {
5734 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
5735 CMSCollector* _collector;
5736 CMSBitMap* _mark_bit_map;
5737 const MemRegion _span;
5738 ProcessTask& _task;
5739
5740 public:
5741 CMSRefProcTaskProxy(ProcessTask& task,
5742 CMSCollector* collector,
5743 const MemRegion& span,
5744 CMSBitMap* mark_bit_map,
5745 AbstractWorkGang* workers,
5746 OopTaskQueueSet* task_queues):
5747 AbstractGangTaskWOopQueues("Process referents by policy in parallel",
5748 task_queues),
5749 _task(task),
5750 _collector(collector), _span(span), _mark_bit_map(mark_bit_map)
5751 {
5752 assert(_collector->_span.equals(_span) && !_span.is_empty(),
5753 "Inconsistency in _span");
5754 }
5755
5756 OopTaskQueueSet* task_queues() { return queues(); }
5757
5758 OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); }
5759
5760 void do_work_steal(int i,
5761 CMSParDrainMarkingStackClosure* drain,
5762 CMSParKeepAliveClosure* keep_alive,
5763 int* seed);
5764
5765 virtual void work(int i);
5766 };
5767
5768 void CMSRefProcTaskProxy::work(int i) {
5769 assert(_collector->_span.equals(_span), "Inconsistency in _span");
5770 CMSParKeepAliveClosure par_keep_alive(_collector, _span,
5771 _mark_bit_map,
5772 &_collector->_revisitStack,
5773 work_queue(i));
5774 CMSParDrainMarkingStackClosure par_drain_stack(_collector, _span,
5775 _mark_bit_map,
5776 &_collector->_revisitStack,
5777 work_queue(i));
5778 CMSIsAliveClosure is_alive_closure(_span, _mark_bit_map);
5779 _task.work(i, is_alive_closure, par_keep_alive, par_drain_stack);
5780 if (_task.marks_oops_alive()) {
5781 do_work_steal(i, &par_drain_stack, &par_keep_alive,
5782 _collector->hash_seed(i));
5783 }
5784 assert(work_queue(i)->size() == 0, "work_queue should be empty");
5785 assert(_collector->_overflow_list == NULL, "non-empty _overflow_list");
5786 }
5787
5788 class CMSRefEnqueueTaskProxy: public AbstractGangTask {
5789 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
5790 EnqueueTask& _task;
5791
5792 public:
5793 CMSRefEnqueueTaskProxy(EnqueueTask& task)
5794 : AbstractGangTask("Enqueue reference objects in parallel"),
5795 _task(task)
5796 { }
5797
5798 virtual void work(int i)
5799 {
5800 _task.work(i);
5801 }
5802 };
5803
5804 CMSParKeepAliveClosure::CMSParKeepAliveClosure(CMSCollector* collector,
5805 MemRegion span, CMSBitMap* bit_map, CMSMarkStack* revisit_stack,
5806 OopTaskQueue* work_queue):
5807 Par_KlassRememberingOopClosure(collector, NULL, revisit_stack),
5808 _span(span),
5809 _bit_map(bit_map),
5810 _work_queue(work_queue),
5811 _mark_and_push(collector, span, bit_map, revisit_stack, work_queue),
5812 _low_water_mark(MIN2((uint)(work_queue->max_elems()/4),
5813 (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads)))
5814 { }
5815
5816 // . see if we can share work_queues with ParNew? XXX
5817 void CMSRefProcTaskProxy::do_work_steal(int i,
5818 CMSParDrainMarkingStackClosure* drain,
5819 CMSParKeepAliveClosure* keep_alive,
5820 int* seed) {
5821 OopTaskQueue* work_q = work_queue(i);
5822 NOT_PRODUCT(int num_steals = 0;)
5823 oop obj_to_scan;
5824
5825 while (true) {
5826 // Completely finish any left over work from (an) earlier round(s)
5827 drain->trim_queue(0);
5828 size_t num_from_overflow_list = MIN2((size_t)(work_q->max_elems() - work_q->size())/4,
5829 (size_t)ParGCDesiredObjsFromOverflowList);
5830 // Now check if there's any work in the overflow list
5831 // Passing ParallelGCThreads as the third parameter, no_of_gc_threads,
5832 // only affects the number of attempts made to get work from the
5833 // overflow list and does not affect the number of workers. Just
5834 // pass ParallelGCThreads so this behavior is unchanged.
5835 if (_collector->par_take_from_overflow_list(num_from_overflow_list,
5836 work_q,
5837 ParallelGCThreads)) {
5838 // Found something in global overflow list;
5839 // not yet ready to go stealing work from others.
5840 // We'd like to assert(work_q->size() != 0, ...)
5841 // because we just took work from the overflow list,
5842 // but of course we can't, since all of that might have
5843 // been already stolen from us.
5844 continue;
5845 }
5846 // Verify that we have no work before we resort to stealing
5847 assert(work_q->size() == 0, "Have work, shouldn't steal");
5848 // Try to steal from other queues that have work
5849 if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) {
5850 NOT_PRODUCT(num_steals++;)
5851 assert(obj_to_scan->is_oop(), "Oops, not an oop!");
5852 assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?");
5853 // Do scanning work
5854 obj_to_scan->oop_iterate(keep_alive);
5855 // Loop around, finish this work, and try to steal some more
5856 } else if (terminator()->offer_termination()) {
5857 break; // nirvana from the infinite cycle
5858 }
5859 }
5860 NOT_PRODUCT(
5861 if (PrintCMSStatistics != 0) {
5862 gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals);
5863 }
5864 )
5865 }
5866
5867 void CMSRefProcTaskExecutor::execute(ProcessTask& task)
5868 {
5869 GenCollectedHeap* gch = GenCollectedHeap::heap();
5870 FlexibleWorkGang* workers = gch->workers();
5871 assert(workers != NULL, "Need parallel worker threads.");
5872 CMSRefProcTaskProxy rp_task(task, &_collector,
5873 _collector.ref_processor()->span(),
5874 _collector.markBitMap(),
5875 workers, _collector.task_queues());
5876 workers->run_task(&rp_task);
5877 }
5878
5879 void CMSRefProcTaskExecutor::execute(EnqueueTask& task)
5880 {
5881
5882 GenCollectedHeap* gch = GenCollectedHeap::heap();
5883 FlexibleWorkGang* workers = gch->workers();
5884 assert(workers != NULL, "Need parallel worker threads.");
5885 CMSRefEnqueueTaskProxy enq_task(task);
5886 workers->run_task(&enq_task);
5887 }
5888
5889 void CMSCollector::refProcessingWork(bool asynch, bool clear_all_soft_refs) {
5890
5891 ResourceMark rm;
5892 HandleMark hm;
5893
5894 ReferenceProcessor* rp = ref_processor();
5895 assert(rp->span().equals(_span), "Spans should be equal");
5896 assert(!rp->enqueuing_is_done(), "Enqueuing should not be complete");
5897 // Process weak references.
5898 rp->setup_policy(clear_all_soft_refs);
5899 verify_work_stacks_empty();
5900
5901 CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap,
5902 &_markStack, &_revisitStack,
5903 false /* !preclean */);
5904 CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this,
5905 _span, &_markBitMap, &_markStack,
5906 &cmsKeepAliveClosure, false /* !preclean */);
5907 {
5908 TraceTime t("weak refs processing", PrintGCDetails, false, gclog_or_tty);
5909 if (rp->processing_is_mt()) {
5910 // Set the degree of MT here. If the discovery is done MT, there
5911 // may have been a different number of threads doing the discovery
5912 // and a different number of discovered lists may have Ref objects.
5913 // That is OK as long as the Reference lists are balanced (see
5914 // balance_all_queues() and balance_queues()).
5915
5916
5917 rp->set_mt_degree(ParallelGCThreads);
5918 CMSRefProcTaskExecutor task_executor(*this);
5919 rp->process_discovered_references(&_is_alive_closure,
5920 &cmsKeepAliveClosure,
5921 &cmsDrainMarkingStackClosure,
5922 &task_executor);
5923 } else {
5924 rp->process_discovered_references(&_is_alive_closure,
5925 &cmsKeepAliveClosure,
5926 &cmsDrainMarkingStackClosure,
5927 NULL);
5928 }
5929 verify_work_stacks_empty();
5930 }
5931
5932 if (should_unload_classes()) {
5933 {
5934 TraceTime t("class unloading", PrintGCDetails, false, gclog_or_tty);
5935
5936 // Follow SystemDictionary roots and unload classes
5937 bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure);
5938
5939 // Follow CodeCache roots and unload any methods marked for unloading
5940 CodeCache::do_unloading(&_is_alive_closure,
5941 &cmsKeepAliveClosure,
5942 purged_class);
5943
5944 cmsDrainMarkingStackClosure.do_void();
5945 verify_work_stacks_empty();
5946
5947 // Update subklass/sibling/implementor links in KlassKlass descendants
5948 assert(!_revisitStack.isEmpty(), "revisit stack should not be empty");
5949 oop k;
5950 while ((k = _revisitStack.pop()) != NULL) {
5951 ((Klass*)(oopDesc*)k)->follow_weak_klass_links(
5952 &_is_alive_closure,
5953 &cmsKeepAliveClosure);
5954 }
5955 assert(!ClassUnloading ||
5956 (_markStack.isEmpty() && overflow_list_is_empty()),
5957 "Should not have found new reachable objects");
5958 assert(_revisitStack.isEmpty(), "revisit stack should have been drained");
5959 cmsDrainMarkingStackClosure.do_void();
5960 verify_work_stacks_empty();
5961 }
5962
5963 {
5964 TraceTime t("scrub symbol & string tables", PrintGCDetails, false, gclog_or_tty);
5965 // Now clean up stale oops in SymbolTable and StringTable
5966 SymbolTable::unlink(&_is_alive_closure);
5967 StringTable::unlink(&_is_alive_closure);
5968 }
5969 }
5970
5971 verify_work_stacks_empty();
5972 // Restore any preserved marks as a result of mark stack or
5973 // work queue overflow
5974 restore_preserved_marks_if_any(); // done single-threaded for now
5975
5976 rp->set_enqueuing_is_done(true);
5977 if (rp->processing_is_mt()) {
5978 rp->balance_all_queues();
5979 CMSRefProcTaskExecutor task_executor(*this);
5980 rp->enqueue_discovered_references(&task_executor);
5981 } else {
5982 rp->enqueue_discovered_references(NULL);
5983 }
5984 rp->verify_no_references_recorded();
5985 assert(!rp->discovery_enabled(), "should have been disabled");
5986
5987 // JVMTI object tagging is based on JNI weak refs. If any of these
5988 // refs were cleared then JVMTI needs to update its maps and
5989 // maybe post ObjectFrees to agents.
5990 JvmtiExport::cms_ref_processing_epilogue();
5991 }
5992
5993 #ifndef PRODUCT
5994 void CMSCollector::check_correct_thread_executing() {
5995 Thread* t = Thread::current();
5996 // Only the VM thread or the CMS thread should be here.
5997 assert(t->is_ConcurrentGC_thread() || t->is_VM_thread(),
5998 "Unexpected thread type");
5999 // If this is the vm thread, the foreground process
6000 // should not be waiting. Note that _foregroundGCIsActive is
6001 // true while the foreground collector is waiting.
6002 if (_foregroundGCShouldWait) {
6003 // We cannot be the VM thread
6004 assert(t->is_ConcurrentGC_thread(),
6005 "Should be CMS thread");
6006 } else {
6007 // We can be the CMS thread only if we are in a stop-world
6008 // phase of CMS collection.
6009 if (t->is_ConcurrentGC_thread()) {
6010 assert(_collectorState == InitialMarking ||
6011 _collectorState == FinalMarking,
6012 "Should be a stop-world phase");
6013 // The CMS thread should be holding the CMS_token.
6014 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6015 "Potential interference with concurrently "
6016 "executing VM thread");
6017 }
6018 }
6019 }
6020 #endif
6021
6022 void CMSCollector::sweep(bool asynch) {
6023 assert(_collectorState == Sweeping, "just checking");
6024 check_correct_thread_executing();
6025 verify_work_stacks_empty();
6026 verify_overflow_empty();
6027 increment_sweep_count();
6028 TraceCMSMemoryManagerStats tms(_collectorState);
6029
6030 _inter_sweep_timer.stop();
6031 _inter_sweep_estimate.sample(_inter_sweep_timer.seconds());
6032 size_policy()->avg_cms_free_at_sweep()->sample(_cmsGen->free());
6033
6034 // PermGen verification support: If perm gen sweeping is disabled in
6035 // this cycle, we preserve the perm gen object "deadness" information
6036 // in the perm_gen_verify_bit_map. In order to do that we traverse
6037 // all blocks in perm gen and mark all dead objects.
6038 if (verifying() && !should_unload_classes()) {
6039 assert(perm_gen_verify_bit_map()->sizeInBits() != 0,
6040 "Should have already been allocated");
6041 MarkDeadObjectsClosure mdo(this, _permGen->cmsSpace(),
6042 markBitMap(), perm_gen_verify_bit_map());
6043 if (asynch) {
6044 CMSTokenSyncWithLocks ts(true, _permGen->freelistLock(),
6045 bitMapLock());
6046 _permGen->cmsSpace()->blk_iterate(&mdo);
6047 } else {
6048 // In the case of synchronous sweep, we already have
6049 // the requisite locks/tokens.
6050 _permGen->cmsSpace()->blk_iterate(&mdo);
6051 }
6052 }
6053
6054 assert(!_intra_sweep_timer.is_active(), "Should not be active");
6055 _intra_sweep_timer.reset();
6056 _intra_sweep_timer.start();
6057 if (asynch) {
6058 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
6059 CMSPhaseAccounting pa(this, "sweep", !PrintGCDetails);
6060 // First sweep the old gen then the perm gen
6061 {
6062 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
6063 bitMapLock());
6064 sweepWork(_cmsGen, asynch);
6065 }
6066
6067 // Now repeat for perm gen
6068 if (should_unload_classes()) {
6069 CMSTokenSyncWithLocks ts(true, _permGen->freelistLock(),
6070 bitMapLock());
6071 sweepWork(_permGen, asynch);
6072 }
6073
6074 // Update Universe::_heap_*_at_gc figures.
6075 // We need all the free list locks to make the abstract state
6076 // transition from Sweeping to Resetting. See detailed note
6077 // further below.
6078 {
6079 CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(),
6080 _permGen->freelistLock());
6081 // Update heap occupancy information which is used as
6082 // input to soft ref clearing policy at the next gc.
6083 Universe::update_heap_info_at_gc();
6084 _collectorState = Resizing;
6085 }
6086 } else {
6087 // already have needed locks
6088 sweepWork(_cmsGen, asynch);
6089
6090 if (should_unload_classes()) {
6091 sweepWork(_permGen, asynch);
6092 }
6093 // Update heap occupancy information which is used as
6094 // input to soft ref clearing policy at the next gc.
6095 Universe::update_heap_info_at_gc();
6096 _collectorState = Resizing;
6097 }
6098 verify_work_stacks_empty();
6099 verify_overflow_empty();
6100
6101 _intra_sweep_timer.stop();
6102 _intra_sweep_estimate.sample(_intra_sweep_timer.seconds());
6103
6104 _inter_sweep_timer.reset();
6105 _inter_sweep_timer.start();
6106
6107 update_time_of_last_gc(os::javaTimeMillis());
6108
6109 // NOTE on abstract state transitions:
6110 // Mutators allocate-live and/or mark the mod-union table dirty
6111 // based on the state of the collection. The former is done in
6112 // the interval [Marking, Sweeping] and the latter in the interval
6113 // [Marking, Sweeping). Thus the transitions into the Marking state
6114 // and out of the Sweeping state must be synchronously visible
6115 // globally to the mutators.
6116 // The transition into the Marking state happens with the world
6117 // stopped so the mutators will globally see it. Sweeping is
6118 // done asynchronously by the background collector so the transition
6119 // from the Sweeping state to the Resizing state must be done
6120 // under the freelistLock (as is the check for whether to
6121 // allocate-live and whether to dirty the mod-union table).
6122 assert(_collectorState == Resizing, "Change of collector state to"
6123 " Resizing must be done under the freelistLocks (plural)");
6124
6125 // Now that sweeping has been completed, if the GCH's
6126 // incremental_collection_will_fail flag is set, clear it,
6127 // thus inviting a younger gen collection to promote into
6128 // this generation. If such a promotion may still fail,
6129 // the flag will be set again when a young collection is
6130 // attempted.
6131 // I think the incremental_collection_will_fail flag's use
6132 // is specific to a 2 generation collection policy, so i'll
6133 // assert that that's the configuration we are operating within.
6134 // The use of the flag can and should be generalized appropriately
6135 // in the future to deal with a general n-generation system.
6136
6137 GenCollectedHeap* gch = GenCollectedHeap::heap();
6138 assert(gch->collector_policy()->is_two_generation_policy(),
6139 "Resetting of incremental_collection_will_fail flag"
6140 " may be incorrect otherwise");
6141 gch->clear_incremental_collection_will_fail();
6142 gch->update_full_collections_completed(_collection_count_start);
6143 }
6144
6145 // FIX ME!!! Looks like this belongs in CFLSpace, with
6146 // CMSGen merely delegating to it.
6147 void ConcurrentMarkSweepGeneration::setNearLargestChunk() {
6148 double nearLargestPercent = FLSLargestBlockCoalesceProximity;
6149 HeapWord* minAddr = _cmsSpace->bottom();
6150 HeapWord* largestAddr =
6151 (HeapWord*) _cmsSpace->dictionary()->findLargestDict();
6152 if (largestAddr == NULL) {
6153 // The dictionary appears to be empty. In this case
6154 // try to coalesce at the end of the heap.
6155 largestAddr = _cmsSpace->end();
6156 }
6157 size_t largestOffset = pointer_delta(largestAddr, minAddr);
6158 size_t nearLargestOffset =
6159 (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize;
6160 if (PrintFLSStatistics != 0) {
6161 gclog_or_tty->print_cr(
6162 "CMS: Large Block: " PTR_FORMAT ";"
6163 " Proximity: " PTR_FORMAT " -> " PTR_FORMAT,
6164 largestAddr,
6165 _cmsSpace->nearLargestChunk(), minAddr + nearLargestOffset);
6166 }
6167 _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset);
6168 }
6169
6170 bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) {
6171 return addr >= _cmsSpace->nearLargestChunk();
6172 }
6173
6174 FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() {
6175 return _cmsSpace->find_chunk_at_end();
6176 }
6177
6178 void ConcurrentMarkSweepGeneration::update_gc_stats(int current_level,
6179 bool full) {
6180 // The next lower level has been collected. Gather any statistics
6181 // that are of interest at this point.
6182 if (!full && (current_level + 1) == level()) {
6183 // Gather statistics on the young generation collection.
6184 collector()->stats().record_gc0_end(used());
6185 }
6186 }
6187
6188 CMSAdaptiveSizePolicy* ConcurrentMarkSweepGeneration::size_policy() {
6189 GenCollectedHeap* gch = GenCollectedHeap::heap();
6190 assert(gch->kind() == CollectedHeap::GenCollectedHeap,
6191 "Wrong type of heap");
6192 CMSAdaptiveSizePolicy* sp = (CMSAdaptiveSizePolicy*)
6193 gch->gen_policy()->size_policy();
6194 assert(sp->is_gc_cms_adaptive_size_policy(),
6195 "Wrong type of size policy");
6196 return sp;
6197 }
6198
6199 void ConcurrentMarkSweepGeneration::rotate_debug_collection_type() {
6200 if (PrintGCDetails && Verbose) {
6201 gclog_or_tty->print("Rotate from %d ", _debug_collection_type);
6202 }
6203 _debug_collection_type = (CollectionTypes) (_debug_collection_type + 1);
6204 _debug_collection_type =
6205 (CollectionTypes) (_debug_collection_type % Unknown_collection_type);
6206 if (PrintGCDetails && Verbose) {
6207 gclog_or_tty->print_cr("to %d ", _debug_collection_type);
6208 }
6209 }
6210
6211 void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* gen,
6212 bool asynch) {
6213 // We iterate over the space(s) underlying this generation,
6214 // checking the mark bit map to see if the bits corresponding
6215 // to specific blocks are marked or not. Blocks that are
6216 // marked are live and are not swept up. All remaining blocks
6217 // are swept up, with coalescing on-the-fly as we sweep up
6218 // contiguous free and/or garbage blocks:
6219 // We need to ensure that the sweeper synchronizes with allocators
6220 // and stop-the-world collectors. In particular, the following
6221 // locks are used:
6222 // . CMS token: if this is held, a stop the world collection cannot occur
6223 // . freelistLock: if this is held no allocation can occur from this
6224 // generation by another thread
6225 // . bitMapLock: if this is held, no other thread can access or update
6226 //
6227
6228 // Note that we need to hold the freelistLock if we use
6229 // block iterate below; else the iterator might go awry if
6230 // a mutator (or promotion) causes block contents to change
6231 // (for instance if the allocator divvies up a block).
6232 // If we hold the free list lock, for all practical purposes
6233 // young generation GC's can't occur (they'll usually need to
6234 // promote), so we might as well prevent all young generation
6235 // GC's while we do a sweeping step. For the same reason, we might
6236 // as well take the bit map lock for the entire duration
6237
6238 // check that we hold the requisite locks
6239 assert(have_cms_token(), "Should hold cms token");
6240 assert( (asynch && ConcurrentMarkSweepThread::cms_thread_has_cms_token())
6241 || (!asynch && ConcurrentMarkSweepThread::vm_thread_has_cms_token()),
6242 "Should possess CMS token to sweep");
6243 assert_lock_strong(gen->freelistLock());
6244 assert_lock_strong(bitMapLock());
6245
6246 assert(!_inter_sweep_timer.is_active(), "Was switched off in an outer context");
6247 assert(_intra_sweep_timer.is_active(), "Was switched on in an outer context");
6248 gen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()),
6249 _inter_sweep_estimate.padded_average(),
6250 _intra_sweep_estimate.padded_average());
6251 gen->setNearLargestChunk();
6252
6253 {
6254 SweepClosure sweepClosure(this, gen, &_markBitMap,
6255 CMSYield && asynch);
6256 gen->cmsSpace()->blk_iterate_careful(&sweepClosure);
6257 // We need to free-up/coalesce garbage/blocks from a
6258 // co-terminal free run. This is done in the SweepClosure
6259 // destructor; so, do not remove this scope, else the
6260 // end-of-sweep-census below will be off by a little bit.
6261 }
6262 gen->cmsSpace()->sweep_completed();
6263 gen->cmsSpace()->endSweepFLCensus(sweep_count());
6264 if (should_unload_classes()) { // unloaded classes this cycle,
6265 _concurrent_cycles_since_last_unload = 0; // ... reset count
6266 } else { // did not unload classes,
6267 _concurrent_cycles_since_last_unload++; // ... increment count
6268 }
6269 }
6270
6271 // Reset CMS data structures (for now just the marking bit map)
6272 // preparatory for the next cycle.
6273 void CMSCollector::reset(bool asynch) {
6274 GenCollectedHeap* gch = GenCollectedHeap::heap();
6275 CMSAdaptiveSizePolicy* sp = size_policy();
6276 AdaptiveSizePolicyOutput(sp, gch->total_collections());
6277 if (asynch) {
6278 CMSTokenSyncWithLocks ts(true, bitMapLock());
6279
6280 // If the state is not "Resetting", the foreground thread
6281 // has done a collection and the resetting.
6282 if (_collectorState != Resetting) {
6283 assert(_collectorState == Idling, "The state should only change"
6284 " because the foreground collector has finished the collection");
6285 return;
6286 }
6287
6288 // Clear the mark bitmap (no grey objects to start with)
6289 // for the next cycle.
6290 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
6291 CMSPhaseAccounting cmspa(this, "reset", !PrintGCDetails);
6292
6293 HeapWord* curAddr = _markBitMap.startWord();
6294 while (curAddr < _markBitMap.endWord()) {
6295 size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr);
6296 MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining));
6297 _markBitMap.clear_large_range(chunk);
6298 if (ConcurrentMarkSweepThread::should_yield() &&
6299 !foregroundGCIsActive() &&
6300 CMSYield) {
6301 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6302 "CMS thread should hold CMS token");
6303 assert_lock_strong(bitMapLock());
6304 bitMapLock()->unlock();
6305 ConcurrentMarkSweepThread::desynchronize(true);
6306 ConcurrentMarkSweepThread::acknowledge_yield_request();
6307 stopTimer();
6308 if (PrintCMSStatistics != 0) {
6309 incrementYields();
6310 }
6311 icms_wait();
6312
6313 // See the comment in coordinator_yield()
6314 for (unsigned i = 0; i < CMSYieldSleepCount &&
6315 ConcurrentMarkSweepThread::should_yield() &&
6316 !CMSCollector::foregroundGCIsActive(); ++i) {
6317 os::sleep(Thread::current(), 1, false);
6318 ConcurrentMarkSweepThread::acknowledge_yield_request();
6319 }
6320
6321 ConcurrentMarkSweepThread::synchronize(true);
6322 bitMapLock()->lock_without_safepoint_check();
6323 startTimer();
6324 }
6325 curAddr = chunk.end();
6326 }
6327 // A successful mostly concurrent collection has been done.
6328 // Because only the full (i.e., concurrent mode failure) collections
6329 // are being measured for gc overhead limits, clean the "near" flag
6330 // and count.
6331 sp->reset_gc_overhead_limit_count();
6332 _collectorState = Idling;
6333 } else {
6334 // already have the lock
6335 assert(_collectorState == Resetting, "just checking");
6336 assert_lock_strong(bitMapLock());
6337 _markBitMap.clear_all();
6338 _collectorState = Idling;
6339 }
6340
6341 // Stop incremental mode after a cycle completes, so that any future cycles
6342 // are triggered by allocation.
6343 stop_icms();
6344
6345 NOT_PRODUCT(
6346 if (RotateCMSCollectionTypes) {
6347 _cmsGen->rotate_debug_collection_type();
6348 }
6349 )
6350 }
6351
6352 void CMSCollector::do_CMS_operation(CMS_op_type op) {
6353 gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
6354 TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
6355 TraceTime t("GC", PrintGC, !PrintGCDetails, gclog_or_tty);
6356 TraceCollectorStats tcs(counters());
6357
6358 switch (op) {
6359 case CMS_op_checkpointRootsInitial: {
6360 checkpointRootsInitial(true); // asynch
6361 if (PrintGC) {
6362 _cmsGen->printOccupancy("initial-mark");
6363 }
6364 break;
6365 }
6366 case CMS_op_checkpointRootsFinal: {
6367 checkpointRootsFinal(true, // asynch
6368 false, // !clear_all_soft_refs
6369 false); // !init_mark_was_synchronous
6370 if (PrintGC) {
6371 _cmsGen->printOccupancy("remark");
6372 }
6373 break;
6374 }
6375 default:
6376 fatal("No such CMS_op");
6377 }
6378 }
6379
6380 #ifndef PRODUCT
6381 size_t const CMSCollector::skip_header_HeapWords() {
6382 return FreeChunk::header_size();
6383 }
6384
6385 // Try and collect here conditions that should hold when
6386 // CMS thread is exiting. The idea is that the foreground GC
6387 // thread should not be blocked if it wants to terminate
6388 // the CMS thread and yet continue to run the VM for a while
6389 // after that.
6390 void CMSCollector::verify_ok_to_terminate() const {
6391 assert(Thread::current()->is_ConcurrentGC_thread(),
6392 "should be called by CMS thread");
6393 assert(!_foregroundGCShouldWait, "should be false");
6394 // We could check here that all the various low-level locks
6395 // are not held by the CMS thread, but that is overkill; see
6396 // also CMSThread::verify_ok_to_terminate() where the CGC_lock
6397 // is checked.
6398 }
6399 #endif
6400
6401 size_t CMSCollector::block_size_using_printezis_bits(HeapWord* addr) const {
6402 assert(_markBitMap.isMarked(addr) && _markBitMap.isMarked(addr + 1),
6403 "missing Printezis mark?");
6404 HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
6405 size_t size = pointer_delta(nextOneAddr + 1, addr);
6406 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6407 "alignment problem");
6408 assert(size >= 3, "Necessary for Printezis marks to work");
6409 return size;
6410 }
6411
6412 // A variant of the above (block_size_using_printezis_bits()) except
6413 // that we return 0 if the P-bits are not yet set.
6414 size_t CMSCollector::block_size_if_printezis_bits(HeapWord* addr) const {
6415 if (_markBitMap.isMarked(addr)) {
6416 assert(_markBitMap.isMarked(addr + 1), "Missing Printezis bit?");
6417 HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
6418 size_t size = pointer_delta(nextOneAddr + 1, addr);
6419 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6420 "alignment problem");
6421 assert(size >= 3, "Necessary for Printezis marks to work");
6422 return size;
6423 } else {
6424 assert(!_markBitMap.isMarked(addr + 1), "Bit map inconsistency?");
6425 return 0;
6426 }
6427 }
6428
6429 HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
6430 size_t sz = 0;
6431 oop p = (oop)addr;
6432 if (p->klass_or_null() != NULL && p->is_parsable()) {
6433 sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
6434 } else {
6435 sz = block_size_using_printezis_bits(addr);
6436 }
6437 assert(sz > 0, "size must be nonzero");
6438 HeapWord* next_block = addr + sz;
6439 HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block,
6440 CardTableModRefBS::card_size);
6441 assert(round_down((uintptr_t)addr, CardTableModRefBS::card_size) <
6442 round_down((uintptr_t)next_card, CardTableModRefBS::card_size),
6443 "must be different cards");
6444 return next_card;
6445 }
6446
6447
6448 // CMS Bit Map Wrapper /////////////////////////////////////////
6449
6450 // Construct a CMS bit map infrastructure, but don't create the
6451 // bit vector itself. That is done by a separate call CMSBitMap::allocate()
6452 // further below.
6453 CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
6454 _bm(),
6455 _shifter(shifter),
6456 _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true) : NULL)
6457 {
6458 _bmStartWord = 0;
6459 _bmWordSize = 0;
6460 }
6461
6462 bool CMSBitMap::allocate(MemRegion mr) {
6463 _bmStartWord = mr.start();
6464 _bmWordSize = mr.word_size();
6465 ReservedSpace brs(ReservedSpace::allocation_align_size_up(
6466 (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1));
6467 if (!brs.is_reserved()) {
6468 warning("CMS bit map allocation failure");
6469 return false;
6470 }
6471 // For now we'll just commit all of the bit map up fromt.
6472 // Later on we'll try to be more parsimonious with swap.
6473 if (!_virtual_space.initialize(brs, brs.size())) {
6474 warning("CMS bit map backing store failure");
6475 return false;
6476 }
6477 assert(_virtual_space.committed_size() == brs.size(),
6478 "didn't reserve backing store for all of CMS bit map?");
6479 _bm.set_map((BitMap::bm_word_t*)_virtual_space.low());
6480 assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
6481 _bmWordSize, "inconsistency in bit map sizing");
6482 _bm.set_size(_bmWordSize >> _shifter);
6483
6484 // bm.clear(); // can we rely on getting zero'd memory? verify below
6485 assert(isAllClear(),
6486 "Expected zero'd memory from ReservedSpace constructor");
6487 assert(_bm.size() == heapWordDiffToOffsetDiff(sizeInWords()),
6488 "consistency check");
6489 return true;
6490 }
6491
6492 void CMSBitMap::dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl) {
6493 HeapWord *next_addr, *end_addr, *last_addr;
6494 assert_locked();
6495 assert(covers(mr), "out-of-range error");
6496 // XXX assert that start and end are appropriately aligned
6497 for (next_addr = mr.start(), end_addr = mr.end();
6498 next_addr < end_addr; next_addr = last_addr) {
6499 MemRegion dirty_region = getAndClearMarkedRegion(next_addr, end_addr);
6500 last_addr = dirty_region.end();
6501 if (!dirty_region.is_empty()) {
6502 cl->do_MemRegion(dirty_region);
6503 } else {
6504 assert(last_addr == end_addr, "program logic");
6505 return;
6506 }
6507 }
6508 }
6509
6510 #ifndef PRODUCT
6511 void CMSBitMap::assert_locked() const {
6512 CMSLockVerifier::assert_locked(lock());
6513 }
6514
6515 bool CMSBitMap::covers(MemRegion mr) const {
6516 // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
6517 assert((size_t)_bm.size() == (_bmWordSize >> _shifter),
6518 "size inconsistency");
6519 return (mr.start() >= _bmStartWord) &&
6520 (mr.end() <= endWord());
6521 }
6522
6523 bool CMSBitMap::covers(HeapWord* start, size_t size) const {
6524 return (start >= _bmStartWord && (start + size) <= endWord());
6525 }
6526
6527 void CMSBitMap::verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) {
6528 // verify that there are no 1 bits in the interval [left, right)
6529 FalseBitMapClosure falseBitMapClosure;
6530 iterate(&falseBitMapClosure, left, right);
6531 }
6532
6533 void CMSBitMap::region_invariant(MemRegion mr)
6534 {
6535 assert_locked();
6536 // mr = mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
6537 assert(!mr.is_empty(), "unexpected empty region");
6538 assert(covers(mr), "mr should be covered by bit map");
6539 // convert address range into offset range
6540 size_t start_ofs = heapWordToOffset(mr.start());
6541 // Make sure that end() is appropriately aligned
6542 assert(mr.end() == (HeapWord*)round_to((intptr_t)mr.end(),
6543 (1 << (_shifter+LogHeapWordSize))),
6544 "Misaligned mr.end()");
6545 size_t end_ofs = heapWordToOffset(mr.end());
6546 assert(end_ofs > start_ofs, "Should mark at least one bit");
6547 }
6548
6549 #endif
6550
6551 bool CMSMarkStack::allocate(size_t size) {
6552 // allocate a stack of the requisite depth
6553 ReservedSpace rs(ReservedSpace::allocation_align_size_up(
6554 size * sizeof(oop)));
6555 if (!rs.is_reserved()) {
6556 warning("CMSMarkStack allocation failure");
6557 return false;
6558 }
6559 if (!_virtual_space.initialize(rs, rs.size())) {
6560 warning("CMSMarkStack backing store failure");
6561 return false;
6562 }
6563 assert(_virtual_space.committed_size() == rs.size(),
6564 "didn't reserve backing store for all of CMS stack?");
6565 _base = (oop*)(_virtual_space.low());
6566 _index = 0;
6567 _capacity = size;
6568 NOT_PRODUCT(_max_depth = 0);
6569 return true;
6570 }
6571
6572 // XXX FIX ME !!! In the MT case we come in here holding a
6573 // leaf lock. For printing we need to take a further lock
6574 // which has lower rank. We need to recallibrate the two
6575 // lock-ranks involved in order to be able to rpint the
6576 // messages below. (Or defer the printing to the caller.
6577 // For now we take the expedient path of just disabling the
6578 // messages for the problematic case.)
6579 void CMSMarkStack::expand() {
6580 assert(_capacity <= MarkStackSizeMax, "stack bigger than permitted");
6581 if (_capacity == MarkStackSizeMax) {
6582 if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
6583 // We print a warning message only once per CMS cycle.
6584 gclog_or_tty->print_cr(" (benign) Hit CMSMarkStack max size limit");
6585 }
6586 return;
6587 }
6588 // Double capacity if possible
6589 size_t new_capacity = MIN2(_capacity*2, MarkStackSizeMax);
6590 // Do not give up existing stack until we have managed to
6591 // get the double capacity that we desired.
6592 ReservedSpace rs(ReservedSpace::allocation_align_size_up(
6593 new_capacity * sizeof(oop)));
6594 if (rs.is_reserved()) {
6595 // Release the backing store associated with old stack
6596 _virtual_space.release();
6597 // Reinitialize virtual space for new stack
6598 if (!_virtual_space.initialize(rs, rs.size())) {
6599 fatal("Not enough swap for expanded marking stack");
6600 }
6601 _base = (oop*)(_virtual_space.low());
6602 _index = 0;
6603 _capacity = new_capacity;
6604 } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) {
6605 // Failed to double capacity, continue;
6606 // we print a detail message only once per CMS cycle.
6607 gclog_or_tty->print(" (benign) Failed to expand marking stack from "SIZE_FORMAT"K to "
6608 SIZE_FORMAT"K",
6609 _capacity / K, new_capacity / K);
6610 }
6611 }
6612
6613
6614 // Closures
6615 // XXX: there seems to be a lot of code duplication here;
6616 // should refactor and consolidate common code.
6617
6618 // This closure is used to mark refs into the CMS generation in
6619 // the CMS bit map. Called at the first checkpoint. This closure
6620 // assumes that we do not need to re-mark dirty cards; if the CMS
6621 // generation on which this is used is not an oldest (modulo perm gen)
6622 // generation then this will lose younger_gen cards!
6623
6624 MarkRefsIntoClosure::MarkRefsIntoClosure(
6625 MemRegion span, CMSBitMap* bitMap):
6626 _span(span),
6627 _bitMap(bitMap)
6628 {
6629 assert(_ref_processor == NULL, "deliberately left NULL");
6630 assert(_bitMap->covers(_span), "_bitMap/_span mismatch");
6631 }
6632
6633 void MarkRefsIntoClosure::do_oop(oop obj) {
6634 // if p points into _span, then mark corresponding bit in _markBitMap
6635 assert(obj->is_oop(), "expected an oop");
6636 HeapWord* addr = (HeapWord*)obj;
6637 if (_span.contains(addr)) {
6638 // this should be made more efficient
6639 _bitMap->mark(addr);
6640 }
6641 }
6642
6643 void MarkRefsIntoClosure::do_oop(oop* p) { MarkRefsIntoClosure::do_oop_work(p); }
6644 void MarkRefsIntoClosure::do_oop(narrowOop* p) { MarkRefsIntoClosure::do_oop_work(p); }
6645
6646 // A variant of the above, used for CMS marking verification.
6647 MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure(
6648 MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm):
6649 _span(span),
6650 _verification_bm(verification_bm),
6651 _cms_bm(cms_bm)
6652 {
6653 assert(_ref_processor == NULL, "deliberately left NULL");
6654 assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch");
6655 }
6656
6657 void MarkRefsIntoVerifyClosure::do_oop(oop obj) {
6658 // if p points into _span, then mark corresponding bit in _markBitMap
6659 assert(obj->is_oop(), "expected an oop");
6660 HeapWord* addr = (HeapWord*)obj;
6661 if (_span.contains(addr)) {
6662 _verification_bm->mark(addr);
6663 if (!_cms_bm->isMarked(addr)) {
6664 oop(addr)->print();
6665 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", addr);
6666 fatal("... aborting");
6667 }
6668 }
6669 }
6670
6671 void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
6672 void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); }
6673
6674 //////////////////////////////////////////////////
6675 // MarkRefsIntoAndScanClosure
6676 //////////////////////////////////////////////////
6677
6678 MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span,
6679 ReferenceProcessor* rp,
6680 CMSBitMap* bit_map,
6681 CMSBitMap* mod_union_table,
6682 CMSMarkStack* mark_stack,
6683 CMSMarkStack* revisit_stack,
6684 CMSCollector* collector,
6685 bool should_yield,
6686 bool concurrent_precleaning):
6687 _collector(collector),
6688 _span(span),
6689 _bit_map(bit_map),
6690 _mark_stack(mark_stack),
6691 _pushAndMarkClosure(collector, span, rp, bit_map, mod_union_table,
6692 mark_stack, revisit_stack, concurrent_precleaning),
6693 _yield(should_yield),
6694 _concurrent_precleaning(concurrent_precleaning),
6695 _freelistLock(NULL)
6696 {
6697 _ref_processor = rp;
6698 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
6699 }
6700
6701 // This closure is used to mark refs into the CMS generation at the
6702 // second (final) checkpoint, and to scan and transitively follow
6703 // the unmarked oops. It is also used during the concurrent precleaning
6704 // phase while scanning objects on dirty cards in the CMS generation.
6705 // The marks are made in the marking bit map and the marking stack is
6706 // used for keeping the (newly) grey objects during the scan.
6707 // The parallel version (Par_...) appears further below.
6708 void MarkRefsIntoAndScanClosure::do_oop(oop obj) {
6709 if (obj != NULL) {
6710 assert(obj->is_oop(), "expected an oop");
6711 HeapWord* addr = (HeapWord*)obj;
6712 assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)");
6713 assert(_collector->overflow_list_is_empty(),
6714 "overflow list should be empty");
6715 if (_span.contains(addr) &&
6716 !_bit_map->isMarked(addr)) {
6717 // mark bit map (object is now grey)
6718 _bit_map->mark(addr);
6719 // push on marking stack (stack should be empty), and drain the
6720 // stack by applying this closure to the oops in the oops popped
6721 // from the stack (i.e. blacken the grey objects)
6722 bool res = _mark_stack->push(obj);
6723 assert(res, "Should have space to push on empty stack");
6724 do {
6725 oop new_oop = _mark_stack->pop();
6726 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
6727 assert(new_oop->is_parsable(), "Found unparsable oop");
6728 assert(_bit_map->isMarked((HeapWord*)new_oop),
6729 "only grey objects on this stack");
6730 // iterate over the oops in this oop, marking and pushing
6731 // the ones in CMS heap (i.e. in _span).
6732 new_oop->oop_iterate(&_pushAndMarkClosure);
6733 // check if it's time to yield
6734 do_yield_check();
6735 } while (!_mark_stack->isEmpty() ||
6736 (!_concurrent_precleaning && take_from_overflow_list()));
6737 // if marking stack is empty, and we are not doing this
6738 // during precleaning, then check the overflow list
6739 }
6740 assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
6741 assert(_collector->overflow_list_is_empty(),
6742 "overflow list was drained above");
6743 // We could restore evacuated mark words, if any, used for
6744 // overflow list links here because the overflow list is
6745 // provably empty here. That would reduce the maximum
6746 // size requirements for preserved_{oop,mark}_stack.
6747 // But we'll just postpone it until we are all done
6748 // so we can just stream through.
6749 if (!_concurrent_precleaning && CMSOverflowEarlyRestoration) {
6750 _collector->restore_preserved_marks_if_any();
6751 assert(_collector->no_preserved_marks(), "No preserved marks");
6752 }
6753 assert(!CMSOverflowEarlyRestoration || _collector->no_preserved_marks(),
6754 "All preserved marks should have been restored above");
6755 }
6756 }
6757
6758 void MarkRefsIntoAndScanClosure::do_oop(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
6759 void MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); }
6760
6761 void MarkRefsIntoAndScanClosure::do_yield_work() {
6762 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6763 "CMS thread should hold CMS token");
6764 assert_lock_strong(_freelistLock);
6765 assert_lock_strong(_bit_map->lock());
6766 // relinquish the free_list_lock and bitMaplock()
6767 DEBUG_ONLY(RememberKlassesChecker mux(false);)
6768 _bit_map->lock()->unlock();
6769 _freelistLock->unlock();
6770 ConcurrentMarkSweepThread::desynchronize(true);
6771 ConcurrentMarkSweepThread::acknowledge_yield_request();
6772 _collector->stopTimer();
6773 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
6774 if (PrintCMSStatistics != 0) {
6775 _collector->incrementYields();
6776 }
6777 _collector->icms_wait();
6778
6779 // See the comment in coordinator_yield()
6780 for (unsigned i = 0;
6781 i < CMSYieldSleepCount &&
6782 ConcurrentMarkSweepThread::should_yield() &&
6783 !CMSCollector::foregroundGCIsActive();
6784 ++i) {
6785 os::sleep(Thread::current(), 1, false);
6786 ConcurrentMarkSweepThread::acknowledge_yield_request();
6787 }
6788
6789 ConcurrentMarkSweepThread::synchronize(true);
6790 _freelistLock->lock_without_safepoint_check();
6791 _bit_map->lock()->lock_without_safepoint_check();
6792 _collector->startTimer();
6793 }
6794
6795 ///////////////////////////////////////////////////////////
6796 // Par_MarkRefsIntoAndScanClosure: a parallel version of
6797 // MarkRefsIntoAndScanClosure
6798 ///////////////////////////////////////////////////////////
6799 Par_MarkRefsIntoAndScanClosure::Par_MarkRefsIntoAndScanClosure(
6800 CMSCollector* collector, MemRegion span, ReferenceProcessor* rp,
6801 CMSBitMap* bit_map, OopTaskQueue* work_queue, CMSMarkStack* revisit_stack):
6802 _span(span),
6803 _bit_map(bit_map),
6804 _work_queue(work_queue),
6805 _low_water_mark(MIN2((uint)(work_queue->max_elems()/4),
6806 (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads))),
6807 _par_pushAndMarkClosure(collector, span, rp, bit_map, work_queue,
6808 revisit_stack)
6809 {
6810 _ref_processor = rp;
6811 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
6812 }
6813
6814 // This closure is used to mark refs into the CMS generation at the
6815 // second (final) checkpoint, and to scan and transitively follow
6816 // the unmarked oops. The marks are made in the marking bit map and
6817 // the work_queue is used for keeping the (newly) grey objects during
6818 // the scan phase whence they are also available for stealing by parallel
6819 // threads. Since the marking bit map is shared, updates are
6820 // synchronized (via CAS).
6821 void Par_MarkRefsIntoAndScanClosure::do_oop(oop obj) {
6822 if (obj != NULL) {
6823 // Ignore mark word because this could be an already marked oop
6824 // that may be chained at the end of the overflow list.
6825 assert(obj->is_oop(true), "expected an oop");
6826 HeapWord* addr = (HeapWord*)obj;
6827 if (_span.contains(addr) &&
6828 !_bit_map->isMarked(addr)) {
6829 // mark bit map (object will become grey):
6830 // It is possible for several threads to be
6831 // trying to "claim" this object concurrently;
6832 // the unique thread that succeeds in marking the
6833 // object first will do the subsequent push on
6834 // to the work queue (or overflow list).
6835 if (_bit_map->par_mark(addr)) {
6836 // push on work_queue (which may not be empty), and trim the
6837 // queue to an appropriate length by applying this closure to
6838 // the oops in the oops popped from the stack (i.e. blacken the
6839 // grey objects)
6840 bool res = _work_queue->push(obj);
6841 assert(res, "Low water mark should be less than capacity?");
6842 trim_queue(_low_water_mark);
6843 } // Else, another thread claimed the object
6844 }
6845 }
6846 }
6847
6848 void Par_MarkRefsIntoAndScanClosure::do_oop(oop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
6849 void Par_MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); }
6850
6851 // This closure is used to rescan the marked objects on the dirty cards
6852 // in the mod union table and the card table proper.
6853 size_t ScanMarkedObjectsAgainCarefullyClosure::do_object_careful_m(
6854 oop p, MemRegion mr) {
6855
6856 size_t size = 0;
6857 HeapWord* addr = (HeapWord*)p;
6858 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6859 assert(_span.contains(addr), "we are scanning the CMS generation");
6860 // check if it's time to yield
6861 if (do_yield_check()) {
6862 // We yielded for some foreground stop-world work,
6863 // and we have been asked to abort this ongoing preclean cycle.
6864 return 0;
6865 }
6866 if (_bitMap->isMarked(addr)) {
6867 // it's marked; is it potentially uninitialized?
6868 if (p->klass_or_null() != NULL) {
6869 // If is_conc_safe is false, the object may be undergoing
6870 // change by the VM outside a safepoint. Don't try to
6871 // scan it, but rather leave it for the remark phase.
6872 if (CMSPermGenPrecleaningEnabled &&
6873 (!p->is_conc_safe() || !p->is_parsable())) {
6874 // Signal precleaning to redirty the card since
6875 // the klass pointer is already installed.
6876 assert(size == 0, "Initial value");
6877 } else {
6878 assert(p->is_parsable(), "must be parsable.");
6879 // an initialized object; ignore mark word in verification below
6880 // since we are running concurrent with mutators
6881 assert(p->is_oop(true), "should be an oop");
6882 if (p->is_objArray()) {
6883 // objArrays are precisely marked; restrict scanning
6884 // to dirty cards only.
6885 size = CompactibleFreeListSpace::adjustObjectSize(
6886 p->oop_iterate(_scanningClosure, mr));
6887 } else {
6888 // A non-array may have been imprecisely marked; we need
6889 // to scan object in its entirety.
6890 size = CompactibleFreeListSpace::adjustObjectSize(
6891 p->oop_iterate(_scanningClosure));
6892 }
6893 #ifdef DEBUG
6894 size_t direct_size =
6895 CompactibleFreeListSpace::adjustObjectSize(p->size());
6896 assert(size == direct_size, "Inconsistency in size");
6897 assert(size >= 3, "Necessary for Printezis marks to work");
6898 if (!_bitMap->isMarked(addr+1)) {
6899 _bitMap->verifyNoOneBitsInRange(addr+2, addr+size);
6900 } else {
6901 _bitMap->verifyNoOneBitsInRange(addr+2, addr+size-1);
6902 assert(_bitMap->isMarked(addr+size-1),
6903 "inconsistent Printezis mark");
6904 }
6905 #endif // DEBUG
6906 }
6907 } else {
6908 // an unitialized object
6909 assert(_bitMap->isMarked(addr+1), "missing Printezis mark?");
6910 HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
6911 size = pointer_delta(nextOneAddr + 1, addr);
6912 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
6913 "alignment problem");
6914 // Note that pre-cleaning needn't redirty the card. OopDesc::set_klass()
6915 // will dirty the card when the klass pointer is installed in the
6916 // object (signalling the completion of initialization).
6917 }
6918 } else {
6919 // Either a not yet marked object or an uninitialized object
6920 if (p->klass_or_null() == NULL || !p->is_parsable()) {
6921 // An uninitialized object, skip to the next card, since
6922 // we may not be able to read its P-bits yet.
6923 assert(size == 0, "Initial value");
6924 } else {
6925 // An object not (yet) reached by marking: we merely need to
6926 // compute its size so as to go look at the next block.
6927 assert(p->is_oop(true), "should be an oop");
6928 size = CompactibleFreeListSpace::adjustObjectSize(p->size());
6929 }
6930 }
6931 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6932 return size;
6933 }
6934
6935 void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() {
6936 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
6937 "CMS thread should hold CMS token");
6938 assert_lock_strong(_freelistLock);
6939 assert_lock_strong(_bitMap->lock());
6940 DEBUG_ONLY(RememberKlassesChecker mux(false);)
6941 // relinquish the free_list_lock and bitMaplock()
6942 _bitMap->lock()->unlock();
6943 _freelistLock->unlock();
6944 ConcurrentMarkSweepThread::desynchronize(true);
6945 ConcurrentMarkSweepThread::acknowledge_yield_request();
6946 _collector->stopTimer();
6947 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
6948 if (PrintCMSStatistics != 0) {
6949 _collector->incrementYields();
6950 }
6951 _collector->icms_wait();
6952
6953 // See the comment in coordinator_yield()
6954 for (unsigned i = 0; i < CMSYieldSleepCount &&
6955 ConcurrentMarkSweepThread::should_yield() &&
6956 !CMSCollector::foregroundGCIsActive(); ++i) {
6957 os::sleep(Thread::current(), 1, false);
6958 ConcurrentMarkSweepThread::acknowledge_yield_request();
6959 }
6960
6961 ConcurrentMarkSweepThread::synchronize(true);
6962 _freelistLock->lock_without_safepoint_check();
6963 _bitMap->lock()->lock_without_safepoint_check();
6964 _collector->startTimer();
6965 }
6966
6967
6968 //////////////////////////////////////////////////////////////////
6969 // SurvivorSpacePrecleanClosure
6970 //////////////////////////////////////////////////////////////////
6971 // This (single-threaded) closure is used to preclean the oops in
6972 // the survivor spaces.
6973 size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) {
6974
6975 HeapWord* addr = (HeapWord*)p;
6976 DEBUG_ONLY(_collector->verify_work_stacks_empty();)
6977 assert(!_span.contains(addr), "we are scanning the survivor spaces");
6978 assert(p->klass_or_null() != NULL, "object should be initializd");
6979 assert(p->is_parsable(), "must be parsable.");
6980 // an initialized object; ignore mark word in verification below
6981 // since we are running concurrent with mutators
6982 assert(p->is_oop(true), "should be an oop");
6983 // Note that we do not yield while we iterate over
6984 // the interior oops of p, pushing the relevant ones
6985 // on our marking stack.
6986 size_t size = p->oop_iterate(_scanning_closure);
6987 do_yield_check();
6988 // Observe that below, we do not abandon the preclean
6989 // phase as soon as we should; rather we empty the
6990 // marking stack before returning. This is to satisfy
6991 // some existing assertions. In general, it may be a
6992 // good idea to abort immediately and complete the marking
6993 // from the grey objects at a later time.
6994 while (!_mark_stack->isEmpty()) {
6995 oop new_oop = _mark_stack->pop();
6996 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
6997 assert(new_oop->is_parsable(), "Found unparsable oop");
6998 assert(_bit_map->isMarked((HeapWord*)new_oop),
6999 "only grey objects on this stack");
7000 // iterate over the oops in this oop, marking and pushing
7001 // the ones in CMS heap (i.e. in _span).
7002 new_oop->oop_iterate(_scanning_closure);
7003 // check if it's time to yield
7004 do_yield_check();
7005 }
7006 unsigned int after_count =
7007 GenCollectedHeap::heap()->total_collections();
7008 bool abort = (_before_count != after_count) ||
7009 _collector->should_abort_preclean();
7010 return abort ? 0 : size;
7011 }
7012
7013 void SurvivorSpacePrecleanClosure::do_yield_work() {
7014 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7015 "CMS thread should hold CMS token");
7016 assert_lock_strong(_bit_map->lock());
7017 DEBUG_ONLY(RememberKlassesChecker smx(false);)
7018 // Relinquish the bit map lock
7019 _bit_map->lock()->unlock();
7020 ConcurrentMarkSweepThread::desynchronize(true);
7021 ConcurrentMarkSweepThread::acknowledge_yield_request();
7022 _collector->stopTimer();
7023 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
7024 if (PrintCMSStatistics != 0) {
7025 _collector->incrementYields();
7026 }
7027 _collector->icms_wait();
7028
7029 // See the comment in coordinator_yield()
7030 for (unsigned i = 0; i < CMSYieldSleepCount &&
7031 ConcurrentMarkSweepThread::should_yield() &&
7032 !CMSCollector::foregroundGCIsActive(); ++i) {
7033 os::sleep(Thread::current(), 1, false);
7034 ConcurrentMarkSweepThread::acknowledge_yield_request();
7035 }
7036
7037 ConcurrentMarkSweepThread::synchronize(true);
7038 _bit_map->lock()->lock_without_safepoint_check();
7039 _collector->startTimer();
7040 }
7041
7042 // This closure is used to rescan the marked objects on the dirty cards
7043 // in the mod union table and the card table proper. In the parallel
7044 // case, although the bitMap is shared, we do a single read so the
7045 // isMarked() query is "safe".
7046 bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) {
7047 // Ignore mark word because we are running concurrent with mutators
7048 assert(p->is_oop_or_null(true), "expected an oop or null");
7049 HeapWord* addr = (HeapWord*)p;
7050 assert(_span.contains(addr), "we are scanning the CMS generation");
7051 bool is_obj_array = false;
7052 #ifdef DEBUG
7053 if (!_parallel) {
7054 assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)");
7055 assert(_collector->overflow_list_is_empty(),
7056 "overflow list should be empty");
7057
7058 }
7059 #endif // DEBUG
7060 if (_bit_map->isMarked(addr)) {
7061 // Obj arrays are precisely marked, non-arrays are not;
7062 // so we scan objArrays precisely and non-arrays in their
7063 // entirety.
7064 if (p->is_objArray()) {
7065 is_obj_array = true;
7066 if (_parallel) {
7067 p->oop_iterate(_par_scan_closure, mr);
7068 } else {
7069 p->oop_iterate(_scan_closure, mr);
7070 }
7071 } else {
7072 if (_parallel) {
7073 p->oop_iterate(_par_scan_closure);
7074 } else {
7075 p->oop_iterate(_scan_closure);
7076 }
7077 }
7078 }
7079 #ifdef DEBUG
7080 if (!_parallel) {
7081 assert(_mark_stack->isEmpty(), "post-condition (eager drainage)");
7082 assert(_collector->overflow_list_is_empty(),
7083 "overflow list should be empty");
7084
7085 }
7086 #endif // DEBUG
7087 return is_obj_array;
7088 }
7089
7090 MarkFromRootsClosure::MarkFromRootsClosure(CMSCollector* collector,
7091 MemRegion span,
7092 CMSBitMap* bitMap, CMSMarkStack* markStack,
7093 CMSMarkStack* revisitStack,
7094 bool should_yield, bool verifying):
7095 _collector(collector),
7096 _span(span),
7097 _bitMap(bitMap),
7098 _mut(&collector->_modUnionTable),
7099 _markStack(markStack),
7100 _revisitStack(revisitStack),
7101 _yield(should_yield),
7102 _skipBits(0)
7103 {
7104 assert(_markStack->isEmpty(), "stack should be empty");
7105 _finger = _bitMap->startWord();
7106 _threshold = _finger;
7107 assert(_collector->_restart_addr == NULL, "Sanity check");
7108 assert(_span.contains(_finger), "Out of bounds _finger?");
7109 DEBUG_ONLY(_verifying = verifying;)
7110 }
7111
7112 void MarkFromRootsClosure::reset(HeapWord* addr) {
7113 assert(_markStack->isEmpty(), "would cause duplicates on stack");
7114 assert(_span.contains(addr), "Out of bounds _finger?");
7115 _finger = addr;
7116 _threshold = (HeapWord*)round_to(
7117 (intptr_t)_finger, CardTableModRefBS::card_size);
7118 }
7119
7120 // Should revisit to see if this should be restructured for
7121 // greater efficiency.
7122 bool MarkFromRootsClosure::do_bit(size_t offset) {
7123 if (_skipBits > 0) {
7124 _skipBits--;
7125 return true;
7126 }
7127 // convert offset into a HeapWord*
7128 HeapWord* addr = _bitMap->startWord() + offset;
7129 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
7130 "address out of range");
7131 assert(_bitMap->isMarked(addr), "tautology");
7132 if (_bitMap->isMarked(addr+1)) {
7133 // this is an allocated but not yet initialized object
7134 assert(_skipBits == 0, "tautology");
7135 _skipBits = 2; // skip next two marked bits ("Printezis-marks")
7136 oop p = oop(addr);
7137 if (p->klass_or_null() == NULL || !p->is_parsable()) {
7138 DEBUG_ONLY(if (!_verifying) {)
7139 // We re-dirty the cards on which this object lies and increase
7140 // the _threshold so that we'll come back to scan this object
7141 // during the preclean or remark phase. (CMSCleanOnEnter)
7142 if (CMSCleanOnEnter) {
7143 size_t sz = _collector->block_size_using_printezis_bits(addr);
7144 HeapWord* end_card_addr = (HeapWord*)round_to(
7145 (intptr_t)(addr+sz), CardTableModRefBS::card_size);
7146 MemRegion redirty_range = MemRegion(addr, end_card_addr);
7147 assert(!redirty_range.is_empty(), "Arithmetical tautology");
7148 // Bump _threshold to end_card_addr; note that
7149 // _threshold cannot possibly exceed end_card_addr, anyhow.
7150 // This prevents future clearing of the card as the scan proceeds
7151 // to the right.
7152 assert(_threshold <= end_card_addr,
7153 "Because we are just scanning into this object");
7154 if (_threshold < end_card_addr) {
7155 _threshold = end_card_addr;
7156 }
7157 if (p->klass_or_null() != NULL) {
7158 // Redirty the range of cards...
7159 _mut->mark_range(redirty_range);
7160 } // ...else the setting of klass will dirty the card anyway.
7161 }
7162 DEBUG_ONLY(})
7163 return true;
7164 }
7165 }
7166 scanOopsInOop(addr);
7167 return true;
7168 }
7169
7170 // We take a break if we've been at this for a while,
7171 // so as to avoid monopolizing the locks involved.
7172 void MarkFromRootsClosure::do_yield_work() {
7173 // First give up the locks, then yield, then re-lock
7174 // We should probably use a constructor/destructor idiom to
7175 // do this unlock/lock or modify the MutexUnlocker class to
7176 // serve our purpose. XXX
7177 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7178 "CMS thread should hold CMS token");
7179 assert_lock_strong(_bitMap->lock());
7180 DEBUG_ONLY(RememberKlassesChecker mux(false);)
7181 _bitMap->lock()->unlock();
7182 ConcurrentMarkSweepThread::desynchronize(true);
7183 ConcurrentMarkSweepThread::acknowledge_yield_request();
7184 _collector->stopTimer();
7185 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
7186 if (PrintCMSStatistics != 0) {
7187 _collector->incrementYields();
7188 }
7189 _collector->icms_wait();
7190
7191 // See the comment in coordinator_yield()
7192 for (unsigned i = 0; i < CMSYieldSleepCount &&
7193 ConcurrentMarkSweepThread::should_yield() &&
7194 !CMSCollector::foregroundGCIsActive(); ++i) {
7195 os::sleep(Thread::current(), 1, false);
7196 ConcurrentMarkSweepThread::acknowledge_yield_request();
7197 }
7198
7199 ConcurrentMarkSweepThread::synchronize(true);
7200 _bitMap->lock()->lock_without_safepoint_check();
7201 _collector->startTimer();
7202 }
7203
7204 void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) {
7205 assert(_bitMap->isMarked(ptr), "expected bit to be set");
7206 assert(_markStack->isEmpty(),
7207 "should drain stack to limit stack usage");
7208 // convert ptr to an oop preparatory to scanning
7209 oop obj = oop(ptr);
7210 // Ignore mark word in verification below, since we
7211 // may be running concurrent with mutators.
7212 assert(obj->is_oop(true), "should be an oop");
7213 assert(_finger <= ptr, "_finger runneth ahead");
7214 // advance the finger to right end of this object
7215 _finger = ptr + obj->size();
7216 assert(_finger > ptr, "we just incremented it above");
7217 // On large heaps, it may take us some time to get through
7218 // the marking phase (especially if running iCMS). During
7219 // this time it's possible that a lot of mutations have
7220 // accumulated in the card table and the mod union table --
7221 // these mutation records are redundant until we have
7222 // actually traced into the corresponding card.
7223 // Here, we check whether advancing the finger would make
7224 // us cross into a new card, and if so clear corresponding
7225 // cards in the MUT (preclean them in the card-table in the
7226 // future).
7227
7228 DEBUG_ONLY(if (!_verifying) {)
7229 // The clean-on-enter optimization is disabled by default,
7230 // until we fix 6178663.
7231 if (CMSCleanOnEnter && (_finger > _threshold)) {
7232 // [_threshold, _finger) represents the interval
7233 // of cards to be cleared in MUT (or precleaned in card table).
7234 // The set of cards to be cleared is all those that overlap
7235 // with the interval [_threshold, _finger); note that
7236 // _threshold is always kept card-aligned but _finger isn't
7237 // always card-aligned.
7238 HeapWord* old_threshold = _threshold;
7239 assert(old_threshold == (HeapWord*)round_to(
7240 (intptr_t)old_threshold, CardTableModRefBS::card_size),
7241 "_threshold should always be card-aligned");
7242 _threshold = (HeapWord*)round_to(
7243 (intptr_t)_finger, CardTableModRefBS::card_size);
7244 MemRegion mr(old_threshold, _threshold);
7245 assert(!mr.is_empty(), "Control point invariant");
7246 assert(_span.contains(mr), "Should clear within span");
7247 // XXX When _finger crosses from old gen into perm gen
7248 // we may be doing unnecessary cleaning; do better in the
7249 // future by detecting that condition and clearing fewer
7250 // MUT/CT entries.
7251 _mut->clear_range(mr);
7252 }
7253 DEBUG_ONLY(})
7254 // Note: the finger doesn't advance while we drain
7255 // the stack below.
7256 PushOrMarkClosure pushOrMarkClosure(_collector,
7257 _span, _bitMap, _markStack,
7258 _revisitStack,
7259 _finger, this);
7260 bool res = _markStack->push(obj);
7261 assert(res, "Empty non-zero size stack should have space for single push");
7262 while (!_markStack->isEmpty()) {
7263 oop new_oop = _markStack->pop();
7264 // Skip verifying header mark word below because we are
7265 // running concurrent with mutators.
7266 assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
7267 // now scan this oop's oops
7268 new_oop->oop_iterate(&pushOrMarkClosure);
7269 do_yield_check();
7270 }
7271 assert(_markStack->isEmpty(), "tautology, emphasizing post-condition");
7272 }
7273
7274 Par_MarkFromRootsClosure::Par_MarkFromRootsClosure(CMSConcMarkingTask* task,
7275 CMSCollector* collector, MemRegion span,
7276 CMSBitMap* bit_map,
7277 OopTaskQueue* work_queue,
7278 CMSMarkStack* overflow_stack,
7279 CMSMarkStack* revisit_stack,
7280 bool should_yield):
7281 _collector(collector),
7282 _whole_span(collector->_span),
7283 _span(span),
7284 _bit_map(bit_map),
7285 _mut(&collector->_modUnionTable),
7286 _work_queue(work_queue),
7287 _overflow_stack(overflow_stack),
7288 _revisit_stack(revisit_stack),
7289 _yield(should_yield),
7290 _skip_bits(0),
7291 _task(task)
7292 {
7293 assert(_work_queue->size() == 0, "work_queue should be empty");
7294 _finger = span.start();
7295 _threshold = _finger; // XXX Defer clear-on-enter optimization for now
7296 assert(_span.contains(_finger), "Out of bounds _finger?");
7297 }
7298
7299 // Should revisit to see if this should be restructured for
7300 // greater efficiency.
7301 bool Par_MarkFromRootsClosure::do_bit(size_t offset) {
7302 if (_skip_bits > 0) {
7303 _skip_bits--;
7304 return true;
7305 }
7306 // convert offset into a HeapWord*
7307 HeapWord* addr = _bit_map->startWord() + offset;
7308 assert(_bit_map->endWord() && addr < _bit_map->endWord(),
7309 "address out of range");
7310 assert(_bit_map->isMarked(addr), "tautology");
7311 if (_bit_map->isMarked(addr+1)) {
7312 // this is an allocated object that might not yet be initialized
7313 assert(_skip_bits == 0, "tautology");
7314 _skip_bits = 2; // skip next two marked bits ("Printezis-marks")
7315 oop p = oop(addr);
7316 if (p->klass_or_null() == NULL || !p->is_parsable()) {
7317 // in the case of Clean-on-Enter optimization, redirty card
7318 // and avoid clearing card by increasing the threshold.
7319 return true;
7320 }
7321 }
7322 scan_oops_in_oop(addr);
7323 return true;
7324 }
7325
7326 void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) {
7327 assert(_bit_map->isMarked(ptr), "expected bit to be set");
7328 // Should we assert that our work queue is empty or
7329 // below some drain limit?
7330 assert(_work_queue->size() == 0,
7331 "should drain stack to limit stack usage");
7332 // convert ptr to an oop preparatory to scanning
7333 oop obj = oop(ptr);
7334 // Ignore mark word in verification below, since we
7335 // may be running concurrent with mutators.
7336 assert(obj->is_oop(true), "should be an oop");
7337 assert(_finger <= ptr, "_finger runneth ahead");
7338 // advance the finger to right end of this object
7339 _finger = ptr + obj->size();
7340 assert(_finger > ptr, "we just incremented it above");
7341 // On large heaps, it may take us some time to get through
7342 // the marking phase (especially if running iCMS). During
7343 // this time it's possible that a lot of mutations have
7344 // accumulated in the card table and the mod union table --
7345 // these mutation records are redundant until we have
7346 // actually traced into the corresponding card.
7347 // Here, we check whether advancing the finger would make
7348 // us cross into a new card, and if so clear corresponding
7349 // cards in the MUT (preclean them in the card-table in the
7350 // future).
7351
7352 // The clean-on-enter optimization is disabled by default,
7353 // until we fix 6178663.
7354 if (CMSCleanOnEnter && (_finger > _threshold)) {
7355 // [_threshold, _finger) represents the interval
7356 // of cards to be cleared in MUT (or precleaned in card table).
7357 // The set of cards to be cleared is all those that overlap
7358 // with the interval [_threshold, _finger); note that
7359 // _threshold is always kept card-aligned but _finger isn't
7360 // always card-aligned.
7361 HeapWord* old_threshold = _threshold;
7362 assert(old_threshold == (HeapWord*)round_to(
7363 (intptr_t)old_threshold, CardTableModRefBS::card_size),
7364 "_threshold should always be card-aligned");
7365 _threshold = (HeapWord*)round_to(
7366 (intptr_t)_finger, CardTableModRefBS::card_size);
7367 MemRegion mr(old_threshold, _threshold);
7368 assert(!mr.is_empty(), "Control point invariant");
7369 assert(_span.contains(mr), "Should clear within span"); // _whole_span ??
7370 // XXX When _finger crosses from old gen into perm gen
7371 // we may be doing unnecessary cleaning; do better in the
7372 // future by detecting that condition and clearing fewer
7373 // MUT/CT entries.
7374 _mut->clear_range(mr);
7375 }
7376
7377 // Note: the local finger doesn't advance while we drain
7378 // the stack below, but the global finger sure can and will.
7379 HeapWord** gfa = _task->global_finger_addr();
7380 Par_PushOrMarkClosure pushOrMarkClosure(_collector,
7381 _span, _bit_map,
7382 _work_queue,
7383 _overflow_stack,
7384 _revisit_stack,
7385 _finger,
7386 gfa, this);
7387 bool res = _work_queue->push(obj); // overflow could occur here
7388 assert(res, "Will hold once we use workqueues");
7389 while (true) {
7390 oop new_oop;
7391 if (!_work_queue->pop_local(new_oop)) {
7392 // We emptied our work_queue; check if there's stuff that can
7393 // be gotten from the overflow stack.
7394 if (CMSConcMarkingTask::get_work_from_overflow_stack(
7395 _overflow_stack, _work_queue)) {
7396 do_yield_check();
7397 continue;
7398 } else { // done
7399 break;
7400 }
7401 }
7402 // Skip verifying header mark word below because we are
7403 // running concurrent with mutators.
7404 assert(new_oop->is_oop(true), "Oops! expected to pop an oop");
7405 // now scan this oop's oops
7406 new_oop->oop_iterate(&pushOrMarkClosure);
7407 do_yield_check();
7408 }
7409 assert(_work_queue->size() == 0, "tautology, emphasizing post-condition");
7410 }
7411
7412 // Yield in response to a request from VM Thread or
7413 // from mutators.
7414 void Par_MarkFromRootsClosure::do_yield_work() {
7415 assert(_task != NULL, "sanity");
7416 _task->yield();
7417 }
7418
7419 // A variant of the above used for verifying CMS marking work.
7420 MarkFromRootsVerifyClosure::MarkFromRootsVerifyClosure(CMSCollector* collector,
7421 MemRegion span,
7422 CMSBitMap* verification_bm, CMSBitMap* cms_bm,
7423 CMSMarkStack* mark_stack):
7424 _collector(collector),
7425 _span(span),
7426 _verification_bm(verification_bm),
7427 _cms_bm(cms_bm),
7428 _mark_stack(mark_stack),
7429 _pam_verify_closure(collector, span, verification_bm, cms_bm,
7430 mark_stack)
7431 {
7432 assert(_mark_stack->isEmpty(), "stack should be empty");
7433 _finger = _verification_bm->startWord();
7434 assert(_collector->_restart_addr == NULL, "Sanity check");
7435 assert(_span.contains(_finger), "Out of bounds _finger?");
7436 }
7437
7438 void MarkFromRootsVerifyClosure::reset(HeapWord* addr) {
7439 assert(_mark_stack->isEmpty(), "would cause duplicates on stack");
7440 assert(_span.contains(addr), "Out of bounds _finger?");
7441 _finger = addr;
7442 }
7443
7444 // Should revisit to see if this should be restructured for
7445 // greater efficiency.
7446 bool MarkFromRootsVerifyClosure::do_bit(size_t offset) {
7447 // convert offset into a HeapWord*
7448 HeapWord* addr = _verification_bm->startWord() + offset;
7449 assert(_verification_bm->endWord() && addr < _verification_bm->endWord(),
7450 "address out of range");
7451 assert(_verification_bm->isMarked(addr), "tautology");
7452 assert(_cms_bm->isMarked(addr), "tautology");
7453
7454 assert(_mark_stack->isEmpty(),
7455 "should drain stack to limit stack usage");
7456 // convert addr to an oop preparatory to scanning
7457 oop obj = oop(addr);
7458 assert(obj->is_oop(), "should be an oop");
7459 assert(_finger <= addr, "_finger runneth ahead");
7460 // advance the finger to right end of this object
7461 _finger = addr + obj->size();
7462 assert(_finger > addr, "we just incremented it above");
7463 // Note: the finger doesn't advance while we drain
7464 // the stack below.
7465 bool res = _mark_stack->push(obj);
7466 assert(res, "Empty non-zero size stack should have space for single push");
7467 while (!_mark_stack->isEmpty()) {
7468 oop new_oop = _mark_stack->pop();
7469 assert(new_oop->is_oop(), "Oops! expected to pop an oop");
7470 // now scan this oop's oops
7471 new_oop->oop_iterate(&_pam_verify_closure);
7472 }
7473 assert(_mark_stack->isEmpty(), "tautology, emphasizing post-condition");
7474 return true;
7475 }
7476
7477 PushAndMarkVerifyClosure::PushAndMarkVerifyClosure(
7478 CMSCollector* collector, MemRegion span,
7479 CMSBitMap* verification_bm, CMSBitMap* cms_bm,
7480 CMSMarkStack* mark_stack):
7481 OopClosure(collector->ref_processor()),
7482 _collector(collector),
7483 _span(span),
7484 _verification_bm(verification_bm),
7485 _cms_bm(cms_bm),
7486 _mark_stack(mark_stack)
7487 { }
7488
7489 void PushAndMarkVerifyClosure::do_oop(oop* p) { PushAndMarkVerifyClosure::do_oop_work(p); }
7490 void PushAndMarkVerifyClosure::do_oop(narrowOop* p) { PushAndMarkVerifyClosure::do_oop_work(p); }
7491
7492 // Upon stack overflow, we discard (part of) the stack,
7493 // remembering the least address amongst those discarded
7494 // in CMSCollector's _restart_address.
7495 void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) {
7496 // Remember the least grey address discarded
7497 HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost);
7498 _collector->lower_restart_addr(ra);
7499 _mark_stack->reset(); // discard stack contents
7500 _mark_stack->expand(); // expand the stack if possible
7501 }
7502
7503 void PushAndMarkVerifyClosure::do_oop(oop obj) {
7504 assert(obj->is_oop_or_null(), "expected an oop or NULL");
7505 HeapWord* addr = (HeapWord*)obj;
7506 if (_span.contains(addr) && !_verification_bm->isMarked(addr)) {
7507 // Oop lies in _span and isn't yet grey or black
7508 _verification_bm->mark(addr); // now grey
7509 if (!_cms_bm->isMarked(addr)) {
7510 oop(addr)->print();
7511 gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)",
7512 addr);
7513 fatal("... aborting");
7514 }
7515
7516 if (!_mark_stack->push(obj)) { // stack overflow
7517 if (PrintCMSStatistics != 0) {
7518 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
7519 SIZE_FORMAT, _mark_stack->capacity());
7520 }
7521 assert(_mark_stack->isFull(), "Else push should have succeeded");
7522 handle_stack_overflow(addr);
7523 }
7524 // anything including and to the right of _finger
7525 // will be scanned as we iterate over the remainder of the
7526 // bit map
7527 }
7528 }
7529
7530 PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector,
7531 MemRegion span,
7532 CMSBitMap* bitMap, CMSMarkStack* markStack,
7533 CMSMarkStack* revisitStack,
7534 HeapWord* finger, MarkFromRootsClosure* parent) :
7535 KlassRememberingOopClosure(collector, collector->ref_processor(), revisitStack),
7536 _span(span),
7537 _bitMap(bitMap),
7538 _markStack(markStack),
7539 _finger(finger),
7540 _parent(parent)
7541 { }
7542
7543 Par_PushOrMarkClosure::Par_PushOrMarkClosure(CMSCollector* collector,
7544 MemRegion span,
7545 CMSBitMap* bit_map,
7546 OopTaskQueue* work_queue,
7547 CMSMarkStack* overflow_stack,
7548 CMSMarkStack* revisit_stack,
7549 HeapWord* finger,
7550 HeapWord** global_finger_addr,
7551 Par_MarkFromRootsClosure* parent) :
7552 Par_KlassRememberingOopClosure(collector,
7553 collector->ref_processor(),
7554 revisit_stack),
7555 _whole_span(collector->_span),
7556 _span(span),
7557 _bit_map(bit_map),
7558 _work_queue(work_queue),
7559 _overflow_stack(overflow_stack),
7560 _finger(finger),
7561 _global_finger_addr(global_finger_addr),
7562 _parent(parent)
7563 { }
7564
7565 // Assumes thread-safe access by callers, who are
7566 // responsible for mutual exclusion.
7567 void CMSCollector::lower_restart_addr(HeapWord* low) {
7568 assert(_span.contains(low), "Out of bounds addr");
7569 if (_restart_addr == NULL) {
7570 _restart_addr = low;
7571 } else {
7572 _restart_addr = MIN2(_restart_addr, low);
7573 }
7574 }
7575
7576 // Upon stack overflow, we discard (part of) the stack,
7577 // remembering the least address amongst those discarded
7578 // in CMSCollector's _restart_address.
7579 void PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
7580 // Remember the least grey address discarded
7581 HeapWord* ra = (HeapWord*)_markStack->least_value(lost);
7582 _collector->lower_restart_addr(ra);
7583 _markStack->reset(); // discard stack contents
7584 _markStack->expand(); // expand the stack if possible
7585 }
7586
7587 // Upon stack overflow, we discard (part of) the stack,
7588 // remembering the least address amongst those discarded
7589 // in CMSCollector's _restart_address.
7590 void Par_PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
7591 // We need to do this under a mutex to prevent other
7592 // workers from interfering with the work done below.
7593 MutexLockerEx ml(_overflow_stack->par_lock(),
7594 Mutex::_no_safepoint_check_flag);
7595 // Remember the least grey address discarded
7596 HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost);
7597 _collector->lower_restart_addr(ra);
7598 _overflow_stack->reset(); // discard stack contents
7599 _overflow_stack->expand(); // expand the stack if possible
7600 }
7601
7602 void PushOrMarkClosure::do_oop(oop obj) {
7603 // Ignore mark word because we are running concurrent with mutators.
7604 assert(obj->is_oop_or_null(true), "expected an oop or NULL");
7605 HeapWord* addr = (HeapWord*)obj;
7606 if (_span.contains(addr) && !_bitMap->isMarked(addr)) {
7607 // Oop lies in _span and isn't yet grey or black
7608 _bitMap->mark(addr); // now grey
7609 if (addr < _finger) {
7610 // the bit map iteration has already either passed, or
7611 // sampled, this bit in the bit map; we'll need to
7612 // use the marking stack to scan this oop's oops.
7613 bool simulate_overflow = false;
7614 NOT_PRODUCT(
7615 if (CMSMarkStackOverflowALot &&
7616 _collector->simulate_overflow()) {
7617 // simulate a stack overflow
7618 simulate_overflow = true;
7619 }
7620 )
7621 if (simulate_overflow || !_markStack->push(obj)) { // stack overflow
7622 if (PrintCMSStatistics != 0) {
7623 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
7624 SIZE_FORMAT, _markStack->capacity());
7625 }
7626 assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded");
7627 handle_stack_overflow(addr);
7628 }
7629 }
7630 // anything including and to the right of _finger
7631 // will be scanned as we iterate over the remainder of the
7632 // bit map
7633 do_yield_check();
7634 }
7635 }
7636
7637 void PushOrMarkClosure::do_oop(oop* p) { PushOrMarkClosure::do_oop_work(p); }
7638 void PushOrMarkClosure::do_oop(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); }
7639
7640 void Par_PushOrMarkClosure::do_oop(oop obj) {
7641 // Ignore mark word because we are running concurrent with mutators.
7642 assert(obj->is_oop_or_null(true), "expected an oop or NULL");
7643 HeapWord* addr = (HeapWord*)obj;
7644 if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) {
7645 // Oop lies in _span and isn't yet grey or black
7646 // We read the global_finger (volatile read) strictly after marking oop
7647 bool res = _bit_map->par_mark(addr); // now grey
7648 volatile HeapWord** gfa = (volatile HeapWord**)_global_finger_addr;
7649 // Should we push this marked oop on our stack?
7650 // -- if someone else marked it, nothing to do
7651 // -- if target oop is above global finger nothing to do
7652 // -- if target oop is in chunk and above local finger
7653 // then nothing to do
7654 // -- else push on work queue
7655 if ( !res // someone else marked it, they will deal with it
7656 || (addr >= *gfa) // will be scanned in a later task
7657 || (_span.contains(addr) && addr >= _finger)) { // later in this chunk
7658 return;
7659 }
7660 // the bit map iteration has already either passed, or
7661 // sampled, this bit in the bit map; we'll need to
7662 // use the marking stack to scan this oop's oops.
7663 bool simulate_overflow = false;
7664 NOT_PRODUCT(
7665 if (CMSMarkStackOverflowALot &&
7666 _collector->simulate_overflow()) {
7667 // simulate a stack overflow
7668 simulate_overflow = true;
7669 }
7670 )
7671 if (simulate_overflow ||
7672 !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) {
7673 // stack overflow
7674 if (PrintCMSStatistics != 0) {
7675 gclog_or_tty->print_cr("CMS marking stack overflow (benign) at "
7676 SIZE_FORMAT, _overflow_stack->capacity());
7677 }
7678 // We cannot assert that the overflow stack is full because
7679 // it may have been emptied since.
7680 assert(simulate_overflow ||
7681 _work_queue->size() == _work_queue->max_elems(),
7682 "Else push should have succeeded");
7683 handle_stack_overflow(addr);
7684 }
7685 do_yield_check();
7686 }
7687 }
7688
7689 void Par_PushOrMarkClosure::do_oop(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
7690 void Par_PushOrMarkClosure::do_oop(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); }
7691
7692 KlassRememberingOopClosure::KlassRememberingOopClosure(CMSCollector* collector,
7693 ReferenceProcessor* rp,
7694 CMSMarkStack* revisit_stack) :
7695 OopClosure(rp),
7696 _collector(collector),
7697 _revisit_stack(revisit_stack),
7698 _should_remember_klasses(collector->should_unload_classes()) {}
7699
7700 PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector,
7701 MemRegion span,
7702 ReferenceProcessor* rp,
7703 CMSBitMap* bit_map,
7704 CMSBitMap* mod_union_table,
7705 CMSMarkStack* mark_stack,
7706 CMSMarkStack* revisit_stack,
7707 bool concurrent_precleaning):
7708 KlassRememberingOopClosure(collector, rp, revisit_stack),
7709 _span(span),
7710 _bit_map(bit_map),
7711 _mod_union_table(mod_union_table),
7712 _mark_stack(mark_stack),
7713 _concurrent_precleaning(concurrent_precleaning)
7714 {
7715 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
7716 }
7717
7718 // Grey object rescan during pre-cleaning and second checkpoint phases --
7719 // the non-parallel version (the parallel version appears further below.)
7720 void PushAndMarkClosure::do_oop(oop obj) {
7721 // Ignore mark word verification. If during concurrent precleaning,
7722 // the object monitor may be locked. If during the checkpoint
7723 // phases, the object may already have been reached by a different
7724 // path and may be at the end of the global overflow list (so
7725 // the mark word may be NULL).
7726 assert(obj->is_oop_or_null(true /* ignore mark word */),
7727 "expected an oop or NULL");
7728 HeapWord* addr = (HeapWord*)obj;
7729 // Check if oop points into the CMS generation
7730 // and is not marked
7731 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
7732 // a white object ...
7733 _bit_map->mark(addr); // ... now grey
7734 // push on the marking stack (grey set)
7735 bool simulate_overflow = false;
7736 NOT_PRODUCT(
7737 if (CMSMarkStackOverflowALot &&
7738 _collector->simulate_overflow()) {
7739 // simulate a stack overflow
7740 simulate_overflow = true;
7741 }
7742 )
7743 if (simulate_overflow || !_mark_stack->push(obj)) {
7744 if (_concurrent_precleaning) {
7745 // During precleaning we can just dirty the appropriate card(s)
7746 // in the mod union table, thus ensuring that the object remains
7747 // in the grey set and continue. In the case of object arrays
7748 // we need to dirty all of the cards that the object spans,
7749 // since the rescan of object arrays will be limited to the
7750 // dirty cards.
7751 // Note that no one can be intefering with us in this action
7752 // of dirtying the mod union table, so no locking or atomics
7753 // are required.
7754 if (obj->is_objArray()) {
7755 size_t sz = obj->size();
7756 HeapWord* end_card_addr = (HeapWord*)round_to(
7757 (intptr_t)(addr+sz), CardTableModRefBS::card_size);
7758 MemRegion redirty_range = MemRegion(addr, end_card_addr);
7759 assert(!redirty_range.is_empty(), "Arithmetical tautology");
7760 _mod_union_table->mark_range(redirty_range);
7761 } else {
7762 _mod_union_table->mark(addr);
7763 }
7764 _collector->_ser_pmc_preclean_ovflw++;
7765 } else {
7766 // During the remark phase, we need to remember this oop
7767 // in the overflow list.
7768 _collector->push_on_overflow_list(obj);
7769 _collector->_ser_pmc_remark_ovflw++;
7770 }
7771 }
7772 }
7773 }
7774
7775 Par_PushAndMarkClosure::Par_PushAndMarkClosure(CMSCollector* collector,
7776 MemRegion span,
7777 ReferenceProcessor* rp,
7778 CMSBitMap* bit_map,
7779 OopTaskQueue* work_queue,
7780 CMSMarkStack* revisit_stack):
7781 Par_KlassRememberingOopClosure(collector, rp, revisit_stack),
7782 _span(span),
7783 _bit_map(bit_map),
7784 _work_queue(work_queue)
7785 {
7786 assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL");
7787 }
7788
7789 void PushAndMarkClosure::do_oop(oop* p) { PushAndMarkClosure::do_oop_work(p); }
7790 void PushAndMarkClosure::do_oop(narrowOop* p) { PushAndMarkClosure::do_oop_work(p); }
7791
7792 // Grey object rescan during second checkpoint phase --
7793 // the parallel version.
7794 void Par_PushAndMarkClosure::do_oop(oop obj) {
7795 // In the assert below, we ignore the mark word because
7796 // this oop may point to an already visited object that is
7797 // on the overflow stack (in which case the mark word has
7798 // been hijacked for chaining into the overflow stack --
7799 // if this is the last object in the overflow stack then
7800 // its mark word will be NULL). Because this object may
7801 // have been subsequently popped off the global overflow
7802 // stack, and the mark word possibly restored to the prototypical
7803 // value, by the time we get to examined this failing assert in
7804 // the debugger, is_oop_or_null(false) may subsequently start
7805 // to hold.
7806 assert(obj->is_oop_or_null(true),
7807 "expected an oop or NULL");
7808 HeapWord* addr = (HeapWord*)obj;
7809 // Check if oop points into the CMS generation
7810 // and is not marked
7811 if (_span.contains(addr) && !_bit_map->isMarked(addr)) {
7812 // a white object ...
7813 // If we manage to "claim" the object, by being the
7814 // first thread to mark it, then we push it on our
7815 // marking stack
7816 if (_bit_map->par_mark(addr)) { // ... now grey
7817 // push on work queue (grey set)
7818 bool simulate_overflow = false;
7819 NOT_PRODUCT(
7820 if (CMSMarkStackOverflowALot &&
7821 _collector->par_simulate_overflow()) {
7822 // simulate a stack overflow
7823 simulate_overflow = true;
7824 }
7825 )
7826 if (simulate_overflow || !_work_queue->push(obj)) {
7827 _collector->par_push_on_overflow_list(obj);
7828 _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS
7829 }
7830 } // Else, some other thread got there first
7831 }
7832 }
7833
7834 void Par_PushAndMarkClosure::do_oop(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
7835 void Par_PushAndMarkClosure::do_oop(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); }
7836
7837 void PushAndMarkClosure::remember_mdo(DataLayout* v) {
7838 // TBD
7839 }
7840
7841 void Par_PushAndMarkClosure::remember_mdo(DataLayout* v) {
7842 // TBD
7843 }
7844
7845 void CMSPrecleanRefsYieldClosure::do_yield_work() {
7846 DEBUG_ONLY(RememberKlassesChecker mux(false);)
7847 Mutex* bml = _collector->bitMapLock();
7848 assert_lock_strong(bml);
7849 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
7850 "CMS thread should hold CMS token");
7851
7852 bml->unlock();
7853 ConcurrentMarkSweepThread::desynchronize(true);
7854
7855 ConcurrentMarkSweepThread::acknowledge_yield_request();
7856
7857 _collector->stopTimer();
7858 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
7859 if (PrintCMSStatistics != 0) {
7860 _collector->incrementYields();
7861 }
7862 _collector->icms_wait();
7863
7864 // See the comment in coordinator_yield()
7865 for (unsigned i = 0; i < CMSYieldSleepCount &&
7866 ConcurrentMarkSweepThread::should_yield() &&
7867 !CMSCollector::foregroundGCIsActive(); ++i) {
7868 os::sleep(Thread::current(), 1, false);
7869 ConcurrentMarkSweepThread::acknowledge_yield_request();
7870 }
7871
7872 ConcurrentMarkSweepThread::synchronize(true);
7873 bml->lock();
7874
7875 _collector->startTimer();
7876 }
7877
7878 bool CMSPrecleanRefsYieldClosure::should_return() {
7879 if (ConcurrentMarkSweepThread::should_yield()) {
7880 do_yield_work();
7881 }
7882 return _collector->foregroundGCIsActive();
7883 }
7884
7885 void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) {
7886 assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0,
7887 "mr should be aligned to start at a card boundary");
7888 // We'd like to assert:
7889 // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0,
7890 // "mr should be a range of cards");
7891 // However, that would be too strong in one case -- the last
7892 // partition ends at _unallocated_block which, in general, can be
7893 // an arbitrary boundary, not necessarily card aligned.
7894 if (PrintCMSStatistics != 0) {
7895 _num_dirty_cards +=
7896 mr.word_size()/CardTableModRefBS::card_size_in_words;
7897 }
7898 _space->object_iterate_mem(mr, &_scan_cl);
7899 }
7900
7901 SweepClosure::SweepClosure(CMSCollector* collector,
7902 ConcurrentMarkSweepGeneration* g,
7903 CMSBitMap* bitMap, bool should_yield) :
7904 _collector(collector),
7905 _g(g),
7906 _sp(g->cmsSpace()),
7907 _limit(_sp->sweep_limit()),
7908 _freelistLock(_sp->freelistLock()),
7909 _bitMap(bitMap),
7910 _yield(should_yield),
7911 _inFreeRange(false), // No free range at beginning of sweep
7912 _freeRangeInFreeLists(false), // No free range at beginning of sweep
7913 _lastFreeRangeCoalesced(false),
7914 _freeFinger(g->used_region().start())
7915 {
7916 NOT_PRODUCT(
7917 _numObjectsFreed = 0;
7918 _numWordsFreed = 0;
7919 _numObjectsLive = 0;
7920 _numWordsLive = 0;
7921 _numObjectsAlreadyFree = 0;
7922 _numWordsAlreadyFree = 0;
7923 _last_fc = NULL;
7924
7925 _sp->initializeIndexedFreeListArrayReturnedBytes();
7926 _sp->dictionary()->initializeDictReturnedBytes();
7927 )
7928 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
7929 "sweep _limit out of bounds");
7930 if (CMSTraceSweeper) {
7931 gclog_or_tty->print("\n====================\nStarting new sweep\n");
7932 }
7933 }
7934
7935 // We need this destructor to reclaim any space at the end
7936 // of the space, which do_blk below may not have added back to
7937 // the free lists. [basically dealing with the "fringe effect"]
7938 SweepClosure::~SweepClosure() {
7939 assert_lock_strong(_freelistLock);
7940 // this should be treated as the end of a free run if any
7941 // The current free range should be returned to the free lists
7942 // as one coalesced chunk.
7943 if (inFreeRange()) {
7944 flushCurFreeChunk(freeFinger(),
7945 pointer_delta(_limit, freeFinger()));
7946 assert(freeFinger() < _limit, "the finger pointeth off base");
7947 if (CMSTraceSweeper) {
7948 gclog_or_tty->print("destructor:");
7949 gclog_or_tty->print("Sweep:put_free_blk 0x%x ("SIZE_FORMAT") "
7950 "[coalesced:"SIZE_FORMAT"]\n",
7951 freeFinger(), pointer_delta(_limit, freeFinger()),
7952 lastFreeRangeCoalesced());
7953 }
7954 }
7955 NOT_PRODUCT(
7956 if (Verbose && PrintGC) {
7957 gclog_or_tty->print("Collected "SIZE_FORMAT" objects, "
7958 SIZE_FORMAT " bytes",
7959 _numObjectsFreed, _numWordsFreed*sizeof(HeapWord));
7960 gclog_or_tty->print_cr("\nLive "SIZE_FORMAT" objects, "
7961 SIZE_FORMAT" bytes "
7962 "Already free "SIZE_FORMAT" objects, "SIZE_FORMAT" bytes",
7963 _numObjectsLive, _numWordsLive*sizeof(HeapWord),
7964 _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord));
7965 size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree) *
7966 sizeof(HeapWord);
7967 gclog_or_tty->print_cr("Total sweep: "SIZE_FORMAT" bytes", totalBytes);
7968
7969 if (PrintCMSStatistics && CMSVerifyReturnedBytes) {
7970 size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes();
7971 size_t dictReturnedBytes = _sp->dictionary()->sumDictReturnedBytes();
7972 size_t returnedBytes = indexListReturnedBytes + dictReturnedBytes;
7973 gclog_or_tty->print("Returned "SIZE_FORMAT" bytes", returnedBytes);
7974 gclog_or_tty->print(" Indexed List Returned "SIZE_FORMAT" bytes",
7975 indexListReturnedBytes);
7976 gclog_or_tty->print_cr(" Dictionary Returned "SIZE_FORMAT" bytes",
7977 dictReturnedBytes);
7978 }
7979 }
7980 )
7981 // Now, in debug mode, just null out the sweep_limit
7982 NOT_PRODUCT(_sp->clear_sweep_limit();)
7983 if (CMSTraceSweeper) {
7984 gclog_or_tty->print("end of sweep\n================\n");
7985 }
7986 }
7987
7988 void SweepClosure::initialize_free_range(HeapWord* freeFinger,
7989 bool freeRangeInFreeLists) {
7990 if (CMSTraceSweeper) {
7991 gclog_or_tty->print("---- Start free range 0x%x with free block [%d] (%d)\n",
7992 freeFinger, _sp->block_size(freeFinger),
7993 freeRangeInFreeLists);
7994 }
7995 assert(!inFreeRange(), "Trampling existing free range");
7996 set_inFreeRange(true);
7997 set_lastFreeRangeCoalesced(false);
7998
7999 set_freeFinger(freeFinger);
8000 set_freeRangeInFreeLists(freeRangeInFreeLists);
8001 if (CMSTestInFreeList) {
8002 if (freeRangeInFreeLists) {
8003 FreeChunk* fc = (FreeChunk*) freeFinger;
8004 assert(fc->isFree(), "A chunk on the free list should be free.");
8005 assert(fc->size() > 0, "Free range should have a size");
8006 assert(_sp->verifyChunkInFreeLists(fc), "Chunk is not in free lists");
8007 }
8008 }
8009 }
8010
8011 // Note that the sweeper runs concurrently with mutators. Thus,
8012 // it is possible for direct allocation in this generation to happen
8013 // in the middle of the sweep. Note that the sweeper also coalesces
8014 // contiguous free blocks. Thus, unless the sweeper and the allocator
8015 // synchronize appropriately freshly allocated blocks may get swept up.
8016 // This is accomplished by the sweeper locking the free lists while
8017 // it is sweeping. Thus blocks that are determined to be free are
8018 // indeed free. There is however one additional complication:
8019 // blocks that have been allocated since the final checkpoint and
8020 // mark, will not have been marked and so would be treated as
8021 // unreachable and swept up. To prevent this, the allocator marks
8022 // the bit map when allocating during the sweep phase. This leads,
8023 // however, to a further complication -- objects may have been allocated
8024 // but not yet initialized -- in the sense that the header isn't yet
8025 // installed. The sweeper can not then determine the size of the block
8026 // in order to skip over it. To deal with this case, we use a technique
8027 // (due to Printezis) to encode such uninitialized block sizes in the
8028 // bit map. Since the bit map uses a bit per every HeapWord, but the
8029 // CMS generation has a minimum object size of 3 HeapWords, it follows
8030 // that "normal marks" won't be adjacent in the bit map (there will
8031 // always be at least two 0 bits between successive 1 bits). We make use
8032 // of these "unused" bits to represent uninitialized blocks -- the bit
8033 // corresponding to the start of the uninitialized object and the next
8034 // bit are both set. Finally, a 1 bit marks the end of the object that
8035 // started with the two consecutive 1 bits to indicate its potentially
8036 // uninitialized state.
8037
8038 size_t SweepClosure::do_blk_careful(HeapWord* addr) {
8039 FreeChunk* fc = (FreeChunk*)addr;
8040 size_t res;
8041
8042 // Check if we are done sweeping. Below we check "addr >= _limit" rather
8043 // than "addr == _limit" because although _limit was a block boundary when
8044 // we started the sweep, it may no longer be one because heap expansion
8045 // may have caused us to coalesce the block ending at the address _limit
8046 // with a newly expanded chunk (this happens when _limit was set to the
8047 // previous _end of the space), so we may have stepped past _limit; see CR 6977970.
8048 if (addr >= _limit) { // we have swept up to or past the limit, do nothing more
8049 assert(_limit >= _sp->bottom() && _limit <= _sp->end(),
8050 "sweep _limit out of bounds");
8051 assert(addr < _sp->end(), "addr out of bounds");
8052 // help the closure application finish
8053 return pointer_delta(_sp->end(), addr);
8054 }
8055 assert(addr < _limit, "sweep invariant");
8056
8057 // check if we should yield
8058 do_yield_check(addr);
8059 if (fc->isFree()) {
8060 // Chunk that is already free
8061 res = fc->size();
8062 doAlreadyFreeChunk(fc);
8063 debug_only(_sp->verifyFreeLists());
8064 assert(res == fc->size(), "Don't expect the size to change");
8065 NOT_PRODUCT(
8066 _numObjectsAlreadyFree++;
8067 _numWordsAlreadyFree += res;
8068 )
8069 NOT_PRODUCT(_last_fc = fc;)
8070 } else if (!_bitMap->isMarked(addr)) {
8071 // Chunk is fresh garbage
8072 res = doGarbageChunk(fc);
8073 debug_only(_sp->verifyFreeLists());
8074 NOT_PRODUCT(
8075 _numObjectsFreed++;
8076 _numWordsFreed += res;
8077 )
8078 } else {
8079 // Chunk that is alive.
8080 res = doLiveChunk(fc);
8081 debug_only(_sp->verifyFreeLists());
8082 NOT_PRODUCT(
8083 _numObjectsLive++;
8084 _numWordsLive += res;
8085 )
8086 }
8087 return res;
8088 }
8089
8090 // For the smart allocation, record following
8091 // split deaths - a free chunk is removed from its free list because
8092 // it is being split into two or more chunks.
8093 // split birth - a free chunk is being added to its free list because
8094 // a larger free chunk has been split and resulted in this free chunk.
8095 // coal death - a free chunk is being removed from its free list because
8096 // it is being coalesced into a large free chunk.
8097 // coal birth - a free chunk is being added to its free list because
8098 // it was created when two or more free chunks where coalesced into
8099 // this free chunk.
8100 //
8101 // These statistics are used to determine the desired number of free
8102 // chunks of a given size. The desired number is chosen to be relative
8103 // to the end of a CMS sweep. The desired number at the end of a sweep
8104 // is the
8105 // count-at-end-of-previous-sweep (an amount that was enough)
8106 // - count-at-beginning-of-current-sweep (the excess)
8107 // + split-births (gains in this size during interval)
8108 // - split-deaths (demands on this size during interval)
8109 // where the interval is from the end of one sweep to the end of the
8110 // next.
8111 //
8112 // When sweeping the sweeper maintains an accumulated chunk which is
8113 // the chunk that is made up of chunks that have been coalesced. That
8114 // will be termed the left-hand chunk. A new chunk of garbage that
8115 // is being considered for coalescing will be referred to as the
8116 // right-hand chunk.
8117 //
8118 // When making a decision on whether to coalesce a right-hand chunk with
8119 // the current left-hand chunk, the current count vs. the desired count
8120 // of the left-hand chunk is considered. Also if the right-hand chunk
8121 // is near the large chunk at the end of the heap (see
8122 // ConcurrentMarkSweepGeneration::isNearLargestChunk()), then the
8123 // left-hand chunk is coalesced.
8124 //
8125 // When making a decision about whether to split a chunk, the desired count
8126 // vs. the current count of the candidate to be split is also considered.
8127 // If the candidate is underpopulated (currently fewer chunks than desired)
8128 // a chunk of an overpopulated (currently more chunks than desired) size may
8129 // be chosen. The "hint" associated with a free list, if non-null, points
8130 // to a free list which may be overpopulated.
8131 //
8132
8133 void SweepClosure::doAlreadyFreeChunk(FreeChunk* fc) {
8134 size_t size = fc->size();
8135 // Chunks that cannot be coalesced are not in the
8136 // free lists.
8137 if (CMSTestInFreeList && !fc->cantCoalesce()) {
8138 assert(_sp->verifyChunkInFreeLists(fc),
8139 "free chunk should be in free lists");
8140 }
8141 // a chunk that is already free, should not have been
8142 // marked in the bit map
8143 HeapWord* addr = (HeapWord*) fc;
8144 assert(!_bitMap->isMarked(addr), "free chunk should be unmarked");
8145 // Verify that the bit map has no bits marked between
8146 // addr and purported end of this block.
8147 _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
8148
8149 // Some chunks cannot be coalesced in under any circumstances.
8150 // See the definition of cantCoalesce().
8151 if (!fc->cantCoalesce()) {
8152 // This chunk can potentially be coalesced.
8153 if (_sp->adaptive_freelists()) {
8154 // All the work is done in
8155 doPostIsFreeOrGarbageChunk(fc, size);
8156 } else { // Not adaptive free lists
8157 // this is a free chunk that can potentially be coalesced by the sweeper;
8158 if (!inFreeRange()) {
8159 // if the next chunk is a free block that can't be coalesced
8160 // it doesn't make sense to remove this chunk from the free lists
8161 FreeChunk* nextChunk = (FreeChunk*)(addr + size);
8162 assert((HeapWord*)nextChunk <= _limit, "sweep invariant");
8163 if ((HeapWord*)nextChunk < _limit && // there's a next chunk...
8164 nextChunk->isFree() && // which is free...
8165 nextChunk->cantCoalesce()) { // ... but cant be coalesced
8166 // nothing to do
8167 } else {
8168 // Potentially the start of a new free range:
8169 // Don't eagerly remove it from the free lists.
8170 // No need to remove it if it will just be put
8171 // back again. (Also from a pragmatic point of view
8172 // if it is a free block in a region that is beyond
8173 // any allocated blocks, an assertion will fail)
8174 // Remember the start of a free run.
8175 initialize_free_range(addr, true);
8176 // end - can coalesce with next chunk
8177 }
8178 } else {
8179 // the midst of a free range, we are coalescing
8180 debug_only(record_free_block_coalesced(fc);)
8181 if (CMSTraceSweeper) {
8182 gclog_or_tty->print(" -- pick up free block 0x%x (%d)\n", fc, size);
8183 }
8184 // remove it from the free lists
8185 _sp->removeFreeChunkFromFreeLists(fc);
8186 set_lastFreeRangeCoalesced(true);
8187 // If the chunk is being coalesced and the current free range is
8188 // in the free lists, remove the current free range so that it
8189 // will be returned to the free lists in its entirety - all
8190 // the coalesced pieces included.
8191 if (freeRangeInFreeLists()) {
8192 FreeChunk* ffc = (FreeChunk*) freeFinger();
8193 assert(ffc->size() == pointer_delta(addr, freeFinger()),
8194 "Size of free range is inconsistent with chunk size.");
8195 if (CMSTestInFreeList) {
8196 assert(_sp->verifyChunkInFreeLists(ffc),
8197 "free range is not in free lists");
8198 }
8199 _sp->removeFreeChunkFromFreeLists(ffc);
8200 set_freeRangeInFreeLists(false);
8201 }
8202 }
8203 }
8204 } else {
8205 // Code path common to both original and adaptive free lists.
8206
8207 // cant coalesce with previous block; this should be treated
8208 // as the end of a free run if any
8209 if (inFreeRange()) {
8210 // we kicked some butt; time to pick up the garbage
8211 assert(freeFinger() < addr, "the finger pointeth off base");
8212 flushCurFreeChunk(freeFinger(), pointer_delta(addr, freeFinger()));
8213 }
8214 // else, nothing to do, just continue
8215 }
8216 }
8217
8218 size_t SweepClosure::doGarbageChunk(FreeChunk* fc) {
8219 // This is a chunk of garbage. It is not in any free list.
8220 // Add it to a free list or let it possibly be coalesced into
8221 // a larger chunk.
8222 HeapWord* addr = (HeapWord*) fc;
8223 size_t size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
8224
8225 if (_sp->adaptive_freelists()) {
8226 // Verify that the bit map has no bits marked between
8227 // addr and purported end of just dead object.
8228 _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
8229
8230 doPostIsFreeOrGarbageChunk(fc, size);
8231 } else {
8232 if (!inFreeRange()) {
8233 // start of a new free range
8234 assert(size > 0, "A free range should have a size");
8235 initialize_free_range(addr, false);
8236
8237 } else {
8238 // this will be swept up when we hit the end of the
8239 // free range
8240 if (CMSTraceSweeper) {
8241 gclog_or_tty->print(" -- pick up garbage 0x%x (%d) \n", fc, size);
8242 }
8243 // If the chunk is being coalesced and the current free range is
8244 // in the free lists, remove the current free range so that it
8245 // will be returned to the free lists in its entirety - all
8246 // the coalesced pieces included.
8247 if (freeRangeInFreeLists()) {
8248 FreeChunk* ffc = (FreeChunk*)freeFinger();
8249 assert(ffc->size() == pointer_delta(addr, freeFinger()),
8250 "Size of free range is inconsistent with chunk size.");
8251 if (CMSTestInFreeList) {
8252 assert(_sp->verifyChunkInFreeLists(ffc),
8253 "free range is not in free lists");
8254 }
8255 _sp->removeFreeChunkFromFreeLists(ffc);
8256 set_freeRangeInFreeLists(false);
8257 }
8258 set_lastFreeRangeCoalesced(true);
8259 }
8260 // this will be swept up when we hit the end of the free range
8261
8262 // Verify that the bit map has no bits marked between
8263 // addr and purported end of just dead object.
8264 _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size);
8265 }
8266 return size;
8267 }
8268
8269 size_t SweepClosure::doLiveChunk(FreeChunk* fc) {
8270 HeapWord* addr = (HeapWord*) fc;
8271 // The sweeper has just found a live object. Return any accumulated
8272 // left hand chunk to the free lists.
8273 if (inFreeRange()) {
8274 if (_sp->adaptive_freelists()) {
8275 flushCurFreeChunk(freeFinger(),
8276 pointer_delta(addr, freeFinger()));
8277 } else { // not adaptive freelists
8278 set_inFreeRange(false);
8279 // Add the free range back to the free list if it is not already
8280 // there.
8281 if (!freeRangeInFreeLists()) {
8282 assert(freeFinger() < addr, "the finger pointeth off base");
8283 if (CMSTraceSweeper) {
8284 gclog_or_tty->print("Sweep:put_free_blk 0x%x (%d) "
8285 "[coalesced:%d]\n",
8286 freeFinger(), pointer_delta(addr, freeFinger()),
8287 lastFreeRangeCoalesced());
8288 }
8289 _sp->addChunkAndRepairOffsetTable(freeFinger(),
8290 pointer_delta(addr, freeFinger()), lastFreeRangeCoalesced());
8291 }
8292 }
8293 }
8294
8295 // Common code path for original and adaptive free lists.
8296
8297 // this object is live: we'd normally expect this to be
8298 // an oop, and like to assert the following:
8299 // assert(oop(addr)->is_oop(), "live block should be an oop");
8300 // However, as we commented above, this may be an object whose
8301 // header hasn't yet been initialized.
8302 size_t size;
8303 assert(_bitMap->isMarked(addr), "Tautology for this control point");
8304 if (_bitMap->isMarked(addr + 1)) {
8305 // Determine the size from the bit map, rather than trying to
8306 // compute it from the object header.
8307 HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2);
8308 size = pointer_delta(nextOneAddr + 1, addr);
8309 assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
8310 "alignment problem");
8311
8312 #ifdef DEBUG
8313 if (oop(addr)->klass_or_null() != NULL &&
8314 ( !_collector->should_unload_classes()
8315 || (oop(addr)->is_parsable()) &&
8316 oop(addr)->is_conc_safe())) {
8317 // Ignore mark word because we are running concurrent with mutators
8318 assert(oop(addr)->is_oop(true), "live block should be an oop");
8319 // is_conc_safe is checked before performing this assertion
8320 // because an object that is not is_conc_safe may yet have
8321 // the return from size() correct.
8322 assert(size ==
8323 CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()),
8324 "P-mark and computed size do not agree");
8325 }
8326 #endif
8327
8328 } else {
8329 // This should be an initialized object that's alive.
8330 assert(oop(addr)->klass_or_null() != NULL &&
8331 (!_collector->should_unload_classes()
8332 || oop(addr)->is_parsable()),
8333 "Should be an initialized object");
8334 // Note that there are objects used during class redefinition
8335 // (e.g., merge_cp in VM_RedefineClasses::merge_cp_and_rewrite()
8336 // which are discarded with their is_conc_safe state still
8337 // false. These object may be floating garbage so may be
8338 // seen here. If they are floating garbage their size
8339 // should be attainable from their klass. Do not that
8340 // is_conc_safe() is true for oop(addr).
8341 // Ignore mark word because we are running concurrent with mutators
8342 assert(oop(addr)->is_oop(true), "live block should be an oop");
8343 // Verify that the bit map has no bits marked between
8344 // addr and purported end of this block.
8345 size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size());
8346 assert(size >= 3, "Necessary for Printezis marks to work");
8347 assert(!_bitMap->isMarked(addr+1), "Tautology for this control point");
8348 DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);)
8349 }
8350 return size;
8351 }
8352
8353 void SweepClosure::doPostIsFreeOrGarbageChunk(FreeChunk* fc,
8354 size_t chunkSize) {
8355 // doPostIsFreeOrGarbageChunk() should only be called in the smart allocation
8356 // scheme.
8357 bool fcInFreeLists = fc->isFree();
8358 assert(_sp->adaptive_freelists(), "Should only be used in this case.");
8359 assert((HeapWord*)fc <= _limit, "sweep invariant");
8360 if (CMSTestInFreeList && fcInFreeLists) {
8361 assert(_sp->verifyChunkInFreeLists(fc),
8362 "free chunk is not in free lists");
8363 }
8364
8365
8366 if (CMSTraceSweeper) {
8367 gclog_or_tty->print_cr(" -- pick up another chunk at 0x%x (%d)", fc, chunkSize);
8368 }
8369
8370 HeapWord* addr = (HeapWord*) fc;
8371
8372 bool coalesce;
8373 size_t left = pointer_delta(addr, freeFinger());
8374 size_t right = chunkSize;
8375 switch (FLSCoalescePolicy) {
8376 // numeric value forms a coalition aggressiveness metric
8377 case 0: { // never coalesce
8378 coalesce = false;
8379 break;
8380 }
8381 case 1: { // coalesce if left & right chunks on overpopulated lists
8382 coalesce = _sp->coalOverPopulated(left) &&
8383 _sp->coalOverPopulated(right);
8384 break;
8385 }
8386 case 2: { // coalesce if left chunk on overpopulated list (default)
8387 coalesce = _sp->coalOverPopulated(left);
8388 break;
8389 }
8390 case 3: { // coalesce if left OR right chunk on overpopulated list
8391 coalesce = _sp->coalOverPopulated(left) ||
8392 _sp->coalOverPopulated(right);
8393 break;
8394 }
8395 case 4: { // always coalesce
8396 coalesce = true;
8397 break;
8398 }
8399 default:
8400 ShouldNotReachHere();
8401 }
8402
8403 // Should the current free range be coalesced?
8404 // If the chunk is in a free range and either we decided to coalesce above
8405 // or the chunk is near the large block at the end of the heap
8406 // (isNearLargestChunk() returns true), then coalesce this chunk.
8407 bool doCoalesce = inFreeRange() &&
8408 (coalesce || _g->isNearLargestChunk((HeapWord*)fc));
8409 if (doCoalesce) {
8410 // Coalesce the current free range on the left with the new
8411 // chunk on the right. If either is on a free list,
8412 // it must be removed from the list and stashed in the closure.
8413 if (freeRangeInFreeLists()) {
8414 FreeChunk* ffc = (FreeChunk*)freeFinger();
8415 assert(ffc->size() == pointer_delta(addr, freeFinger()),
8416 "Size of free range is inconsistent with chunk size.");
8417 if (CMSTestInFreeList) {
8418 assert(_sp->verifyChunkInFreeLists(ffc),
8419 "Chunk is not in free lists");
8420 }
8421 _sp->coalDeath(ffc->size());
8422 _sp->removeFreeChunkFromFreeLists(ffc);
8423 set_freeRangeInFreeLists(false);
8424 }
8425 if (fcInFreeLists) {
8426 _sp->coalDeath(chunkSize);
8427 assert(fc->size() == chunkSize,
8428 "The chunk has the wrong size or is not in the free lists");
8429 _sp->removeFreeChunkFromFreeLists(fc);
8430 }
8431 set_lastFreeRangeCoalesced(true);
8432 } else { // not in a free range and/or should not coalesce
8433 // Return the current free range and start a new one.
8434 if (inFreeRange()) {
8435 // In a free range but cannot coalesce with the right hand chunk.
8436 // Put the current free range into the free lists.
8437 flushCurFreeChunk(freeFinger(),
8438 pointer_delta(addr, freeFinger()));
8439 }
8440 // Set up for new free range. Pass along whether the right hand
8441 // chunk is in the free lists.
8442 initialize_free_range((HeapWord*)fc, fcInFreeLists);
8443 }
8444 }
8445 void SweepClosure::flushCurFreeChunk(HeapWord* chunk, size_t size) {
8446 assert(inFreeRange(), "Should only be called if currently in a free range.");
8447 assert(size > 0,
8448 "A zero sized chunk cannot be added to the free lists.");
8449 if (!freeRangeInFreeLists()) {
8450 if(CMSTestInFreeList) {
8451 FreeChunk* fc = (FreeChunk*) chunk;
8452 fc->setSize(size);
8453 assert(!_sp->verifyChunkInFreeLists(fc),
8454 "chunk should not be in free lists yet");
8455 }
8456 if (CMSTraceSweeper) {
8457 gclog_or_tty->print_cr(" -- add free block 0x%x (%d) to free lists",
8458 chunk, size);
8459 }
8460 // A new free range is going to be starting. The current
8461 // free range has not been added to the free lists yet or
8462 // was removed so add it back.
8463 // If the current free range was coalesced, then the death
8464 // of the free range was recorded. Record a birth now.
8465 if (lastFreeRangeCoalesced()) {
8466 _sp->coalBirth(size);
8467 }
8468 _sp->addChunkAndRepairOffsetTable(chunk, size,
8469 lastFreeRangeCoalesced());
8470 }
8471 set_inFreeRange(false);
8472 set_freeRangeInFreeLists(false);
8473 }
8474
8475 // We take a break if we've been at this for a while,
8476 // so as to avoid monopolizing the locks involved.
8477 void SweepClosure::do_yield_work(HeapWord* addr) {
8478 // Return current free chunk being used for coalescing (if any)
8479 // to the appropriate freelist. After yielding, the next
8480 // free block encountered will start a coalescing range of
8481 // free blocks. If the next free block is adjacent to the
8482 // chunk just flushed, they will need to wait for the next
8483 // sweep to be coalesced.
8484 if (inFreeRange()) {
8485 flushCurFreeChunk(freeFinger(), pointer_delta(addr, freeFinger()));
8486 }
8487
8488 // First give up the locks, then yield, then re-lock.
8489 // We should probably use a constructor/destructor idiom to
8490 // do this unlock/lock or modify the MutexUnlocker class to
8491 // serve our purpose. XXX
8492 assert_lock_strong(_bitMap->lock());
8493 assert_lock_strong(_freelistLock);
8494 assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),
8495 "CMS thread should hold CMS token");
8496 _bitMap->lock()->unlock();
8497 _freelistLock->unlock();
8498 ConcurrentMarkSweepThread::desynchronize(true);
8499 ConcurrentMarkSweepThread::acknowledge_yield_request();
8500 _collector->stopTimer();
8501 GCPauseTimer p(_collector->size_policy()->concurrent_timer_ptr());
8502 if (PrintCMSStatistics != 0) {
8503 _collector->incrementYields();
8504 }
8505 _collector->icms_wait();
8506
8507 // See the comment in coordinator_yield()
8508 for (unsigned i = 0; i < CMSYieldSleepCount &&
8509 ConcurrentMarkSweepThread::should_yield() &&
8510 !CMSCollector::foregroundGCIsActive(); ++i) {
8511 os::sleep(Thread::current(), 1, false);
8512 ConcurrentMarkSweepThread::acknowledge_yield_request();
8513 }
8514
8515 ConcurrentMarkSweepThread::synchronize(true);
8516 _freelistLock->lock();
8517 _bitMap->lock()->lock_without_safepoint_check();
8518 _collector->startTimer();
8519 }
8520
8521 #ifndef PRODUCT
8522 // This is actually very useful in a product build if it can
8523 // be called from the debugger. Compile it into the product
8524 // as needed.
8525 bool debug_verifyChunkInFreeLists(FreeChunk* fc) {
8526 return debug_cms_space->verifyChunkInFreeLists(fc);
8527 }
8528
8529 void SweepClosure::record_free_block_coalesced(FreeChunk* fc) const {
8530 if (CMSTraceSweeper) {
8531 gclog_or_tty->print("Sweep:coal_free_blk 0x%x (%d)\n", fc, fc->size());
8532 }
8533 }
8534 #endif
8535
8536 // CMSIsAliveClosure
8537 bool CMSIsAliveClosure::do_object_b(oop obj) {
8538 HeapWord* addr = (HeapWord*)obj;
8539 return addr != NULL &&
8540 (!_span.contains(addr) || _bit_map->isMarked(addr));
8541 }
8542
8543 CMSKeepAliveClosure::CMSKeepAliveClosure( CMSCollector* collector,
8544 MemRegion span,
8545 CMSBitMap* bit_map, CMSMarkStack* mark_stack,
8546 CMSMarkStack* revisit_stack, bool cpc):
8547 KlassRememberingOopClosure(collector, NULL, revisit_stack),
8548 _span(span),
8549 _bit_map(bit_map),
8550 _mark_stack(mark_stack),
8551 _concurrent_precleaning(cpc) {
8552 assert(!_span.is_empty(), "Empty span could spell trouble");
8553 }
8554
8555
8556 // CMSKeepAliveClosure: the serial version
8557 void CMSKeepAliveClosure::do_oop(oop obj) {
8558 HeapWord* addr = (HeapWord*)obj;
8559 if (_span.contains(addr) &&
8560 !_bit_map->isMarked(addr)) {
8561 _bit_map->mark(addr);
8562 bool simulate_overflow = false;
8563 NOT_PRODUCT(
8564 if (CMSMarkStackOverflowALot &&
8565 _collector->simulate_overflow()) {
8566 // simulate a stack overflow
8567 simulate_overflow = true;
8568 }
8569 )
8570 if (simulate_overflow || !_mark_stack->push(obj)) {
8571 if (_concurrent_precleaning) {
8572 // We dirty the overflown object and let the remark
8573 // phase deal with it.
8574 assert(_collector->overflow_list_is_empty(), "Error");
8575 // In the case of object arrays, we need to dirty all of
8576 // the cards that the object spans. No locking or atomics
8577 // are needed since no one else can be mutating the mod union
8578 // table.
8579 if (obj->is_objArray()) {
8580 size_t sz = obj->size();
8581 HeapWord* end_card_addr =
8582 (HeapWord*)round_to((intptr_t)(addr+sz), CardTableModRefBS::card_size);
8583 MemRegion redirty_range = MemRegion(addr, end_card_addr);
8584 assert(!redirty_range.is_empty(), "Arithmetical tautology");
8585 _collector->_modUnionTable.mark_range(redirty_range);
8586 } else {
8587 _collector->_modUnionTable.mark(addr);
8588 }
8589 _collector->_ser_kac_preclean_ovflw++;
8590 } else {
8591 _collector->push_on_overflow_list(obj);
8592 _collector->_ser_kac_ovflw++;
8593 }
8594 }
8595 }
8596 }
8597
8598 void CMSKeepAliveClosure::do_oop(oop* p) { CMSKeepAliveClosure::do_oop_work(p); }
8599 void CMSKeepAliveClosure::do_oop(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); }
8600
8601 // CMSParKeepAliveClosure: a parallel version of the above.
8602 // The work queues are private to each closure (thread),
8603 // but (may be) available for stealing by other threads.
8604 void CMSParKeepAliveClosure::do_oop(oop obj) {
8605 HeapWord* addr = (HeapWord*)obj;
8606 if (_span.contains(addr) &&
8607 !_bit_map->isMarked(addr)) {
8608 // In general, during recursive tracing, several threads
8609 // may be concurrently getting here; the first one to
8610 // "tag" it, claims it.
8611 if (_bit_map->par_mark(addr)) {
8612 bool res = _work_queue->push(obj);
8613 assert(res, "Low water mark should be much less than capacity");
8614 // Do a recursive trim in the hope that this will keep
8615 // stack usage lower, but leave some oops for potential stealers
8616 trim_queue(_low_water_mark);
8617 } // Else, another thread got there first
8618 }
8619 }
8620
8621 void CMSParKeepAliveClosure::do_oop(oop* p) { CMSParKeepAliveClosure::do_oop_work(p); }
8622 void CMSParKeepAliveClosure::do_oop(narrowOop* p) { CMSParKeepAliveClosure::do_oop_work(p); }
8623
8624 void CMSParKeepAliveClosure::trim_queue(uint max) {
8625 while (_work_queue->size() > max) {
8626 oop new_oop;
8627 if (_work_queue->pop_local(new_oop)) {
8628 assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop");
8629 assert(_bit_map->isMarked((HeapWord*)new_oop),
8630 "no white objects on this stack!");
8631 assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop");
8632 // iterate over the oops in this oop, marking and pushing
8633 // the ones in CMS heap (i.e. in _span).
8634 new_oop->oop_iterate(&_mark_and_push);
8635 }
8636 }
8637 }
8638
8639 CMSInnerParMarkAndPushClosure::CMSInnerParMarkAndPushClosure(
8640 CMSCollector* collector,
8641 MemRegion span, CMSBitMap* bit_map,
8642 CMSMarkStack* revisit_stack,
8643 OopTaskQueue* work_queue):
8644 Par_KlassRememberingOopClosure(collector, NULL, revisit_stack),
8645 _span(span),
8646 _bit_map(bit_map),
8647 _work_queue(work_queue) { }
8648
8649 void CMSInnerParMarkAndPushClosure::do_oop(oop obj) {
8650 HeapWord* addr = (HeapWord*)obj;
8651 if (_span.contains(addr) &&
8652 !_bit_map->isMarked(addr)) {
8653 if (_bit_map->par_mark(addr)) {
8654 bool simulate_overflow = false;
8655 NOT_PRODUCT(
8656 if (CMSMarkStackOverflowALot &&
8657 _collector->par_simulate_overflow()) {
8658 // simulate a stack overflow
8659 simulate_overflow = true;
8660 }
8661 )
8662 if (simulate_overflow || !_work_queue->push(obj)) {
8663 _collector->par_push_on_overflow_list(obj);
8664 _collector->_par_kac_ovflw++;
8665 }
8666 } // Else another thread got there already
8667 }
8668 }
8669
8670 void CMSInnerParMarkAndPushClosure::do_oop(oop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); }
8671 void CMSInnerParMarkAndPushClosure::do_oop(narrowOop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); }
8672
8673 //////////////////////////////////////////////////////////////////
8674 // CMSExpansionCause /////////////////////////////
8675 //////////////////////////////////////////////////////////////////
8676 const char* CMSExpansionCause::to_string(CMSExpansionCause::Cause cause) {
8677 switch (cause) {
8678 case _no_expansion:
8679 return "No expansion";
8680 case _satisfy_free_ratio:
8681 return "Free ratio";
8682 case _satisfy_promotion:
8683 return "Satisfy promotion";
8684 case _satisfy_allocation:
8685 return "allocation";
8686 case _allocate_par_lab:
8687 return "Par LAB";
8688 case _allocate_par_spooling_space:
8689 return "Par Spooling Space";
8690 case _adaptive_size_policy:
8691 return "Ergonomics";
8692 default:
8693 return "unknown";
8694 }
8695 }
8696
8697 void CMSDrainMarkingStackClosure::do_void() {
8698 // the max number to take from overflow list at a time
8699 const size_t num = _mark_stack->capacity()/4;
8700 assert(!_concurrent_precleaning || _collector->overflow_list_is_empty(),
8701 "Overflow list should be NULL during concurrent phases");
8702 while (!_mark_stack->isEmpty() ||
8703 // if stack is empty, check the overflow list
8704 _collector->take_from_overflow_list(num, _mark_stack)) {
8705 oop obj = _mark_stack->pop();
8706 HeapWord* addr = (HeapWord*)obj;
8707 assert(_span.contains(addr), "Should be within span");
8708 assert(_bit_map->isMarked(addr), "Should be marked");
8709 assert(obj->is_oop(), "Should be an oop");
8710 obj->oop_iterate(_keep_alive);
8711 }
8712 }
8713
8714 void CMSParDrainMarkingStackClosure::do_void() {
8715 // drain queue
8716 trim_queue(0);
8717 }
8718
8719 // Trim our work_queue so its length is below max at return
8720 void CMSParDrainMarkingStackClosure::trim_queue(uint max) {
8721 while (_work_queue->size() > max) {
8722 oop new_oop;
8723 if (_work_queue->pop_local(new_oop)) {
8724 assert(new_oop->is_oop(), "Expected an oop");
8725 assert(_bit_map->isMarked((HeapWord*)new_oop),
8726 "no white objects on this stack!");
8727 assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop");
8728 // iterate over the oops in this oop, marking and pushing
8729 // the ones in CMS heap (i.e. in _span).
8730 new_oop->oop_iterate(&_mark_and_push);
8731 }
8732 }
8733 }
8734
8735 ////////////////////////////////////////////////////////////////////
8736 // Support for Marking Stack Overflow list handling and related code
8737 ////////////////////////////////////////////////////////////////////
8738 // Much of the following code is similar in shape and spirit to the
8739 // code used in ParNewGC. We should try and share that code
8740 // as much as possible in the future.
8741
8742 #ifndef PRODUCT
8743 // Debugging support for CMSStackOverflowALot
8744
8745 // It's OK to call this multi-threaded; the worst thing
8746 // that can happen is that we'll get a bunch of closely
8747 // spaced simulated oveflows, but that's OK, in fact
8748 // probably good as it would exercise the overflow code
8749 // under contention.
8750 bool CMSCollector::simulate_overflow() {
8751 if (_overflow_counter-- <= 0) { // just being defensive
8752 _overflow_counter = CMSMarkStackOverflowInterval;
8753 return true;
8754 } else {
8755 return false;
8756 }
8757 }
8758
8759 bool CMSCollector::par_simulate_overflow() {
8760 return simulate_overflow();
8761 }
8762 #endif
8763
8764 // Single-threaded
8765 bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) {
8766 assert(stack->isEmpty(), "Expected precondition");
8767 assert(stack->capacity() > num, "Shouldn't bite more than can chew");
8768 size_t i = num;
8769 oop cur = _overflow_list;
8770 const markOop proto = markOopDesc::prototype();
8771 NOT_PRODUCT(ssize_t n = 0;)
8772 for (oop next; i > 0 && cur != NULL; cur = next, i--) {
8773 next = oop(cur->mark());
8774 cur->set_mark(proto); // until proven otherwise
8775 assert(cur->is_oop(), "Should be an oop");
8776 bool res = stack->push(cur);
8777 assert(res, "Bit off more than can chew?");
8778 NOT_PRODUCT(n++;)
8779 }
8780 _overflow_list = cur;
8781 #ifndef PRODUCT
8782 assert(_num_par_pushes >= n, "Too many pops?");
8783 _num_par_pushes -=n;
8784 #endif
8785 return !stack->isEmpty();
8786 }
8787
8788 #define BUSY (oop(0x1aff1aff))
8789 // (MT-safe) Get a prefix of at most "num" from the list.
8790 // The overflow list is chained through the mark word of
8791 // each object in the list. We fetch the entire list,
8792 // break off a prefix of the right size and return the
8793 // remainder. If other threads try to take objects from
8794 // the overflow list at that time, they will wait for
8795 // some time to see if data becomes available. If (and
8796 // only if) another thread places one or more object(s)
8797 // on the global list before we have returned the suffix
8798 // to the global list, we will walk down our local list
8799 // to find its end and append the global list to
8800 // our suffix before returning it. This suffix walk can
8801 // prove to be expensive (quadratic in the amount of traffic)
8802 // when there are many objects in the overflow list and
8803 // there is much producer-consumer contention on the list.
8804 // *NOTE*: The overflow list manipulation code here and
8805 // in ParNewGeneration:: are very similar in shape,
8806 // except that in the ParNew case we use the old (from/eden)
8807 // copy of the object to thread the list via its klass word.
8808 // Because of the common code, if you make any changes in
8809 // the code below, please check the ParNew version to see if
8810 // similar changes might be needed.
8811 // CR 6797058 has been filed to consolidate the common code.
8812 bool CMSCollector::par_take_from_overflow_list(size_t num,
8813 OopTaskQueue* work_q,
8814 int no_of_gc_threads) {
8815 assert(work_q->size() == 0, "First empty local work queue");
8816 assert(num < work_q->max_elems(), "Can't bite more than we can chew");
8817 if (_overflow_list == NULL) {
8818 return false;
8819 }
8820 // Grab the entire list; we'll put back a suffix
8821 oop prefix = (oop)Atomic::xchg_ptr(BUSY, &_overflow_list);
8822 Thread* tid = Thread::current();
8823 // Before "no_of_gc_threads" was introduced CMSOverflowSpinCount was
8824 // set to ParallelGCThreads.
8825 size_t CMSOverflowSpinCount = (size_t) no_of_gc_threads; // was ParallelGCThreads;
8826 size_t sleep_time_millis = MAX2((size_t)1, num/100);
8827 // If the list is busy, we spin for a short while,
8828 // sleeping between attempts to get the list.
8829 for (size_t spin = 0; prefix == BUSY && spin < CMSOverflowSpinCount; spin++) {
8830 os::sleep(tid, sleep_time_millis, false);
8831 if (_overflow_list == NULL) {
8832 // Nothing left to take
8833 return false;
8834 } else if (_overflow_list != BUSY) {
8835 // Try and grab the prefix
8836 prefix = (oop)Atomic::xchg_ptr(BUSY, &_overflow_list);
8837 }
8838 }
8839 // If the list was found to be empty, or we spun long
8840 // enough, we give up and return empty-handed. If we leave
8841 // the list in the BUSY state below, it must be the case that
8842 // some other thread holds the overflow list and will set it
8843 // to a non-BUSY state in the future.
8844 if (prefix == NULL || prefix == BUSY) {
8845 // Nothing to take or waited long enough
8846 if (prefix == NULL) {
8847 // Write back the NULL in case we overwrote it with BUSY above
8848 // and it is still the same value.
8849 (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY);
8850 }
8851 return false;
8852 }
8853 assert(prefix != NULL && prefix != BUSY, "Error");
8854 size_t i = num;
8855 oop cur = prefix;
8856 // Walk down the first "num" objects, unless we reach the end.
8857 for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--);
8858 if (cur->mark() == NULL) {
8859 // We have "num" or fewer elements in the list, so there
8860 // is nothing to return to the global list.
8861 // Write back the NULL in lieu of the BUSY we wrote
8862 // above, if it is still the same value.
8863 if (_overflow_list == BUSY) {
8864 (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY);
8865 }
8866 } else {
8867 // Chop off the suffix and rerturn it to the global list.
8868 assert(cur->mark() != BUSY, "Error");
8869 oop suffix_head = cur->mark(); // suffix will be put back on global list
8870 cur->set_mark(NULL); // break off suffix
8871 // It's possible that the list is still in the empty(busy) state
8872 // we left it in a short while ago; in that case we may be
8873 // able to place back the suffix without incurring the cost
8874 // of a walk down the list.
8875 oop observed_overflow_list = _overflow_list;
8876 oop cur_overflow_list = observed_overflow_list;
8877 bool attached = false;
8878 while (observed_overflow_list == BUSY || observed_overflow_list == NULL) {
8879 observed_overflow_list =
8880 (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur_overflow_list);
8881 if (cur_overflow_list == observed_overflow_list) {
8882 attached = true;
8883 break;
8884 } else cur_overflow_list = observed_overflow_list;
8885 }
8886 if (!attached) {
8887 // Too bad, someone else sneaked in (at least) an element; we'll need
8888 // to do a splice. Find tail of suffix so we can prepend suffix to global
8889 // list.
8890 for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark()));
8891 oop suffix_tail = cur;
8892 assert(suffix_tail != NULL && suffix_tail->mark() == NULL,
8893 "Tautology");
8894 observed_overflow_list = _overflow_list;
8895 do {
8896 cur_overflow_list = observed_overflow_list;
8897 if (cur_overflow_list != BUSY) {
8898 // Do the splice ...
8899 suffix_tail->set_mark(markOop(cur_overflow_list));
8900 } else { // cur_overflow_list == BUSY
8901 suffix_tail->set_mark(NULL);
8902 }
8903 // ... and try to place spliced list back on overflow_list ...
8904 observed_overflow_list =
8905 (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur_overflow_list);
8906 } while (cur_overflow_list != observed_overflow_list);
8907 // ... until we have succeeded in doing so.
8908 }
8909 }
8910
8911 // Push the prefix elements on work_q
8912 assert(prefix != NULL, "control point invariant");
8913 const markOop proto = markOopDesc::prototype();
8914 oop next;
8915 NOT_PRODUCT(ssize_t n = 0;)
8916 for (cur = prefix; cur != NULL; cur = next) {
8917 next = oop(cur->mark());
8918 cur->set_mark(proto); // until proven otherwise
8919 assert(cur->is_oop(), "Should be an oop");
8920 bool res = work_q->push(cur);
8921 assert(res, "Bit off more than we can chew?");
8922 NOT_PRODUCT(n++;)
8923 }
8924 #ifndef PRODUCT
8925 assert(_num_par_pushes >= n, "Too many pops?");
8926 Atomic::add_ptr(-(intptr_t)n, &_num_par_pushes);
8927 #endif
8928 return true;
8929 }
8930
8931 // Single-threaded
8932 void CMSCollector::push_on_overflow_list(oop p) {
8933 NOT_PRODUCT(_num_par_pushes++;)
8934 assert(p->is_oop(), "Not an oop");
8935 preserve_mark_if_necessary(p);
8936 p->set_mark((markOop)_overflow_list);
8937 _overflow_list = p;
8938 }
8939
8940 // Multi-threaded; use CAS to prepend to overflow list
8941 void CMSCollector::par_push_on_overflow_list(oop p) {
8942 NOT_PRODUCT(Atomic::inc_ptr(&_num_par_pushes);)
8943 assert(p->is_oop(), "Not an oop");
8944 par_preserve_mark_if_necessary(p);
8945 oop observed_overflow_list = _overflow_list;
8946 oop cur_overflow_list;
8947 do {
8948 cur_overflow_list = observed_overflow_list;
8949 if (cur_overflow_list != BUSY) {
8950 p->set_mark(markOop(cur_overflow_list));
8951 } else {
8952 p->set_mark(NULL);
8953 }
8954 observed_overflow_list =
8955 (oop) Atomic::cmpxchg_ptr(p, &_overflow_list, cur_overflow_list);
8956 } while (cur_overflow_list != observed_overflow_list);
8957 }
8958 #undef BUSY
8959
8960 // Single threaded
8961 // General Note on GrowableArray: pushes may silently fail
8962 // because we are (temporarily) out of C-heap for expanding
8963 // the stack. The problem is quite ubiquitous and affects
8964 // a lot of code in the JVM. The prudent thing for GrowableArray
8965 // to do (for now) is to exit with an error. However, that may
8966 // be too draconian in some cases because the caller may be
8967 // able to recover without much harm. For such cases, we
8968 // should probably introduce a "soft_push" method which returns
8969 // an indication of success or failure with the assumption that
8970 // the caller may be able to recover from a failure; code in
8971 // the VM can then be changed, incrementally, to deal with such
8972 // failures where possible, thus, incrementally hardening the VM
8973 // in such low resource situations.
8974 void CMSCollector::preserve_mark_work(oop p, markOop m) {
8975 _preserved_oop_stack.push(p);
8976 _preserved_mark_stack.push(m);
8977 assert(m == p->mark(), "Mark word changed");
8978 assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
8979 "bijection");
8980 }
8981
8982 // Single threaded
8983 void CMSCollector::preserve_mark_if_necessary(oop p) {
8984 markOop m = p->mark();
8985 if (m->must_be_preserved(p)) {
8986 preserve_mark_work(p, m);
8987 }
8988 }
8989
8990 void CMSCollector::par_preserve_mark_if_necessary(oop p) {
8991 markOop m = p->mark();
8992 if (m->must_be_preserved(p)) {
8993 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
8994 // Even though we read the mark word without holding
8995 // the lock, we are assured that it will not change
8996 // because we "own" this oop, so no other thread can
8997 // be trying to push it on the overflow list; see
8998 // the assertion in preserve_mark_work() that checks
8999 // that m == p->mark().
9000 preserve_mark_work(p, m);
9001 }
9002 }
9003
9004 // We should be able to do this multi-threaded,
9005 // a chunk of stack being a task (this is
9006 // correct because each oop only ever appears
9007 // once in the overflow list. However, it's
9008 // not very easy to completely overlap this with
9009 // other operations, so will generally not be done
9010 // until all work's been completed. Because we
9011 // expect the preserved oop stack (set) to be small,
9012 // it's probably fine to do this single-threaded.
9013 // We can explore cleverer concurrent/overlapped/parallel
9014 // processing of preserved marks if we feel the
9015 // need for this in the future. Stack overflow should
9016 // be so rare in practice and, when it happens, its
9017 // effect on performance so great that this will
9018 // likely just be in the noise anyway.
9019 void CMSCollector::restore_preserved_marks_if_any() {
9020 assert(SafepointSynchronize::is_at_safepoint(),
9021 "world should be stopped");
9022 assert(Thread::current()->is_ConcurrentGC_thread() ||
9023 Thread::current()->is_VM_thread(),
9024 "should be single-threaded");
9025 assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
9026 "bijection");
9027
9028 while (!_preserved_oop_stack.is_empty()) {
9029 oop p = _preserved_oop_stack.pop();
9030 assert(p->is_oop(), "Should be an oop");
9031 assert(_span.contains(p), "oop should be in _span");
9032 assert(p->mark() == markOopDesc::prototype(),
9033 "Set when taken from overflow list");
9034 markOop m = _preserved_mark_stack.pop();
9035 p->set_mark(m);
9036 }
9037 assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(),
9038 "stacks were cleared above");
9039 }
9040
9041 #ifndef PRODUCT
9042 bool CMSCollector::no_preserved_marks() const {
9043 return _preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty();
9044 }
9045 #endif
9046
9047 CMSAdaptiveSizePolicy* ASConcurrentMarkSweepGeneration::cms_size_policy() const
9048 {
9049 GenCollectedHeap* gch = (GenCollectedHeap*) GenCollectedHeap::heap();
9050 CMSAdaptiveSizePolicy* size_policy =
9051 (CMSAdaptiveSizePolicy*) gch->gen_policy()->size_policy();
9052 assert(size_policy->is_gc_cms_adaptive_size_policy(),
9053 "Wrong type for size policy");
9054 return size_policy;
9055 }
9056
9057 void ASConcurrentMarkSweepGeneration::resize(size_t cur_promo_size,
9058 size_t desired_promo_size) {
9059 if (cur_promo_size < desired_promo_size) {
9060 size_t expand_bytes = desired_promo_size - cur_promo_size;
9061 if (PrintAdaptiveSizePolicy && Verbose) {
9062 gclog_or_tty->print_cr(" ASConcurrentMarkSweepGeneration::resize "
9063 "Expanding tenured generation by " SIZE_FORMAT " (bytes)",
9064 expand_bytes);
9065 }
9066 expand(expand_bytes,
9067 MinHeapDeltaBytes,
9068 CMSExpansionCause::_adaptive_size_policy);
9069 } else if (desired_promo_size < cur_promo_size) {
9070 size_t shrink_bytes = cur_promo_size - desired_promo_size;
9071 if (PrintAdaptiveSizePolicy && Verbose) {
9072 gclog_or_tty->print_cr(" ASConcurrentMarkSweepGeneration::resize "
9073 "Shrinking tenured generation by " SIZE_FORMAT " (bytes)",
9074 shrink_bytes);
9075 }
9076 shrink(shrink_bytes);
9077 }
9078 }
9079
9080 CMSGCAdaptivePolicyCounters* ASConcurrentMarkSweepGeneration::gc_adaptive_policy_counters() {
9081 GenCollectedHeap* gch = GenCollectedHeap::heap();
9082 CMSGCAdaptivePolicyCounters* counters =
9083 (CMSGCAdaptivePolicyCounters*) gch->collector_policy()->counters();
9084 assert(counters->kind() == GCPolicyCounters::CMSGCAdaptivePolicyCountersKind,
9085 "Wrong kind of counters");
9086 return counters;
9087 }
9088
9089
9090 void ASConcurrentMarkSweepGeneration::update_counters() {
9091 if (UsePerfData) {
9092 _space_counters->update_all();
9093 _gen_counters->update_all();
9094 CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
9095 GenCollectedHeap* gch = GenCollectedHeap::heap();
9096 CMSGCStats* gc_stats_l = (CMSGCStats*) gc_stats();
9097 assert(gc_stats_l->kind() == GCStats::CMSGCStatsKind,
9098 "Wrong gc statistics type");
9099 counters->update_counters(gc_stats_l);
9100 }
9101 }
9102
9103 void ASConcurrentMarkSweepGeneration::update_counters(size_t used) {
9104 if (UsePerfData) {
9105 _space_counters->update_used(used);
9106 _space_counters->update_capacity();
9107 _gen_counters->update_all();
9108
9109 CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
9110 GenCollectedHeap* gch = GenCollectedHeap::heap();
9111 CMSGCStats* gc_stats_l = (CMSGCStats*) gc_stats();
9112 assert(gc_stats_l->kind() == GCStats::CMSGCStatsKind,
9113 "Wrong gc statistics type");
9114 counters->update_counters(gc_stats_l);
9115 }
9116 }
9117
9118 // The desired expansion delta is computed so that:
9119 // . desired free percentage or greater is used
9120 void ASConcurrentMarkSweepGeneration::compute_new_size() {
9121 assert_locked_or_safepoint(Heap_lock);
9122
9123 GenCollectedHeap* gch = (GenCollectedHeap*) GenCollectedHeap::heap();
9124
9125 // If incremental collection failed, we just want to expand
9126 // to the limit.
9127 if (incremental_collection_failed()) {
9128 clear_incremental_collection_failed();
9129 grow_to_reserved();
9130 return;
9131 }
9132
9133 assert(UseAdaptiveSizePolicy, "Should be using adaptive sizing");
9134
9135 assert(gch->kind() == CollectedHeap::GenCollectedHeap,
9136 "Wrong type of heap");
9137 int prev_level = level() - 1;
9138 assert(prev_level >= 0, "The cms generation is the lowest generation");
9139 Generation* prev_gen = gch->get_gen(prev_level);
9140 assert(prev_gen->kind() == Generation::ASParNew,
9141 "Wrong type of young generation");
9142 ParNewGeneration* younger_gen = (ParNewGeneration*) prev_gen;
9143 size_t cur_eden = younger_gen->eden()->capacity();
9144 CMSAdaptiveSizePolicy* size_policy = cms_size_policy();
9145 size_t cur_promo = free();
9146 size_policy->compute_tenured_generation_free_space(cur_promo,
9147 max_available(),
9148 cur_eden);
9149 resize(cur_promo, size_policy->promo_size());
9150
9151 // Record the new size of the space in the cms generation
9152 // that is available for promotions. This is temporary.
9153 // It should be the desired promo size.
9154 size_policy->avg_cms_promo()->sample(free());
9155 size_policy->avg_old_live()->sample(used());
9156
9157 if (UsePerfData) {
9158 CMSGCAdaptivePolicyCounters* counters = gc_adaptive_policy_counters();
9159 counters->update_cms_capacity_counter(capacity());
9160 }
9161 }
9162
9163 void ASConcurrentMarkSweepGeneration::shrink_by(size_t desired_bytes) {
9164 assert_locked_or_safepoint(Heap_lock);
9165 assert_lock_strong(freelistLock());
9166 HeapWord* old_end = _cmsSpace->end();
9167 HeapWord* unallocated_start = _cmsSpace->unallocated_block();
9168 assert(old_end >= unallocated_start, "Miscalculation of unallocated_start");
9169 FreeChunk* chunk_at_end = find_chunk_at_end();
9170 if (chunk_at_end == NULL) {
9171 // No room to shrink
9172 if (PrintGCDetails && Verbose) {
9173 gclog_or_tty->print_cr("No room to shrink: old_end "
9174 PTR_FORMAT " unallocated_start " PTR_FORMAT
9175 " chunk_at_end " PTR_FORMAT,
9176 old_end, unallocated_start, chunk_at_end);
9177 }
9178 return;
9179 } else {
9180
9181 // Find the chunk at the end of the space and determine
9182 // how much it can be shrunk.
9183 size_t shrinkable_size_in_bytes = chunk_at_end->size();
9184 size_t aligned_shrinkable_size_in_bytes =
9185 align_size_down(shrinkable_size_in_bytes, os::vm_page_size());
9186 assert(unallocated_start <= chunk_at_end->end(),
9187 "Inconsistent chunk at end of space");
9188 size_t bytes = MIN2(desired_bytes, aligned_shrinkable_size_in_bytes);
9189 size_t word_size_before = heap_word_size(_virtual_space.committed_size());
9190
9191 // Shrink the underlying space
9192 _virtual_space.shrink_by(bytes);
9193 if (PrintGCDetails && Verbose) {
9194 gclog_or_tty->print_cr("ConcurrentMarkSweepGeneration::shrink_by:"
9195 " desired_bytes " SIZE_FORMAT
9196 " shrinkable_size_in_bytes " SIZE_FORMAT
9197 " aligned_shrinkable_size_in_bytes " SIZE_FORMAT
9198 " bytes " SIZE_FORMAT,
9199 desired_bytes, shrinkable_size_in_bytes,
9200 aligned_shrinkable_size_in_bytes, bytes);
9201 gclog_or_tty->print_cr(" old_end " SIZE_FORMAT
9202 " unallocated_start " SIZE_FORMAT,
9203 old_end, unallocated_start);
9204 }
9205
9206 // If the space did shrink (shrinking is not guaranteed),
9207 // shrink the chunk at the end by the appropriate amount.
9208 if (((HeapWord*)_virtual_space.high()) < old_end) {
9209 size_t new_word_size =
9210 heap_word_size(_virtual_space.committed_size());
9211
9212 // Have to remove the chunk from the dictionary because it is changing
9213 // size and might be someplace elsewhere in the dictionary.
9214
9215 // Get the chunk at end, shrink it, and put it
9216 // back.
9217 _cmsSpace->removeChunkFromDictionary(chunk_at_end);
9218 size_t word_size_change = word_size_before - new_word_size;
9219 size_t chunk_at_end_old_size = chunk_at_end->size();
9220 assert(chunk_at_end_old_size >= word_size_change,
9221 "Shrink is too large");
9222 chunk_at_end->setSize(chunk_at_end_old_size -
9223 word_size_change);
9224 _cmsSpace->freed((HeapWord*) chunk_at_end->end(),
9225 word_size_change);
9226
9227 _cmsSpace->returnChunkToDictionary(chunk_at_end);
9228
9229 MemRegion mr(_cmsSpace->bottom(), new_word_size);
9230 _bts->resize(new_word_size); // resize the block offset shared array
9231 Universe::heap()->barrier_set()->resize_covered_region(mr);
9232 _cmsSpace->assert_locked();
9233 _cmsSpace->set_end((HeapWord*)_virtual_space.high());
9234
9235 NOT_PRODUCT(_cmsSpace->dictionary()->verify());
9236
9237 // update the space and generation capacity counters
9238 if (UsePerfData) {
9239 _space_counters->update_capacity();
9240 _gen_counters->update_all();
9241 }
9242
9243 if (Verbose && PrintGCDetails) {
9244 size_t new_mem_size = _virtual_space.committed_size();
9245 size_t old_mem_size = new_mem_size + bytes;
9246 gclog_or_tty->print_cr("Shrinking %s from %ldK by %ldK to %ldK",
9247 name(), old_mem_size/K, bytes/K, new_mem_size/K);
9248 }
9249 }
9250
9251 assert(_cmsSpace->unallocated_block() <= _cmsSpace->end(),
9252 "Inconsistency at end of space");
9253 assert(chunk_at_end->end() == _cmsSpace->end(),
9254 "Shrinking is inconsistent");
9255 return;
9256 }
9257 }
9258
9259 // Transfer some number of overflown objects to usual marking
9260 // stack. Return true if some objects were transferred.
9261 bool MarkRefsIntoAndScanClosure::take_from_overflow_list() {
9262 size_t num = MIN2((size_t)(_mark_stack->capacity() - _mark_stack->length())/4,
9263 (size_t)ParGCDesiredObjsFromOverflowList);
9264
9265 bool res = _collector->take_from_overflow_list(num, _mark_stack);
9266 assert(_collector->overflow_list_is_empty() || res,
9267 "If list is not empty, we should have taken something");
9268 assert(!res || !_mark_stack->isEmpty(),
9269 "If we took something, it should now be on our stack");
9270 return res;
9271 }
9272
9273 size_t MarkDeadObjectsClosure::do_blk(HeapWord* addr) {
9274 size_t res = _sp->block_size_no_stall(addr, _collector);
9275 assert(res != 0, "Should always be able to compute a size");
9276 if (_sp->block_is_obj(addr)) {
9277 if (_live_bit_map->isMarked(addr)) {
9278 // It can't have been dead in a previous cycle
9279 guarantee(!_dead_bit_map->isMarked(addr), "No resurrection!");
9280 } else {
9281 _dead_bit_map->mark(addr); // mark the dead object
9282 }
9283 }
9284 return res;
9285 }
9286
9287 TraceCMSMemoryManagerStats::TraceCMSMemoryManagerStats(CMSCollector::CollectorState phase): TraceMemoryManagerStats() {
9288
9289 switch (phase) {
9290 case CMSCollector::InitialMarking:
9291 initialize(true /* fullGC */ ,
9292 true /* recordGCBeginTime */,
9293 true /* recordPreGCUsage */,
9294 false /* recordPeakUsage */,
9295 false /* recordPostGCusage */,
9296 true /* recordAccumulatedGCTime */,
9297 false /* recordGCEndTime */,
9298 false /* countCollection */ );
9299 break;
9300
9301 case CMSCollector::FinalMarking:
9302 initialize(true /* fullGC */ ,
9303 false /* recordGCBeginTime */,
9304 false /* recordPreGCUsage */,
9305 false /* recordPeakUsage */,
9306 false /* recordPostGCusage */,
9307 true /* recordAccumulatedGCTime */,
9308 false /* recordGCEndTime */,
9309 false /* countCollection */ );
9310 break;
9311
9312 case CMSCollector::Sweeping:
9313 initialize(true /* fullGC */ ,
9314 false /* recordGCBeginTime */,
9315 false /* recordPreGCUsage */,
9316 true /* recordPeakUsage */,
9317 true /* recordPostGCusage */,
9318 false /* recordAccumulatedGCTime */,
9319 true /* recordGCEndTime */,
9320 true /* countCollection */ );
9321 break;
9322
9323 default:
9324 ShouldNotReachHere();
9325 }
9326 }
9327
9328 // when bailing out of cms in concurrent mode failure
9329 TraceCMSMemoryManagerStats::TraceCMSMemoryManagerStats(): TraceMemoryManagerStats() {
9330 initialize(true /* fullGC */ ,
9331 true /* recordGCBeginTime */,
9332 true /* recordPreGCUsage */,
9333 true /* recordPeakUsage */,
9334 true /* recordPostGCusage */,
9335 true /* recordAccumulatedGCTime */,
9336 true /* recordGCEndTime */,
9337 true /* countCollection */ );
9338 }