1 /*
2 * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP
26 #define SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP
27
28 #include "gc/cms/cmsLockVerifier.hpp"
29 #include "gc/cms/compactibleFreeListSpace.hpp"
30 #include "gc/cms/concurrentMarkSweepGeneration.hpp"
31 #include "gc/cms/concurrentMarkSweepThread.hpp"
32 #include "gc/cms/parNewGeneration.hpp"
33 #include "gc/shared/gcUtil.hpp"
34 #include "gc/shared/genCollectedHeap.hpp"
35 #include "utilities/align.hpp"
36 #include "utilities/bitMap.inline.hpp"
37
38 inline void CMSBitMap::clear_all() {
39 assert_locked();
40 // CMS bitmaps are usually cover large memory regions
41 _bm.clear_large();
42 return;
43 }
44
45 inline size_t CMSBitMap::heapWordToOffset(HeapWord* addr) const {
46 return (pointer_delta(addr, _bmStartWord)) >> _shifter;
47 }
48
49 inline HeapWord* CMSBitMap::offsetToHeapWord(size_t offset) const {
50 return _bmStartWord + (offset << _shifter);
51 }
52
53 inline size_t CMSBitMap::heapWordDiffToOffsetDiff(size_t diff) const {
54 assert((diff & ((1 << _shifter) - 1)) == 0, "argument check");
55 return diff >> _shifter;
56 }
57
58 inline void CMSBitMap::mark(HeapWord* addr) {
59 assert_locked();
60 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
61 "outside underlying space?");
62 _bm.set_bit(heapWordToOffset(addr));
63 }
64
65 inline bool CMSBitMap::par_mark(HeapWord* addr) {
66 assert_locked();
67 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
68 "outside underlying space?");
69 return _bm.par_at_put(heapWordToOffset(addr), true);
70 }
71
72 inline void CMSBitMap::par_clear(HeapWord* addr) {
73 assert_locked();
74 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
75 "outside underlying space?");
76 _bm.par_at_put(heapWordToOffset(addr), false);
77 }
78
79 inline void CMSBitMap::mark_range(MemRegion mr) {
80 NOT_PRODUCT(region_invariant(mr));
81 // Range size is usually just 1 bit.
82 _bm.set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
83 BitMap::small_range);
84 }
85
86 inline void CMSBitMap::clear_range(MemRegion mr) {
87 NOT_PRODUCT(region_invariant(mr));
88 // Range size is usually just 1 bit.
89 _bm.clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
90 BitMap::small_range);
91 }
92
93 inline void CMSBitMap::par_mark_range(MemRegion mr) {
94 NOT_PRODUCT(region_invariant(mr));
95 // Range size is usually just 1 bit.
96 _bm.par_set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
97 BitMap::small_range);
98 }
99
100 inline void CMSBitMap::par_clear_range(MemRegion mr) {
101 NOT_PRODUCT(region_invariant(mr));
102 // Range size is usually just 1 bit.
103 _bm.par_clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
104 BitMap::small_range);
105 }
106
107 inline void CMSBitMap::mark_large_range(MemRegion mr) {
108 NOT_PRODUCT(region_invariant(mr));
109 // Range size must be greater than 32 bytes.
110 _bm.set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
111 BitMap::large_range);
112 }
113
114 inline void CMSBitMap::clear_large_range(MemRegion mr) {
115 NOT_PRODUCT(region_invariant(mr));
116 // Range size must be greater than 32 bytes.
117 _bm.clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
118 BitMap::large_range);
119 }
120
121 inline void CMSBitMap::par_mark_large_range(MemRegion mr) {
122 NOT_PRODUCT(region_invariant(mr));
123 // Range size must be greater than 32 bytes.
124 _bm.par_set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
125 BitMap::large_range);
126 }
127
128 inline void CMSBitMap::par_clear_large_range(MemRegion mr) {
129 NOT_PRODUCT(region_invariant(mr));
130 // Range size must be greater than 32 bytes.
131 _bm.par_clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()),
132 BitMap::large_range);
133 }
134
135 // Starting at "addr" (inclusive) return a memory region
136 // corresponding to the first maximally contiguous marked ("1") region.
137 inline MemRegion CMSBitMap::getAndClearMarkedRegion(HeapWord* addr) {
138 return getAndClearMarkedRegion(addr, endWord());
139 }
140
141 // Starting at "start_addr" (inclusive) return a memory region
142 // corresponding to the first maximal contiguous marked ("1") region
143 // strictly less than end_addr.
144 inline MemRegion CMSBitMap::getAndClearMarkedRegion(HeapWord* start_addr,
145 HeapWord* end_addr) {
146 HeapWord *start, *end;
147 assert_locked();
148 start = getNextMarkedWordAddress (start_addr, end_addr);
149 end = getNextUnmarkedWordAddress(start, end_addr);
150 assert(start <= end, "Consistency check");
151 MemRegion mr(start, end);
152 if (!mr.is_empty()) {
153 clear_range(mr);
154 }
155 return mr;
156 }
157
158 inline bool CMSBitMap::isMarked(HeapWord* addr) const {
159 assert_locked();
160 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
161 "outside underlying space?");
162 return _bm.at(heapWordToOffset(addr));
163 }
164
165 // The same as isMarked() but without a lock check.
166 inline bool CMSBitMap::par_isMarked(HeapWord* addr) const {
167 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
168 "outside underlying space?");
169 return _bm.at(heapWordToOffset(addr));
170 }
171
172
173 inline bool CMSBitMap::isUnmarked(HeapWord* addr) const {
174 assert_locked();
175 assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
176 "outside underlying space?");
177 return !_bm.at(heapWordToOffset(addr));
178 }
179
180 // Return the HeapWord address corresponding to next "1" bit
181 // (inclusive).
182 inline HeapWord* CMSBitMap::getNextMarkedWordAddress(HeapWord* addr) const {
183 return getNextMarkedWordAddress(addr, endWord());
184 }
185
186 // Return the least HeapWord address corresponding to next "1" bit
187 // starting at start_addr (inclusive) but strictly less than end_addr.
188 inline HeapWord* CMSBitMap::getNextMarkedWordAddress(
189 HeapWord* start_addr, HeapWord* end_addr) const {
190 assert_locked();
191 size_t nextOffset = _bm.get_next_one_offset(
192 heapWordToOffset(start_addr),
193 heapWordToOffset(end_addr));
194 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
195 assert(nextAddr >= start_addr &&
196 nextAddr <= end_addr, "get_next_one postcondition");
197 assert((nextAddr == end_addr) ||
198 isMarked(nextAddr), "get_next_one postcondition");
199 return nextAddr;
200 }
201
202
203 // Return the HeapWord address corresponding to the next "0" bit
204 // (inclusive).
205 inline HeapWord* CMSBitMap::getNextUnmarkedWordAddress(HeapWord* addr) const {
206 return getNextUnmarkedWordAddress(addr, endWord());
207 }
208
209 // Return the HeapWord address corresponding to the next "0" bit
210 // (inclusive).
211 inline HeapWord* CMSBitMap::getNextUnmarkedWordAddress(
212 HeapWord* start_addr, HeapWord* end_addr) const {
213 assert_locked();
214 size_t nextOffset = _bm.get_next_zero_offset(
215 heapWordToOffset(start_addr),
216 heapWordToOffset(end_addr));
217 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
218 assert(nextAddr >= start_addr &&
219 nextAddr <= end_addr, "get_next_zero postcondition");
220 assert((nextAddr == end_addr) ||
221 isUnmarked(nextAddr), "get_next_zero postcondition");
222 return nextAddr;
223 }
224
225 inline bool CMSBitMap::isAllClear() const {
226 assert_locked();
227 return getNextMarkedWordAddress(startWord()) >= endWord();
228 }
229
230 inline void CMSBitMap::iterate(BitMapClosure* cl, HeapWord* left,
231 HeapWord* right) {
232 assert_locked();
233 left = MAX2(_bmStartWord, left);
234 right = MIN2(_bmStartWord + _bmWordSize, right);
235 if (right > left) {
236 _bm.iterate(cl, heapWordToOffset(left), heapWordToOffset(right));
237 }
238 }
239
240 inline void CMSCollector::save_sweep_limits() {
241 _cmsGen->save_sweep_limit();
242 }
243
244 inline bool CMSCollector::is_dead_obj(oop obj) const {
245 HeapWord* addr = (HeapWord*)obj;
246 assert((_cmsGen->cmsSpace()->is_in_reserved(addr)
247 && _cmsGen->cmsSpace()->block_is_obj(addr)),
248 "must be object");
249 return should_unload_classes() &&
250 _collectorState == Sweeping &&
251 !_markBitMap.isMarked(addr);
252 }
253
254 inline bool CMSCollector::should_abort_preclean() const {
255 // We are in the midst of an "abortable preclean" and either
256 // scavenge is done or foreground GC wants to take over collection
257 return _collectorState == AbortablePreclean &&
258 (_abort_preclean || _foregroundGCIsActive ||
259 GenCollectedHeap::heap()->incremental_collection_will_fail(true /* consult_young */));
260 }
261
262 inline size_t CMSCollector::get_eden_used() const {
263 return _young_gen->eden()->used();
264 }
265
266 inline size_t CMSCollector::get_eden_capacity() const {
267 return _young_gen->eden()->capacity();
268 }
269
270 inline bool CMSStats::valid() const {
271 return _valid_bits == _ALL_VALID;
272 }
273
274 inline void CMSStats::record_gc0_begin() {
275 if (_gc0_begin_time.is_updated()) {
276 float last_gc0_period = _gc0_begin_time.seconds();
277 _gc0_period = AdaptiveWeightedAverage::exp_avg(_gc0_period,
278 last_gc0_period, _gc0_alpha);
279 _gc0_alpha = _saved_alpha;
280 _valid_bits |= _GC0_VALID;
281 }
282 _cms_used_at_gc0_begin = _cms_gen->cmsSpace()->used();
283
284 _gc0_begin_time.update();
285 }
286
287 inline void CMSStats::record_gc0_end(size_t cms_gen_bytes_used) {
288 float last_gc0_duration = _gc0_begin_time.seconds();
289 _gc0_duration = AdaptiveWeightedAverage::exp_avg(_gc0_duration,
290 last_gc0_duration, _gc0_alpha);
291
292 // Amount promoted.
293 _cms_used_at_gc0_end = cms_gen_bytes_used;
294
295 size_t promoted_bytes = 0;
296 if (_cms_used_at_gc0_end >= _cms_used_at_gc0_begin) {
297 promoted_bytes = _cms_used_at_gc0_end - _cms_used_at_gc0_begin;
298 }
299
300 // If the young gen collection was skipped, then the
301 // number of promoted bytes will be 0 and adding it to the
302 // average will incorrectly lessen the average. It is, however,
303 // also possible that no promotion was needed.
304 //
305 // _gc0_promoted used to be calculated as
306 // _gc0_promoted = AdaptiveWeightedAverage::exp_avg(_gc0_promoted,
307 // promoted_bytes, _gc0_alpha);
308 _cms_gen->gc_stats()->avg_promoted()->sample(promoted_bytes);
309 _gc0_promoted = (size_t) _cms_gen->gc_stats()->avg_promoted()->average();
310
311 // Amount directly allocated.
312 size_t allocated_bytes = _cms_gen->direct_allocated_words() * HeapWordSize;
313 _cms_gen->reset_direct_allocated_words();
314 _cms_allocated = AdaptiveWeightedAverage::exp_avg(_cms_allocated,
315 allocated_bytes, _gc0_alpha);
316 }
317
318 inline void CMSStats::record_cms_begin() {
319 _cms_timer.stop();
320
321 // This is just an approximate value, but is good enough.
322 _cms_used_at_cms_begin = _cms_used_at_gc0_end;
323
324 _cms_period = AdaptiveWeightedAverage::exp_avg((float)_cms_period,
325 (float) _cms_timer.seconds(), _cms_alpha);
326 _cms_begin_time.update();
327
328 _cms_timer.reset();
329 _cms_timer.start();
330 }
331
332 inline void CMSStats::record_cms_end() {
333 _cms_timer.stop();
334
335 float cur_duration = _cms_timer.seconds();
336 _cms_duration = AdaptiveWeightedAverage::exp_avg(_cms_duration,
337 cur_duration, _cms_alpha);
338
339 _cms_end_time.update();
340 _cms_alpha = _saved_alpha;
341 _allow_duty_cycle_reduction = true;
342 _valid_bits |= _CMS_VALID;
343
344 _cms_timer.start();
345 }
346
347 inline double CMSStats::cms_time_since_begin() const {
348 return _cms_begin_time.seconds();
349 }
350
351 inline double CMSStats::cms_time_since_end() const {
352 return _cms_end_time.seconds();
353 }
354
355 inline double CMSStats::promotion_rate() const {
356 assert(valid(), "statistics not valid yet");
357 return gc0_promoted() / gc0_period();
358 }
359
360 inline double CMSStats::cms_allocation_rate() const {
361 assert(valid(), "statistics not valid yet");
362 return cms_allocated() / gc0_period();
363 }
364
365 inline double CMSStats::cms_consumption_rate() const {
366 assert(valid(), "statistics not valid yet");
367 return (gc0_promoted() + cms_allocated()) / gc0_period();
368 }
369
370 inline void ConcurrentMarkSweepGeneration::save_sweep_limit() {
371 cmsSpace()->save_sweep_limit();
372 }
373
374 inline MemRegion ConcurrentMarkSweepGeneration::used_region_at_save_marks() const {
375 return _cmsSpace->used_region_at_save_marks();
376 }
377
378 inline void MarkFromRootsClosure::do_yield_check() {
379 if (ConcurrentMarkSweepThread::should_yield() &&
380 !_collector->foregroundGCIsActive() &&
381 _yield) {
382 do_yield_work();
383 }
384 }
385
386 inline void ParMarkFromRootsClosure::do_yield_check() {
387 if (ConcurrentMarkSweepThread::should_yield() &&
388 !_collector->foregroundGCIsActive()) {
389 do_yield_work();
390 }
391 }
392
393 inline void PushOrMarkClosure::do_yield_check() {
394 _parent->do_yield_check();
395 }
396
397 inline void ParPushOrMarkClosure::do_yield_check() {
398 _parent->do_yield_check();
399 }
400
401 // Return value of "true" indicates that the on-going preclean
402 // should be aborted.
403 inline bool ScanMarkedObjectsAgainCarefullyClosure::do_yield_check() {
404 if (ConcurrentMarkSweepThread::should_yield() &&
405 !_collector->foregroundGCIsActive() &&
406 _yield) {
407 // Sample young gen size before and after yield
408 _collector->sample_eden();
409 do_yield_work();
410 _collector->sample_eden();
411 return _collector->should_abort_preclean();
412 }
413 return false;
414 }
415
416 inline void SurvivorSpacePrecleanClosure::do_yield_check() {
417 if (ConcurrentMarkSweepThread::should_yield() &&
418 !_collector->foregroundGCIsActive() &&
419 _yield) {
420 // Sample young gen size before and after yield
421 _collector->sample_eden();
422 do_yield_work();
423 _collector->sample_eden();
424 }
425 }
426
427 inline void SweepClosure::do_yield_check(HeapWord* addr) {
428 if (ConcurrentMarkSweepThread::should_yield() &&
429 !_collector->foregroundGCIsActive() &&
430 _yield) {
431 do_yield_work(addr);
432 }
433 }
434
435 inline void MarkRefsIntoAndScanClosure::do_yield_check() {
436 // The conditions are ordered for the remarking phase
437 // when _yield is false.
438 if (_yield &&
439 !_collector->foregroundGCIsActive() &&
440 ConcurrentMarkSweepThread::should_yield()) {
441 do_yield_work();
442 }
443 }
444
445
446 inline void ModUnionClosure::do_MemRegion(MemRegion mr) {
447 // Align the end of mr so it's at a card boundary.
448 // This is superfluous except at the end of the space;
449 // we should do better than this XXX
450 MemRegion mr2(mr.start(), align_up(mr.end(),
451 CardTableModRefBS::card_size /* bytes */));
452 _t->mark_range(mr2);
453 }
454
455 inline void ModUnionClosurePar::do_MemRegion(MemRegion mr) {
456 // Align the end of mr so it's at a card boundary.
457 // This is superfluous except at the end of the space;
458 // we should do better than this XXX
459 MemRegion mr2(mr.start(), align_up(mr.end(),
460 CardTableModRefBS::card_size /* bytes */));
461 _t->par_mark_range(mr2);
462 }
463
464 #endif // SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP