Print this page
rev 2691 : [mq]: g1-reference-processing
| Split |
Close |
| Expand all |
| Collapse all |
--- old/src/share/vm/gc_implementation/g1/concurrentMark.cpp
+++ new/src/share/vm/gc_implementation/g1/concurrentMark.cpp
1 1 /*
2 2 * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
3 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 4 *
5 5 * This code is free software; you can redistribute it and/or modify it
6 6 * under the terms of the GNU General Public License version 2 only, as
7 7 * published by the Free Software Foundation.
8 8 *
9 9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 12 * version 2 for more details (a copy is included in the LICENSE file that
13 13 * accompanied this code).
14 14 *
15 15 * You should have received a copy of the GNU General Public License version
16 16 * 2 along with this work; if not, write to the Free Software Foundation,
17 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 18 *
19 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 20 * or visit www.oracle.com if you need additional information or have any
21 21 * questions.
22 22 *
23 23 */
24 24
25 25 #include "precompiled.hpp"
26 26 #include "classfile/symbolTable.hpp"
27 27 #include "gc_implementation/g1/concurrentMark.inline.hpp"
28 28 #include "gc_implementation/g1/concurrentMarkThread.inline.hpp"
29 29 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
30 30 #include "gc_implementation/g1/g1CollectorPolicy.hpp"
31 31 #include "gc_implementation/g1/g1ErgoVerbose.hpp"
32 32 #include "gc_implementation/g1/g1OopClosures.inline.hpp"
33 33 #include "gc_implementation/g1/g1RemSet.hpp"
34 34 #include "gc_implementation/g1/heapRegionRemSet.hpp"
35 35 #include "gc_implementation/g1/heapRegionSeq.inline.hpp"
36 36 #include "gc_implementation/shared/vmGCOperations.hpp"
37 37 #include "memory/genOopClosures.inline.hpp"
38 38 #include "memory/referencePolicy.hpp"
39 39 #include "memory/resourceArea.hpp"
40 40 #include "oops/oop.inline.hpp"
41 41 #include "runtime/handles.inline.hpp"
42 42 #include "runtime/java.hpp"
43 43
44 44 //
45 45 // CMS Bit Map Wrapper
46 46
47 47 CMBitMapRO::CMBitMapRO(ReservedSpace rs, int shifter):
48 48 _bm((uintptr_t*)NULL,0),
49 49 _shifter(shifter) {
50 50 _bmStartWord = (HeapWord*)(rs.base());
51 51 _bmWordSize = rs.size()/HeapWordSize; // rs.size() is in bytes
52 52 ReservedSpace brs(ReservedSpace::allocation_align_size_up(
53 53 (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1));
54 54
55 55 guarantee(brs.is_reserved(), "couldn't allocate CMS bit map");
56 56 // For now we'll just commit all of the bit map up fromt.
57 57 // Later on we'll try to be more parsimonious with swap.
58 58 guarantee(_virtual_space.initialize(brs, brs.size()),
59 59 "couldn't reseve backing store for CMS bit map");
60 60 assert(_virtual_space.committed_size() == brs.size(),
61 61 "didn't reserve backing store for all of CMS bit map?");
62 62 _bm.set_map((uintptr_t*)_virtual_space.low());
63 63 assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
64 64 _bmWordSize, "inconsistency in bit map sizing");
65 65 _bm.set_size(_bmWordSize >> _shifter);
66 66 }
67 67
68 68 HeapWord* CMBitMapRO::getNextMarkedWordAddress(HeapWord* addr,
69 69 HeapWord* limit) const {
70 70 // First we must round addr *up* to a possible object boundary.
71 71 addr = (HeapWord*)align_size_up((intptr_t)addr,
72 72 HeapWordSize << _shifter);
73 73 size_t addrOffset = heapWordToOffset(addr);
74 74 if (limit == NULL) {
75 75 limit = _bmStartWord + _bmWordSize;
76 76 }
77 77 size_t limitOffset = heapWordToOffset(limit);
78 78 size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset);
79 79 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
80 80 assert(nextAddr >= addr, "get_next_one postcondition");
81 81 assert(nextAddr == limit || isMarked(nextAddr),
82 82 "get_next_one postcondition");
83 83 return nextAddr;
84 84 }
85 85
86 86 HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(HeapWord* addr,
87 87 HeapWord* limit) const {
88 88 size_t addrOffset = heapWordToOffset(addr);
89 89 if (limit == NULL) {
90 90 limit = _bmStartWord + _bmWordSize;
91 91 }
92 92 size_t limitOffset = heapWordToOffset(limit);
93 93 size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset);
94 94 HeapWord* nextAddr = offsetToHeapWord(nextOffset);
95 95 assert(nextAddr >= addr, "get_next_one postcondition");
96 96 assert(nextAddr == limit || !isMarked(nextAddr),
97 97 "get_next_one postcondition");
98 98 return nextAddr;
99 99 }
100 100
101 101 int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const {
102 102 assert((diff & ((1 << _shifter) - 1)) == 0, "argument check");
103 103 return (int) (diff >> _shifter);
104 104 }
105 105
106 106 bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) {
107 107 HeapWord* left = MAX2(_bmStartWord, mr.start());
108 108 HeapWord* right = MIN2(_bmStartWord + _bmWordSize, mr.end());
109 109 if (right > left) {
110 110 // Right-open interval [leftOffset, rightOffset).
111 111 return _bm.iterate(cl, heapWordToOffset(left), heapWordToOffset(right));
112 112 } else {
113 113 return true;
114 114 }
115 115 }
116 116
117 117 void CMBitMapRO::mostly_disjoint_range_union(BitMap* from_bitmap,
118 118 size_t from_start_index,
119 119 HeapWord* to_start_word,
120 120 size_t word_num) {
121 121 _bm.mostly_disjoint_range_union(from_bitmap,
122 122 from_start_index,
123 123 heapWordToOffset(to_start_word),
124 124 word_num);
125 125 }
126 126
127 127 #ifndef PRODUCT
128 128 bool CMBitMapRO::covers(ReservedSpace rs) const {
129 129 // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
130 130 assert(((size_t)_bm.size() * (size_t)(1 << _shifter)) == _bmWordSize,
131 131 "size inconsistency");
132 132 return _bmStartWord == (HeapWord*)(rs.base()) &&
133 133 _bmWordSize == rs.size()>>LogHeapWordSize;
134 134 }
135 135 #endif
136 136
137 137 void CMBitMap::clearAll() {
138 138 _bm.clear();
139 139 return;
140 140 }
141 141
142 142 void CMBitMap::markRange(MemRegion mr) {
143 143 mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
144 144 assert(!mr.is_empty(), "unexpected empty region");
145 145 assert((offsetToHeapWord(heapWordToOffset(mr.end())) ==
146 146 ((HeapWord *) mr.end())),
147 147 "markRange memory region end is not card aligned");
148 148 // convert address range into offset range
149 149 _bm.at_put_range(heapWordToOffset(mr.start()),
150 150 heapWordToOffset(mr.end()), true);
151 151 }
152 152
153 153 void CMBitMap::clearRange(MemRegion mr) {
154 154 mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
155 155 assert(!mr.is_empty(), "unexpected empty region");
156 156 // convert address range into offset range
157 157 _bm.at_put_range(heapWordToOffset(mr.start()),
158 158 heapWordToOffset(mr.end()), false);
159 159 }
160 160
161 161 MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr,
162 162 HeapWord* end_addr) {
163 163 HeapWord* start = getNextMarkedWordAddress(addr);
164 164 start = MIN2(start, end_addr);
165 165 HeapWord* end = getNextUnmarkedWordAddress(start);
166 166 end = MIN2(end, end_addr);
167 167 assert(start <= end, "Consistency check");
168 168 MemRegion mr(start, end);
169 169 if (!mr.is_empty()) {
170 170 clearRange(mr);
171 171 }
172 172 return mr;
173 173 }
174 174
175 175 CMMarkStack::CMMarkStack(ConcurrentMark* cm) :
176 176 _base(NULL), _cm(cm)
177 177 #ifdef ASSERT
178 178 , _drain_in_progress(false)
179 179 , _drain_in_progress_yields(false)
180 180 #endif
181 181 {}
182 182
183 183 void CMMarkStack::allocate(size_t size) {
184 184 _base = NEW_C_HEAP_ARRAY(oop, size);
185 185 if (_base == NULL) {
186 186 vm_exit_during_initialization("Failed to allocate "
187 187 "CM region mark stack");
188 188 }
189 189 _index = 0;
190 190 _capacity = (jint) size;
191 191 _oops_do_bound = -1;
192 192 NOT_PRODUCT(_max_depth = 0);
193 193 }
194 194
195 195 CMMarkStack::~CMMarkStack() {
196 196 if (_base != NULL) {
197 197 FREE_C_HEAP_ARRAY(oop, _base);
198 198 }
199 199 }
200 200
201 201 void CMMarkStack::par_push(oop ptr) {
202 202 while (true) {
203 203 if (isFull()) {
204 204 _overflow = true;
205 205 return;
206 206 }
207 207 // Otherwise...
208 208 jint index = _index;
209 209 jint next_index = index+1;
210 210 jint res = Atomic::cmpxchg(next_index, &_index, index);
211 211 if (res == index) {
212 212 _base[index] = ptr;
213 213 // Note that we don't maintain this atomically. We could, but it
214 214 // doesn't seem necessary.
215 215 NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index));
216 216 return;
217 217 }
218 218 // Otherwise, we need to try again.
219 219 }
220 220 }
221 221
222 222 void CMMarkStack::par_adjoin_arr(oop* ptr_arr, int n) {
223 223 while (true) {
224 224 if (isFull()) {
225 225 _overflow = true;
226 226 return;
227 227 }
228 228 // Otherwise...
229 229 jint index = _index;
230 230 jint next_index = index + n;
231 231 if (next_index > _capacity) {
232 232 _overflow = true;
233 233 return;
234 234 }
235 235 jint res = Atomic::cmpxchg(next_index, &_index, index);
236 236 if (res == index) {
237 237 for (int i = 0; i < n; i++) {
238 238 int ind = index + i;
239 239 assert(ind < _capacity, "By overflow test above.");
240 240 _base[ind] = ptr_arr[i];
241 241 }
242 242 NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index));
243 243 return;
244 244 }
245 245 // Otherwise, we need to try again.
246 246 }
247 247 }
248 248
249 249
250 250 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
251 251 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
252 252 jint start = _index;
253 253 jint next_index = start + n;
254 254 if (next_index > _capacity) {
255 255 _overflow = true;
256 256 return;
257 257 }
258 258 // Otherwise.
259 259 _index = next_index;
260 260 for (int i = 0; i < n; i++) {
261 261 int ind = start + i;
262 262 assert(ind < _capacity, "By overflow test above.");
263 263 _base[ind] = ptr_arr[i];
264 264 }
265 265 }
266 266
267 267
268 268 bool CMMarkStack::par_pop_arr(oop* ptr_arr, int max, int* n) {
269 269 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
270 270 jint index = _index;
271 271 if (index == 0) {
272 272 *n = 0;
273 273 return false;
274 274 } else {
275 275 int k = MIN2(max, index);
276 276 jint new_ind = index - k;
277 277 for (int j = 0; j < k; j++) {
278 278 ptr_arr[j] = _base[new_ind + j];
279 279 }
280 280 _index = new_ind;
281 281 *n = k;
282 282 return true;
283 283 }
284 284 }
285 285
286 286
287 287 CMRegionStack::CMRegionStack() : _base(NULL) {}
288 288
289 289 void CMRegionStack::allocate(size_t size) {
290 290 _base = NEW_C_HEAP_ARRAY(MemRegion, size);
291 291 if (_base == NULL) {
292 292 vm_exit_during_initialization("Failed to allocate CM region mark stack");
293 293 }
294 294 _index = 0;
295 295 _capacity = (jint) size;
296 296 }
297 297
298 298 CMRegionStack::~CMRegionStack() {
299 299 if (_base != NULL) {
300 300 FREE_C_HEAP_ARRAY(oop, _base);
301 301 }
302 302 }
303 303
304 304 void CMRegionStack::push_lock_free(MemRegion mr) {
305 305 assert(mr.word_size() > 0, "Precondition");
306 306 while (true) {
307 307 jint index = _index;
308 308
309 309 if (index >= _capacity) {
310 310 _overflow = true;
311 311 return;
312 312 }
313 313 // Otherwise...
314 314 jint next_index = index+1;
315 315 jint res = Atomic::cmpxchg(next_index, &_index, index);
316 316 if (res == index) {
317 317 _base[index] = mr;
318 318 return;
319 319 }
320 320 // Otherwise, we need to try again.
321 321 }
322 322 }
323 323
324 324 // Lock-free pop of the region stack. Called during the concurrent
325 325 // marking / remark phases. Should only be called in tandem with
326 326 // other lock-free pops.
327 327 MemRegion CMRegionStack::pop_lock_free() {
328 328 while (true) {
329 329 jint index = _index;
330 330
331 331 if (index == 0) {
332 332 return MemRegion();
333 333 }
334 334 // Otherwise...
335 335 jint next_index = index-1;
336 336 jint res = Atomic::cmpxchg(next_index, &_index, index);
337 337 if (res == index) {
338 338 MemRegion mr = _base[next_index];
339 339 if (mr.start() != NULL) {
340 340 assert(mr.end() != NULL, "invariant");
341 341 assert(mr.word_size() > 0, "invariant");
342 342 return mr;
343 343 } else {
344 344 // that entry was invalidated... let's skip it
345 345 assert(mr.end() == NULL, "invariant");
346 346 }
347 347 }
348 348 // Otherwise, we need to try again.
349 349 }
350 350 }
351 351
352 352 #if 0
353 353 // The routines that manipulate the region stack with a lock are
354 354 // not currently used. They should be retained, however, as a
355 355 // diagnostic aid.
356 356
357 357 void CMRegionStack::push_with_lock(MemRegion mr) {
358 358 assert(mr.word_size() > 0, "Precondition");
359 359 MutexLockerEx x(CMRegionStack_lock, Mutex::_no_safepoint_check_flag);
360 360
361 361 if (isFull()) {
362 362 _overflow = true;
363 363 return;
364 364 }
365 365
366 366 _base[_index] = mr;
367 367 _index += 1;
368 368 }
369 369
370 370 MemRegion CMRegionStack::pop_with_lock() {
371 371 MutexLockerEx x(CMRegionStack_lock, Mutex::_no_safepoint_check_flag);
372 372
373 373 while (true) {
374 374 if (_index == 0) {
375 375 return MemRegion();
376 376 }
377 377 _index -= 1;
378 378
379 379 MemRegion mr = _base[_index];
380 380 if (mr.start() != NULL) {
381 381 assert(mr.end() != NULL, "invariant");
382 382 assert(mr.word_size() > 0, "invariant");
383 383 return mr;
384 384 } else {
385 385 // that entry was invalidated... let's skip it
386 386 assert(mr.end() == NULL, "invariant");
387 387 }
388 388 }
389 389 }
390 390 #endif
391 391
392 392 bool CMRegionStack::invalidate_entries_into_cset() {
393 393 bool result = false;
394 394 G1CollectedHeap* g1h = G1CollectedHeap::heap();
395 395 for (int i = 0; i < _oops_do_bound; ++i) {
396 396 MemRegion mr = _base[i];
397 397 if (mr.start() != NULL) {
398 398 assert(mr.end() != NULL, "invariant");
399 399 assert(mr.word_size() > 0, "invariant");
400 400 HeapRegion* hr = g1h->heap_region_containing(mr.start());
401 401 assert(hr != NULL, "invariant");
402 402 if (hr->in_collection_set()) {
403 403 // The region points into the collection set
404 404 _base[i] = MemRegion();
405 405 result = true;
406 406 }
407 407 } else {
408 408 // that entry was invalidated... let's skip it
409 409 assert(mr.end() == NULL, "invariant");
410 410 }
411 411 }
412 412 return result;
413 413 }
414 414
415 415 template<class OopClosureClass>
416 416 bool CMMarkStack::drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after) {
417 417 assert(!_drain_in_progress || !_drain_in_progress_yields || yield_after
418 418 || SafepointSynchronize::is_at_safepoint(),
419 419 "Drain recursion must be yield-safe.");
420 420 bool res = true;
421 421 debug_only(_drain_in_progress = true);
422 422 debug_only(_drain_in_progress_yields = yield_after);
423 423 while (!isEmpty()) {
424 424 oop newOop = pop();
425 425 assert(G1CollectedHeap::heap()->is_in_reserved(newOop), "Bad pop");
426 426 assert(newOop->is_oop(), "Expected an oop");
427 427 assert(bm == NULL || bm->isMarked((HeapWord*)newOop),
428 428 "only grey objects on this stack");
429 429 // iterate over the oops in this oop, marking and pushing
430 430 // the ones in CMS generation.
431 431 newOop->oop_iterate(cl);
432 432 if (yield_after && _cm->do_yield_check()) {
433 433 res = false;
434 434 break;
435 435 }
436 436 }
437 437 debug_only(_drain_in_progress = false);
438 438 return res;
439 439 }
440 440
441 441 void CMMarkStack::oops_do(OopClosure* f) {
442 442 if (_index == 0) return;
443 443 assert(_oops_do_bound != -1 && _oops_do_bound <= _index,
444 444 "Bound must be set.");
445 445 for (int i = 0; i < _oops_do_bound; i++) {
446 446 f->do_oop(&_base[i]);
447 447 }
448 448 _oops_do_bound = -1;
449 449 }
450 450
451 451 bool ConcurrentMark::not_yet_marked(oop obj) const {
452 452 return (_g1h->is_obj_ill(obj)
453 453 || (_g1h->is_in_permanent(obj)
454 454 && !nextMarkBitMap()->isMarked((HeapWord*)obj)));
455 455 }
456 456
457 457 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
458 458 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
459 459 #endif // _MSC_VER
460 460
461 461 ConcurrentMark::ConcurrentMark(ReservedSpace rs,
462 462 int max_regions) :
463 463 _markBitMap1(rs, MinObjAlignment - 1),
464 464 _markBitMap2(rs, MinObjAlignment - 1),
465 465
466 466 _parallel_marking_threads(0),
467 467 _sleep_factor(0.0),
468 468 _marking_task_overhead(1.0),
469 469 _cleanup_sleep_factor(0.0),
470 470 _cleanup_task_overhead(1.0),
471 471 _cleanup_list("Cleanup List"),
472 472 _region_bm(max_regions, false /* in_resource_area*/),
473 473 _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >>
474 474 CardTableModRefBS::card_shift,
475 475 false /* in_resource_area*/),
476 476 _prevMarkBitMap(&_markBitMap1),
477 477 _nextMarkBitMap(&_markBitMap2),
478 478 _at_least_one_mark_complete(false),
479 479
480 480 _markStack(this),
481 481 _regionStack(),
482 482 // _finger set in set_non_marking_state
483 483
484 484 _max_task_num(MAX2(ParallelGCThreads, (size_t)1)),
485 485 // _active_tasks set in set_non_marking_state
486 486 // _tasks set inside the constructor
487 487 _task_queues(new CMTaskQueueSet((int) _max_task_num)),
488 488 _terminator(ParallelTaskTerminator((int) _max_task_num, _task_queues)),
489 489
490 490 _has_overflown(false),
491 491 _concurrent(false),
492 492 _has_aborted(false),
493 493 _restart_for_overflow(false),
494 494 _concurrent_marking_in_progress(false),
495 495 _should_gray_objects(false),
496 496
497 497 // _verbose_level set below
498 498
499 499 _init_times(),
500 500 _remark_times(), _remark_mark_times(), _remark_weak_ref_times(),
501 501 _cleanup_times(),
502 502 _total_counting_time(0.0),
503 503 _total_rs_scrub_time(0.0),
504 504
505 505 _parallel_workers(NULL) {
506 506 CMVerboseLevel verbose_level = (CMVerboseLevel) G1MarkingVerboseLevel;
507 507 if (verbose_level < no_verbose) {
508 508 verbose_level = no_verbose;
509 509 }
510 510 if (verbose_level > high_verbose) {
511 511 verbose_level = high_verbose;
512 512 }
513 513 _verbose_level = verbose_level;
514 514
515 515 if (verbose_low()) {
516 516 gclog_or_tty->print_cr("[global] init, heap start = "PTR_FORMAT", "
517 517 "heap end = "PTR_FORMAT, _heap_start, _heap_end);
518 518 }
519 519
520 520 _markStack.allocate(MarkStackSize);
521 521 _regionStack.allocate(G1MarkRegionStackSize);
522 522
523 523 // Create & start a ConcurrentMark thread.
524 524 _cmThread = new ConcurrentMarkThread(this);
525 525 assert(cmThread() != NULL, "CM Thread should have been created");
526 526 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
527 527
528 528 _g1h = G1CollectedHeap::heap();
529 529 assert(CGC_lock != NULL, "Where's the CGC_lock?");
530 530 assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency");
531 531 assert(_markBitMap2.covers(rs), "_markBitMap2 inconsistency");
532 532
533 533 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
534 534 satb_qs.set_buffer_size(G1SATBBufferSize);
535 535
536 536 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num);
537 537 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num);
538 538
539 539 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
540 540 _active_tasks = _max_task_num;
541 541 for (int i = 0; i < (int) _max_task_num; ++i) {
542 542 CMTaskQueue* task_queue = new CMTaskQueue();
543 543 task_queue->initialize();
544 544 _task_queues->register_queue(i, task_queue);
545 545
546 546 _tasks[i] = new CMTask(i, this, task_queue, _task_queues);
547 547 _accum_task_vtime[i] = 0.0;
548 548 }
549 549
550 550 if (ConcGCThreads > ParallelGCThreads) {
551 551 vm_exit_during_initialization("Can't have more ConcGCThreads "
552 552 "than ParallelGCThreads.");
553 553 }
554 554 if (ParallelGCThreads == 0) {
555 555 // if we are not running with any parallel GC threads we will not
556 556 // spawn any marking threads either
557 557 _parallel_marking_threads = 0;
558 558 _sleep_factor = 0.0;
559 559 _marking_task_overhead = 1.0;
560 560 } else {
561 561 if (ConcGCThreads > 0) {
562 562 // notice that ConcGCThreads overwrites G1MarkingOverheadPercent
563 563 // if both are set
564 564
565 565 _parallel_marking_threads = ConcGCThreads;
566 566 _sleep_factor = 0.0;
567 567 _marking_task_overhead = 1.0;
568 568 } else if (G1MarkingOverheadPercent > 0) {
569 569 // we will calculate the number of parallel marking threads
570 570 // based on a target overhead with respect to the soft real-time
571 571 // goal
572 572
573 573 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0;
574 574 double overall_cm_overhead =
575 575 (double) MaxGCPauseMillis * marking_overhead /
576 576 (double) GCPauseIntervalMillis;
577 577 double cpu_ratio = 1.0 / (double) os::processor_count();
578 578 double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio);
579 579 double marking_task_overhead =
580 580 overall_cm_overhead / marking_thread_num *
581 581 (double) os::processor_count();
582 582 double sleep_factor =
583 583 (1.0 - marking_task_overhead) / marking_task_overhead;
584 584
585 585 _parallel_marking_threads = (size_t) marking_thread_num;
586 586 _sleep_factor = sleep_factor;
587 587 _marking_task_overhead = marking_task_overhead;
588 588 } else {
589 589 _parallel_marking_threads = MAX2((ParallelGCThreads + 2) / 4, (size_t)1);
590 590 _sleep_factor = 0.0;
591 591 _marking_task_overhead = 1.0;
592 592 }
593 593
594 594 if (parallel_marking_threads() > 1) {
595 595 _cleanup_task_overhead = 1.0;
596 596 } else {
597 597 _cleanup_task_overhead = marking_task_overhead();
598 598 }
599 599 _cleanup_sleep_factor =
600 600 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
601 601
602 602 #if 0
603 603 gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads());
604 604 gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead());
605 605 gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor());
606 606 gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead());
607 607 gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor());
608 608 #endif
609 609
610 610 guarantee(parallel_marking_threads() > 0, "peace of mind");
611 611 _parallel_workers = new FlexibleWorkGang("G1 Parallel Marking Threads",
612 612 (int) _parallel_marking_threads, false, true);
613 613 if (_parallel_workers == NULL) {
614 614 vm_exit_during_initialization("Failed necessary allocation.");
615 615 } else {
616 616 _parallel_workers->initialize_workers();
617 617 }
618 618 }
619 619
620 620 // so that the call below can read a sensible value
621 621 _heap_start = (HeapWord*) rs.base();
622 622 set_non_marking_state();
623 623 }
624 624
625 625 void ConcurrentMark::update_g1_committed(bool force) {
626 626 // If concurrent marking is not in progress, then we do not need to
627 627 // update _heap_end. This has a subtle and important
628 628 // side-effect. Imagine that two evacuation pauses happen between
629 629 // marking completion and remark. The first one can grow the
630 630 // heap (hence now the finger is below the heap end). Then, the
631 631 // second one could unnecessarily push regions on the region
632 632 // stack. This causes the invariant that the region stack is empty
633 633 // at the beginning of remark to be false. By ensuring that we do
634 634 // not observe heap expansions after marking is complete, then we do
635 635 // not have this problem.
636 636 if (!concurrent_marking_in_progress() && !force) return;
637 637
638 638 MemRegion committed = _g1h->g1_committed();
639 639 assert(committed.start() == _heap_start, "start shouldn't change");
640 640 HeapWord* new_end = committed.end();
641 641 if (new_end > _heap_end) {
642 642 // The heap has been expanded.
643 643
644 644 _heap_end = new_end;
645 645 }
646 646 // Notice that the heap can also shrink. However, this only happens
647 647 // during a Full GC (at least currently) and the entire marking
648 648 // phase will bail out and the task will not be restarted. So, let's
649 649 // do nothing.
650 650 }
651 651
652 652 void ConcurrentMark::reset() {
653 653 // Starting values for these two. This should be called in a STW
654 654 // phase. CM will be notified of any future g1_committed expansions
655 655 // will be at the end of evacuation pauses, when tasks are
656 656 // inactive.
657 657 MemRegion committed = _g1h->g1_committed();
658 658 _heap_start = committed.start();
659 659 _heap_end = committed.end();
660 660
661 661 // Separated the asserts so that we know which one fires.
662 662 assert(_heap_start != NULL, "heap bounds should look ok");
663 663 assert(_heap_end != NULL, "heap bounds should look ok");
664 664 assert(_heap_start < _heap_end, "heap bounds should look ok");
665 665
666 666 // reset all the marking data structures and any necessary flags
667 667 clear_marking_state();
668 668
669 669 if (verbose_low()) {
670 670 gclog_or_tty->print_cr("[global] resetting");
671 671 }
672 672
673 673 // We do reset all of them, since different phases will use
674 674 // different number of active threads. So, it's easiest to have all
675 675 // of them ready.
676 676 for (int i = 0; i < (int) _max_task_num; ++i) {
677 677 _tasks[i]->reset(_nextMarkBitMap);
678 678 }
679 679
680 680 // we need this to make sure that the flag is on during the evac
681 681 // pause with initial mark piggy-backed
682 682 set_concurrent_marking_in_progress();
683 683 }
684 684
685 685 void ConcurrentMark::set_phase(size_t active_tasks, bool concurrent) {
686 686 assert(active_tasks <= _max_task_num, "we should not have more");
687 687
688 688 _active_tasks = active_tasks;
689 689 // Need to update the three data structures below according to the
690 690 // number of active threads for this phase.
691 691 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues);
692 692 _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
693 693 _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
694 694
695 695 _concurrent = concurrent;
696 696 // We propagate this to all tasks, not just the active ones.
697 697 for (int i = 0; i < (int) _max_task_num; ++i)
698 698 _tasks[i]->set_concurrent(concurrent);
699 699
700 700 if (concurrent) {
701 701 set_concurrent_marking_in_progress();
702 702 } else {
703 703 // We currently assume that the concurrent flag has been set to
704 704 // false before we start remark. At this point we should also be
705 705 // in a STW phase.
706 706 assert(!concurrent_marking_in_progress(), "invariant");
707 707 assert(_finger == _heap_end, "only way to get here");
708 708 update_g1_committed(true);
709 709 }
710 710 }
711 711
712 712 void ConcurrentMark::set_non_marking_state() {
713 713 // We set the global marking state to some default values when we're
714 714 // not doing marking.
715 715 clear_marking_state();
716 716 _active_tasks = 0;
717 717 clear_concurrent_marking_in_progress();
718 718 }
719 719
720 720 ConcurrentMark::~ConcurrentMark() {
721 721 for (int i = 0; i < (int) _max_task_num; ++i) {
722 722 delete _task_queues->queue(i);
723 723 delete _tasks[i];
724 724 }
725 725 delete _task_queues;
726 726 FREE_C_HEAP_ARRAY(CMTask*, _max_task_num);
727 727 }
728 728
729 729 // This closure is used to mark refs into the g1 generation
730 730 // from external roots in the CMS bit map.
731 731 // Called at the first checkpoint.
732 732 //
733 733
734 734 void ConcurrentMark::clearNextBitmap() {
735 735 G1CollectedHeap* g1h = G1CollectedHeap::heap();
736 736 G1CollectorPolicy* g1p = g1h->g1_policy();
737 737
738 738 // Make sure that the concurrent mark thread looks to still be in
739 739 // the current cycle.
740 740 guarantee(cmThread()->during_cycle(), "invariant");
741 741
742 742 // We are finishing up the current cycle by clearing the next
743 743 // marking bitmap and getting it ready for the next cycle. During
744 744 // this time no other cycle can start. So, let's make sure that this
745 745 // is the case.
746 746 guarantee(!g1h->mark_in_progress(), "invariant");
747 747
748 748 // clear the mark bitmap (no grey objects to start with).
749 749 // We need to do this in chunks and offer to yield in between
750 750 // each chunk.
751 751 HeapWord* start = _nextMarkBitMap->startWord();
752 752 HeapWord* end = _nextMarkBitMap->endWord();
753 753 HeapWord* cur = start;
754 754 size_t chunkSize = M;
755 755 while (cur < end) {
756 756 HeapWord* next = cur + chunkSize;
757 757 if (next > end) {
758 758 next = end;
759 759 }
760 760 MemRegion mr(cur,next);
761 761 _nextMarkBitMap->clearRange(mr);
762 762 cur = next;
763 763 do_yield_check();
764 764
765 765 // Repeat the asserts from above. We'll do them as asserts here to
766 766 // minimize their overhead on the product. However, we'll have
767 767 // them as guarantees at the beginning / end of the bitmap
768 768 // clearing to get some checking in the product.
769 769 assert(cmThread()->during_cycle(), "invariant");
770 770 assert(!g1h->mark_in_progress(), "invariant");
771 771 }
772 772
773 773 // Repeat the asserts from above.
774 774 guarantee(cmThread()->during_cycle(), "invariant");
775 775 guarantee(!g1h->mark_in_progress(), "invariant");
776 776 }
777 777
778 778 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
779 779 public:
780 780 bool doHeapRegion(HeapRegion* r) {
781 781 if (!r->continuesHumongous()) {
782 782 r->note_start_of_marking(true);
783 783 }
784 784 return false;
785 785 }
786 786 };
787 787
788 788 void ConcurrentMark::checkpointRootsInitialPre() {
789 789 G1CollectedHeap* g1h = G1CollectedHeap::heap();
790 790 G1CollectorPolicy* g1p = g1h->g1_policy();
791 791
792 792 _has_aborted = false;
793 793
794 794 #ifndef PRODUCT
795 795 if (G1PrintReachableAtInitialMark) {
796 796 print_reachable("at-cycle-start",
797 797 VerifyOption_G1UsePrevMarking, true /* all */);
798 798 }
799 799 #endif
800 800
801 801 // Initialise marking structures. This has to be done in a STW phase.
802 802 reset();
803 803 }
804 804
805 805
806 806 void ConcurrentMark::checkpointRootsInitialPost() {
807 807 G1CollectedHeap* g1h = G1CollectedHeap::heap();
808 808
809 809 // If we force an overflow during remark, the remark operation will
810 810 // actually abort and we'll restart concurrent marking. If we always
|
↓ open down ↓ |
810 lines elided |
↑ open up ↑ |
811 811 // force an oveflow during remark we'll never actually complete the
812 812 // marking phase. So, we initilize this here, at the start of the
813 813 // cycle, so that at the remaining overflow number will decrease at
814 814 // every remark and we'll eventually not need to cause one.
815 815 force_overflow_stw()->init();
816 816
817 817 // For each region note start of marking.
818 818 NoteStartOfMarkHRClosure startcl;
819 819 g1h->heap_region_iterate(&startcl);
820 820
821 - // Start weak-reference discovery.
822 - ReferenceProcessor* rp = g1h->ref_processor();
823 - rp->verify_no_references_recorded();
824 - rp->enable_discovery(); // enable ("weak") refs discovery
821 + // Start Concurrent Marking weak-reference discovery.
822 + ReferenceProcessor* rp = g1h->ref_processor_cm();
823 + // enable ("weak") refs discovery
824 + rp->enable_discovery(true /*verify_disabled*/, true /*verify_no_refs*/);
825 825 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
826 826
827 827 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
828 828 // This is the start of the marking cycle, we're expected all
829 829 // threads to have SATB queues with active set to false.
830 830 satb_mq_set.set_active_all_threads(true, /* new active value */
831 831 false /* expected_active */);
832 832
833 833 // update_g1_committed() will be called at the end of an evac pause
834 834 // when marking is on. So, it's also called at the end of the
835 835 // initial-mark pause to update the heap end, if the heap expands
836 836 // during it. No need to call it here.
837 837 }
838 838
839 839 /*
840 840 * Notice that in the next two methods, we actually leave the STS
841 841 * during the barrier sync and join it immediately afterwards. If we
842 842 * do not do this, the following deadlock can occur: one thread could
843 843 * be in the barrier sync code, waiting for the other thread to also
844 844 * sync up, whereas another one could be trying to yield, while also
845 845 * waiting for the other threads to sync up too.
846 846 *
847 847 * Note, however, that this code is also used during remark and in
848 848 * this case we should not attempt to leave / enter the STS, otherwise
849 849 * we'll either hit an asseert (debug / fastdebug) or deadlock
850 850 * (product). So we should only leave / enter the STS if we are
851 851 * operating concurrently.
852 852 *
853 853 * Because the thread that does the sync barrier has left the STS, it
854 854 * is possible to be suspended for a Full GC or an evacuation pause
855 855 * could occur. This is actually safe, since the entering the sync
856 856 * barrier is one of the last things do_marking_step() does, and it
857 857 * doesn't manipulate any data structures afterwards.
858 858 */
859 859
860 860 void ConcurrentMark::enter_first_sync_barrier(int task_num) {
861 861 if (verbose_low()) {
862 862 gclog_or_tty->print_cr("[%d] entering first barrier", task_num);
863 863 }
864 864
865 865 if (concurrent()) {
866 866 ConcurrentGCThread::stsLeave();
867 867 }
868 868 _first_overflow_barrier_sync.enter();
869 869 if (concurrent()) {
870 870 ConcurrentGCThread::stsJoin();
871 871 }
872 872 // at this point everyone should have synced up and not be doing any
873 873 // more work
874 874
875 875 if (verbose_low()) {
876 876 gclog_or_tty->print_cr("[%d] leaving first barrier", task_num);
877 877 }
878 878
879 879 // let task 0 do this
880 880 if (task_num == 0) {
881 881 // task 0 is responsible for clearing the global data structures
882 882 // We should be here because of an overflow. During STW we should
883 883 // not clear the overflow flag since we rely on it being true when
884 884 // we exit this method to abort the pause and restart concurent
885 885 // marking.
886 886 clear_marking_state(concurrent() /* clear_overflow */);
887 887 force_overflow()->update();
888 888
889 889 if (PrintGC) {
890 890 gclog_or_tty->date_stamp(PrintGCDateStamps);
891 891 gclog_or_tty->stamp(PrintGCTimeStamps);
892 892 gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]");
893 893 }
894 894 }
895 895
896 896 // after this, each task should reset its own data structures then
897 897 // then go into the second barrier
898 898 }
899 899
900 900 void ConcurrentMark::enter_second_sync_barrier(int task_num) {
901 901 if (verbose_low()) {
902 902 gclog_or_tty->print_cr("[%d] entering second barrier", task_num);
903 903 }
904 904
905 905 if (concurrent()) {
906 906 ConcurrentGCThread::stsLeave();
907 907 }
908 908 _second_overflow_barrier_sync.enter();
909 909 if (concurrent()) {
910 910 ConcurrentGCThread::stsJoin();
911 911 }
912 912 // at this point everything should be re-initialised and ready to go
913 913
914 914 if (verbose_low()) {
915 915 gclog_or_tty->print_cr("[%d] leaving second barrier", task_num);
916 916 }
917 917 }
918 918
919 919 #ifndef PRODUCT
920 920 void ForceOverflowSettings::init() {
921 921 _num_remaining = G1ConcMarkForceOverflow;
922 922 _force = false;
923 923 update();
924 924 }
925 925
926 926 void ForceOverflowSettings::update() {
927 927 if (_num_remaining > 0) {
928 928 _num_remaining -= 1;
929 929 _force = true;
930 930 } else {
931 931 _force = false;
932 932 }
933 933 }
934 934
935 935 bool ForceOverflowSettings::should_force() {
936 936 if (_force) {
937 937 _force = false;
938 938 return true;
939 939 } else {
940 940 return false;
941 941 }
942 942 }
943 943 #endif // !PRODUCT
944 944
945 945 void ConcurrentMark::grayRoot(oop p) {
946 946 HeapWord* addr = (HeapWord*) p;
947 947 // We can't really check against _heap_start and _heap_end, since it
948 948 // is possible during an evacuation pause with piggy-backed
949 949 // initial-mark that the committed space is expanded during the
950 950 // pause without CM observing this change. So the assertions below
951 951 // is a bit conservative; but better than nothing.
952 952 assert(_g1h->g1_committed().contains(addr),
953 953 "address should be within the heap bounds");
954 954
955 955 if (!_nextMarkBitMap->isMarked(addr)) {
956 956 _nextMarkBitMap->parMark(addr);
957 957 }
958 958 }
959 959
960 960 void ConcurrentMark::grayRegionIfNecessary(MemRegion mr) {
961 961 // The objects on the region have already been marked "in bulk" by
962 962 // the caller. We only need to decide whether to push the region on
963 963 // the region stack or not.
964 964
965 965 if (!concurrent_marking_in_progress() || !_should_gray_objects) {
966 966 // We're done with marking and waiting for remark. We do not need to
967 967 // push anything else on the region stack.
968 968 return;
969 969 }
970 970
971 971 HeapWord* finger = _finger;
972 972
973 973 if (verbose_low()) {
974 974 gclog_or_tty->print_cr("[global] attempting to push "
975 975 "region ["PTR_FORMAT", "PTR_FORMAT"), finger is at "
976 976 PTR_FORMAT, mr.start(), mr.end(), finger);
977 977 }
978 978
979 979 if (mr.start() < finger) {
980 980 // The finger is always heap region aligned and it is not possible
981 981 // for mr to span heap regions.
982 982 assert(mr.end() <= finger, "invariant");
983 983
984 984 // Separated the asserts so that we know which one fires.
985 985 assert(mr.start() <= mr.end(),
986 986 "region boundaries should fall within the committed space");
987 987 assert(_heap_start <= mr.start(),
988 988 "region boundaries should fall within the committed space");
989 989 assert(mr.end() <= _heap_end,
990 990 "region boundaries should fall within the committed space");
991 991 if (verbose_low()) {
992 992 gclog_or_tty->print_cr("[global] region ["PTR_FORMAT", "PTR_FORMAT") "
993 993 "below the finger, pushing it",
994 994 mr.start(), mr.end());
995 995 }
996 996
997 997 if (!region_stack_push_lock_free(mr)) {
998 998 if (verbose_low()) {
999 999 gclog_or_tty->print_cr("[global] region stack has overflown.");
1000 1000 }
1001 1001 }
1002 1002 }
1003 1003 }
1004 1004
1005 1005 void ConcurrentMark::markAndGrayObjectIfNecessary(oop p) {
1006 1006 // The object is not marked by the caller. We need to at least mark
1007 1007 // it and maybe push in on the stack.
1008 1008
1009 1009 HeapWord* addr = (HeapWord*)p;
1010 1010 if (!_nextMarkBitMap->isMarked(addr)) {
1011 1011 // We definitely need to mark it, irrespective whether we bail out
1012 1012 // because we're done with marking.
1013 1013 if (_nextMarkBitMap->parMark(addr)) {
1014 1014 if (!concurrent_marking_in_progress() || !_should_gray_objects) {
1015 1015 // If we're done with concurrent marking and we're waiting for
1016 1016 // remark, then we're not pushing anything on the stack.
1017 1017 return;
1018 1018 }
1019 1019
1020 1020 // No OrderAccess:store_load() is needed. It is implicit in the
1021 1021 // CAS done in parMark(addr) above
1022 1022 HeapWord* finger = _finger;
1023 1023
1024 1024 if (addr < finger) {
1025 1025 if (!mark_stack_push(oop(addr))) {
1026 1026 if (verbose_low()) {
1027 1027 gclog_or_tty->print_cr("[global] global stack overflow "
1028 1028 "during parMark");
1029 1029 }
1030 1030 }
1031 1031 }
1032 1032 }
1033 1033 }
1034 1034 }
1035 1035
1036 1036 class CMConcurrentMarkingTask: public AbstractGangTask {
1037 1037 private:
1038 1038 ConcurrentMark* _cm;
1039 1039 ConcurrentMarkThread* _cmt;
1040 1040
1041 1041 public:
1042 1042 void work(int worker_i) {
1043 1043 assert(Thread::current()->is_ConcurrentGC_thread(),
1044 1044 "this should only be done by a conc GC thread");
1045 1045 ResourceMark rm;
1046 1046
1047 1047 double start_vtime = os::elapsedVTime();
1048 1048
1049 1049 ConcurrentGCThread::stsJoin();
1050 1050
1051 1051 assert((size_t) worker_i < _cm->active_tasks(), "invariant");
1052 1052 CMTask* the_task = _cm->task(worker_i);
1053 1053 the_task->record_start_time();
1054 1054 if (!_cm->has_aborted()) {
1055 1055 do {
1056 1056 double start_vtime_sec = os::elapsedVTime();
1057 1057 double start_time_sec = os::elapsedTime();
1058 1058 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
1059 1059
1060 1060 the_task->do_marking_step(mark_step_duration_ms,
1061 1061 true /* do_stealing */,
1062 1062 true /* do_termination */);
1063 1063
1064 1064 double end_time_sec = os::elapsedTime();
1065 1065 double end_vtime_sec = os::elapsedVTime();
1066 1066 double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec;
1067 1067 double elapsed_time_sec = end_time_sec - start_time_sec;
1068 1068 _cm->clear_has_overflown();
1069 1069
1070 1070 bool ret = _cm->do_yield_check(worker_i);
1071 1071
1072 1072 jlong sleep_time_ms;
1073 1073 if (!_cm->has_aborted() && the_task->has_aborted()) {
1074 1074 sleep_time_ms =
1075 1075 (jlong) (elapsed_vtime_sec * _cm->sleep_factor() * 1000.0);
1076 1076 ConcurrentGCThread::stsLeave();
1077 1077 os::sleep(Thread::current(), sleep_time_ms, false);
1078 1078 ConcurrentGCThread::stsJoin();
1079 1079 }
1080 1080 double end_time2_sec = os::elapsedTime();
1081 1081 double elapsed_time2_sec = end_time2_sec - start_time_sec;
1082 1082
1083 1083 #if 0
1084 1084 gclog_or_tty->print_cr("CM: elapsed %1.4lf ms, sleep %1.4lf ms, "
1085 1085 "overhead %1.4lf",
1086 1086 elapsed_vtime_sec * 1000.0, (double) sleep_time_ms,
1087 1087 the_task->conc_overhead(os::elapsedTime()) * 8.0);
1088 1088 gclog_or_tty->print_cr("elapsed time %1.4lf ms, time 2: %1.4lf ms",
1089 1089 elapsed_time_sec * 1000.0, elapsed_time2_sec * 1000.0);
1090 1090 #endif
1091 1091 } while (!_cm->has_aborted() && the_task->has_aborted());
1092 1092 }
1093 1093 the_task->record_end_time();
1094 1094 guarantee(!the_task->has_aborted() || _cm->has_aborted(), "invariant");
1095 1095
1096 1096 ConcurrentGCThread::stsLeave();
1097 1097
1098 1098 double end_vtime = os::elapsedVTime();
1099 1099 _cm->update_accum_task_vtime(worker_i, end_vtime - start_vtime);
1100 1100 }
1101 1101
1102 1102 CMConcurrentMarkingTask(ConcurrentMark* cm,
1103 1103 ConcurrentMarkThread* cmt) :
1104 1104 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { }
1105 1105
1106 1106 ~CMConcurrentMarkingTask() { }
1107 1107 };
1108 1108
1109 1109 void ConcurrentMark::markFromRoots() {
1110 1110 // we might be tempted to assert that:
1111 1111 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
1112 1112 // "inconsistent argument?");
1113 1113 // However that wouldn't be right, because it's possible that
1114 1114 // a safepoint is indeed in progress as a younger generation
1115 1115 // stop-the-world GC happens even as we mark in this generation.
1116 1116
1117 1117 _restart_for_overflow = false;
1118 1118
1119 1119 size_t active_workers = MAX2((size_t) 1, parallel_marking_threads());
1120 1120 force_overflow_conc()->init();
1121 1121 set_phase(active_workers, true /* concurrent */);
1122 1122
1123 1123 CMConcurrentMarkingTask markingTask(this, cmThread());
1124 1124 if (parallel_marking_threads() > 0) {
1125 1125 _parallel_workers->run_task(&markingTask);
|
↓ open down ↓ |
291 lines elided |
↑ open up ↑ |
1126 1126 } else {
1127 1127 markingTask.work(0);
1128 1128 }
1129 1129 print_stats();
1130 1130 }
1131 1131
1132 1132 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
1133 1133 // world is stopped at this checkpoint
1134 1134 assert(SafepointSynchronize::is_at_safepoint(),
1135 1135 "world should be stopped");
1136 +
1136 1137 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1137 1138
1138 1139 // If a full collection has happened, we shouldn't do this.
1139 1140 if (has_aborted()) {
1140 1141 g1h->set_marking_complete(); // So bitmap clearing isn't confused
1141 1142 return;
1142 1143 }
1143 1144
1144 1145 SvcGCMarker sgcm(SvcGCMarker::OTHER);
1145 1146
1146 1147 if (VerifyDuringGC) {
1147 1148 HandleMark hm; // handle scope
1148 1149 gclog_or_tty->print(" VerifyDuringGC:(before)");
1149 1150 Universe::heap()->prepare_for_verify();
1150 1151 Universe::verify(/* allow dirty */ true,
1151 1152 /* silent */ false,
1152 1153 /* option */ VerifyOption_G1UsePrevMarking);
1153 1154 }
1154 1155
1155 1156 G1CollectorPolicy* g1p = g1h->g1_policy();
1156 1157 g1p->record_concurrent_mark_remark_start();
1157 1158
1158 1159 double start = os::elapsedTime();
1159 1160
1160 1161 checkpointRootsFinalWork();
1161 1162
1162 1163 double mark_work_end = os::elapsedTime();
1163 1164
1164 1165 weakRefsWork(clear_all_soft_refs);
1165 1166
1166 1167 if (has_overflown()) {
1167 1168 // Oops. We overflowed. Restart concurrent marking.
1168 1169 _restart_for_overflow = true;
1169 1170 // Clear the flag. We do not need it any more.
1170 1171 clear_has_overflown();
1171 1172 if (G1TraceMarkStackOverflow) {
1172 1173 gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
1173 1174 }
1174 1175 } else {
1175 1176 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1176 1177 // We're done with marking.
1177 1178 // This is the end of the marking cycle, we're expected all
1178 1179 // threads to have SATB queues with active set to true.
1179 1180 satb_mq_set.set_active_all_threads(false, /* new active value */
1180 1181 true /* expected_active */);
1181 1182
1182 1183 if (VerifyDuringGC) {
1183 1184 HandleMark hm; // handle scope
1184 1185 gclog_or_tty->print(" VerifyDuringGC:(after)");
1185 1186 Universe::heap()->prepare_for_verify();
1186 1187 Universe::verify(/* allow dirty */ true,
1187 1188 /* silent */ false,
1188 1189 /* option */ VerifyOption_G1UseNextMarking);
1189 1190 }
1190 1191 assert(!restart_for_overflow(), "sanity");
1191 1192 }
1192 1193
1193 1194 // Reset the marking state if marking completed
1194 1195 if (!restart_for_overflow()) {
1195 1196 set_non_marking_state();
1196 1197 }
1197 1198
1198 1199 #if VERIFY_OBJS_PROCESSED
1199 1200 _scan_obj_cl.objs_processed = 0;
1200 1201 ThreadLocalObjQueue::objs_enqueued = 0;
1201 1202 #endif
1202 1203
1203 1204 // Statistics
1204 1205 double now = os::elapsedTime();
1205 1206 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1206 1207 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1207 1208 _remark_times.add((now - start) * 1000.0);
1208 1209
1209 1210 g1p->record_concurrent_mark_remark_end();
1210 1211 }
1211 1212
1212 1213 #define CARD_BM_TEST_MODE 0
1213 1214
1214 1215 class CalcLiveObjectsClosure: public HeapRegionClosure {
1215 1216
1216 1217 CMBitMapRO* _bm;
1217 1218 ConcurrentMark* _cm;
1218 1219 bool _changed;
1219 1220 bool _yield;
1220 1221 size_t _words_done;
1221 1222 size_t _tot_live;
1222 1223 size_t _tot_used;
1223 1224 size_t _regions_done;
1224 1225 double _start_vtime_sec;
1225 1226
1226 1227 BitMap* _region_bm;
1227 1228 BitMap* _card_bm;
1228 1229 intptr_t _bottom_card_num;
1229 1230 bool _final;
1230 1231
1231 1232 void mark_card_num_range(intptr_t start_card_num, intptr_t last_card_num) {
1232 1233 for (intptr_t i = start_card_num; i <= last_card_num; i++) {
1233 1234 #if CARD_BM_TEST_MODE
1234 1235 guarantee(_card_bm->at(i - _bottom_card_num), "Should already be set.");
1235 1236 #else
1236 1237 _card_bm->par_at_put(i - _bottom_card_num, 1);
1237 1238 #endif
1238 1239 }
1239 1240 }
1240 1241
1241 1242 public:
1242 1243 CalcLiveObjectsClosure(bool final,
1243 1244 CMBitMapRO *bm, ConcurrentMark *cm,
1244 1245 BitMap* region_bm, BitMap* card_bm) :
1245 1246 _bm(bm), _cm(cm), _changed(false), _yield(true),
1246 1247 _words_done(0), _tot_live(0), _tot_used(0),
1247 1248 _region_bm(region_bm), _card_bm(card_bm),_final(final),
1248 1249 _regions_done(0), _start_vtime_sec(0.0)
1249 1250 {
1250 1251 _bottom_card_num =
1251 1252 intptr_t(uintptr_t(G1CollectedHeap::heap()->reserved_region().start()) >>
1252 1253 CardTableModRefBS::card_shift);
1253 1254 }
1254 1255
1255 1256 // It takes a region that's not empty (i.e., it has at least one
1256 1257 // live object in it and sets its corresponding bit on the region
1257 1258 // bitmap to 1. If the region is "starts humongous" it will also set
1258 1259 // to 1 the bits on the region bitmap that correspond to its
1259 1260 // associated "continues humongous" regions.
1260 1261 void set_bit_for_region(HeapRegion* hr) {
1261 1262 assert(!hr->continuesHumongous(), "should have filtered those out");
1262 1263
1263 1264 size_t index = hr->hrs_index();
1264 1265 if (!hr->startsHumongous()) {
1265 1266 // Normal (non-humongous) case: just set the bit.
1266 1267 _region_bm->par_at_put((BitMap::idx_t) index, true);
1267 1268 } else {
1268 1269 // Starts humongous case: calculate how many regions are part of
1269 1270 // this humongous region and then set the bit range. It might
1270 1271 // have been a bit more efficient to look at the object that
1271 1272 // spans these humongous regions to calculate their number from
1272 1273 // the object's size. However, it's a good idea to calculate
1273 1274 // this based on the metadata itself, and not the region
1274 1275 // contents, so that this code is not aware of what goes into
1275 1276 // the humongous regions (in case this changes in the future).
1276 1277 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1277 1278 size_t end_index = index + 1;
1278 1279 while (end_index < g1h->n_regions()) {
1279 1280 HeapRegion* chr = g1h->region_at(end_index);
1280 1281 if (!chr->continuesHumongous()) break;
1281 1282 end_index += 1;
1282 1283 }
1283 1284 _region_bm->par_at_put_range((BitMap::idx_t) index,
1284 1285 (BitMap::idx_t) end_index, true);
1285 1286 }
1286 1287 }
1287 1288
1288 1289 bool doHeapRegion(HeapRegion* hr) {
1289 1290 if (!_final && _regions_done == 0) {
1290 1291 _start_vtime_sec = os::elapsedVTime();
1291 1292 }
1292 1293
1293 1294 if (hr->continuesHumongous()) {
1294 1295 // We will ignore these here and process them when their
1295 1296 // associated "starts humongous" region is processed (see
1296 1297 // set_bit_for_heap_region()). Note that we cannot rely on their
1297 1298 // associated "starts humongous" region to have their bit set to
1298 1299 // 1 since, due to the region chunking in the parallel region
1299 1300 // iteration, a "continues humongous" region might be visited
1300 1301 // before its associated "starts humongous".
1301 1302 return false;
1302 1303 }
1303 1304
1304 1305 HeapWord* nextTop = hr->next_top_at_mark_start();
1305 1306 HeapWord* start = hr->top_at_conc_mark_count();
1306 1307 assert(hr->bottom() <= start && start <= hr->end() &&
1307 1308 hr->bottom() <= nextTop && nextTop <= hr->end() &&
1308 1309 start <= nextTop,
1309 1310 "Preconditions.");
1310 1311 // Otherwise, record the number of word's we'll examine.
1311 1312 size_t words_done = (nextTop - start);
1312 1313 // Find the first marked object at or after "start".
1313 1314 start = _bm->getNextMarkedWordAddress(start, nextTop);
1314 1315 size_t marked_bytes = 0;
1315 1316
1316 1317 // Below, the term "card num" means the result of shifting an address
1317 1318 // by the card shift -- address 0 corresponds to card number 0. One
1318 1319 // must subtract the card num of the bottom of the heap to obtain a
1319 1320 // card table index.
1320 1321 // The first card num of the sequence of live cards currently being
1321 1322 // constructed. -1 ==> no sequence.
1322 1323 intptr_t start_card_num = -1;
1323 1324 // The last card num of the sequence of live cards currently being
1324 1325 // constructed. -1 ==> no sequence.
1325 1326 intptr_t last_card_num = -1;
1326 1327
1327 1328 while (start < nextTop) {
1328 1329 if (_yield && _cm->do_yield_check()) {
1329 1330 // We yielded. It might be for a full collection, in which case
1330 1331 // all bets are off; terminate the traversal.
1331 1332 if (_cm->has_aborted()) {
1332 1333 _changed = false;
1333 1334 return true;
1334 1335 } else {
1335 1336 // Otherwise, it might be a collection pause, and the region
1336 1337 // we're looking at might be in the collection set. We'll
1337 1338 // abandon this region.
1338 1339 return false;
1339 1340 }
1340 1341 }
1341 1342 oop obj = oop(start);
1342 1343 int obj_sz = obj->size();
1343 1344 // The card num of the start of the current object.
1344 1345 intptr_t obj_card_num =
1345 1346 intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
1346 1347
1347 1348 HeapWord* obj_last = start + obj_sz - 1;
1348 1349 intptr_t obj_last_card_num =
1349 1350 intptr_t(uintptr_t(obj_last) >> CardTableModRefBS::card_shift);
1350 1351
1351 1352 if (obj_card_num != last_card_num) {
1352 1353 if (start_card_num == -1) {
1353 1354 assert(last_card_num == -1, "Both or neither.");
1354 1355 start_card_num = obj_card_num;
1355 1356 } else {
1356 1357 assert(last_card_num != -1, "Both or neither.");
1357 1358 assert(obj_card_num >= last_card_num, "Inv");
1358 1359 if ((obj_card_num - last_card_num) > 1) {
1359 1360 // Mark the last run, and start a new one.
1360 1361 mark_card_num_range(start_card_num, last_card_num);
1361 1362 start_card_num = obj_card_num;
1362 1363 }
1363 1364 }
1364 1365 #if CARD_BM_TEST_MODE
1365 1366 /*
1366 1367 gclog_or_tty->print_cr("Setting bits from %d/%d.",
1367 1368 obj_card_num - _bottom_card_num,
1368 1369 obj_last_card_num - _bottom_card_num);
1369 1370 */
1370 1371 for (intptr_t j = obj_card_num; j <= obj_last_card_num; j++) {
1371 1372 _card_bm->par_at_put(j - _bottom_card_num, 1);
1372 1373 }
1373 1374 #endif
1374 1375 }
1375 1376 // In any case, we set the last card num.
1376 1377 last_card_num = obj_last_card_num;
1377 1378
1378 1379 marked_bytes += (size_t)obj_sz * HeapWordSize;
1379 1380 // Find the next marked object after this one.
1380 1381 start = _bm->getNextMarkedWordAddress(start + 1, nextTop);
1381 1382 _changed = true;
1382 1383 }
1383 1384 // Handle the last range, if any.
1384 1385 if (start_card_num != -1) {
1385 1386 mark_card_num_range(start_card_num, last_card_num);
1386 1387 }
1387 1388 if (_final) {
1388 1389 // Mark the allocated-since-marking portion...
1389 1390 HeapWord* tp = hr->top();
1390 1391 if (nextTop < tp) {
1391 1392 start_card_num =
1392 1393 intptr_t(uintptr_t(nextTop) >> CardTableModRefBS::card_shift);
1393 1394 last_card_num =
1394 1395 intptr_t(uintptr_t(tp) >> CardTableModRefBS::card_shift);
1395 1396 mark_card_num_range(start_card_num, last_card_num);
1396 1397 // This definitely means the region has live objects.
1397 1398 set_bit_for_region(hr);
1398 1399 }
1399 1400 }
1400 1401
1401 1402 hr->add_to_marked_bytes(marked_bytes);
1402 1403 // Update the live region bitmap.
1403 1404 if (marked_bytes > 0) {
1404 1405 set_bit_for_region(hr);
1405 1406 }
1406 1407 hr->set_top_at_conc_mark_count(nextTop);
1407 1408 _tot_live += hr->next_live_bytes();
1408 1409 _tot_used += hr->used();
1409 1410 _words_done = words_done;
1410 1411
1411 1412 if (!_final) {
1412 1413 ++_regions_done;
1413 1414 if (_regions_done % 10 == 0) {
1414 1415 double end_vtime_sec = os::elapsedVTime();
1415 1416 double elapsed_vtime_sec = end_vtime_sec - _start_vtime_sec;
1416 1417 if (elapsed_vtime_sec > (10.0 / 1000.0)) {
1417 1418 jlong sleep_time_ms =
1418 1419 (jlong) (elapsed_vtime_sec * _cm->cleanup_sleep_factor() * 1000.0);
1419 1420 os::sleep(Thread::current(), sleep_time_ms, false);
1420 1421 _start_vtime_sec = end_vtime_sec;
1421 1422 }
1422 1423 }
1423 1424 }
1424 1425
1425 1426 return false;
1426 1427 }
1427 1428
1428 1429 bool changed() { return _changed; }
1429 1430 void reset() { _changed = false; _words_done = 0; }
1430 1431 void no_yield() { _yield = false; }
1431 1432 size_t words_done() { return _words_done; }
1432 1433 size_t tot_live() { return _tot_live; }
1433 1434 size_t tot_used() { return _tot_used; }
1434 1435 };
1435 1436
1436 1437
1437 1438 void ConcurrentMark::calcDesiredRegions() {
1438 1439 _region_bm.clear();
1439 1440 _card_bm.clear();
1440 1441 CalcLiveObjectsClosure calccl(false /*final*/,
1441 1442 nextMarkBitMap(), this,
1442 1443 &_region_bm, &_card_bm);
1443 1444 G1CollectedHeap *g1h = G1CollectedHeap::heap();
1444 1445 g1h->heap_region_iterate(&calccl);
1445 1446
1446 1447 do {
1447 1448 calccl.reset();
1448 1449 g1h->heap_region_iterate(&calccl);
1449 1450 } while (calccl.changed());
1450 1451 }
1451 1452
1452 1453 class G1ParFinalCountTask: public AbstractGangTask {
1453 1454 protected:
1454 1455 G1CollectedHeap* _g1h;
1455 1456 CMBitMap* _bm;
1456 1457 size_t _n_workers;
1457 1458 size_t *_live_bytes;
1458 1459 size_t *_used_bytes;
1459 1460 BitMap* _region_bm;
1460 1461 BitMap* _card_bm;
1461 1462 public:
1462 1463 G1ParFinalCountTask(G1CollectedHeap* g1h, CMBitMap* bm,
1463 1464 BitMap* region_bm, BitMap* card_bm)
1464 1465 : AbstractGangTask("G1 final counting"), _g1h(g1h),
1465 1466 _bm(bm), _region_bm(region_bm), _card_bm(card_bm) {
1466 1467 if (ParallelGCThreads > 0) {
1467 1468 _n_workers = _g1h->workers()->total_workers();
1468 1469 } else {
1469 1470 _n_workers = 1;
1470 1471 }
1471 1472 _live_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers);
1472 1473 _used_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers);
1473 1474 }
1474 1475
1475 1476 ~G1ParFinalCountTask() {
1476 1477 FREE_C_HEAP_ARRAY(size_t, _live_bytes);
1477 1478 FREE_C_HEAP_ARRAY(size_t, _used_bytes);
1478 1479 }
1479 1480
1480 1481 void work(int i) {
1481 1482 CalcLiveObjectsClosure calccl(true /*final*/,
1482 1483 _bm, _g1h->concurrent_mark(),
1483 1484 _region_bm, _card_bm);
1484 1485 calccl.no_yield();
1485 1486 if (G1CollectedHeap::use_parallel_gc_threads()) {
1486 1487 _g1h->heap_region_par_iterate_chunked(&calccl, i,
1487 1488 HeapRegion::FinalCountClaimValue);
1488 1489 } else {
1489 1490 _g1h->heap_region_iterate(&calccl);
1490 1491 }
1491 1492 assert(calccl.complete(), "Shouldn't have yielded!");
1492 1493
1493 1494 assert((size_t) i < _n_workers, "invariant");
1494 1495 _live_bytes[i] = calccl.tot_live();
1495 1496 _used_bytes[i] = calccl.tot_used();
1496 1497 }
1497 1498 size_t live_bytes() {
1498 1499 size_t live_bytes = 0;
1499 1500 for (size_t i = 0; i < _n_workers; ++i)
1500 1501 live_bytes += _live_bytes[i];
1501 1502 return live_bytes;
1502 1503 }
1503 1504 size_t used_bytes() {
1504 1505 size_t used_bytes = 0;
1505 1506 for (size_t i = 0; i < _n_workers; ++i)
1506 1507 used_bytes += _used_bytes[i];
1507 1508 return used_bytes;
1508 1509 }
1509 1510 };
1510 1511
1511 1512 class G1ParNoteEndTask;
1512 1513
1513 1514 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1514 1515 G1CollectedHeap* _g1;
1515 1516 int _worker_num;
1516 1517 size_t _max_live_bytes;
1517 1518 size_t _regions_claimed;
1518 1519 size_t _freed_bytes;
1519 1520 FreeRegionList* _local_cleanup_list;
1520 1521 HumongousRegionSet* _humongous_proxy_set;
1521 1522 HRRSCleanupTask* _hrrs_cleanup_task;
1522 1523 double _claimed_region_time;
1523 1524 double _max_region_time;
1524 1525
1525 1526 public:
1526 1527 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1527 1528 int worker_num,
1528 1529 FreeRegionList* local_cleanup_list,
1529 1530 HumongousRegionSet* humongous_proxy_set,
1530 1531 HRRSCleanupTask* hrrs_cleanup_task);
1531 1532 size_t freed_bytes() { return _freed_bytes; }
1532 1533
1533 1534 bool doHeapRegion(HeapRegion *r);
1534 1535
1535 1536 size_t max_live_bytes() { return _max_live_bytes; }
1536 1537 size_t regions_claimed() { return _regions_claimed; }
1537 1538 double claimed_region_time_sec() { return _claimed_region_time; }
1538 1539 double max_region_time_sec() { return _max_region_time; }
1539 1540 };
1540 1541
1541 1542 class G1ParNoteEndTask: public AbstractGangTask {
1542 1543 friend class G1NoteEndOfConcMarkClosure;
1543 1544
1544 1545 protected:
1545 1546 G1CollectedHeap* _g1h;
1546 1547 size_t _max_live_bytes;
1547 1548 size_t _freed_bytes;
1548 1549 FreeRegionList* _cleanup_list;
1549 1550
1550 1551 public:
1551 1552 G1ParNoteEndTask(G1CollectedHeap* g1h,
1552 1553 FreeRegionList* cleanup_list) :
1553 1554 AbstractGangTask("G1 note end"), _g1h(g1h),
1554 1555 _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list) { }
1555 1556
1556 1557 void work(int i) {
1557 1558 double start = os::elapsedTime();
1558 1559 FreeRegionList local_cleanup_list("Local Cleanup List");
1559 1560 HumongousRegionSet humongous_proxy_set("Local Cleanup Humongous Proxy Set");
1560 1561 HRRSCleanupTask hrrs_cleanup_task;
1561 1562 G1NoteEndOfConcMarkClosure g1_note_end(_g1h, i, &local_cleanup_list,
1562 1563 &humongous_proxy_set,
1563 1564 &hrrs_cleanup_task);
1564 1565 if (G1CollectedHeap::use_parallel_gc_threads()) {
1565 1566 _g1h->heap_region_par_iterate_chunked(&g1_note_end, i,
1566 1567 HeapRegion::NoteEndClaimValue);
1567 1568 } else {
1568 1569 _g1h->heap_region_iterate(&g1_note_end);
1569 1570 }
1570 1571 assert(g1_note_end.complete(), "Shouldn't have yielded!");
1571 1572
1572 1573 // Now update the lists
1573 1574 _g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(),
1574 1575 NULL /* free_list */,
1575 1576 &humongous_proxy_set,
1576 1577 true /* par */);
1577 1578 {
1578 1579 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1579 1580 _max_live_bytes += g1_note_end.max_live_bytes();
1580 1581 _freed_bytes += g1_note_end.freed_bytes();
1581 1582
1582 1583 // If we iterate over the global cleanup list at the end of
1583 1584 // cleanup to do this printing we will not guarantee to only
1584 1585 // generate output for the newly-reclaimed regions (the list
1585 1586 // might not be empty at the beginning of cleanup; we might
1586 1587 // still be working on its previous contents). So we do the
1587 1588 // printing here, before we append the new regions to the global
1588 1589 // cleanup list.
1589 1590
1590 1591 G1HRPrinter* hr_printer = _g1h->hr_printer();
1591 1592 if (hr_printer->is_active()) {
1592 1593 HeapRegionLinkedListIterator iter(&local_cleanup_list);
1593 1594 while (iter.more_available()) {
1594 1595 HeapRegion* hr = iter.get_next();
1595 1596 hr_printer->cleanup(hr);
1596 1597 }
1597 1598 }
1598 1599
1599 1600 _cleanup_list->add_as_tail(&local_cleanup_list);
1600 1601 assert(local_cleanup_list.is_empty(), "post-condition");
1601 1602
1602 1603 HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task);
1603 1604 }
1604 1605 double end = os::elapsedTime();
1605 1606 if (G1PrintParCleanupStats) {
1606 1607 gclog_or_tty->print(" Worker thread %d [%8.3f..%8.3f = %8.3f ms] "
1607 1608 "claimed %d regions (tot = %8.3f ms, max = %8.3f ms).\n",
1608 1609 i, start, end, (end-start)*1000.0,
1609 1610 g1_note_end.regions_claimed(),
1610 1611 g1_note_end.claimed_region_time_sec()*1000.0,
1611 1612 g1_note_end.max_region_time_sec()*1000.0);
1612 1613 }
1613 1614 }
1614 1615 size_t max_live_bytes() { return _max_live_bytes; }
1615 1616 size_t freed_bytes() { return _freed_bytes; }
1616 1617 };
1617 1618
1618 1619 class G1ParScrubRemSetTask: public AbstractGangTask {
1619 1620 protected:
1620 1621 G1RemSet* _g1rs;
1621 1622 BitMap* _region_bm;
1622 1623 BitMap* _card_bm;
1623 1624 public:
1624 1625 G1ParScrubRemSetTask(G1CollectedHeap* g1h,
1625 1626 BitMap* region_bm, BitMap* card_bm) :
1626 1627 AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()),
1627 1628 _region_bm(region_bm), _card_bm(card_bm)
1628 1629 {}
1629 1630
1630 1631 void work(int i) {
1631 1632 if (G1CollectedHeap::use_parallel_gc_threads()) {
1632 1633 _g1rs->scrub_par(_region_bm, _card_bm, i,
1633 1634 HeapRegion::ScrubRemSetClaimValue);
1634 1635 } else {
1635 1636 _g1rs->scrub(_region_bm, _card_bm);
1636 1637 }
1637 1638 }
1638 1639
1639 1640 };
1640 1641
1641 1642 G1NoteEndOfConcMarkClosure::
1642 1643 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1643 1644 int worker_num,
1644 1645 FreeRegionList* local_cleanup_list,
1645 1646 HumongousRegionSet* humongous_proxy_set,
1646 1647 HRRSCleanupTask* hrrs_cleanup_task)
1647 1648 : _g1(g1), _worker_num(worker_num),
1648 1649 _max_live_bytes(0), _regions_claimed(0),
1649 1650 _freed_bytes(0),
1650 1651 _claimed_region_time(0.0), _max_region_time(0.0),
1651 1652 _local_cleanup_list(local_cleanup_list),
1652 1653 _humongous_proxy_set(humongous_proxy_set),
1653 1654 _hrrs_cleanup_task(hrrs_cleanup_task) { }
1654 1655
1655 1656 bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *hr) {
1656 1657 // We use a claim value of zero here because all regions
1657 1658 // were claimed with value 1 in the FinalCount task.
1658 1659 hr->reset_gc_time_stamp();
1659 1660 if (!hr->continuesHumongous()) {
1660 1661 double start = os::elapsedTime();
1661 1662 _regions_claimed++;
1662 1663 hr->note_end_of_marking();
1663 1664 _max_live_bytes += hr->max_live_bytes();
1664 1665 _g1->free_region_if_empty(hr,
1665 1666 &_freed_bytes,
1666 1667 _local_cleanup_list,
1667 1668 _humongous_proxy_set,
1668 1669 _hrrs_cleanup_task,
1669 1670 true /* par */);
1670 1671 double region_time = (os::elapsedTime() - start);
1671 1672 _claimed_region_time += region_time;
1672 1673 if (region_time > _max_region_time) {
1673 1674 _max_region_time = region_time;
1674 1675 }
1675 1676 }
1676 1677 return false;
1677 1678 }
1678 1679
1679 1680 void ConcurrentMark::cleanup() {
1680 1681 // world is stopped at this checkpoint
1681 1682 assert(SafepointSynchronize::is_at_safepoint(),
1682 1683 "world should be stopped");
1683 1684 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1684 1685
1685 1686 // If a full collection has happened, we shouldn't do this.
1686 1687 if (has_aborted()) {
1687 1688 g1h->set_marking_complete(); // So bitmap clearing isn't confused
1688 1689 return;
1689 1690 }
1690 1691
1691 1692 g1h->verify_region_sets_optional();
1692 1693
1693 1694 if (VerifyDuringGC) {
1694 1695 HandleMark hm; // handle scope
1695 1696 gclog_or_tty->print(" VerifyDuringGC:(before)");
1696 1697 Universe::heap()->prepare_for_verify();
1697 1698 Universe::verify(/* allow dirty */ true,
1698 1699 /* silent */ false,
1699 1700 /* option */ VerifyOption_G1UsePrevMarking);
1700 1701 }
1701 1702
1702 1703 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
1703 1704 g1p->record_concurrent_mark_cleanup_start();
1704 1705
1705 1706 double start = os::elapsedTime();
1706 1707
1707 1708 HeapRegionRemSet::reset_for_cleanup_tasks();
1708 1709
1709 1710 // Do counting once more with the world stopped for good measure.
1710 1711 G1ParFinalCountTask g1_par_count_task(g1h, nextMarkBitMap(),
1711 1712 &_region_bm, &_card_bm);
1712 1713 if (G1CollectedHeap::use_parallel_gc_threads()) {
1713 1714 assert(g1h->check_heap_region_claim_values(
1714 1715 HeapRegion::InitialClaimValue),
1715 1716 "sanity check");
1716 1717
1717 1718 int n_workers = g1h->workers()->total_workers();
1718 1719 g1h->set_par_threads(n_workers);
1719 1720 g1h->workers()->run_task(&g1_par_count_task);
1720 1721 g1h->set_par_threads(0);
1721 1722
1722 1723 assert(g1h->check_heap_region_claim_values(
1723 1724 HeapRegion::FinalCountClaimValue),
1724 1725 "sanity check");
1725 1726 } else {
1726 1727 g1_par_count_task.work(0);
1727 1728 }
1728 1729
1729 1730 size_t known_garbage_bytes =
1730 1731 g1_par_count_task.used_bytes() - g1_par_count_task.live_bytes();
1731 1732 g1p->set_known_garbage_bytes(known_garbage_bytes);
1732 1733
1733 1734 size_t start_used_bytes = g1h->used();
1734 1735 _at_least_one_mark_complete = true;
1735 1736 g1h->set_marking_complete();
1736 1737
1737 1738 ergo_verbose4(ErgoConcCycles,
1738 1739 "finish cleanup",
1739 1740 ergo_format_byte("occupancy")
1740 1741 ergo_format_byte("capacity")
1741 1742 ergo_format_byte_perc("known garbage"),
1742 1743 start_used_bytes, g1h->capacity(),
1743 1744 known_garbage_bytes,
1744 1745 ((double) known_garbage_bytes / (double) g1h->capacity()) * 100.0);
1745 1746
1746 1747 double count_end = os::elapsedTime();
1747 1748 double this_final_counting_time = (count_end - start);
1748 1749 if (G1PrintParCleanupStats) {
1749 1750 gclog_or_tty->print_cr("Cleanup:");
1750 1751 gclog_or_tty->print_cr(" Finalize counting: %8.3f ms",
1751 1752 this_final_counting_time*1000.0);
1752 1753 }
1753 1754 _total_counting_time += this_final_counting_time;
1754 1755
1755 1756 if (G1PrintRegionLivenessInfo) {
1756 1757 G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking");
1757 1758 _g1h->heap_region_iterate(&cl);
1758 1759 }
1759 1760
1760 1761 // Install newly created mark bitMap as "prev".
1761 1762 swapMarkBitMaps();
1762 1763
1763 1764 g1h->reset_gc_time_stamp();
1764 1765
1765 1766 // Note end of marking in all heap regions.
1766 1767 double note_end_start = os::elapsedTime();
1767 1768 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list);
1768 1769 if (G1CollectedHeap::use_parallel_gc_threads()) {
1769 1770 int n_workers = g1h->workers()->total_workers();
1770 1771 g1h->set_par_threads(n_workers);
1771 1772 g1h->workers()->run_task(&g1_par_note_end_task);
1772 1773 g1h->set_par_threads(0);
1773 1774
1774 1775 assert(g1h->check_heap_region_claim_values(HeapRegion::NoteEndClaimValue),
1775 1776 "sanity check");
1776 1777 } else {
1777 1778 g1_par_note_end_task.work(0);
1778 1779 }
1779 1780
1780 1781 if (!cleanup_list_is_empty()) {
1781 1782 // The cleanup list is not empty, so we'll have to process it
1782 1783 // concurrently. Notify anyone else that might be wanting free
1783 1784 // regions that there will be more free regions coming soon.
1784 1785 g1h->set_free_regions_coming();
1785 1786 }
1786 1787 double note_end_end = os::elapsedTime();
1787 1788 if (G1PrintParCleanupStats) {
1788 1789 gclog_or_tty->print_cr(" note end of marking: %8.3f ms.",
1789 1790 (note_end_end - note_end_start)*1000.0);
1790 1791 }
1791 1792
1792 1793
1793 1794 // call below, since it affects the metric by which we sort the heap
1794 1795 // regions.
1795 1796 if (G1ScrubRemSets) {
1796 1797 double rs_scrub_start = os::elapsedTime();
1797 1798 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm);
1798 1799 if (G1CollectedHeap::use_parallel_gc_threads()) {
1799 1800 int n_workers = g1h->workers()->total_workers();
1800 1801 g1h->set_par_threads(n_workers);
1801 1802 g1h->workers()->run_task(&g1_par_scrub_rs_task);
1802 1803 g1h->set_par_threads(0);
1803 1804
1804 1805 assert(g1h->check_heap_region_claim_values(
1805 1806 HeapRegion::ScrubRemSetClaimValue),
1806 1807 "sanity check");
1807 1808 } else {
1808 1809 g1_par_scrub_rs_task.work(0);
1809 1810 }
1810 1811
1811 1812 double rs_scrub_end = os::elapsedTime();
1812 1813 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
1813 1814 _total_rs_scrub_time += this_rs_scrub_time;
1814 1815 }
1815 1816
1816 1817 // this will also free any regions totally full of garbage objects,
1817 1818 // and sort the regions.
1818 1819 g1h->g1_policy()->record_concurrent_mark_cleanup_end(
1819 1820 g1_par_note_end_task.freed_bytes(),
1820 1821 g1_par_note_end_task.max_live_bytes());
1821 1822
1822 1823 // Statistics.
1823 1824 double end = os::elapsedTime();
1824 1825 _cleanup_times.add((end - start) * 1000.0);
1825 1826
1826 1827 // G1CollectedHeap::heap()->print();
1827 1828 // gclog_or_tty->print_cr("HEAP GC TIME STAMP : %d",
1828 1829 // G1CollectedHeap::heap()->get_gc_time_stamp());
1829 1830
|
↓ open down ↓ |
684 lines elided |
↑ open up ↑ |
1830 1831 if (PrintGC || PrintGCDetails) {
1831 1832 g1h->print_size_transition(gclog_or_tty,
1832 1833 start_used_bytes,
1833 1834 g1h->used(),
1834 1835 g1h->capacity());
1835 1836 }
1836 1837
1837 1838 size_t cleaned_up_bytes = start_used_bytes - g1h->used();
1838 1839 g1p->decrease_known_garbage_bytes(cleaned_up_bytes);
1839 1840
1841 + // Clean up will have freed any regions completely full of garbage.
1842 + // Update the soft reference policy with the new heap occupancy.
1843 + Universe::update_heap_info_at_gc();
1844 +
1840 1845 // We need to make this be a "collection" so any collection pause that
1841 1846 // races with it goes around and waits for completeCleanup to finish.
1842 1847 g1h->increment_total_collections();
1843 1848
1844 1849 if (VerifyDuringGC) {
1845 1850 HandleMark hm; // handle scope
1846 1851 gclog_or_tty->print(" VerifyDuringGC:(after)");
1847 1852 Universe::heap()->prepare_for_verify();
1848 1853 Universe::verify(/* allow dirty */ true,
1849 1854 /* silent */ false,
1850 1855 /* option */ VerifyOption_G1UsePrevMarking);
1851 1856 }
1852 1857
1853 1858 g1h->verify_region_sets_optional();
1854 1859 }
1855 1860
1856 1861 void ConcurrentMark::completeCleanup() {
1857 1862 if (has_aborted()) return;
1858 1863
1859 1864 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1860 1865
1861 1866 _cleanup_list.verify_optional();
1862 1867 FreeRegionList tmp_free_list("Tmp Free List");
1863 1868
1864 1869 if (G1ConcRegionFreeingVerbose) {
1865 1870 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1866 1871 "cleanup list has "SIZE_FORMAT" entries",
1867 1872 _cleanup_list.length());
1868 1873 }
1869 1874
1870 1875 // Noone else should be accessing the _cleanup_list at this point,
1871 1876 // so it's not necessary to take any locks
1872 1877 while (!_cleanup_list.is_empty()) {
1873 1878 HeapRegion* hr = _cleanup_list.remove_head();
1874 1879 assert(hr != NULL, "the list was not empty");
1875 1880 hr->par_clear();
1876 1881 tmp_free_list.add_as_tail(hr);
1877 1882
1878 1883 // Instead of adding one region at a time to the secondary_free_list,
1879 1884 // we accumulate them in the local list and move them a few at a
1880 1885 // time. This also cuts down on the number of notify_all() calls
1881 1886 // we do during this process. We'll also append the local list when
1882 1887 // _cleanup_list is empty (which means we just removed the last
1883 1888 // region from the _cleanup_list).
1884 1889 if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1885 1890 _cleanup_list.is_empty()) {
1886 1891 if (G1ConcRegionFreeingVerbose) {
1887 1892 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
1888 1893 "appending "SIZE_FORMAT" entries to the "
1889 1894 "secondary_free_list, clean list still has "
1890 1895 SIZE_FORMAT" entries",
1891 1896 tmp_free_list.length(),
1892 1897 _cleanup_list.length());
1893 1898 }
1894 1899
1895 1900 {
1896 1901 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1897 1902 g1h->secondary_free_list_add_as_tail(&tmp_free_list);
1898 1903 SecondaryFreeList_lock->notify_all();
1899 1904 }
1900 1905
1901 1906 if (G1StressConcRegionFreeing) {
1902 1907 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1903 1908 os::sleep(Thread::current(), (jlong) 1, false);
1904 1909 }
1905 1910 }
1906 1911 }
1907 1912 }
1908 1913 assert(tmp_free_list.is_empty(), "post-condition");
1909 1914 }
1910 1915
1911 1916 // Support closures for reference procssing in G1
1912 1917
1913 1918 bool G1CMIsAliveClosure::do_object_b(oop obj) {
1914 1919 HeapWord* addr = (HeapWord*)obj;
1915 1920 return addr != NULL &&
1916 1921 (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj));
1917 1922 }
1918 1923
1919 1924 class G1CMKeepAliveClosure: public OopClosure {
1920 1925 G1CollectedHeap* _g1;
1921 1926 ConcurrentMark* _cm;
1922 1927 CMBitMap* _bitMap;
1923 1928 public:
1924 1929 G1CMKeepAliveClosure(G1CollectedHeap* g1, ConcurrentMark* cm,
1925 1930 CMBitMap* bitMap) :
1926 1931 _g1(g1), _cm(cm),
1927 1932 _bitMap(bitMap) {}
1928 1933
1929 1934 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
1930 1935 virtual void do_oop( oop* p) { do_oop_work(p); }
1931 1936
1932 1937 template <class T> void do_oop_work(T* p) {
1933 1938 oop obj = oopDesc::load_decode_heap_oop(p);
1934 1939 HeapWord* addr = (HeapWord*)obj;
1935 1940
1936 1941 if (_cm->verbose_high()) {
1937 1942 gclog_or_tty->print_cr("\t[0] we're looking at location "
1938 1943 "*"PTR_FORMAT" = "PTR_FORMAT,
1939 1944 p, (void*) obj);
1940 1945 }
1941 1946
1942 1947 if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(obj)) {
1943 1948 _bitMap->mark(addr);
1944 1949 _cm->mark_stack_push(obj);
1945 1950 }
1946 1951 }
1947 1952 };
1948 1953
1949 1954 class G1CMDrainMarkingStackClosure: public VoidClosure {
1950 1955 CMMarkStack* _markStack;
1951 1956 CMBitMap* _bitMap;
1952 1957 G1CMKeepAliveClosure* _oopClosure;
1953 1958 public:
1954 1959 G1CMDrainMarkingStackClosure(CMBitMap* bitMap, CMMarkStack* markStack,
1955 1960 G1CMKeepAliveClosure* oopClosure) :
1956 1961 _bitMap(bitMap),
1957 1962 _markStack(markStack),
1958 1963 _oopClosure(oopClosure)
1959 1964 {}
1960 1965
1961 1966 void do_void() {
1962 1967 _markStack->drain((OopClosure*)_oopClosure, _bitMap, false);
1963 1968 }
1964 1969 };
1965 1970
1966 1971 // 'Keep Alive' closure used by parallel reference processing.
1967 1972 // An instance of this closure is used in the parallel reference processing
1968 1973 // code rather than an instance of G1CMKeepAliveClosure. We could have used
1969 1974 // the G1CMKeepAliveClosure as it is MT-safe. Also reference objects are
1970 1975 // placed on to discovered ref lists once so we can mark and push with no
1971 1976 // need to check whether the object has already been marked. Using the
1972 1977 // G1CMKeepAliveClosure would mean, however, having all the worker threads
1973 1978 // operating on the global mark stack. This means that an individual
1974 1979 // worker would be doing lock-free pushes while it processes its own
1975 1980 // discovered ref list followed by drain call. If the discovered ref lists
1976 1981 // are unbalanced then this could cause interference with the other
1977 1982 // workers. Using a CMTask (and its embedded local data structures)
1978 1983 // avoids that potential interference.
1979 1984 class G1CMParKeepAliveAndDrainClosure: public OopClosure {
1980 1985 ConcurrentMark* _cm;
1981 1986 CMTask* _task;
1982 1987 CMBitMap* _bitMap;
1983 1988 int _ref_counter_limit;
1984 1989 int _ref_counter;
1985 1990 public:
1986 1991 G1CMParKeepAliveAndDrainClosure(ConcurrentMark* cm,
1987 1992 CMTask* task,
1988 1993 CMBitMap* bitMap) :
1989 1994 _cm(cm), _task(task), _bitMap(bitMap),
1990 1995 _ref_counter_limit(G1RefProcDrainInterval)
1991 1996 {
1992 1997 assert(_ref_counter_limit > 0, "sanity");
1993 1998 _ref_counter = _ref_counter_limit;
1994 1999 }
1995 2000
1996 2001 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
1997 2002 virtual void do_oop( oop* p) { do_oop_work(p); }
1998 2003
1999 2004 template <class T> void do_oop_work(T* p) {
2000 2005 if (!_cm->has_overflown()) {
2001 2006 oop obj = oopDesc::load_decode_heap_oop(p);
2002 2007 if (_cm->verbose_high()) {
2003 2008 gclog_or_tty->print_cr("\t[%d] we're looking at location "
2004 2009 "*"PTR_FORMAT" = "PTR_FORMAT,
2005 2010 _task->task_id(), p, (void*) obj);
2006 2011 }
2007 2012
2008 2013 _task->deal_with_reference(obj);
2009 2014 _ref_counter--;
2010 2015
2011 2016 if (_ref_counter == 0) {
2012 2017 // We have dealt with _ref_counter_limit references, pushing them and objects
2013 2018 // reachable from them on to the local stack (and possibly the global stack).
2014 2019 // Call do_marking_step() to process these entries. We call the routine in a
2015 2020 // loop, which we'll exit if there's nothing more to do (i.e. we're done
2016 2021 // with the entries that we've pushed as a result of the deal_with_reference
2017 2022 // calls above) or we overflow.
2018 2023 // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() flag
2019 2024 // while there may still be some work to do. (See the comment at the
2020 2025 // beginning of CMTask::do_marking_step() for those conditions - one of which
2021 2026 // is reaching the specified time target.) It is only when
2022 2027 // CMTask::do_marking_step() returns without setting the has_aborted() flag
2023 2028 // that the marking has completed.
2024 2029 do {
2025 2030 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
2026 2031 _task->do_marking_step(mark_step_duration_ms,
2027 2032 false /* do_stealing */,
2028 2033 false /* do_termination */);
2029 2034 } while (_task->has_aborted() && !_cm->has_overflown());
2030 2035 _ref_counter = _ref_counter_limit;
2031 2036 }
2032 2037 } else {
2033 2038 if (_cm->verbose_high()) {
2034 2039 gclog_or_tty->print_cr("\t[%d] CM Overflow", _task->task_id());
2035 2040 }
2036 2041 }
2037 2042 }
2038 2043 };
2039 2044
2040 2045 class G1CMParDrainMarkingStackClosure: public VoidClosure {
2041 2046 ConcurrentMark* _cm;
2042 2047 CMTask* _task;
2043 2048 public:
2044 2049 G1CMParDrainMarkingStackClosure(ConcurrentMark* cm, CMTask* task) :
2045 2050 _cm(cm), _task(task)
2046 2051 {}
2047 2052
2048 2053 void do_void() {
2049 2054 do {
2050 2055 if (_cm->verbose_high()) {
2051 2056 gclog_or_tty->print_cr("\t[%d] Drain: Calling do marking_step",
2052 2057 _task->task_id());
2053 2058 }
2054 2059
2055 2060 // We call CMTask::do_marking_step() to completely drain the local and
2056 2061 // global marking stacks. The routine is called in a loop, which we'll
2057 2062 // exit if there's nothing more to do (i.e. we'completely drained the
2058 2063 // entries that were pushed as a result of applying the
2059 2064 // G1CMParKeepAliveAndDrainClosure to the entries on the discovered ref
2060 2065 // lists above) or we overflow the global marking stack.
2061 2066 // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() flag
2062 2067 // while there may still be some work to do. (See the comment at the
2063 2068 // beginning of CMTask::do_marking_step() for those conditions - one of which
2064 2069 // is reaching the specified time target.) It is only when
|
↓ open down ↓ |
215 lines elided |
↑ open up ↑ |
2065 2070 // CMTask::do_marking_step() returns without setting the has_aborted() flag
2066 2071 // that the marking has completed.
2067 2072
2068 2073 _task->do_marking_step(1000000000.0 /* something very large */,
2069 2074 true /* do_stealing */,
2070 2075 true /* do_termination */);
2071 2076 } while (_task->has_aborted() && !_cm->has_overflown());
2072 2077 }
2073 2078 };
2074 2079
2075 -// Implementation of AbstractRefProcTaskExecutor for G1
2076 -class G1RefProcTaskExecutor: public AbstractRefProcTaskExecutor {
2080 +// Implementation of AbstractRefProcTaskExecutor for parallel
2081 +// reference processing at the end of G1 concurrent marking
2082 +
2083 +class G1CMRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
2077 2084 private:
2078 2085 G1CollectedHeap* _g1h;
2079 2086 ConcurrentMark* _cm;
2080 2087 CMBitMap* _bitmap;
2081 2088 WorkGang* _workers;
2082 2089 int _active_workers;
2083 2090
2084 2091 public:
2085 - G1RefProcTaskExecutor(G1CollectedHeap* g1h,
2092 + G1CMRefProcTaskExecutor(G1CollectedHeap* g1h,
2086 2093 ConcurrentMark* cm,
2087 2094 CMBitMap* bitmap,
2088 2095 WorkGang* workers,
2089 2096 int n_workers) :
2090 2097 _g1h(g1h), _cm(cm), _bitmap(bitmap),
2091 2098 _workers(workers), _active_workers(n_workers)
2092 2099 { }
2093 2100
2094 2101 // Executes the given task using concurrent marking worker threads.
2095 2102 virtual void execute(ProcessTask& task);
2096 2103 virtual void execute(EnqueueTask& task);
2097 2104 };
2098 2105
2099 -class G1RefProcTaskProxy: public AbstractGangTask {
2106 +class G1CMRefProcTaskProxy: public AbstractGangTask {
2100 2107 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
2101 2108 ProcessTask& _proc_task;
2102 2109 G1CollectedHeap* _g1h;
2103 2110 ConcurrentMark* _cm;
2104 2111 CMBitMap* _bitmap;
2105 2112
2106 2113 public:
2107 - G1RefProcTaskProxy(ProcessTask& proc_task,
2114 + G1CMRefProcTaskProxy(ProcessTask& proc_task,
2108 2115 G1CollectedHeap* g1h,
2109 2116 ConcurrentMark* cm,
2110 2117 CMBitMap* bitmap) :
2111 2118 AbstractGangTask("Process reference objects in parallel"),
2112 2119 _proc_task(proc_task), _g1h(g1h), _cm(cm), _bitmap(bitmap)
2113 2120 {}
2114 2121
2115 2122 virtual void work(int i) {
2116 2123 CMTask* marking_task = _cm->task(i);
2117 2124 G1CMIsAliveClosure g1_is_alive(_g1h);
2118 2125 G1CMParKeepAliveAndDrainClosure g1_par_keep_alive(_cm, marking_task, _bitmap);
2119 2126 G1CMParDrainMarkingStackClosure g1_par_drain(_cm, marking_task);
2120 2127
2121 2128 _proc_task.work(i, g1_is_alive, g1_par_keep_alive, g1_par_drain);
2122 2129 }
2123 2130 };
2124 2131
2125 -void G1RefProcTaskExecutor::execute(ProcessTask& proc_task) {
2132 +void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) {
2126 2133 assert(_workers != NULL, "Need parallel worker threads.");
2127 2134
2128 - G1RefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm, _bitmap);
2135 + G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm, _bitmap);
2129 2136
2130 2137 // We need to reset the phase for each task execution so that
2131 2138 // the termination protocol of CMTask::do_marking_step works.
2132 2139 _cm->set_phase(_active_workers, false /* concurrent */);
2133 2140 _g1h->set_par_threads(_active_workers);
2134 2141 _workers->run_task(&proc_task_proxy);
2135 2142 _g1h->set_par_threads(0);
2136 2143 }
2137 2144
2138 -class G1RefEnqueueTaskProxy: public AbstractGangTask {
2145 +class G1CMRefEnqueueTaskProxy: public AbstractGangTask {
2139 2146 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
2140 2147 EnqueueTask& _enq_task;
2141 2148
2142 2149 public:
2143 - G1RefEnqueueTaskProxy(EnqueueTask& enq_task) :
2150 + G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) :
2144 2151 AbstractGangTask("Enqueue reference objects in parallel"),
2145 2152 _enq_task(enq_task)
2146 2153 { }
2147 2154
2148 2155 virtual void work(int i) {
2149 2156 _enq_task.work(i);
2150 2157 }
2151 2158 };
2152 2159
2153 -void G1RefProcTaskExecutor::execute(EnqueueTask& enq_task) {
2160 +void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
2154 2161 assert(_workers != NULL, "Need parallel worker threads.");
2155 2162
2156 - G1RefEnqueueTaskProxy enq_task_proxy(enq_task);
2163 + G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task);
2157 2164
2158 2165 _g1h->set_par_threads(_active_workers);
2159 2166 _workers->run_task(&enq_task_proxy);
2160 2167 _g1h->set_par_threads(0);
2161 2168 }
2162 2169
2163 2170 void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {
2164 2171 ResourceMark rm;
2165 2172 HandleMark hm;
2166 2173 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2167 - ReferenceProcessor* rp = g1h->ref_processor();
2174 + ReferenceProcessor* rp = g1h->ref_processor_cm();
2168 2175
2169 2176 // See the comment in G1CollectedHeap::ref_processing_init()
2170 2177 // about how reference processing currently works in G1.
2171 2178
2172 2179 // Process weak references.
2173 2180 rp->setup_policy(clear_all_soft_refs);
2174 2181 assert(_markStack.isEmpty(), "mark stack should be empty");
2175 2182
2176 2183 G1CMIsAliveClosure g1_is_alive(g1h);
2177 2184 G1CMKeepAliveClosure g1_keep_alive(g1h, this, nextMarkBitMap());
2178 2185 G1CMDrainMarkingStackClosure
2179 2186 g1_drain_mark_stack(nextMarkBitMap(), &_markStack, &g1_keep_alive);
2180 2187 // We use the work gang from the G1CollectedHeap and we utilize all
2181 2188 // the worker threads.
2182 2189 int active_workers = g1h->workers() ? g1h->workers()->total_workers() : 1;
2183 2190 active_workers = MAX2(MIN2(active_workers, (int)_max_task_num), 1);
2184 2191
2185 - G1RefProcTaskExecutor par_task_executor(g1h, this, nextMarkBitMap(),
2192 + G1CMRefProcTaskExecutor par_task_executor(g1h, this, nextMarkBitMap(),
2186 2193 g1h->workers(), active_workers);
2187 2194
2188 2195
2189 2196 if (rp->processing_is_mt()) {
2190 2197 // Set the degree of MT here. If the discovery is done MT, there
2191 2198 // may have been a different number of threads doing the discovery
2192 2199 // and a different number of discovered lists may have Ref objects.
2193 2200 // That is OK as long as the Reference lists are balanced (see
2194 2201 // balance_all_queues() and balance_queues()).
2195 2202 rp->set_active_mt_degree(active_workers);
2196 2203
2197 2204 rp->process_discovered_references(&g1_is_alive,
2198 2205 &g1_keep_alive,
2199 2206 &g1_drain_mark_stack,
2200 2207 &par_task_executor);
2201 2208
2202 2209 // The work routines of the parallel keep_alive and drain_marking_stack
2203 2210 // will set the has_overflown flag if we overflow the global marking
2204 2211 // stack.
2205 2212 } else {
2206 2213 rp->process_discovered_references(&g1_is_alive,
2207 2214 &g1_keep_alive,
2208 2215 &g1_drain_mark_stack,
2209 2216 NULL);
2210 2217
2211 2218 }
2212 2219
2213 2220 assert(_markStack.overflow() || _markStack.isEmpty(),
2214 2221 "mark stack should be empty (unless it overflowed)");
2215 2222 if (_markStack.overflow()) {
2216 2223 // Should have been done already when we tried to push an
2217 2224 // entry on to the global mark stack. But let's do it again.
2218 2225 set_has_overflown();
|
↓ open down ↓ |
23 lines elided |
↑ open up ↑ |
2219 2226 }
2220 2227
2221 2228 if (rp->processing_is_mt()) {
2222 2229 assert(rp->num_q() == active_workers, "why not");
2223 2230 rp->enqueue_discovered_references(&par_task_executor);
2224 2231 } else {
2225 2232 rp->enqueue_discovered_references();
2226 2233 }
2227 2234
2228 2235 rp->verify_no_references_recorded();
2229 - assert(!rp->discovery_enabled(), "should have been disabled");
2236 + assert(!rp->discovery_enabled(), "Post condition");
2230 2237
2231 2238 // Now clean up stale oops in StringTable
2232 2239 StringTable::unlink(&g1_is_alive);
2233 2240 // Clean up unreferenced symbols in symbol table.
2234 2241 SymbolTable::unlink();
2235 2242 }
2236 2243
2237 2244 void ConcurrentMark::swapMarkBitMaps() {
2238 2245 CMBitMapRO* temp = _prevMarkBitMap;
2239 2246 _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap;
2240 2247 _nextMarkBitMap = (CMBitMap*) temp;
2241 2248 }
2242 2249
2243 2250 class CMRemarkTask: public AbstractGangTask {
2244 2251 private:
2245 2252 ConcurrentMark *_cm;
2246 2253
2247 2254 public:
2248 2255 void work(int worker_i) {
2249 2256 // Since all available tasks are actually started, we should
2250 2257 // only proceed if we're supposed to be actived.
2251 2258 if ((size_t)worker_i < _cm->active_tasks()) {
2252 2259 CMTask* task = _cm->task(worker_i);
2253 2260 task->record_start_time();
2254 2261 do {
2255 2262 task->do_marking_step(1000000000.0 /* something very large */,
2256 2263 true /* do_stealing */,
2257 2264 true /* do_termination */);
2258 2265 } while (task->has_aborted() && !_cm->has_overflown());
2259 2266 // If we overflow, then we do not want to restart. We instead
2260 2267 // want to abort remark and do concurrent marking again.
2261 2268 task->record_end_time();
2262 2269 }
2263 2270 }
2264 2271
2265 2272 CMRemarkTask(ConcurrentMark* cm) :
2266 2273 AbstractGangTask("Par Remark"), _cm(cm) { }
2267 2274 };
2268 2275
2269 2276 void ConcurrentMark::checkpointRootsFinalWork() {
2270 2277 ResourceMark rm;
2271 2278 HandleMark hm;
2272 2279 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2273 2280
2274 2281 g1h->ensure_parsability(false);
2275 2282
2276 2283 if (G1CollectedHeap::use_parallel_gc_threads()) {
2277 2284 G1CollectedHeap::StrongRootsScope srs(g1h);
2278 2285 // this is remark, so we'll use up all available threads
2279 2286 int active_workers = ParallelGCThreads;
2280 2287 set_phase(active_workers, false /* concurrent */);
2281 2288
2282 2289 CMRemarkTask remarkTask(this);
2283 2290 // We will start all available threads, even if we decide that the
2284 2291 // active_workers will be fewer. The extra ones will just bail out
2285 2292 // immediately.
2286 2293 int n_workers = g1h->workers()->total_workers();
2287 2294 g1h->set_par_threads(n_workers);
2288 2295 g1h->workers()->run_task(&remarkTask);
2289 2296 g1h->set_par_threads(0);
2290 2297 } else {
2291 2298 G1CollectedHeap::StrongRootsScope srs(g1h);
2292 2299 // this is remark, so we'll use up all available threads
2293 2300 int active_workers = 1;
2294 2301 set_phase(active_workers, false /* concurrent */);
2295 2302
2296 2303 CMRemarkTask remarkTask(this);
2297 2304 // We will start all available threads, even if we decide that the
2298 2305 // active_workers will be fewer. The extra ones will just bail out
2299 2306 // immediately.
2300 2307 remarkTask.work(0);
2301 2308 }
2302 2309 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2303 2310 guarantee(satb_mq_set.completed_buffers_num() == 0, "invariant");
2304 2311
2305 2312 print_stats();
2306 2313
2307 2314 #if VERIFY_OBJS_PROCESSED
2308 2315 if (_scan_obj_cl.objs_processed != ThreadLocalObjQueue::objs_enqueued) {
2309 2316 gclog_or_tty->print_cr("Processed = %d, enqueued = %d.",
2310 2317 _scan_obj_cl.objs_processed,
2311 2318 ThreadLocalObjQueue::objs_enqueued);
2312 2319 guarantee(_scan_obj_cl.objs_processed ==
2313 2320 ThreadLocalObjQueue::objs_enqueued,
2314 2321 "Different number of objs processed and enqueued.");
2315 2322 }
2316 2323 #endif
2317 2324 }
2318 2325
2319 2326 #ifndef PRODUCT
2320 2327
2321 2328 class PrintReachableOopClosure: public OopClosure {
2322 2329 private:
2323 2330 G1CollectedHeap* _g1h;
2324 2331 outputStream* _out;
2325 2332 VerifyOption _vo;
2326 2333 bool _all;
2327 2334
2328 2335 public:
2329 2336 PrintReachableOopClosure(outputStream* out,
2330 2337 VerifyOption vo,
2331 2338 bool all) :
2332 2339 _g1h(G1CollectedHeap::heap()),
2333 2340 _out(out), _vo(vo), _all(all) { }
2334 2341
2335 2342 void do_oop(narrowOop* p) { do_oop_work(p); }
2336 2343 void do_oop( oop* p) { do_oop_work(p); }
2337 2344
2338 2345 template <class T> void do_oop_work(T* p) {
2339 2346 oop obj = oopDesc::load_decode_heap_oop(p);
2340 2347 const char* str = NULL;
2341 2348 const char* str2 = "";
2342 2349
2343 2350 if (obj == NULL) {
2344 2351 str = "";
2345 2352 } else if (!_g1h->is_in_g1_reserved(obj)) {
2346 2353 str = " O";
2347 2354 } else {
2348 2355 HeapRegion* hr = _g1h->heap_region_containing(obj);
2349 2356 guarantee(hr != NULL, "invariant");
2350 2357 bool over_tams = false;
2351 2358 bool marked = false;
2352 2359
2353 2360 switch (_vo) {
2354 2361 case VerifyOption_G1UsePrevMarking:
2355 2362 over_tams = hr->obj_allocated_since_prev_marking(obj);
2356 2363 marked = _g1h->isMarkedPrev(obj);
2357 2364 break;
2358 2365 case VerifyOption_G1UseNextMarking:
2359 2366 over_tams = hr->obj_allocated_since_next_marking(obj);
2360 2367 marked = _g1h->isMarkedNext(obj);
2361 2368 break;
2362 2369 case VerifyOption_G1UseMarkWord:
2363 2370 marked = obj->is_gc_marked();
2364 2371 break;
2365 2372 default:
2366 2373 ShouldNotReachHere();
2367 2374 }
2368 2375
2369 2376 if (over_tams) {
2370 2377 str = " >";
2371 2378 if (marked) {
2372 2379 str2 = " AND MARKED";
2373 2380 }
2374 2381 } else if (marked) {
2375 2382 str = " M";
2376 2383 } else {
2377 2384 str = " NOT";
2378 2385 }
2379 2386 }
2380 2387
2381 2388 _out->print_cr(" "PTR_FORMAT": "PTR_FORMAT"%s%s",
2382 2389 p, (void*) obj, str, str2);
2383 2390 }
2384 2391 };
2385 2392
2386 2393 class PrintReachableObjectClosure : public ObjectClosure {
2387 2394 private:
2388 2395 G1CollectedHeap* _g1h;
2389 2396 outputStream* _out;
2390 2397 VerifyOption _vo;
2391 2398 bool _all;
2392 2399 HeapRegion* _hr;
2393 2400
2394 2401 public:
2395 2402 PrintReachableObjectClosure(outputStream* out,
2396 2403 VerifyOption vo,
2397 2404 bool all,
2398 2405 HeapRegion* hr) :
2399 2406 _g1h(G1CollectedHeap::heap()),
2400 2407 _out(out), _vo(vo), _all(all), _hr(hr) { }
2401 2408
2402 2409 void do_object(oop o) {
2403 2410 bool over_tams = false;
2404 2411 bool marked = false;
2405 2412
2406 2413 switch (_vo) {
2407 2414 case VerifyOption_G1UsePrevMarking:
2408 2415 over_tams = _hr->obj_allocated_since_prev_marking(o);
2409 2416 marked = _g1h->isMarkedPrev(o);
2410 2417 break;
2411 2418 case VerifyOption_G1UseNextMarking:
2412 2419 over_tams = _hr->obj_allocated_since_next_marking(o);
2413 2420 marked = _g1h->isMarkedNext(o);
2414 2421 break;
2415 2422 case VerifyOption_G1UseMarkWord:
2416 2423 marked = o->is_gc_marked();
2417 2424 break;
2418 2425 default:
2419 2426 ShouldNotReachHere();
2420 2427 }
2421 2428 bool print_it = _all || over_tams || marked;
2422 2429
2423 2430 if (print_it) {
2424 2431 _out->print_cr(" "PTR_FORMAT"%s",
2425 2432 o, (over_tams) ? " >" : (marked) ? " M" : "");
2426 2433 PrintReachableOopClosure oopCl(_out, _vo, _all);
2427 2434 o->oop_iterate(&oopCl);
2428 2435 }
2429 2436 }
2430 2437 };
2431 2438
2432 2439 class PrintReachableRegionClosure : public HeapRegionClosure {
2433 2440 private:
2434 2441 outputStream* _out;
2435 2442 VerifyOption _vo;
2436 2443 bool _all;
2437 2444
2438 2445 public:
2439 2446 bool doHeapRegion(HeapRegion* hr) {
2440 2447 HeapWord* b = hr->bottom();
2441 2448 HeapWord* e = hr->end();
2442 2449 HeapWord* t = hr->top();
2443 2450 HeapWord* p = NULL;
2444 2451
2445 2452 switch (_vo) {
2446 2453 case VerifyOption_G1UsePrevMarking:
2447 2454 p = hr->prev_top_at_mark_start();
2448 2455 break;
2449 2456 case VerifyOption_G1UseNextMarking:
2450 2457 p = hr->next_top_at_mark_start();
2451 2458 break;
2452 2459 case VerifyOption_G1UseMarkWord:
2453 2460 // When we are verifying marking using the mark word
2454 2461 // TAMS has no relevance.
2455 2462 assert(p == NULL, "post-condition");
2456 2463 break;
2457 2464 default:
2458 2465 ShouldNotReachHere();
2459 2466 }
2460 2467 _out->print_cr("** ["PTR_FORMAT", "PTR_FORMAT"] top: "PTR_FORMAT" "
2461 2468 "TAMS: "PTR_FORMAT, b, e, t, p);
2462 2469 _out->cr();
2463 2470
2464 2471 HeapWord* from = b;
2465 2472 HeapWord* to = t;
2466 2473
2467 2474 if (to > from) {
2468 2475 _out->print_cr("Objects in ["PTR_FORMAT", "PTR_FORMAT"]", from, to);
2469 2476 _out->cr();
2470 2477 PrintReachableObjectClosure ocl(_out, _vo, _all, hr);
2471 2478 hr->object_iterate_mem_careful(MemRegion(from, to), &ocl);
2472 2479 _out->cr();
2473 2480 }
2474 2481
2475 2482 return false;
2476 2483 }
2477 2484
2478 2485 PrintReachableRegionClosure(outputStream* out,
2479 2486 VerifyOption vo,
2480 2487 bool all) :
2481 2488 _out(out), _vo(vo), _all(all) { }
2482 2489 };
2483 2490
2484 2491 static const char* verify_option_to_tams(VerifyOption vo) {
2485 2492 switch (vo) {
2486 2493 case VerifyOption_G1UsePrevMarking:
2487 2494 return "PTAMS";
2488 2495 case VerifyOption_G1UseNextMarking:
2489 2496 return "NTAMS";
2490 2497 default:
2491 2498 return "NONE";
2492 2499 }
2493 2500 }
2494 2501
2495 2502 void ConcurrentMark::print_reachable(const char* str,
2496 2503 VerifyOption vo,
2497 2504 bool all) {
2498 2505 gclog_or_tty->cr();
2499 2506 gclog_or_tty->print_cr("== Doing heap dump... ");
2500 2507
2501 2508 if (G1PrintReachableBaseFile == NULL) {
2502 2509 gclog_or_tty->print_cr(" #### error: no base file defined");
2503 2510 return;
2504 2511 }
2505 2512
2506 2513 if (strlen(G1PrintReachableBaseFile) + 1 + strlen(str) >
2507 2514 (JVM_MAXPATHLEN - 1)) {
2508 2515 gclog_or_tty->print_cr(" #### error: file name too long");
2509 2516 return;
2510 2517 }
2511 2518
2512 2519 char file_name[JVM_MAXPATHLEN];
2513 2520 sprintf(file_name, "%s.%s", G1PrintReachableBaseFile, str);
2514 2521 gclog_or_tty->print_cr(" dumping to file %s", file_name);
2515 2522
2516 2523 fileStream fout(file_name);
2517 2524 if (!fout.is_open()) {
2518 2525 gclog_or_tty->print_cr(" #### error: could not open file");
2519 2526 return;
2520 2527 }
2521 2528
2522 2529 outputStream* out = &fout;
2523 2530 out->print_cr("-- USING %s", verify_option_to_tams(vo));
2524 2531 out->cr();
2525 2532
2526 2533 out->print_cr("--- ITERATING OVER REGIONS");
2527 2534 out->cr();
2528 2535 PrintReachableRegionClosure rcl(out, vo, all);
2529 2536 _g1h->heap_region_iterate(&rcl);
2530 2537 out->cr();
2531 2538
2532 2539 gclog_or_tty->print_cr(" done");
2533 2540 gclog_or_tty->flush();
2534 2541 }
2535 2542
2536 2543 #endif // PRODUCT
2537 2544
2538 2545 // This note is for drainAllSATBBuffers and the code in between.
2539 2546 // In the future we could reuse a task to do this work during an
2540 2547 // evacuation pause (since now tasks are not active and can be claimed
2541 2548 // during an evacuation pause). This was a late change to the code and
2542 2549 // is currently not being taken advantage of.
2543 2550
2544 2551 class CMGlobalObjectClosure : public ObjectClosure {
2545 2552 private:
2546 2553 ConcurrentMark* _cm;
2547 2554
2548 2555 public:
2549 2556 void do_object(oop obj) {
2550 2557 _cm->deal_with_reference(obj);
2551 2558 }
2552 2559
2553 2560 CMGlobalObjectClosure(ConcurrentMark* cm) : _cm(cm) { }
2554 2561 };
2555 2562
2556 2563 void ConcurrentMark::deal_with_reference(oop obj) {
2557 2564 if (verbose_high()) {
2558 2565 gclog_or_tty->print_cr("[global] we're dealing with reference "PTR_FORMAT,
2559 2566 (void*) obj);
2560 2567 }
2561 2568
2562 2569 HeapWord* objAddr = (HeapWord*) obj;
2563 2570 assert(obj->is_oop_or_null(true /* ignore mark word */), "Error");
2564 2571 if (_g1h->is_in_g1_reserved(objAddr)) {
2565 2572 assert(obj != NULL, "null check is implicit");
2566 2573 if (!_nextMarkBitMap->isMarked(objAddr)) {
2567 2574 // Only get the containing region if the object is not marked on the
2568 2575 // bitmap (otherwise, it's a waste of time since we won't do
2569 2576 // anything with it).
2570 2577 HeapRegion* hr = _g1h->heap_region_containing_raw(obj);
2571 2578 if (!hr->obj_allocated_since_next_marking(obj)) {
2572 2579 if (verbose_high()) {
2573 2580 gclog_or_tty->print_cr("[global] "PTR_FORMAT" is not considered "
2574 2581 "marked", (void*) obj);
2575 2582 }
2576 2583
2577 2584 // we need to mark it first
2578 2585 if (_nextMarkBitMap->parMark(objAddr)) {
2579 2586 // No OrderAccess:store_load() is needed. It is implicit in the
2580 2587 // CAS done in parMark(objAddr) above
2581 2588 HeapWord* finger = _finger;
2582 2589 if (objAddr < finger) {
2583 2590 if (verbose_high()) {
2584 2591 gclog_or_tty->print_cr("[global] below the global finger "
2585 2592 "("PTR_FORMAT"), pushing it", finger);
2586 2593 }
2587 2594 if (!mark_stack_push(obj)) {
2588 2595 if (verbose_low()) {
2589 2596 gclog_or_tty->print_cr("[global] global stack overflow during "
2590 2597 "deal_with_reference");
2591 2598 }
2592 2599 }
2593 2600 }
2594 2601 }
2595 2602 }
2596 2603 }
2597 2604 }
2598 2605 }
2599 2606
2600 2607 void ConcurrentMark::drainAllSATBBuffers() {
2601 2608 CMGlobalObjectClosure oc(this);
2602 2609 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2603 2610 satb_mq_set.set_closure(&oc);
2604 2611
2605 2612 while (satb_mq_set.apply_closure_to_completed_buffer()) {
2606 2613 if (verbose_medium()) {
2607 2614 gclog_or_tty->print_cr("[global] processed an SATB buffer");
2608 2615 }
2609 2616 }
2610 2617
2611 2618 // no need to check whether we should do this, as this is only
2612 2619 // called during an evacuation pause
2613 2620 satb_mq_set.iterate_closure_all_threads();
2614 2621
2615 2622 satb_mq_set.set_closure(NULL);
2616 2623 assert(satb_mq_set.completed_buffers_num() == 0, "invariant");
2617 2624 }
2618 2625
2619 2626 void ConcurrentMark::markPrev(oop p) {
2620 2627 // Note we are overriding the read-only view of the prev map here, via
2621 2628 // the cast.
2622 2629 ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*)p);
2623 2630 }
2624 2631
2625 2632 void ConcurrentMark::clear(oop p) {
2626 2633 assert(p != NULL && p->is_oop(), "expected an oop");
2627 2634 HeapWord* addr = (HeapWord*)p;
2628 2635 assert(addr >= _nextMarkBitMap->startWord() ||
2629 2636 addr < _nextMarkBitMap->endWord(), "in a region");
2630 2637
2631 2638 _nextMarkBitMap->clear(addr);
2632 2639 }
2633 2640
2634 2641 void ConcurrentMark::clearRangeBothMaps(MemRegion mr) {
2635 2642 // Note we are overriding the read-only view of the prev map here, via
2636 2643 // the cast.
2637 2644 ((CMBitMap*)_prevMarkBitMap)->clearRange(mr);
2638 2645 _nextMarkBitMap->clearRange(mr);
2639 2646 }
2640 2647
2641 2648 HeapRegion*
2642 2649 ConcurrentMark::claim_region(int task_num) {
2643 2650 // "checkpoint" the finger
2644 2651 HeapWord* finger = _finger;
2645 2652
2646 2653 // _heap_end will not change underneath our feet; it only changes at
2647 2654 // yield points.
2648 2655 while (finger < _heap_end) {
2649 2656 assert(_g1h->is_in_g1_reserved(finger), "invariant");
2650 2657
2651 2658 // Note on how this code handles humongous regions. In the
2652 2659 // normal case the finger will reach the start of a "starts
2653 2660 // humongous" (SH) region. Its end will either be the end of the
2654 2661 // last "continues humongous" (CH) region in the sequence, or the
2655 2662 // standard end of the SH region (if the SH is the only region in
2656 2663 // the sequence). That way claim_region() will skip over the CH
2657 2664 // regions. However, there is a subtle race between a CM thread
2658 2665 // executing this method and a mutator thread doing a humongous
2659 2666 // object allocation. The two are not mutually exclusive as the CM
2660 2667 // thread does not need to hold the Heap_lock when it gets
2661 2668 // here. So there is a chance that claim_region() will come across
2662 2669 // a free region that's in the progress of becoming a SH or a CH
2663 2670 // region. In the former case, it will either
2664 2671 // a) Miss the update to the region's end, in which case it will
2665 2672 // visit every subsequent CH region, will find their bitmaps
2666 2673 // empty, and do nothing, or
2667 2674 // b) Will observe the update of the region's end (in which case
2668 2675 // it will skip the subsequent CH regions).
2669 2676 // If it comes across a region that suddenly becomes CH, the
2670 2677 // scenario will be similar to b). So, the race between
2671 2678 // claim_region() and a humongous object allocation might force us
2672 2679 // to do a bit of unnecessary work (due to some unnecessary bitmap
2673 2680 // iterations) but it should not introduce and correctness issues.
2674 2681 HeapRegion* curr_region = _g1h->heap_region_containing_raw(finger);
2675 2682 HeapWord* bottom = curr_region->bottom();
2676 2683 HeapWord* end = curr_region->end();
2677 2684 HeapWord* limit = curr_region->next_top_at_mark_start();
2678 2685
2679 2686 if (verbose_low()) {
2680 2687 gclog_or_tty->print_cr("[%d] curr_region = "PTR_FORMAT" "
2681 2688 "["PTR_FORMAT", "PTR_FORMAT"), "
2682 2689 "limit = "PTR_FORMAT,
2683 2690 task_num, curr_region, bottom, end, limit);
2684 2691 }
2685 2692
2686 2693 // Is the gap between reading the finger and doing the CAS too long?
2687 2694 HeapWord* res = (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger);
2688 2695 if (res == finger) {
2689 2696 // we succeeded
2690 2697
2691 2698 // notice that _finger == end cannot be guaranteed here since,
2692 2699 // someone else might have moved the finger even further
2693 2700 assert(_finger >= end, "the finger should have moved forward");
2694 2701
2695 2702 if (verbose_low()) {
2696 2703 gclog_or_tty->print_cr("[%d] we were successful with region = "
2697 2704 PTR_FORMAT, task_num, curr_region);
2698 2705 }
2699 2706
2700 2707 if (limit > bottom) {
2701 2708 if (verbose_low()) {
2702 2709 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is not empty, "
2703 2710 "returning it ", task_num, curr_region);
2704 2711 }
2705 2712 return curr_region;
2706 2713 } else {
2707 2714 assert(limit == bottom,
2708 2715 "the region limit should be at bottom");
2709 2716 if (verbose_low()) {
2710 2717 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is empty, "
2711 2718 "returning NULL", task_num, curr_region);
2712 2719 }
2713 2720 // we return NULL and the caller should try calling
2714 2721 // claim_region() again.
2715 2722 return NULL;
2716 2723 }
2717 2724 } else {
2718 2725 assert(_finger > finger, "the finger should have moved forward");
2719 2726 if (verbose_low()) {
2720 2727 gclog_or_tty->print_cr("[%d] somebody else moved the finger, "
2721 2728 "global finger = "PTR_FORMAT", "
2722 2729 "our finger = "PTR_FORMAT,
2723 2730 task_num, _finger, finger);
2724 2731 }
2725 2732
2726 2733 // read it again
2727 2734 finger = _finger;
2728 2735 }
2729 2736 }
2730 2737
2731 2738 return NULL;
2732 2739 }
2733 2740
2734 2741 bool ConcurrentMark::invalidate_aborted_regions_in_cset() {
2735 2742 bool result = false;
2736 2743 for (int i = 0; i < (int)_max_task_num; ++i) {
2737 2744 CMTask* the_task = _tasks[i];
2738 2745 MemRegion mr = the_task->aborted_region();
2739 2746 if (mr.start() != NULL) {
2740 2747 assert(mr.end() != NULL, "invariant");
2741 2748 assert(mr.word_size() > 0, "invariant");
2742 2749 HeapRegion* hr = _g1h->heap_region_containing(mr.start());
2743 2750 assert(hr != NULL, "invariant");
2744 2751 if (hr->in_collection_set()) {
2745 2752 // The region points into the collection set
2746 2753 the_task->set_aborted_region(MemRegion());
2747 2754 result = true;
2748 2755 }
2749 2756 }
2750 2757 }
2751 2758 return result;
2752 2759 }
2753 2760
2754 2761 bool ConcurrentMark::has_aborted_regions() {
2755 2762 for (int i = 0; i < (int)_max_task_num; ++i) {
2756 2763 CMTask* the_task = _tasks[i];
2757 2764 MemRegion mr = the_task->aborted_region();
2758 2765 if (mr.start() != NULL) {
2759 2766 assert(mr.end() != NULL, "invariant");
2760 2767 assert(mr.word_size() > 0, "invariant");
2761 2768 return true;
2762 2769 }
2763 2770 }
2764 2771 return false;
2765 2772 }
2766 2773
2767 2774 void ConcurrentMark::oops_do(OopClosure* cl) {
2768 2775 if (_markStack.size() > 0 && verbose_low()) {
2769 2776 gclog_or_tty->print_cr("[global] scanning the global marking stack, "
2770 2777 "size = %d", _markStack.size());
2771 2778 }
2772 2779 // we first iterate over the contents of the mark stack...
2773 2780 _markStack.oops_do(cl);
2774 2781
2775 2782 for (int i = 0; i < (int)_max_task_num; ++i) {
2776 2783 OopTaskQueue* queue = _task_queues->queue((int)i);
2777 2784
2778 2785 if (queue->size() > 0 && verbose_low()) {
2779 2786 gclog_or_tty->print_cr("[global] scanning task queue of task %d, "
2780 2787 "size = %d", i, queue->size());
2781 2788 }
2782 2789
2783 2790 // ...then over the contents of the all the task queues.
2784 2791 queue->oops_do(cl);
2785 2792 }
2786 2793
2787 2794 // Invalidate any entries, that are in the region stack, that
2788 2795 // point into the collection set
2789 2796 if (_regionStack.invalidate_entries_into_cset()) {
2790 2797 // otherwise, any gray objects copied during the evacuation pause
2791 2798 // might not be visited.
2792 2799 assert(_should_gray_objects, "invariant");
2793 2800 }
2794 2801
2795 2802 // Invalidate any aborted regions, recorded in the individual CM
2796 2803 // tasks, that point into the collection set.
2797 2804 if (invalidate_aborted_regions_in_cset()) {
2798 2805 // otherwise, any gray objects copied during the evacuation pause
2799 2806 // might not be visited.
2800 2807 assert(_should_gray_objects, "invariant");
2801 2808 }
2802 2809
2803 2810 }
2804 2811
2805 2812 void ConcurrentMark::clear_marking_state(bool clear_overflow) {
2806 2813 _markStack.setEmpty();
2807 2814 _markStack.clear_overflow();
2808 2815 _regionStack.setEmpty();
2809 2816 _regionStack.clear_overflow();
2810 2817 if (clear_overflow) {
2811 2818 clear_has_overflown();
2812 2819 } else {
2813 2820 assert(has_overflown(), "pre-condition");
2814 2821 }
2815 2822 _finger = _heap_start;
2816 2823
2817 2824 for (int i = 0; i < (int)_max_task_num; ++i) {
2818 2825 OopTaskQueue* queue = _task_queues->queue(i);
2819 2826 queue->set_empty();
2820 2827 // Clear any partial regions from the CMTasks
2821 2828 _tasks[i]->clear_aborted_region();
2822 2829 }
2823 2830 }
2824 2831
2825 2832 void ConcurrentMark::print_stats() {
2826 2833 if (verbose_stats()) {
2827 2834 gclog_or_tty->print_cr("---------------------------------------------------------------------");
2828 2835 for (size_t i = 0; i < _active_tasks; ++i) {
2829 2836 _tasks[i]->print_stats();
2830 2837 gclog_or_tty->print_cr("---------------------------------------------------------------------");
2831 2838 }
2832 2839 }
2833 2840 }
2834 2841
2835 2842 class CSMarkOopClosure: public OopClosure {
2836 2843 friend class CSMarkBitMapClosure;
2837 2844
2838 2845 G1CollectedHeap* _g1h;
2839 2846 CMBitMap* _bm;
2840 2847 ConcurrentMark* _cm;
2841 2848 oop* _ms;
2842 2849 jint* _array_ind_stack;
2843 2850 int _ms_size;
2844 2851 int _ms_ind;
2845 2852 int _array_increment;
2846 2853
2847 2854 bool push(oop obj, int arr_ind = 0) {
2848 2855 if (_ms_ind == _ms_size) {
2849 2856 gclog_or_tty->print_cr("Mark stack is full.");
2850 2857 return false;
2851 2858 }
2852 2859 _ms[_ms_ind] = obj;
2853 2860 if (obj->is_objArray()) {
2854 2861 _array_ind_stack[_ms_ind] = arr_ind;
2855 2862 }
2856 2863 _ms_ind++;
2857 2864 return true;
2858 2865 }
2859 2866
2860 2867 oop pop() {
2861 2868 if (_ms_ind == 0) {
2862 2869 return NULL;
2863 2870 } else {
2864 2871 _ms_ind--;
2865 2872 return _ms[_ms_ind];
2866 2873 }
2867 2874 }
2868 2875
2869 2876 template <class T> bool drain() {
2870 2877 while (_ms_ind > 0) {
2871 2878 oop obj = pop();
2872 2879 assert(obj != NULL, "Since index was non-zero.");
2873 2880 if (obj->is_objArray()) {
2874 2881 jint arr_ind = _array_ind_stack[_ms_ind];
2875 2882 objArrayOop aobj = objArrayOop(obj);
2876 2883 jint len = aobj->length();
2877 2884 jint next_arr_ind = arr_ind + _array_increment;
2878 2885 if (next_arr_ind < len) {
2879 2886 push(obj, next_arr_ind);
2880 2887 }
2881 2888 // Now process this portion of this one.
2882 2889 int lim = MIN2(next_arr_ind, len);
2883 2890 for (int j = arr_ind; j < lim; j++) {
2884 2891 do_oop(aobj->objArrayOopDesc::obj_at_addr<T>(j));
2885 2892 }
2886 2893
2887 2894 } else {
2888 2895 obj->oop_iterate(this);
2889 2896 }
2890 2897 if (abort()) return false;
2891 2898 }
2892 2899 return true;
2893 2900 }
2894 2901
2895 2902 public:
2896 2903 CSMarkOopClosure(ConcurrentMark* cm, int ms_size) :
2897 2904 _g1h(G1CollectedHeap::heap()),
2898 2905 _cm(cm),
2899 2906 _bm(cm->nextMarkBitMap()),
2900 2907 _ms_size(ms_size), _ms_ind(0),
2901 2908 _ms(NEW_C_HEAP_ARRAY(oop, ms_size)),
2902 2909 _array_ind_stack(NEW_C_HEAP_ARRAY(jint, ms_size)),
2903 2910 _array_increment(MAX2(ms_size/8, 16))
2904 2911 {}
2905 2912
2906 2913 ~CSMarkOopClosure() {
2907 2914 FREE_C_HEAP_ARRAY(oop, _ms);
2908 2915 FREE_C_HEAP_ARRAY(jint, _array_ind_stack);
2909 2916 }
2910 2917
2911 2918 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
2912 2919 virtual void do_oop( oop* p) { do_oop_work(p); }
2913 2920
2914 2921 template <class T> void do_oop_work(T* p) {
2915 2922 T heap_oop = oopDesc::load_heap_oop(p);
2916 2923 if (oopDesc::is_null(heap_oop)) return;
2917 2924 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
2918 2925 if (obj->is_forwarded()) {
2919 2926 // If the object has already been forwarded, we have to make sure
2920 2927 // that it's marked. So follow the forwarding pointer. Note that
2921 2928 // this does the right thing for self-forwarding pointers in the
2922 2929 // evacuation failure case.
2923 2930 obj = obj->forwardee();
2924 2931 }
2925 2932 HeapRegion* hr = _g1h->heap_region_containing(obj);
2926 2933 if (hr != NULL) {
2927 2934 if (hr->in_collection_set()) {
2928 2935 if (_g1h->is_obj_ill(obj)) {
2929 2936 _bm->mark((HeapWord*)obj);
2930 2937 if (!push(obj)) {
2931 2938 gclog_or_tty->print_cr("Setting abort in CSMarkOopClosure because push failed.");
2932 2939 set_abort();
2933 2940 }
2934 2941 }
2935 2942 } else {
2936 2943 // Outside the collection set; we need to gray it
2937 2944 _cm->deal_with_reference(obj);
2938 2945 }
2939 2946 }
2940 2947 }
2941 2948 };
2942 2949
2943 2950 class CSMarkBitMapClosure: public BitMapClosure {
2944 2951 G1CollectedHeap* _g1h;
2945 2952 CMBitMap* _bitMap;
2946 2953 ConcurrentMark* _cm;
2947 2954 CSMarkOopClosure _oop_cl;
2948 2955 public:
2949 2956 CSMarkBitMapClosure(ConcurrentMark* cm, int ms_size) :
2950 2957 _g1h(G1CollectedHeap::heap()),
2951 2958 _bitMap(cm->nextMarkBitMap()),
2952 2959 _oop_cl(cm, ms_size)
2953 2960 {}
2954 2961
2955 2962 ~CSMarkBitMapClosure() {}
2956 2963
2957 2964 bool do_bit(size_t offset) {
2958 2965 // convert offset into a HeapWord*
2959 2966 HeapWord* addr = _bitMap->offsetToHeapWord(offset);
2960 2967 assert(_bitMap->endWord() && addr < _bitMap->endWord(),
2961 2968 "address out of range");
2962 2969 assert(_bitMap->isMarked(addr), "tautology");
2963 2970 oop obj = oop(addr);
2964 2971 if (!obj->is_forwarded()) {
2965 2972 if (!_oop_cl.push(obj)) return false;
2966 2973 if (UseCompressedOops) {
2967 2974 if (!_oop_cl.drain<narrowOop>()) return false;
2968 2975 } else {
2969 2976 if (!_oop_cl.drain<oop>()) return false;
2970 2977 }
2971 2978 }
2972 2979 // Otherwise...
2973 2980 return true;
2974 2981 }
2975 2982 };
2976 2983
2977 2984
2978 2985 class CompleteMarkingInCSHRClosure: public HeapRegionClosure {
2979 2986 CMBitMap* _bm;
2980 2987 CSMarkBitMapClosure _bit_cl;
2981 2988 enum SomePrivateConstants {
2982 2989 MSSize = 1000
2983 2990 };
2984 2991 bool _completed;
2985 2992 public:
2986 2993 CompleteMarkingInCSHRClosure(ConcurrentMark* cm) :
2987 2994 _bm(cm->nextMarkBitMap()),
2988 2995 _bit_cl(cm, MSSize),
2989 2996 _completed(true)
2990 2997 {}
2991 2998
2992 2999 ~CompleteMarkingInCSHRClosure() {}
2993 3000
2994 3001 bool doHeapRegion(HeapRegion* r) {
2995 3002 if (!r->evacuation_failed()) {
2996 3003 MemRegion mr = MemRegion(r->bottom(), r->next_top_at_mark_start());
2997 3004 if (!mr.is_empty()) {
2998 3005 if (!_bm->iterate(&_bit_cl, mr)) {
2999 3006 _completed = false;
3000 3007 return true;
3001 3008 }
3002 3009 }
3003 3010 }
3004 3011 return false;
3005 3012 }
3006 3013
3007 3014 bool completed() { return _completed; }
3008 3015 };
3009 3016
3010 3017 class ClearMarksInHRClosure: public HeapRegionClosure {
3011 3018 CMBitMap* _bm;
3012 3019 public:
3013 3020 ClearMarksInHRClosure(CMBitMap* bm): _bm(bm) { }
3014 3021
3015 3022 bool doHeapRegion(HeapRegion* r) {
3016 3023 if (!r->used_region().is_empty() && !r->evacuation_failed()) {
3017 3024 MemRegion usedMR = r->used_region();
3018 3025 _bm->clearRange(r->used_region());
3019 3026 }
3020 3027 return false;
3021 3028 }
3022 3029 };
3023 3030
3024 3031 void ConcurrentMark::complete_marking_in_collection_set() {
3025 3032 G1CollectedHeap* g1h = G1CollectedHeap::heap();
3026 3033
3027 3034 if (!g1h->mark_in_progress()) {
3028 3035 g1h->g1_policy()->record_mark_closure_time(0.0);
3029 3036 return;
3030 3037 }
3031 3038
3032 3039 int i = 1;
3033 3040 double start = os::elapsedTime();
3034 3041 while (true) {
3035 3042 i++;
3036 3043 CompleteMarkingInCSHRClosure cmplt(this);
3037 3044 g1h->collection_set_iterate(&cmplt);
3038 3045 if (cmplt.completed()) break;
3039 3046 }
3040 3047 double end_time = os::elapsedTime();
3041 3048 double elapsed_time_ms = (end_time - start) * 1000.0;
3042 3049 g1h->g1_policy()->record_mark_closure_time(elapsed_time_ms);
3043 3050
3044 3051 ClearMarksInHRClosure clr(nextMarkBitMap());
3045 3052 g1h->collection_set_iterate(&clr);
3046 3053 }
3047 3054
3048 3055 // The next two methods deal with the following optimisation. Some
3049 3056 // objects are gray by being marked and located above the finger. If
3050 3057 // they are copied, during an evacuation pause, below the finger then
3051 3058 // the need to be pushed on the stack. The observation is that, if
3052 3059 // there are no regions in the collection set located above the
3053 3060 // finger, then the above cannot happen, hence we do not need to
3054 3061 // explicitly gray any objects when copying them to below the
3055 3062 // finger. The global stack will be scanned to ensure that, if it
3056 3063 // points to objects being copied, it will update their
3057 3064 // location. There is a tricky situation with the gray objects in
3058 3065 // region stack that are being coped, however. See the comment in
3059 3066 // newCSet().
3060 3067
3061 3068 void ConcurrentMark::newCSet() {
3062 3069 if (!concurrent_marking_in_progress()) {
3063 3070 // nothing to do if marking is not in progress
3064 3071 return;
3065 3072 }
3066 3073
3067 3074 // find what the lowest finger is among the global and local fingers
3068 3075 _min_finger = _finger;
3069 3076 for (int i = 0; i < (int)_max_task_num; ++i) {
3070 3077 CMTask* task = _tasks[i];
3071 3078 HeapWord* task_finger = task->finger();
3072 3079 if (task_finger != NULL && task_finger < _min_finger) {
3073 3080 _min_finger = task_finger;
3074 3081 }
3075 3082 }
3076 3083
3077 3084 _should_gray_objects = false;
3078 3085
3079 3086 // This fixes a very subtle and fustrating bug. It might be the case
3080 3087 // that, during en evacuation pause, heap regions that contain
3081 3088 // objects that are gray (by being in regions contained in the
3082 3089 // region stack) are included in the collection set. Since such gray
3083 3090 // objects will be moved, and because it's not easy to redirect
3084 3091 // region stack entries to point to a new location (because objects
3085 3092 // in one region might be scattered to multiple regions after they
3086 3093 // are copied), one option is to ensure that all marked objects
3087 3094 // copied during a pause are pushed on the stack. Notice, however,
3088 3095 // that this problem can only happen when the region stack is not
3089 3096 // empty during an evacuation pause. So, we make the fix a bit less
3090 3097 // conservative and ensure that regions are pushed on the stack,
3091 3098 // irrespective whether all collection set regions are below the
3092 3099 // finger, if the region stack is not empty. This is expected to be
3093 3100 // a rare case, so I don't think it's necessary to be smarted about it.
3094 3101 if (!region_stack_empty() || has_aborted_regions()) {
3095 3102 _should_gray_objects = true;
3096 3103 }
3097 3104 }
3098 3105
3099 3106 void ConcurrentMark::registerCSetRegion(HeapRegion* hr) {
3100 3107 if (!concurrent_marking_in_progress()) return;
3101 3108
3102 3109 HeapWord* region_end = hr->end();
3103 3110 if (region_end > _min_finger) {
3104 3111 _should_gray_objects = true;
3105 3112 }
3106 3113 }
3107 3114
3108 3115 // Resets the region fields of active CMTasks whose values point
3109 3116 // into the collection set.
3110 3117 void ConcurrentMark::reset_active_task_region_fields_in_cset() {
3111 3118 assert(SafepointSynchronize::is_at_safepoint(), "should be in STW");
3112 3119 assert(parallel_marking_threads() <= _max_task_num, "sanity");
3113 3120
3114 3121 for (int i = 0; i < (int)parallel_marking_threads(); i += 1) {
3115 3122 CMTask* task = _tasks[i];
3116 3123 HeapWord* task_finger = task->finger();
3117 3124 if (task_finger != NULL) {
3118 3125 assert(_g1h->is_in_g1_reserved(task_finger), "not in heap");
3119 3126 HeapRegion* finger_region = _g1h->heap_region_containing(task_finger);
3120 3127 if (finger_region->in_collection_set()) {
3121 3128 // The task's current region is in the collection set.
3122 3129 // This region will be evacuated in the current GC and
3123 3130 // the region fields in the task will be stale.
3124 3131 task->giveup_current_region();
3125 3132 }
3126 3133 }
3127 3134 }
3128 3135 }
3129 3136
3130 3137 // abandon current marking iteration due to a Full GC
3131 3138 void ConcurrentMark::abort() {
3132 3139 // Clear all marks to force marking thread to do nothing
3133 3140 _nextMarkBitMap->clearAll();
3134 3141 // Empty mark stack
3135 3142 clear_marking_state();
3136 3143 for (int i = 0; i < (int)_max_task_num; ++i) {
3137 3144 _tasks[i]->clear_region_fields();
3138 3145 }
3139 3146 _has_aborted = true;
3140 3147
3141 3148 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3142 3149 satb_mq_set.abandon_partial_marking();
3143 3150 // This can be called either during or outside marking, we'll read
3144 3151 // the expected_active value from the SATB queue set.
3145 3152 satb_mq_set.set_active_all_threads(
3146 3153 false, /* new active value */
3147 3154 satb_mq_set.is_active() /* expected_active */);
3148 3155 }
3149 3156
3150 3157 static void print_ms_time_info(const char* prefix, const char* name,
3151 3158 NumberSeq& ns) {
3152 3159 gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
3153 3160 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
3154 3161 if (ns.num() > 0) {
3155 3162 gclog_or_tty->print_cr("%s [std. dev = %8.2f ms, max = %8.2f ms]",
3156 3163 prefix, ns.sd(), ns.maximum());
3157 3164 }
3158 3165 }
3159 3166
3160 3167 void ConcurrentMark::print_summary_info() {
3161 3168 gclog_or_tty->print_cr(" Concurrent marking:");
3162 3169 print_ms_time_info(" ", "init marks", _init_times);
3163 3170 print_ms_time_info(" ", "remarks", _remark_times);
3164 3171 {
3165 3172 print_ms_time_info(" ", "final marks", _remark_mark_times);
3166 3173 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
3167 3174
3168 3175 }
3169 3176 print_ms_time_info(" ", "cleanups", _cleanup_times);
3170 3177 gclog_or_tty->print_cr(" Final counting total time = %8.2f s (avg = %8.2f ms).",
3171 3178 _total_counting_time,
3172 3179 (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 /
3173 3180 (double)_cleanup_times.num()
3174 3181 : 0.0));
3175 3182 if (G1ScrubRemSets) {
3176 3183 gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
3177 3184 _total_rs_scrub_time,
3178 3185 (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 /
3179 3186 (double)_cleanup_times.num()
3180 3187 : 0.0));
3181 3188 }
3182 3189 gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.",
3183 3190 (_init_times.sum() + _remark_times.sum() +
3184 3191 _cleanup_times.sum())/1000.0);
3185 3192 gclog_or_tty->print_cr(" Total concurrent time = %8.2f s "
3186 3193 "(%8.2f s marking, %8.2f s counting).",
3187 3194 cmThread()->vtime_accum(),
3188 3195 cmThread()->vtime_mark_accum(),
3189 3196 cmThread()->vtime_count_accum());
3190 3197 }
3191 3198
3192 3199 void ConcurrentMark::print_worker_threads_on(outputStream* st) const {
3193 3200 _parallel_workers->print_worker_threads_on(st);
3194 3201 }
3195 3202
3196 3203 // Closures
3197 3204 // XXX: there seems to be a lot of code duplication here;
3198 3205 // should refactor and consolidate the shared code.
3199 3206
3200 3207 // This closure is used to mark refs into the CMS generation in
3201 3208 // the CMS bit map. Called at the first checkpoint.
3202 3209
3203 3210 // We take a break if someone is trying to stop the world.
3204 3211 bool ConcurrentMark::do_yield_check(int worker_i) {
3205 3212 if (should_yield()) {
3206 3213 if (worker_i == 0) {
3207 3214 _g1h->g1_policy()->record_concurrent_pause();
3208 3215 }
3209 3216 cmThread()->yield();
3210 3217 if (worker_i == 0) {
3211 3218 _g1h->g1_policy()->record_concurrent_pause_end();
3212 3219 }
3213 3220 return true;
3214 3221 } else {
3215 3222 return false;
3216 3223 }
3217 3224 }
3218 3225
3219 3226 bool ConcurrentMark::should_yield() {
3220 3227 return cmThread()->should_yield();
3221 3228 }
3222 3229
3223 3230 bool ConcurrentMark::containing_card_is_marked(void* p) {
3224 3231 size_t offset = pointer_delta(p, _g1h->reserved_region().start(), 1);
3225 3232 return _card_bm.at(offset >> CardTableModRefBS::card_shift);
3226 3233 }
3227 3234
3228 3235 bool ConcurrentMark::containing_cards_are_marked(void* start,
3229 3236 void* last) {
3230 3237 return containing_card_is_marked(start) &&
3231 3238 containing_card_is_marked(last);
3232 3239 }
3233 3240
3234 3241 #ifndef PRODUCT
3235 3242 // for debugging purposes
3236 3243 void ConcurrentMark::print_finger() {
3237 3244 gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT,
3238 3245 _heap_start, _heap_end, _finger);
3239 3246 for (int i = 0; i < (int) _max_task_num; ++i) {
3240 3247 gclog_or_tty->print(" %d: "PTR_FORMAT, i, _tasks[i]->finger());
3241 3248 }
3242 3249 gclog_or_tty->print_cr("");
3243 3250 }
3244 3251 #endif
3245 3252
3246 3253 void CMTask::scan_object(oop obj) {
3247 3254 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), "invariant");
3248 3255
3249 3256 if (_cm->verbose_high()) {
3250 3257 gclog_or_tty->print_cr("[%d] we're scanning object "PTR_FORMAT,
3251 3258 _task_id, (void*) obj);
3252 3259 }
3253 3260
3254 3261 size_t obj_size = obj->size();
3255 3262 _words_scanned += obj_size;
3256 3263
3257 3264 obj->oop_iterate(_cm_oop_closure);
3258 3265 statsOnly( ++_objs_scanned );
3259 3266 check_limits();
3260 3267 }
3261 3268
3262 3269 // Closure for iteration over bitmaps
3263 3270 class CMBitMapClosure : public BitMapClosure {
3264 3271 private:
3265 3272 // the bitmap that is being iterated over
3266 3273 CMBitMap* _nextMarkBitMap;
3267 3274 ConcurrentMark* _cm;
3268 3275 CMTask* _task;
3269 3276 // true if we're scanning a heap region claimed by the task (so that
3270 3277 // we move the finger along), false if we're not, i.e. currently when
3271 3278 // scanning a heap region popped from the region stack (so that we
3272 3279 // do not move the task finger along; it'd be a mistake if we did so).
3273 3280 bool _scanning_heap_region;
3274 3281
3275 3282 public:
3276 3283 CMBitMapClosure(CMTask *task,
3277 3284 ConcurrentMark* cm,
3278 3285 CMBitMap* nextMarkBitMap)
3279 3286 : _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { }
3280 3287
3281 3288 void set_scanning_heap_region(bool scanning_heap_region) {
3282 3289 _scanning_heap_region = scanning_heap_region;
3283 3290 }
3284 3291
3285 3292 bool do_bit(size_t offset) {
3286 3293 HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset);
3287 3294 assert(_nextMarkBitMap->isMarked(addr), "invariant");
3288 3295 assert( addr < _cm->finger(), "invariant");
3289 3296
3290 3297 if (_scanning_heap_region) {
3291 3298 statsOnly( _task->increase_objs_found_on_bitmap() );
3292 3299 assert(addr >= _task->finger(), "invariant");
3293 3300 // We move that task's local finger along.
3294 3301 _task->move_finger_to(addr);
3295 3302 } else {
3296 3303 // We move the task's region finger along.
3297 3304 _task->move_region_finger_to(addr);
3298 3305 }
3299 3306
3300 3307 _task->scan_object(oop(addr));
3301 3308 // we only partially drain the local queue and global stack
3302 3309 _task->drain_local_queue(true);
3303 3310 _task->drain_global_stack(true);
3304 3311
3305 3312 // if the has_aborted flag has been raised, we need to bail out of
3306 3313 // the iteration
3307 3314 return !_task->has_aborted();
3308 3315 }
3309 3316 };
3310 3317
3311 3318 // Closure for iterating over objects, currently only used for
3312 3319 // processing SATB buffers.
3313 3320 class CMObjectClosure : public ObjectClosure {
3314 3321 private:
3315 3322 CMTask* _task;
3316 3323
3317 3324 public:
3318 3325 void do_object(oop obj) {
3319 3326 _task->deal_with_reference(obj);
3320 3327 }
3321 3328
|
↓ open down ↓ |
1082 lines elided |
↑ open up ↑ |
3322 3329 CMObjectClosure(CMTask* task) : _task(task) { }
3323 3330 };
3324 3331
3325 3332 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
3326 3333 ConcurrentMark* cm,
3327 3334 CMTask* task)
3328 3335 : _g1h(g1h), _cm(cm), _task(task) {
3329 3336 assert(_ref_processor == NULL, "should be initialized to NULL");
3330 3337
3331 3338 if (G1UseConcMarkReferenceProcessing) {
3332 - _ref_processor = g1h->ref_processor();
3339 + _ref_processor = g1h->ref_processor_cm();
3333 3340 assert(_ref_processor != NULL, "should not be NULL");
3334 3341 }
3335 3342 }
3336 3343
3337 3344 void CMTask::setup_for_region(HeapRegion* hr) {
3338 3345 // Separated the asserts so that we know which one fires.
3339 3346 assert(hr != NULL,
3340 3347 "claim_region() should have filtered out continues humongous regions");
3341 3348 assert(!hr->continuesHumongous(),
3342 3349 "claim_region() should have filtered out continues humongous regions");
3343 3350
3344 3351 if (_cm->verbose_low()) {
3345 3352 gclog_or_tty->print_cr("[%d] setting up for region "PTR_FORMAT,
3346 3353 _task_id, hr);
3347 3354 }
3348 3355
3349 3356 _curr_region = hr;
3350 3357 _finger = hr->bottom();
3351 3358 update_region_limit();
3352 3359 }
3353 3360
3354 3361 void CMTask::update_region_limit() {
3355 3362 HeapRegion* hr = _curr_region;
3356 3363 HeapWord* bottom = hr->bottom();
3357 3364 HeapWord* limit = hr->next_top_at_mark_start();
3358 3365
3359 3366 if (limit == bottom) {
3360 3367 if (_cm->verbose_low()) {
3361 3368 gclog_or_tty->print_cr("[%d] found an empty region "
3362 3369 "["PTR_FORMAT", "PTR_FORMAT")",
3363 3370 _task_id, bottom, limit);
3364 3371 }
3365 3372 // The region was collected underneath our feet.
3366 3373 // We set the finger to bottom to ensure that the bitmap
3367 3374 // iteration that will follow this will not do anything.
3368 3375 // (this is not a condition that holds when we set the region up,
3369 3376 // as the region is not supposed to be empty in the first place)
3370 3377 _finger = bottom;
3371 3378 } else if (limit >= _region_limit) {
3372 3379 assert(limit >= _finger, "peace of mind");
3373 3380 } else {
3374 3381 assert(limit < _region_limit, "only way to get here");
3375 3382 // This can happen under some pretty unusual circumstances. An
3376 3383 // evacuation pause empties the region underneath our feet (NTAMS
3377 3384 // at bottom). We then do some allocation in the region (NTAMS
3378 3385 // stays at bottom), followed by the region being used as a GC
3379 3386 // alloc region (NTAMS will move to top() and the objects
3380 3387 // originally below it will be grayed). All objects now marked in
3381 3388 // the region are explicitly grayed, if below the global finger,
3382 3389 // and we do not need in fact to scan anything else. So, we simply
3383 3390 // set _finger to be limit to ensure that the bitmap iteration
3384 3391 // doesn't do anything.
3385 3392 _finger = limit;
3386 3393 }
3387 3394
3388 3395 _region_limit = limit;
3389 3396 }
3390 3397
3391 3398 void CMTask::giveup_current_region() {
3392 3399 assert(_curr_region != NULL, "invariant");
3393 3400 if (_cm->verbose_low()) {
3394 3401 gclog_or_tty->print_cr("[%d] giving up region "PTR_FORMAT,
3395 3402 _task_id, _curr_region);
3396 3403 }
3397 3404 clear_region_fields();
3398 3405 }
3399 3406
3400 3407 void CMTask::clear_region_fields() {
3401 3408 // Values for these three fields that indicate that we're not
3402 3409 // holding on to a region.
3403 3410 _curr_region = NULL;
3404 3411 _finger = NULL;
3405 3412 _region_limit = NULL;
3406 3413
3407 3414 _region_finger = NULL;
3408 3415 }
3409 3416
3410 3417 void CMTask::set_cm_oop_closure(G1CMOopClosure* cm_oop_closure) {
3411 3418 if (cm_oop_closure == NULL) {
3412 3419 assert(_cm_oop_closure != NULL, "invariant");
3413 3420 } else {
3414 3421 assert(_cm_oop_closure == NULL, "invariant");
3415 3422 }
3416 3423 _cm_oop_closure = cm_oop_closure;
3417 3424 }
3418 3425
3419 3426 void CMTask::reset(CMBitMap* nextMarkBitMap) {
3420 3427 guarantee(nextMarkBitMap != NULL, "invariant");
3421 3428
3422 3429 if (_cm->verbose_low()) {
3423 3430 gclog_or_tty->print_cr("[%d] resetting", _task_id);
3424 3431 }
3425 3432
3426 3433 _nextMarkBitMap = nextMarkBitMap;
3427 3434 clear_region_fields();
3428 3435 assert(_aborted_region.is_empty(), "should have been cleared");
3429 3436
3430 3437 _calls = 0;
3431 3438 _elapsed_time_ms = 0.0;
3432 3439 _termination_time_ms = 0.0;
3433 3440 _termination_start_time_ms = 0.0;
3434 3441
3435 3442 #if _MARKING_STATS_
3436 3443 _local_pushes = 0;
3437 3444 _local_pops = 0;
3438 3445 _local_max_size = 0;
3439 3446 _objs_scanned = 0;
3440 3447 _global_pushes = 0;
3441 3448 _global_pops = 0;
3442 3449 _global_max_size = 0;
3443 3450 _global_transfers_to = 0;
3444 3451 _global_transfers_from = 0;
3445 3452 _region_stack_pops = 0;
3446 3453 _regions_claimed = 0;
3447 3454 _objs_found_on_bitmap = 0;
3448 3455 _satb_buffers_processed = 0;
3449 3456 _steal_attempts = 0;
3450 3457 _steals = 0;
3451 3458 _aborted = 0;
3452 3459 _aborted_overflow = 0;
3453 3460 _aborted_cm_aborted = 0;
3454 3461 _aborted_yield = 0;
3455 3462 _aborted_timed_out = 0;
3456 3463 _aborted_satb = 0;
3457 3464 _aborted_termination = 0;
3458 3465 #endif // _MARKING_STATS_
3459 3466 }
3460 3467
3461 3468 bool CMTask::should_exit_termination() {
3462 3469 regular_clock_call();
3463 3470 // This is called when we are in the termination protocol. We should
3464 3471 // quit if, for some reason, this task wants to abort or the global
3465 3472 // stack is not empty (this means that we can get work from it).
3466 3473 return !_cm->mark_stack_empty() || has_aborted();
3467 3474 }
3468 3475
3469 3476 void CMTask::reached_limit() {
3470 3477 assert(_words_scanned >= _words_scanned_limit ||
3471 3478 _refs_reached >= _refs_reached_limit ,
3472 3479 "shouldn't have been called otherwise");
3473 3480 regular_clock_call();
3474 3481 }
3475 3482
3476 3483 void CMTask::regular_clock_call() {
3477 3484 if (has_aborted()) return;
3478 3485
3479 3486 // First, we need to recalculate the words scanned and refs reached
3480 3487 // limits for the next clock call.
3481 3488 recalculate_limits();
3482 3489
3483 3490 // During the regular clock call we do the following
3484 3491
3485 3492 // (1) If an overflow has been flagged, then we abort.
3486 3493 if (_cm->has_overflown()) {
3487 3494 set_has_aborted();
3488 3495 return;
3489 3496 }
3490 3497
3491 3498 // If we are not concurrent (i.e. we're doing remark) we don't need
3492 3499 // to check anything else. The other steps are only needed during
3493 3500 // the concurrent marking phase.
3494 3501 if (!concurrent()) return;
3495 3502
3496 3503 // (2) If marking has been aborted for Full GC, then we also abort.
3497 3504 if (_cm->has_aborted()) {
3498 3505 set_has_aborted();
3499 3506 statsOnly( ++_aborted_cm_aborted );
3500 3507 return;
3501 3508 }
3502 3509
3503 3510 double curr_time_ms = os::elapsedVTime() * 1000.0;
3504 3511
3505 3512 // (3) If marking stats are enabled, then we update the step history.
3506 3513 #if _MARKING_STATS_
3507 3514 if (_words_scanned >= _words_scanned_limit) {
3508 3515 ++_clock_due_to_scanning;
3509 3516 }
3510 3517 if (_refs_reached >= _refs_reached_limit) {
3511 3518 ++_clock_due_to_marking;
3512 3519 }
3513 3520
3514 3521 double last_interval_ms = curr_time_ms - _interval_start_time_ms;
3515 3522 _interval_start_time_ms = curr_time_ms;
3516 3523 _all_clock_intervals_ms.add(last_interval_ms);
3517 3524
3518 3525 if (_cm->verbose_medium()) {
3519 3526 gclog_or_tty->print_cr("[%d] regular clock, interval = %1.2lfms, "
3520 3527 "scanned = %d%s, refs reached = %d%s",
3521 3528 _task_id, last_interval_ms,
3522 3529 _words_scanned,
3523 3530 (_words_scanned >= _words_scanned_limit) ? " (*)" : "",
3524 3531 _refs_reached,
3525 3532 (_refs_reached >= _refs_reached_limit) ? " (*)" : "");
3526 3533 }
3527 3534 #endif // _MARKING_STATS_
3528 3535
3529 3536 // (4) We check whether we should yield. If we have to, then we abort.
3530 3537 if (_cm->should_yield()) {
3531 3538 // We should yield. To do this we abort the task. The caller is
3532 3539 // responsible for yielding.
3533 3540 set_has_aborted();
3534 3541 statsOnly( ++_aborted_yield );
3535 3542 return;
3536 3543 }
3537 3544
3538 3545 // (5) We check whether we've reached our time quota. If we have,
3539 3546 // then we abort.
3540 3547 double elapsed_time_ms = curr_time_ms - _start_time_ms;
3541 3548 if (elapsed_time_ms > _time_target_ms) {
3542 3549 set_has_aborted();
3543 3550 _has_timed_out = true;
3544 3551 statsOnly( ++_aborted_timed_out );
3545 3552 return;
3546 3553 }
3547 3554
3548 3555 // (6) Finally, we check whether there are enough completed STAB
3549 3556 // buffers available for processing. If there are, we abort.
3550 3557 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3551 3558 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) {
3552 3559 if (_cm->verbose_low()) {
3553 3560 gclog_or_tty->print_cr("[%d] aborting to deal with pending SATB buffers",
3554 3561 _task_id);
3555 3562 }
3556 3563 // we do need to process SATB buffers, we'll abort and restart
3557 3564 // the marking task to do so
3558 3565 set_has_aborted();
3559 3566 statsOnly( ++_aborted_satb );
3560 3567 return;
3561 3568 }
3562 3569 }
3563 3570
3564 3571 void CMTask::recalculate_limits() {
3565 3572 _real_words_scanned_limit = _words_scanned + words_scanned_period;
3566 3573 _words_scanned_limit = _real_words_scanned_limit;
3567 3574
3568 3575 _real_refs_reached_limit = _refs_reached + refs_reached_period;
3569 3576 _refs_reached_limit = _real_refs_reached_limit;
3570 3577 }
3571 3578
3572 3579 void CMTask::decrease_limits() {
3573 3580 // This is called when we believe that we're going to do an infrequent
3574 3581 // operation which will increase the per byte scanned cost (i.e. move
3575 3582 // entries to/from the global stack). It basically tries to decrease the
3576 3583 // scanning limit so that the clock is called earlier.
3577 3584
3578 3585 if (_cm->verbose_medium()) {
3579 3586 gclog_or_tty->print_cr("[%d] decreasing limits", _task_id);
3580 3587 }
3581 3588
3582 3589 _words_scanned_limit = _real_words_scanned_limit -
3583 3590 3 * words_scanned_period / 4;
3584 3591 _refs_reached_limit = _real_refs_reached_limit -
3585 3592 3 * refs_reached_period / 4;
3586 3593 }
3587 3594
3588 3595 void CMTask::move_entries_to_global_stack() {
3589 3596 // local array where we'll store the entries that will be popped
3590 3597 // from the local queue
3591 3598 oop buffer[global_stack_transfer_size];
3592 3599
3593 3600 int n = 0;
3594 3601 oop obj;
3595 3602 while (n < global_stack_transfer_size && _task_queue->pop_local(obj)) {
3596 3603 buffer[n] = obj;
3597 3604 ++n;
3598 3605 }
3599 3606
3600 3607 if (n > 0) {
3601 3608 // we popped at least one entry from the local queue
3602 3609
3603 3610 statsOnly( ++_global_transfers_to; _local_pops += n );
3604 3611
3605 3612 if (!_cm->mark_stack_push(buffer, n)) {
3606 3613 if (_cm->verbose_low()) {
3607 3614 gclog_or_tty->print_cr("[%d] aborting due to global stack overflow",
3608 3615 _task_id);
3609 3616 }
3610 3617 set_has_aborted();
3611 3618 } else {
3612 3619 // the transfer was successful
3613 3620
3614 3621 if (_cm->verbose_medium()) {
3615 3622 gclog_or_tty->print_cr("[%d] pushed %d entries to the global stack",
3616 3623 _task_id, n);
3617 3624 }
3618 3625 statsOnly( int tmp_size = _cm->mark_stack_size();
3619 3626 if (tmp_size > _global_max_size) {
3620 3627 _global_max_size = tmp_size;
3621 3628 }
3622 3629 _global_pushes += n );
3623 3630 }
3624 3631 }
3625 3632
3626 3633 // this operation was quite expensive, so decrease the limits
3627 3634 decrease_limits();
3628 3635 }
3629 3636
3630 3637 void CMTask::get_entries_from_global_stack() {
3631 3638 // local array where we'll store the entries that will be popped
3632 3639 // from the global stack.
3633 3640 oop buffer[global_stack_transfer_size];
3634 3641 int n;
3635 3642 _cm->mark_stack_pop(buffer, global_stack_transfer_size, &n);
3636 3643 assert(n <= global_stack_transfer_size,
3637 3644 "we should not pop more than the given limit");
3638 3645 if (n > 0) {
3639 3646 // yes, we did actually pop at least one entry
3640 3647
3641 3648 statsOnly( ++_global_transfers_from; _global_pops += n );
3642 3649 if (_cm->verbose_medium()) {
3643 3650 gclog_or_tty->print_cr("[%d] popped %d entries from the global stack",
3644 3651 _task_id, n);
3645 3652 }
3646 3653 for (int i = 0; i < n; ++i) {
3647 3654 bool success = _task_queue->push(buffer[i]);
3648 3655 // We only call this when the local queue is empty or under a
3649 3656 // given target limit. So, we do not expect this push to fail.
3650 3657 assert(success, "invariant");
3651 3658 }
3652 3659
3653 3660 statsOnly( int tmp_size = _task_queue->size();
3654 3661 if (tmp_size > _local_max_size) {
3655 3662 _local_max_size = tmp_size;
3656 3663 }
3657 3664 _local_pushes += n );
3658 3665 }
3659 3666
3660 3667 // this operation was quite expensive, so decrease the limits
3661 3668 decrease_limits();
3662 3669 }
3663 3670
3664 3671 void CMTask::drain_local_queue(bool partially) {
3665 3672 if (has_aborted()) return;
3666 3673
3667 3674 // Decide what the target size is, depending whether we're going to
3668 3675 // drain it partially (so that other tasks can steal if they run out
3669 3676 // of things to do) or totally (at the very end).
3670 3677 size_t target_size;
3671 3678 if (partially) {
3672 3679 target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize);
3673 3680 } else {
3674 3681 target_size = 0;
3675 3682 }
3676 3683
3677 3684 if (_task_queue->size() > target_size) {
3678 3685 if (_cm->verbose_high()) {
3679 3686 gclog_or_tty->print_cr("[%d] draining local queue, target size = %d",
3680 3687 _task_id, target_size);
3681 3688 }
3682 3689
3683 3690 oop obj;
3684 3691 bool ret = _task_queue->pop_local(obj);
3685 3692 while (ret) {
3686 3693 statsOnly( ++_local_pops );
3687 3694
3688 3695 if (_cm->verbose_high()) {
3689 3696 gclog_or_tty->print_cr("[%d] popped "PTR_FORMAT, _task_id,
3690 3697 (void*) obj);
3691 3698 }
3692 3699
3693 3700 assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" );
3694 3701 assert(!_g1h->is_on_master_free_list(
3695 3702 _g1h->heap_region_containing((HeapWord*) obj)), "invariant");
3696 3703
3697 3704 scan_object(obj);
3698 3705
3699 3706 if (_task_queue->size() <= target_size || has_aborted()) {
3700 3707 ret = false;
3701 3708 } else {
3702 3709 ret = _task_queue->pop_local(obj);
3703 3710 }
3704 3711 }
3705 3712
3706 3713 if (_cm->verbose_high()) {
3707 3714 gclog_or_tty->print_cr("[%d] drained local queue, size = %d",
3708 3715 _task_id, _task_queue->size());
3709 3716 }
3710 3717 }
3711 3718 }
3712 3719
3713 3720 void CMTask::drain_global_stack(bool partially) {
3714 3721 if (has_aborted()) return;
3715 3722
3716 3723 // We have a policy to drain the local queue before we attempt to
3717 3724 // drain the global stack.
3718 3725 assert(partially || _task_queue->size() == 0, "invariant");
3719 3726
3720 3727 // Decide what the target size is, depending whether we're going to
3721 3728 // drain it partially (so that other tasks can steal if they run out
3722 3729 // of things to do) or totally (at the very end). Notice that,
3723 3730 // because we move entries from the global stack in chunks or
3724 3731 // because another task might be doing the same, we might in fact
3725 3732 // drop below the target. But, this is not a problem.
3726 3733 size_t target_size;
3727 3734 if (partially) {
3728 3735 target_size = _cm->partial_mark_stack_size_target();
3729 3736 } else {
3730 3737 target_size = 0;
3731 3738 }
3732 3739
3733 3740 if (_cm->mark_stack_size() > target_size) {
3734 3741 if (_cm->verbose_low()) {
3735 3742 gclog_or_tty->print_cr("[%d] draining global_stack, target size %d",
3736 3743 _task_id, target_size);
3737 3744 }
3738 3745
3739 3746 while (!has_aborted() && _cm->mark_stack_size() > target_size) {
3740 3747 get_entries_from_global_stack();
3741 3748 drain_local_queue(partially);
3742 3749 }
3743 3750
3744 3751 if (_cm->verbose_low()) {
3745 3752 gclog_or_tty->print_cr("[%d] drained global stack, size = %d",
3746 3753 _task_id, _cm->mark_stack_size());
3747 3754 }
3748 3755 }
3749 3756 }
3750 3757
3751 3758 // SATB Queue has several assumptions on whether to call the par or
3752 3759 // non-par versions of the methods. this is why some of the code is
3753 3760 // replicated. We should really get rid of the single-threaded version
3754 3761 // of the code to simplify things.
3755 3762 void CMTask::drain_satb_buffers() {
3756 3763 if (has_aborted()) return;
3757 3764
3758 3765 // We set this so that the regular clock knows that we're in the
3759 3766 // middle of draining buffers and doesn't set the abort flag when it
3760 3767 // notices that SATB buffers are available for draining. It'd be
3761 3768 // very counter productive if it did that. :-)
3762 3769 _draining_satb_buffers = true;
3763 3770
3764 3771 CMObjectClosure oc(this);
3765 3772 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
3766 3773 if (G1CollectedHeap::use_parallel_gc_threads()) {
3767 3774 satb_mq_set.set_par_closure(_task_id, &oc);
3768 3775 } else {
3769 3776 satb_mq_set.set_closure(&oc);
3770 3777 }
3771 3778
3772 3779 // This keeps claiming and applying the closure to completed buffers
3773 3780 // until we run out of buffers or we need to abort.
3774 3781 if (G1CollectedHeap::use_parallel_gc_threads()) {
3775 3782 while (!has_aborted() &&
3776 3783 satb_mq_set.par_apply_closure_to_completed_buffer(_task_id)) {
3777 3784 if (_cm->verbose_medium()) {
3778 3785 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id);
3779 3786 }
3780 3787 statsOnly( ++_satb_buffers_processed );
3781 3788 regular_clock_call();
3782 3789 }
3783 3790 } else {
3784 3791 while (!has_aborted() &&
3785 3792 satb_mq_set.apply_closure_to_completed_buffer()) {
3786 3793 if (_cm->verbose_medium()) {
3787 3794 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id);
3788 3795 }
3789 3796 statsOnly( ++_satb_buffers_processed );
3790 3797 regular_clock_call();
3791 3798 }
3792 3799 }
3793 3800
3794 3801 if (!concurrent() && !has_aborted()) {
3795 3802 // We should only do this during remark.
3796 3803 if (G1CollectedHeap::use_parallel_gc_threads()) {
3797 3804 satb_mq_set.par_iterate_closure_all_threads(_task_id);
3798 3805 } else {
3799 3806 satb_mq_set.iterate_closure_all_threads();
3800 3807 }
3801 3808 }
3802 3809
3803 3810 _draining_satb_buffers = false;
3804 3811
3805 3812 assert(has_aborted() ||
3806 3813 concurrent() ||
3807 3814 satb_mq_set.completed_buffers_num() == 0, "invariant");
3808 3815
3809 3816 if (G1CollectedHeap::use_parallel_gc_threads()) {
3810 3817 satb_mq_set.set_par_closure(_task_id, NULL);
3811 3818 } else {
3812 3819 satb_mq_set.set_closure(NULL);
3813 3820 }
3814 3821
3815 3822 // again, this was a potentially expensive operation, decrease the
3816 3823 // limits to get the regular clock call early
3817 3824 decrease_limits();
3818 3825 }
3819 3826
3820 3827 void CMTask::drain_region_stack(BitMapClosure* bc) {
3821 3828 if (has_aborted()) return;
3822 3829
3823 3830 assert(_region_finger == NULL,
3824 3831 "it should be NULL when we're not scanning a region");
3825 3832
3826 3833 if (!_cm->region_stack_empty() || !_aborted_region.is_empty()) {
3827 3834 if (_cm->verbose_low()) {
3828 3835 gclog_or_tty->print_cr("[%d] draining region stack, size = %d",
3829 3836 _task_id, _cm->region_stack_size());
3830 3837 }
3831 3838
3832 3839 MemRegion mr;
3833 3840
3834 3841 if (!_aborted_region.is_empty()) {
3835 3842 mr = _aborted_region;
3836 3843 _aborted_region = MemRegion();
3837 3844
3838 3845 if (_cm->verbose_low()) {
3839 3846 gclog_or_tty->print_cr("[%d] scanning aborted region "
3840 3847 "[ " PTR_FORMAT ", " PTR_FORMAT " )",
3841 3848 _task_id, mr.start(), mr.end());
3842 3849 }
3843 3850 } else {
3844 3851 mr = _cm->region_stack_pop_lock_free();
3845 3852 // it returns MemRegion() if the pop fails
3846 3853 statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
3847 3854 }
3848 3855
3849 3856 while (mr.start() != NULL) {
3850 3857 if (_cm->verbose_medium()) {
3851 3858 gclog_or_tty->print_cr("[%d] we are scanning region "
3852 3859 "["PTR_FORMAT", "PTR_FORMAT")",
3853 3860 _task_id, mr.start(), mr.end());
3854 3861 }
3855 3862
3856 3863 assert(mr.end() <= _cm->finger(),
3857 3864 "otherwise the region shouldn't be on the stack");
3858 3865 assert(!mr.is_empty(), "Only non-empty regions live on the region stack");
3859 3866 if (_nextMarkBitMap->iterate(bc, mr)) {
3860 3867 assert(!has_aborted(),
3861 3868 "cannot abort the task without aborting the bitmap iteration");
3862 3869
3863 3870 // We finished iterating over the region without aborting.
3864 3871 regular_clock_call();
3865 3872 if (has_aborted()) {
3866 3873 mr = MemRegion();
3867 3874 } else {
3868 3875 mr = _cm->region_stack_pop_lock_free();
3869 3876 // it returns MemRegion() if the pop fails
3870 3877 statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
3871 3878 }
3872 3879 } else {
3873 3880 assert(has_aborted(), "currently the only way to do so");
3874 3881
3875 3882 // The only way to abort the bitmap iteration is to return
3876 3883 // false from the do_bit() method. However, inside the
3877 3884 // do_bit() method we move the _region_finger to point to the
3878 3885 // object currently being looked at. So, if we bail out, we
3879 3886 // have definitely set _region_finger to something non-null.
3880 3887 assert(_region_finger != NULL, "invariant");
3881 3888
3882 3889 // Make sure that any previously aborted region has been
3883 3890 // cleared.
3884 3891 assert(_aborted_region.is_empty(), "aborted region not cleared");
3885 3892
3886 3893 // The iteration was actually aborted. So now _region_finger
3887 3894 // points to the address of the object we last scanned. If we
3888 3895 // leave it there, when we restart this task, we will rescan
3889 3896 // the object. It is easy to avoid this. We move the finger by
3890 3897 // enough to point to the next possible object header (the
3891 3898 // bitmap knows by how much we need to move it as it knows its
3892 3899 // granularity).
3893 3900 MemRegion newRegion =
3894 3901 MemRegion(_nextMarkBitMap->nextWord(_region_finger), mr.end());
3895 3902
3896 3903 if (!newRegion.is_empty()) {
3897 3904 if (_cm->verbose_low()) {
3898 3905 gclog_or_tty->print_cr("[%d] recording unscanned region"
3899 3906 "[" PTR_FORMAT "," PTR_FORMAT ") in CMTask",
3900 3907 _task_id,
3901 3908 newRegion.start(), newRegion.end());
3902 3909 }
3903 3910 // Now record the part of the region we didn't scan to
3904 3911 // make sure this task scans it later.
3905 3912 _aborted_region = newRegion;
3906 3913 }
3907 3914 // break from while
3908 3915 mr = MemRegion();
3909 3916 }
3910 3917 _region_finger = NULL;
3911 3918 }
3912 3919
3913 3920 if (_cm->verbose_low()) {
3914 3921 gclog_or_tty->print_cr("[%d] drained region stack, size = %d",
3915 3922 _task_id, _cm->region_stack_size());
3916 3923 }
3917 3924 }
3918 3925 }
3919 3926
3920 3927 void CMTask::print_stats() {
3921 3928 gclog_or_tty->print_cr("Marking Stats, task = %d, calls = %d",
3922 3929 _task_id, _calls);
3923 3930 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
3924 3931 _elapsed_time_ms, _termination_time_ms);
3925 3932 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3926 3933 _step_times_ms.num(), _step_times_ms.avg(),
3927 3934 _step_times_ms.sd());
3928 3935 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
3929 3936 _step_times_ms.maximum(), _step_times_ms.sum());
3930 3937
3931 3938 #if _MARKING_STATS_
3932 3939 gclog_or_tty->print_cr(" Clock Intervals (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
3933 3940 _all_clock_intervals_ms.num(), _all_clock_intervals_ms.avg(),
3934 3941 _all_clock_intervals_ms.sd());
3935 3942 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
3936 3943 _all_clock_intervals_ms.maximum(),
3937 3944 _all_clock_intervals_ms.sum());
3938 3945 gclog_or_tty->print_cr(" Clock Causes (cum): scanning = %d, marking = %d",
3939 3946 _clock_due_to_scanning, _clock_due_to_marking);
3940 3947 gclog_or_tty->print_cr(" Objects: scanned = %d, found on the bitmap = %d",
3941 3948 _objs_scanned, _objs_found_on_bitmap);
3942 3949 gclog_or_tty->print_cr(" Local Queue: pushes = %d, pops = %d, max size = %d",
3943 3950 _local_pushes, _local_pops, _local_max_size);
3944 3951 gclog_or_tty->print_cr(" Global Stack: pushes = %d, pops = %d, max size = %d",
3945 3952 _global_pushes, _global_pops, _global_max_size);
3946 3953 gclog_or_tty->print_cr(" transfers to = %d, transfers from = %d",
3947 3954 _global_transfers_to,_global_transfers_from);
3948 3955 gclog_or_tty->print_cr(" Regions: claimed = %d, Region Stack: pops = %d",
3949 3956 _regions_claimed, _region_stack_pops);
3950 3957 gclog_or_tty->print_cr(" SATB buffers: processed = %d", _satb_buffers_processed);
3951 3958 gclog_or_tty->print_cr(" Steals: attempts = %d, successes = %d",
3952 3959 _steal_attempts, _steals);
3953 3960 gclog_or_tty->print_cr(" Aborted: %d, due to", _aborted);
3954 3961 gclog_or_tty->print_cr(" overflow: %d, global abort: %d, yield: %d",
3955 3962 _aborted_overflow, _aborted_cm_aborted, _aborted_yield);
3956 3963 gclog_or_tty->print_cr(" time out: %d, SATB: %d, termination: %d",
3957 3964 _aborted_timed_out, _aborted_satb, _aborted_termination);
3958 3965 #endif // _MARKING_STATS_
3959 3966 }
3960 3967
3961 3968 /*****************************************************************************
3962 3969
3963 3970 The do_marking_step(time_target_ms) method is the building block
3964 3971 of the parallel marking framework. It can be called in parallel
3965 3972 with other invocations of do_marking_step() on different tasks
3966 3973 (but only one per task, obviously) and concurrently with the
3967 3974 mutator threads, or during remark, hence it eliminates the need
3968 3975 for two versions of the code. When called during remark, it will
3969 3976 pick up from where the task left off during the concurrent marking
3970 3977 phase. Interestingly, tasks are also claimable during evacuation
3971 3978 pauses too, since do_marking_step() ensures that it aborts before
3972 3979 it needs to yield.
3973 3980
3974 3981 The data structures that is uses to do marking work are the
3975 3982 following:
3976 3983
3977 3984 (1) Marking Bitmap. If there are gray objects that appear only
3978 3985 on the bitmap (this happens either when dealing with an overflow
3979 3986 or when the initial marking phase has simply marked the roots
3980 3987 and didn't push them on the stack), then tasks claim heap
3981 3988 regions whose bitmap they then scan to find gray objects. A
3982 3989 global finger indicates where the end of the last claimed region
3983 3990 is. A local finger indicates how far into the region a task has
3984 3991 scanned. The two fingers are used to determine how to gray an
3985 3992 object (i.e. whether simply marking it is OK, as it will be
3986 3993 visited by a task in the future, or whether it needs to be also
3987 3994 pushed on a stack).
3988 3995
3989 3996 (2) Local Queue. The local queue of the task which is accessed
3990 3997 reasonably efficiently by the task. Other tasks can steal from
3991 3998 it when they run out of work. Throughout the marking phase, a
3992 3999 task attempts to keep its local queue short but not totally
3993 4000 empty, so that entries are available for stealing by other
3994 4001 tasks. Only when there is no more work, a task will totally
3995 4002 drain its local queue.
3996 4003
3997 4004 (3) Global Mark Stack. This handles local queue overflow. During
3998 4005 marking only sets of entries are moved between it and the local
3999 4006 queues, as access to it requires a mutex and more fine-grain
4000 4007 interaction with it which might cause contention. If it
4001 4008 overflows, then the marking phase should restart and iterate
4002 4009 over the bitmap to identify gray objects. Throughout the marking
4003 4010 phase, tasks attempt to keep the global mark stack at a small
4004 4011 length but not totally empty, so that entries are available for
4005 4012 popping by other tasks. Only when there is no more work, tasks
4006 4013 will totally drain the global mark stack.
4007 4014
4008 4015 (4) Global Region Stack. Entries on it correspond to areas of
4009 4016 the bitmap that need to be scanned since they contain gray
4010 4017 objects. Pushes on the region stack only happen during
4011 4018 evacuation pauses and typically correspond to areas covered by
4012 4019 GC LABS. If it overflows, then the marking phase should restart
4013 4020 and iterate over the bitmap to identify gray objects. Tasks will
4014 4021 try to totally drain the region stack as soon as possible.
4015 4022
4016 4023 (5) SATB Buffer Queue. This is where completed SATB buffers are
4017 4024 made available. Buffers are regularly removed from this queue
4018 4025 and scanned for roots, so that the queue doesn't get too
4019 4026 long. During remark, all completed buffers are processed, as
4020 4027 well as the filled in parts of any uncompleted buffers.
4021 4028
4022 4029 The do_marking_step() method tries to abort when the time target
4023 4030 has been reached. There are a few other cases when the
4024 4031 do_marking_step() method also aborts:
4025 4032
4026 4033 (1) When the marking phase has been aborted (after a Full GC).
4027 4034
4028 4035 (2) When a global overflow (either on the global stack or the
4029 4036 region stack) has been triggered. Before the task aborts, it
4030 4037 will actually sync up with the other tasks to ensure that all
4031 4038 the marking data structures (local queues, stacks, fingers etc.)
4032 4039 are re-initialised so that when do_marking_step() completes,
4033 4040 the marking phase can immediately restart.
4034 4041
4035 4042 (3) When enough completed SATB buffers are available. The
4036 4043 do_marking_step() method only tries to drain SATB buffers right
4037 4044 at the beginning. So, if enough buffers are available, the
4038 4045 marking step aborts and the SATB buffers are processed at
4039 4046 the beginning of the next invocation.
4040 4047
4041 4048 (4) To yield. when we have to yield then we abort and yield
4042 4049 right at the end of do_marking_step(). This saves us from a lot
4043 4050 of hassle as, by yielding we might allow a Full GC. If this
4044 4051 happens then objects will be compacted underneath our feet, the
4045 4052 heap might shrink, etc. We save checking for this by just
4046 4053 aborting and doing the yield right at the end.
4047 4054
4048 4055 From the above it follows that the do_marking_step() method should
4049 4056 be called in a loop (or, otherwise, regularly) until it completes.
4050 4057
4051 4058 If a marking step completes without its has_aborted() flag being
4052 4059 true, it means it has completed the current marking phase (and
4053 4060 also all other marking tasks have done so and have all synced up).
4054 4061
4055 4062 A method called regular_clock_call() is invoked "regularly" (in
4056 4063 sub ms intervals) throughout marking. It is this clock method that
4057 4064 checks all the abort conditions which were mentioned above and
4058 4065 decides when the task should abort. A work-based scheme is used to
4059 4066 trigger this clock method: when the number of object words the
4060 4067 marking phase has scanned or the number of references the marking
4061 4068 phase has visited reach a given limit. Additional invocations to
4062 4069 the method clock have been planted in a few other strategic places
4063 4070 too. The initial reason for the clock method was to avoid calling
4064 4071 vtime too regularly, as it is quite expensive. So, once it was in
4065 4072 place, it was natural to piggy-back all the other conditions on it
4066 4073 too and not constantly check them throughout the code.
4067 4074
4068 4075 *****************************************************************************/
4069 4076
4070 4077 void CMTask::do_marking_step(double time_target_ms,
4071 4078 bool do_stealing,
4072 4079 bool do_termination) {
4073 4080 assert(time_target_ms >= 1.0, "minimum granularity is 1ms");
4074 4081 assert(concurrent() == _cm->concurrent(), "they should be the same");
4075 4082
4076 4083 assert(concurrent() || _cm->region_stack_empty(),
4077 4084 "the region stack should have been cleared before remark");
4078 4085 assert(concurrent() || !_cm->has_aborted_regions(),
4079 4086 "aborted regions should have been cleared before remark");
4080 4087 assert(_region_finger == NULL,
4081 4088 "this should be non-null only when a region is being scanned");
4082 4089
4083 4090 G1CollectorPolicy* g1_policy = _g1h->g1_policy();
4084 4091 assert(_task_queues != NULL, "invariant");
4085 4092 assert(_task_queue != NULL, "invariant");
4086 4093 assert(_task_queues->queue(_task_id) == _task_queue, "invariant");
4087 4094
4088 4095 assert(!_claimed,
4089 4096 "only one thread should claim this task at any one time");
4090 4097
4091 4098 // OK, this doesn't safeguard again all possible scenarios, as it is
4092 4099 // possible for two threads to set the _claimed flag at the same
4093 4100 // time. But it is only for debugging purposes anyway and it will
4094 4101 // catch most problems.
4095 4102 _claimed = true;
4096 4103
4097 4104 _start_time_ms = os::elapsedVTime() * 1000.0;
4098 4105 statsOnly( _interval_start_time_ms = _start_time_ms );
4099 4106
4100 4107 double diff_prediction_ms =
4101 4108 g1_policy->get_new_prediction(&_marking_step_diffs_ms);
4102 4109 _time_target_ms = time_target_ms - diff_prediction_ms;
4103 4110
4104 4111 // set up the variables that are used in the work-based scheme to
4105 4112 // call the regular clock method
4106 4113 _words_scanned = 0;
4107 4114 _refs_reached = 0;
4108 4115 recalculate_limits();
4109 4116
4110 4117 // clear all flags
4111 4118 clear_has_aborted();
4112 4119 _has_timed_out = false;
4113 4120 _draining_satb_buffers = false;
4114 4121
4115 4122 ++_calls;
4116 4123
4117 4124 if (_cm->verbose_low()) {
4118 4125 gclog_or_tty->print_cr("[%d] >>>>>>>>>> START, call = %d, "
4119 4126 "target = %1.2lfms >>>>>>>>>>",
4120 4127 _task_id, _calls, _time_target_ms);
4121 4128 }
4122 4129
4123 4130 // Set up the bitmap and oop closures. Anything that uses them is
4124 4131 // eventually called from this method, so it is OK to allocate these
4125 4132 // statically.
4126 4133 CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap);
4127 4134 G1CMOopClosure cm_oop_closure(_g1h, _cm, this);
4128 4135 set_cm_oop_closure(&cm_oop_closure);
4129 4136
4130 4137 if (_cm->has_overflown()) {
4131 4138 // This can happen if the region stack or the mark stack overflows
4132 4139 // during a GC pause and this task, after a yield point,
4133 4140 // restarts. We have to abort as we need to get into the overflow
4134 4141 // protocol which happens right at the end of this task.
4135 4142 set_has_aborted();
4136 4143 }
4137 4144
4138 4145 // First drain any available SATB buffers. After this, we will not
4139 4146 // look at SATB buffers before the next invocation of this method.
4140 4147 // If enough completed SATB buffers are queued up, the regular clock
4141 4148 // will abort this task so that it restarts.
4142 4149 drain_satb_buffers();
4143 4150 // ...then partially drain the local queue and the global stack
4144 4151 drain_local_queue(true);
4145 4152 drain_global_stack(true);
4146 4153
4147 4154 // Then totally drain the region stack. We will not look at
4148 4155 // it again before the next invocation of this method. Entries on
4149 4156 // the region stack are only added during evacuation pauses, for
4150 4157 // which we have to yield. When we do, we abort the task anyway so
4151 4158 // it will look at the region stack again when it restarts.
4152 4159 bitmap_closure.set_scanning_heap_region(false);
4153 4160 drain_region_stack(&bitmap_closure);
4154 4161 // ...then partially drain the local queue and the global stack
4155 4162 drain_local_queue(true);
4156 4163 drain_global_stack(true);
4157 4164
4158 4165 do {
4159 4166 if (!has_aborted() && _curr_region != NULL) {
4160 4167 // This means that we're already holding on to a region.
4161 4168 assert(_finger != NULL, "if region is not NULL, then the finger "
4162 4169 "should not be NULL either");
4163 4170
4164 4171 // We might have restarted this task after an evacuation pause
4165 4172 // which might have evacuated the region we're holding on to
4166 4173 // underneath our feet. Let's read its limit again to make sure
4167 4174 // that we do not iterate over a region of the heap that
4168 4175 // contains garbage (update_region_limit() will also move
4169 4176 // _finger to the start of the region if it is found empty).
4170 4177 update_region_limit();
4171 4178 // We will start from _finger not from the start of the region,
4172 4179 // as we might be restarting this task after aborting half-way
4173 4180 // through scanning this region. In this case, _finger points to
4174 4181 // the address where we last found a marked object. If this is a
4175 4182 // fresh region, _finger points to start().
4176 4183 MemRegion mr = MemRegion(_finger, _region_limit);
4177 4184
4178 4185 if (_cm->verbose_low()) {
4179 4186 gclog_or_tty->print_cr("[%d] we're scanning part "
4180 4187 "["PTR_FORMAT", "PTR_FORMAT") "
4181 4188 "of region "PTR_FORMAT,
4182 4189 _task_id, _finger, _region_limit, _curr_region);
4183 4190 }
4184 4191
4185 4192 // Let's iterate over the bitmap of the part of the
4186 4193 // region that is left.
4187 4194 bitmap_closure.set_scanning_heap_region(true);
4188 4195 if (mr.is_empty() ||
4189 4196 _nextMarkBitMap->iterate(&bitmap_closure, mr)) {
4190 4197 // We successfully completed iterating over the region. Now,
4191 4198 // let's give up the region.
4192 4199 giveup_current_region();
4193 4200 regular_clock_call();
4194 4201 } else {
4195 4202 assert(has_aborted(), "currently the only way to do so");
4196 4203 // The only way to abort the bitmap iteration is to return
4197 4204 // false from the do_bit() method. However, inside the
4198 4205 // do_bit() method we move the _finger to point to the
4199 4206 // object currently being looked at. So, if we bail out, we
4200 4207 // have definitely set _finger to something non-null.
4201 4208 assert(_finger != NULL, "invariant");
4202 4209
4203 4210 // Region iteration was actually aborted. So now _finger
4204 4211 // points to the address of the object we last scanned. If we
4205 4212 // leave it there, when we restart this task, we will rescan
4206 4213 // the object. It is easy to avoid this. We move the finger by
4207 4214 // enough to point to the next possible object header (the
4208 4215 // bitmap knows by how much we need to move it as it knows its
4209 4216 // granularity).
4210 4217 assert(_finger < _region_limit, "invariant");
4211 4218 HeapWord* new_finger = _nextMarkBitMap->nextWord(_finger);
4212 4219 // Check if bitmap iteration was aborted while scanning the last object
4213 4220 if (new_finger >= _region_limit) {
4214 4221 giveup_current_region();
4215 4222 } else {
4216 4223 move_finger_to(new_finger);
4217 4224 }
4218 4225 }
4219 4226 }
4220 4227 // At this point we have either completed iterating over the
4221 4228 // region we were holding on to, or we have aborted.
4222 4229
4223 4230 // We then partially drain the local queue and the global stack.
4224 4231 // (Do we really need this?)
4225 4232 drain_local_queue(true);
4226 4233 drain_global_stack(true);
4227 4234
4228 4235 // Read the note on the claim_region() method on why it might
4229 4236 // return NULL with potentially more regions available for
4230 4237 // claiming and why we have to check out_of_regions() to determine
4231 4238 // whether we're done or not.
4232 4239 while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) {
4233 4240 // We are going to try to claim a new region. We should have
4234 4241 // given up on the previous one.
4235 4242 // Separated the asserts so that we know which one fires.
4236 4243 assert(_curr_region == NULL, "invariant");
4237 4244 assert(_finger == NULL, "invariant");
4238 4245 assert(_region_limit == NULL, "invariant");
4239 4246 if (_cm->verbose_low()) {
4240 4247 gclog_or_tty->print_cr("[%d] trying to claim a new region", _task_id);
4241 4248 }
4242 4249 HeapRegion* claimed_region = _cm->claim_region(_task_id);
4243 4250 if (claimed_region != NULL) {
4244 4251 // Yes, we managed to claim one
4245 4252 statsOnly( ++_regions_claimed );
4246 4253
4247 4254 if (_cm->verbose_low()) {
4248 4255 gclog_or_tty->print_cr("[%d] we successfully claimed "
4249 4256 "region "PTR_FORMAT,
4250 4257 _task_id, claimed_region);
4251 4258 }
4252 4259
4253 4260 setup_for_region(claimed_region);
4254 4261 assert(_curr_region == claimed_region, "invariant");
4255 4262 }
4256 4263 // It is important to call the regular clock here. It might take
4257 4264 // a while to claim a region if, for example, we hit a large
4258 4265 // block of empty regions. So we need to call the regular clock
4259 4266 // method once round the loop to make sure it's called
4260 4267 // frequently enough.
4261 4268 regular_clock_call();
4262 4269 }
4263 4270
4264 4271 if (!has_aborted() && _curr_region == NULL) {
4265 4272 assert(_cm->out_of_regions(),
4266 4273 "at this point we should be out of regions");
4267 4274 }
4268 4275 } while ( _curr_region != NULL && !has_aborted());
4269 4276
4270 4277 if (!has_aborted()) {
4271 4278 // We cannot check whether the global stack is empty, since other
4272 4279 // tasks might be pushing objects to it concurrently. We also cannot
4273 4280 // check if the region stack is empty because if a thread is aborting
4274 4281 // it can push a partially done region back.
4275 4282 assert(_cm->out_of_regions(),
4276 4283 "at this point we should be out of regions");
4277 4284
4278 4285 if (_cm->verbose_low()) {
4279 4286 gclog_or_tty->print_cr("[%d] all regions claimed", _task_id);
4280 4287 }
4281 4288
4282 4289 // Try to reduce the number of available SATB buffers so that
4283 4290 // remark has less work to do.
4284 4291 drain_satb_buffers();
4285 4292 }
4286 4293
4287 4294 // Since we've done everything else, we can now totally drain the
4288 4295 // local queue and global stack.
4289 4296 drain_local_queue(false);
4290 4297 drain_global_stack(false);
4291 4298
4292 4299 // Attempt at work stealing from other task's queues.
4293 4300 if (do_stealing && !has_aborted()) {
4294 4301 // We have not aborted. This means that we have finished all that
4295 4302 // we could. Let's try to do some stealing...
4296 4303
4297 4304 // We cannot check whether the global stack is empty, since other
4298 4305 // tasks might be pushing objects to it concurrently. We also cannot
4299 4306 // check if the region stack is empty because if a thread is aborting
4300 4307 // it can push a partially done region back.
4301 4308 assert(_cm->out_of_regions() && _task_queue->size() == 0,
4302 4309 "only way to reach here");
4303 4310
4304 4311 if (_cm->verbose_low()) {
4305 4312 gclog_or_tty->print_cr("[%d] starting to steal", _task_id);
4306 4313 }
4307 4314
4308 4315 while (!has_aborted()) {
4309 4316 oop obj;
4310 4317 statsOnly( ++_steal_attempts );
4311 4318
4312 4319 if (_cm->try_stealing(_task_id, &_hash_seed, obj)) {
4313 4320 if (_cm->verbose_medium()) {
4314 4321 gclog_or_tty->print_cr("[%d] stolen "PTR_FORMAT" successfully",
4315 4322 _task_id, (void*) obj);
4316 4323 }
4317 4324
4318 4325 statsOnly( ++_steals );
4319 4326
4320 4327 assert(_nextMarkBitMap->isMarked((HeapWord*) obj),
4321 4328 "any stolen object should be marked");
4322 4329 scan_object(obj);
4323 4330
4324 4331 // And since we're towards the end, let's totally drain the
4325 4332 // local queue and global stack.
4326 4333 drain_local_queue(false);
4327 4334 drain_global_stack(false);
4328 4335 } else {
4329 4336 break;
4330 4337 }
4331 4338 }
4332 4339 }
4333 4340
4334 4341 // If we are about to wrap up and go into termination, check if we
4335 4342 // should raise the overflow flag.
4336 4343 if (do_termination && !has_aborted()) {
4337 4344 if (_cm->force_overflow()->should_force()) {
4338 4345 _cm->set_has_overflown();
4339 4346 regular_clock_call();
4340 4347 }
4341 4348 }
4342 4349
4343 4350 // We still haven't aborted. Now, let's try to get into the
4344 4351 // termination protocol.
4345 4352 if (do_termination && !has_aborted()) {
4346 4353 // We cannot check whether the global stack is empty, since other
4347 4354 // tasks might be concurrently pushing objects on it. We also cannot
4348 4355 // check if the region stack is empty because if a thread is aborting
4349 4356 // it can push a partially done region back.
4350 4357 // Separated the asserts so that we know which one fires.
4351 4358 assert(_cm->out_of_regions(), "only way to reach here");
4352 4359 assert(_task_queue->size() == 0, "only way to reach here");
4353 4360
4354 4361 if (_cm->verbose_low()) {
4355 4362 gclog_or_tty->print_cr("[%d] starting termination protocol", _task_id);
4356 4363 }
4357 4364
4358 4365 _termination_start_time_ms = os::elapsedVTime() * 1000.0;
4359 4366 // The CMTask class also extends the TerminatorTerminator class,
4360 4367 // hence its should_exit_termination() method will also decide
4361 4368 // whether to exit the termination protocol or not.
4362 4369 bool finished = _cm->terminator()->offer_termination(this);
4363 4370 double termination_end_time_ms = os::elapsedVTime() * 1000.0;
4364 4371 _termination_time_ms +=
4365 4372 termination_end_time_ms - _termination_start_time_ms;
4366 4373
4367 4374 if (finished) {
4368 4375 // We're all done.
4369 4376
4370 4377 if (_task_id == 0) {
4371 4378 // let's allow task 0 to do this
4372 4379 if (concurrent()) {
4373 4380 assert(_cm->concurrent_marking_in_progress(), "invariant");
4374 4381 // we need to set this to false before the next
4375 4382 // safepoint. This way we ensure that the marking phase
4376 4383 // doesn't observe any more heap expansions.
4377 4384 _cm->clear_concurrent_marking_in_progress();
4378 4385 }
4379 4386 }
4380 4387
4381 4388 // We can now guarantee that the global stack is empty, since
4382 4389 // all other tasks have finished. We separated the guarantees so
4383 4390 // that, if a condition is false, we can immediately find out
4384 4391 // which one.
4385 4392 guarantee(_cm->out_of_regions(), "only way to reach here");
4386 4393 guarantee(_aborted_region.is_empty(), "only way to reach here");
4387 4394 guarantee(_cm->region_stack_empty(), "only way to reach here");
4388 4395 guarantee(_cm->mark_stack_empty(), "only way to reach here");
4389 4396 guarantee(_task_queue->size() == 0, "only way to reach here");
4390 4397 guarantee(!_cm->has_overflown(), "only way to reach here");
4391 4398 guarantee(!_cm->mark_stack_overflow(), "only way to reach here");
4392 4399 guarantee(!_cm->region_stack_overflow(), "only way to reach here");
4393 4400
4394 4401 if (_cm->verbose_low()) {
4395 4402 gclog_or_tty->print_cr("[%d] all tasks terminated", _task_id);
4396 4403 }
4397 4404 } else {
4398 4405 // Apparently there's more work to do. Let's abort this task. It
4399 4406 // will restart it and we can hopefully find more things to do.
4400 4407
4401 4408 if (_cm->verbose_low()) {
4402 4409 gclog_or_tty->print_cr("[%d] apparently there is more work to do",
4403 4410 _task_id);
4404 4411 }
4405 4412
4406 4413 set_has_aborted();
4407 4414 statsOnly( ++_aborted_termination );
4408 4415 }
4409 4416 }
4410 4417
4411 4418 // Mainly for debugging purposes to make sure that a pointer to the
4412 4419 // closure which was statically allocated in this frame doesn't
4413 4420 // escape it by accident.
4414 4421 set_cm_oop_closure(NULL);
4415 4422 double end_time_ms = os::elapsedVTime() * 1000.0;
4416 4423 double elapsed_time_ms = end_time_ms - _start_time_ms;
4417 4424 // Update the step history.
4418 4425 _step_times_ms.add(elapsed_time_ms);
4419 4426
4420 4427 if (has_aborted()) {
4421 4428 // The task was aborted for some reason.
4422 4429
4423 4430 statsOnly( ++_aborted );
4424 4431
4425 4432 if (_has_timed_out) {
4426 4433 double diff_ms = elapsed_time_ms - _time_target_ms;
4427 4434 // Keep statistics of how well we did with respect to hitting
4428 4435 // our target only if we actually timed out (if we aborted for
4429 4436 // other reasons, then the results might get skewed).
4430 4437 _marking_step_diffs_ms.add(diff_ms);
4431 4438 }
4432 4439
4433 4440 if (_cm->has_overflown()) {
4434 4441 // This is the interesting one. We aborted because a global
4435 4442 // overflow was raised. This means we have to restart the
4436 4443 // marking phase and start iterating over regions. However, in
4437 4444 // order to do this we have to make sure that all tasks stop
4438 4445 // what they are doing and re-initialise in a safe manner. We
4439 4446 // will achieve this with the use of two barrier sync points.
4440 4447
4441 4448 if (_cm->verbose_low()) {
4442 4449 gclog_or_tty->print_cr("[%d] detected overflow", _task_id);
4443 4450 }
4444 4451
4445 4452 _cm->enter_first_sync_barrier(_task_id);
4446 4453 // When we exit this sync barrier we know that all tasks have
4447 4454 // stopped doing marking work. So, it's now safe to
4448 4455 // re-initialise our data structures. At the end of this method,
4449 4456 // task 0 will clear the global data structures.
4450 4457
4451 4458 statsOnly( ++_aborted_overflow );
4452 4459
4453 4460 // We clear the local state of this task...
4454 4461 clear_region_fields();
4455 4462
4456 4463 // ...and enter the second barrier.
4457 4464 _cm->enter_second_sync_barrier(_task_id);
4458 4465 // At this point everything has bee re-initialised and we're
4459 4466 // ready to restart.
4460 4467 }
4461 4468
4462 4469 if (_cm->verbose_low()) {
4463 4470 gclog_or_tty->print_cr("[%d] <<<<<<<<<< ABORTING, target = %1.2lfms, "
4464 4471 "elapsed = %1.2lfms <<<<<<<<<<",
4465 4472 _task_id, _time_target_ms, elapsed_time_ms);
4466 4473 if (_cm->has_aborted()) {
4467 4474 gclog_or_tty->print_cr("[%d] ========== MARKING ABORTED ==========",
4468 4475 _task_id);
4469 4476 }
4470 4477 }
4471 4478 } else {
4472 4479 if (_cm->verbose_low()) {
4473 4480 gclog_or_tty->print_cr("[%d] <<<<<<<<<< FINISHED, target = %1.2lfms, "
4474 4481 "elapsed = %1.2lfms <<<<<<<<<<",
4475 4482 _task_id, _time_target_ms, elapsed_time_ms);
4476 4483 }
4477 4484 }
4478 4485
4479 4486 _claimed = false;
4480 4487 }
4481 4488
4482 4489 CMTask::CMTask(int task_id,
4483 4490 ConcurrentMark* cm,
4484 4491 CMTaskQueue* task_queue,
4485 4492 CMTaskQueueSet* task_queues)
4486 4493 : _g1h(G1CollectedHeap::heap()),
4487 4494 _task_id(task_id), _cm(cm),
4488 4495 _claimed(false),
4489 4496 _nextMarkBitMap(NULL), _hash_seed(17),
4490 4497 _task_queue(task_queue),
4491 4498 _task_queues(task_queues),
4492 4499 _cm_oop_closure(NULL),
4493 4500 _aborted_region(MemRegion()) {
4494 4501 guarantee(task_queue != NULL, "invariant");
4495 4502 guarantee(task_queues != NULL, "invariant");
4496 4503
4497 4504 statsOnly( _clock_due_to_scanning = 0;
4498 4505 _clock_due_to_marking = 0 );
4499 4506
4500 4507 _marking_step_diffs_ms.add(0.5);
4501 4508 }
4502 4509
4503 4510 // These are formatting macros that are used below to ensure
4504 4511 // consistent formatting. The *_H_* versions are used to format the
4505 4512 // header for a particular value and they should be kept consistent
4506 4513 // with the corresponding macro. Also note that most of the macros add
4507 4514 // the necessary white space (as a prefix) which makes them a bit
4508 4515 // easier to compose.
4509 4516
4510 4517 // All the output lines are prefixed with this string to be able to
4511 4518 // identify them easily in a large log file.
4512 4519 #define G1PPRL_LINE_PREFIX "###"
4513 4520
4514 4521 #define G1PPRL_ADDR_BASE_FORMAT " "PTR_FORMAT"-"PTR_FORMAT
4515 4522 #ifdef _LP64
4516 4523 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
4517 4524 #else // _LP64
4518 4525 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s"
4519 4526 #endif // _LP64
4520 4527
4521 4528 // For per-region info
4522 4529 #define G1PPRL_TYPE_FORMAT " %-4s"
4523 4530 #define G1PPRL_TYPE_H_FORMAT " %4s"
4524 4531 #define G1PPRL_BYTE_FORMAT " "SIZE_FORMAT_W(9)
4525 4532 #define G1PPRL_BYTE_H_FORMAT " %9s"
4526 4533 #define G1PPRL_DOUBLE_FORMAT " %14.1f"
4527 4534 #define G1PPRL_DOUBLE_H_FORMAT " %14s"
4528 4535
4529 4536 // For summary info
4530 4537 #define G1PPRL_SUM_ADDR_FORMAT(tag) " "tag":"G1PPRL_ADDR_BASE_FORMAT
4531 4538 #define G1PPRL_SUM_BYTE_FORMAT(tag) " "tag": "SIZE_FORMAT
4532 4539 #define G1PPRL_SUM_MB_FORMAT(tag) " "tag": %1.2f MB"
4533 4540 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag)" / %1.2f %%"
4534 4541
4535 4542 G1PrintRegionLivenessInfoClosure::
4536 4543 G1PrintRegionLivenessInfoClosure(outputStream* out, const char* phase_name)
4537 4544 : _out(out),
4538 4545 _total_used_bytes(0), _total_capacity_bytes(0),
4539 4546 _total_prev_live_bytes(0), _total_next_live_bytes(0),
4540 4547 _hum_used_bytes(0), _hum_capacity_bytes(0),
4541 4548 _hum_prev_live_bytes(0), _hum_next_live_bytes(0) {
4542 4549 G1CollectedHeap* g1h = G1CollectedHeap::heap();
4543 4550 MemRegion g1_committed = g1h->g1_committed();
4544 4551 MemRegion g1_reserved = g1h->g1_reserved();
4545 4552 double now = os::elapsedTime();
4546 4553
4547 4554 // Print the header of the output.
4548 4555 _out->cr();
4549 4556 _out->print_cr(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
4550 4557 _out->print_cr(G1PPRL_LINE_PREFIX" HEAP"
4551 4558 G1PPRL_SUM_ADDR_FORMAT("committed")
4552 4559 G1PPRL_SUM_ADDR_FORMAT("reserved")
4553 4560 G1PPRL_SUM_BYTE_FORMAT("region-size"),
4554 4561 g1_committed.start(), g1_committed.end(),
4555 4562 g1_reserved.start(), g1_reserved.end(),
4556 4563 HeapRegion::GrainBytes);
4557 4564 _out->print_cr(G1PPRL_LINE_PREFIX);
4558 4565 _out->print_cr(G1PPRL_LINE_PREFIX
4559 4566 G1PPRL_TYPE_H_FORMAT
4560 4567 G1PPRL_ADDR_BASE_H_FORMAT
4561 4568 G1PPRL_BYTE_H_FORMAT
4562 4569 G1PPRL_BYTE_H_FORMAT
4563 4570 G1PPRL_BYTE_H_FORMAT
4564 4571 G1PPRL_DOUBLE_H_FORMAT,
4565 4572 "type", "address-range",
4566 4573 "used", "prev-live", "next-live", "gc-eff");
4567 4574 }
4568 4575
4569 4576 // It takes as a parameter a reference to one of the _hum_* fields, it
4570 4577 // deduces the corresponding value for a region in a humongous region
4571 4578 // series (either the region size, or what's left if the _hum_* field
4572 4579 // is < the region size), and updates the _hum_* field accordingly.
4573 4580 size_t G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* hum_bytes) {
4574 4581 size_t bytes = 0;
4575 4582 // The > 0 check is to deal with the prev and next live bytes which
4576 4583 // could be 0.
4577 4584 if (*hum_bytes > 0) {
4578 4585 bytes = MIN2((size_t) HeapRegion::GrainBytes, *hum_bytes);
4579 4586 *hum_bytes -= bytes;
4580 4587 }
4581 4588 return bytes;
4582 4589 }
4583 4590
4584 4591 // It deduces the values for a region in a humongous region series
4585 4592 // from the _hum_* fields and updates those accordingly. It assumes
4586 4593 // that that _hum_* fields have already been set up from the "starts
4587 4594 // humongous" region and we visit the regions in address order.
4588 4595 void G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* used_bytes,
4589 4596 size_t* capacity_bytes,
4590 4597 size_t* prev_live_bytes,
4591 4598 size_t* next_live_bytes) {
4592 4599 assert(_hum_used_bytes > 0 && _hum_capacity_bytes > 0, "pre-condition");
4593 4600 *used_bytes = get_hum_bytes(&_hum_used_bytes);
4594 4601 *capacity_bytes = get_hum_bytes(&_hum_capacity_bytes);
4595 4602 *prev_live_bytes = get_hum_bytes(&_hum_prev_live_bytes);
4596 4603 *next_live_bytes = get_hum_bytes(&_hum_next_live_bytes);
4597 4604 }
4598 4605
4599 4606 bool G1PrintRegionLivenessInfoClosure::doHeapRegion(HeapRegion* r) {
4600 4607 const char* type = "";
4601 4608 HeapWord* bottom = r->bottom();
4602 4609 HeapWord* end = r->end();
4603 4610 size_t capacity_bytes = r->capacity();
4604 4611 size_t used_bytes = r->used();
4605 4612 size_t prev_live_bytes = r->live_bytes();
4606 4613 size_t next_live_bytes = r->next_live_bytes();
4607 4614 double gc_eff = r->gc_efficiency();
4608 4615 if (r->used() == 0) {
4609 4616 type = "FREE";
4610 4617 } else if (r->is_survivor()) {
4611 4618 type = "SURV";
4612 4619 } else if (r->is_young()) {
4613 4620 type = "EDEN";
4614 4621 } else if (r->startsHumongous()) {
4615 4622 type = "HUMS";
4616 4623
4617 4624 assert(_hum_used_bytes == 0 && _hum_capacity_bytes == 0 &&
4618 4625 _hum_prev_live_bytes == 0 && _hum_next_live_bytes == 0,
4619 4626 "they should have been zeroed after the last time we used them");
4620 4627 // Set up the _hum_* fields.
4621 4628 _hum_capacity_bytes = capacity_bytes;
4622 4629 _hum_used_bytes = used_bytes;
4623 4630 _hum_prev_live_bytes = prev_live_bytes;
4624 4631 _hum_next_live_bytes = next_live_bytes;
4625 4632 get_hum_bytes(&used_bytes, &capacity_bytes,
4626 4633 &prev_live_bytes, &next_live_bytes);
4627 4634 end = bottom + HeapRegion::GrainWords;
4628 4635 } else if (r->continuesHumongous()) {
4629 4636 type = "HUMC";
4630 4637 get_hum_bytes(&used_bytes, &capacity_bytes,
4631 4638 &prev_live_bytes, &next_live_bytes);
4632 4639 assert(end == bottom + HeapRegion::GrainWords, "invariant");
4633 4640 } else {
4634 4641 type = "OLD";
4635 4642 }
4636 4643
4637 4644 _total_used_bytes += used_bytes;
4638 4645 _total_capacity_bytes += capacity_bytes;
4639 4646 _total_prev_live_bytes += prev_live_bytes;
4640 4647 _total_next_live_bytes += next_live_bytes;
4641 4648
4642 4649 // Print a line for this particular region.
4643 4650 _out->print_cr(G1PPRL_LINE_PREFIX
4644 4651 G1PPRL_TYPE_FORMAT
4645 4652 G1PPRL_ADDR_BASE_FORMAT
4646 4653 G1PPRL_BYTE_FORMAT
4647 4654 G1PPRL_BYTE_FORMAT
4648 4655 G1PPRL_BYTE_FORMAT
4649 4656 G1PPRL_DOUBLE_FORMAT,
4650 4657 type, bottom, end,
4651 4658 used_bytes, prev_live_bytes, next_live_bytes, gc_eff);
4652 4659
4653 4660 return false;
4654 4661 }
4655 4662
4656 4663 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
4657 4664 // Print the footer of the output.
4658 4665 _out->print_cr(G1PPRL_LINE_PREFIX);
4659 4666 _out->print_cr(G1PPRL_LINE_PREFIX
4660 4667 " SUMMARY"
4661 4668 G1PPRL_SUM_MB_FORMAT("capacity")
4662 4669 G1PPRL_SUM_MB_PERC_FORMAT("used")
4663 4670 G1PPRL_SUM_MB_PERC_FORMAT("prev-live")
4664 4671 G1PPRL_SUM_MB_PERC_FORMAT("next-live"),
4665 4672 bytes_to_mb(_total_capacity_bytes),
4666 4673 bytes_to_mb(_total_used_bytes),
4667 4674 perc(_total_used_bytes, _total_capacity_bytes),
4668 4675 bytes_to_mb(_total_prev_live_bytes),
4669 4676 perc(_total_prev_live_bytes, _total_capacity_bytes),
4670 4677 bytes_to_mb(_total_next_live_bytes),
4671 4678 perc(_total_next_live_bytes, _total_capacity_bytes));
4672 4679 _out->cr();
4673 4680 }
|
↓ open down ↓ |
1331 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX