1 /*
2 * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/metadataOnStackMark.hpp"
27 #include "classfile/symbolTable.hpp"
28 #include "code/codeCache.hpp"
29 #include "gc/g1/concurrentMarkThread.inline.hpp"
30 #include "gc/g1/g1CollectedHeap.inline.hpp"
31 #include "gc/g1/g1CollectorState.hpp"
32 #include "gc/g1/g1ConcurrentMark.inline.hpp"
33 #include "gc/g1/g1HeapVerifier.hpp"
34 #include "gc/g1/g1OopClosures.inline.hpp"
35 #include "gc/g1/g1CardLiveData.inline.hpp"
36 #include "gc/g1/g1Policy.hpp"
37 #include "gc/g1/g1StringDedup.hpp"
38 #include "gc/g1/heapRegion.inline.hpp"
39 #include "gc/g1/heapRegionRemSet.hpp"
40 #include "gc/g1/heapRegionSet.inline.hpp"
41 #include "gc/shared/adaptiveSizePolicy.hpp"
42 #include "gc/shared/gcId.hpp"
43 #include "gc/shared/gcTimer.hpp"
44 #include "gc/shared/gcTrace.hpp"
45 #include "gc/shared/gcTraceTime.inline.hpp"
46 #include "gc/shared/genOopClosures.inline.hpp"
47 #include "gc/shared/referencePolicy.hpp"
48 #include "gc/shared/strongRootsScope.hpp"
49 #include "gc/shared/suspendibleThreadSet.hpp"
50 #include "gc/shared/taskqueue.inline.hpp"
51 #include "gc/shared/vmGCOperations.hpp"
52 #include "gc/shared/weakProcessor.hpp"
53 #include "logging/log.hpp"
54 #include "memory/allocation.hpp"
55 #include "memory/resourceArea.hpp"
56 #include "oops/oop.inline.hpp"
57 #include "runtime/atomic.hpp"
58 #include "runtime/handles.inline.hpp"
59 #include "runtime/java.hpp"
60 #include "runtime/prefetch.inline.hpp"
61 #include "services/memTracker.hpp"
62 #include "utilities/align.hpp"
63 #include "utilities/growableArray.hpp"
64
65 bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
66 assert(addr < _cm->finger(), "invariant");
67 assert(addr >= _task->finger(), "invariant");
68
69 // We move that task's local finger along.
70 _task->move_finger_to(addr);
71
72 _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr)));
73 // we only partially drain the local queue and global stack
74 _task->drain_local_queue(true);
75 _task->drain_global_stack(true);
76
77 // if the has_aborted flag has been raised, we need to bail out of
78 // the iteration
79 return !_task->has_aborted();
80 }
81
82 G1CMMarkStack::G1CMMarkStack() :
83 _max_chunk_capacity(0),
84 _base(NULL),
85 _chunk_capacity(0) {
86 set_empty();
87 }
88
89 bool G1CMMarkStack::resize(size_t new_capacity) {
90 assert(is_empty(), "Only resize when stack is empty.");
91 assert(new_capacity <= _max_chunk_capacity,
92 "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity);
93
94 TaskQueueEntryChunk* new_base = MmapArrayAllocator<TaskQueueEntryChunk>::allocate_or_null(new_capacity, mtGC);
95
96 if (new_base == NULL) {
97 log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
98 return false;
99 }
100 // Release old mapping.
101 if (_base != NULL) {
102 MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity);
103 }
104
105 _base = new_base;
106 _chunk_capacity = new_capacity;
107 set_empty();
108
109 return true;
110 }
111
112 size_t G1CMMarkStack::capacity_alignment() {
113 return (size_t)lcm(os::vm_allocation_granularity(), sizeof(TaskQueueEntryChunk)) / sizeof(G1TaskQueueEntry);
114 }
115
116 bool G1CMMarkStack::initialize(size_t initial_capacity, size_t max_capacity) {
117 guarantee(_max_chunk_capacity == 0, "G1CMMarkStack already initialized.");
118
119 size_t const TaskEntryChunkSizeInVoidStar = sizeof(TaskQueueEntryChunk) / sizeof(G1TaskQueueEntry);
120
121 _max_chunk_capacity = align_up(max_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
122 size_t initial_chunk_capacity = align_up(initial_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
123
124 guarantee(initial_chunk_capacity <= _max_chunk_capacity,
125 "Maximum chunk capacity " SIZE_FORMAT " smaller than initial capacity " SIZE_FORMAT,
126 _max_chunk_capacity,
127 initial_chunk_capacity);
128
129 log_debug(gc)("Initialize mark stack with " SIZE_FORMAT " chunks, maximum " SIZE_FORMAT,
130 initial_chunk_capacity, _max_chunk_capacity);
131
132 return resize(initial_chunk_capacity);
133 }
134
135 void G1CMMarkStack::expand() {
136 if (_chunk_capacity == _max_chunk_capacity) {
137 log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _chunk_capacity);
138 return;
139 }
140 size_t old_capacity = _chunk_capacity;
141 // Double capacity if possible
142 size_t new_capacity = MIN2(old_capacity * 2, _max_chunk_capacity);
143
144 if (resize(new_capacity)) {
145 log_debug(gc)("Expanded mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
146 old_capacity, new_capacity);
147 } else {
148 log_warning(gc)("Failed to expand mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
149 old_capacity, new_capacity);
150 }
151 }
152
153 G1CMMarkStack::~G1CMMarkStack() {
154 if (_base != NULL) {
155 MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity);
156 }
157 }
158
159 void G1CMMarkStack::add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem) {
160 elem->next = *list;
161 *list = elem;
162 }
163
164 void G1CMMarkStack::add_chunk_to_chunk_list(TaskQueueEntryChunk* elem) {
165 MutexLockerEx x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
166 add_chunk_to_list(&_chunk_list, elem);
167 _chunks_in_chunk_list++;
168 }
169
170 void G1CMMarkStack::add_chunk_to_free_list(TaskQueueEntryChunk* elem) {
171 MutexLockerEx x(MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag);
172 add_chunk_to_list(&_free_list, elem);
173 }
174
175 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_list(TaskQueueEntryChunk* volatile* list) {
176 TaskQueueEntryChunk* result = *list;
177 if (result != NULL) {
178 *list = (*list)->next;
179 }
180 return result;
181 }
182
183 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_chunk_list() {
184 MutexLockerEx x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
185 TaskQueueEntryChunk* result = remove_chunk_from_list(&_chunk_list);
186 if (result != NULL) {
187 _chunks_in_chunk_list--;
188 }
189 return result;
190 }
191
192 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_free_list() {
193 MutexLockerEx x(MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag);
194 return remove_chunk_from_list(&_free_list);
195 }
196
197 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::allocate_new_chunk() {
198 // This dirty read of _hwm is okay because we only ever increase the _hwm in parallel code.
199 // Further this limits _hwm to a value of _chunk_capacity + #threads, avoiding
200 // wraparound of _hwm.
201 if (_hwm >= _chunk_capacity) {
202 return NULL;
203 }
204
205 size_t cur_idx = Atomic::add(1u, &_hwm) - 1;
206 if (cur_idx >= _chunk_capacity) {
207 return NULL;
208 }
209
210 TaskQueueEntryChunk* result = ::new (&_base[cur_idx]) TaskQueueEntryChunk;
211 result->next = NULL;
212 return result;
213 }
214
215 bool G1CMMarkStack::par_push_chunk(G1TaskQueueEntry* ptr_arr) {
216 // Get a new chunk.
217 TaskQueueEntryChunk* new_chunk = remove_chunk_from_free_list();
218
219 if (new_chunk == NULL) {
220 // Did not get a chunk from the free list. Allocate from backing memory.
221 new_chunk = allocate_new_chunk();
222
223 if (new_chunk == NULL) {
224 return false;
225 }
226 }
227
228 Copy::conjoint_memory_atomic(ptr_arr, new_chunk->data, EntriesPerChunk * sizeof(G1TaskQueueEntry));
229
230 add_chunk_to_chunk_list(new_chunk);
231
232 return true;
233 }
234
235 bool G1CMMarkStack::par_pop_chunk(G1TaskQueueEntry* ptr_arr) {
236 TaskQueueEntryChunk* cur = remove_chunk_from_chunk_list();
237
238 if (cur == NULL) {
239 return false;
240 }
241
242 Copy::conjoint_memory_atomic(cur->data, ptr_arr, EntriesPerChunk * sizeof(G1TaskQueueEntry));
243
244 add_chunk_to_free_list(cur);
245 return true;
246 }
247
248 void G1CMMarkStack::set_empty() {
249 _chunks_in_chunk_list = 0;
250 _hwm = 0;
251 _chunk_list = NULL;
252 _free_list = NULL;
253 }
254
255 G1CMRootRegions::G1CMRootRegions() :
256 _cm(NULL), _scan_in_progress(false),
257 _should_abort(false), _claimed_survivor_index(0) { }
258
259 void G1CMRootRegions::init(const G1SurvivorRegions* survivors, G1ConcurrentMark* cm) {
260 _survivors = survivors;
261 _cm = cm;
262 }
263
264 void G1CMRootRegions::prepare_for_scan() {
265 assert(!scan_in_progress(), "pre-condition");
266
267 // Currently, only survivors can be root regions.
268 _claimed_survivor_index = 0;
269 _scan_in_progress = _survivors->regions()->is_nonempty();
270 _should_abort = false;
271 }
272
273 HeapRegion* G1CMRootRegions::claim_next() {
274 if (_should_abort) {
275 // If someone has set the should_abort flag, we return NULL to
276 // force the caller to bail out of their loop.
277 return NULL;
278 }
279
280 // Currently, only survivors can be root regions.
281 const GrowableArray<HeapRegion*>* survivor_regions = _survivors->regions();
282
283 int claimed_index = Atomic::add(1, &_claimed_survivor_index) - 1;
284 if (claimed_index < survivor_regions->length()) {
285 return survivor_regions->at(claimed_index);
286 }
287 return NULL;
288 }
289
290 uint G1CMRootRegions::num_root_regions() const {
291 return (uint)_survivors->regions()->length();
292 }
293
294 void G1CMRootRegions::notify_scan_done() {
295 MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
296 _scan_in_progress = false;
297 RootRegionScan_lock->notify_all();
298 }
299
300 void G1CMRootRegions::cancel_scan() {
301 notify_scan_done();
302 }
303
304 void G1CMRootRegions::scan_finished() {
305 assert(scan_in_progress(), "pre-condition");
306
307 // Currently, only survivors can be root regions.
308 if (!_should_abort) {
309 assert(_claimed_survivor_index >= 0, "otherwise comparison is invalid: %d", _claimed_survivor_index);
310 assert((uint)_claimed_survivor_index >= _survivors->length(),
311 "we should have claimed all survivors, claimed index = %u, length = %u",
312 (uint)_claimed_survivor_index, _survivors->length());
313 }
314
315 notify_scan_done();
316 }
317
318 bool G1CMRootRegions::wait_until_scan_finished() {
319 if (!scan_in_progress()) return false;
320
321 {
322 MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
323 while (scan_in_progress()) {
324 RootRegionScan_lock->wait(Mutex::_no_safepoint_check_flag);
325 }
326 }
327 return true;
328 }
329
330 // Returns the maximum number of workers to be used in a concurrent
331 // phase based on the number of GC workers being used in a STW
332 // phase.
333 static uint scale_concurrent_worker_threads(uint num_gc_workers) {
334 return MAX2((num_gc_workers + 2) / 4, 1U);
335 }
336
337 G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
338 G1RegionToSpaceMapper* prev_bitmap_storage,
339 G1RegionToSpaceMapper* next_bitmap_storage) :
340 // _cm_thread set inside the constructor
341 _g1h(g1h),
342 _completed_initialization(false),
343
344 _cleanup_list("Concurrent Mark Cleanup List"),
345 _mark_bitmap_1(),
346 _mark_bitmap_2(),
347 _prev_mark_bitmap(&_mark_bitmap_1),
348 _next_mark_bitmap(&_mark_bitmap_2),
349
350 _heap_start(_g1h->reserved_region().start()),
351 _heap_end(_g1h->reserved_region().end()),
352
353 _root_regions(),
354
355 _global_mark_stack(),
356
357 // _finger set in set_non_marking_state
358
359 _max_num_tasks(ParallelGCThreads),
360 // _num_active_tasks set in set_non_marking_state()
361 // _tasks set inside the constructor
362
363 _task_queues(new G1CMTaskQueueSet((int) _max_num_tasks)),
364 _terminator(ParallelTaskTerminator((int) _max_num_tasks, _task_queues)),
365
366 _first_overflow_barrier_sync(),
367 _second_overflow_barrier_sync(),
368
369 _has_overflown(false),
370 _concurrent(false),
371 _has_aborted(false),
372 _restart_for_overflow(false),
373 _concurrent_marking_in_progress(false),
374 _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
375 _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) G1OldTracer()),
376
377 // _verbose_level set below
378
379 _init_times(),
380 _remark_times(),
381 _remark_mark_times(),
382 _remark_weak_ref_times(),
383 _cleanup_times(),
384 _total_counting_time(0.0),
385 _total_rs_scrub_time(0.0),
386
387 _accum_task_vtime(NULL),
388
389 _concurrent_workers(NULL),
390 _num_concurrent_workers(0),
391 _max_concurrent_workers(0)
392 {
393 _mark_bitmap_1.initialize(g1h->reserved_region(), prev_bitmap_storage);
394 _mark_bitmap_2.initialize(g1h->reserved_region(), next_bitmap_storage);
395
396 // Create & start ConcurrentMark thread.
397 _cm_thread = new ConcurrentMarkThread(this);
398 if (_cm_thread->osthread() == NULL) {
399 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
400 }
401
402 assert(CGC_lock != NULL, "CGC_lock must be initialized");
403
404 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
405 satb_qs.set_buffer_size(G1SATBBufferSize);
406
407 _root_regions.init(_g1h->survivor(), this);
408
409 if (FLAG_IS_DEFAULT(ConcGCThreads) || ConcGCThreads == 0) {
410 // Calculate the number of concurrent worker threads by scaling
411 // the number of parallel GC threads.
412 uint marking_thread_num = scale_concurrent_worker_threads(ParallelGCThreads);
413 FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num);
414 }
415
416 assert(ConcGCThreads > 0, "ConcGCThreads have been set.");
417 if (ConcGCThreads > ParallelGCThreads) {
418 log_warning(gc)("More ConcGCThreads (%u) than ParallelGCThreads (%u).",
419 ConcGCThreads, ParallelGCThreads);
420 return;
421 }
422
423 log_debug(gc)("ConcGCThreads: %u", ConcGCThreads);
424 log_debug(gc)("ParallelGCThreads: %u", ParallelGCThreads);
425
426 _num_concurrent_workers = ConcGCThreads;
427 _max_concurrent_workers = _num_concurrent_workers;
428
429 _concurrent_workers = new WorkGang("G1 Conc", _max_concurrent_workers, false, true);
430 _concurrent_workers->initialize_workers();
431
432 if (FLAG_IS_DEFAULT(MarkStackSize)) {
433 size_t mark_stack_size =
434 MIN2(MarkStackSizeMax,
435 MAX2(MarkStackSize, (size_t) (_max_concurrent_workers * TASKQUEUE_SIZE)));
436 // Verify that the calculated value for MarkStackSize is in range.
437 // It would be nice to use the private utility routine from Arguments.
438 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
439 log_warning(gc)("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): "
440 "must be between 1 and " SIZE_FORMAT,
441 mark_stack_size, MarkStackSizeMax);
442 return;
443 }
444 FLAG_SET_ERGO(size_t, MarkStackSize, mark_stack_size);
445 } else {
446 // Verify MarkStackSize is in range.
447 if (FLAG_IS_CMDLINE(MarkStackSize)) {
448 if (FLAG_IS_DEFAULT(MarkStackSizeMax)) {
449 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
450 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT "): "
451 "must be between 1 and " SIZE_FORMAT,
452 MarkStackSize, MarkStackSizeMax);
453 return;
454 }
455 } else if (FLAG_IS_CMDLINE(MarkStackSizeMax)) {
456 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
457 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT ")"
458 " or for MarkStackSizeMax (" SIZE_FORMAT ")",
459 MarkStackSize, MarkStackSizeMax);
460 return;
461 }
462 }
463 }
464 }
465
466 if (!_global_mark_stack.initialize(MarkStackSize, MarkStackSizeMax)) {
467 vm_exit_during_initialization("Failed to allocate initial concurrent mark overflow mark stack.");
468 }
469
470 _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_num_tasks, mtGC);
471 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_num_tasks, mtGC);
472
473 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
474 _num_active_tasks = _max_num_tasks;
475
476 for (uint i = 0; i < _max_num_tasks; ++i) {
477 G1CMTaskQueue* task_queue = new G1CMTaskQueue();
478 task_queue->initialize();
479 _task_queues->register_queue(i, task_queue);
480
481 _tasks[i] = new G1CMTask(i, this, task_queue);
482
483 _accum_task_vtime[i] = 0.0;
484 }
485
486 set_non_marking_state();
487 _completed_initialization = true;
488 }
489
490 void G1ConcurrentMark::reset() {
491 // Starting values for these two. This should be called in a STW
492 // phase.
493 MemRegion reserved = _g1h->g1_reserved();
494 _heap_start = reserved.start();
495 _heap_end = reserved.end();
496
497 // Separated the asserts so that we know which one fires.
498 assert(_heap_start != NULL, "heap bounds should look ok");
499 assert(_heap_end != NULL, "heap bounds should look ok");
500 assert(_heap_start < _heap_end, "heap bounds should look ok");
501
502 // Reset all the marking data structures and any necessary flags
503 reset_marking_state();
504
505 // We reset all of them, since different phases will use
506 // different number of active threads. So, it's easiest to have all
507 // of them ready.
508 for (uint i = 0; i < _max_num_tasks; ++i) {
509 _tasks[i]->reset(_next_mark_bitmap);
510 }
511
512 // we need this to make sure that the flag is on during the evac
513 // pause with initial mark piggy-backed
514 set_concurrent_marking_in_progress();
515 }
516
517
518 void G1ConcurrentMark::reset_marking_state() {
519 _global_mark_stack.set_empty();
520
521 // Expand the marking stack, if we have to and if we can.
522 if (has_overflown()) {
523 _global_mark_stack.expand();
524 }
525
526 clear_has_overflown();
527 _finger = _heap_start;
528
529 for (uint i = 0; i < _max_num_tasks; ++i) {
530 G1CMTaskQueue* queue = _task_queues->queue(i);
531 queue->set_empty();
532 }
533 }
534
535 void G1ConcurrentMark::set_concurrency(uint active_tasks) {
536 assert(active_tasks <= _max_num_tasks, "we should not have more");
537
538 _num_active_tasks = active_tasks;
539 // Need to update the three data structures below according to the
540 // number of active threads for this phase.
541 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues);
542 _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
543 _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
544 }
545
546 void G1ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurrent) {
547 set_concurrency(active_tasks);
548
549 _concurrent = concurrent;
550 // We propagate this to all tasks, not just the active ones.
551 for (uint i = 0; i < _max_num_tasks; ++i) {
552 _tasks[i]->set_concurrent(concurrent);
553 }
554
555 if (concurrent) {
556 set_concurrent_marking_in_progress();
557 } else {
558 // We currently assume that the concurrent flag has been set to
559 // false before we start remark. At this point we should also be
560 // in a STW phase.
561 assert(!concurrent_marking_in_progress(), "invariant");
562 assert(out_of_regions(),
563 "only way to get here: _finger: " PTR_FORMAT ", _heap_end: " PTR_FORMAT,
564 p2i(_finger), p2i(_heap_end));
565 }
566 }
567
568 void G1ConcurrentMark::set_non_marking_state() {
569 // We set the global marking state to some default values when we're
570 // not doing marking.
571 reset_marking_state();
572 _num_active_tasks = 0;
573 clear_concurrent_marking_in_progress();
574 }
575
576 G1ConcurrentMark::~G1ConcurrentMark() {
577 // The G1ConcurrentMark instance is never freed.
578 ShouldNotReachHere();
579 }
580
581 class G1ClearBitMapTask : public AbstractGangTask {
582 public:
583 static size_t chunk_size() { return M; }
584
585 private:
586 // Heap region closure used for clearing the given mark bitmap.
587 class G1ClearBitmapHRClosure : public HeapRegionClosure {
588 private:
589 G1CMBitMap* _bitmap;
590 G1ConcurrentMark* _cm;
591 public:
592 G1ClearBitmapHRClosure(G1CMBitMap* bitmap, G1ConcurrentMark* cm) : HeapRegionClosure(), _cm(cm), _bitmap(bitmap) {
593 }
594
595 virtual bool do_heap_region(HeapRegion* r) {
596 size_t const chunk_size_in_words = G1ClearBitMapTask::chunk_size() / HeapWordSize;
597
598 HeapWord* cur = r->bottom();
599 HeapWord* const end = r->end();
600
601 while (cur < end) {
602 MemRegion mr(cur, MIN2(cur + chunk_size_in_words, end));
603 _bitmap->clear_range(mr);
604
605 cur += chunk_size_in_words;
606
607 // Abort iteration if after yielding the marking has been aborted.
608 if (_cm != NULL && _cm->do_yield_check() && _cm->has_aborted()) {
609 return true;
610 }
611 // Repeat the asserts from before the start of the closure. We will do them
612 // as asserts here to minimize their overhead on the product. However, we
613 // will have them as guarantees at the beginning / end of the bitmap
614 // clearing to get some checking in the product.
615 assert(_cm == NULL || _cm->cm_thread()->during_cycle(), "invariant");
616 assert(_cm == NULL || !G1CollectedHeap::heap()->collector_state()->mark_in_progress(), "invariant");
617 }
618 assert(cur == end, "Must have completed iteration over the bitmap for region %u.", r->hrm_index());
619
620 return false;
621 }
622 };
623
624 G1ClearBitmapHRClosure _cl;
625 HeapRegionClaimer _hr_claimer;
626 bool _suspendible; // If the task is suspendible, workers must join the STS.
627
628 public:
629 G1ClearBitMapTask(G1CMBitMap* bitmap, G1ConcurrentMark* cm, uint n_workers, bool suspendible) :
630 AbstractGangTask("G1 Clear Bitmap"),
631 _cl(bitmap, suspendible ? cm : NULL),
632 _hr_claimer(n_workers),
633 _suspendible(suspendible)
634 { }
635
636 void work(uint worker_id) {
637 SuspendibleThreadSetJoiner sts_join(_suspendible);
638 G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_cl, &_hr_claimer, worker_id);
639 }
640
641 bool is_complete() {
642 return _cl.is_complete();
643 }
644 };
645
646 void G1ConcurrentMark::clear_bitmap(G1CMBitMap* bitmap, WorkGang* workers, bool may_yield) {
647 assert(may_yield || SafepointSynchronize::is_at_safepoint(), "Non-yielding bitmap clear only allowed at safepoint.");
648
649 size_t const num_bytes_to_clear = (HeapRegion::GrainBytes * _g1h->num_regions()) / G1CMBitMap::heap_map_factor();
650 size_t const num_chunks = align_up(num_bytes_to_clear, G1ClearBitMapTask::chunk_size()) / G1ClearBitMapTask::chunk_size();
651
652 uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers());
653
654 G1ClearBitMapTask cl(bitmap, this, num_workers, may_yield);
655
656 log_debug(gc, ergo)("Running %s with %u workers for " SIZE_FORMAT " work units.", cl.name(), num_workers, num_chunks);
657 workers->run_task(&cl, num_workers);
658 guarantee(!may_yield || cl.is_complete(), "Must have completed iteration when not yielding.");
659 }
660
661 void G1ConcurrentMark::cleanup_for_next_mark() {
662 // Make sure that the concurrent mark thread looks to still be in
663 // the current cycle.
664 guarantee(cm_thread()->during_cycle(), "invariant");
665
666 // We are finishing up the current cycle by clearing the next
667 // marking bitmap and getting it ready for the next cycle. During
668 // this time no other cycle can start. So, let's make sure that this
669 // is the case.
670 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
671
672 clear_bitmap(_next_mark_bitmap, _concurrent_workers, true);
673
674 // Clear the live count data. If the marking has been aborted, the abort()
675 // call already did that.
676 if (!has_aborted()) {
677 clear_live_data(_concurrent_workers);
678 DEBUG_ONLY(verify_live_data_clear());
679 }
680
681 // Repeat the asserts from above.
682 guarantee(cm_thread()->during_cycle(), "invariant");
683 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
684 }
685
686 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) {
687 assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint.");
688 clear_bitmap(_prev_mark_bitmap, workers, false);
689 }
690
691 class CheckBitmapClearHRClosure : public HeapRegionClosure {
692 G1CMBitMap* _bitmap;
693 bool _error;
694 public:
695 CheckBitmapClearHRClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) {
696 }
697
698 virtual bool do_heap_region(HeapRegion* r) {
699 // This closure can be called concurrently to the mutator, so we must make sure
700 // that the result of the getNextMarkedWordAddress() call is compared to the
701 // value passed to it as limit to detect any found bits.
702 // end never changes in G1.
703 HeapWord* end = r->end();
704 return _bitmap->get_next_marked_addr(r->bottom(), end) != end;
705 }
706 };
707
708 bool G1ConcurrentMark::next_mark_bitmap_is_clear() {
709 CheckBitmapClearHRClosure cl(_next_mark_bitmap);
710 _g1h->heap_region_iterate(&cl);
711 return cl.is_complete();
712 }
713
714 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
715 public:
716 bool do_heap_region(HeapRegion* r) {
717 r->note_start_of_marking();
718 return false;
719 }
720 };
721
722 void G1ConcurrentMark::checkpoint_roots_initial_pre() {
723 G1CollectedHeap* g1h = G1CollectedHeap::heap();
724
725 _has_aborted = false;
726
727 // Initialize marking structures. This has to be done in a STW phase.
728 reset();
729
730 // For each region note start of marking.
731 NoteStartOfMarkHRClosure startcl;
732 g1h->heap_region_iterate(&startcl);
733 }
734
735
736 void G1ConcurrentMark::checkpoint_roots_initial_post() {
737 G1CollectedHeap* g1h = G1CollectedHeap::heap();
738
739 // Start Concurrent Marking weak-reference discovery.
740 ReferenceProcessor* rp = g1h->ref_processor_cm();
741 // enable ("weak") refs discovery
742 rp->enable_discovery();
743 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
744
745 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
746 // This is the start of the marking cycle, we're expected all
747 // threads to have SATB queues with active set to false.
748 satb_mq_set.set_active_all_threads(true, /* new active value */
749 false /* expected_active */);
750
751 _root_regions.prepare_for_scan();
752
753 // update_g1_committed() will be called at the end of an evac pause
754 // when marking is on. So, it's also called at the end of the
755 // initial-mark pause to update the heap end, if the heap expands
756 // during it. No need to call it here.
757 }
758
759 /*
760 * Notice that in the next two methods, we actually leave the STS
761 * during the barrier sync and join it immediately afterwards. If we
762 * do not do this, the following deadlock can occur: one thread could
763 * be in the barrier sync code, waiting for the other thread to also
764 * sync up, whereas another one could be trying to yield, while also
765 * waiting for the other threads to sync up too.
766 *
767 * Note, however, that this code is also used during remark and in
768 * this case we should not attempt to leave / enter the STS, otherwise
769 * we'll either hit an assert (debug / fastdebug) or deadlock
770 * (product). So we should only leave / enter the STS if we are
771 * operating concurrently.
772 *
773 * Because the thread that does the sync barrier has left the STS, it
774 * is possible to be suspended for a Full GC or an evacuation pause
775 * could occur. This is actually safe, since the entering the sync
776 * barrier is one of the last things do_marking_step() does, and it
777 * doesn't manipulate any data structures afterwards.
778 */
779
780 void G1ConcurrentMark::enter_first_sync_barrier(uint worker_id) {
781 bool barrier_aborted;
782 {
783 SuspendibleThreadSetLeaver sts_leave(concurrent());
784 barrier_aborted = !_first_overflow_barrier_sync.enter();
785 }
786
787 // at this point everyone should have synced up and not be doing any
788 // more work
789
790 if (barrier_aborted) {
791 // If the barrier aborted we ignore the overflow condition and
792 // just abort the whole marking phase as quickly as possible.
793 return;
794 }
795
796 // If we're executing the concurrent phase of marking, reset the marking
797 // state; otherwise the marking state is reset after reference processing,
798 // during the remark pause.
799 // If we reset here as a result of an overflow during the remark we will
800 // see assertion failures from any subsequent set_concurrency_and_phase()
801 // calls.
802 if (concurrent()) {
803 // let the task associated with with worker 0 do this
804 if (worker_id == 0) {
805 // task 0 is responsible for clearing the global data structures
806 // We should be here because of an overflow. During STW we should
807 // not clear the overflow flag since we rely on it being true when
808 // we exit this method to abort the pause and restart concurrent
809 // marking.
810 reset_marking_state();
811
812 log_info(gc, marking)("Concurrent Mark reset for overflow");
813 }
814 }
815
816 // after this, each task should reset its own data structures then
817 // then go into the second barrier
818 }
819
820 void G1ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
821 SuspendibleThreadSetLeaver sts_leave(concurrent());
822 _second_overflow_barrier_sync.enter();
823
824 // at this point everything should be re-initialized and ready to go
825 }
826
827 class G1CMConcurrentMarkingTask: public AbstractGangTask {
828 private:
829 G1ConcurrentMark* _cm;
830 ConcurrentMarkThread* _cmt;
831
832 public:
833 void work(uint worker_id) {
834 assert(Thread::current()->is_ConcurrentGC_thread(), "Not a concurrent GC thread");
835 ResourceMark rm;
836
837 double start_vtime = os::elapsedVTime();
838
839 {
840 SuspendibleThreadSetJoiner sts_join;
841
842 assert(worker_id < _cm->active_tasks(), "invariant");
843
844 G1CMTask* task = _cm->task(worker_id);
845 task->record_start_time();
846 if (!_cm->has_aborted()) {
847 do {
848 task->do_marking_step(G1ConcMarkStepDurationMillis,
849 true /* do_termination */,
850 false /* is_serial*/);
851
852 _cm->do_yield_check();
853 } while (!_cm->has_aborted() && task->has_aborted());
854 }
855 task->record_end_time();
856 guarantee(!task->has_aborted() || _cm->has_aborted(), "invariant");
857 }
858
859 double end_vtime = os::elapsedVTime();
860 _cm->update_accum_task_vtime(worker_id, end_vtime - start_vtime);
861 }
862
863 G1CMConcurrentMarkingTask(G1ConcurrentMark* cm,
864 ConcurrentMarkThread* cmt) :
865 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { }
866
867 ~G1CMConcurrentMarkingTask() { }
868 };
869
870 uint G1ConcurrentMark::calc_active_marking_workers() {
871 uint result = 0;
872 if (!UseDynamicNumberOfGCThreads ||
873 (!FLAG_IS_DEFAULT(ConcGCThreads) &&
874 !ForceDynamicNumberOfGCThreads)) {
875 result = _max_concurrent_workers;
876 } else {
877 result =
878 AdaptiveSizePolicy::calc_default_active_workers(_max_concurrent_workers,
879 1, /* Minimum workers */
880 _num_concurrent_workers,
881 Threads::number_of_non_daemon_threads());
882 // Don't scale the result down by scale_concurrent_workers() because
883 // that scaling has already gone into "_max_concurrent_workers".
884 }
885 assert(result > 0 && result <= _max_concurrent_workers,
886 "Calculated number of marking workers must be larger than zero and at most the maximum %u, but is %u",
887 _max_concurrent_workers, result);
888 return result;
889 }
890
891 void G1ConcurrentMark::scan_root_region(HeapRegion* hr) {
892 // Currently, only survivors can be root regions.
893 assert(hr->next_top_at_mark_start() == hr->bottom(), "invariant");
894 G1RootRegionScanClosure cl(_g1h, this);
895
896 const uintx interval = PrefetchScanIntervalInBytes;
897 HeapWord* curr = hr->bottom();
898 const HeapWord* end = hr->top();
899 while (curr < end) {
900 Prefetch::read(curr, interval);
901 oop obj = oop(curr);
902 int size = obj->oop_iterate_size(&cl);
903 assert(size == obj->size(), "sanity");
904 curr += size;
905 }
906 }
907
908 class G1CMRootRegionScanTask : public AbstractGangTask {
909 private:
910 G1ConcurrentMark* _cm;
911
912 public:
913 G1CMRootRegionScanTask(G1ConcurrentMark* cm) :
914 AbstractGangTask("G1 Root Region Scan"), _cm(cm) { }
915
916 void work(uint worker_id) {
917 assert(Thread::current()->is_ConcurrentGC_thread(),
918 "this should only be done by a conc GC thread");
919
920 G1CMRootRegions* root_regions = _cm->root_regions();
921 HeapRegion* hr = root_regions->claim_next();
922 while (hr != NULL) {
923 _cm->scan_root_region(hr);
924 hr = root_regions->claim_next();
925 }
926 }
927 };
928
929 void G1ConcurrentMark::scan_root_regions() {
930 // scan_in_progress() will have been set to true only if there was
931 // at least one root region to scan. So, if it's false, we
932 // should not attempt to do any further work.
933 if (root_regions()->scan_in_progress()) {
934 assert(!has_aborted(), "Aborting before root region scanning is finished not supported.");
935
936 _num_concurrent_workers = MIN2(calc_active_marking_workers(),
937 // We distribute work on a per-region basis, so starting
938 // more threads than that is useless.
939 root_regions()->num_root_regions());
940 assert(_num_concurrent_workers <= _max_concurrent_workers,
941 "Maximum number of marking threads exceeded");
942
943 G1CMRootRegionScanTask task(this);
944 log_debug(gc, ergo)("Running %s using %u workers for %u work units.",
945 task.name(), _num_concurrent_workers, root_regions()->num_root_regions());
946 _concurrent_workers->run_task(&task, _num_concurrent_workers);
947
948 // It's possible that has_aborted() is true here without actually
949 // aborting the survivor scan earlier. This is OK as it's
950 // mainly used for sanity checking.
951 root_regions()->scan_finished();
952 }
953 }
954
955 void G1ConcurrentMark::concurrent_cycle_start() {
956 _gc_timer_cm->register_gc_start();
957
958 _gc_tracer_cm->report_gc_start(GCCause::_no_gc /* first parameter is not used */, _gc_timer_cm->gc_start());
959
960 _g1h->trace_heap_before_gc(_gc_tracer_cm);
961 }
962
963 void G1ConcurrentMark::concurrent_cycle_end() {
964 _g1h->trace_heap_after_gc(_gc_tracer_cm);
965
966 if (has_aborted()) {
967 _gc_tracer_cm->report_concurrent_mode_failure();
968 }
969
970 _gc_timer_cm->register_gc_end();
971
972 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
973 }
974
975 void G1ConcurrentMark::mark_from_roots() {
976 // we might be tempted to assert that:
977 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
978 // "inconsistent argument?");
979 // However that wouldn't be right, because it's possible that
980 // a safepoint is indeed in progress as a younger generation
981 // stop-the-world GC happens even as we mark in this generation.
982
983 _restart_for_overflow = false;
984
985 _num_concurrent_workers = calc_active_marking_workers();
986
987 uint active_workers = MAX2(1U, _num_concurrent_workers);
988
989 // Setting active workers is not guaranteed since fewer
990 // worker threads may currently exist and more may not be
991 // available.
992 active_workers = _concurrent_workers->update_active_workers(active_workers);
993 log_info(gc, task)("Using %u workers of %u for marking", active_workers, _concurrent_workers->total_workers());
994
995 // Parallel task terminator is set in "set_concurrency_and_phase()"
996 set_concurrency_and_phase(active_workers, true /* concurrent */);
997
998 G1CMConcurrentMarkingTask marking_task(this, cm_thread());
999 _concurrent_workers->run_task(&marking_task);
1000 print_stats();
1001 }
1002
1003 void G1ConcurrentMark::checkpoint_roots_final(bool clear_all_soft_refs) {
1004 // world is stopped at this checkpoint
1005 assert(SafepointSynchronize::is_at_safepoint(),
1006 "world should be stopped");
1007
1008 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1009
1010 // If a full collection has happened, we shouldn't do this.
1011 if (has_aborted()) {
1012 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1013 return;
1014 }
1015
1016 if (VerifyDuringGC) {
1017 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (before)");
1018 }
1019 g1h->verifier()->check_bitmaps("Remark Start");
1020
1021 G1Policy* g1p = g1h->g1_policy();
1022 g1p->record_concurrent_mark_remark_start();
1023
1024 double start = os::elapsedTime();
1025
1026 checkpoint_roots_final_work();
1027
1028 double mark_work_end = os::elapsedTime();
1029
1030 weak_refs_work(clear_all_soft_refs);
1031
1032 if (has_overflown()) {
1033 // We overflowed. Restart concurrent marking.
1034 _restart_for_overflow = true;
1035
1036 // Verify the heap w.r.t. the previous marking bitmap.
1037 if (VerifyDuringGC) {
1038 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (overflow)");
1039 }
1040
1041 // Clear the marking state because we will be restarting
1042 // marking due to overflowing the global mark stack.
1043 reset_marking_state();
1044 } else {
1045 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1046 // We're done with marking.
1047 // This is the end of the marking cycle, we're expected all
1048 // threads to have SATB queues with active set to true.
1049 satb_mq_set.set_active_all_threads(false, /* new active value */
1050 true /* expected_active */);
1051
1052 if (VerifyDuringGC) {
1053 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UseNextMarking, "During GC (after)");
1054 }
1055 g1h->verifier()->check_bitmaps("Remark End");
1056 assert(!restart_for_overflow(), "sanity");
1057 // Completely reset the marking state since marking completed
1058 set_non_marking_state();
1059 }
1060
1061 // Statistics
1062 double now = os::elapsedTime();
1063 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1064 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1065 _remark_times.add((now - start) * 1000.0);
1066
1067 g1p->record_concurrent_mark_remark_end();
1068
1069 G1CMIsAliveClosure is_alive(g1h);
1070 _gc_tracer_cm->report_object_count_after_gc(&is_alive);
1071 }
1072
1073 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1074 G1CollectedHeap* _g1;
1075 size_t _freed_bytes;
1076 FreeRegionList* _local_cleanup_list;
1077 uint _old_regions_removed;
1078 uint _humongous_regions_removed;
1079 HRRSCleanupTask* _hrrs_cleanup_task;
1080
1081 public:
1082 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1083 FreeRegionList* local_cleanup_list,
1084 HRRSCleanupTask* hrrs_cleanup_task) :
1085 _g1(g1),
1086 _freed_bytes(0),
1087 _local_cleanup_list(local_cleanup_list),
1088 _old_regions_removed(0),
1089 _humongous_regions_removed(0),
1090 _hrrs_cleanup_task(hrrs_cleanup_task) { }
1091
1092 size_t freed_bytes() { return _freed_bytes; }
1093 const uint old_regions_removed() { return _old_regions_removed; }
1094 const uint humongous_regions_removed() { return _humongous_regions_removed; }
1095
1096 bool do_heap_region(HeapRegion *hr) {
1097 _g1->reset_gc_time_stamps(hr);
1098 hr->note_end_of_marking();
1099
1100 if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young() && !hr->is_archive()) {
1101 _freed_bytes += hr->used();
1102 hr->set_containing_set(NULL);
1103 if (hr->is_humongous()) {
1104 _humongous_regions_removed++;
1105 _g1->free_humongous_region(hr, _local_cleanup_list, true /* skip_remset */);
1106 } else {
1107 _old_regions_removed++;
1108 _g1->free_region(hr, _local_cleanup_list, true /* skip_remset */);
1109 }
1110 } else {
1111 hr->rem_set()->do_cleanup_work(_hrrs_cleanup_task);
1112 }
1113
1114 return false;
1115 }
1116 };
1117
1118 class G1ParNoteEndTask: public AbstractGangTask {
1119 friend class G1NoteEndOfConcMarkClosure;
1120
1121 protected:
1122 G1CollectedHeap* _g1h;
1123 FreeRegionList* _cleanup_list;
1124 HeapRegionClaimer _hrclaimer;
1125
1126 public:
1127 G1ParNoteEndTask(G1CollectedHeap* g1h, FreeRegionList* cleanup_list, uint n_workers) :
1128 AbstractGangTask("G1 note end"), _g1h(g1h), _cleanup_list(cleanup_list), _hrclaimer(n_workers) {
1129 }
1130
1131 void work(uint worker_id) {
1132 FreeRegionList local_cleanup_list("Local Cleanup List");
1133 HRRSCleanupTask hrrs_cleanup_task;
1134 G1NoteEndOfConcMarkClosure g1_note_end(_g1h, &local_cleanup_list,
1135 &hrrs_cleanup_task);
1136 _g1h->heap_region_par_iterate_from_worker_offset(&g1_note_end, &_hrclaimer, worker_id);
1137 assert(g1_note_end.is_complete(), "Shouldn't have yielded!");
1138
1139 // Now update the lists
1140 _g1h->remove_from_old_sets(g1_note_end.old_regions_removed(), g1_note_end.humongous_regions_removed());
1141 {
1142 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1143 _g1h->decrement_summary_bytes(g1_note_end.freed_bytes());
1144
1145 // If we iterate over the global cleanup list at the end of
1146 // cleanup to do this printing we will not guarantee to only
1147 // generate output for the newly-reclaimed regions (the list
1148 // might not be empty at the beginning of cleanup; we might
1149 // still be working on its previous contents). So we do the
1150 // printing here, before we append the new regions to the global
1151 // cleanup list.
1152
1153 G1HRPrinter* hr_printer = _g1h->hr_printer();
1154 if (hr_printer->is_active()) {
1155 FreeRegionListIterator iter(&local_cleanup_list);
1156 while (iter.more_available()) {
1157 HeapRegion* hr = iter.get_next();
1158 hr_printer->cleanup(hr);
1159 }
1160 }
1161
1162 _cleanup_list->add_ordered(&local_cleanup_list);
1163 assert(local_cleanup_list.is_empty(), "post-condition");
1164
1165 HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task);
1166 }
1167 }
1168 };
1169
1170 void G1ConcurrentMark::cleanup() {
1171 // world is stopped at this checkpoint
1172 assert(SafepointSynchronize::is_at_safepoint(),
1173 "world should be stopped");
1174 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1175
1176 // If a full collection has happened, we shouldn't do this.
1177 if (has_aborted()) {
1178 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1179 return;
1180 }
1181
1182 g1h->verifier()->verify_region_sets_optional();
1183
1184 if (VerifyDuringGC) {
1185 g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (before)");
1186 }
1187 g1h->verifier()->check_bitmaps("Cleanup Start");
1188
1189 G1Policy* g1p = g1h->g1_policy();
1190 g1p->record_concurrent_mark_cleanup_start();
1191
1192 double start = os::elapsedTime();
1193
1194 HeapRegionRemSet::reset_for_cleanup_tasks();
1195
1196 {
1197 GCTraceTime(Debug, gc)("Finalize Live Data");
1198 finalize_live_data();
1199 }
1200
1201 if (VerifyDuringGC) {
1202 GCTraceTime(Debug, gc)("Verify Live Data");
1203 verify_live_data();
1204 }
1205
1206 g1h->collector_state()->set_mark_in_progress(false);
1207
1208 double count_end = os::elapsedTime();
1209 double this_final_counting_time = (count_end - start);
1210 _total_counting_time += this_final_counting_time;
1211
1212 if (log_is_enabled(Trace, gc, liveness)) {
1213 G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1214 _g1h->heap_region_iterate(&cl);
1215 }
1216
1217 // Install newly created mark bitMap as "prev".
1218 swap_mark_bitmaps();
1219
1220 g1h->reset_gc_time_stamp();
1221
1222 uint n_workers = _g1h->workers()->active_workers();
1223
1224 // Note end of marking in all heap regions.
1225 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1226 g1h->workers()->run_task(&g1_par_note_end_task);
1227 g1h->check_gc_time_stamps();
1228
1229 if (!cleanup_list_is_empty()) {
1230 // The cleanup list is not empty, so we'll have to process it
1231 // concurrently. Notify anyone else that might be wanting free
1232 // regions that there will be more free regions coming soon.
1233 g1h->set_free_regions_coming();
1234 }
1235
1236 // call below, since it affects the metric by which we sort the heap
1237 // regions.
1238 if (G1ScrubRemSets) {
1239 double rs_scrub_start = os::elapsedTime();
1240 g1h->scrub_rem_set();
1241 _total_rs_scrub_time += (os::elapsedTime() - rs_scrub_start);
1242 }
1243
1244 // this will also free any regions totally full of garbage objects,
1245 // and sort the regions.
1246 g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1247
1248 // Statistics.
1249 double end = os::elapsedTime();
1250 _cleanup_times.add((end - start) * 1000.0);
1251
1252 // Clean up will have freed any regions completely full of garbage.
1253 // Update the soft reference policy with the new heap occupancy.
1254 Universe::update_heap_info_at_gc();
1255
1256 if (VerifyDuringGC) {
1257 g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (after)");
1258 }
1259
1260 g1h->verifier()->check_bitmaps("Cleanup End");
1261
1262 g1h->verifier()->verify_region_sets_optional();
1263
1264 // We need to make this be a "collection" so any collection pause that
1265 // races with it goes around and waits for completeCleanup to finish.
1266 g1h->increment_total_collections();
1267
1268 // Clean out dead classes and update Metaspace sizes.
1269 if (ClassUnloadingWithConcurrentMark) {
1270 ClassLoaderDataGraph::purge();
1271 }
1272 MetaspaceGC::compute_new_size();
1273
1274 // We reclaimed old regions so we should calculate the sizes to make
1275 // sure we update the old gen/space data.
1276 g1h->g1mm()->update_sizes();
1277 }
1278
1279 void G1ConcurrentMark::complete_cleanup() {
1280 if (has_aborted()) return;
1281
1282 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1283
1284 _cleanup_list.verify_optional();
1285 FreeRegionList tmp_free_list("Tmp Free List");
1286
1287 log_develop_trace(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "
1288 "cleanup list has %u entries",
1289 _cleanup_list.length());
1290
1291 // No one else should be accessing the _cleanup_list at this point,
1292 // so it is not necessary to take any locks
1293 while (!_cleanup_list.is_empty()) {
1294 HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */);
1295 assert(hr != NULL, "Got NULL from a non-empty list");
1296 hr->par_clear();
1297 tmp_free_list.add_ordered(hr);
1298
1299 // Instead of adding one region at a time to the secondary_free_list,
1300 // we accumulate them in the local list and move them a few at a
1301 // time. This also cuts down on the number of notify_all() calls
1302 // we do during this process. We'll also append the local list when
1303 // _cleanup_list is empty (which means we just removed the last
1304 // region from the _cleanup_list).
1305 if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1306 _cleanup_list.is_empty()) {
1307 log_develop_trace(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "
1308 "appending %u entries to the secondary_free_list, "
1309 "cleanup list still has %u entries",
1310 tmp_free_list.length(),
1311 _cleanup_list.length());
1312
1313 {
1314 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1315 g1h->secondary_free_list_add(&tmp_free_list);
1316 SecondaryFreeList_lock->notify_all();
1317 }
1318 #ifndef PRODUCT
1319 if (G1StressConcRegionFreeing) {
1320 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1321 os::sleep(Thread::current(), (jlong) 1, false);
1322 }
1323 }
1324 #endif
1325 }
1326 }
1327 assert(tmp_free_list.is_empty(), "post-condition");
1328 }
1329
1330 // Supporting Object and Oop closures for reference discovery
1331 // and processing in during marking
1332
1333 bool G1CMIsAliveClosure::do_object_b(oop obj) {
1334 HeapWord* addr = (HeapWord*)obj;
1335 return addr != NULL &&
1336 (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj));
1337 }
1338
1339 // 'Keep Alive' oop closure used by both serial parallel reference processing.
1340 // Uses the G1CMTask associated with a worker thread (for serial reference
1341 // processing the G1CMTask for worker 0 is used) to preserve (mark) and
1342 // trace referent objects.
1343 //
1344 // Using the G1CMTask and embedded local queues avoids having the worker
1345 // threads operating on the global mark stack. This reduces the risk
1346 // of overflowing the stack - which we would rather avoid at this late
1347 // state. Also using the tasks' local queues removes the potential
1348 // of the workers interfering with each other that could occur if
1349 // operating on the global stack.
1350
1351 class G1CMKeepAliveAndDrainClosure: public OopClosure {
1352 G1ConcurrentMark* _cm;
1353 G1CMTask* _task;
1354 int _ref_counter_limit;
1355 int _ref_counter;
1356 bool _is_serial;
1357 public:
1358 G1CMKeepAliveAndDrainClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) :
1359 _cm(cm), _task(task), _is_serial(is_serial),
1360 _ref_counter_limit(G1RefProcDrainInterval) {
1361 assert(_ref_counter_limit > 0, "sanity");
1362 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code");
1363 _ref_counter = _ref_counter_limit;
1364 }
1365
1366 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
1367 virtual void do_oop( oop* p) { do_oop_work(p); }
1368
1369 template <class T> void do_oop_work(T* p) {
1370 if (!_cm->has_overflown()) {
1371 oop obj = oopDesc::load_decode_heap_oop(p);
1372 _task->deal_with_reference(obj);
1373 _ref_counter--;
1374
1375 if (_ref_counter == 0) {
1376 // We have dealt with _ref_counter_limit references, pushing them
1377 // and objects reachable from them on to the local stack (and
1378 // possibly the global stack). Call G1CMTask::do_marking_step() to
1379 // process these entries.
1380 //
1381 // We call G1CMTask::do_marking_step() in a loop, which we'll exit if
1382 // there's nothing more to do (i.e. we're done with the entries that
1383 // were pushed as a result of the G1CMTask::deal_with_reference() calls
1384 // above) or we overflow.
1385 //
1386 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted()
1387 // flag while there may still be some work to do. (See the comment at
1388 // the beginning of G1CMTask::do_marking_step() for those conditions -
1389 // one of which is reaching the specified time target.) It is only
1390 // when G1CMTask::do_marking_step() returns without setting the
1391 // has_aborted() flag that the marking step has completed.
1392 do {
1393 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
1394 _task->do_marking_step(mark_step_duration_ms,
1395 false /* do_termination */,
1396 _is_serial);
1397 } while (_task->has_aborted() && !_cm->has_overflown());
1398 _ref_counter = _ref_counter_limit;
1399 }
1400 }
1401 }
1402 };
1403
1404 // 'Drain' oop closure used by both serial and parallel reference processing.
1405 // Uses the G1CMTask associated with a given worker thread (for serial
1406 // reference processing the G1CMtask for worker 0 is used). Calls the
1407 // do_marking_step routine, with an unbelievably large timeout value,
1408 // to drain the marking data structures of the remaining entries
1409 // added by the 'keep alive' oop closure above.
1410
1411 class G1CMDrainMarkingStackClosure: public VoidClosure {
1412 G1ConcurrentMark* _cm;
1413 G1CMTask* _task;
1414 bool _is_serial;
1415 public:
1416 G1CMDrainMarkingStackClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) :
1417 _cm(cm), _task(task), _is_serial(is_serial) {
1418 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code");
1419 }
1420
1421 void do_void() {
1422 do {
1423 // We call G1CMTask::do_marking_step() to completely drain the local
1424 // and global marking stacks of entries pushed by the 'keep alive'
1425 // oop closure (an instance of G1CMKeepAliveAndDrainClosure above).
1426 //
1427 // G1CMTask::do_marking_step() is called in a loop, which we'll exit
1428 // if there's nothing more to do (i.e. we've completely drained the
1429 // entries that were pushed as a a result of applying the 'keep alive'
1430 // closure to the entries on the discovered ref lists) or we overflow
1431 // the global marking stack.
1432 //
1433 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted()
1434 // flag while there may still be some work to do. (See the comment at
1435 // the beginning of G1CMTask::do_marking_step() for those conditions -
1436 // one of which is reaching the specified time target.) It is only
1437 // when G1CMTask::do_marking_step() returns without setting the
1438 // has_aborted() flag that the marking step has completed.
1439
1440 _task->do_marking_step(1000000000.0 /* something very large */,
1441 true /* do_termination */,
1442 _is_serial);
1443 } while (_task->has_aborted() && !_cm->has_overflown());
1444 }
1445 };
1446
1447 // Implementation of AbstractRefProcTaskExecutor for parallel
1448 // reference processing at the end of G1 concurrent marking
1449
1450 class G1CMRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
1451 private:
1452 G1CollectedHeap* _g1h;
1453 G1ConcurrentMark* _cm;
1454 WorkGang* _workers;
1455 uint _active_workers;
1456
1457 public:
1458 G1CMRefProcTaskExecutor(G1CollectedHeap* g1h,
1459 G1ConcurrentMark* cm,
1460 WorkGang* workers,
1461 uint n_workers) :
1462 _g1h(g1h), _cm(cm),
1463 _workers(workers), _active_workers(n_workers) { }
1464
1465 // Executes the given task using concurrent marking worker threads.
1466 virtual void execute(ProcessTask& task);
1467 virtual void execute(EnqueueTask& task);
1468 };
1469
1470 class G1CMRefProcTaskProxy: public AbstractGangTask {
1471 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
1472 ProcessTask& _proc_task;
1473 G1CollectedHeap* _g1h;
1474 G1ConcurrentMark* _cm;
1475
1476 public:
1477 G1CMRefProcTaskProxy(ProcessTask& proc_task,
1478 G1CollectedHeap* g1h,
1479 G1ConcurrentMark* cm) :
1480 AbstractGangTask("Process reference objects in parallel"),
1481 _proc_task(proc_task), _g1h(g1h), _cm(cm) {
1482 ReferenceProcessor* rp = _g1h->ref_processor_cm();
1483 assert(rp->processing_is_mt(), "shouldn't be here otherwise");
1484 }
1485
1486 virtual void work(uint worker_id) {
1487 ResourceMark rm;
1488 HandleMark hm;
1489 G1CMTask* task = _cm->task(worker_id);
1490 G1CMIsAliveClosure g1_is_alive(_g1h);
1491 G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */);
1492 G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */);
1493
1494 _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain);
1495 }
1496 };
1497
1498 void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) {
1499 assert(_workers != NULL, "Need parallel worker threads.");
1500 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
1501
1502 G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm);
1503
1504 // We need to reset the concurrency level before each
1505 // proxy task execution, so that the termination protocol
1506 // and overflow handling in G1CMTask::do_marking_step() knows
1507 // how many workers to wait for.
1508 _cm->set_concurrency(_active_workers);
1509 _workers->run_task(&proc_task_proxy);
1510 }
1511
1512 class G1CMRefEnqueueTaskProxy: public AbstractGangTask {
1513 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
1514 EnqueueTask& _enq_task;
1515
1516 public:
1517 G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) :
1518 AbstractGangTask("Enqueue reference objects in parallel"),
1519 _enq_task(enq_task) { }
1520
1521 virtual void work(uint worker_id) {
1522 _enq_task.work(worker_id);
1523 }
1524 };
1525
1526 void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
1527 assert(_workers != NULL, "Need parallel worker threads.");
1528 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
1529
1530 G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task);
1531
1532 // Not strictly necessary but...
1533 //
1534 // We need to reset the concurrency level before each
1535 // proxy task execution, so that the termination protocol
1536 // and overflow handling in G1CMTask::do_marking_step() knows
1537 // how many workers to wait for.
1538 _cm->set_concurrency(_active_workers);
1539 _workers->run_task(&enq_task_proxy);
1540 }
1541
1542 void G1ConcurrentMark::weak_refs_work(bool clear_all_soft_refs) {
1543 if (has_overflown()) {
1544 // Skip processing the discovered references if we have
1545 // overflown the global marking stack. Reference objects
1546 // only get discovered once so it is OK to not
1547 // de-populate the discovered reference lists. We could have,
1548 // but the only benefit would be that, when marking restarts,
1549 // less reference objects are discovered.
1550 return;
1551 }
1552
1553 ResourceMark rm;
1554 HandleMark hm;
1555
1556 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1557
1558 // Is alive closure.
1559 G1CMIsAliveClosure g1_is_alive(g1h);
1560
1561 // Inner scope to exclude the cleaning of the string and symbol
1562 // tables from the displayed time.
1563 {
1564 GCTraceTime(Debug, gc, phases) trace("Reference Processing", _gc_timer_cm);
1565
1566 ReferenceProcessor* rp = g1h->ref_processor_cm();
1567
1568 // See the comment in G1CollectedHeap::ref_processing_init()
1569 // about how reference processing currently works in G1.
1570
1571 // Set the soft reference policy
1572 rp->setup_policy(clear_all_soft_refs);
1573 assert(_global_mark_stack.is_empty(), "mark stack should be empty");
1574
1575 // Instances of the 'Keep Alive' and 'Complete GC' closures used
1576 // in serial reference processing. Note these closures are also
1577 // used for serially processing (by the the current thread) the
1578 // JNI references during parallel reference processing.
1579 //
1580 // These closures do not need to synchronize with the worker
1581 // threads involved in parallel reference processing as these
1582 // instances are executed serially by the current thread (e.g.
1583 // reference processing is not multi-threaded and is thus
1584 // performed by the current thread instead of a gang worker).
1585 //
1586 // The gang tasks involved in parallel reference processing create
1587 // their own instances of these closures, which do their own
1588 // synchronization among themselves.
1589 G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */);
1590 G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */);
1591
1592 // We need at least one active thread. If reference processing
1593 // is not multi-threaded we use the current (VMThread) thread,
1594 // otherwise we use the work gang from the G1CollectedHeap and
1595 // we utilize all the worker threads we can.
1596 bool processing_is_mt = rp->processing_is_mt();
1597 uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U);
1598 active_workers = MAX2(MIN2(active_workers, _max_num_tasks), 1U);
1599
1600 // Parallel processing task executor.
1601 G1CMRefProcTaskExecutor par_task_executor(g1h, this,
1602 g1h->workers(), active_workers);
1603 AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL);
1604
1605 // Set the concurrency level. The phase was already set prior to
1606 // executing the remark task.
1607 set_concurrency(active_workers);
1608
1609 // Set the degree of MT processing here. If the discovery was done MT,
1610 // the number of threads involved during discovery could differ from
1611 // the number of active workers. This is OK as long as the discovered
1612 // Reference lists are balanced (see balance_all_queues() and balance_queues()).
1613 rp->set_active_mt_degree(active_workers);
1614
1615 ReferenceProcessorPhaseTimes pt(_gc_timer_cm, rp->num_q());
1616
1617 // Process the weak references.
1618 const ReferenceProcessorStats& stats =
1619 rp->process_discovered_references(&g1_is_alive,
1620 &g1_keep_alive,
1621 &g1_drain_mark_stack,
1622 executor,
1623 &pt);
1624 _gc_tracer_cm->report_gc_reference_stats(stats);
1625 pt.print_all_references();
1626
1627 // The do_oop work routines of the keep_alive and drain_marking_stack
1628 // oop closures will set the has_overflown flag if we overflow the
1629 // global marking stack.
1630
1631 assert(has_overflown() || _global_mark_stack.is_empty(),
1632 "Mark stack should be empty (unless it has overflown)");
1633
1634 assert(rp->num_q() == active_workers, "why not");
1635
1636 rp->enqueue_discovered_references(executor, &pt);
1637
1638 rp->verify_no_references_recorded();
1639
1640 pt.print_enqueue_phase();
1641
1642 assert(!rp->discovery_enabled(), "Post condition");
1643 }
1644
1645 assert(has_overflown() || _global_mark_stack.is_empty(),
1646 "Mark stack should be empty (unless it has overflown)");
1647
1648 {
1649 GCTraceTime(Debug, gc, phases) debug("Weak Processing", _gc_timer_cm);
1650 WeakProcessor::weak_oops_do(&g1_is_alive, &do_nothing_cl);
1651 }
1652
1653 if (has_overflown()) {
1654 // We can not trust g1_is_alive if the marking stack overflowed
1655 return;
1656 }
1657
1658 assert(_global_mark_stack.is_empty(), "Marking should have completed");
1659
1660 // Unload Klasses, String, Symbols, Code Cache, etc.
1661 if (ClassUnloadingWithConcurrentMark) {
1662 GCTraceTime(Debug, gc, phases) debug("Class Unloading", _gc_timer_cm);
1663 bool purged_classes = SystemDictionary::do_unloading(&g1_is_alive, _gc_timer_cm, false /* Defer cleaning */);
1664 g1h->complete_cleaning(&g1_is_alive, purged_classes);
1665 } else {
1666 GCTraceTime(Debug, gc, phases) debug("Cleanup", _gc_timer_cm);
1667 // No need to clean string table and symbol table as they are treated as strong roots when
1668 // class unloading is disabled.
1669 g1h->partial_cleaning(&g1_is_alive, false, false, G1StringDedup::is_enabled());
1670
1671 }
1672 }
1673
1674 void G1ConcurrentMark::swap_mark_bitmaps() {
1675 G1CMBitMap* temp = _prev_mark_bitmap;
1676 _prev_mark_bitmap = _next_mark_bitmap;
1677 _next_mark_bitmap = temp;
1678 }
1679
1680 // Closure for marking entries in SATB buffers.
1681 class G1CMSATBBufferClosure : public SATBBufferClosure {
1682 private:
1683 G1CMTask* _task;
1684 G1CollectedHeap* _g1h;
1685
1686 // This is very similar to G1CMTask::deal_with_reference, but with
1687 // more relaxed requirements for the argument, so this must be more
1688 // circumspect about treating the argument as an object.
1689 void do_entry(void* entry) const {
1690 _task->increment_refs_reached();
1691 oop const obj = static_cast<oop>(entry);
1692 _task->make_reference_grey(obj);
1693 }
1694
1695 public:
1696 G1CMSATBBufferClosure(G1CMTask* task, G1CollectedHeap* g1h)
1697 : _task(task), _g1h(g1h) { }
1698
1699 virtual void do_buffer(void** buffer, size_t size) {
1700 for (size_t i = 0; i < size; ++i) {
1701 do_entry(buffer[i]);
1702 }
1703 }
1704 };
1705
1706 class G1RemarkThreadsClosure : public ThreadClosure {
1707 G1CMSATBBufferClosure _cm_satb_cl;
1708 G1CMOopClosure _cm_cl;
1709 MarkingCodeBlobClosure _code_cl;
1710 int _thread_parity;
1711
1712 public:
1713 G1RemarkThreadsClosure(G1CollectedHeap* g1h, G1CMTask* task) :
1714 _cm_satb_cl(task, g1h),
1715 _cm_cl(g1h, g1h->concurrent_mark(), task),
1716 _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations),
1717 _thread_parity(Threads::thread_claim_parity()) {}
1718
1719 void do_thread(Thread* thread) {
1720 if (thread->is_Java_thread()) {
1721 if (thread->claim_oops_do(true, _thread_parity)) {
1722 JavaThread* jt = (JavaThread*)thread;
1723
1724 // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
1725 // however the liveness of oops reachable from nmethods have very complex lifecycles:
1726 // * Alive if on the stack of an executing method
1727 // * Weakly reachable otherwise
1728 // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
1729 // live by the SATB invariant but other oops recorded in nmethods may behave differently.
1730 jt->nmethods_do(&_code_cl);
1731
1732 jt->satb_mark_queue().apply_closure_and_empty(&_cm_satb_cl);
1733 }
1734 } else if (thread->is_VM_thread()) {
1735 if (thread->claim_oops_do(true, _thread_parity)) {
1736 JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(&_cm_satb_cl);
1737 }
1738 }
1739 }
1740 };
1741
1742 class G1CMRemarkTask: public AbstractGangTask {
1743 private:
1744 G1ConcurrentMark* _cm;
1745 public:
1746 void work(uint worker_id) {
1747 G1CMTask* task = _cm->task(worker_id);
1748 task->record_start_time();
1749 {
1750 ResourceMark rm;
1751 HandleMark hm;
1752
1753 G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task);
1754 Threads::threads_do(&threads_f);
1755 }
1756
1757 do {
1758 task->do_marking_step(1000000000.0 /* something very large */,
1759 true /* do_termination */,
1760 false /* is_serial */);
1761 } while (task->has_aborted() && !_cm->has_overflown());
1762 // If we overflow, then we do not want to restart. We instead
1763 // want to abort remark and do concurrent marking again.
1764 task->record_end_time();
1765 }
1766
1767 G1CMRemarkTask(G1ConcurrentMark* cm, uint active_workers) :
1768 AbstractGangTask("Par Remark"), _cm(cm) {
1769 _cm->terminator()->reset_for_reuse(active_workers);
1770 }
1771 };
1772
1773 void G1ConcurrentMark::checkpoint_roots_final_work() {
1774 ResourceMark rm;
1775 HandleMark hm;
1776 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1777
1778 GCTraceTime(Debug, gc, phases) trace("Finalize Marking", _gc_timer_cm);
1779
1780 g1h->ensure_parsability(false);
1781
1782 // this is remark, so we'll use up all active threads
1783 uint active_workers = g1h->workers()->active_workers();
1784 set_concurrency_and_phase(active_workers, false /* concurrent */);
1785 // Leave _parallel_marking_threads at it's
1786 // value originally calculated in the G1ConcurrentMark
1787 // constructor and pass values of the active workers
1788 // through the gang in the task.
1789
1790 {
1791 StrongRootsScope srs(active_workers);
1792
1793 G1CMRemarkTask remarkTask(this, active_workers);
1794 // We will start all available threads, even if we decide that the
1795 // active_workers will be fewer. The extra ones will just bail out
1796 // immediately.
1797 g1h->workers()->run_task(&remarkTask);
1798 }
1799
1800 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1801 guarantee(has_overflown() ||
1802 satb_mq_set.completed_buffers_num() == 0,
1803 "Invariant: has_overflown = %s, num buffers = " SIZE_FORMAT,
1804 BOOL_TO_STR(has_overflown()),
1805 satb_mq_set.completed_buffers_num());
1806
1807 print_stats();
1808 }
1809
1810 void G1ConcurrentMark::clear_range_in_prev_bitmap(MemRegion mr) {
1811 _prev_mark_bitmap->clear_range(mr);
1812 }
1813
1814 HeapRegion*
1815 G1ConcurrentMark::claim_region(uint worker_id) {
1816 // "checkpoint" the finger
1817 HeapWord* finger = _finger;
1818
1819 // _heap_end will not change underneath our feet; it only changes at
1820 // yield points.
1821 while (finger < _heap_end) {
1822 assert(_g1h->is_in_g1_reserved(finger), "invariant");
1823
1824 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
1825 // Make sure that the reads below do not float before loading curr_region.
1826 OrderAccess::loadload();
1827 // Above heap_region_containing may return NULL as we always scan claim
1828 // until the end of the heap. In this case, just jump to the next region.
1829 HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords;
1830
1831 // Is the gap between reading the finger and doing the CAS too long?
1832 HeapWord* res = Atomic::cmpxchg(end, &_finger, finger);
1833 if (res == finger && curr_region != NULL) {
1834 // we succeeded
1835 HeapWord* bottom = curr_region->bottom();
1836 HeapWord* limit = curr_region->next_top_at_mark_start();
1837
1838 // notice that _finger == end cannot be guaranteed here since,
1839 // someone else might have moved the finger even further
1840 assert(_finger >= end, "the finger should have moved forward");
1841
1842 if (limit > bottom) {
1843 return curr_region;
1844 } else {
1845 assert(limit == bottom,
1846 "the region limit should be at bottom");
1847 // we return NULL and the caller should try calling
1848 // claim_region() again.
1849 return NULL;
1850 }
1851 } else {
1852 assert(_finger > finger, "the finger should have moved forward");
1853 // read it again
1854 finger = _finger;
1855 }
1856 }
1857
1858 return NULL;
1859 }
1860
1861 #ifndef PRODUCT
1862 class VerifyNoCSetOops {
1863 private:
1864 G1CollectedHeap* _g1h;
1865 const char* _phase;
1866 int _info;
1867
1868 public:
1869 VerifyNoCSetOops(const char* phase, int info = -1) :
1870 _g1h(G1CollectedHeap::heap()),
1871 _phase(phase),
1872 _info(info)
1873 { }
1874
1875 void operator()(G1TaskQueueEntry task_entry) const {
1876 if (task_entry.is_array_slice()) {
1877 guarantee(_g1h->is_in_reserved(task_entry.slice()), "Slice " PTR_FORMAT " must be in heap.", p2i(task_entry.slice()));
1878 return;
1879 }
1880 guarantee(oopDesc::is_oop(task_entry.obj()),
1881 "Non-oop " PTR_FORMAT ", phase: %s, info: %d",
1882 p2i(task_entry.obj()), _phase, _info);
1883 guarantee(!_g1h->is_in_cset(task_entry.obj()),
1884 "obj: " PTR_FORMAT " in CSet, phase: %s, info: %d",
1885 p2i(task_entry.obj()), _phase, _info);
1886 }
1887 };
1888
1889 void G1ConcurrentMark::verify_no_cset_oops() {
1890 assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint");
1891 if (!G1CollectedHeap::heap()->collector_state()->mark_in_progress()) {
1892 return;
1893 }
1894
1895 // Verify entries on the global mark stack
1896 _global_mark_stack.iterate(VerifyNoCSetOops("Stack"));
1897
1898 // Verify entries on the task queues
1899 for (uint i = 0; i < _max_num_tasks; ++i) {
1900 G1CMTaskQueue* queue = _task_queues->queue(i);
1901 queue->iterate(VerifyNoCSetOops("Queue", i));
1902 }
1903
1904 // Verify the global finger
1905 HeapWord* global_finger = finger();
1906 if (global_finger != NULL && global_finger < _heap_end) {
1907 // Since we always iterate over all regions, we might get a NULL HeapRegion
1908 // here.
1909 HeapRegion* global_hr = _g1h->heap_region_containing(global_finger);
1910 guarantee(global_hr == NULL || global_finger == global_hr->bottom(),
1911 "global finger: " PTR_FORMAT " region: " HR_FORMAT,
1912 p2i(global_finger), HR_FORMAT_PARAMS(global_hr));
1913 }
1914
1915 // Verify the task fingers
1916 assert(_num_concurrent_workers <= _max_num_tasks, "sanity");
1917 for (uint i = 0; i < _num_concurrent_workers; ++i) {
1918 G1CMTask* task = _tasks[i];
1919 HeapWord* task_finger = task->finger();
1920 if (task_finger != NULL && task_finger < _heap_end) {
1921 // See above note on the global finger verification.
1922 HeapRegion* task_hr = _g1h->heap_region_containing(task_finger);
1923 guarantee(task_hr == NULL || task_finger == task_hr->bottom() ||
1924 !task_hr->in_collection_set(),
1925 "task finger: " PTR_FORMAT " region: " HR_FORMAT,
1926 p2i(task_finger), HR_FORMAT_PARAMS(task_hr));
1927 }
1928 }
1929 }
1930 #endif // PRODUCT
1931 void G1ConcurrentMark::create_live_data() {
1932 _g1h->g1_rem_set()->create_card_live_data(_concurrent_workers, _next_mark_bitmap);
1933 }
1934
1935 void G1ConcurrentMark::finalize_live_data() {
1936 _g1h->g1_rem_set()->finalize_card_live_data(_g1h->workers(), _next_mark_bitmap);
1937 }
1938
1939 void G1ConcurrentMark::verify_live_data() {
1940 _g1h->g1_rem_set()->verify_card_live_data(_g1h->workers(), _next_mark_bitmap);
1941 }
1942
1943 void G1ConcurrentMark::clear_live_data(WorkGang* workers) {
1944 _g1h->g1_rem_set()->clear_card_live_data(workers);
1945 }
1946
1947 #ifdef ASSERT
1948 void G1ConcurrentMark::verify_live_data_clear() {
1949 _g1h->g1_rem_set()->verify_card_live_data_is_clear();
1950 }
1951 #endif
1952
1953 void G1ConcurrentMark::print_stats() {
1954 if (!log_is_enabled(Debug, gc, stats)) {
1955 return;
1956 }
1957 log_debug(gc, stats)("---------------------------------------------------------------------");
1958 for (size_t i = 0; i < _num_active_tasks; ++i) {
1959 _tasks[i]->print_stats();
1960 log_debug(gc, stats)("---------------------------------------------------------------------");
1961 }
1962 }
1963
1964 void G1ConcurrentMark::abort() {
1965 if (!cm_thread()->during_cycle() || _has_aborted) {
1966 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
1967 return;
1968 }
1969
1970 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
1971 // concurrent bitmap clearing.
1972 {
1973 GCTraceTime(Debug, gc)("Clear Next Bitmap");
1974 clear_bitmap(_next_mark_bitmap, _g1h->workers(), false);
1975 }
1976 // Note we cannot clear the previous marking bitmap here
1977 // since VerifyDuringGC verifies the objects marked during
1978 // a full GC against the previous bitmap.
1979
1980 {
1981 GCTraceTime(Debug, gc)("Clear Live Data");
1982 clear_live_data(_g1h->workers());
1983 }
1984 DEBUG_ONLY({
1985 GCTraceTime(Debug, gc)("Verify Live Data Clear");
1986 verify_live_data_clear();
1987 })
1988 // Empty mark stack
1989 reset_marking_state();
1990 for (uint i = 0; i < _max_num_tasks; ++i) {
1991 _tasks[i]->clear_region_fields();
1992 }
1993 _first_overflow_barrier_sync.abort();
1994 _second_overflow_barrier_sync.abort();
1995 _has_aborted = true;
1996
1997 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1998 satb_mq_set.abandon_partial_marking();
1999 // This can be called either during or outside marking, we'll read
2000 // the expected_active value from the SATB queue set.
2001 satb_mq_set.set_active_all_threads(
2002 false, /* new active value */
2003 satb_mq_set.is_active() /* expected_active */);
2004 }
2005
2006 static void print_ms_time_info(const char* prefix, const char* name,
2007 NumberSeq& ns) {
2008 log_trace(gc, marking)("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2009 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2010 if (ns.num() > 0) {
2011 log_trace(gc, marking)("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2012 prefix, ns.sd(), ns.maximum());
2013 }
2014 }
2015
2016 void G1ConcurrentMark::print_summary_info() {
2017 Log(gc, marking) log;
2018 if (!log.is_trace()) {
2019 return;
2020 }
2021
2022 log.trace(" Concurrent marking:");
2023 print_ms_time_info(" ", "init marks", _init_times);
2024 print_ms_time_info(" ", "remarks", _remark_times);
2025 {
2026 print_ms_time_info(" ", "final marks", _remark_mark_times);
2027 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2028
2029 }
2030 print_ms_time_info(" ", "cleanups", _cleanup_times);
2031 log.trace(" Finalize live data total time = %8.2f s (avg = %8.2f ms).",
2032 _total_counting_time, (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2033 if (G1ScrubRemSets) {
2034 log.trace(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
2035 _total_rs_scrub_time, (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2036 }
2037 log.trace(" Total stop_world time = %8.2f s.",
2038 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2039 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2040 cm_thread()->vtime_accum(), cm_thread()->vtime_mark_accum());
2041 }
2042
2043 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2044 _concurrent_workers->print_worker_threads_on(st);
2045 }
2046
2047 void G1ConcurrentMark::threads_do(ThreadClosure* tc) const {
2048 _concurrent_workers->threads_do(tc);
2049 }
2050
2051 void G1ConcurrentMark::print_on_error(outputStream* st) const {
2052 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2053 p2i(_prev_mark_bitmap), p2i(_next_mark_bitmap));
2054 _prev_mark_bitmap->print_on_error(st, " Prev Bits: ");
2055 _next_mark_bitmap->print_on_error(st, " Next Bits: ");
2056 }
2057
2058 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) {
2059 ReferenceProcessor* result = g1h->ref_processor_cm();
2060 assert(result != NULL, "CM reference processor should not be NULL");
2061 return result;
2062 }
2063
2064 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
2065 G1ConcurrentMark* cm,
2066 G1CMTask* task)
2067 : MetadataAwareOopClosure(get_cm_oop_closure_ref_processor(g1h)),
2068 _g1h(g1h), _cm(cm), _task(task)
2069 { }
2070
2071 void G1CMTask::setup_for_region(HeapRegion* hr) {
2072 assert(hr != NULL,
2073 "claim_region() should have filtered out NULL regions");
2074 _curr_region = hr;
2075 _finger = hr->bottom();
2076 update_region_limit();
2077 }
2078
2079 void G1CMTask::update_region_limit() {
2080 HeapRegion* hr = _curr_region;
2081 HeapWord* bottom = hr->bottom();
2082 HeapWord* limit = hr->next_top_at_mark_start();
2083
2084 if (limit == bottom) {
2085 // The region was collected underneath our feet.
2086 // We set the finger to bottom to ensure that the bitmap
2087 // iteration that will follow this will not do anything.
2088 // (this is not a condition that holds when we set the region up,
2089 // as the region is not supposed to be empty in the first place)
2090 _finger = bottom;
2091 } else if (limit >= _region_limit) {
2092 assert(limit >= _finger, "peace of mind");
2093 } else {
2094 assert(limit < _region_limit, "only way to get here");
2095 // This can happen under some pretty unusual circumstances. An
2096 // evacuation pause empties the region underneath our feet (NTAMS
2097 // at bottom). We then do some allocation in the region (NTAMS
2098 // stays at bottom), followed by the region being used as a GC
2099 // alloc region (NTAMS will move to top() and the objects
2100 // originally below it will be grayed). All objects now marked in
2101 // the region are explicitly grayed, if below the global finger,
2102 // and we do not need in fact to scan anything else. So, we simply
2103 // set _finger to be limit to ensure that the bitmap iteration
2104 // doesn't do anything.
2105 _finger = limit;
2106 }
2107
2108 _region_limit = limit;
2109 }
2110
2111 void G1CMTask::giveup_current_region() {
2112 assert(_curr_region != NULL, "invariant");
2113 clear_region_fields();
2114 }
2115
2116 void G1CMTask::clear_region_fields() {
2117 // Values for these three fields that indicate that we're not
2118 // holding on to a region.
2119 _curr_region = NULL;
2120 _finger = NULL;
2121 _region_limit = NULL;
2122 }
2123
2124 void G1CMTask::set_cm_oop_closure(G1CMOopClosure* cm_oop_closure) {
2125 if (cm_oop_closure == NULL) {
2126 assert(_cm_oop_closure != NULL, "invariant");
2127 } else {
2128 assert(_cm_oop_closure == NULL, "invariant");
2129 }
2130 _cm_oop_closure = cm_oop_closure;
2131 }
2132
2133 void G1CMTask::reset(G1CMBitMap* next_mark_bitmap) {
2134 guarantee(next_mark_bitmap != NULL, "invariant");
2135 _next_mark_bitmap = next_mark_bitmap;
2136 clear_region_fields();
2137
2138 _calls = 0;
2139 _elapsed_time_ms = 0.0;
2140 _termination_time_ms = 0.0;
2141 _termination_start_time_ms = 0.0;
2142 }
2143
2144 bool G1CMTask::should_exit_termination() {
2145 regular_clock_call();
2146 // This is called when we are in the termination protocol. We should
2147 // quit if, for some reason, this task wants to abort or the global
2148 // stack is not empty (this means that we can get work from it).
2149 return !_cm->mark_stack_empty() || has_aborted();
2150 }
2151
2152 void G1CMTask::reached_limit() {
2153 assert(_words_scanned >= _words_scanned_limit ||
2154 _refs_reached >= _refs_reached_limit ,
2155 "shouldn't have been called otherwise");
2156 regular_clock_call();
2157 }
2158
2159 void G1CMTask::regular_clock_call() {
2160 if (has_aborted()) return;
2161
2162 // First, we need to recalculate the words scanned and refs reached
2163 // limits for the next clock call.
2164 recalculate_limits();
2165
2166 // During the regular clock call we do the following
2167
2168 // (1) If an overflow has been flagged, then we abort.
2169 if (_cm->has_overflown()) {
2170 set_has_aborted();
2171 return;
2172 }
2173
2174 // If we are not concurrent (i.e. we're doing remark) we don't need
2175 // to check anything else. The other steps are only needed during
2176 // the concurrent marking phase.
2177 if (!_concurrent) {
2178 return;
2179 }
2180
2181 // (2) If marking has been aborted for Full GC, then we also abort.
2182 if (_cm->has_aborted()) {
2183 set_has_aborted();
2184 return;
2185 }
2186
2187 double curr_time_ms = os::elapsedVTime() * 1000.0;
2188
2189 // (4) We check whether we should yield. If we have to, then we abort.
2190 if (SuspendibleThreadSet::should_yield()) {
2191 // We should yield. To do this we abort the task. The caller is
2192 // responsible for yielding.
2193 set_has_aborted();
2194 return;
2195 }
2196
2197 // (5) We check whether we've reached our time quota. If we have,
2198 // then we abort.
2199 double elapsed_time_ms = curr_time_ms - _start_time_ms;
2200 if (elapsed_time_ms > _time_target_ms) {
2201 set_has_aborted();
2202 _has_timed_out = true;
2203 return;
2204 }
2205
2206 // (6) Finally, we check whether there are enough completed STAB
2207 // buffers available for processing. If there are, we abort.
2208 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2209 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) {
2210 // we do need to process SATB buffers, we'll abort and restart
2211 // the marking task to do so
2212 set_has_aborted();
2213 return;
2214 }
2215 }
2216
2217 void G1CMTask::recalculate_limits() {
2218 _real_words_scanned_limit = _words_scanned + words_scanned_period;
2219 _words_scanned_limit = _real_words_scanned_limit;
2220
2221 _real_refs_reached_limit = _refs_reached + refs_reached_period;
2222 _refs_reached_limit = _real_refs_reached_limit;
2223 }
2224
2225 void G1CMTask::decrease_limits() {
2226 // This is called when we believe that we're going to do an infrequent
2227 // operation which will increase the per byte scanned cost (i.e. move
2228 // entries to/from the global stack). It basically tries to decrease the
2229 // scanning limit so that the clock is called earlier.
2230
2231 _words_scanned_limit = _real_words_scanned_limit - 3 * words_scanned_period / 4;
2232 _refs_reached_limit = _real_refs_reached_limit - 3 * refs_reached_period / 4;
2233 }
2234
2235 void G1CMTask::move_entries_to_global_stack() {
2236 // Local array where we'll store the entries that will be popped
2237 // from the local queue.
2238 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk];
2239
2240 size_t n = 0;
2241 G1TaskQueueEntry task_entry;
2242 while (n < G1CMMarkStack::EntriesPerChunk && _task_queue->pop_local(task_entry)) {
2243 buffer[n] = task_entry;
2244 ++n;
2245 }
2246 if (n < G1CMMarkStack::EntriesPerChunk) {
2247 buffer[n] = G1TaskQueueEntry();
2248 }
2249
2250 if (n > 0) {
2251 if (!_cm->mark_stack_push(buffer)) {
2252 set_has_aborted();
2253 }
2254 }
2255
2256 // This operation was quite expensive, so decrease the limits.
2257 decrease_limits();
2258 }
2259
2260 bool G1CMTask::get_entries_from_global_stack() {
2261 // Local array where we'll store the entries that will be popped
2262 // from the global stack.
2263 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk];
2264
2265 if (!_cm->mark_stack_pop(buffer)) {
2266 return false;
2267 }
2268
2269 // We did actually pop at least one entry.
2270 for (size_t i = 0; i < G1CMMarkStack::EntriesPerChunk; ++i) {
2271 G1TaskQueueEntry task_entry = buffer[i];
2272 if (task_entry.is_null()) {
2273 break;
2274 }
2275 assert(task_entry.is_array_slice() || oopDesc::is_oop(task_entry.obj()), "Element " PTR_FORMAT " must be an array slice or oop", p2i(task_entry.obj()));
2276 bool success = _task_queue->push(task_entry);
2277 // We only call this when the local queue is empty or under a
2278 // given target limit. So, we do not expect this push to fail.
2279 assert(success, "invariant");
2280 }
2281
2282 // This operation was quite expensive, so decrease the limits
2283 decrease_limits();
2284 return true;
2285 }
2286
2287 void G1CMTask::drain_local_queue(bool partially) {
2288 if (has_aborted()) {
2289 return;
2290 }
2291
2292 // Decide what the target size is, depending whether we're going to
2293 // drain it partially (so that other tasks can steal if they run out
2294 // of things to do) or totally (at the very end).
2295 size_t target_size;
2296 if (partially) {
2297 target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize);
2298 } else {
2299 target_size = 0;
2300 }
2301
2302 if (_task_queue->size() > target_size) {
2303 G1TaskQueueEntry entry;
2304 bool ret = _task_queue->pop_local(entry);
2305 while (ret) {
2306 scan_task_entry(entry);
2307 if (_task_queue->size() <= target_size || has_aborted()) {
2308 ret = false;
2309 } else {
2310 ret = _task_queue->pop_local(entry);
2311 }
2312 }
2313 }
2314 }
2315
2316 void G1CMTask::drain_global_stack(bool partially) {
2317 if (has_aborted()) return;
2318
2319 // We have a policy to drain the local queue before we attempt to
2320 // drain the global stack.
2321 assert(partially || _task_queue->size() == 0, "invariant");
2322
2323 // Decide what the target size is, depending whether we're going to
2324 // drain it partially (so that other tasks can steal if they run out
2325 // of things to do) or totally (at the very end).
2326 // Notice that when draining the global mark stack partially, due to the racyness
2327 // of the mark stack size update we might in fact drop below the target. But,
2328 // this is not a problem.
2329 // In case of total draining, we simply process until the global mark stack is
2330 // totally empty, disregarding the size counter.
2331 if (partially) {
2332 size_t const target_size = _cm->partial_mark_stack_size_target();
2333 while (!has_aborted() && _cm->mark_stack_size() > target_size) {
2334 if (get_entries_from_global_stack()) {
2335 drain_local_queue(partially);
2336 }
2337 }
2338 } else {
2339 while (!has_aborted() && get_entries_from_global_stack()) {
2340 drain_local_queue(partially);
2341 }
2342 }
2343 }
2344
2345 // SATB Queue has several assumptions on whether to call the par or
2346 // non-par versions of the methods. this is why some of the code is
2347 // replicated. We should really get rid of the single-threaded version
2348 // of the code to simplify things.
2349 void G1CMTask::drain_satb_buffers() {
2350 if (has_aborted()) return;
2351
2352 // We set this so that the regular clock knows that we're in the
2353 // middle of draining buffers and doesn't set the abort flag when it
2354 // notices that SATB buffers are available for draining. It'd be
2355 // very counter productive if it did that. :-)
2356 _draining_satb_buffers = true;
2357
2358 G1CMSATBBufferClosure satb_cl(this, _g1h);
2359 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2360
2361 // This keeps claiming and applying the closure to completed buffers
2362 // until we run out of buffers or we need to abort.
2363 while (!has_aborted() &&
2364 satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
2365 regular_clock_call();
2366 }
2367
2368 _draining_satb_buffers = false;
2369
2370 assert(has_aborted() ||
2371 _concurrent ||
2372 satb_mq_set.completed_buffers_num() == 0, "invariant");
2373
2374 // again, this was a potentially expensive operation, decrease the
2375 // limits to get the regular clock call early
2376 decrease_limits();
2377 }
2378
2379 void G1CMTask::print_stats() {
2380 log_debug(gc, stats)("Marking Stats, task = %u, calls = %u",
2381 _worker_id, _calls);
2382 log_debug(gc, stats)(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
2383 _elapsed_time_ms, _termination_time_ms);
2384 log_debug(gc, stats)(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
2385 _step_times_ms.num(), _step_times_ms.avg(),
2386 _step_times_ms.sd());
2387 log_debug(gc, stats)(" max = %1.2lfms, total = %1.2lfms",
2388 _step_times_ms.maximum(), _step_times_ms.sum());
2389 }
2390
2391 bool G1ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, G1TaskQueueEntry& task_entry) {
2392 return _task_queues->steal(worker_id, hash_seed, task_entry);
2393 }
2394
2395 /*****************************************************************************
2396
2397 The do_marking_step(time_target_ms, ...) method is the building
2398 block of the parallel marking framework. It can be called in parallel
2399 with other invocations of do_marking_step() on different tasks
2400 (but only one per task, obviously) and concurrently with the
2401 mutator threads, or during remark, hence it eliminates the need
2402 for two versions of the code. When called during remark, it will
2403 pick up from where the task left off during the concurrent marking
2404 phase. Interestingly, tasks are also claimable during evacuation
2405 pauses too, since do_marking_step() ensures that it aborts before
2406 it needs to yield.
2407
2408 The data structures that it uses to do marking work are the
2409 following:
2410
2411 (1) Marking Bitmap. If there are gray objects that appear only
2412 on the bitmap (this happens either when dealing with an overflow
2413 or when the initial marking phase has simply marked the roots
2414 and didn't push them on the stack), then tasks claim heap
2415 regions whose bitmap they then scan to find gray objects. A
2416 global finger indicates where the end of the last claimed region
2417 is. A local finger indicates how far into the region a task has
2418 scanned. The two fingers are used to determine how to gray an
2419 object (i.e. whether simply marking it is OK, as it will be
2420 visited by a task in the future, or whether it needs to be also
2421 pushed on a stack).
2422
2423 (2) Local Queue. The local queue of the task which is accessed
2424 reasonably efficiently by the task. Other tasks can steal from
2425 it when they run out of work. Throughout the marking phase, a
2426 task attempts to keep its local queue short but not totally
2427 empty, so that entries are available for stealing by other
2428 tasks. Only when there is no more work, a task will totally
2429 drain its local queue.
2430
2431 (3) Global Mark Stack. This handles local queue overflow. During
2432 marking only sets of entries are moved between it and the local
2433 queues, as access to it requires a mutex and more fine-grain
2434 interaction with it which might cause contention. If it
2435 overflows, then the marking phase should restart and iterate
2436 over the bitmap to identify gray objects. Throughout the marking
2437 phase, tasks attempt to keep the global mark stack at a small
2438 length but not totally empty, so that entries are available for
2439 popping by other tasks. Only when there is no more work, tasks
2440 will totally drain the global mark stack.
2441
2442 (4) SATB Buffer Queue. This is where completed SATB buffers are
2443 made available. Buffers are regularly removed from this queue
2444 and scanned for roots, so that the queue doesn't get too
2445 long. During remark, all completed buffers are processed, as
2446 well as the filled in parts of any uncompleted buffers.
2447
2448 The do_marking_step() method tries to abort when the time target
2449 has been reached. There are a few other cases when the
2450 do_marking_step() method also aborts:
2451
2452 (1) When the marking phase has been aborted (after a Full GC).
2453
2454 (2) When a global overflow (on the global stack) has been
2455 triggered. Before the task aborts, it will actually sync up with
2456 the other tasks to ensure that all the marking data structures
2457 (local queues, stacks, fingers etc.) are re-initialized so that
2458 when do_marking_step() completes, the marking phase can
2459 immediately restart.
2460
2461 (3) When enough completed SATB buffers are available. The
2462 do_marking_step() method only tries to drain SATB buffers right
2463 at the beginning. So, if enough buffers are available, the
2464 marking step aborts and the SATB buffers are processed at
2465 the beginning of the next invocation.
2466
2467 (4) To yield. when we have to yield then we abort and yield
2468 right at the end of do_marking_step(). This saves us from a lot
2469 of hassle as, by yielding we might allow a Full GC. If this
2470 happens then objects will be compacted underneath our feet, the
2471 heap might shrink, etc. We save checking for this by just
2472 aborting and doing the yield right at the end.
2473
2474 From the above it follows that the do_marking_step() method should
2475 be called in a loop (or, otherwise, regularly) until it completes.
2476
2477 If a marking step completes without its has_aborted() flag being
2478 true, it means it has completed the current marking phase (and
2479 also all other marking tasks have done so and have all synced up).
2480
2481 A method called regular_clock_call() is invoked "regularly" (in
2482 sub ms intervals) throughout marking. It is this clock method that
2483 checks all the abort conditions which were mentioned above and
2484 decides when the task should abort. A work-based scheme is used to
2485 trigger this clock method: when the number of object words the
2486 marking phase has scanned or the number of references the marking
2487 phase has visited reach a given limit. Additional invocations to
2488 the method clock have been planted in a few other strategic places
2489 too. The initial reason for the clock method was to avoid calling
2490 vtime too regularly, as it is quite expensive. So, once it was in
2491 place, it was natural to piggy-back all the other conditions on it
2492 too and not constantly check them throughout the code.
2493
2494 If do_termination is true then do_marking_step will enter its
2495 termination protocol.
2496
2497 The value of is_serial must be true when do_marking_step is being
2498 called serially (i.e. by the VMThread) and do_marking_step should
2499 skip any synchronization in the termination and overflow code.
2500 Examples include the serial remark code and the serial reference
2501 processing closures.
2502
2503 The value of is_serial must be false when do_marking_step is
2504 being called by any of the worker threads in a work gang.
2505 Examples include the concurrent marking code (CMMarkingTask),
2506 the MT remark code, and the MT reference processing closures.
2507
2508 *****************************************************************************/
2509
2510 void G1CMTask::do_marking_step(double time_target_ms,
2511 bool do_termination,
2512 bool is_serial) {
2513 assert(time_target_ms >= 1.0, "minimum granularity is 1ms");
2514 assert(_concurrent == _cm->concurrent(), "they should be the same");
2515
2516 _start_time_ms = os::elapsedVTime() * 1000.0;
2517
2518 // If do_stealing is true then do_marking_step will attempt to
2519 // steal work from the other G1CMTasks. It only makes sense to
2520 // enable stealing when the termination protocol is enabled
2521 // and do_marking_step() is not being called serially.
2522 bool do_stealing = do_termination && !is_serial;
2523
2524 double diff_prediction_ms = _g1h->g1_policy()->predictor().get_new_prediction(&_marking_step_diffs_ms);
2525 _time_target_ms = time_target_ms - diff_prediction_ms;
2526
2527 // set up the variables that are used in the work-based scheme to
2528 // call the regular clock method
2529 _words_scanned = 0;
2530 _refs_reached = 0;
2531 recalculate_limits();
2532
2533 // clear all flags
2534 clear_has_aborted();
2535 _has_timed_out = false;
2536 _draining_satb_buffers = false;
2537
2538 ++_calls;
2539
2540 // Set up the bitmap and oop closures. Anything that uses them is
2541 // eventually called from this method, so it is OK to allocate these
2542 // statically.
2543 G1CMBitMapClosure bitmap_closure(this, _cm);
2544 G1CMOopClosure cm_oop_closure(_g1h, _cm, this);
2545 set_cm_oop_closure(&cm_oop_closure);
2546
2547 if (_cm->has_overflown()) {
2548 // This can happen if the mark stack overflows during a GC pause
2549 // and this task, after a yield point, restarts. We have to abort
2550 // as we need to get into the overflow protocol which happens
2551 // right at the end of this task.
2552 set_has_aborted();
2553 }
2554
2555 // First drain any available SATB buffers. After this, we will not
2556 // look at SATB buffers before the next invocation of this method.
2557 // If enough completed SATB buffers are queued up, the regular clock
2558 // will abort this task so that it restarts.
2559 drain_satb_buffers();
2560 // ...then partially drain the local queue and the global stack
2561 drain_local_queue(true);
2562 drain_global_stack(true);
2563
2564 do {
2565 if (!has_aborted() && _curr_region != NULL) {
2566 // This means that we're already holding on to a region.
2567 assert(_finger != NULL, "if region is not NULL, then the finger "
2568 "should not be NULL either");
2569
2570 // We might have restarted this task after an evacuation pause
2571 // which might have evacuated the region we're holding on to
2572 // underneath our feet. Let's read its limit again to make sure
2573 // that we do not iterate over a region of the heap that
2574 // contains garbage (update_region_limit() will also move
2575 // _finger to the start of the region if it is found empty).
2576 update_region_limit();
2577 // We will start from _finger not from the start of the region,
2578 // as we might be restarting this task after aborting half-way
2579 // through scanning this region. In this case, _finger points to
2580 // the address where we last found a marked object. If this is a
2581 // fresh region, _finger points to start().
2582 MemRegion mr = MemRegion(_finger, _region_limit);
2583
2584 assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(),
2585 "humongous regions should go around loop once only");
2586
2587 // Some special cases:
2588 // If the memory region is empty, we can just give up the region.
2589 // If the current region is humongous then we only need to check
2590 // the bitmap for the bit associated with the start of the object,
2591 // scan the object if it's live, and give up the region.
2592 // Otherwise, let's iterate over the bitmap of the part of the region
2593 // that is left.
2594 // If the iteration is successful, give up the region.
2595 if (mr.is_empty()) {
2596 giveup_current_region();
2597 regular_clock_call();
2598 } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) {
2599 if (_next_mark_bitmap->is_marked(mr.start())) {
2600 // The object is marked - apply the closure
2601 bitmap_closure.do_addr(mr.start());
2602 }
2603 // Even if this task aborted while scanning the humongous object
2604 // we can (and should) give up the current region.
2605 giveup_current_region();
2606 regular_clock_call();
2607 } else if (_next_mark_bitmap->iterate(&bitmap_closure, mr)) {
2608 giveup_current_region();
2609 regular_clock_call();
2610 } else {
2611 assert(has_aborted(), "currently the only way to do so");
2612 // The only way to abort the bitmap iteration is to return
2613 // false from the do_bit() method. However, inside the
2614 // do_bit() method we move the _finger to point to the
2615 // object currently being looked at. So, if we bail out, we
2616 // have definitely set _finger to something non-null.
2617 assert(_finger != NULL, "invariant");
2618
2619 // Region iteration was actually aborted. So now _finger
2620 // points to the address of the object we last scanned. If we
2621 // leave it there, when we restart this task, we will rescan
2622 // the object. It is easy to avoid this. We move the finger by
2623 // enough to point to the next possible object header.
2624 assert(_finger < _region_limit, "invariant");
2625 HeapWord* const new_finger = _finger + ((oop)_finger)->size();
2626 // Check if bitmap iteration was aborted while scanning the last object
2627 if (new_finger >= _region_limit) {
2628 giveup_current_region();
2629 } else {
2630 move_finger_to(new_finger);
2631 }
2632 }
2633 }
2634 // At this point we have either completed iterating over the
2635 // region we were holding on to, or we have aborted.
2636
2637 // We then partially drain the local queue and the global stack.
2638 // (Do we really need this?)
2639 drain_local_queue(true);
2640 drain_global_stack(true);
2641
2642 // Read the note on the claim_region() method on why it might
2643 // return NULL with potentially more regions available for
2644 // claiming and why we have to check out_of_regions() to determine
2645 // whether we're done or not.
2646 while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) {
2647 // We are going to try to claim a new region. We should have
2648 // given up on the previous one.
2649 // Separated the asserts so that we know which one fires.
2650 assert(_curr_region == NULL, "invariant");
2651 assert(_finger == NULL, "invariant");
2652 assert(_region_limit == NULL, "invariant");
2653 HeapRegion* claimed_region = _cm->claim_region(_worker_id);
2654 if (claimed_region != NULL) {
2655 // Yes, we managed to claim one
2656 setup_for_region(claimed_region);
2657 assert(_curr_region == claimed_region, "invariant");
2658 }
2659 // It is important to call the regular clock here. It might take
2660 // a while to claim a region if, for example, we hit a large
2661 // block of empty regions. So we need to call the regular clock
2662 // method once round the loop to make sure it's called
2663 // frequently enough.
2664 regular_clock_call();
2665 }
2666
2667 if (!has_aborted() && _curr_region == NULL) {
2668 assert(_cm->out_of_regions(),
2669 "at this point we should be out of regions");
2670 }
2671 } while ( _curr_region != NULL && !has_aborted());
2672
2673 if (!has_aborted()) {
2674 // We cannot check whether the global stack is empty, since other
2675 // tasks might be pushing objects to it concurrently.
2676 assert(_cm->out_of_regions(),
2677 "at this point we should be out of regions");
2678 // Try to reduce the number of available SATB buffers so that
2679 // remark has less work to do.
2680 drain_satb_buffers();
2681 }
2682
2683 // Since we've done everything else, we can now totally drain the
2684 // local queue and global stack.
2685 drain_local_queue(false);
2686 drain_global_stack(false);
2687
2688 // Attempt at work stealing from other task's queues.
2689 if (do_stealing && !has_aborted()) {
2690 // We have not aborted. This means that we have finished all that
2691 // we could. Let's try to do some stealing...
2692
2693 // We cannot check whether the global stack is empty, since other
2694 // tasks might be pushing objects to it concurrently.
2695 assert(_cm->out_of_regions() && _task_queue->size() == 0,
2696 "only way to reach here");
2697 while (!has_aborted()) {
2698 G1TaskQueueEntry entry;
2699 if (_cm->try_stealing(_worker_id, &_hash_seed, entry)) {
2700 scan_task_entry(entry);
2701
2702 // And since we're towards the end, let's totally drain the
2703 // local queue and global stack.
2704 drain_local_queue(false);
2705 drain_global_stack(false);
2706 } else {
2707 break;
2708 }
2709 }
2710 }
2711
2712 // We still haven't aborted. Now, let's try to get into the
2713 // termination protocol.
2714 if (do_termination && !has_aborted()) {
2715 // We cannot check whether the global stack is empty, since other
2716 // tasks might be concurrently pushing objects on it.
2717 // Separated the asserts so that we know which one fires.
2718 assert(_cm->out_of_regions(), "only way to reach here");
2719 assert(_task_queue->size() == 0, "only way to reach here");
2720 _termination_start_time_ms = os::elapsedVTime() * 1000.0;
2721
2722 // The G1CMTask class also extends the TerminatorTerminator class,
2723 // hence its should_exit_termination() method will also decide
2724 // whether to exit the termination protocol or not.
2725 bool finished = (is_serial ||
2726 _cm->terminator()->offer_termination(this));
2727 double termination_end_time_ms = os::elapsedVTime() * 1000.0;
2728 _termination_time_ms +=
2729 termination_end_time_ms - _termination_start_time_ms;
2730
2731 if (finished) {
2732 // We're all done.
2733
2734 if (_worker_id == 0) {
2735 // Let's allow task 0 to do this
2736 if (_concurrent) {
2737 assert(_cm->concurrent_marking_in_progress(), "invariant");
2738 // We need to set this to false before the next
2739 // safepoint. This way we ensure that the marking phase
2740 // doesn't observe any more heap expansions.
2741 _cm->clear_concurrent_marking_in_progress();
2742 }
2743 }
2744
2745 // We can now guarantee that the global stack is empty, since
2746 // all other tasks have finished. We separated the guarantees so
2747 // that, if a condition is false, we can immediately find out
2748 // which one.
2749 guarantee(_cm->out_of_regions(), "only way to reach here");
2750 guarantee(_cm->mark_stack_empty(), "only way to reach here");
2751 guarantee(_task_queue->size() == 0, "only way to reach here");
2752 guarantee(!_cm->has_overflown(), "only way to reach here");
2753 } else {
2754 // Apparently there's more work to do. Let's abort this task. It
2755 // will restart it and we can hopefully find more things to do.
2756 set_has_aborted();
2757 }
2758 }
2759
2760 // Mainly for debugging purposes to make sure that a pointer to the
2761 // closure which was statically allocated in this frame doesn't
2762 // escape it by accident.
2763 set_cm_oop_closure(NULL);
2764 double end_time_ms = os::elapsedVTime() * 1000.0;
2765 double elapsed_time_ms = end_time_ms - _start_time_ms;
2766 // Update the step history.
2767 _step_times_ms.add(elapsed_time_ms);
2768
2769 if (has_aborted()) {
2770 // The task was aborted for some reason.
2771 if (_has_timed_out) {
2772 double diff_ms = elapsed_time_ms - _time_target_ms;
2773 // Keep statistics of how well we did with respect to hitting
2774 // our target only if we actually timed out (if we aborted for
2775 // other reasons, then the results might get skewed).
2776 _marking_step_diffs_ms.add(diff_ms);
2777 }
2778
2779 if (_cm->has_overflown()) {
2780 // This is the interesting one. We aborted because a global
2781 // overflow was raised. This means we have to restart the
2782 // marking phase and start iterating over regions. However, in
2783 // order to do this we have to make sure that all tasks stop
2784 // what they are doing and re-initialize in a safe manner. We
2785 // will achieve this with the use of two barrier sync points.
2786
2787 if (!is_serial) {
2788 // We only need to enter the sync barrier if being called
2789 // from a parallel context
2790 _cm->enter_first_sync_barrier(_worker_id);
2791
2792 // When we exit this sync barrier we know that all tasks have
2793 // stopped doing marking work. So, it's now safe to
2794 // re-initialize our data structures. At the end of this method,
2795 // task 0 will clear the global data structures.
2796 }
2797
2798 // We clear the local state of this task...
2799 clear_region_fields();
2800
2801 if (!is_serial) {
2802 // ...and enter the second barrier.
2803 _cm->enter_second_sync_barrier(_worker_id);
2804 }
2805 // At this point, if we're during the concurrent phase of
2806 // marking, everything has been re-initialized and we're
2807 // ready to restart.
2808 }
2809 }
2810 }
2811
2812 G1CMTask::G1CMTask(uint worker_id, G1ConcurrentMark* cm, G1CMTaskQueue* task_queue) :
2813 _objArray_processor(this),
2814 _worker_id(worker_id),
2815 _g1h(G1CollectedHeap::heap()),
2816 _cm(cm),
2817 _next_mark_bitmap(NULL),
2818 _task_queue(task_queue),
2819 _calls(0),
2820 _time_target_ms(0.0),
2821 _start_time_ms(0.0),
2822 _cm_oop_closure(NULL),
2823 _curr_region(NULL),
2824 _finger(NULL),
2825 _region_limit(NULL),
2826 _words_scanned(0),
2827 _words_scanned_limit(0),
2828 _real_words_scanned_limit(0),
2829 _refs_reached(0),
2830 _refs_reached_limit(0),
2831 _real_refs_reached_limit(0),
2832 _hash_seed(17),
2833 _has_aborted(false),
2834 _has_timed_out(false),
2835 _draining_satb_buffers(false),
2836 _step_times_ms(),
2837 _elapsed_time_ms(0.0),
2838 _termination_time_ms(0.0),
2839 _termination_start_time_ms(0.0),
2840 _concurrent(false),
2841 _marking_step_diffs_ms()
2842 {
2843 guarantee(task_queue != NULL, "invariant");
2844
2845 _marking_step_diffs_ms.add(0.5);
2846 }
2847
2848 // These are formatting macros that are used below to ensure
2849 // consistent formatting. The *_H_* versions are used to format the
2850 // header for a particular value and they should be kept consistent
2851 // with the corresponding macro. Also note that most of the macros add
2852 // the necessary white space (as a prefix) which makes them a bit
2853 // easier to compose.
2854
2855 // All the output lines are prefixed with this string to be able to
2856 // identify them easily in a large log file.
2857 #define G1PPRL_LINE_PREFIX "###"
2858
2859 #define G1PPRL_ADDR_BASE_FORMAT " " PTR_FORMAT "-" PTR_FORMAT
2860 #ifdef _LP64
2861 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
2862 #else // _LP64
2863 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s"
2864 #endif // _LP64
2865
2866 // For per-region info
2867 #define G1PPRL_TYPE_FORMAT " %-4s"
2868 #define G1PPRL_TYPE_H_FORMAT " %4s"
2869 #define G1PPRL_BYTE_FORMAT " " SIZE_FORMAT_W(9)
2870 #define G1PPRL_BYTE_H_FORMAT " %9s"
2871 #define G1PPRL_DOUBLE_FORMAT " %14.1f"
2872 #define G1PPRL_DOUBLE_H_FORMAT " %14s"
2873
2874 // For summary info
2875 #define G1PPRL_SUM_ADDR_FORMAT(tag) " " tag ":" G1PPRL_ADDR_BASE_FORMAT
2876 #define G1PPRL_SUM_BYTE_FORMAT(tag) " " tag ": " SIZE_FORMAT
2877 #define G1PPRL_SUM_MB_FORMAT(tag) " " tag ": %1.2f MB"
2878 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
2879
2880 G1PrintRegionLivenessInfoClosure::G1PrintRegionLivenessInfoClosure(const char* phase_name) :
2881 _total_used_bytes(0), _total_capacity_bytes(0),
2882 _total_prev_live_bytes(0), _total_next_live_bytes(0),
2883 _total_remset_bytes(0), _total_strong_code_roots_bytes(0)
2884 {
2885 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2886 MemRegion g1_reserved = g1h->g1_reserved();
2887 double now = os::elapsedTime();
2888
2889 // Print the header of the output.
2890 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
2891 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" HEAP"
2892 G1PPRL_SUM_ADDR_FORMAT("reserved")
2893 G1PPRL_SUM_BYTE_FORMAT("region-size"),
2894 p2i(g1_reserved.start()), p2i(g1_reserved.end()),
2895 HeapRegion::GrainBytes);
2896 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
2897 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2898 G1PPRL_TYPE_H_FORMAT
2899 G1PPRL_ADDR_BASE_H_FORMAT
2900 G1PPRL_BYTE_H_FORMAT
2901 G1PPRL_BYTE_H_FORMAT
2902 G1PPRL_BYTE_H_FORMAT
2903 G1PPRL_DOUBLE_H_FORMAT
2904 G1PPRL_BYTE_H_FORMAT
2905 G1PPRL_BYTE_H_FORMAT,
2906 "type", "address-range",
2907 "used", "prev-live", "next-live", "gc-eff",
2908 "remset", "code-roots");
2909 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2910 G1PPRL_TYPE_H_FORMAT
2911 G1PPRL_ADDR_BASE_H_FORMAT
2912 G1PPRL_BYTE_H_FORMAT
2913 G1PPRL_BYTE_H_FORMAT
2914 G1PPRL_BYTE_H_FORMAT
2915 G1PPRL_DOUBLE_H_FORMAT
2916 G1PPRL_BYTE_H_FORMAT
2917 G1PPRL_BYTE_H_FORMAT,
2918 "", "",
2919 "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)",
2920 "(bytes)", "(bytes)");
2921 }
2922
2923 bool G1PrintRegionLivenessInfoClosure::do_heap_region(HeapRegion* r) {
2924 const char* type = r->get_type_str();
2925 HeapWord* bottom = r->bottom();
2926 HeapWord* end = r->end();
2927 size_t capacity_bytes = r->capacity();
2928 size_t used_bytes = r->used();
2929 size_t prev_live_bytes = r->live_bytes();
2930 size_t next_live_bytes = r->next_live_bytes();
2931 double gc_eff = r->gc_efficiency();
2932 size_t remset_bytes = r->rem_set()->mem_size();
2933 size_t strong_code_roots_bytes = r->rem_set()->strong_code_roots_mem_size();
2934
2935 _total_used_bytes += used_bytes;
2936 _total_capacity_bytes += capacity_bytes;
2937 _total_prev_live_bytes += prev_live_bytes;
2938 _total_next_live_bytes += next_live_bytes;
2939 _total_remset_bytes += remset_bytes;
2940 _total_strong_code_roots_bytes += strong_code_roots_bytes;
2941
2942 // Print a line for this particular region.
2943 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2944 G1PPRL_TYPE_FORMAT
2945 G1PPRL_ADDR_BASE_FORMAT
2946 G1PPRL_BYTE_FORMAT
2947 G1PPRL_BYTE_FORMAT
2948 G1PPRL_BYTE_FORMAT
2949 G1PPRL_DOUBLE_FORMAT
2950 G1PPRL_BYTE_FORMAT
2951 G1PPRL_BYTE_FORMAT,
2952 type, p2i(bottom), p2i(end),
2953 used_bytes, prev_live_bytes, next_live_bytes, gc_eff,
2954 remset_bytes, strong_code_roots_bytes);
2955
2956 return false;
2957 }
2958
2959 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
2960 // add static memory usages to remembered set sizes
2961 _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
2962 // Print the footer of the output.
2963 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
2964 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2965 " SUMMARY"
2966 G1PPRL_SUM_MB_FORMAT("capacity")
2967 G1PPRL_SUM_MB_PERC_FORMAT("used")
2968 G1PPRL_SUM_MB_PERC_FORMAT("prev-live")
2969 G1PPRL_SUM_MB_PERC_FORMAT("next-live")
2970 G1PPRL_SUM_MB_FORMAT("remset")
2971 G1PPRL_SUM_MB_FORMAT("code-roots"),
2972 bytes_to_mb(_total_capacity_bytes),
2973 bytes_to_mb(_total_used_bytes),
2974 percent_of(_total_used_bytes, _total_capacity_bytes),
2975 bytes_to_mb(_total_prev_live_bytes),
2976 percent_of(_total_prev_live_bytes, _total_capacity_bytes),
2977 bytes_to_mb(_total_next_live_bytes),
2978 percent_of(_total_next_live_bytes, _total_capacity_bytes),
2979 bytes_to_mb(_total_remset_bytes),
2980 bytes_to_mb(_total_strong_code_roots_bytes));
2981 }