1 /*
2 * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/metadataOnStackMark.hpp"
27 #include "classfile/symbolTable.hpp"
28 #include "code/codeCache.hpp"
29 #include "gc/g1/concurrentMarkThread.inline.hpp"
30 #include "gc/g1/g1CollectedHeap.inline.hpp"
31 #include "gc/g1/g1CollectorState.hpp"
32 #include "gc/g1/g1ConcurrentMark.inline.hpp"
33 #include "gc/g1/g1HeapVerifier.hpp"
34 #include "gc/g1/g1OopClosures.inline.hpp"
35 #include "gc/g1/g1CardLiveData.inline.hpp"
36 #include "gc/g1/g1Policy.hpp"
37 #include "gc/g1/g1StringDedup.hpp"
38 #include "gc/g1/heapRegion.inline.hpp"
39 #include "gc/g1/heapRegionRemSet.hpp"
40 #include "gc/g1/heapRegionSet.inline.hpp"
41 #include "gc/shared/adaptiveSizePolicy.hpp"
42 #include "gc/shared/gcId.hpp"
43 #include "gc/shared/gcTimer.hpp"
44 #include "gc/shared/gcTrace.hpp"
45 #include "gc/shared/gcTraceTime.inline.hpp"
46 #include "gc/shared/genOopClosures.inline.hpp"
47 #include "gc/shared/referencePolicy.hpp"
48 #include "gc/shared/strongRootsScope.hpp"
49 #include "gc/shared/suspendibleThreadSet.hpp"
50 #include "gc/shared/taskqueue.inline.hpp"
51 #include "gc/shared/vmGCOperations.hpp"
52 #include "gc/shared/weakProcessor.hpp"
53 #include "logging/log.hpp"
54 #include "memory/allocation.hpp"
55 #include "memory/resourceArea.hpp"
56 #include "oops/access.inline.hpp"
57 #include "oops/oop.inline.hpp"
58 #include "runtime/atomic.hpp"
59 #include "runtime/handles.inline.hpp"
60 #include "runtime/java.hpp"
61 #include "runtime/prefetch.inline.hpp"
62 #include "services/memTracker.hpp"
63 #include "utilities/align.hpp"
64 #include "utilities/growableArray.hpp"
65
66 bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
67 assert(addr < _cm->finger(), "invariant");
68 assert(addr >= _task->finger(), "invariant");
69
70 // We move that task's local finger along.
71 _task->move_finger_to(addr);
72
73 _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr)));
74 // we only partially drain the local queue and global stack
75 _task->drain_local_queue(true);
76 _task->drain_global_stack(true);
77
78 // if the has_aborted flag has been raised, we need to bail out of
79 // the iteration
80 return !_task->has_aborted();
81 }
82
83 G1CMMarkStack::G1CMMarkStack() :
84 _max_chunk_capacity(0),
85 _base(NULL),
86 _chunk_capacity(0) {
87 set_empty();
88 }
89
90 bool G1CMMarkStack::resize(size_t new_capacity) {
91 assert(is_empty(), "Only resize when stack is empty.");
92 assert(new_capacity <= _max_chunk_capacity,
93 "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity);
94
95 TaskQueueEntryChunk* new_base = MmapArrayAllocator<TaskQueueEntryChunk>::allocate_or_null(new_capacity, mtGC);
96
97 if (new_base == NULL) {
98 log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
99 return false;
100 }
101 // Release old mapping.
102 if (_base != NULL) {
103 MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity);
104 }
105
106 _base = new_base;
107 _chunk_capacity = new_capacity;
108 set_empty();
109
110 return true;
111 }
112
113 size_t G1CMMarkStack::capacity_alignment() {
114 return (size_t)lcm(os::vm_allocation_granularity(), sizeof(TaskQueueEntryChunk)) / sizeof(G1TaskQueueEntry);
115 }
116
117 bool G1CMMarkStack::initialize(size_t initial_capacity, size_t max_capacity) {
118 guarantee(_max_chunk_capacity == 0, "G1CMMarkStack already initialized.");
119
120 size_t const TaskEntryChunkSizeInVoidStar = sizeof(TaskQueueEntryChunk) / sizeof(G1TaskQueueEntry);
121
122 _max_chunk_capacity = align_up(max_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
123 size_t initial_chunk_capacity = align_up(initial_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
124
125 guarantee(initial_chunk_capacity <= _max_chunk_capacity,
126 "Maximum chunk capacity " SIZE_FORMAT " smaller than initial capacity " SIZE_FORMAT,
127 _max_chunk_capacity,
128 initial_chunk_capacity);
129
130 log_debug(gc)("Initialize mark stack with " SIZE_FORMAT " chunks, maximum " SIZE_FORMAT,
131 initial_chunk_capacity, _max_chunk_capacity);
132
133 return resize(initial_chunk_capacity);
134 }
135
136 void G1CMMarkStack::expand() {
137 if (_chunk_capacity == _max_chunk_capacity) {
138 log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _chunk_capacity);
139 return;
140 }
141 size_t old_capacity = _chunk_capacity;
142 // Double capacity if possible
143 size_t new_capacity = MIN2(old_capacity * 2, _max_chunk_capacity);
144
145 if (resize(new_capacity)) {
146 log_debug(gc)("Expanded mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
147 old_capacity, new_capacity);
148 } else {
149 log_warning(gc)("Failed to expand mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
150 old_capacity, new_capacity);
151 }
152 }
153
154 G1CMMarkStack::~G1CMMarkStack() {
155 if (_base != NULL) {
156 MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity);
157 }
158 }
159
160 void G1CMMarkStack::add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem) {
161 elem->next = *list;
162 *list = elem;
163 }
164
165 void G1CMMarkStack::add_chunk_to_chunk_list(TaskQueueEntryChunk* elem) {
166 MutexLockerEx x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
167 add_chunk_to_list(&_chunk_list, elem);
168 _chunks_in_chunk_list++;
169 }
170
171 void G1CMMarkStack::add_chunk_to_free_list(TaskQueueEntryChunk* elem) {
172 MutexLockerEx x(MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag);
173 add_chunk_to_list(&_free_list, elem);
174 }
175
176 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_list(TaskQueueEntryChunk* volatile* list) {
177 TaskQueueEntryChunk* result = *list;
178 if (result != NULL) {
179 *list = (*list)->next;
180 }
181 return result;
182 }
183
184 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_chunk_list() {
185 MutexLockerEx x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
186 TaskQueueEntryChunk* result = remove_chunk_from_list(&_chunk_list);
187 if (result != NULL) {
188 _chunks_in_chunk_list--;
189 }
190 return result;
191 }
192
193 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_free_list() {
194 MutexLockerEx x(MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag);
195 return remove_chunk_from_list(&_free_list);
196 }
197
198 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::allocate_new_chunk() {
199 // This dirty read of _hwm is okay because we only ever increase the _hwm in parallel code.
200 // Further this limits _hwm to a value of _chunk_capacity + #threads, avoiding
201 // wraparound of _hwm.
202 if (_hwm >= _chunk_capacity) {
203 return NULL;
204 }
205
206 size_t cur_idx = Atomic::add(1u, &_hwm) - 1;
207 if (cur_idx >= _chunk_capacity) {
208 return NULL;
209 }
210
211 TaskQueueEntryChunk* result = ::new (&_base[cur_idx]) TaskQueueEntryChunk;
212 result->next = NULL;
213 return result;
214 }
215
216 bool G1CMMarkStack::par_push_chunk(G1TaskQueueEntry* ptr_arr) {
217 // Get a new chunk.
218 TaskQueueEntryChunk* new_chunk = remove_chunk_from_free_list();
219
220 if (new_chunk == NULL) {
221 // Did not get a chunk from the free list. Allocate from backing memory.
222 new_chunk = allocate_new_chunk();
223
224 if (new_chunk == NULL) {
225 return false;
226 }
227 }
228
229 Copy::conjoint_memory_atomic(ptr_arr, new_chunk->data, EntriesPerChunk * sizeof(G1TaskQueueEntry));
230
231 add_chunk_to_chunk_list(new_chunk);
232
233 return true;
234 }
235
236 bool G1CMMarkStack::par_pop_chunk(G1TaskQueueEntry* ptr_arr) {
237 TaskQueueEntryChunk* cur = remove_chunk_from_chunk_list();
238
239 if (cur == NULL) {
240 return false;
241 }
242
243 Copy::conjoint_memory_atomic(cur->data, ptr_arr, EntriesPerChunk * sizeof(G1TaskQueueEntry));
244
245 add_chunk_to_free_list(cur);
246 return true;
247 }
248
249 void G1CMMarkStack::set_empty() {
250 _chunks_in_chunk_list = 0;
251 _hwm = 0;
252 _chunk_list = NULL;
253 _free_list = NULL;
254 }
255
256 G1CMRootRegions::G1CMRootRegions() :
257 _cm(NULL), _scan_in_progress(false),
258 _should_abort(false), _claimed_survivor_index(0) { }
259
260 void G1CMRootRegions::init(const G1SurvivorRegions* survivors, G1ConcurrentMark* cm) {
261 _survivors = survivors;
262 _cm = cm;
263 }
264
265 void G1CMRootRegions::prepare_for_scan() {
266 assert(!scan_in_progress(), "pre-condition");
267
268 // Currently, only survivors can be root regions.
269 _claimed_survivor_index = 0;
270 _scan_in_progress = _survivors->regions()->is_nonempty();
271 _should_abort = false;
272 }
273
274 HeapRegion* G1CMRootRegions::claim_next() {
275 if (_should_abort) {
276 // If someone has set the should_abort flag, we return NULL to
277 // force the caller to bail out of their loop.
278 return NULL;
279 }
280
281 // Currently, only survivors can be root regions.
282 const GrowableArray<HeapRegion*>* survivor_regions = _survivors->regions();
283
284 int claimed_index = Atomic::add(1, &_claimed_survivor_index) - 1;
285 if (claimed_index < survivor_regions->length()) {
286 return survivor_regions->at(claimed_index);
287 }
288 return NULL;
289 }
290
291 uint G1CMRootRegions::num_root_regions() const {
292 return (uint)_survivors->regions()->length();
293 }
294
295 void G1CMRootRegions::notify_scan_done() {
296 MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
297 _scan_in_progress = false;
298 RootRegionScan_lock->notify_all();
299 }
300
301 void G1CMRootRegions::cancel_scan() {
302 notify_scan_done();
303 }
304
305 void G1CMRootRegions::scan_finished() {
306 assert(scan_in_progress(), "pre-condition");
307
308 // Currently, only survivors can be root regions.
309 if (!_should_abort) {
310 assert(_claimed_survivor_index >= 0, "otherwise comparison is invalid: %d", _claimed_survivor_index);
311 assert((uint)_claimed_survivor_index >= _survivors->length(),
312 "we should have claimed all survivors, claimed index = %u, length = %u",
313 (uint)_claimed_survivor_index, _survivors->length());
314 }
315
316 notify_scan_done();
317 }
318
319 bool G1CMRootRegions::wait_until_scan_finished() {
320 if (!scan_in_progress()) return false;
321
322 {
323 MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag);
324 while (scan_in_progress()) {
325 RootRegionScan_lock->wait(Mutex::_no_safepoint_check_flag);
326 }
327 }
328 return true;
329 }
330
331 // Returns the maximum number of workers to be used in a concurrent
332 // phase based on the number of GC workers being used in a STW
333 // phase.
334 static uint scale_concurrent_worker_threads(uint num_gc_workers) {
335 return MAX2((num_gc_workers + 2) / 4, 1U);
336 }
337
338 G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
339 G1RegionToSpaceMapper* prev_bitmap_storage,
340 G1RegionToSpaceMapper* next_bitmap_storage) :
341 // _cm_thread set inside the constructor
342 _g1h(g1h),
343 _completed_initialization(false),
344
345 _cleanup_list("Concurrent Mark Cleanup List"),
346 _mark_bitmap_1(),
347 _mark_bitmap_2(),
348 _prev_mark_bitmap(&_mark_bitmap_1),
349 _next_mark_bitmap(&_mark_bitmap_2),
350
351 _heap_start(_g1h->reserved_region().start()),
352 _heap_end(_g1h->reserved_region().end()),
353
354 _root_regions(),
355
356 _global_mark_stack(),
357
358 // _finger set in set_non_marking_state
359
360 _max_num_tasks(ParallelGCThreads),
361 // _num_active_tasks set in set_non_marking_state()
362 // _tasks set inside the constructor
363
364 _task_queues(new G1CMTaskQueueSet((int) _max_num_tasks)),
365 _terminator(ParallelTaskTerminator((int) _max_num_tasks, _task_queues)),
366
367 _first_overflow_barrier_sync(),
368 _second_overflow_barrier_sync(),
369
370 _has_overflown(false),
371 _concurrent(false),
372 _has_aborted(false),
373 _restart_for_overflow(false),
374 _concurrent_marking_in_progress(false),
375 _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
376 _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) G1OldTracer()),
377
378 // _verbose_level set below
379
380 _init_times(),
381 _remark_times(),
382 _remark_mark_times(),
383 _remark_weak_ref_times(),
384 _cleanup_times(),
385 _total_counting_time(0.0),
386 _total_rs_scrub_time(0.0),
387
388 _accum_task_vtime(NULL),
389
390 _concurrent_workers(NULL),
391 _num_concurrent_workers(0),
392 _max_concurrent_workers(0)
393 {
394 _mark_bitmap_1.initialize(g1h->reserved_region(), prev_bitmap_storage);
395 _mark_bitmap_2.initialize(g1h->reserved_region(), next_bitmap_storage);
396
397 // Create & start ConcurrentMark thread.
398 _cm_thread = new ConcurrentMarkThread(this);
399 if (_cm_thread->osthread() == NULL) {
400 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
401 }
402
403 assert(CGC_lock != NULL, "CGC_lock must be initialized");
404
405 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
406 satb_qs.set_buffer_size(G1SATBBufferSize);
407
408 _root_regions.init(_g1h->survivor(), this);
409
410 if (FLAG_IS_DEFAULT(ConcGCThreads) || ConcGCThreads == 0) {
411 // Calculate the number of concurrent worker threads by scaling
412 // the number of parallel GC threads.
413 uint marking_thread_num = scale_concurrent_worker_threads(ParallelGCThreads);
414 FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num);
415 }
416
417 assert(ConcGCThreads > 0, "ConcGCThreads have been set.");
418 if (ConcGCThreads > ParallelGCThreads) {
419 log_warning(gc)("More ConcGCThreads (%u) than ParallelGCThreads (%u).",
420 ConcGCThreads, ParallelGCThreads);
421 return;
422 }
423
424 log_debug(gc)("ConcGCThreads: %u", ConcGCThreads);
425 log_debug(gc)("ParallelGCThreads: %u", ParallelGCThreads);
426
427 _num_concurrent_workers = ConcGCThreads;
428 _max_concurrent_workers = _num_concurrent_workers;
429
430 _concurrent_workers = new WorkGang("G1 Conc", _max_concurrent_workers, false, true);
431 _concurrent_workers->initialize_workers();
432
433 if (FLAG_IS_DEFAULT(MarkStackSize)) {
434 size_t mark_stack_size =
435 MIN2(MarkStackSizeMax,
436 MAX2(MarkStackSize, (size_t) (_max_concurrent_workers * TASKQUEUE_SIZE)));
437 // Verify that the calculated value for MarkStackSize is in range.
438 // It would be nice to use the private utility routine from Arguments.
439 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) {
440 log_warning(gc)("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): "
441 "must be between 1 and " SIZE_FORMAT,
442 mark_stack_size, MarkStackSizeMax);
443 return;
444 }
445 FLAG_SET_ERGO(size_t, MarkStackSize, mark_stack_size);
446 } else {
447 // Verify MarkStackSize is in range.
448 if (FLAG_IS_CMDLINE(MarkStackSize)) {
449 if (FLAG_IS_DEFAULT(MarkStackSizeMax)) {
450 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
451 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT "): "
452 "must be between 1 and " SIZE_FORMAT,
453 MarkStackSize, MarkStackSizeMax);
454 return;
455 }
456 } else if (FLAG_IS_CMDLINE(MarkStackSizeMax)) {
457 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) {
458 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT ")"
459 " or for MarkStackSizeMax (" SIZE_FORMAT ")",
460 MarkStackSize, MarkStackSizeMax);
461 return;
462 }
463 }
464 }
465 }
466
467 if (!_global_mark_stack.initialize(MarkStackSize, MarkStackSizeMax)) {
468 vm_exit_during_initialization("Failed to allocate initial concurrent mark overflow mark stack.");
469 }
470
471 _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_num_tasks, mtGC);
472 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_num_tasks, mtGC);
473
474 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
475 _num_active_tasks = _max_num_tasks;
476
477 for (uint i = 0; i < _max_num_tasks; ++i) {
478 G1CMTaskQueue* task_queue = new G1CMTaskQueue();
479 task_queue->initialize();
480 _task_queues->register_queue(i, task_queue);
481
482 _tasks[i] = new G1CMTask(i, this, task_queue);
483
484 _accum_task_vtime[i] = 0.0;
485 }
486
487 set_non_marking_state();
488 _completed_initialization = true;
489 }
490
491 void G1ConcurrentMark::reset() {
492 // Starting values for these two. This should be called in a STW
493 // phase.
494 MemRegion reserved = _g1h->g1_reserved();
495 _heap_start = reserved.start();
496 _heap_end = reserved.end();
497
498 // Separated the asserts so that we know which one fires.
499 assert(_heap_start != NULL, "heap bounds should look ok");
500 assert(_heap_end != NULL, "heap bounds should look ok");
501 assert(_heap_start < _heap_end, "heap bounds should look ok");
502
503 // Reset all the marking data structures and any necessary flags
504 reset_marking_state();
505
506 // We reset all of them, since different phases will use
507 // different number of active threads. So, it's easiest to have all
508 // of them ready.
509 for (uint i = 0; i < _max_num_tasks; ++i) {
510 _tasks[i]->reset(_next_mark_bitmap);
511 }
512
513 // we need this to make sure that the flag is on during the evac
514 // pause with initial mark piggy-backed
515 set_concurrent_marking_in_progress();
516 }
517
518
519 void G1ConcurrentMark::reset_marking_state() {
520 _global_mark_stack.set_empty();
521
522 // Expand the marking stack, if we have to and if we can.
523 if (has_overflown()) {
524 _global_mark_stack.expand();
525 }
526
527 clear_has_overflown();
528 _finger = _heap_start;
529
530 for (uint i = 0; i < _max_num_tasks; ++i) {
531 G1CMTaskQueue* queue = _task_queues->queue(i);
532 queue->set_empty();
533 }
534 }
535
536 void G1ConcurrentMark::set_concurrency(uint active_tasks) {
537 assert(active_tasks <= _max_num_tasks, "we should not have more");
538
539 _num_active_tasks = active_tasks;
540 // Need to update the three data structures below according to the
541 // number of active threads for this phase.
542 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues);
543 _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
544 _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
545 }
546
547 void G1ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurrent) {
548 set_concurrency(active_tasks);
549
550 _concurrent = concurrent;
551 // We propagate this to all tasks, not just the active ones.
552 for (uint i = 0; i < _max_num_tasks; ++i) {
553 _tasks[i]->set_concurrent(concurrent);
554 }
555
556 if (concurrent) {
557 set_concurrent_marking_in_progress();
558 } else {
559 // We currently assume that the concurrent flag has been set to
560 // false before we start remark. At this point we should also be
561 // in a STW phase.
562 assert(!concurrent_marking_in_progress(), "invariant");
563 assert(out_of_regions(),
564 "only way to get here: _finger: " PTR_FORMAT ", _heap_end: " PTR_FORMAT,
565 p2i(_finger), p2i(_heap_end));
566 }
567 }
568
569 void G1ConcurrentMark::set_non_marking_state() {
570 // We set the global marking state to some default values when we're
571 // not doing marking.
572 reset_marking_state();
573 _num_active_tasks = 0;
574 clear_concurrent_marking_in_progress();
575 }
576
577 G1ConcurrentMark::~G1ConcurrentMark() {
578 // The G1ConcurrentMark instance is never freed.
579 ShouldNotReachHere();
580 }
581
582 class G1ClearBitMapTask : public AbstractGangTask {
583 public:
584 static size_t chunk_size() { return M; }
585
586 private:
587 // Heap region closure used for clearing the given mark bitmap.
588 class G1ClearBitmapHRClosure : public HeapRegionClosure {
589 private:
590 G1CMBitMap* _bitmap;
591 G1ConcurrentMark* _cm;
592 public:
593 G1ClearBitmapHRClosure(G1CMBitMap* bitmap, G1ConcurrentMark* cm) : HeapRegionClosure(), _cm(cm), _bitmap(bitmap) {
594 }
595
596 virtual bool do_heap_region(HeapRegion* r) {
597 size_t const chunk_size_in_words = G1ClearBitMapTask::chunk_size() / HeapWordSize;
598
599 HeapWord* cur = r->bottom();
600 HeapWord* const end = r->end();
601
602 while (cur < end) {
603 MemRegion mr(cur, MIN2(cur + chunk_size_in_words, end));
604 _bitmap->clear_range(mr);
605
606 cur += chunk_size_in_words;
607
608 // Abort iteration if after yielding the marking has been aborted.
609 if (_cm != NULL && _cm->do_yield_check() && _cm->has_aborted()) {
610 return true;
611 }
612 // Repeat the asserts from before the start of the closure. We will do them
613 // as asserts here to minimize their overhead on the product. However, we
614 // will have them as guarantees at the beginning / end of the bitmap
615 // clearing to get some checking in the product.
616 assert(_cm == NULL || _cm->cm_thread()->during_cycle(), "invariant");
617 assert(_cm == NULL || !G1CollectedHeap::heap()->collector_state()->mark_in_progress(), "invariant");
618 }
619 assert(cur == end, "Must have completed iteration over the bitmap for region %u.", r->hrm_index());
620
621 return false;
622 }
623 };
624
625 G1ClearBitmapHRClosure _cl;
626 HeapRegionClaimer _hr_claimer;
627 bool _suspendible; // If the task is suspendible, workers must join the STS.
628
629 public:
630 G1ClearBitMapTask(G1CMBitMap* bitmap, G1ConcurrentMark* cm, uint n_workers, bool suspendible) :
631 AbstractGangTask("G1 Clear Bitmap"),
632 _cl(bitmap, suspendible ? cm : NULL),
633 _hr_claimer(n_workers),
634 _suspendible(suspendible)
635 { }
636
637 void work(uint worker_id) {
638 SuspendibleThreadSetJoiner sts_join(_suspendible);
639 G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_cl, &_hr_claimer, worker_id);
640 }
641
642 bool is_complete() {
643 return _cl.is_complete();
644 }
645 };
646
647 void G1ConcurrentMark::clear_bitmap(G1CMBitMap* bitmap, WorkGang* workers, bool may_yield) {
648 assert(may_yield || SafepointSynchronize::is_at_safepoint(), "Non-yielding bitmap clear only allowed at safepoint.");
649
650 size_t const num_bytes_to_clear = (HeapRegion::GrainBytes * _g1h->num_regions()) / G1CMBitMap::heap_map_factor();
651 size_t const num_chunks = align_up(num_bytes_to_clear, G1ClearBitMapTask::chunk_size()) / G1ClearBitMapTask::chunk_size();
652
653 uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers());
654
655 G1ClearBitMapTask cl(bitmap, this, num_workers, may_yield);
656
657 log_debug(gc, ergo)("Running %s with %u workers for " SIZE_FORMAT " work units.", cl.name(), num_workers, num_chunks);
658 workers->run_task(&cl, num_workers);
659 guarantee(!may_yield || cl.is_complete(), "Must have completed iteration when not yielding.");
660 }
661
662 void G1ConcurrentMark::cleanup_for_next_mark() {
663 // Make sure that the concurrent mark thread looks to still be in
664 // the current cycle.
665 guarantee(cm_thread()->during_cycle(), "invariant");
666
667 // We are finishing up the current cycle by clearing the next
668 // marking bitmap and getting it ready for the next cycle. During
669 // this time no other cycle can start. So, let's make sure that this
670 // is the case.
671 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
672
673 clear_bitmap(_next_mark_bitmap, _concurrent_workers, true);
674
675 // Clear the live count data. If the marking has been aborted, the abort()
676 // call already did that.
677 if (!has_aborted()) {
678 clear_live_data(_concurrent_workers);
679 DEBUG_ONLY(verify_live_data_clear());
680 }
681
682 // Repeat the asserts from above.
683 guarantee(cm_thread()->during_cycle(), "invariant");
684 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
685 }
686
687 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) {
688 assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint.");
689 clear_bitmap(_prev_mark_bitmap, workers, false);
690 }
691
692 class CheckBitmapClearHRClosure : public HeapRegionClosure {
693 G1CMBitMap* _bitmap;
694 bool _error;
695 public:
696 CheckBitmapClearHRClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) {
697 }
698
699 virtual bool do_heap_region(HeapRegion* r) {
700 // This closure can be called concurrently to the mutator, so we must make sure
701 // that the result of the getNextMarkedWordAddress() call is compared to the
702 // value passed to it as limit to detect any found bits.
703 // end never changes in G1.
704 HeapWord* end = r->end();
705 return _bitmap->get_next_marked_addr(r->bottom(), end) != end;
706 }
707 };
708
709 bool G1ConcurrentMark::next_mark_bitmap_is_clear() {
710 CheckBitmapClearHRClosure cl(_next_mark_bitmap);
711 _g1h->heap_region_iterate(&cl);
712 return cl.is_complete();
713 }
714
715 class NoteStartOfMarkHRClosure: public HeapRegionClosure {
716 public:
717 bool do_heap_region(HeapRegion* r) {
718 r->note_start_of_marking();
719 return false;
720 }
721 };
722
723 void G1ConcurrentMark::checkpoint_roots_initial_pre() {
724 G1CollectedHeap* g1h = G1CollectedHeap::heap();
725
726 _has_aborted = false;
727
728 // Initialize marking structures. This has to be done in a STW phase.
729 reset();
730
731 // For each region note start of marking.
732 NoteStartOfMarkHRClosure startcl;
733 g1h->heap_region_iterate(&startcl);
734 }
735
736
737 void G1ConcurrentMark::checkpoint_roots_initial_post() {
738 G1CollectedHeap* g1h = G1CollectedHeap::heap();
739
740 // Start Concurrent Marking weak-reference discovery.
741 ReferenceProcessor* rp = g1h->ref_processor_cm();
742 // enable ("weak") refs discovery
743 rp->enable_discovery();
744 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
745
746 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
747 // This is the start of the marking cycle, we're expected all
748 // threads to have SATB queues with active set to false.
749 satb_mq_set.set_active_all_threads(true, /* new active value */
750 false /* expected_active */);
751
752 _root_regions.prepare_for_scan();
753
754 // update_g1_committed() will be called at the end of an evac pause
755 // when marking is on. So, it's also called at the end of the
756 // initial-mark pause to update the heap end, if the heap expands
757 // during it. No need to call it here.
758 }
759
760 /*
761 * Notice that in the next two methods, we actually leave the STS
762 * during the barrier sync and join it immediately afterwards. If we
763 * do not do this, the following deadlock can occur: one thread could
764 * be in the barrier sync code, waiting for the other thread to also
765 * sync up, whereas another one could be trying to yield, while also
766 * waiting for the other threads to sync up too.
767 *
768 * Note, however, that this code is also used during remark and in
769 * this case we should not attempt to leave / enter the STS, otherwise
770 * we'll either hit an assert (debug / fastdebug) or deadlock
771 * (product). So we should only leave / enter the STS if we are
772 * operating concurrently.
773 *
774 * Because the thread that does the sync barrier has left the STS, it
775 * is possible to be suspended for a Full GC or an evacuation pause
776 * could occur. This is actually safe, since the entering the sync
777 * barrier is one of the last things do_marking_step() does, and it
778 * doesn't manipulate any data structures afterwards.
779 */
780
781 void G1ConcurrentMark::enter_first_sync_barrier(uint worker_id) {
782 bool barrier_aborted;
783 {
784 SuspendibleThreadSetLeaver sts_leave(concurrent());
785 barrier_aborted = !_first_overflow_barrier_sync.enter();
786 }
787
788 // at this point everyone should have synced up and not be doing any
789 // more work
790
791 if (barrier_aborted) {
792 // If the barrier aborted we ignore the overflow condition and
793 // just abort the whole marking phase as quickly as possible.
794 return;
795 }
796
797 // If we're executing the concurrent phase of marking, reset the marking
798 // state; otherwise the marking state is reset after reference processing,
799 // during the remark pause.
800 // If we reset here as a result of an overflow during the remark we will
801 // see assertion failures from any subsequent set_concurrency_and_phase()
802 // calls.
803 if (concurrent()) {
804 // let the task associated with with worker 0 do this
805 if (worker_id == 0) {
806 // task 0 is responsible for clearing the global data structures
807 // We should be here because of an overflow. During STW we should
808 // not clear the overflow flag since we rely on it being true when
809 // we exit this method to abort the pause and restart concurrent
810 // marking.
811 reset_marking_state();
812
813 log_info(gc, marking)("Concurrent Mark reset for overflow");
814 }
815 }
816
817 // after this, each task should reset its own data structures then
818 // then go into the second barrier
819 }
820
821 void G1ConcurrentMark::enter_second_sync_barrier(uint worker_id) {
822 SuspendibleThreadSetLeaver sts_leave(concurrent());
823 _second_overflow_barrier_sync.enter();
824
825 // at this point everything should be re-initialized and ready to go
826 }
827
828 class G1CMConcurrentMarkingTask: public AbstractGangTask {
829 private:
830 G1ConcurrentMark* _cm;
831 ConcurrentMarkThread* _cmt;
832
833 public:
834 void work(uint worker_id) {
835 assert(Thread::current()->is_ConcurrentGC_thread(), "Not a concurrent GC thread");
836 ResourceMark rm;
837
838 double start_vtime = os::elapsedVTime();
839
840 {
841 SuspendibleThreadSetJoiner sts_join;
842
843 assert(worker_id < _cm->active_tasks(), "invariant");
844
845 G1CMTask* task = _cm->task(worker_id);
846 task->record_start_time();
847 if (!_cm->has_aborted()) {
848 do {
849 task->do_marking_step(G1ConcMarkStepDurationMillis,
850 true /* do_termination */,
851 false /* is_serial*/);
852
853 _cm->do_yield_check();
854 } while (!_cm->has_aborted() && task->has_aborted());
855 }
856 task->record_end_time();
857 guarantee(!task->has_aborted() || _cm->has_aborted(), "invariant");
858 }
859
860 double end_vtime = os::elapsedVTime();
861 _cm->update_accum_task_vtime(worker_id, end_vtime - start_vtime);
862 }
863
864 G1CMConcurrentMarkingTask(G1ConcurrentMark* cm,
865 ConcurrentMarkThread* cmt) :
866 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { }
867
868 ~G1CMConcurrentMarkingTask() { }
869 };
870
871 uint G1ConcurrentMark::calc_active_marking_workers() {
872 uint result = 0;
873 if (!UseDynamicNumberOfGCThreads ||
874 (!FLAG_IS_DEFAULT(ConcGCThreads) &&
875 !ForceDynamicNumberOfGCThreads)) {
876 result = _max_concurrent_workers;
877 } else {
878 result =
879 AdaptiveSizePolicy::calc_default_active_workers(_max_concurrent_workers,
880 1, /* Minimum workers */
881 _num_concurrent_workers,
882 Threads::number_of_non_daemon_threads());
883 // Don't scale the result down by scale_concurrent_workers() because
884 // that scaling has already gone into "_max_concurrent_workers".
885 }
886 assert(result > 0 && result <= _max_concurrent_workers,
887 "Calculated number of marking workers must be larger than zero and at most the maximum %u, but is %u",
888 _max_concurrent_workers, result);
889 return result;
890 }
891
892 void G1ConcurrentMark::scan_root_region(HeapRegion* hr) {
893 // Currently, only survivors can be root regions.
894 assert(hr->next_top_at_mark_start() == hr->bottom(), "invariant");
895 G1RootRegionScanClosure cl(_g1h, this);
896
897 const uintx interval = PrefetchScanIntervalInBytes;
898 HeapWord* curr = hr->bottom();
899 const HeapWord* end = hr->top();
900 while (curr < end) {
901 Prefetch::read(curr, interval);
902 oop obj = oop(curr);
903 int size = obj->oop_iterate_size(&cl);
904 assert(size == obj->size(), "sanity");
905 curr += size;
906 }
907 }
908
909 class G1CMRootRegionScanTask : public AbstractGangTask {
910 private:
911 G1ConcurrentMark* _cm;
912
913 public:
914 G1CMRootRegionScanTask(G1ConcurrentMark* cm) :
915 AbstractGangTask("G1 Root Region Scan"), _cm(cm) { }
916
917 void work(uint worker_id) {
918 assert(Thread::current()->is_ConcurrentGC_thread(),
919 "this should only be done by a conc GC thread");
920
921 G1CMRootRegions* root_regions = _cm->root_regions();
922 HeapRegion* hr = root_regions->claim_next();
923 while (hr != NULL) {
924 _cm->scan_root_region(hr);
925 hr = root_regions->claim_next();
926 }
927 }
928 };
929
930 void G1ConcurrentMark::scan_root_regions() {
931 // scan_in_progress() will have been set to true only if there was
932 // at least one root region to scan. So, if it's false, we
933 // should not attempt to do any further work.
934 if (root_regions()->scan_in_progress()) {
935 assert(!has_aborted(), "Aborting before root region scanning is finished not supported.");
936
937 _num_concurrent_workers = MIN2(calc_active_marking_workers(),
938 // We distribute work on a per-region basis, so starting
939 // more threads than that is useless.
940 root_regions()->num_root_regions());
941 assert(_num_concurrent_workers <= _max_concurrent_workers,
942 "Maximum number of marking threads exceeded");
943
944 G1CMRootRegionScanTask task(this);
945 log_debug(gc, ergo)("Running %s using %u workers for %u work units.",
946 task.name(), _num_concurrent_workers, root_regions()->num_root_regions());
947 _concurrent_workers->run_task(&task, _num_concurrent_workers);
948
949 // It's possible that has_aborted() is true here without actually
950 // aborting the survivor scan earlier. This is OK as it's
951 // mainly used for sanity checking.
952 root_regions()->scan_finished();
953 }
954 }
955
956 void G1ConcurrentMark::concurrent_cycle_start() {
957 _gc_timer_cm->register_gc_start();
958
959 _gc_tracer_cm->report_gc_start(GCCause::_no_gc /* first parameter is not used */, _gc_timer_cm->gc_start());
960
961 _g1h->trace_heap_before_gc(_gc_tracer_cm);
962 }
963
964 void G1ConcurrentMark::concurrent_cycle_end() {
965 _g1h->trace_heap_after_gc(_gc_tracer_cm);
966
967 if (has_aborted()) {
968 _gc_tracer_cm->report_concurrent_mode_failure();
969 }
970
971 _gc_timer_cm->register_gc_end();
972
973 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
974 }
975
976 void G1ConcurrentMark::mark_from_roots() {
977 // we might be tempted to assert that:
978 // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
979 // "inconsistent argument?");
980 // However that wouldn't be right, because it's possible that
981 // a safepoint is indeed in progress as a younger generation
982 // stop-the-world GC happens even as we mark in this generation.
983
984 _restart_for_overflow = false;
985
986 _num_concurrent_workers = calc_active_marking_workers();
987
988 uint active_workers = MAX2(1U, _num_concurrent_workers);
989
990 // Setting active workers is not guaranteed since fewer
991 // worker threads may currently exist and more may not be
992 // available.
993 active_workers = _concurrent_workers->update_active_workers(active_workers);
994 log_info(gc, task)("Using %u workers of %u for marking", active_workers, _concurrent_workers->total_workers());
995
996 // Parallel task terminator is set in "set_concurrency_and_phase()"
997 set_concurrency_and_phase(active_workers, true /* concurrent */);
998
999 G1CMConcurrentMarkingTask marking_task(this, cm_thread());
1000 _concurrent_workers->run_task(&marking_task);
1001 print_stats();
1002 }
1003
1004 void G1ConcurrentMark::checkpoint_roots_final(bool clear_all_soft_refs) {
1005 // world is stopped at this checkpoint
1006 assert(SafepointSynchronize::is_at_safepoint(),
1007 "world should be stopped");
1008
1009 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1010
1011 // If a full collection has happened, we shouldn't do this.
1012 if (has_aborted()) {
1013 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1014 return;
1015 }
1016
1017 if (VerifyDuringGC) {
1018 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (before)");
1019 }
1020 g1h->verifier()->check_bitmaps("Remark Start");
1021
1022 G1Policy* g1p = g1h->g1_policy();
1023 g1p->record_concurrent_mark_remark_start();
1024
1025 double start = os::elapsedTime();
1026
1027 checkpoint_roots_final_work();
1028
1029 double mark_work_end = os::elapsedTime();
1030
1031 weak_refs_work(clear_all_soft_refs);
1032
1033 if (has_overflown()) {
1034 // We overflowed. Restart concurrent marking.
1035 _restart_for_overflow = true;
1036
1037 // Verify the heap w.r.t. the previous marking bitmap.
1038 if (VerifyDuringGC) {
1039 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (overflow)");
1040 }
1041
1042 // Clear the marking state because we will be restarting
1043 // marking due to overflowing the global mark stack.
1044 reset_marking_state();
1045 } else {
1046 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1047 // We're done with marking.
1048 // This is the end of the marking cycle, we're expected all
1049 // threads to have SATB queues with active set to true.
1050 satb_mq_set.set_active_all_threads(false, /* new active value */
1051 true /* expected_active */);
1052
1053 if (VerifyDuringGC) {
1054 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UseNextMarking, "During GC (after)");
1055 }
1056 g1h->verifier()->check_bitmaps("Remark End");
1057 assert(!restart_for_overflow(), "sanity");
1058 // Completely reset the marking state since marking completed
1059 set_non_marking_state();
1060 }
1061
1062 // Statistics
1063 double now = os::elapsedTime();
1064 _remark_mark_times.add((mark_work_end - start) * 1000.0);
1065 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
1066 _remark_times.add((now - start) * 1000.0);
1067
1068 g1p->record_concurrent_mark_remark_end();
1069
1070 G1CMIsAliveClosure is_alive(g1h);
1071 _gc_tracer_cm->report_object_count_after_gc(&is_alive);
1072 }
1073
1074 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
1075 G1CollectedHeap* _g1;
1076 size_t _freed_bytes;
1077 FreeRegionList* _local_cleanup_list;
1078 uint _old_regions_removed;
1079 uint _humongous_regions_removed;
1080 HRRSCleanupTask* _hrrs_cleanup_task;
1081
1082 public:
1083 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
1084 FreeRegionList* local_cleanup_list,
1085 HRRSCleanupTask* hrrs_cleanup_task) :
1086 _g1(g1),
1087 _freed_bytes(0),
1088 _local_cleanup_list(local_cleanup_list),
1089 _old_regions_removed(0),
1090 _humongous_regions_removed(0),
1091 _hrrs_cleanup_task(hrrs_cleanup_task) { }
1092
1093 size_t freed_bytes() { return _freed_bytes; }
1094 const uint old_regions_removed() { return _old_regions_removed; }
1095 const uint humongous_regions_removed() { return _humongous_regions_removed; }
1096
1097 bool do_heap_region(HeapRegion *hr) {
1098 _g1->reset_gc_time_stamps(hr);
1099 hr->note_end_of_marking();
1100
1101 if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young() && !hr->is_archive()) {
1102 _freed_bytes += hr->used();
1103 hr->set_containing_set(NULL);
1104 if (hr->is_humongous()) {
1105 _humongous_regions_removed++;
1106 _g1->free_humongous_region(hr, _local_cleanup_list, true /* skip_remset */);
1107 } else {
1108 _old_regions_removed++;
1109 _g1->free_region(hr, _local_cleanup_list, true /* skip_remset */);
1110 }
1111 } else {
1112 hr->rem_set()->do_cleanup_work(_hrrs_cleanup_task);
1113 }
1114
1115 return false;
1116 }
1117 };
1118
1119 class G1ParNoteEndTask: public AbstractGangTask {
1120 friend class G1NoteEndOfConcMarkClosure;
1121
1122 protected:
1123 G1CollectedHeap* _g1h;
1124 FreeRegionList* _cleanup_list;
1125 HeapRegionClaimer _hrclaimer;
1126
1127 public:
1128 G1ParNoteEndTask(G1CollectedHeap* g1h, FreeRegionList* cleanup_list, uint n_workers) :
1129 AbstractGangTask("G1 note end"), _g1h(g1h), _cleanup_list(cleanup_list), _hrclaimer(n_workers) {
1130 }
1131
1132 void work(uint worker_id) {
1133 FreeRegionList local_cleanup_list("Local Cleanup List");
1134 HRRSCleanupTask hrrs_cleanup_task;
1135 G1NoteEndOfConcMarkClosure g1_note_end(_g1h, &local_cleanup_list,
1136 &hrrs_cleanup_task);
1137 _g1h->heap_region_par_iterate_from_worker_offset(&g1_note_end, &_hrclaimer, worker_id);
1138 assert(g1_note_end.is_complete(), "Shouldn't have yielded!");
1139
1140 // Now update the lists
1141 _g1h->remove_from_old_sets(g1_note_end.old_regions_removed(), g1_note_end.humongous_regions_removed());
1142 {
1143 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
1144 _g1h->decrement_summary_bytes(g1_note_end.freed_bytes());
1145
1146 // If we iterate over the global cleanup list at the end of
1147 // cleanup to do this printing we will not guarantee to only
1148 // generate output for the newly-reclaimed regions (the list
1149 // might not be empty at the beginning of cleanup; we might
1150 // still be working on its previous contents). So we do the
1151 // printing here, before we append the new regions to the global
1152 // cleanup list.
1153
1154 G1HRPrinter* hr_printer = _g1h->hr_printer();
1155 if (hr_printer->is_active()) {
1156 FreeRegionListIterator iter(&local_cleanup_list);
1157 while (iter.more_available()) {
1158 HeapRegion* hr = iter.get_next();
1159 hr_printer->cleanup(hr);
1160 }
1161 }
1162
1163 _cleanup_list->add_ordered(&local_cleanup_list);
1164 assert(local_cleanup_list.is_empty(), "post-condition");
1165
1166 HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task);
1167 }
1168 }
1169 };
1170
1171 void G1ConcurrentMark::cleanup() {
1172 // world is stopped at this checkpoint
1173 assert(SafepointSynchronize::is_at_safepoint(),
1174 "world should be stopped");
1175 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1176
1177 // If a full collection has happened, we shouldn't do this.
1178 if (has_aborted()) {
1179 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused
1180 return;
1181 }
1182
1183 g1h->verifier()->verify_region_sets_optional();
1184
1185 if (VerifyDuringGC) {
1186 g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (before)");
1187 }
1188 g1h->verifier()->check_bitmaps("Cleanup Start");
1189
1190 G1Policy* g1p = g1h->g1_policy();
1191 g1p->record_concurrent_mark_cleanup_start();
1192
1193 double start = os::elapsedTime();
1194
1195 HeapRegionRemSet::reset_for_cleanup_tasks();
1196
1197 {
1198 GCTraceTime(Debug, gc)("Finalize Live Data");
1199 finalize_live_data();
1200 }
1201
1202 if (VerifyDuringGC) {
1203 GCTraceTime(Debug, gc)("Verify Live Data");
1204 verify_live_data();
1205 }
1206
1207 g1h->collector_state()->set_mark_in_progress(false);
1208
1209 double count_end = os::elapsedTime();
1210 double this_final_counting_time = (count_end - start);
1211 _total_counting_time += this_final_counting_time;
1212
1213 if (log_is_enabled(Trace, gc, liveness)) {
1214 G1PrintRegionLivenessInfoClosure cl("Post-Marking");
1215 _g1h->heap_region_iterate(&cl);
1216 }
1217
1218 // Install newly created mark bitMap as "prev".
1219 swap_mark_bitmaps();
1220
1221 g1h->reset_gc_time_stamp();
1222
1223 uint n_workers = _g1h->workers()->active_workers();
1224
1225 // Note end of marking in all heap regions.
1226 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers);
1227 g1h->workers()->run_task(&g1_par_note_end_task);
1228 g1h->check_gc_time_stamps();
1229
1230 if (!cleanup_list_is_empty()) {
1231 // The cleanup list is not empty, so we'll have to process it
1232 // concurrently. Notify anyone else that might be wanting free
1233 // regions that there will be more free regions coming soon.
1234 g1h->set_free_regions_coming();
1235 }
1236
1237 // call below, since it affects the metric by which we sort the heap
1238 // regions.
1239 if (G1ScrubRemSets) {
1240 double rs_scrub_start = os::elapsedTime();
1241 g1h->scrub_rem_set();
1242 _total_rs_scrub_time += (os::elapsedTime() - rs_scrub_start);
1243 }
1244
1245 // this will also free any regions totally full of garbage objects,
1246 // and sort the regions.
1247 g1h->g1_policy()->record_concurrent_mark_cleanup_end();
1248
1249 // Statistics.
1250 double end = os::elapsedTime();
1251 _cleanup_times.add((end - start) * 1000.0);
1252
1253 // Clean up will have freed any regions completely full of garbage.
1254 // Update the soft reference policy with the new heap occupancy.
1255 Universe::update_heap_info_at_gc();
1256
1257 if (VerifyDuringGC) {
1258 g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (after)");
1259 }
1260
1261 g1h->verifier()->check_bitmaps("Cleanup End");
1262
1263 g1h->verifier()->verify_region_sets_optional();
1264
1265 // We need to make this be a "collection" so any collection pause that
1266 // races with it goes around and waits for completeCleanup to finish.
1267 g1h->increment_total_collections();
1268
1269 // Clean out dead classes and update Metaspace sizes.
1270 if (ClassUnloadingWithConcurrentMark) {
1271 ClassLoaderDataGraph::purge();
1272 }
1273 MetaspaceGC::compute_new_size();
1274
1275 // We reclaimed old regions so we should calculate the sizes to make
1276 // sure we update the old gen/space data.
1277 g1h->g1mm()->update_sizes();
1278 }
1279
1280 void G1ConcurrentMark::complete_cleanup() {
1281 if (has_aborted()) return;
1282
1283 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1284
1285 _cleanup_list.verify_optional();
1286 FreeRegionList tmp_free_list("Tmp Free List");
1287
1288 log_develop_trace(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "
1289 "cleanup list has %u entries",
1290 _cleanup_list.length());
1291
1292 // No one else should be accessing the _cleanup_list at this point,
1293 // so it is not necessary to take any locks
1294 while (!_cleanup_list.is_empty()) {
1295 HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */);
1296 assert(hr != NULL, "Got NULL from a non-empty list");
1297 hr->par_clear();
1298 tmp_free_list.add_ordered(hr);
1299
1300 // Instead of adding one region at a time to the secondary_free_list,
1301 // we accumulate them in the local list and move them a few at a
1302 // time. This also cuts down on the number of notify_all() calls
1303 // we do during this process. We'll also append the local list when
1304 // _cleanup_list is empty (which means we just removed the last
1305 // region from the _cleanup_list).
1306 if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
1307 _cleanup_list.is_empty()) {
1308 log_develop_trace(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : "
1309 "appending %u entries to the secondary_free_list, "
1310 "cleanup list still has %u entries",
1311 tmp_free_list.length(),
1312 _cleanup_list.length());
1313
1314 {
1315 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
1316 g1h->secondary_free_list_add(&tmp_free_list);
1317 SecondaryFreeList_lock->notify_all();
1318 }
1319 #ifndef PRODUCT
1320 if (G1StressConcRegionFreeing) {
1321 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
1322 os::sleep(Thread::current(), (jlong) 1, false);
1323 }
1324 }
1325 #endif
1326 }
1327 }
1328 assert(tmp_free_list.is_empty(), "post-condition");
1329 }
1330
1331 // Supporting Object and Oop closures for reference discovery
1332 // and processing in during marking
1333
1334 bool G1CMIsAliveClosure::do_object_b(oop obj) {
1335 HeapWord* addr = (HeapWord*)obj;
1336 return addr != NULL &&
1337 (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj));
1338 }
1339
1340 // 'Keep Alive' oop closure used by both serial parallel reference processing.
1341 // Uses the G1CMTask associated with a worker thread (for serial reference
1342 // processing the G1CMTask for worker 0 is used) to preserve (mark) and
1343 // trace referent objects.
1344 //
1345 // Using the G1CMTask and embedded local queues avoids having the worker
1346 // threads operating on the global mark stack. This reduces the risk
1347 // of overflowing the stack - which we would rather avoid at this late
1348 // state. Also using the tasks' local queues removes the potential
1349 // of the workers interfering with each other that could occur if
1350 // operating on the global stack.
1351
1352 class G1CMKeepAliveAndDrainClosure: public OopClosure {
1353 G1ConcurrentMark* _cm;
1354 G1CMTask* _task;
1355 int _ref_counter_limit;
1356 int _ref_counter;
1357 bool _is_serial;
1358 public:
1359 G1CMKeepAliveAndDrainClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) :
1360 _cm(cm), _task(task), _is_serial(is_serial),
1361 _ref_counter_limit(G1RefProcDrainInterval) {
1362 assert(_ref_counter_limit > 0, "sanity");
1363 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code");
1364 _ref_counter = _ref_counter_limit;
1365 }
1366
1367 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
1368 virtual void do_oop( oop* p) { do_oop_work(p); }
1369
1370 template <class T> void do_oop_work(T* p) {
1371 if (!_cm->has_overflown()) {
1372 oop obj = RawAccess<>::oop_load(p);
1373 _task->deal_with_reference(obj);
1374 _ref_counter--;
1375
1376 if (_ref_counter == 0) {
1377 // We have dealt with _ref_counter_limit references, pushing them
1378 // and objects reachable from them on to the local stack (and
1379 // possibly the global stack). Call G1CMTask::do_marking_step() to
1380 // process these entries.
1381 //
1382 // We call G1CMTask::do_marking_step() in a loop, which we'll exit if
1383 // there's nothing more to do (i.e. we're done with the entries that
1384 // were pushed as a result of the G1CMTask::deal_with_reference() calls
1385 // above) or we overflow.
1386 //
1387 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted()
1388 // flag while there may still be some work to do. (See the comment at
1389 // the beginning of G1CMTask::do_marking_step() for those conditions -
1390 // one of which is reaching the specified time target.) It is only
1391 // when G1CMTask::do_marking_step() returns without setting the
1392 // has_aborted() flag that the marking step has completed.
1393 do {
1394 double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
1395 _task->do_marking_step(mark_step_duration_ms,
1396 false /* do_termination */,
1397 _is_serial);
1398 } while (_task->has_aborted() && !_cm->has_overflown());
1399 _ref_counter = _ref_counter_limit;
1400 }
1401 }
1402 }
1403 };
1404
1405 // 'Drain' oop closure used by both serial and parallel reference processing.
1406 // Uses the G1CMTask associated with a given worker thread (for serial
1407 // reference processing the G1CMtask for worker 0 is used). Calls the
1408 // do_marking_step routine, with an unbelievably large timeout value,
1409 // to drain the marking data structures of the remaining entries
1410 // added by the 'keep alive' oop closure above.
1411
1412 class G1CMDrainMarkingStackClosure: public VoidClosure {
1413 G1ConcurrentMark* _cm;
1414 G1CMTask* _task;
1415 bool _is_serial;
1416 public:
1417 G1CMDrainMarkingStackClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) :
1418 _cm(cm), _task(task), _is_serial(is_serial) {
1419 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code");
1420 }
1421
1422 void do_void() {
1423 do {
1424 // We call G1CMTask::do_marking_step() to completely drain the local
1425 // and global marking stacks of entries pushed by the 'keep alive'
1426 // oop closure (an instance of G1CMKeepAliveAndDrainClosure above).
1427 //
1428 // G1CMTask::do_marking_step() is called in a loop, which we'll exit
1429 // if there's nothing more to do (i.e. we've completely drained the
1430 // entries that were pushed as a a result of applying the 'keep alive'
1431 // closure to the entries on the discovered ref lists) or we overflow
1432 // the global marking stack.
1433 //
1434 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted()
1435 // flag while there may still be some work to do. (See the comment at
1436 // the beginning of G1CMTask::do_marking_step() for those conditions -
1437 // one of which is reaching the specified time target.) It is only
1438 // when G1CMTask::do_marking_step() returns without setting the
1439 // has_aborted() flag that the marking step has completed.
1440
1441 _task->do_marking_step(1000000000.0 /* something very large */,
1442 true /* do_termination */,
1443 _is_serial);
1444 } while (_task->has_aborted() && !_cm->has_overflown());
1445 }
1446 };
1447
1448 // Implementation of AbstractRefProcTaskExecutor for parallel
1449 // reference processing at the end of G1 concurrent marking
1450
1451 class G1CMRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
1452 private:
1453 G1CollectedHeap* _g1h;
1454 G1ConcurrentMark* _cm;
1455 WorkGang* _workers;
1456 uint _active_workers;
1457
1458 public:
1459 G1CMRefProcTaskExecutor(G1CollectedHeap* g1h,
1460 G1ConcurrentMark* cm,
1461 WorkGang* workers,
1462 uint n_workers) :
1463 _g1h(g1h), _cm(cm),
1464 _workers(workers), _active_workers(n_workers) { }
1465
1466 // Executes the given task using concurrent marking worker threads.
1467 virtual void execute(ProcessTask& task);
1468 virtual void execute(EnqueueTask& task);
1469 };
1470
1471 class G1CMRefProcTaskProxy: public AbstractGangTask {
1472 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
1473 ProcessTask& _proc_task;
1474 G1CollectedHeap* _g1h;
1475 G1ConcurrentMark* _cm;
1476
1477 public:
1478 G1CMRefProcTaskProxy(ProcessTask& proc_task,
1479 G1CollectedHeap* g1h,
1480 G1ConcurrentMark* cm) :
1481 AbstractGangTask("Process reference objects in parallel"),
1482 _proc_task(proc_task), _g1h(g1h), _cm(cm) {
1483 ReferenceProcessor* rp = _g1h->ref_processor_cm();
1484 assert(rp->processing_is_mt(), "shouldn't be here otherwise");
1485 }
1486
1487 virtual void work(uint worker_id) {
1488 ResourceMark rm;
1489 HandleMark hm;
1490 G1CMTask* task = _cm->task(worker_id);
1491 G1CMIsAliveClosure g1_is_alive(_g1h);
1492 G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */);
1493 G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */);
1494
1495 _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain);
1496 }
1497 };
1498
1499 void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) {
1500 assert(_workers != NULL, "Need parallel worker threads.");
1501 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
1502
1503 G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm);
1504
1505 // We need to reset the concurrency level before each
1506 // proxy task execution, so that the termination protocol
1507 // and overflow handling in G1CMTask::do_marking_step() knows
1508 // how many workers to wait for.
1509 _cm->set_concurrency(_active_workers);
1510 _workers->run_task(&proc_task_proxy);
1511 }
1512
1513 class G1CMRefEnqueueTaskProxy: public AbstractGangTask {
1514 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
1515 EnqueueTask& _enq_task;
1516
1517 public:
1518 G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) :
1519 AbstractGangTask("Enqueue reference objects in parallel"),
1520 _enq_task(enq_task) { }
1521
1522 virtual void work(uint worker_id) {
1523 _enq_task.work(worker_id);
1524 }
1525 };
1526
1527 void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
1528 assert(_workers != NULL, "Need parallel worker threads.");
1529 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT");
1530
1531 G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task);
1532
1533 // Not strictly necessary but...
1534 //
1535 // We need to reset the concurrency level before each
1536 // proxy task execution, so that the termination protocol
1537 // and overflow handling in G1CMTask::do_marking_step() knows
1538 // how many workers to wait for.
1539 _cm->set_concurrency(_active_workers);
1540 _workers->run_task(&enq_task_proxy);
1541 }
1542
1543 void G1ConcurrentMark::weak_refs_work(bool clear_all_soft_refs) {
1544 if (has_overflown()) {
1545 // Skip processing the discovered references if we have
1546 // overflown the global marking stack. Reference objects
1547 // only get discovered once so it is OK to not
1548 // de-populate the discovered reference lists. We could have,
1549 // but the only benefit would be that, when marking restarts,
1550 // less reference objects are discovered.
1551 return;
1552 }
1553
1554 ResourceMark rm;
1555 HandleMark hm;
1556
1557 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1558
1559 // Is alive closure.
1560 G1CMIsAliveClosure g1_is_alive(g1h);
1561
1562 // Inner scope to exclude the cleaning of the string and symbol
1563 // tables from the displayed time.
1564 {
1565 GCTraceTime(Debug, gc, phases) trace("Reference Processing", _gc_timer_cm);
1566
1567 ReferenceProcessor* rp = g1h->ref_processor_cm();
1568
1569 // See the comment in G1CollectedHeap::ref_processing_init()
1570 // about how reference processing currently works in G1.
1571
1572 // Set the soft reference policy
1573 rp->setup_policy(clear_all_soft_refs);
1574 assert(_global_mark_stack.is_empty(), "mark stack should be empty");
1575
1576 // Instances of the 'Keep Alive' and 'Complete GC' closures used
1577 // in serial reference processing. Note these closures are also
1578 // used for serially processing (by the the current thread) the
1579 // JNI references during parallel reference processing.
1580 //
1581 // These closures do not need to synchronize with the worker
1582 // threads involved in parallel reference processing as these
1583 // instances are executed serially by the current thread (e.g.
1584 // reference processing is not multi-threaded and is thus
1585 // performed by the current thread instead of a gang worker).
1586 //
1587 // The gang tasks involved in parallel reference processing create
1588 // their own instances of these closures, which do their own
1589 // synchronization among themselves.
1590 G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */);
1591 G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */);
1592
1593 // We need at least one active thread. If reference processing
1594 // is not multi-threaded we use the current (VMThread) thread,
1595 // otherwise we use the work gang from the G1CollectedHeap and
1596 // we utilize all the worker threads we can.
1597 bool processing_is_mt = rp->processing_is_mt();
1598 uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U);
1599 active_workers = MAX2(MIN2(active_workers, _max_num_tasks), 1U);
1600
1601 // Parallel processing task executor.
1602 G1CMRefProcTaskExecutor par_task_executor(g1h, this,
1603 g1h->workers(), active_workers);
1604 AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL);
1605
1606 // Set the concurrency level. The phase was already set prior to
1607 // executing the remark task.
1608 set_concurrency(active_workers);
1609
1610 // Set the degree of MT processing here. If the discovery was done MT,
1611 // the number of threads involved during discovery could differ from
1612 // the number of active workers. This is OK as long as the discovered
1613 // Reference lists are balanced (see balance_all_queues() and balance_queues()).
1614 rp->set_active_mt_degree(active_workers);
1615
1616 ReferenceProcessorPhaseTimes pt(_gc_timer_cm, rp->num_q());
1617
1618 // Process the weak references.
1619 const ReferenceProcessorStats& stats =
1620 rp->process_discovered_references(&g1_is_alive,
1621 &g1_keep_alive,
1622 &g1_drain_mark_stack,
1623 executor,
1624 &pt);
1625 _gc_tracer_cm->report_gc_reference_stats(stats);
1626 pt.print_all_references();
1627
1628 // The do_oop work routines of the keep_alive and drain_marking_stack
1629 // oop closures will set the has_overflown flag if we overflow the
1630 // global marking stack.
1631
1632 assert(has_overflown() || _global_mark_stack.is_empty(),
1633 "Mark stack should be empty (unless it has overflown)");
1634
1635 assert(rp->num_q() == active_workers, "why not");
1636
1637 rp->enqueue_discovered_references(executor, &pt);
1638
1639 rp->verify_no_references_recorded();
1640
1641 pt.print_enqueue_phase();
1642
1643 assert(!rp->discovery_enabled(), "Post condition");
1644 }
1645
1646 assert(has_overflown() || _global_mark_stack.is_empty(),
1647 "Mark stack should be empty (unless it has overflown)");
1648
1649 {
1650 GCTraceTime(Debug, gc, phases) debug("Weak Processing", _gc_timer_cm);
1651 WeakProcessor::weak_oops_do(&g1_is_alive, &do_nothing_cl);
1652 }
1653
1654 if (has_overflown()) {
1655 // We can not trust g1_is_alive if the marking stack overflowed
1656 return;
1657 }
1658
1659 assert(_global_mark_stack.is_empty(), "Marking should have completed");
1660
1661 // Unload Klasses, String, Symbols, Code Cache, etc.
1662 if (ClassUnloadingWithConcurrentMark) {
1663 GCTraceTime(Debug, gc, phases) debug("Class Unloading", _gc_timer_cm);
1664 bool purged_classes = SystemDictionary::do_unloading(&g1_is_alive, _gc_timer_cm, false /* Defer cleaning */);
1665 g1h->complete_cleaning(&g1_is_alive, purged_classes);
1666 } else {
1667 GCTraceTime(Debug, gc, phases) debug("Cleanup", _gc_timer_cm);
1668 // No need to clean string table and symbol table as they are treated as strong roots when
1669 // class unloading is disabled.
1670 g1h->partial_cleaning(&g1_is_alive, false, false, G1StringDedup::is_enabled());
1671
1672 }
1673 }
1674
1675 void G1ConcurrentMark::swap_mark_bitmaps() {
1676 G1CMBitMap* temp = _prev_mark_bitmap;
1677 _prev_mark_bitmap = _next_mark_bitmap;
1678 _next_mark_bitmap = temp;
1679 }
1680
1681 // Closure for marking entries in SATB buffers.
1682 class G1CMSATBBufferClosure : public SATBBufferClosure {
1683 private:
1684 G1CMTask* _task;
1685 G1CollectedHeap* _g1h;
1686
1687 // This is very similar to G1CMTask::deal_with_reference, but with
1688 // more relaxed requirements for the argument, so this must be more
1689 // circumspect about treating the argument as an object.
1690 void do_entry(void* entry) const {
1691 _task->increment_refs_reached();
1692 oop const obj = static_cast<oop>(entry);
1693 _task->make_reference_grey(obj);
1694 }
1695
1696 public:
1697 G1CMSATBBufferClosure(G1CMTask* task, G1CollectedHeap* g1h)
1698 : _task(task), _g1h(g1h) { }
1699
1700 virtual void do_buffer(void** buffer, size_t size) {
1701 for (size_t i = 0; i < size; ++i) {
1702 do_entry(buffer[i]);
1703 }
1704 }
1705 };
1706
1707 class G1RemarkThreadsClosure : public ThreadClosure {
1708 G1CMSATBBufferClosure _cm_satb_cl;
1709 G1CMOopClosure _cm_cl;
1710 MarkingCodeBlobClosure _code_cl;
1711 int _thread_parity;
1712
1713 public:
1714 G1RemarkThreadsClosure(G1CollectedHeap* g1h, G1CMTask* task) :
1715 _cm_satb_cl(task, g1h),
1716 _cm_cl(g1h, g1h->concurrent_mark(), task),
1717 _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations),
1718 _thread_parity(Threads::thread_claim_parity()) {}
1719
1720 void do_thread(Thread* thread) {
1721 if (thread->is_Java_thread()) {
1722 if (thread->claim_oops_do(true, _thread_parity)) {
1723 JavaThread* jt = (JavaThread*)thread;
1724
1725 // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
1726 // however the liveness of oops reachable from nmethods have very complex lifecycles:
1727 // * Alive if on the stack of an executing method
1728 // * Weakly reachable otherwise
1729 // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
1730 // live by the SATB invariant but other oops recorded in nmethods may behave differently.
1731 jt->nmethods_do(&_code_cl);
1732
1733 jt->satb_mark_queue().apply_closure_and_empty(&_cm_satb_cl);
1734 }
1735 } else if (thread->is_VM_thread()) {
1736 if (thread->claim_oops_do(true, _thread_parity)) {
1737 JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(&_cm_satb_cl);
1738 }
1739 }
1740 }
1741 };
1742
1743 class G1CMRemarkTask: public AbstractGangTask {
1744 private:
1745 G1ConcurrentMark* _cm;
1746 public:
1747 void work(uint worker_id) {
1748 G1CMTask* task = _cm->task(worker_id);
1749 task->record_start_time();
1750 {
1751 ResourceMark rm;
1752 HandleMark hm;
1753
1754 G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task);
1755 Threads::threads_do(&threads_f);
1756 }
1757
1758 do {
1759 task->do_marking_step(1000000000.0 /* something very large */,
1760 true /* do_termination */,
1761 false /* is_serial */);
1762 } while (task->has_aborted() && !_cm->has_overflown());
1763 // If we overflow, then we do not want to restart. We instead
1764 // want to abort remark and do concurrent marking again.
1765 task->record_end_time();
1766 }
1767
1768 G1CMRemarkTask(G1ConcurrentMark* cm, uint active_workers) :
1769 AbstractGangTask("Par Remark"), _cm(cm) {
1770 _cm->terminator()->reset_for_reuse(active_workers);
1771 }
1772 };
1773
1774 void G1ConcurrentMark::checkpoint_roots_final_work() {
1775 ResourceMark rm;
1776 HandleMark hm;
1777 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1778
1779 GCTraceTime(Debug, gc, phases) trace("Finalize Marking", _gc_timer_cm);
1780
1781 g1h->ensure_parsability(false);
1782
1783 // this is remark, so we'll use up all active threads
1784 uint active_workers = g1h->workers()->active_workers();
1785 set_concurrency_and_phase(active_workers, false /* concurrent */);
1786 // Leave _parallel_marking_threads at it's
1787 // value originally calculated in the G1ConcurrentMark
1788 // constructor and pass values of the active workers
1789 // through the gang in the task.
1790
1791 {
1792 StrongRootsScope srs(active_workers);
1793
1794 G1CMRemarkTask remarkTask(this, active_workers);
1795 // We will start all available threads, even if we decide that the
1796 // active_workers will be fewer. The extra ones will just bail out
1797 // immediately.
1798 g1h->workers()->run_task(&remarkTask);
1799 }
1800
1801 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1802 guarantee(has_overflown() ||
1803 satb_mq_set.completed_buffers_num() == 0,
1804 "Invariant: has_overflown = %s, num buffers = " SIZE_FORMAT,
1805 BOOL_TO_STR(has_overflown()),
1806 satb_mq_set.completed_buffers_num());
1807
1808 print_stats();
1809 }
1810
1811 void G1ConcurrentMark::clear_range_in_prev_bitmap(MemRegion mr) {
1812 _prev_mark_bitmap->clear_range(mr);
1813 }
1814
1815 HeapRegion*
1816 G1ConcurrentMark::claim_region(uint worker_id) {
1817 // "checkpoint" the finger
1818 HeapWord* finger = _finger;
1819
1820 // _heap_end will not change underneath our feet; it only changes at
1821 // yield points.
1822 while (finger < _heap_end) {
1823 assert(_g1h->is_in_g1_reserved(finger), "invariant");
1824
1825 HeapRegion* curr_region = _g1h->heap_region_containing(finger);
1826 // Make sure that the reads below do not float before loading curr_region.
1827 OrderAccess::loadload();
1828 // Above heap_region_containing may return NULL as we always scan claim
1829 // until the end of the heap. In this case, just jump to the next region.
1830 HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords;
1831
1832 // Is the gap between reading the finger and doing the CAS too long?
1833 HeapWord* res = Atomic::cmpxchg(end, &_finger, finger);
1834 if (res == finger && curr_region != NULL) {
1835 // we succeeded
1836 HeapWord* bottom = curr_region->bottom();
1837 HeapWord* limit = curr_region->next_top_at_mark_start();
1838
1839 // notice that _finger == end cannot be guaranteed here since,
1840 // someone else might have moved the finger even further
1841 assert(_finger >= end, "the finger should have moved forward");
1842
1843 if (limit > bottom) {
1844 return curr_region;
1845 } else {
1846 assert(limit == bottom,
1847 "the region limit should be at bottom");
1848 // we return NULL and the caller should try calling
1849 // claim_region() again.
1850 return NULL;
1851 }
1852 } else {
1853 assert(_finger > finger, "the finger should have moved forward");
1854 // read it again
1855 finger = _finger;
1856 }
1857 }
1858
1859 return NULL;
1860 }
1861
1862 #ifndef PRODUCT
1863 class VerifyNoCSetOops {
1864 private:
1865 G1CollectedHeap* _g1h;
1866 const char* _phase;
1867 int _info;
1868
1869 public:
1870 VerifyNoCSetOops(const char* phase, int info = -1) :
1871 _g1h(G1CollectedHeap::heap()),
1872 _phase(phase),
1873 _info(info)
1874 { }
1875
1876 void operator()(G1TaskQueueEntry task_entry) const {
1877 if (task_entry.is_array_slice()) {
1878 guarantee(_g1h->is_in_reserved(task_entry.slice()), "Slice " PTR_FORMAT " must be in heap.", p2i(task_entry.slice()));
1879 return;
1880 }
1881 guarantee(oopDesc::is_oop(task_entry.obj()),
1882 "Non-oop " PTR_FORMAT ", phase: %s, info: %d",
1883 p2i(task_entry.obj()), _phase, _info);
1884 guarantee(!_g1h->is_in_cset(task_entry.obj()),
1885 "obj: " PTR_FORMAT " in CSet, phase: %s, info: %d",
1886 p2i(task_entry.obj()), _phase, _info);
1887 }
1888 };
1889
1890 void G1ConcurrentMark::verify_no_cset_oops() {
1891 assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint");
1892 if (!G1CollectedHeap::heap()->collector_state()->mark_in_progress()) {
1893 return;
1894 }
1895
1896 // Verify entries on the global mark stack
1897 _global_mark_stack.iterate(VerifyNoCSetOops("Stack"));
1898
1899 // Verify entries on the task queues
1900 for (uint i = 0; i < _max_num_tasks; ++i) {
1901 G1CMTaskQueue* queue = _task_queues->queue(i);
1902 queue->iterate(VerifyNoCSetOops("Queue", i));
1903 }
1904
1905 // Verify the global finger
1906 HeapWord* global_finger = finger();
1907 if (global_finger != NULL && global_finger < _heap_end) {
1908 // Since we always iterate over all regions, we might get a NULL HeapRegion
1909 // here.
1910 HeapRegion* global_hr = _g1h->heap_region_containing(global_finger);
1911 guarantee(global_hr == NULL || global_finger == global_hr->bottom(),
1912 "global finger: " PTR_FORMAT " region: " HR_FORMAT,
1913 p2i(global_finger), HR_FORMAT_PARAMS(global_hr));
1914 }
1915
1916 // Verify the task fingers
1917 assert(_num_concurrent_workers <= _max_num_tasks, "sanity");
1918 for (uint i = 0; i < _num_concurrent_workers; ++i) {
1919 G1CMTask* task = _tasks[i];
1920 HeapWord* task_finger = task->finger();
1921 if (task_finger != NULL && task_finger < _heap_end) {
1922 // See above note on the global finger verification.
1923 HeapRegion* task_hr = _g1h->heap_region_containing(task_finger);
1924 guarantee(task_hr == NULL || task_finger == task_hr->bottom() ||
1925 !task_hr->in_collection_set(),
1926 "task finger: " PTR_FORMAT " region: " HR_FORMAT,
1927 p2i(task_finger), HR_FORMAT_PARAMS(task_hr));
1928 }
1929 }
1930 }
1931 #endif // PRODUCT
1932 void G1ConcurrentMark::create_live_data() {
1933 _g1h->g1_rem_set()->create_card_live_data(_concurrent_workers, _next_mark_bitmap);
1934 }
1935
1936 void G1ConcurrentMark::finalize_live_data() {
1937 _g1h->g1_rem_set()->finalize_card_live_data(_g1h->workers(), _next_mark_bitmap);
1938 }
1939
1940 void G1ConcurrentMark::verify_live_data() {
1941 _g1h->g1_rem_set()->verify_card_live_data(_g1h->workers(), _next_mark_bitmap);
1942 }
1943
1944 void G1ConcurrentMark::clear_live_data(WorkGang* workers) {
1945 _g1h->g1_rem_set()->clear_card_live_data(workers);
1946 }
1947
1948 #ifdef ASSERT
1949 void G1ConcurrentMark::verify_live_data_clear() {
1950 _g1h->g1_rem_set()->verify_card_live_data_is_clear();
1951 }
1952 #endif
1953
1954 void G1ConcurrentMark::print_stats() {
1955 if (!log_is_enabled(Debug, gc, stats)) {
1956 return;
1957 }
1958 log_debug(gc, stats)("---------------------------------------------------------------------");
1959 for (size_t i = 0; i < _num_active_tasks; ++i) {
1960 _tasks[i]->print_stats();
1961 log_debug(gc, stats)("---------------------------------------------------------------------");
1962 }
1963 }
1964
1965 void G1ConcurrentMark::abort() {
1966 if (!cm_thread()->during_cycle() || _has_aborted) {
1967 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything.
1968 return;
1969 }
1970
1971 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next
1972 // concurrent bitmap clearing.
1973 {
1974 GCTraceTime(Debug, gc)("Clear Next Bitmap");
1975 clear_bitmap(_next_mark_bitmap, _g1h->workers(), false);
1976 }
1977 // Note we cannot clear the previous marking bitmap here
1978 // since VerifyDuringGC verifies the objects marked during
1979 // a full GC against the previous bitmap.
1980
1981 {
1982 GCTraceTime(Debug, gc)("Clear Live Data");
1983 clear_live_data(_g1h->workers());
1984 }
1985 DEBUG_ONLY({
1986 GCTraceTime(Debug, gc)("Verify Live Data Clear");
1987 verify_live_data_clear();
1988 })
1989 // Empty mark stack
1990 reset_marking_state();
1991 for (uint i = 0; i < _max_num_tasks; ++i) {
1992 _tasks[i]->clear_region_fields();
1993 }
1994 _first_overflow_barrier_sync.abort();
1995 _second_overflow_barrier_sync.abort();
1996 _has_aborted = true;
1997
1998 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
1999 satb_mq_set.abandon_partial_marking();
2000 // This can be called either during or outside marking, we'll read
2001 // the expected_active value from the SATB queue set.
2002 satb_mq_set.set_active_all_threads(
2003 false, /* new active value */
2004 satb_mq_set.is_active() /* expected_active */);
2005 }
2006
2007 static void print_ms_time_info(const char* prefix, const char* name,
2008 NumberSeq& ns) {
2009 log_trace(gc, marking)("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
2010 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
2011 if (ns.num() > 0) {
2012 log_trace(gc, marking)("%s [std. dev = %8.2f ms, max = %8.2f ms]",
2013 prefix, ns.sd(), ns.maximum());
2014 }
2015 }
2016
2017 void G1ConcurrentMark::print_summary_info() {
2018 Log(gc, marking) log;
2019 if (!log.is_trace()) {
2020 return;
2021 }
2022
2023 log.trace(" Concurrent marking:");
2024 print_ms_time_info(" ", "init marks", _init_times);
2025 print_ms_time_info(" ", "remarks", _remark_times);
2026 {
2027 print_ms_time_info(" ", "final marks", _remark_mark_times);
2028 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
2029
2030 }
2031 print_ms_time_info(" ", "cleanups", _cleanup_times);
2032 log.trace(" Finalize live data total time = %8.2f s (avg = %8.2f ms).",
2033 _total_counting_time, (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2034 if (G1ScrubRemSets) {
2035 log.trace(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
2036 _total_rs_scrub_time, (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
2037 }
2038 log.trace(" Total stop_world time = %8.2f s.",
2039 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
2040 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
2041 cm_thread()->vtime_accum(), cm_thread()->vtime_mark_accum());
2042 }
2043
2044 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const {
2045 _concurrent_workers->print_worker_threads_on(st);
2046 }
2047
2048 void G1ConcurrentMark::threads_do(ThreadClosure* tc) const {
2049 _concurrent_workers->threads_do(tc);
2050 }
2051
2052 void G1ConcurrentMark::print_on_error(outputStream* st) const {
2053 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT,
2054 p2i(_prev_mark_bitmap), p2i(_next_mark_bitmap));
2055 _prev_mark_bitmap->print_on_error(st, " Prev Bits: ");
2056 _next_mark_bitmap->print_on_error(st, " Next Bits: ");
2057 }
2058
2059 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) {
2060 ReferenceProcessor* result = g1h->ref_processor_cm();
2061 assert(result != NULL, "CM reference processor should not be NULL");
2062 return result;
2063 }
2064
2065 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
2066 G1ConcurrentMark* cm,
2067 G1CMTask* task)
2068 : MetadataAwareOopClosure(get_cm_oop_closure_ref_processor(g1h)),
2069 _g1h(g1h), _cm(cm), _task(task)
2070 { }
2071
2072 void G1CMTask::setup_for_region(HeapRegion* hr) {
2073 assert(hr != NULL,
2074 "claim_region() should have filtered out NULL regions");
2075 _curr_region = hr;
2076 _finger = hr->bottom();
2077 update_region_limit();
2078 }
2079
2080 void G1CMTask::update_region_limit() {
2081 HeapRegion* hr = _curr_region;
2082 HeapWord* bottom = hr->bottom();
2083 HeapWord* limit = hr->next_top_at_mark_start();
2084
2085 if (limit == bottom) {
2086 // The region was collected underneath our feet.
2087 // We set the finger to bottom to ensure that the bitmap
2088 // iteration that will follow this will not do anything.
2089 // (this is not a condition that holds when we set the region up,
2090 // as the region is not supposed to be empty in the first place)
2091 _finger = bottom;
2092 } else if (limit >= _region_limit) {
2093 assert(limit >= _finger, "peace of mind");
2094 } else {
2095 assert(limit < _region_limit, "only way to get here");
2096 // This can happen under some pretty unusual circumstances. An
2097 // evacuation pause empties the region underneath our feet (NTAMS
2098 // at bottom). We then do some allocation in the region (NTAMS
2099 // stays at bottom), followed by the region being used as a GC
2100 // alloc region (NTAMS will move to top() and the objects
2101 // originally below it will be grayed). All objects now marked in
2102 // the region are explicitly grayed, if below the global finger,
2103 // and we do not need in fact to scan anything else. So, we simply
2104 // set _finger to be limit to ensure that the bitmap iteration
2105 // doesn't do anything.
2106 _finger = limit;
2107 }
2108
2109 _region_limit = limit;
2110 }
2111
2112 void G1CMTask::giveup_current_region() {
2113 assert(_curr_region != NULL, "invariant");
2114 clear_region_fields();
2115 }
2116
2117 void G1CMTask::clear_region_fields() {
2118 // Values for these three fields that indicate that we're not
2119 // holding on to a region.
2120 _curr_region = NULL;
2121 _finger = NULL;
2122 _region_limit = NULL;
2123 }
2124
2125 void G1CMTask::set_cm_oop_closure(G1CMOopClosure* cm_oop_closure) {
2126 if (cm_oop_closure == NULL) {
2127 assert(_cm_oop_closure != NULL, "invariant");
2128 } else {
2129 assert(_cm_oop_closure == NULL, "invariant");
2130 }
2131 _cm_oop_closure = cm_oop_closure;
2132 }
2133
2134 void G1CMTask::reset(G1CMBitMap* next_mark_bitmap) {
2135 guarantee(next_mark_bitmap != NULL, "invariant");
2136 _next_mark_bitmap = next_mark_bitmap;
2137 clear_region_fields();
2138
2139 _calls = 0;
2140 _elapsed_time_ms = 0.0;
2141 _termination_time_ms = 0.0;
2142 _termination_start_time_ms = 0.0;
2143 }
2144
2145 bool G1CMTask::should_exit_termination() {
2146 regular_clock_call();
2147 // This is called when we are in the termination protocol. We should
2148 // quit if, for some reason, this task wants to abort or the global
2149 // stack is not empty (this means that we can get work from it).
2150 return !_cm->mark_stack_empty() || has_aborted();
2151 }
2152
2153 void G1CMTask::reached_limit() {
2154 assert(_words_scanned >= _words_scanned_limit ||
2155 _refs_reached >= _refs_reached_limit ,
2156 "shouldn't have been called otherwise");
2157 regular_clock_call();
2158 }
2159
2160 void G1CMTask::regular_clock_call() {
2161 if (has_aborted()) return;
2162
2163 // First, we need to recalculate the words scanned and refs reached
2164 // limits for the next clock call.
2165 recalculate_limits();
2166
2167 // During the regular clock call we do the following
2168
2169 // (1) If an overflow has been flagged, then we abort.
2170 if (_cm->has_overflown()) {
2171 set_has_aborted();
2172 return;
2173 }
2174
2175 // If we are not concurrent (i.e. we're doing remark) we don't need
2176 // to check anything else. The other steps are only needed during
2177 // the concurrent marking phase.
2178 if (!_concurrent) {
2179 return;
2180 }
2181
2182 // (2) If marking has been aborted for Full GC, then we also abort.
2183 if (_cm->has_aborted()) {
2184 set_has_aborted();
2185 return;
2186 }
2187
2188 double curr_time_ms = os::elapsedVTime() * 1000.0;
2189
2190 // (4) We check whether we should yield. If we have to, then we abort.
2191 if (SuspendibleThreadSet::should_yield()) {
2192 // We should yield. To do this we abort the task. The caller is
2193 // responsible for yielding.
2194 set_has_aborted();
2195 return;
2196 }
2197
2198 // (5) We check whether we've reached our time quota. If we have,
2199 // then we abort.
2200 double elapsed_time_ms = curr_time_ms - _start_time_ms;
2201 if (elapsed_time_ms > _time_target_ms) {
2202 set_has_aborted();
2203 _has_timed_out = true;
2204 return;
2205 }
2206
2207 // (6) Finally, we check whether there are enough completed STAB
2208 // buffers available for processing. If there are, we abort.
2209 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2210 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) {
2211 // we do need to process SATB buffers, we'll abort and restart
2212 // the marking task to do so
2213 set_has_aborted();
2214 return;
2215 }
2216 }
2217
2218 void G1CMTask::recalculate_limits() {
2219 _real_words_scanned_limit = _words_scanned + words_scanned_period;
2220 _words_scanned_limit = _real_words_scanned_limit;
2221
2222 _real_refs_reached_limit = _refs_reached + refs_reached_period;
2223 _refs_reached_limit = _real_refs_reached_limit;
2224 }
2225
2226 void G1CMTask::decrease_limits() {
2227 // This is called when we believe that we're going to do an infrequent
2228 // operation which will increase the per byte scanned cost (i.e. move
2229 // entries to/from the global stack). It basically tries to decrease the
2230 // scanning limit so that the clock is called earlier.
2231
2232 _words_scanned_limit = _real_words_scanned_limit - 3 * words_scanned_period / 4;
2233 _refs_reached_limit = _real_refs_reached_limit - 3 * refs_reached_period / 4;
2234 }
2235
2236 void G1CMTask::move_entries_to_global_stack() {
2237 // Local array where we'll store the entries that will be popped
2238 // from the local queue.
2239 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk];
2240
2241 size_t n = 0;
2242 G1TaskQueueEntry task_entry;
2243 while (n < G1CMMarkStack::EntriesPerChunk && _task_queue->pop_local(task_entry)) {
2244 buffer[n] = task_entry;
2245 ++n;
2246 }
2247 if (n < G1CMMarkStack::EntriesPerChunk) {
2248 buffer[n] = G1TaskQueueEntry();
2249 }
2250
2251 if (n > 0) {
2252 if (!_cm->mark_stack_push(buffer)) {
2253 set_has_aborted();
2254 }
2255 }
2256
2257 // This operation was quite expensive, so decrease the limits.
2258 decrease_limits();
2259 }
2260
2261 bool G1CMTask::get_entries_from_global_stack() {
2262 // Local array where we'll store the entries that will be popped
2263 // from the global stack.
2264 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk];
2265
2266 if (!_cm->mark_stack_pop(buffer)) {
2267 return false;
2268 }
2269
2270 // We did actually pop at least one entry.
2271 for (size_t i = 0; i < G1CMMarkStack::EntriesPerChunk; ++i) {
2272 G1TaskQueueEntry task_entry = buffer[i];
2273 if (task_entry.is_null()) {
2274 break;
2275 }
2276 assert(task_entry.is_array_slice() || oopDesc::is_oop(task_entry.obj()), "Element " PTR_FORMAT " must be an array slice or oop", p2i(task_entry.obj()));
2277 bool success = _task_queue->push(task_entry);
2278 // We only call this when the local queue is empty or under a
2279 // given target limit. So, we do not expect this push to fail.
2280 assert(success, "invariant");
2281 }
2282
2283 // This operation was quite expensive, so decrease the limits
2284 decrease_limits();
2285 return true;
2286 }
2287
2288 void G1CMTask::drain_local_queue(bool partially) {
2289 if (has_aborted()) {
2290 return;
2291 }
2292
2293 // Decide what the target size is, depending whether we're going to
2294 // drain it partially (so that other tasks can steal if they run out
2295 // of things to do) or totally (at the very end).
2296 size_t target_size;
2297 if (partially) {
2298 target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize);
2299 } else {
2300 target_size = 0;
2301 }
2302
2303 if (_task_queue->size() > target_size) {
2304 G1TaskQueueEntry entry;
2305 bool ret = _task_queue->pop_local(entry);
2306 while (ret) {
2307 scan_task_entry(entry);
2308 if (_task_queue->size() <= target_size || has_aborted()) {
2309 ret = false;
2310 } else {
2311 ret = _task_queue->pop_local(entry);
2312 }
2313 }
2314 }
2315 }
2316
2317 void G1CMTask::drain_global_stack(bool partially) {
2318 if (has_aborted()) return;
2319
2320 // We have a policy to drain the local queue before we attempt to
2321 // drain the global stack.
2322 assert(partially || _task_queue->size() == 0, "invariant");
2323
2324 // Decide what the target size is, depending whether we're going to
2325 // drain it partially (so that other tasks can steal if they run out
2326 // of things to do) or totally (at the very end).
2327 // Notice that when draining the global mark stack partially, due to the racyness
2328 // of the mark stack size update we might in fact drop below the target. But,
2329 // this is not a problem.
2330 // In case of total draining, we simply process until the global mark stack is
2331 // totally empty, disregarding the size counter.
2332 if (partially) {
2333 size_t const target_size = _cm->partial_mark_stack_size_target();
2334 while (!has_aborted() && _cm->mark_stack_size() > target_size) {
2335 if (get_entries_from_global_stack()) {
2336 drain_local_queue(partially);
2337 }
2338 }
2339 } else {
2340 while (!has_aborted() && get_entries_from_global_stack()) {
2341 drain_local_queue(partially);
2342 }
2343 }
2344 }
2345
2346 // SATB Queue has several assumptions on whether to call the par or
2347 // non-par versions of the methods. this is why some of the code is
2348 // replicated. We should really get rid of the single-threaded version
2349 // of the code to simplify things.
2350 void G1CMTask::drain_satb_buffers() {
2351 if (has_aborted()) return;
2352
2353 // We set this so that the regular clock knows that we're in the
2354 // middle of draining buffers and doesn't set the abort flag when it
2355 // notices that SATB buffers are available for draining. It'd be
2356 // very counter productive if it did that. :-)
2357 _draining_satb_buffers = true;
2358
2359 G1CMSATBBufferClosure satb_cl(this, _g1h);
2360 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
2361
2362 // This keeps claiming and applying the closure to completed buffers
2363 // until we run out of buffers or we need to abort.
2364 while (!has_aborted() &&
2365 satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) {
2366 regular_clock_call();
2367 }
2368
2369 _draining_satb_buffers = false;
2370
2371 assert(has_aborted() ||
2372 _concurrent ||
2373 satb_mq_set.completed_buffers_num() == 0, "invariant");
2374
2375 // again, this was a potentially expensive operation, decrease the
2376 // limits to get the regular clock call early
2377 decrease_limits();
2378 }
2379
2380 void G1CMTask::print_stats() {
2381 log_debug(gc, stats)("Marking Stats, task = %u, calls = %u",
2382 _worker_id, _calls);
2383 log_debug(gc, stats)(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
2384 _elapsed_time_ms, _termination_time_ms);
2385 log_debug(gc, stats)(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
2386 _step_times_ms.num(), _step_times_ms.avg(),
2387 _step_times_ms.sd());
2388 log_debug(gc, stats)(" max = %1.2lfms, total = %1.2lfms",
2389 _step_times_ms.maximum(), _step_times_ms.sum());
2390 }
2391
2392 bool G1ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, G1TaskQueueEntry& task_entry) {
2393 return _task_queues->steal(worker_id, hash_seed, task_entry);
2394 }
2395
2396 /*****************************************************************************
2397
2398 The do_marking_step(time_target_ms, ...) method is the building
2399 block of the parallel marking framework. It can be called in parallel
2400 with other invocations of do_marking_step() on different tasks
2401 (but only one per task, obviously) and concurrently with the
2402 mutator threads, or during remark, hence it eliminates the need
2403 for two versions of the code. When called during remark, it will
2404 pick up from where the task left off during the concurrent marking
2405 phase. Interestingly, tasks are also claimable during evacuation
2406 pauses too, since do_marking_step() ensures that it aborts before
2407 it needs to yield.
2408
2409 The data structures that it uses to do marking work are the
2410 following:
2411
2412 (1) Marking Bitmap. If there are gray objects that appear only
2413 on the bitmap (this happens either when dealing with an overflow
2414 or when the initial marking phase has simply marked the roots
2415 and didn't push them on the stack), then tasks claim heap
2416 regions whose bitmap they then scan to find gray objects. A
2417 global finger indicates where the end of the last claimed region
2418 is. A local finger indicates how far into the region a task has
2419 scanned. The two fingers are used to determine how to gray an
2420 object (i.e. whether simply marking it is OK, as it will be
2421 visited by a task in the future, or whether it needs to be also
2422 pushed on a stack).
2423
2424 (2) Local Queue. The local queue of the task which is accessed
2425 reasonably efficiently by the task. Other tasks can steal from
2426 it when they run out of work. Throughout the marking phase, a
2427 task attempts to keep its local queue short but not totally
2428 empty, so that entries are available for stealing by other
2429 tasks. Only when there is no more work, a task will totally
2430 drain its local queue.
2431
2432 (3) Global Mark Stack. This handles local queue overflow. During
2433 marking only sets of entries are moved between it and the local
2434 queues, as access to it requires a mutex and more fine-grain
2435 interaction with it which might cause contention. If it
2436 overflows, then the marking phase should restart and iterate
2437 over the bitmap to identify gray objects. Throughout the marking
2438 phase, tasks attempt to keep the global mark stack at a small
2439 length but not totally empty, so that entries are available for
2440 popping by other tasks. Only when there is no more work, tasks
2441 will totally drain the global mark stack.
2442
2443 (4) SATB Buffer Queue. This is where completed SATB buffers are
2444 made available. Buffers are regularly removed from this queue
2445 and scanned for roots, so that the queue doesn't get too
2446 long. During remark, all completed buffers are processed, as
2447 well as the filled in parts of any uncompleted buffers.
2448
2449 The do_marking_step() method tries to abort when the time target
2450 has been reached. There are a few other cases when the
2451 do_marking_step() method also aborts:
2452
2453 (1) When the marking phase has been aborted (after a Full GC).
2454
2455 (2) When a global overflow (on the global stack) has been
2456 triggered. Before the task aborts, it will actually sync up with
2457 the other tasks to ensure that all the marking data structures
2458 (local queues, stacks, fingers etc.) are re-initialized so that
2459 when do_marking_step() completes, the marking phase can
2460 immediately restart.
2461
2462 (3) When enough completed SATB buffers are available. The
2463 do_marking_step() method only tries to drain SATB buffers right
2464 at the beginning. So, if enough buffers are available, the
2465 marking step aborts and the SATB buffers are processed at
2466 the beginning of the next invocation.
2467
2468 (4) To yield. when we have to yield then we abort and yield
2469 right at the end of do_marking_step(). This saves us from a lot
2470 of hassle as, by yielding we might allow a Full GC. If this
2471 happens then objects will be compacted underneath our feet, the
2472 heap might shrink, etc. We save checking for this by just
2473 aborting and doing the yield right at the end.
2474
2475 From the above it follows that the do_marking_step() method should
2476 be called in a loop (or, otherwise, regularly) until it completes.
2477
2478 If a marking step completes without its has_aborted() flag being
2479 true, it means it has completed the current marking phase (and
2480 also all other marking tasks have done so and have all synced up).
2481
2482 A method called regular_clock_call() is invoked "regularly" (in
2483 sub ms intervals) throughout marking. It is this clock method that
2484 checks all the abort conditions which were mentioned above and
2485 decides when the task should abort. A work-based scheme is used to
2486 trigger this clock method: when the number of object words the
2487 marking phase has scanned or the number of references the marking
2488 phase has visited reach a given limit. Additional invocations to
2489 the method clock have been planted in a few other strategic places
2490 too. The initial reason for the clock method was to avoid calling
2491 vtime too regularly, as it is quite expensive. So, once it was in
2492 place, it was natural to piggy-back all the other conditions on it
2493 too and not constantly check them throughout the code.
2494
2495 If do_termination is true then do_marking_step will enter its
2496 termination protocol.
2497
2498 The value of is_serial must be true when do_marking_step is being
2499 called serially (i.e. by the VMThread) and do_marking_step should
2500 skip any synchronization in the termination and overflow code.
2501 Examples include the serial remark code and the serial reference
2502 processing closures.
2503
2504 The value of is_serial must be false when do_marking_step is
2505 being called by any of the worker threads in a work gang.
2506 Examples include the concurrent marking code (CMMarkingTask),
2507 the MT remark code, and the MT reference processing closures.
2508
2509 *****************************************************************************/
2510
2511 void G1CMTask::do_marking_step(double time_target_ms,
2512 bool do_termination,
2513 bool is_serial) {
2514 assert(time_target_ms >= 1.0, "minimum granularity is 1ms");
2515 assert(_concurrent == _cm->concurrent(), "they should be the same");
2516
2517 _start_time_ms = os::elapsedVTime() * 1000.0;
2518
2519 // If do_stealing is true then do_marking_step will attempt to
2520 // steal work from the other G1CMTasks. It only makes sense to
2521 // enable stealing when the termination protocol is enabled
2522 // and do_marking_step() is not being called serially.
2523 bool do_stealing = do_termination && !is_serial;
2524
2525 double diff_prediction_ms = _g1h->g1_policy()->predictor().get_new_prediction(&_marking_step_diffs_ms);
2526 _time_target_ms = time_target_ms - diff_prediction_ms;
2527
2528 // set up the variables that are used in the work-based scheme to
2529 // call the regular clock method
2530 _words_scanned = 0;
2531 _refs_reached = 0;
2532 recalculate_limits();
2533
2534 // clear all flags
2535 clear_has_aborted();
2536 _has_timed_out = false;
2537 _draining_satb_buffers = false;
2538
2539 ++_calls;
2540
2541 // Set up the bitmap and oop closures. Anything that uses them is
2542 // eventually called from this method, so it is OK to allocate these
2543 // statically.
2544 G1CMBitMapClosure bitmap_closure(this, _cm);
2545 G1CMOopClosure cm_oop_closure(_g1h, _cm, this);
2546 set_cm_oop_closure(&cm_oop_closure);
2547
2548 if (_cm->has_overflown()) {
2549 // This can happen if the mark stack overflows during a GC pause
2550 // and this task, after a yield point, restarts. We have to abort
2551 // as we need to get into the overflow protocol which happens
2552 // right at the end of this task.
2553 set_has_aborted();
2554 }
2555
2556 // First drain any available SATB buffers. After this, we will not
2557 // look at SATB buffers before the next invocation of this method.
2558 // If enough completed SATB buffers are queued up, the regular clock
2559 // will abort this task so that it restarts.
2560 drain_satb_buffers();
2561 // ...then partially drain the local queue and the global stack
2562 drain_local_queue(true);
2563 drain_global_stack(true);
2564
2565 do {
2566 if (!has_aborted() && _curr_region != NULL) {
2567 // This means that we're already holding on to a region.
2568 assert(_finger != NULL, "if region is not NULL, then the finger "
2569 "should not be NULL either");
2570
2571 // We might have restarted this task after an evacuation pause
2572 // which might have evacuated the region we're holding on to
2573 // underneath our feet. Let's read its limit again to make sure
2574 // that we do not iterate over a region of the heap that
2575 // contains garbage (update_region_limit() will also move
2576 // _finger to the start of the region if it is found empty).
2577 update_region_limit();
2578 // We will start from _finger not from the start of the region,
2579 // as we might be restarting this task after aborting half-way
2580 // through scanning this region. In this case, _finger points to
2581 // the address where we last found a marked object. If this is a
2582 // fresh region, _finger points to start().
2583 MemRegion mr = MemRegion(_finger, _region_limit);
2584
2585 assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(),
2586 "humongous regions should go around loop once only");
2587
2588 // Some special cases:
2589 // If the memory region is empty, we can just give up the region.
2590 // If the current region is humongous then we only need to check
2591 // the bitmap for the bit associated with the start of the object,
2592 // scan the object if it's live, and give up the region.
2593 // Otherwise, let's iterate over the bitmap of the part of the region
2594 // that is left.
2595 // If the iteration is successful, give up the region.
2596 if (mr.is_empty()) {
2597 giveup_current_region();
2598 regular_clock_call();
2599 } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) {
2600 if (_next_mark_bitmap->is_marked(mr.start())) {
2601 // The object is marked - apply the closure
2602 bitmap_closure.do_addr(mr.start());
2603 }
2604 // Even if this task aborted while scanning the humongous object
2605 // we can (and should) give up the current region.
2606 giveup_current_region();
2607 regular_clock_call();
2608 } else if (_next_mark_bitmap->iterate(&bitmap_closure, mr)) {
2609 giveup_current_region();
2610 regular_clock_call();
2611 } else {
2612 assert(has_aborted(), "currently the only way to do so");
2613 // The only way to abort the bitmap iteration is to return
2614 // false from the do_bit() method. However, inside the
2615 // do_bit() method we move the _finger to point to the
2616 // object currently being looked at. So, if we bail out, we
2617 // have definitely set _finger to something non-null.
2618 assert(_finger != NULL, "invariant");
2619
2620 // Region iteration was actually aborted. So now _finger
2621 // points to the address of the object we last scanned. If we
2622 // leave it there, when we restart this task, we will rescan
2623 // the object. It is easy to avoid this. We move the finger by
2624 // enough to point to the next possible object header.
2625 assert(_finger < _region_limit, "invariant");
2626 HeapWord* const new_finger = _finger + ((oop)_finger)->size();
2627 // Check if bitmap iteration was aborted while scanning the last object
2628 if (new_finger >= _region_limit) {
2629 giveup_current_region();
2630 } else {
2631 move_finger_to(new_finger);
2632 }
2633 }
2634 }
2635 // At this point we have either completed iterating over the
2636 // region we were holding on to, or we have aborted.
2637
2638 // We then partially drain the local queue and the global stack.
2639 // (Do we really need this?)
2640 drain_local_queue(true);
2641 drain_global_stack(true);
2642
2643 // Read the note on the claim_region() method on why it might
2644 // return NULL with potentially more regions available for
2645 // claiming and why we have to check out_of_regions() to determine
2646 // whether we're done or not.
2647 while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) {
2648 // We are going to try to claim a new region. We should have
2649 // given up on the previous one.
2650 // Separated the asserts so that we know which one fires.
2651 assert(_curr_region == NULL, "invariant");
2652 assert(_finger == NULL, "invariant");
2653 assert(_region_limit == NULL, "invariant");
2654 HeapRegion* claimed_region = _cm->claim_region(_worker_id);
2655 if (claimed_region != NULL) {
2656 // Yes, we managed to claim one
2657 setup_for_region(claimed_region);
2658 assert(_curr_region == claimed_region, "invariant");
2659 }
2660 // It is important to call the regular clock here. It might take
2661 // a while to claim a region if, for example, we hit a large
2662 // block of empty regions. So we need to call the regular clock
2663 // method once round the loop to make sure it's called
2664 // frequently enough.
2665 regular_clock_call();
2666 }
2667
2668 if (!has_aborted() && _curr_region == NULL) {
2669 assert(_cm->out_of_regions(),
2670 "at this point we should be out of regions");
2671 }
2672 } while ( _curr_region != NULL && !has_aborted());
2673
2674 if (!has_aborted()) {
2675 // We cannot check whether the global stack is empty, since other
2676 // tasks might be pushing objects to it concurrently.
2677 assert(_cm->out_of_regions(),
2678 "at this point we should be out of regions");
2679 // Try to reduce the number of available SATB buffers so that
2680 // remark has less work to do.
2681 drain_satb_buffers();
2682 }
2683
2684 // Since we've done everything else, we can now totally drain the
2685 // local queue and global stack.
2686 drain_local_queue(false);
2687 drain_global_stack(false);
2688
2689 // Attempt at work stealing from other task's queues.
2690 if (do_stealing && !has_aborted()) {
2691 // We have not aborted. This means that we have finished all that
2692 // we could. Let's try to do some stealing...
2693
2694 // We cannot check whether the global stack is empty, since other
2695 // tasks might be pushing objects to it concurrently.
2696 assert(_cm->out_of_regions() && _task_queue->size() == 0,
2697 "only way to reach here");
2698 while (!has_aborted()) {
2699 G1TaskQueueEntry entry;
2700 if (_cm->try_stealing(_worker_id, &_hash_seed, entry)) {
2701 scan_task_entry(entry);
2702
2703 // And since we're towards the end, let's totally drain the
2704 // local queue and global stack.
2705 drain_local_queue(false);
2706 drain_global_stack(false);
2707 } else {
2708 break;
2709 }
2710 }
2711 }
2712
2713 // We still haven't aborted. Now, let's try to get into the
2714 // termination protocol.
2715 if (do_termination && !has_aborted()) {
2716 // We cannot check whether the global stack is empty, since other
2717 // tasks might be concurrently pushing objects on it.
2718 // Separated the asserts so that we know which one fires.
2719 assert(_cm->out_of_regions(), "only way to reach here");
2720 assert(_task_queue->size() == 0, "only way to reach here");
2721 _termination_start_time_ms = os::elapsedVTime() * 1000.0;
2722
2723 // The G1CMTask class also extends the TerminatorTerminator class,
2724 // hence its should_exit_termination() method will also decide
2725 // whether to exit the termination protocol or not.
2726 bool finished = (is_serial ||
2727 _cm->terminator()->offer_termination(this));
2728 double termination_end_time_ms = os::elapsedVTime() * 1000.0;
2729 _termination_time_ms +=
2730 termination_end_time_ms - _termination_start_time_ms;
2731
2732 if (finished) {
2733 // We're all done.
2734
2735 if (_worker_id == 0) {
2736 // Let's allow task 0 to do this
2737 if (_concurrent) {
2738 assert(_cm->concurrent_marking_in_progress(), "invariant");
2739 // We need to set this to false before the next
2740 // safepoint. This way we ensure that the marking phase
2741 // doesn't observe any more heap expansions.
2742 _cm->clear_concurrent_marking_in_progress();
2743 }
2744 }
2745
2746 // We can now guarantee that the global stack is empty, since
2747 // all other tasks have finished. We separated the guarantees so
2748 // that, if a condition is false, we can immediately find out
2749 // which one.
2750 guarantee(_cm->out_of_regions(), "only way to reach here");
2751 guarantee(_cm->mark_stack_empty(), "only way to reach here");
2752 guarantee(_task_queue->size() == 0, "only way to reach here");
2753 guarantee(!_cm->has_overflown(), "only way to reach here");
2754 } else {
2755 // Apparently there's more work to do. Let's abort this task. It
2756 // will restart it and we can hopefully find more things to do.
2757 set_has_aborted();
2758 }
2759 }
2760
2761 // Mainly for debugging purposes to make sure that a pointer to the
2762 // closure which was statically allocated in this frame doesn't
2763 // escape it by accident.
2764 set_cm_oop_closure(NULL);
2765 double end_time_ms = os::elapsedVTime() * 1000.0;
2766 double elapsed_time_ms = end_time_ms - _start_time_ms;
2767 // Update the step history.
2768 _step_times_ms.add(elapsed_time_ms);
2769
2770 if (has_aborted()) {
2771 // The task was aborted for some reason.
2772 if (_has_timed_out) {
2773 double diff_ms = elapsed_time_ms - _time_target_ms;
2774 // Keep statistics of how well we did with respect to hitting
2775 // our target only if we actually timed out (if we aborted for
2776 // other reasons, then the results might get skewed).
2777 _marking_step_diffs_ms.add(diff_ms);
2778 }
2779
2780 if (_cm->has_overflown()) {
2781 // This is the interesting one. We aborted because a global
2782 // overflow was raised. This means we have to restart the
2783 // marking phase and start iterating over regions. However, in
2784 // order to do this we have to make sure that all tasks stop
2785 // what they are doing and re-initialize in a safe manner. We
2786 // will achieve this with the use of two barrier sync points.
2787
2788 if (!is_serial) {
2789 // We only need to enter the sync barrier if being called
2790 // from a parallel context
2791 _cm->enter_first_sync_barrier(_worker_id);
2792
2793 // When we exit this sync barrier we know that all tasks have
2794 // stopped doing marking work. So, it's now safe to
2795 // re-initialize our data structures. At the end of this method,
2796 // task 0 will clear the global data structures.
2797 }
2798
2799 // We clear the local state of this task...
2800 clear_region_fields();
2801
2802 if (!is_serial) {
2803 // ...and enter the second barrier.
2804 _cm->enter_second_sync_barrier(_worker_id);
2805 }
2806 // At this point, if we're during the concurrent phase of
2807 // marking, everything has been re-initialized and we're
2808 // ready to restart.
2809 }
2810 }
2811 }
2812
2813 G1CMTask::G1CMTask(uint worker_id, G1ConcurrentMark* cm, G1CMTaskQueue* task_queue) :
2814 _objArray_processor(this),
2815 _worker_id(worker_id),
2816 _g1h(G1CollectedHeap::heap()),
2817 _cm(cm),
2818 _next_mark_bitmap(NULL),
2819 _task_queue(task_queue),
2820 _calls(0),
2821 _time_target_ms(0.0),
2822 _start_time_ms(0.0),
2823 _cm_oop_closure(NULL),
2824 _curr_region(NULL),
2825 _finger(NULL),
2826 _region_limit(NULL),
2827 _words_scanned(0),
2828 _words_scanned_limit(0),
2829 _real_words_scanned_limit(0),
2830 _refs_reached(0),
2831 _refs_reached_limit(0),
2832 _real_refs_reached_limit(0),
2833 _hash_seed(17),
2834 _has_aborted(false),
2835 _has_timed_out(false),
2836 _draining_satb_buffers(false),
2837 _step_times_ms(),
2838 _elapsed_time_ms(0.0),
2839 _termination_time_ms(0.0),
2840 _termination_start_time_ms(0.0),
2841 _concurrent(false),
2842 _marking_step_diffs_ms()
2843 {
2844 guarantee(task_queue != NULL, "invariant");
2845
2846 _marking_step_diffs_ms.add(0.5);
2847 }
2848
2849 // These are formatting macros that are used below to ensure
2850 // consistent formatting. The *_H_* versions are used to format the
2851 // header for a particular value and they should be kept consistent
2852 // with the corresponding macro. Also note that most of the macros add
2853 // the necessary white space (as a prefix) which makes them a bit
2854 // easier to compose.
2855
2856 // All the output lines are prefixed with this string to be able to
2857 // identify them easily in a large log file.
2858 #define G1PPRL_LINE_PREFIX "###"
2859
2860 #define G1PPRL_ADDR_BASE_FORMAT " " PTR_FORMAT "-" PTR_FORMAT
2861 #ifdef _LP64
2862 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s"
2863 #else // _LP64
2864 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s"
2865 #endif // _LP64
2866
2867 // For per-region info
2868 #define G1PPRL_TYPE_FORMAT " %-4s"
2869 #define G1PPRL_TYPE_H_FORMAT " %4s"
2870 #define G1PPRL_BYTE_FORMAT " " SIZE_FORMAT_W(9)
2871 #define G1PPRL_BYTE_H_FORMAT " %9s"
2872 #define G1PPRL_DOUBLE_FORMAT " %14.1f"
2873 #define G1PPRL_DOUBLE_H_FORMAT " %14s"
2874
2875 // For summary info
2876 #define G1PPRL_SUM_ADDR_FORMAT(tag) " " tag ":" G1PPRL_ADDR_BASE_FORMAT
2877 #define G1PPRL_SUM_BYTE_FORMAT(tag) " " tag ": " SIZE_FORMAT
2878 #define G1PPRL_SUM_MB_FORMAT(tag) " " tag ": %1.2f MB"
2879 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%"
2880
2881 G1PrintRegionLivenessInfoClosure::G1PrintRegionLivenessInfoClosure(const char* phase_name) :
2882 _total_used_bytes(0), _total_capacity_bytes(0),
2883 _total_prev_live_bytes(0), _total_next_live_bytes(0),
2884 _total_remset_bytes(0), _total_strong_code_roots_bytes(0)
2885 {
2886 G1CollectedHeap* g1h = G1CollectedHeap::heap();
2887 MemRegion g1_reserved = g1h->g1_reserved();
2888 double now = os::elapsedTime();
2889
2890 // Print the header of the output.
2891 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now);
2892 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" HEAP"
2893 G1PPRL_SUM_ADDR_FORMAT("reserved")
2894 G1PPRL_SUM_BYTE_FORMAT("region-size"),
2895 p2i(g1_reserved.start()), p2i(g1_reserved.end()),
2896 HeapRegion::GrainBytes);
2897 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
2898 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2899 G1PPRL_TYPE_H_FORMAT
2900 G1PPRL_ADDR_BASE_H_FORMAT
2901 G1PPRL_BYTE_H_FORMAT
2902 G1PPRL_BYTE_H_FORMAT
2903 G1PPRL_BYTE_H_FORMAT
2904 G1PPRL_DOUBLE_H_FORMAT
2905 G1PPRL_BYTE_H_FORMAT
2906 G1PPRL_BYTE_H_FORMAT,
2907 "type", "address-range",
2908 "used", "prev-live", "next-live", "gc-eff",
2909 "remset", "code-roots");
2910 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2911 G1PPRL_TYPE_H_FORMAT
2912 G1PPRL_ADDR_BASE_H_FORMAT
2913 G1PPRL_BYTE_H_FORMAT
2914 G1PPRL_BYTE_H_FORMAT
2915 G1PPRL_BYTE_H_FORMAT
2916 G1PPRL_DOUBLE_H_FORMAT
2917 G1PPRL_BYTE_H_FORMAT
2918 G1PPRL_BYTE_H_FORMAT,
2919 "", "",
2920 "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)",
2921 "(bytes)", "(bytes)");
2922 }
2923
2924 bool G1PrintRegionLivenessInfoClosure::do_heap_region(HeapRegion* r) {
2925 const char* type = r->get_type_str();
2926 HeapWord* bottom = r->bottom();
2927 HeapWord* end = r->end();
2928 size_t capacity_bytes = r->capacity();
2929 size_t used_bytes = r->used();
2930 size_t prev_live_bytes = r->live_bytes();
2931 size_t next_live_bytes = r->next_live_bytes();
2932 double gc_eff = r->gc_efficiency();
2933 size_t remset_bytes = r->rem_set()->mem_size();
2934 size_t strong_code_roots_bytes = r->rem_set()->strong_code_roots_mem_size();
2935
2936 _total_used_bytes += used_bytes;
2937 _total_capacity_bytes += capacity_bytes;
2938 _total_prev_live_bytes += prev_live_bytes;
2939 _total_next_live_bytes += next_live_bytes;
2940 _total_remset_bytes += remset_bytes;
2941 _total_strong_code_roots_bytes += strong_code_roots_bytes;
2942
2943 // Print a line for this particular region.
2944 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2945 G1PPRL_TYPE_FORMAT
2946 G1PPRL_ADDR_BASE_FORMAT
2947 G1PPRL_BYTE_FORMAT
2948 G1PPRL_BYTE_FORMAT
2949 G1PPRL_BYTE_FORMAT
2950 G1PPRL_DOUBLE_FORMAT
2951 G1PPRL_BYTE_FORMAT
2952 G1PPRL_BYTE_FORMAT,
2953 type, p2i(bottom), p2i(end),
2954 used_bytes, prev_live_bytes, next_live_bytes, gc_eff,
2955 remset_bytes, strong_code_roots_bytes);
2956
2957 return false;
2958 }
2959
2960 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
2961 // add static memory usages to remembered set sizes
2962 _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
2963 // Print the footer of the output.
2964 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
2965 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX
2966 " SUMMARY"
2967 G1PPRL_SUM_MB_FORMAT("capacity")
2968 G1PPRL_SUM_MB_PERC_FORMAT("used")
2969 G1PPRL_SUM_MB_PERC_FORMAT("prev-live")
2970 G1PPRL_SUM_MB_PERC_FORMAT("next-live")
2971 G1PPRL_SUM_MB_FORMAT("remset")
2972 G1PPRL_SUM_MB_FORMAT("code-roots"),
2973 bytes_to_mb(_total_capacity_bytes),
2974 bytes_to_mb(_total_used_bytes),
2975 percent_of(_total_used_bytes, _total_capacity_bytes),
2976 bytes_to_mb(_total_prev_live_bytes),
2977 percent_of(_total_prev_live_bytes, _total_capacity_bytes),
2978 bytes_to_mb(_total_next_live_bytes),
2979 percent_of(_total_next_live_bytes, _total_capacity_bytes),
2980 bytes_to_mb(_total_remset_bytes),
2981 bytes_to_mb(_total_strong_code_roots_bytes));
2982 }