1 /* 2 * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/metadataOnStackMark.hpp" 27 #include "classfile/symbolTable.hpp" 28 #include "code/codeCache.hpp" 29 #include "gc/g1/concurrentMarkThread.inline.hpp" 30 #include "gc/g1/g1CollectedHeap.inline.hpp" 31 #include "gc/g1/g1CollectorState.hpp" 32 #include "gc/g1/g1ConcurrentMark.inline.hpp" 33 #include "gc/g1/g1HeapVerifier.hpp" 34 #include "gc/g1/g1OopClosures.inline.hpp" 35 #include "gc/g1/g1CardLiveData.inline.hpp" 36 #include "gc/g1/g1Policy.hpp" 37 #include "gc/g1/g1StringDedup.hpp" 38 #include "gc/g1/heapRegion.inline.hpp" 39 #include "gc/g1/heapRegionRemSet.hpp" 40 #include "gc/g1/heapRegionSet.inline.hpp" 41 #include "gc/shared/adaptiveSizePolicy.hpp" 42 #include "gc/shared/gcId.hpp" 43 #include "gc/shared/gcTimer.hpp" 44 #include "gc/shared/gcTrace.hpp" 45 #include "gc/shared/gcTraceTime.inline.hpp" 46 #include "gc/shared/genOopClosures.inline.hpp" 47 #include "gc/shared/referencePolicy.hpp" 48 #include "gc/shared/strongRootsScope.hpp" 49 #include "gc/shared/suspendibleThreadSet.hpp" 50 #include "gc/shared/taskqueue.inline.hpp" 51 #include "gc/shared/vmGCOperations.hpp" 52 #include "gc/shared/weakProcessor.hpp" 53 #include "logging/log.hpp" 54 #include "memory/allocation.hpp" 55 #include "memory/resourceArea.hpp" 56 #include "oops/access.inline.hpp" 57 #include "oops/oop.inline.hpp" 58 #include "runtime/atomic.hpp" 59 #include "runtime/handles.inline.hpp" 60 #include "runtime/java.hpp" 61 #include "runtime/prefetch.inline.hpp" 62 #include "services/memTracker.hpp" 63 #include "utilities/align.hpp" 64 #include "utilities/growableArray.hpp" 65 66 bool G1CMBitMapClosure::do_addr(HeapWord* const addr) { 67 assert(addr < _cm->finger(), "invariant"); 68 assert(addr >= _task->finger(), "invariant"); 69 70 // We move that task's local finger along. 71 _task->move_finger_to(addr); 72 73 _task->scan_task_entry(G1TaskQueueEntry::from_oop(oop(addr))); 74 // we only partially drain the local queue and global stack 75 _task->drain_local_queue(true); 76 _task->drain_global_stack(true); 77 78 // if the has_aborted flag has been raised, we need to bail out of 79 // the iteration 80 return !_task->has_aborted(); 81 } 82 83 G1CMMarkStack::G1CMMarkStack() : 84 _max_chunk_capacity(0), 85 _base(NULL), 86 _chunk_capacity(0) { 87 set_empty(); 88 } 89 90 bool G1CMMarkStack::resize(size_t new_capacity) { 91 assert(is_empty(), "Only resize when stack is empty."); 92 assert(new_capacity <= _max_chunk_capacity, 93 "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity); 94 95 TaskQueueEntryChunk* new_base = MmapArrayAllocator<TaskQueueEntryChunk>::allocate_or_null(new_capacity, mtGC); 96 97 if (new_base == NULL) { 98 log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk)); 99 return false; 100 } 101 // Release old mapping. 102 if (_base != NULL) { 103 MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity); 104 } 105 106 _base = new_base; 107 _chunk_capacity = new_capacity; 108 set_empty(); 109 110 return true; 111 } 112 113 size_t G1CMMarkStack::capacity_alignment() { 114 return (size_t)lcm(os::vm_allocation_granularity(), sizeof(TaskQueueEntryChunk)) / sizeof(G1TaskQueueEntry); 115 } 116 117 bool G1CMMarkStack::initialize(size_t initial_capacity, size_t max_capacity) { 118 guarantee(_max_chunk_capacity == 0, "G1CMMarkStack already initialized."); 119 120 size_t const TaskEntryChunkSizeInVoidStar = sizeof(TaskQueueEntryChunk) / sizeof(G1TaskQueueEntry); 121 122 _max_chunk_capacity = align_up(max_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar; 123 size_t initial_chunk_capacity = align_up(initial_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar; 124 125 guarantee(initial_chunk_capacity <= _max_chunk_capacity, 126 "Maximum chunk capacity " SIZE_FORMAT " smaller than initial capacity " SIZE_FORMAT, 127 _max_chunk_capacity, 128 initial_chunk_capacity); 129 130 log_debug(gc)("Initialize mark stack with " SIZE_FORMAT " chunks, maximum " SIZE_FORMAT, 131 initial_chunk_capacity, _max_chunk_capacity); 132 133 return resize(initial_chunk_capacity); 134 } 135 136 void G1CMMarkStack::expand() { 137 if (_chunk_capacity == _max_chunk_capacity) { 138 log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _chunk_capacity); 139 return; 140 } 141 size_t old_capacity = _chunk_capacity; 142 // Double capacity if possible 143 size_t new_capacity = MIN2(old_capacity * 2, _max_chunk_capacity); 144 145 if (resize(new_capacity)) { 146 log_debug(gc)("Expanded mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks", 147 old_capacity, new_capacity); 148 } else { 149 log_warning(gc)("Failed to expand mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks", 150 old_capacity, new_capacity); 151 } 152 } 153 154 G1CMMarkStack::~G1CMMarkStack() { 155 if (_base != NULL) { 156 MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity); 157 } 158 } 159 160 void G1CMMarkStack::add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem) { 161 elem->next = *list; 162 *list = elem; 163 } 164 165 void G1CMMarkStack::add_chunk_to_chunk_list(TaskQueueEntryChunk* elem) { 166 MutexLockerEx x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag); 167 add_chunk_to_list(&_chunk_list, elem); 168 _chunks_in_chunk_list++; 169 } 170 171 void G1CMMarkStack::add_chunk_to_free_list(TaskQueueEntryChunk* elem) { 172 MutexLockerEx x(MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag); 173 add_chunk_to_list(&_free_list, elem); 174 } 175 176 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_list(TaskQueueEntryChunk* volatile* list) { 177 TaskQueueEntryChunk* result = *list; 178 if (result != NULL) { 179 *list = (*list)->next; 180 } 181 return result; 182 } 183 184 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_chunk_list() { 185 MutexLockerEx x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag); 186 TaskQueueEntryChunk* result = remove_chunk_from_list(&_chunk_list); 187 if (result != NULL) { 188 _chunks_in_chunk_list--; 189 } 190 return result; 191 } 192 193 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_free_list() { 194 MutexLockerEx x(MarkStackFreeList_lock, Mutex::_no_safepoint_check_flag); 195 return remove_chunk_from_list(&_free_list); 196 } 197 198 G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::allocate_new_chunk() { 199 // This dirty read of _hwm is okay because we only ever increase the _hwm in parallel code. 200 // Further this limits _hwm to a value of _chunk_capacity + #threads, avoiding 201 // wraparound of _hwm. 202 if (_hwm >= _chunk_capacity) { 203 return NULL; 204 } 205 206 size_t cur_idx = Atomic::add(1u, &_hwm) - 1; 207 if (cur_idx >= _chunk_capacity) { 208 return NULL; 209 } 210 211 TaskQueueEntryChunk* result = ::new (&_base[cur_idx]) TaskQueueEntryChunk; 212 result->next = NULL; 213 return result; 214 } 215 216 bool G1CMMarkStack::par_push_chunk(G1TaskQueueEntry* ptr_arr) { 217 // Get a new chunk. 218 TaskQueueEntryChunk* new_chunk = remove_chunk_from_free_list(); 219 220 if (new_chunk == NULL) { 221 // Did not get a chunk from the free list. Allocate from backing memory. 222 new_chunk = allocate_new_chunk(); 223 224 if (new_chunk == NULL) { 225 return false; 226 } 227 } 228 229 Copy::conjoint_memory_atomic(ptr_arr, new_chunk->data, EntriesPerChunk * sizeof(G1TaskQueueEntry)); 230 231 add_chunk_to_chunk_list(new_chunk); 232 233 return true; 234 } 235 236 bool G1CMMarkStack::par_pop_chunk(G1TaskQueueEntry* ptr_arr) { 237 TaskQueueEntryChunk* cur = remove_chunk_from_chunk_list(); 238 239 if (cur == NULL) { 240 return false; 241 } 242 243 Copy::conjoint_memory_atomic(cur->data, ptr_arr, EntriesPerChunk * sizeof(G1TaskQueueEntry)); 244 245 add_chunk_to_free_list(cur); 246 return true; 247 } 248 249 void G1CMMarkStack::set_empty() { 250 _chunks_in_chunk_list = 0; 251 _hwm = 0; 252 _chunk_list = NULL; 253 _free_list = NULL; 254 } 255 256 G1CMRootRegions::G1CMRootRegions() : 257 _cm(NULL), _scan_in_progress(false), 258 _should_abort(false), _claimed_survivor_index(0) { } 259 260 void G1CMRootRegions::init(const G1SurvivorRegions* survivors, G1ConcurrentMark* cm) { 261 _survivors = survivors; 262 _cm = cm; 263 } 264 265 void G1CMRootRegions::prepare_for_scan() { 266 assert(!scan_in_progress(), "pre-condition"); 267 268 // Currently, only survivors can be root regions. 269 _claimed_survivor_index = 0; 270 _scan_in_progress = _survivors->regions()->is_nonempty(); 271 _should_abort = false; 272 } 273 274 HeapRegion* G1CMRootRegions::claim_next() { 275 if (_should_abort) { 276 // If someone has set the should_abort flag, we return NULL to 277 // force the caller to bail out of their loop. 278 return NULL; 279 } 280 281 // Currently, only survivors can be root regions. 282 const GrowableArray<HeapRegion*>* survivor_regions = _survivors->regions(); 283 284 int claimed_index = Atomic::add(1, &_claimed_survivor_index) - 1; 285 if (claimed_index < survivor_regions->length()) { 286 return survivor_regions->at(claimed_index); 287 } 288 return NULL; 289 } 290 291 uint G1CMRootRegions::num_root_regions() const { 292 return (uint)_survivors->regions()->length(); 293 } 294 295 void G1CMRootRegions::notify_scan_done() { 296 MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); 297 _scan_in_progress = false; 298 RootRegionScan_lock->notify_all(); 299 } 300 301 void G1CMRootRegions::cancel_scan() { 302 notify_scan_done(); 303 } 304 305 void G1CMRootRegions::scan_finished() { 306 assert(scan_in_progress(), "pre-condition"); 307 308 // Currently, only survivors can be root regions. 309 if (!_should_abort) { 310 assert(_claimed_survivor_index >= 0, "otherwise comparison is invalid: %d", _claimed_survivor_index); 311 assert((uint)_claimed_survivor_index >= _survivors->length(), 312 "we should have claimed all survivors, claimed index = %u, length = %u", 313 (uint)_claimed_survivor_index, _survivors->length()); 314 } 315 316 notify_scan_done(); 317 } 318 319 bool G1CMRootRegions::wait_until_scan_finished() { 320 if (!scan_in_progress()) return false; 321 322 { 323 MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); 324 while (scan_in_progress()) { 325 RootRegionScan_lock->wait(Mutex::_no_safepoint_check_flag); 326 } 327 } 328 return true; 329 } 330 331 // Returns the maximum number of workers to be used in a concurrent 332 // phase based on the number of GC workers being used in a STW 333 // phase. 334 static uint scale_concurrent_worker_threads(uint num_gc_workers) { 335 return MAX2((num_gc_workers + 2) / 4, 1U); 336 } 337 338 G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h, 339 G1RegionToSpaceMapper* prev_bitmap_storage, 340 G1RegionToSpaceMapper* next_bitmap_storage) : 341 // _cm_thread set inside the constructor 342 _g1h(g1h), 343 _completed_initialization(false), 344 345 _cleanup_list("Concurrent Mark Cleanup List"), 346 _mark_bitmap_1(), 347 _mark_bitmap_2(), 348 _prev_mark_bitmap(&_mark_bitmap_1), 349 _next_mark_bitmap(&_mark_bitmap_2), 350 351 _heap_start(_g1h->reserved_region().start()), 352 _heap_end(_g1h->reserved_region().end()), 353 354 _root_regions(), 355 356 _global_mark_stack(), 357 358 // _finger set in set_non_marking_state 359 360 _max_num_tasks(ParallelGCThreads), 361 // _num_active_tasks set in set_non_marking_state() 362 // _tasks set inside the constructor 363 364 _task_queues(new G1CMTaskQueueSet((int) _max_num_tasks)), 365 _terminator(ParallelTaskTerminator((int) _max_num_tasks, _task_queues)), 366 367 _first_overflow_barrier_sync(), 368 _second_overflow_barrier_sync(), 369 370 _has_overflown(false), 371 _concurrent(false), 372 _has_aborted(false), 373 _restart_for_overflow(false), 374 _concurrent_marking_in_progress(false), 375 _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()), 376 _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) G1OldTracer()), 377 378 // _verbose_level set below 379 380 _init_times(), 381 _remark_times(), 382 _remark_mark_times(), 383 _remark_weak_ref_times(), 384 _cleanup_times(), 385 _total_counting_time(0.0), 386 _total_rs_scrub_time(0.0), 387 388 _accum_task_vtime(NULL), 389 390 _concurrent_workers(NULL), 391 _num_concurrent_workers(0), 392 _max_concurrent_workers(0) 393 { 394 _mark_bitmap_1.initialize(g1h->reserved_region(), prev_bitmap_storage); 395 _mark_bitmap_2.initialize(g1h->reserved_region(), next_bitmap_storage); 396 397 // Create & start ConcurrentMark thread. 398 _cm_thread = new ConcurrentMarkThread(this); 399 if (_cm_thread->osthread() == NULL) { 400 vm_shutdown_during_initialization("Could not create ConcurrentMarkThread"); 401 } 402 403 assert(CGC_lock != NULL, "CGC_lock must be initialized"); 404 405 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); 406 satb_qs.set_buffer_size(G1SATBBufferSize); 407 408 _root_regions.init(_g1h->survivor(), this); 409 410 if (FLAG_IS_DEFAULT(ConcGCThreads) || ConcGCThreads == 0) { 411 // Calculate the number of concurrent worker threads by scaling 412 // the number of parallel GC threads. 413 uint marking_thread_num = scale_concurrent_worker_threads(ParallelGCThreads); 414 FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num); 415 } 416 417 assert(ConcGCThreads > 0, "ConcGCThreads have been set."); 418 if (ConcGCThreads > ParallelGCThreads) { 419 log_warning(gc)("More ConcGCThreads (%u) than ParallelGCThreads (%u).", 420 ConcGCThreads, ParallelGCThreads); 421 return; 422 } 423 424 log_debug(gc)("ConcGCThreads: %u", ConcGCThreads); 425 log_debug(gc)("ParallelGCThreads: %u", ParallelGCThreads); 426 427 _num_concurrent_workers = ConcGCThreads; 428 _max_concurrent_workers = _num_concurrent_workers; 429 430 _concurrent_workers = new WorkGang("G1 Conc", _max_concurrent_workers, false, true); 431 _concurrent_workers->initialize_workers(); 432 433 if (FLAG_IS_DEFAULT(MarkStackSize)) { 434 size_t mark_stack_size = 435 MIN2(MarkStackSizeMax, 436 MAX2(MarkStackSize, (size_t) (_max_concurrent_workers * TASKQUEUE_SIZE))); 437 // Verify that the calculated value for MarkStackSize is in range. 438 // It would be nice to use the private utility routine from Arguments. 439 if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) { 440 log_warning(gc)("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): " 441 "must be between 1 and " SIZE_FORMAT, 442 mark_stack_size, MarkStackSizeMax); 443 return; 444 } 445 FLAG_SET_ERGO(size_t, MarkStackSize, mark_stack_size); 446 } else { 447 // Verify MarkStackSize is in range. 448 if (FLAG_IS_CMDLINE(MarkStackSize)) { 449 if (FLAG_IS_DEFAULT(MarkStackSizeMax)) { 450 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) { 451 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT "): " 452 "must be between 1 and " SIZE_FORMAT, 453 MarkStackSize, MarkStackSizeMax); 454 return; 455 } 456 } else if (FLAG_IS_CMDLINE(MarkStackSizeMax)) { 457 if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) { 458 log_warning(gc)("Invalid value specified for MarkStackSize (" SIZE_FORMAT ")" 459 " or for MarkStackSizeMax (" SIZE_FORMAT ")", 460 MarkStackSize, MarkStackSizeMax); 461 return; 462 } 463 } 464 } 465 } 466 467 if (!_global_mark_stack.initialize(MarkStackSize, MarkStackSizeMax)) { 468 vm_exit_during_initialization("Failed to allocate initial concurrent mark overflow mark stack."); 469 } 470 471 _tasks = NEW_C_HEAP_ARRAY(G1CMTask*, _max_num_tasks, mtGC); 472 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_num_tasks, mtGC); 473 474 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail 475 _num_active_tasks = _max_num_tasks; 476 477 for (uint i = 0; i < _max_num_tasks; ++i) { 478 G1CMTaskQueue* task_queue = new G1CMTaskQueue(); 479 task_queue->initialize(); 480 _task_queues->register_queue(i, task_queue); 481 482 _tasks[i] = new G1CMTask(i, this, task_queue); 483 484 _accum_task_vtime[i] = 0.0; 485 } 486 487 set_non_marking_state(); 488 _completed_initialization = true; 489 } 490 491 void G1ConcurrentMark::reset() { 492 // Starting values for these two. This should be called in a STW 493 // phase. 494 MemRegion reserved = _g1h->g1_reserved(); 495 _heap_start = reserved.start(); 496 _heap_end = reserved.end(); 497 498 // Separated the asserts so that we know which one fires. 499 assert(_heap_start != NULL, "heap bounds should look ok"); 500 assert(_heap_end != NULL, "heap bounds should look ok"); 501 assert(_heap_start < _heap_end, "heap bounds should look ok"); 502 503 // Reset all the marking data structures and any necessary flags 504 reset_marking_state(); 505 506 // We reset all of them, since different phases will use 507 // different number of active threads. So, it's easiest to have all 508 // of them ready. 509 for (uint i = 0; i < _max_num_tasks; ++i) { 510 _tasks[i]->reset(_next_mark_bitmap); 511 } 512 513 // we need this to make sure that the flag is on during the evac 514 // pause with initial mark piggy-backed 515 set_concurrent_marking_in_progress(); 516 } 517 518 519 void G1ConcurrentMark::reset_marking_state() { 520 _global_mark_stack.set_empty(); 521 522 // Expand the marking stack, if we have to and if we can. 523 if (has_overflown()) { 524 _global_mark_stack.expand(); 525 } 526 527 clear_has_overflown(); 528 _finger = _heap_start; 529 530 for (uint i = 0; i < _max_num_tasks; ++i) { 531 G1CMTaskQueue* queue = _task_queues->queue(i); 532 queue->set_empty(); 533 } 534 } 535 536 void G1ConcurrentMark::set_concurrency(uint active_tasks) { 537 assert(active_tasks <= _max_num_tasks, "we should not have more"); 538 539 _num_active_tasks = active_tasks; 540 // Need to update the three data structures below according to the 541 // number of active threads for this phase. 542 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues); 543 _first_overflow_barrier_sync.set_n_workers((int) active_tasks); 544 _second_overflow_barrier_sync.set_n_workers((int) active_tasks); 545 } 546 547 void G1ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurrent) { 548 set_concurrency(active_tasks); 549 550 _concurrent = concurrent; 551 // We propagate this to all tasks, not just the active ones. 552 for (uint i = 0; i < _max_num_tasks; ++i) { 553 _tasks[i]->set_concurrent(concurrent); 554 } 555 556 if (concurrent) { 557 set_concurrent_marking_in_progress(); 558 } else { 559 // We currently assume that the concurrent flag has been set to 560 // false before we start remark. At this point we should also be 561 // in a STW phase. 562 assert(!concurrent_marking_in_progress(), "invariant"); 563 assert(out_of_regions(), 564 "only way to get here: _finger: " PTR_FORMAT ", _heap_end: " PTR_FORMAT, 565 p2i(_finger), p2i(_heap_end)); 566 } 567 } 568 569 void G1ConcurrentMark::set_non_marking_state() { 570 // We set the global marking state to some default values when we're 571 // not doing marking. 572 reset_marking_state(); 573 _num_active_tasks = 0; 574 clear_concurrent_marking_in_progress(); 575 } 576 577 G1ConcurrentMark::~G1ConcurrentMark() { 578 // The G1ConcurrentMark instance is never freed. 579 ShouldNotReachHere(); 580 } 581 582 class G1ClearBitMapTask : public AbstractGangTask { 583 public: 584 static size_t chunk_size() { return M; } 585 586 private: 587 // Heap region closure used for clearing the given mark bitmap. 588 class G1ClearBitmapHRClosure : public HeapRegionClosure { 589 private: 590 G1CMBitMap* _bitmap; 591 G1ConcurrentMark* _cm; 592 public: 593 G1ClearBitmapHRClosure(G1CMBitMap* bitmap, G1ConcurrentMark* cm) : HeapRegionClosure(), _cm(cm), _bitmap(bitmap) { 594 } 595 596 virtual bool do_heap_region(HeapRegion* r) { 597 size_t const chunk_size_in_words = G1ClearBitMapTask::chunk_size() / HeapWordSize; 598 599 HeapWord* cur = r->bottom(); 600 HeapWord* const end = r->end(); 601 602 while (cur < end) { 603 MemRegion mr(cur, MIN2(cur + chunk_size_in_words, end)); 604 _bitmap->clear_range(mr); 605 606 cur += chunk_size_in_words; 607 608 // Abort iteration if after yielding the marking has been aborted. 609 if (_cm != NULL && _cm->do_yield_check() && _cm->has_aborted()) { 610 return true; 611 } 612 // Repeat the asserts from before the start of the closure. We will do them 613 // as asserts here to minimize their overhead on the product. However, we 614 // will have them as guarantees at the beginning / end of the bitmap 615 // clearing to get some checking in the product. 616 assert(_cm == NULL || _cm->cm_thread()->during_cycle(), "invariant"); 617 assert(_cm == NULL || !G1CollectedHeap::heap()->collector_state()->mark_in_progress(), "invariant"); 618 } 619 assert(cur == end, "Must have completed iteration over the bitmap for region %u.", r->hrm_index()); 620 621 return false; 622 } 623 }; 624 625 G1ClearBitmapHRClosure _cl; 626 HeapRegionClaimer _hr_claimer; 627 bool _suspendible; // If the task is suspendible, workers must join the STS. 628 629 public: 630 G1ClearBitMapTask(G1CMBitMap* bitmap, G1ConcurrentMark* cm, uint n_workers, bool suspendible) : 631 AbstractGangTask("G1 Clear Bitmap"), 632 _cl(bitmap, suspendible ? cm : NULL), 633 _hr_claimer(n_workers), 634 _suspendible(suspendible) 635 { } 636 637 void work(uint worker_id) { 638 SuspendibleThreadSetJoiner sts_join(_suspendible); 639 G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_cl, &_hr_claimer, worker_id); 640 } 641 642 bool is_complete() { 643 return _cl.is_complete(); 644 } 645 }; 646 647 void G1ConcurrentMark::clear_bitmap(G1CMBitMap* bitmap, WorkGang* workers, bool may_yield) { 648 assert(may_yield || SafepointSynchronize::is_at_safepoint(), "Non-yielding bitmap clear only allowed at safepoint."); 649 650 size_t const num_bytes_to_clear = (HeapRegion::GrainBytes * _g1h->num_regions()) / G1CMBitMap::heap_map_factor(); 651 size_t const num_chunks = align_up(num_bytes_to_clear, G1ClearBitMapTask::chunk_size()) / G1ClearBitMapTask::chunk_size(); 652 653 uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers()); 654 655 G1ClearBitMapTask cl(bitmap, this, num_workers, may_yield); 656 657 log_debug(gc, ergo)("Running %s with %u workers for " SIZE_FORMAT " work units.", cl.name(), num_workers, num_chunks); 658 workers->run_task(&cl, num_workers); 659 guarantee(!may_yield || cl.is_complete(), "Must have completed iteration when not yielding."); 660 } 661 662 void G1ConcurrentMark::cleanup_for_next_mark() { 663 // Make sure that the concurrent mark thread looks to still be in 664 // the current cycle. 665 guarantee(cm_thread()->during_cycle(), "invariant"); 666 667 // We are finishing up the current cycle by clearing the next 668 // marking bitmap and getting it ready for the next cycle. During 669 // this time no other cycle can start. So, let's make sure that this 670 // is the case. 671 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant"); 672 673 clear_bitmap(_next_mark_bitmap, _concurrent_workers, true); 674 675 // Clear the live count data. If the marking has been aborted, the abort() 676 // call already did that. 677 if (!has_aborted()) { 678 clear_live_data(_concurrent_workers); 679 DEBUG_ONLY(verify_live_data_clear()); 680 } 681 682 // Repeat the asserts from above. 683 guarantee(cm_thread()->during_cycle(), "invariant"); 684 guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant"); 685 } 686 687 void G1ConcurrentMark::clear_prev_bitmap(WorkGang* workers) { 688 assert(SafepointSynchronize::is_at_safepoint(), "Should only clear the entire prev bitmap at a safepoint."); 689 clear_bitmap(_prev_mark_bitmap, workers, false); 690 } 691 692 class CheckBitmapClearHRClosure : public HeapRegionClosure { 693 G1CMBitMap* _bitmap; 694 bool _error; 695 public: 696 CheckBitmapClearHRClosure(G1CMBitMap* bitmap) : _bitmap(bitmap) { 697 } 698 699 virtual bool do_heap_region(HeapRegion* r) { 700 // This closure can be called concurrently to the mutator, so we must make sure 701 // that the result of the getNextMarkedWordAddress() call is compared to the 702 // value passed to it as limit to detect any found bits. 703 // end never changes in G1. 704 HeapWord* end = r->end(); 705 return _bitmap->get_next_marked_addr(r->bottom(), end) != end; 706 } 707 }; 708 709 bool G1ConcurrentMark::next_mark_bitmap_is_clear() { 710 CheckBitmapClearHRClosure cl(_next_mark_bitmap); 711 _g1h->heap_region_iterate(&cl); 712 return cl.is_complete(); 713 } 714 715 class NoteStartOfMarkHRClosure: public HeapRegionClosure { 716 public: 717 bool do_heap_region(HeapRegion* r) { 718 r->note_start_of_marking(); 719 return false; 720 } 721 }; 722 723 void G1ConcurrentMark::checkpoint_roots_initial_pre() { 724 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 725 726 _has_aborted = false; 727 728 // Initialize marking structures. This has to be done in a STW phase. 729 reset(); 730 731 // For each region note start of marking. 732 NoteStartOfMarkHRClosure startcl; 733 g1h->heap_region_iterate(&startcl); 734 } 735 736 737 void G1ConcurrentMark::checkpoint_roots_initial_post() { 738 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 739 740 // Start Concurrent Marking weak-reference discovery. 741 ReferenceProcessor* rp = g1h->ref_processor_cm(); 742 // enable ("weak") refs discovery 743 rp->enable_discovery(); 744 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle 745 746 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 747 // This is the start of the marking cycle, we're expected all 748 // threads to have SATB queues with active set to false. 749 satb_mq_set.set_active_all_threads(true, /* new active value */ 750 false /* expected_active */); 751 752 _root_regions.prepare_for_scan(); 753 754 // update_g1_committed() will be called at the end of an evac pause 755 // when marking is on. So, it's also called at the end of the 756 // initial-mark pause to update the heap end, if the heap expands 757 // during it. No need to call it here. 758 } 759 760 /* 761 * Notice that in the next two methods, we actually leave the STS 762 * during the barrier sync and join it immediately afterwards. If we 763 * do not do this, the following deadlock can occur: one thread could 764 * be in the barrier sync code, waiting for the other thread to also 765 * sync up, whereas another one could be trying to yield, while also 766 * waiting for the other threads to sync up too. 767 * 768 * Note, however, that this code is also used during remark and in 769 * this case we should not attempt to leave / enter the STS, otherwise 770 * we'll either hit an assert (debug / fastdebug) or deadlock 771 * (product). So we should only leave / enter the STS if we are 772 * operating concurrently. 773 * 774 * Because the thread that does the sync barrier has left the STS, it 775 * is possible to be suspended for a Full GC or an evacuation pause 776 * could occur. This is actually safe, since the entering the sync 777 * barrier is one of the last things do_marking_step() does, and it 778 * doesn't manipulate any data structures afterwards. 779 */ 780 781 void G1ConcurrentMark::enter_first_sync_barrier(uint worker_id) { 782 bool barrier_aborted; 783 { 784 SuspendibleThreadSetLeaver sts_leave(concurrent()); 785 barrier_aborted = !_first_overflow_barrier_sync.enter(); 786 } 787 788 // at this point everyone should have synced up and not be doing any 789 // more work 790 791 if (barrier_aborted) { 792 // If the barrier aborted we ignore the overflow condition and 793 // just abort the whole marking phase as quickly as possible. 794 return; 795 } 796 797 // If we're executing the concurrent phase of marking, reset the marking 798 // state; otherwise the marking state is reset after reference processing, 799 // during the remark pause. 800 // If we reset here as a result of an overflow during the remark we will 801 // see assertion failures from any subsequent set_concurrency_and_phase() 802 // calls. 803 if (concurrent()) { 804 // let the task associated with with worker 0 do this 805 if (worker_id == 0) { 806 // task 0 is responsible for clearing the global data structures 807 // We should be here because of an overflow. During STW we should 808 // not clear the overflow flag since we rely on it being true when 809 // we exit this method to abort the pause and restart concurrent 810 // marking. 811 reset_marking_state(); 812 813 log_info(gc, marking)("Concurrent Mark reset for overflow"); 814 } 815 } 816 817 // after this, each task should reset its own data structures then 818 // then go into the second barrier 819 } 820 821 void G1ConcurrentMark::enter_second_sync_barrier(uint worker_id) { 822 SuspendibleThreadSetLeaver sts_leave(concurrent()); 823 _second_overflow_barrier_sync.enter(); 824 825 // at this point everything should be re-initialized and ready to go 826 } 827 828 class G1CMConcurrentMarkingTask: public AbstractGangTask { 829 private: 830 G1ConcurrentMark* _cm; 831 ConcurrentMarkThread* _cmt; 832 833 public: 834 void work(uint worker_id) { 835 assert(Thread::current()->is_ConcurrentGC_thread(), "Not a concurrent GC thread"); 836 ResourceMark rm; 837 838 double start_vtime = os::elapsedVTime(); 839 840 { 841 SuspendibleThreadSetJoiner sts_join; 842 843 assert(worker_id < _cm->active_tasks(), "invariant"); 844 845 G1CMTask* task = _cm->task(worker_id); 846 task->record_start_time(); 847 if (!_cm->has_aborted()) { 848 do { 849 task->do_marking_step(G1ConcMarkStepDurationMillis, 850 true /* do_termination */, 851 false /* is_serial*/); 852 853 _cm->do_yield_check(); 854 } while (!_cm->has_aborted() && task->has_aborted()); 855 } 856 task->record_end_time(); 857 guarantee(!task->has_aborted() || _cm->has_aborted(), "invariant"); 858 } 859 860 double end_vtime = os::elapsedVTime(); 861 _cm->update_accum_task_vtime(worker_id, end_vtime - start_vtime); 862 } 863 864 G1CMConcurrentMarkingTask(G1ConcurrentMark* cm, 865 ConcurrentMarkThread* cmt) : 866 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { } 867 868 ~G1CMConcurrentMarkingTask() { } 869 }; 870 871 uint G1ConcurrentMark::calc_active_marking_workers() { 872 uint result = 0; 873 if (!UseDynamicNumberOfGCThreads || 874 (!FLAG_IS_DEFAULT(ConcGCThreads) && 875 !ForceDynamicNumberOfGCThreads)) { 876 result = _max_concurrent_workers; 877 } else { 878 result = 879 AdaptiveSizePolicy::calc_default_active_workers(_max_concurrent_workers, 880 1, /* Minimum workers */ 881 _num_concurrent_workers, 882 Threads::number_of_non_daemon_threads()); 883 // Don't scale the result down by scale_concurrent_workers() because 884 // that scaling has already gone into "_max_concurrent_workers". 885 } 886 assert(result > 0 && result <= _max_concurrent_workers, 887 "Calculated number of marking workers must be larger than zero and at most the maximum %u, but is %u", 888 _max_concurrent_workers, result); 889 return result; 890 } 891 892 void G1ConcurrentMark::scan_root_region(HeapRegion* hr) { 893 // Currently, only survivors can be root regions. 894 assert(hr->next_top_at_mark_start() == hr->bottom(), "invariant"); 895 G1RootRegionScanClosure cl(_g1h, this); 896 897 const uintx interval = PrefetchScanIntervalInBytes; 898 HeapWord* curr = hr->bottom(); 899 const HeapWord* end = hr->top(); 900 while (curr < end) { 901 Prefetch::read(curr, interval); 902 oop obj = oop(curr); 903 int size = obj->oop_iterate_size(&cl); 904 assert(size == obj->size(), "sanity"); 905 curr += size; 906 } 907 } 908 909 class G1CMRootRegionScanTask : public AbstractGangTask { 910 private: 911 G1ConcurrentMark* _cm; 912 913 public: 914 G1CMRootRegionScanTask(G1ConcurrentMark* cm) : 915 AbstractGangTask("G1 Root Region Scan"), _cm(cm) { } 916 917 void work(uint worker_id) { 918 assert(Thread::current()->is_ConcurrentGC_thread(), 919 "this should only be done by a conc GC thread"); 920 921 G1CMRootRegions* root_regions = _cm->root_regions(); 922 HeapRegion* hr = root_regions->claim_next(); 923 while (hr != NULL) { 924 _cm->scan_root_region(hr); 925 hr = root_regions->claim_next(); 926 } 927 } 928 }; 929 930 void G1ConcurrentMark::scan_root_regions() { 931 // scan_in_progress() will have been set to true only if there was 932 // at least one root region to scan. So, if it's false, we 933 // should not attempt to do any further work. 934 if (root_regions()->scan_in_progress()) { 935 assert(!has_aborted(), "Aborting before root region scanning is finished not supported."); 936 937 _num_concurrent_workers = MIN2(calc_active_marking_workers(), 938 // We distribute work on a per-region basis, so starting 939 // more threads than that is useless. 940 root_regions()->num_root_regions()); 941 assert(_num_concurrent_workers <= _max_concurrent_workers, 942 "Maximum number of marking threads exceeded"); 943 944 G1CMRootRegionScanTask task(this); 945 log_debug(gc, ergo)("Running %s using %u workers for %u work units.", 946 task.name(), _num_concurrent_workers, root_regions()->num_root_regions()); 947 _concurrent_workers->run_task(&task, _num_concurrent_workers); 948 949 // It's possible that has_aborted() is true here without actually 950 // aborting the survivor scan earlier. This is OK as it's 951 // mainly used for sanity checking. 952 root_regions()->scan_finished(); 953 } 954 } 955 956 void G1ConcurrentMark::concurrent_cycle_start() { 957 _gc_timer_cm->register_gc_start(); 958 959 _gc_tracer_cm->report_gc_start(GCCause::_no_gc /* first parameter is not used */, _gc_timer_cm->gc_start()); 960 961 _g1h->trace_heap_before_gc(_gc_tracer_cm); 962 } 963 964 void G1ConcurrentMark::concurrent_cycle_end() { 965 _g1h->trace_heap_after_gc(_gc_tracer_cm); 966 967 if (has_aborted()) { 968 _gc_tracer_cm->report_concurrent_mode_failure(); 969 } 970 971 _gc_timer_cm->register_gc_end(); 972 973 _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions()); 974 } 975 976 void G1ConcurrentMark::mark_from_roots() { 977 // we might be tempted to assert that: 978 // assert(asynch == !SafepointSynchronize::is_at_safepoint(), 979 // "inconsistent argument?"); 980 // However that wouldn't be right, because it's possible that 981 // a safepoint is indeed in progress as a younger generation 982 // stop-the-world GC happens even as we mark in this generation. 983 984 _restart_for_overflow = false; 985 986 _num_concurrent_workers = calc_active_marking_workers(); 987 988 uint active_workers = MAX2(1U, _num_concurrent_workers); 989 990 // Setting active workers is not guaranteed since fewer 991 // worker threads may currently exist and more may not be 992 // available. 993 active_workers = _concurrent_workers->update_active_workers(active_workers); 994 log_info(gc, task)("Using %u workers of %u for marking", active_workers, _concurrent_workers->total_workers()); 995 996 // Parallel task terminator is set in "set_concurrency_and_phase()" 997 set_concurrency_and_phase(active_workers, true /* concurrent */); 998 999 G1CMConcurrentMarkingTask marking_task(this, cm_thread()); 1000 _concurrent_workers->run_task(&marking_task); 1001 print_stats(); 1002 } 1003 1004 void G1ConcurrentMark::checkpoint_roots_final(bool clear_all_soft_refs) { 1005 // world is stopped at this checkpoint 1006 assert(SafepointSynchronize::is_at_safepoint(), 1007 "world should be stopped"); 1008 1009 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 1010 1011 // If a full collection has happened, we shouldn't do this. 1012 if (has_aborted()) { 1013 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused 1014 return; 1015 } 1016 1017 if (VerifyDuringGC) { 1018 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (before)"); 1019 } 1020 g1h->verifier()->check_bitmaps("Remark Start"); 1021 1022 G1Policy* g1p = g1h->g1_policy(); 1023 g1p->record_concurrent_mark_remark_start(); 1024 1025 double start = os::elapsedTime(); 1026 1027 checkpoint_roots_final_work(); 1028 1029 double mark_work_end = os::elapsedTime(); 1030 1031 weak_refs_work(clear_all_soft_refs); 1032 1033 if (has_overflown()) { 1034 // We overflowed. Restart concurrent marking. 1035 _restart_for_overflow = true; 1036 1037 // Verify the heap w.r.t. the previous marking bitmap. 1038 if (VerifyDuringGC) { 1039 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (overflow)"); 1040 } 1041 1042 // Clear the marking state because we will be restarting 1043 // marking due to overflowing the global mark stack. 1044 reset_marking_state(); 1045 } else { 1046 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 1047 // We're done with marking. 1048 // This is the end of the marking cycle, we're expected all 1049 // threads to have SATB queues with active set to true. 1050 satb_mq_set.set_active_all_threads(false, /* new active value */ 1051 true /* expected_active */); 1052 1053 if (VerifyDuringGC) { 1054 g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UseNextMarking, "During GC (after)"); 1055 } 1056 g1h->verifier()->check_bitmaps("Remark End"); 1057 assert(!restart_for_overflow(), "sanity"); 1058 // Completely reset the marking state since marking completed 1059 set_non_marking_state(); 1060 } 1061 1062 // Statistics 1063 double now = os::elapsedTime(); 1064 _remark_mark_times.add((mark_work_end - start) * 1000.0); 1065 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0); 1066 _remark_times.add((now - start) * 1000.0); 1067 1068 g1p->record_concurrent_mark_remark_end(); 1069 1070 G1CMIsAliveClosure is_alive(g1h); 1071 _gc_tracer_cm->report_object_count_after_gc(&is_alive); 1072 } 1073 1074 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure { 1075 G1CollectedHeap* _g1; 1076 size_t _freed_bytes; 1077 FreeRegionList* _local_cleanup_list; 1078 uint _old_regions_removed; 1079 uint _humongous_regions_removed; 1080 HRRSCleanupTask* _hrrs_cleanup_task; 1081 1082 public: 1083 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, 1084 FreeRegionList* local_cleanup_list, 1085 HRRSCleanupTask* hrrs_cleanup_task) : 1086 _g1(g1), 1087 _freed_bytes(0), 1088 _local_cleanup_list(local_cleanup_list), 1089 _old_regions_removed(0), 1090 _humongous_regions_removed(0), 1091 _hrrs_cleanup_task(hrrs_cleanup_task) { } 1092 1093 size_t freed_bytes() { return _freed_bytes; } 1094 const uint old_regions_removed() { return _old_regions_removed; } 1095 const uint humongous_regions_removed() { return _humongous_regions_removed; } 1096 1097 bool do_heap_region(HeapRegion *hr) { 1098 _g1->reset_gc_time_stamps(hr); 1099 hr->note_end_of_marking(); 1100 1101 if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young() && !hr->is_archive()) { 1102 _freed_bytes += hr->used(); 1103 hr->set_containing_set(NULL); 1104 if (hr->is_humongous()) { 1105 _humongous_regions_removed++; 1106 _g1->free_humongous_region(hr, _local_cleanup_list, true /* skip_remset */); 1107 } else { 1108 _old_regions_removed++; 1109 _g1->free_region(hr, _local_cleanup_list, true /* skip_remset */); 1110 } 1111 } else { 1112 hr->rem_set()->do_cleanup_work(_hrrs_cleanup_task); 1113 } 1114 1115 return false; 1116 } 1117 }; 1118 1119 class G1ParNoteEndTask: public AbstractGangTask { 1120 friend class G1NoteEndOfConcMarkClosure; 1121 1122 protected: 1123 G1CollectedHeap* _g1h; 1124 FreeRegionList* _cleanup_list; 1125 HeapRegionClaimer _hrclaimer; 1126 1127 public: 1128 G1ParNoteEndTask(G1CollectedHeap* g1h, FreeRegionList* cleanup_list, uint n_workers) : 1129 AbstractGangTask("G1 note end"), _g1h(g1h), _cleanup_list(cleanup_list), _hrclaimer(n_workers) { 1130 } 1131 1132 void work(uint worker_id) { 1133 FreeRegionList local_cleanup_list("Local Cleanup List"); 1134 HRRSCleanupTask hrrs_cleanup_task; 1135 G1NoteEndOfConcMarkClosure g1_note_end(_g1h, &local_cleanup_list, 1136 &hrrs_cleanup_task); 1137 _g1h->heap_region_par_iterate_from_worker_offset(&g1_note_end, &_hrclaimer, worker_id); 1138 assert(g1_note_end.is_complete(), "Shouldn't have yielded!"); 1139 1140 // Now update the lists 1141 _g1h->remove_from_old_sets(g1_note_end.old_regions_removed(), g1_note_end.humongous_regions_removed()); 1142 { 1143 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); 1144 _g1h->decrement_summary_bytes(g1_note_end.freed_bytes()); 1145 1146 // If we iterate over the global cleanup list at the end of 1147 // cleanup to do this printing we will not guarantee to only 1148 // generate output for the newly-reclaimed regions (the list 1149 // might not be empty at the beginning of cleanup; we might 1150 // still be working on its previous contents). So we do the 1151 // printing here, before we append the new regions to the global 1152 // cleanup list. 1153 1154 G1HRPrinter* hr_printer = _g1h->hr_printer(); 1155 if (hr_printer->is_active()) { 1156 FreeRegionListIterator iter(&local_cleanup_list); 1157 while (iter.more_available()) { 1158 HeapRegion* hr = iter.get_next(); 1159 hr_printer->cleanup(hr); 1160 } 1161 } 1162 1163 _cleanup_list->add_ordered(&local_cleanup_list); 1164 assert(local_cleanup_list.is_empty(), "post-condition"); 1165 1166 HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task); 1167 } 1168 } 1169 }; 1170 1171 void G1ConcurrentMark::cleanup() { 1172 // world is stopped at this checkpoint 1173 assert(SafepointSynchronize::is_at_safepoint(), 1174 "world should be stopped"); 1175 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 1176 1177 // If a full collection has happened, we shouldn't do this. 1178 if (has_aborted()) { 1179 g1h->collector_state()->set_mark_in_progress(false); // So bitmap clearing isn't confused 1180 return; 1181 } 1182 1183 g1h->verifier()->verify_region_sets_optional(); 1184 1185 if (VerifyDuringGC) { 1186 g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (before)"); 1187 } 1188 g1h->verifier()->check_bitmaps("Cleanup Start"); 1189 1190 G1Policy* g1p = g1h->g1_policy(); 1191 g1p->record_concurrent_mark_cleanup_start(); 1192 1193 double start = os::elapsedTime(); 1194 1195 HeapRegionRemSet::reset_for_cleanup_tasks(); 1196 1197 { 1198 GCTraceTime(Debug, gc)("Finalize Live Data"); 1199 finalize_live_data(); 1200 } 1201 1202 if (VerifyDuringGC) { 1203 GCTraceTime(Debug, gc)("Verify Live Data"); 1204 verify_live_data(); 1205 } 1206 1207 g1h->collector_state()->set_mark_in_progress(false); 1208 1209 double count_end = os::elapsedTime(); 1210 double this_final_counting_time = (count_end - start); 1211 _total_counting_time += this_final_counting_time; 1212 1213 if (log_is_enabled(Trace, gc, liveness)) { 1214 G1PrintRegionLivenessInfoClosure cl("Post-Marking"); 1215 _g1h->heap_region_iterate(&cl); 1216 } 1217 1218 // Install newly created mark bitMap as "prev". 1219 swap_mark_bitmaps(); 1220 1221 g1h->reset_gc_time_stamp(); 1222 1223 uint n_workers = _g1h->workers()->active_workers(); 1224 1225 // Note end of marking in all heap regions. 1226 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers); 1227 g1h->workers()->run_task(&g1_par_note_end_task); 1228 g1h->check_gc_time_stamps(); 1229 1230 if (!cleanup_list_is_empty()) { 1231 // The cleanup list is not empty, so we'll have to process it 1232 // concurrently. Notify anyone else that might be wanting free 1233 // regions that there will be more free regions coming soon. 1234 g1h->set_free_regions_coming(); 1235 } 1236 1237 // call below, since it affects the metric by which we sort the heap 1238 // regions. 1239 if (G1ScrubRemSets) { 1240 double rs_scrub_start = os::elapsedTime(); 1241 g1h->scrub_rem_set(); 1242 _total_rs_scrub_time += (os::elapsedTime() - rs_scrub_start); 1243 } 1244 1245 // this will also free any regions totally full of garbage objects, 1246 // and sort the regions. 1247 g1h->g1_policy()->record_concurrent_mark_cleanup_end(); 1248 1249 // Statistics. 1250 double end = os::elapsedTime(); 1251 _cleanup_times.add((end - start) * 1000.0); 1252 1253 // Clean up will have freed any regions completely full of garbage. 1254 // Update the soft reference policy with the new heap occupancy. 1255 Universe::update_heap_info_at_gc(); 1256 1257 if (VerifyDuringGC) { 1258 g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (after)"); 1259 } 1260 1261 g1h->verifier()->check_bitmaps("Cleanup End"); 1262 1263 g1h->verifier()->verify_region_sets_optional(); 1264 1265 // We need to make this be a "collection" so any collection pause that 1266 // races with it goes around and waits for completeCleanup to finish. 1267 g1h->increment_total_collections(); 1268 1269 // Clean out dead classes and update Metaspace sizes. 1270 if (ClassUnloadingWithConcurrentMark) { 1271 ClassLoaderDataGraph::purge(); 1272 } 1273 MetaspaceGC::compute_new_size(); 1274 1275 // We reclaimed old regions so we should calculate the sizes to make 1276 // sure we update the old gen/space data. 1277 g1h->g1mm()->update_sizes(); 1278 } 1279 1280 void G1ConcurrentMark::complete_cleanup() { 1281 if (has_aborted()) return; 1282 1283 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 1284 1285 _cleanup_list.verify_optional(); 1286 FreeRegionList tmp_free_list("Tmp Free List"); 1287 1288 log_develop_trace(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : " 1289 "cleanup list has %u entries", 1290 _cleanup_list.length()); 1291 1292 // No one else should be accessing the _cleanup_list at this point, 1293 // so it is not necessary to take any locks 1294 while (!_cleanup_list.is_empty()) { 1295 HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */); 1296 assert(hr != NULL, "Got NULL from a non-empty list"); 1297 hr->par_clear(); 1298 tmp_free_list.add_ordered(hr); 1299 1300 // Instead of adding one region at a time to the secondary_free_list, 1301 // we accumulate them in the local list and move them a few at a 1302 // time. This also cuts down on the number of notify_all() calls 1303 // we do during this process. We'll also append the local list when 1304 // _cleanup_list is empty (which means we just removed the last 1305 // region from the _cleanup_list). 1306 if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) || 1307 _cleanup_list.is_empty()) { 1308 log_develop_trace(gc, freelist)("G1ConcRegionFreeing [complete cleanup] : " 1309 "appending %u entries to the secondary_free_list, " 1310 "cleanup list still has %u entries", 1311 tmp_free_list.length(), 1312 _cleanup_list.length()); 1313 1314 { 1315 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); 1316 g1h->secondary_free_list_add(&tmp_free_list); 1317 SecondaryFreeList_lock->notify_all(); 1318 } 1319 #ifndef PRODUCT 1320 if (G1StressConcRegionFreeing) { 1321 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) { 1322 os::sleep(Thread::current(), (jlong) 1, false); 1323 } 1324 } 1325 #endif 1326 } 1327 } 1328 assert(tmp_free_list.is_empty(), "post-condition"); 1329 } 1330 1331 // Supporting Object and Oop closures for reference discovery 1332 // and processing in during marking 1333 1334 bool G1CMIsAliveClosure::do_object_b(oop obj) { 1335 HeapWord* addr = (HeapWord*)obj; 1336 return addr != NULL && 1337 (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj)); 1338 } 1339 1340 // 'Keep Alive' oop closure used by both serial parallel reference processing. 1341 // Uses the G1CMTask associated with a worker thread (for serial reference 1342 // processing the G1CMTask for worker 0 is used) to preserve (mark) and 1343 // trace referent objects. 1344 // 1345 // Using the G1CMTask and embedded local queues avoids having the worker 1346 // threads operating on the global mark stack. This reduces the risk 1347 // of overflowing the stack - which we would rather avoid at this late 1348 // state. Also using the tasks' local queues removes the potential 1349 // of the workers interfering with each other that could occur if 1350 // operating on the global stack. 1351 1352 class G1CMKeepAliveAndDrainClosure: public OopClosure { 1353 G1ConcurrentMark* _cm; 1354 G1CMTask* _task; 1355 int _ref_counter_limit; 1356 int _ref_counter; 1357 bool _is_serial; 1358 public: 1359 G1CMKeepAliveAndDrainClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) : 1360 _cm(cm), _task(task), _is_serial(is_serial), 1361 _ref_counter_limit(G1RefProcDrainInterval) { 1362 assert(_ref_counter_limit > 0, "sanity"); 1363 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code"); 1364 _ref_counter = _ref_counter_limit; 1365 } 1366 1367 virtual void do_oop(narrowOop* p) { do_oop_work(p); } 1368 virtual void do_oop( oop* p) { do_oop_work(p); } 1369 1370 template <class T> void do_oop_work(T* p) { 1371 if (!_cm->has_overflown()) { 1372 oop obj = RawAccess<>::oop_load(p); 1373 _task->deal_with_reference(obj); 1374 _ref_counter--; 1375 1376 if (_ref_counter == 0) { 1377 // We have dealt with _ref_counter_limit references, pushing them 1378 // and objects reachable from them on to the local stack (and 1379 // possibly the global stack). Call G1CMTask::do_marking_step() to 1380 // process these entries. 1381 // 1382 // We call G1CMTask::do_marking_step() in a loop, which we'll exit if 1383 // there's nothing more to do (i.e. we're done with the entries that 1384 // were pushed as a result of the G1CMTask::deal_with_reference() calls 1385 // above) or we overflow. 1386 // 1387 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted() 1388 // flag while there may still be some work to do. (See the comment at 1389 // the beginning of G1CMTask::do_marking_step() for those conditions - 1390 // one of which is reaching the specified time target.) It is only 1391 // when G1CMTask::do_marking_step() returns without setting the 1392 // has_aborted() flag that the marking step has completed. 1393 do { 1394 double mark_step_duration_ms = G1ConcMarkStepDurationMillis; 1395 _task->do_marking_step(mark_step_duration_ms, 1396 false /* do_termination */, 1397 _is_serial); 1398 } while (_task->has_aborted() && !_cm->has_overflown()); 1399 _ref_counter = _ref_counter_limit; 1400 } 1401 } 1402 } 1403 }; 1404 1405 // 'Drain' oop closure used by both serial and parallel reference processing. 1406 // Uses the G1CMTask associated with a given worker thread (for serial 1407 // reference processing the G1CMtask for worker 0 is used). Calls the 1408 // do_marking_step routine, with an unbelievably large timeout value, 1409 // to drain the marking data structures of the remaining entries 1410 // added by the 'keep alive' oop closure above. 1411 1412 class G1CMDrainMarkingStackClosure: public VoidClosure { 1413 G1ConcurrentMark* _cm; 1414 G1CMTask* _task; 1415 bool _is_serial; 1416 public: 1417 G1CMDrainMarkingStackClosure(G1ConcurrentMark* cm, G1CMTask* task, bool is_serial) : 1418 _cm(cm), _task(task), _is_serial(is_serial) { 1419 assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code"); 1420 } 1421 1422 void do_void() { 1423 do { 1424 // We call G1CMTask::do_marking_step() to completely drain the local 1425 // and global marking stacks of entries pushed by the 'keep alive' 1426 // oop closure (an instance of G1CMKeepAliveAndDrainClosure above). 1427 // 1428 // G1CMTask::do_marking_step() is called in a loop, which we'll exit 1429 // if there's nothing more to do (i.e. we've completely drained the 1430 // entries that were pushed as a a result of applying the 'keep alive' 1431 // closure to the entries on the discovered ref lists) or we overflow 1432 // the global marking stack. 1433 // 1434 // Note: G1CMTask::do_marking_step() can set the G1CMTask::has_aborted() 1435 // flag while there may still be some work to do. (See the comment at 1436 // the beginning of G1CMTask::do_marking_step() for those conditions - 1437 // one of which is reaching the specified time target.) It is only 1438 // when G1CMTask::do_marking_step() returns without setting the 1439 // has_aborted() flag that the marking step has completed. 1440 1441 _task->do_marking_step(1000000000.0 /* something very large */, 1442 true /* do_termination */, 1443 _is_serial); 1444 } while (_task->has_aborted() && !_cm->has_overflown()); 1445 } 1446 }; 1447 1448 // Implementation of AbstractRefProcTaskExecutor for parallel 1449 // reference processing at the end of G1 concurrent marking 1450 1451 class G1CMRefProcTaskExecutor: public AbstractRefProcTaskExecutor { 1452 private: 1453 G1CollectedHeap* _g1h; 1454 G1ConcurrentMark* _cm; 1455 WorkGang* _workers; 1456 uint _active_workers; 1457 1458 public: 1459 G1CMRefProcTaskExecutor(G1CollectedHeap* g1h, 1460 G1ConcurrentMark* cm, 1461 WorkGang* workers, 1462 uint n_workers) : 1463 _g1h(g1h), _cm(cm), 1464 _workers(workers), _active_workers(n_workers) { } 1465 1466 // Executes the given task using concurrent marking worker threads. 1467 virtual void execute(ProcessTask& task); 1468 virtual void execute(EnqueueTask& task); 1469 }; 1470 1471 class G1CMRefProcTaskProxy: public AbstractGangTask { 1472 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; 1473 ProcessTask& _proc_task; 1474 G1CollectedHeap* _g1h; 1475 G1ConcurrentMark* _cm; 1476 1477 public: 1478 G1CMRefProcTaskProxy(ProcessTask& proc_task, 1479 G1CollectedHeap* g1h, 1480 G1ConcurrentMark* cm) : 1481 AbstractGangTask("Process reference objects in parallel"), 1482 _proc_task(proc_task), _g1h(g1h), _cm(cm) { 1483 ReferenceProcessor* rp = _g1h->ref_processor_cm(); 1484 assert(rp->processing_is_mt(), "shouldn't be here otherwise"); 1485 } 1486 1487 virtual void work(uint worker_id) { 1488 ResourceMark rm; 1489 HandleMark hm; 1490 G1CMTask* task = _cm->task(worker_id); 1491 G1CMIsAliveClosure g1_is_alive(_g1h); 1492 G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */); 1493 G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */); 1494 1495 _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain); 1496 } 1497 }; 1498 1499 void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) { 1500 assert(_workers != NULL, "Need parallel worker threads."); 1501 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT"); 1502 1503 G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm); 1504 1505 // We need to reset the concurrency level before each 1506 // proxy task execution, so that the termination protocol 1507 // and overflow handling in G1CMTask::do_marking_step() knows 1508 // how many workers to wait for. 1509 _cm->set_concurrency(_active_workers); 1510 _workers->run_task(&proc_task_proxy); 1511 } 1512 1513 class G1CMRefEnqueueTaskProxy: public AbstractGangTask { 1514 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; 1515 EnqueueTask& _enq_task; 1516 1517 public: 1518 G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) : 1519 AbstractGangTask("Enqueue reference objects in parallel"), 1520 _enq_task(enq_task) { } 1521 1522 virtual void work(uint worker_id) { 1523 _enq_task.work(worker_id); 1524 } 1525 }; 1526 1527 void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) { 1528 assert(_workers != NULL, "Need parallel worker threads."); 1529 assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT"); 1530 1531 G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task); 1532 1533 // Not strictly necessary but... 1534 // 1535 // We need to reset the concurrency level before each 1536 // proxy task execution, so that the termination protocol 1537 // and overflow handling in G1CMTask::do_marking_step() knows 1538 // how many workers to wait for. 1539 _cm->set_concurrency(_active_workers); 1540 _workers->run_task(&enq_task_proxy); 1541 } 1542 1543 void G1ConcurrentMark::weak_refs_work(bool clear_all_soft_refs) { 1544 if (has_overflown()) { 1545 // Skip processing the discovered references if we have 1546 // overflown the global marking stack. Reference objects 1547 // only get discovered once so it is OK to not 1548 // de-populate the discovered reference lists. We could have, 1549 // but the only benefit would be that, when marking restarts, 1550 // less reference objects are discovered. 1551 return; 1552 } 1553 1554 ResourceMark rm; 1555 HandleMark hm; 1556 1557 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 1558 1559 // Is alive closure. 1560 G1CMIsAliveClosure g1_is_alive(g1h); 1561 1562 // Inner scope to exclude the cleaning of the string and symbol 1563 // tables from the displayed time. 1564 { 1565 GCTraceTime(Debug, gc, phases) trace("Reference Processing", _gc_timer_cm); 1566 1567 ReferenceProcessor* rp = g1h->ref_processor_cm(); 1568 1569 // See the comment in G1CollectedHeap::ref_processing_init() 1570 // about how reference processing currently works in G1. 1571 1572 // Set the soft reference policy 1573 rp->setup_policy(clear_all_soft_refs); 1574 assert(_global_mark_stack.is_empty(), "mark stack should be empty"); 1575 1576 // Instances of the 'Keep Alive' and 'Complete GC' closures used 1577 // in serial reference processing. Note these closures are also 1578 // used for serially processing (by the the current thread) the 1579 // JNI references during parallel reference processing. 1580 // 1581 // These closures do not need to synchronize with the worker 1582 // threads involved in parallel reference processing as these 1583 // instances are executed serially by the current thread (e.g. 1584 // reference processing is not multi-threaded and is thus 1585 // performed by the current thread instead of a gang worker). 1586 // 1587 // The gang tasks involved in parallel reference processing create 1588 // their own instances of these closures, which do their own 1589 // synchronization among themselves. 1590 G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */); 1591 G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */); 1592 1593 // We need at least one active thread. If reference processing 1594 // is not multi-threaded we use the current (VMThread) thread, 1595 // otherwise we use the work gang from the G1CollectedHeap and 1596 // we utilize all the worker threads we can. 1597 bool processing_is_mt = rp->processing_is_mt(); 1598 uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U); 1599 active_workers = MAX2(MIN2(active_workers, _max_num_tasks), 1U); 1600 1601 // Parallel processing task executor. 1602 G1CMRefProcTaskExecutor par_task_executor(g1h, this, 1603 g1h->workers(), active_workers); 1604 AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL); 1605 1606 // Set the concurrency level. The phase was already set prior to 1607 // executing the remark task. 1608 set_concurrency(active_workers); 1609 1610 // Set the degree of MT processing here. If the discovery was done MT, 1611 // the number of threads involved during discovery could differ from 1612 // the number of active workers. This is OK as long as the discovered 1613 // Reference lists are balanced (see balance_all_queues() and balance_queues()). 1614 rp->set_active_mt_degree(active_workers); 1615 1616 ReferenceProcessorPhaseTimes pt(_gc_timer_cm, rp->num_q()); 1617 1618 // Process the weak references. 1619 const ReferenceProcessorStats& stats = 1620 rp->process_discovered_references(&g1_is_alive, 1621 &g1_keep_alive, 1622 &g1_drain_mark_stack, 1623 executor, 1624 &pt); 1625 _gc_tracer_cm->report_gc_reference_stats(stats); 1626 pt.print_all_references(); 1627 1628 // The do_oop work routines of the keep_alive and drain_marking_stack 1629 // oop closures will set the has_overflown flag if we overflow the 1630 // global marking stack. 1631 1632 assert(has_overflown() || _global_mark_stack.is_empty(), 1633 "Mark stack should be empty (unless it has overflown)"); 1634 1635 assert(rp->num_q() == active_workers, "why not"); 1636 1637 rp->enqueue_discovered_references(executor, &pt); 1638 1639 rp->verify_no_references_recorded(); 1640 1641 pt.print_enqueue_phase(); 1642 1643 assert(!rp->discovery_enabled(), "Post condition"); 1644 } 1645 1646 assert(has_overflown() || _global_mark_stack.is_empty(), 1647 "Mark stack should be empty (unless it has overflown)"); 1648 1649 { 1650 GCTraceTime(Debug, gc, phases) debug("Weak Processing", _gc_timer_cm); 1651 WeakProcessor::weak_oops_do(&g1_is_alive, &do_nothing_cl); 1652 } 1653 1654 if (has_overflown()) { 1655 // We can not trust g1_is_alive if the marking stack overflowed 1656 return; 1657 } 1658 1659 assert(_global_mark_stack.is_empty(), "Marking should have completed"); 1660 1661 // Unload Klasses, String, Symbols, Code Cache, etc. 1662 if (ClassUnloadingWithConcurrentMark) { 1663 GCTraceTime(Debug, gc, phases) debug("Class Unloading", _gc_timer_cm); 1664 bool purged_classes = SystemDictionary::do_unloading(&g1_is_alive, _gc_timer_cm, false /* Defer cleaning */); 1665 g1h->complete_cleaning(&g1_is_alive, purged_classes); 1666 } else { 1667 GCTraceTime(Debug, gc, phases) debug("Cleanup", _gc_timer_cm); 1668 // No need to clean string table and symbol table as they are treated as strong roots when 1669 // class unloading is disabled. 1670 g1h->partial_cleaning(&g1_is_alive, false, false, G1StringDedup::is_enabled()); 1671 1672 } 1673 } 1674 1675 void G1ConcurrentMark::swap_mark_bitmaps() { 1676 G1CMBitMap* temp = _prev_mark_bitmap; 1677 _prev_mark_bitmap = _next_mark_bitmap; 1678 _next_mark_bitmap = temp; 1679 } 1680 1681 // Closure for marking entries in SATB buffers. 1682 class G1CMSATBBufferClosure : public SATBBufferClosure { 1683 private: 1684 G1CMTask* _task; 1685 G1CollectedHeap* _g1h; 1686 1687 // This is very similar to G1CMTask::deal_with_reference, but with 1688 // more relaxed requirements for the argument, so this must be more 1689 // circumspect about treating the argument as an object. 1690 void do_entry(void* entry) const { 1691 _task->increment_refs_reached(); 1692 oop const obj = static_cast<oop>(entry); 1693 _task->make_reference_grey(obj); 1694 } 1695 1696 public: 1697 G1CMSATBBufferClosure(G1CMTask* task, G1CollectedHeap* g1h) 1698 : _task(task), _g1h(g1h) { } 1699 1700 virtual void do_buffer(void** buffer, size_t size) { 1701 for (size_t i = 0; i < size; ++i) { 1702 do_entry(buffer[i]); 1703 } 1704 } 1705 }; 1706 1707 class G1RemarkThreadsClosure : public ThreadClosure { 1708 G1CMSATBBufferClosure _cm_satb_cl; 1709 G1CMOopClosure _cm_cl; 1710 MarkingCodeBlobClosure _code_cl; 1711 int _thread_parity; 1712 1713 public: 1714 G1RemarkThreadsClosure(G1CollectedHeap* g1h, G1CMTask* task) : 1715 _cm_satb_cl(task, g1h), 1716 _cm_cl(g1h, g1h->concurrent_mark(), task), 1717 _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations), 1718 _thread_parity(Threads::thread_claim_parity()) {} 1719 1720 void do_thread(Thread* thread) { 1721 if (thread->is_Java_thread()) { 1722 if (thread->claim_oops_do(true, _thread_parity)) { 1723 JavaThread* jt = (JavaThread*)thread; 1724 1725 // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking 1726 // however the liveness of oops reachable from nmethods have very complex lifecycles: 1727 // * Alive if on the stack of an executing method 1728 // * Weakly reachable otherwise 1729 // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be 1730 // live by the SATB invariant but other oops recorded in nmethods may behave differently. 1731 jt->nmethods_do(&_code_cl); 1732 1733 jt->satb_mark_queue().apply_closure_and_empty(&_cm_satb_cl); 1734 } 1735 } else if (thread->is_VM_thread()) { 1736 if (thread->claim_oops_do(true, _thread_parity)) { 1737 JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(&_cm_satb_cl); 1738 } 1739 } 1740 } 1741 }; 1742 1743 class G1CMRemarkTask: public AbstractGangTask { 1744 private: 1745 G1ConcurrentMark* _cm; 1746 public: 1747 void work(uint worker_id) { 1748 G1CMTask* task = _cm->task(worker_id); 1749 task->record_start_time(); 1750 { 1751 ResourceMark rm; 1752 HandleMark hm; 1753 1754 G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task); 1755 Threads::threads_do(&threads_f); 1756 } 1757 1758 do { 1759 task->do_marking_step(1000000000.0 /* something very large */, 1760 true /* do_termination */, 1761 false /* is_serial */); 1762 } while (task->has_aborted() && !_cm->has_overflown()); 1763 // If we overflow, then we do not want to restart. We instead 1764 // want to abort remark and do concurrent marking again. 1765 task->record_end_time(); 1766 } 1767 1768 G1CMRemarkTask(G1ConcurrentMark* cm, uint active_workers) : 1769 AbstractGangTask("Par Remark"), _cm(cm) { 1770 _cm->terminator()->reset_for_reuse(active_workers); 1771 } 1772 }; 1773 1774 void G1ConcurrentMark::checkpoint_roots_final_work() { 1775 ResourceMark rm; 1776 HandleMark hm; 1777 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 1778 1779 GCTraceTime(Debug, gc, phases) trace("Finalize Marking", _gc_timer_cm); 1780 1781 g1h->ensure_parsability(false); 1782 1783 // this is remark, so we'll use up all active threads 1784 uint active_workers = g1h->workers()->active_workers(); 1785 set_concurrency_and_phase(active_workers, false /* concurrent */); 1786 // Leave _parallel_marking_threads at it's 1787 // value originally calculated in the G1ConcurrentMark 1788 // constructor and pass values of the active workers 1789 // through the gang in the task. 1790 1791 { 1792 StrongRootsScope srs(active_workers); 1793 1794 G1CMRemarkTask remarkTask(this, active_workers); 1795 // We will start all available threads, even if we decide that the 1796 // active_workers will be fewer. The extra ones will just bail out 1797 // immediately. 1798 g1h->workers()->run_task(&remarkTask); 1799 } 1800 1801 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 1802 guarantee(has_overflown() || 1803 satb_mq_set.completed_buffers_num() == 0, 1804 "Invariant: has_overflown = %s, num buffers = " SIZE_FORMAT, 1805 BOOL_TO_STR(has_overflown()), 1806 satb_mq_set.completed_buffers_num()); 1807 1808 print_stats(); 1809 } 1810 1811 void G1ConcurrentMark::clear_range_in_prev_bitmap(MemRegion mr) { 1812 _prev_mark_bitmap->clear_range(mr); 1813 } 1814 1815 HeapRegion* 1816 G1ConcurrentMark::claim_region(uint worker_id) { 1817 // "checkpoint" the finger 1818 HeapWord* finger = _finger; 1819 1820 // _heap_end will not change underneath our feet; it only changes at 1821 // yield points. 1822 while (finger < _heap_end) { 1823 assert(_g1h->is_in_g1_reserved(finger), "invariant"); 1824 1825 HeapRegion* curr_region = _g1h->heap_region_containing(finger); 1826 // Make sure that the reads below do not float before loading curr_region. 1827 OrderAccess::loadload(); 1828 // Above heap_region_containing may return NULL as we always scan claim 1829 // until the end of the heap. In this case, just jump to the next region. 1830 HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords; 1831 1832 // Is the gap between reading the finger and doing the CAS too long? 1833 HeapWord* res = Atomic::cmpxchg(end, &_finger, finger); 1834 if (res == finger && curr_region != NULL) { 1835 // we succeeded 1836 HeapWord* bottom = curr_region->bottom(); 1837 HeapWord* limit = curr_region->next_top_at_mark_start(); 1838 1839 // notice that _finger == end cannot be guaranteed here since, 1840 // someone else might have moved the finger even further 1841 assert(_finger >= end, "the finger should have moved forward"); 1842 1843 if (limit > bottom) { 1844 return curr_region; 1845 } else { 1846 assert(limit == bottom, 1847 "the region limit should be at bottom"); 1848 // we return NULL and the caller should try calling 1849 // claim_region() again. 1850 return NULL; 1851 } 1852 } else { 1853 assert(_finger > finger, "the finger should have moved forward"); 1854 // read it again 1855 finger = _finger; 1856 } 1857 } 1858 1859 return NULL; 1860 } 1861 1862 #ifndef PRODUCT 1863 class VerifyNoCSetOops { 1864 private: 1865 G1CollectedHeap* _g1h; 1866 const char* _phase; 1867 int _info; 1868 1869 public: 1870 VerifyNoCSetOops(const char* phase, int info = -1) : 1871 _g1h(G1CollectedHeap::heap()), 1872 _phase(phase), 1873 _info(info) 1874 { } 1875 1876 void operator()(G1TaskQueueEntry task_entry) const { 1877 if (task_entry.is_array_slice()) { 1878 guarantee(_g1h->is_in_reserved(task_entry.slice()), "Slice " PTR_FORMAT " must be in heap.", p2i(task_entry.slice())); 1879 return; 1880 } 1881 guarantee(oopDesc::is_oop(task_entry.obj()), 1882 "Non-oop " PTR_FORMAT ", phase: %s, info: %d", 1883 p2i(task_entry.obj()), _phase, _info); 1884 guarantee(!_g1h->is_in_cset(task_entry.obj()), 1885 "obj: " PTR_FORMAT " in CSet, phase: %s, info: %d", 1886 p2i(task_entry.obj()), _phase, _info); 1887 } 1888 }; 1889 1890 void G1ConcurrentMark::verify_no_cset_oops() { 1891 assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint"); 1892 if (!G1CollectedHeap::heap()->collector_state()->mark_in_progress()) { 1893 return; 1894 } 1895 1896 // Verify entries on the global mark stack 1897 _global_mark_stack.iterate(VerifyNoCSetOops("Stack")); 1898 1899 // Verify entries on the task queues 1900 for (uint i = 0; i < _max_num_tasks; ++i) { 1901 G1CMTaskQueue* queue = _task_queues->queue(i); 1902 queue->iterate(VerifyNoCSetOops("Queue", i)); 1903 } 1904 1905 // Verify the global finger 1906 HeapWord* global_finger = finger(); 1907 if (global_finger != NULL && global_finger < _heap_end) { 1908 // Since we always iterate over all regions, we might get a NULL HeapRegion 1909 // here. 1910 HeapRegion* global_hr = _g1h->heap_region_containing(global_finger); 1911 guarantee(global_hr == NULL || global_finger == global_hr->bottom(), 1912 "global finger: " PTR_FORMAT " region: " HR_FORMAT, 1913 p2i(global_finger), HR_FORMAT_PARAMS(global_hr)); 1914 } 1915 1916 // Verify the task fingers 1917 assert(_num_concurrent_workers <= _max_num_tasks, "sanity"); 1918 for (uint i = 0; i < _num_concurrent_workers; ++i) { 1919 G1CMTask* task = _tasks[i]; 1920 HeapWord* task_finger = task->finger(); 1921 if (task_finger != NULL && task_finger < _heap_end) { 1922 // See above note on the global finger verification. 1923 HeapRegion* task_hr = _g1h->heap_region_containing(task_finger); 1924 guarantee(task_hr == NULL || task_finger == task_hr->bottom() || 1925 !task_hr->in_collection_set(), 1926 "task finger: " PTR_FORMAT " region: " HR_FORMAT, 1927 p2i(task_finger), HR_FORMAT_PARAMS(task_hr)); 1928 } 1929 } 1930 } 1931 #endif // PRODUCT 1932 void G1ConcurrentMark::create_live_data() { 1933 _g1h->g1_rem_set()->create_card_live_data(_concurrent_workers, _next_mark_bitmap); 1934 } 1935 1936 void G1ConcurrentMark::finalize_live_data() { 1937 _g1h->g1_rem_set()->finalize_card_live_data(_g1h->workers(), _next_mark_bitmap); 1938 } 1939 1940 void G1ConcurrentMark::verify_live_data() { 1941 _g1h->g1_rem_set()->verify_card_live_data(_g1h->workers(), _next_mark_bitmap); 1942 } 1943 1944 void G1ConcurrentMark::clear_live_data(WorkGang* workers) { 1945 _g1h->g1_rem_set()->clear_card_live_data(workers); 1946 } 1947 1948 #ifdef ASSERT 1949 void G1ConcurrentMark::verify_live_data_clear() { 1950 _g1h->g1_rem_set()->verify_card_live_data_is_clear(); 1951 } 1952 #endif 1953 1954 void G1ConcurrentMark::print_stats() { 1955 if (!log_is_enabled(Debug, gc, stats)) { 1956 return; 1957 } 1958 log_debug(gc, stats)("---------------------------------------------------------------------"); 1959 for (size_t i = 0; i < _num_active_tasks; ++i) { 1960 _tasks[i]->print_stats(); 1961 log_debug(gc, stats)("---------------------------------------------------------------------"); 1962 } 1963 } 1964 1965 void G1ConcurrentMark::abort() { 1966 if (!cm_thread()->during_cycle() || _has_aborted) { 1967 // We haven't started a concurrent cycle or we have already aborted it. No need to do anything. 1968 return; 1969 } 1970 1971 // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next 1972 // concurrent bitmap clearing. 1973 { 1974 GCTraceTime(Debug, gc)("Clear Next Bitmap"); 1975 clear_bitmap(_next_mark_bitmap, _g1h->workers(), false); 1976 } 1977 // Note we cannot clear the previous marking bitmap here 1978 // since VerifyDuringGC verifies the objects marked during 1979 // a full GC against the previous bitmap. 1980 1981 { 1982 GCTraceTime(Debug, gc)("Clear Live Data"); 1983 clear_live_data(_g1h->workers()); 1984 } 1985 DEBUG_ONLY({ 1986 GCTraceTime(Debug, gc)("Verify Live Data Clear"); 1987 verify_live_data_clear(); 1988 }) 1989 // Empty mark stack 1990 reset_marking_state(); 1991 for (uint i = 0; i < _max_num_tasks; ++i) { 1992 _tasks[i]->clear_region_fields(); 1993 } 1994 _first_overflow_barrier_sync.abort(); 1995 _second_overflow_barrier_sync.abort(); 1996 _has_aborted = true; 1997 1998 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 1999 satb_mq_set.abandon_partial_marking(); 2000 // This can be called either during or outside marking, we'll read 2001 // the expected_active value from the SATB queue set. 2002 satb_mq_set.set_active_all_threads( 2003 false, /* new active value */ 2004 satb_mq_set.is_active() /* expected_active */); 2005 } 2006 2007 static void print_ms_time_info(const char* prefix, const char* name, 2008 NumberSeq& ns) { 2009 log_trace(gc, marking)("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).", 2010 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg()); 2011 if (ns.num() > 0) { 2012 log_trace(gc, marking)("%s [std. dev = %8.2f ms, max = %8.2f ms]", 2013 prefix, ns.sd(), ns.maximum()); 2014 } 2015 } 2016 2017 void G1ConcurrentMark::print_summary_info() { 2018 Log(gc, marking) log; 2019 if (!log.is_trace()) { 2020 return; 2021 } 2022 2023 log.trace(" Concurrent marking:"); 2024 print_ms_time_info(" ", "init marks", _init_times); 2025 print_ms_time_info(" ", "remarks", _remark_times); 2026 { 2027 print_ms_time_info(" ", "final marks", _remark_mark_times); 2028 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times); 2029 2030 } 2031 print_ms_time_info(" ", "cleanups", _cleanup_times); 2032 log.trace(" Finalize live data total time = %8.2f s (avg = %8.2f ms).", 2033 _total_counting_time, (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / (double)_cleanup_times.num() : 0.0)); 2034 if (G1ScrubRemSets) { 2035 log.trace(" RS scrub total time = %8.2f s (avg = %8.2f ms).", 2036 _total_rs_scrub_time, (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / (double)_cleanup_times.num() : 0.0)); 2037 } 2038 log.trace(" Total stop_world time = %8.2f s.", 2039 (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0); 2040 log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).", 2041 cm_thread()->vtime_accum(), cm_thread()->vtime_mark_accum()); 2042 } 2043 2044 void G1ConcurrentMark::print_worker_threads_on(outputStream* st) const { 2045 _concurrent_workers->print_worker_threads_on(st); 2046 } 2047 2048 void G1ConcurrentMark::threads_do(ThreadClosure* tc) const { 2049 _concurrent_workers->threads_do(tc); 2050 } 2051 2052 void G1ConcurrentMark::print_on_error(outputStream* st) const { 2053 st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT, 2054 p2i(_prev_mark_bitmap), p2i(_next_mark_bitmap)); 2055 _prev_mark_bitmap->print_on_error(st, " Prev Bits: "); 2056 _next_mark_bitmap->print_on_error(st, " Next Bits: "); 2057 } 2058 2059 static ReferenceProcessor* get_cm_oop_closure_ref_processor(G1CollectedHeap* g1h) { 2060 ReferenceProcessor* result = g1h->ref_processor_cm(); 2061 assert(result != NULL, "CM reference processor should not be NULL"); 2062 return result; 2063 } 2064 2065 G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h, 2066 G1ConcurrentMark* cm, 2067 G1CMTask* task) 2068 : MetadataAwareOopClosure(get_cm_oop_closure_ref_processor(g1h)), 2069 _g1h(g1h), _cm(cm), _task(task) 2070 { } 2071 2072 void G1CMTask::setup_for_region(HeapRegion* hr) { 2073 assert(hr != NULL, 2074 "claim_region() should have filtered out NULL regions"); 2075 _curr_region = hr; 2076 _finger = hr->bottom(); 2077 update_region_limit(); 2078 } 2079 2080 void G1CMTask::update_region_limit() { 2081 HeapRegion* hr = _curr_region; 2082 HeapWord* bottom = hr->bottom(); 2083 HeapWord* limit = hr->next_top_at_mark_start(); 2084 2085 if (limit == bottom) { 2086 // The region was collected underneath our feet. 2087 // We set the finger to bottom to ensure that the bitmap 2088 // iteration that will follow this will not do anything. 2089 // (this is not a condition that holds when we set the region up, 2090 // as the region is not supposed to be empty in the first place) 2091 _finger = bottom; 2092 } else if (limit >= _region_limit) { 2093 assert(limit >= _finger, "peace of mind"); 2094 } else { 2095 assert(limit < _region_limit, "only way to get here"); 2096 // This can happen under some pretty unusual circumstances. An 2097 // evacuation pause empties the region underneath our feet (NTAMS 2098 // at bottom). We then do some allocation in the region (NTAMS 2099 // stays at bottom), followed by the region being used as a GC 2100 // alloc region (NTAMS will move to top() and the objects 2101 // originally below it will be grayed). All objects now marked in 2102 // the region are explicitly grayed, if below the global finger, 2103 // and we do not need in fact to scan anything else. So, we simply 2104 // set _finger to be limit to ensure that the bitmap iteration 2105 // doesn't do anything. 2106 _finger = limit; 2107 } 2108 2109 _region_limit = limit; 2110 } 2111 2112 void G1CMTask::giveup_current_region() { 2113 assert(_curr_region != NULL, "invariant"); 2114 clear_region_fields(); 2115 } 2116 2117 void G1CMTask::clear_region_fields() { 2118 // Values for these three fields that indicate that we're not 2119 // holding on to a region. 2120 _curr_region = NULL; 2121 _finger = NULL; 2122 _region_limit = NULL; 2123 } 2124 2125 void G1CMTask::set_cm_oop_closure(G1CMOopClosure* cm_oop_closure) { 2126 if (cm_oop_closure == NULL) { 2127 assert(_cm_oop_closure != NULL, "invariant"); 2128 } else { 2129 assert(_cm_oop_closure == NULL, "invariant"); 2130 } 2131 _cm_oop_closure = cm_oop_closure; 2132 } 2133 2134 void G1CMTask::reset(G1CMBitMap* next_mark_bitmap) { 2135 guarantee(next_mark_bitmap != NULL, "invariant"); 2136 _next_mark_bitmap = next_mark_bitmap; 2137 clear_region_fields(); 2138 2139 _calls = 0; 2140 _elapsed_time_ms = 0.0; 2141 _termination_time_ms = 0.0; 2142 _termination_start_time_ms = 0.0; 2143 } 2144 2145 bool G1CMTask::should_exit_termination() { 2146 regular_clock_call(); 2147 // This is called when we are in the termination protocol. We should 2148 // quit if, for some reason, this task wants to abort or the global 2149 // stack is not empty (this means that we can get work from it). 2150 return !_cm->mark_stack_empty() || has_aborted(); 2151 } 2152 2153 void G1CMTask::reached_limit() { 2154 assert(_words_scanned >= _words_scanned_limit || 2155 _refs_reached >= _refs_reached_limit , 2156 "shouldn't have been called otherwise"); 2157 regular_clock_call(); 2158 } 2159 2160 void G1CMTask::regular_clock_call() { 2161 if (has_aborted()) return; 2162 2163 // First, we need to recalculate the words scanned and refs reached 2164 // limits for the next clock call. 2165 recalculate_limits(); 2166 2167 // During the regular clock call we do the following 2168 2169 // (1) If an overflow has been flagged, then we abort. 2170 if (_cm->has_overflown()) { 2171 set_has_aborted(); 2172 return; 2173 } 2174 2175 // If we are not concurrent (i.e. we're doing remark) we don't need 2176 // to check anything else. The other steps are only needed during 2177 // the concurrent marking phase. 2178 if (!_concurrent) { 2179 return; 2180 } 2181 2182 // (2) If marking has been aborted for Full GC, then we also abort. 2183 if (_cm->has_aborted()) { 2184 set_has_aborted(); 2185 return; 2186 } 2187 2188 double curr_time_ms = os::elapsedVTime() * 1000.0; 2189 2190 // (4) We check whether we should yield. If we have to, then we abort. 2191 if (SuspendibleThreadSet::should_yield()) { 2192 // We should yield. To do this we abort the task. The caller is 2193 // responsible for yielding. 2194 set_has_aborted(); 2195 return; 2196 } 2197 2198 // (5) We check whether we've reached our time quota. If we have, 2199 // then we abort. 2200 double elapsed_time_ms = curr_time_ms - _start_time_ms; 2201 if (elapsed_time_ms > _time_target_ms) { 2202 set_has_aborted(); 2203 _has_timed_out = true; 2204 return; 2205 } 2206 2207 // (6) Finally, we check whether there are enough completed STAB 2208 // buffers available for processing. If there are, we abort. 2209 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 2210 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) { 2211 // we do need to process SATB buffers, we'll abort and restart 2212 // the marking task to do so 2213 set_has_aborted(); 2214 return; 2215 } 2216 } 2217 2218 void G1CMTask::recalculate_limits() { 2219 _real_words_scanned_limit = _words_scanned + words_scanned_period; 2220 _words_scanned_limit = _real_words_scanned_limit; 2221 2222 _real_refs_reached_limit = _refs_reached + refs_reached_period; 2223 _refs_reached_limit = _real_refs_reached_limit; 2224 } 2225 2226 void G1CMTask::decrease_limits() { 2227 // This is called when we believe that we're going to do an infrequent 2228 // operation which will increase the per byte scanned cost (i.e. move 2229 // entries to/from the global stack). It basically tries to decrease the 2230 // scanning limit so that the clock is called earlier. 2231 2232 _words_scanned_limit = _real_words_scanned_limit - 3 * words_scanned_period / 4; 2233 _refs_reached_limit = _real_refs_reached_limit - 3 * refs_reached_period / 4; 2234 } 2235 2236 void G1CMTask::move_entries_to_global_stack() { 2237 // Local array where we'll store the entries that will be popped 2238 // from the local queue. 2239 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk]; 2240 2241 size_t n = 0; 2242 G1TaskQueueEntry task_entry; 2243 while (n < G1CMMarkStack::EntriesPerChunk && _task_queue->pop_local(task_entry)) { 2244 buffer[n] = task_entry; 2245 ++n; 2246 } 2247 if (n < G1CMMarkStack::EntriesPerChunk) { 2248 buffer[n] = G1TaskQueueEntry(); 2249 } 2250 2251 if (n > 0) { 2252 if (!_cm->mark_stack_push(buffer)) { 2253 set_has_aborted(); 2254 } 2255 } 2256 2257 // This operation was quite expensive, so decrease the limits. 2258 decrease_limits(); 2259 } 2260 2261 bool G1CMTask::get_entries_from_global_stack() { 2262 // Local array where we'll store the entries that will be popped 2263 // from the global stack. 2264 G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk]; 2265 2266 if (!_cm->mark_stack_pop(buffer)) { 2267 return false; 2268 } 2269 2270 // We did actually pop at least one entry. 2271 for (size_t i = 0; i < G1CMMarkStack::EntriesPerChunk; ++i) { 2272 G1TaskQueueEntry task_entry = buffer[i]; 2273 if (task_entry.is_null()) { 2274 break; 2275 } 2276 assert(task_entry.is_array_slice() || oopDesc::is_oop(task_entry.obj()), "Element " PTR_FORMAT " must be an array slice or oop", p2i(task_entry.obj())); 2277 bool success = _task_queue->push(task_entry); 2278 // We only call this when the local queue is empty or under a 2279 // given target limit. So, we do not expect this push to fail. 2280 assert(success, "invariant"); 2281 } 2282 2283 // This operation was quite expensive, so decrease the limits 2284 decrease_limits(); 2285 return true; 2286 } 2287 2288 void G1CMTask::drain_local_queue(bool partially) { 2289 if (has_aborted()) { 2290 return; 2291 } 2292 2293 // Decide what the target size is, depending whether we're going to 2294 // drain it partially (so that other tasks can steal if they run out 2295 // of things to do) or totally (at the very end). 2296 size_t target_size; 2297 if (partially) { 2298 target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize); 2299 } else { 2300 target_size = 0; 2301 } 2302 2303 if (_task_queue->size() > target_size) { 2304 G1TaskQueueEntry entry; 2305 bool ret = _task_queue->pop_local(entry); 2306 while (ret) { 2307 scan_task_entry(entry); 2308 if (_task_queue->size() <= target_size || has_aborted()) { 2309 ret = false; 2310 } else { 2311 ret = _task_queue->pop_local(entry); 2312 } 2313 } 2314 } 2315 } 2316 2317 void G1CMTask::drain_global_stack(bool partially) { 2318 if (has_aborted()) return; 2319 2320 // We have a policy to drain the local queue before we attempt to 2321 // drain the global stack. 2322 assert(partially || _task_queue->size() == 0, "invariant"); 2323 2324 // Decide what the target size is, depending whether we're going to 2325 // drain it partially (so that other tasks can steal if they run out 2326 // of things to do) or totally (at the very end). 2327 // Notice that when draining the global mark stack partially, due to the racyness 2328 // of the mark stack size update we might in fact drop below the target. But, 2329 // this is not a problem. 2330 // In case of total draining, we simply process until the global mark stack is 2331 // totally empty, disregarding the size counter. 2332 if (partially) { 2333 size_t const target_size = _cm->partial_mark_stack_size_target(); 2334 while (!has_aborted() && _cm->mark_stack_size() > target_size) { 2335 if (get_entries_from_global_stack()) { 2336 drain_local_queue(partially); 2337 } 2338 } 2339 } else { 2340 while (!has_aborted() && get_entries_from_global_stack()) { 2341 drain_local_queue(partially); 2342 } 2343 } 2344 } 2345 2346 // SATB Queue has several assumptions on whether to call the par or 2347 // non-par versions of the methods. this is why some of the code is 2348 // replicated. We should really get rid of the single-threaded version 2349 // of the code to simplify things. 2350 void G1CMTask::drain_satb_buffers() { 2351 if (has_aborted()) return; 2352 2353 // We set this so that the regular clock knows that we're in the 2354 // middle of draining buffers and doesn't set the abort flag when it 2355 // notices that SATB buffers are available for draining. It'd be 2356 // very counter productive if it did that. :-) 2357 _draining_satb_buffers = true; 2358 2359 G1CMSATBBufferClosure satb_cl(this, _g1h); 2360 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); 2361 2362 // This keeps claiming and applying the closure to completed buffers 2363 // until we run out of buffers or we need to abort. 2364 while (!has_aborted() && 2365 satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) { 2366 regular_clock_call(); 2367 } 2368 2369 _draining_satb_buffers = false; 2370 2371 assert(has_aborted() || 2372 _concurrent || 2373 satb_mq_set.completed_buffers_num() == 0, "invariant"); 2374 2375 // again, this was a potentially expensive operation, decrease the 2376 // limits to get the regular clock call early 2377 decrease_limits(); 2378 } 2379 2380 void G1CMTask::print_stats() { 2381 log_debug(gc, stats)("Marking Stats, task = %u, calls = %u", 2382 _worker_id, _calls); 2383 log_debug(gc, stats)(" Elapsed time = %1.2lfms, Termination time = %1.2lfms", 2384 _elapsed_time_ms, _termination_time_ms); 2385 log_debug(gc, stats)(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", 2386 _step_times_ms.num(), _step_times_ms.avg(), 2387 _step_times_ms.sd()); 2388 log_debug(gc, stats)(" max = %1.2lfms, total = %1.2lfms", 2389 _step_times_ms.maximum(), _step_times_ms.sum()); 2390 } 2391 2392 bool G1ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, G1TaskQueueEntry& task_entry) { 2393 return _task_queues->steal(worker_id, hash_seed, task_entry); 2394 } 2395 2396 /***************************************************************************** 2397 2398 The do_marking_step(time_target_ms, ...) method is the building 2399 block of the parallel marking framework. It can be called in parallel 2400 with other invocations of do_marking_step() on different tasks 2401 (but only one per task, obviously) and concurrently with the 2402 mutator threads, or during remark, hence it eliminates the need 2403 for two versions of the code. When called during remark, it will 2404 pick up from where the task left off during the concurrent marking 2405 phase. Interestingly, tasks are also claimable during evacuation 2406 pauses too, since do_marking_step() ensures that it aborts before 2407 it needs to yield. 2408 2409 The data structures that it uses to do marking work are the 2410 following: 2411 2412 (1) Marking Bitmap. If there are gray objects that appear only 2413 on the bitmap (this happens either when dealing with an overflow 2414 or when the initial marking phase has simply marked the roots 2415 and didn't push them on the stack), then tasks claim heap 2416 regions whose bitmap they then scan to find gray objects. A 2417 global finger indicates where the end of the last claimed region 2418 is. A local finger indicates how far into the region a task has 2419 scanned. The two fingers are used to determine how to gray an 2420 object (i.e. whether simply marking it is OK, as it will be 2421 visited by a task in the future, or whether it needs to be also 2422 pushed on a stack). 2423 2424 (2) Local Queue. The local queue of the task which is accessed 2425 reasonably efficiently by the task. Other tasks can steal from 2426 it when they run out of work. Throughout the marking phase, a 2427 task attempts to keep its local queue short but not totally 2428 empty, so that entries are available for stealing by other 2429 tasks. Only when there is no more work, a task will totally 2430 drain its local queue. 2431 2432 (3) Global Mark Stack. This handles local queue overflow. During 2433 marking only sets of entries are moved between it and the local 2434 queues, as access to it requires a mutex and more fine-grain 2435 interaction with it which might cause contention. If it 2436 overflows, then the marking phase should restart and iterate 2437 over the bitmap to identify gray objects. Throughout the marking 2438 phase, tasks attempt to keep the global mark stack at a small 2439 length but not totally empty, so that entries are available for 2440 popping by other tasks. Only when there is no more work, tasks 2441 will totally drain the global mark stack. 2442 2443 (4) SATB Buffer Queue. This is where completed SATB buffers are 2444 made available. Buffers are regularly removed from this queue 2445 and scanned for roots, so that the queue doesn't get too 2446 long. During remark, all completed buffers are processed, as 2447 well as the filled in parts of any uncompleted buffers. 2448 2449 The do_marking_step() method tries to abort when the time target 2450 has been reached. There are a few other cases when the 2451 do_marking_step() method also aborts: 2452 2453 (1) When the marking phase has been aborted (after a Full GC). 2454 2455 (2) When a global overflow (on the global stack) has been 2456 triggered. Before the task aborts, it will actually sync up with 2457 the other tasks to ensure that all the marking data structures 2458 (local queues, stacks, fingers etc.) are re-initialized so that 2459 when do_marking_step() completes, the marking phase can 2460 immediately restart. 2461 2462 (3) When enough completed SATB buffers are available. The 2463 do_marking_step() method only tries to drain SATB buffers right 2464 at the beginning. So, if enough buffers are available, the 2465 marking step aborts and the SATB buffers are processed at 2466 the beginning of the next invocation. 2467 2468 (4) To yield. when we have to yield then we abort and yield 2469 right at the end of do_marking_step(). This saves us from a lot 2470 of hassle as, by yielding we might allow a Full GC. If this 2471 happens then objects will be compacted underneath our feet, the 2472 heap might shrink, etc. We save checking for this by just 2473 aborting and doing the yield right at the end. 2474 2475 From the above it follows that the do_marking_step() method should 2476 be called in a loop (or, otherwise, regularly) until it completes. 2477 2478 If a marking step completes without its has_aborted() flag being 2479 true, it means it has completed the current marking phase (and 2480 also all other marking tasks have done so and have all synced up). 2481 2482 A method called regular_clock_call() is invoked "regularly" (in 2483 sub ms intervals) throughout marking. It is this clock method that 2484 checks all the abort conditions which were mentioned above and 2485 decides when the task should abort. A work-based scheme is used to 2486 trigger this clock method: when the number of object words the 2487 marking phase has scanned or the number of references the marking 2488 phase has visited reach a given limit. Additional invocations to 2489 the method clock have been planted in a few other strategic places 2490 too. The initial reason for the clock method was to avoid calling 2491 vtime too regularly, as it is quite expensive. So, once it was in 2492 place, it was natural to piggy-back all the other conditions on it 2493 too and not constantly check them throughout the code. 2494 2495 If do_termination is true then do_marking_step will enter its 2496 termination protocol. 2497 2498 The value of is_serial must be true when do_marking_step is being 2499 called serially (i.e. by the VMThread) and do_marking_step should 2500 skip any synchronization in the termination and overflow code. 2501 Examples include the serial remark code and the serial reference 2502 processing closures. 2503 2504 The value of is_serial must be false when do_marking_step is 2505 being called by any of the worker threads in a work gang. 2506 Examples include the concurrent marking code (CMMarkingTask), 2507 the MT remark code, and the MT reference processing closures. 2508 2509 *****************************************************************************/ 2510 2511 void G1CMTask::do_marking_step(double time_target_ms, 2512 bool do_termination, 2513 bool is_serial) { 2514 assert(time_target_ms >= 1.0, "minimum granularity is 1ms"); 2515 assert(_concurrent == _cm->concurrent(), "they should be the same"); 2516 2517 _start_time_ms = os::elapsedVTime() * 1000.0; 2518 2519 // If do_stealing is true then do_marking_step will attempt to 2520 // steal work from the other G1CMTasks. It only makes sense to 2521 // enable stealing when the termination protocol is enabled 2522 // and do_marking_step() is not being called serially. 2523 bool do_stealing = do_termination && !is_serial; 2524 2525 double diff_prediction_ms = _g1h->g1_policy()->predictor().get_new_prediction(&_marking_step_diffs_ms); 2526 _time_target_ms = time_target_ms - diff_prediction_ms; 2527 2528 // set up the variables that are used in the work-based scheme to 2529 // call the regular clock method 2530 _words_scanned = 0; 2531 _refs_reached = 0; 2532 recalculate_limits(); 2533 2534 // clear all flags 2535 clear_has_aborted(); 2536 _has_timed_out = false; 2537 _draining_satb_buffers = false; 2538 2539 ++_calls; 2540 2541 // Set up the bitmap and oop closures. Anything that uses them is 2542 // eventually called from this method, so it is OK to allocate these 2543 // statically. 2544 G1CMBitMapClosure bitmap_closure(this, _cm); 2545 G1CMOopClosure cm_oop_closure(_g1h, _cm, this); 2546 set_cm_oop_closure(&cm_oop_closure); 2547 2548 if (_cm->has_overflown()) { 2549 // This can happen if the mark stack overflows during a GC pause 2550 // and this task, after a yield point, restarts. We have to abort 2551 // as we need to get into the overflow protocol which happens 2552 // right at the end of this task. 2553 set_has_aborted(); 2554 } 2555 2556 // First drain any available SATB buffers. After this, we will not 2557 // look at SATB buffers before the next invocation of this method. 2558 // If enough completed SATB buffers are queued up, the regular clock 2559 // will abort this task so that it restarts. 2560 drain_satb_buffers(); 2561 // ...then partially drain the local queue and the global stack 2562 drain_local_queue(true); 2563 drain_global_stack(true); 2564 2565 do { 2566 if (!has_aborted() && _curr_region != NULL) { 2567 // This means that we're already holding on to a region. 2568 assert(_finger != NULL, "if region is not NULL, then the finger " 2569 "should not be NULL either"); 2570 2571 // We might have restarted this task after an evacuation pause 2572 // which might have evacuated the region we're holding on to 2573 // underneath our feet. Let's read its limit again to make sure 2574 // that we do not iterate over a region of the heap that 2575 // contains garbage (update_region_limit() will also move 2576 // _finger to the start of the region if it is found empty). 2577 update_region_limit(); 2578 // We will start from _finger not from the start of the region, 2579 // as we might be restarting this task after aborting half-way 2580 // through scanning this region. In this case, _finger points to 2581 // the address where we last found a marked object. If this is a 2582 // fresh region, _finger points to start(). 2583 MemRegion mr = MemRegion(_finger, _region_limit); 2584 2585 assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(), 2586 "humongous regions should go around loop once only"); 2587 2588 // Some special cases: 2589 // If the memory region is empty, we can just give up the region. 2590 // If the current region is humongous then we only need to check 2591 // the bitmap for the bit associated with the start of the object, 2592 // scan the object if it's live, and give up the region. 2593 // Otherwise, let's iterate over the bitmap of the part of the region 2594 // that is left. 2595 // If the iteration is successful, give up the region. 2596 if (mr.is_empty()) { 2597 giveup_current_region(); 2598 regular_clock_call(); 2599 } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) { 2600 if (_next_mark_bitmap->is_marked(mr.start())) { 2601 // The object is marked - apply the closure 2602 bitmap_closure.do_addr(mr.start()); 2603 } 2604 // Even if this task aborted while scanning the humongous object 2605 // we can (and should) give up the current region. 2606 giveup_current_region(); 2607 regular_clock_call(); 2608 } else if (_next_mark_bitmap->iterate(&bitmap_closure, mr)) { 2609 giveup_current_region(); 2610 regular_clock_call(); 2611 } else { 2612 assert(has_aborted(), "currently the only way to do so"); 2613 // The only way to abort the bitmap iteration is to return 2614 // false from the do_bit() method. However, inside the 2615 // do_bit() method we move the _finger to point to the 2616 // object currently being looked at. So, if we bail out, we 2617 // have definitely set _finger to something non-null. 2618 assert(_finger != NULL, "invariant"); 2619 2620 // Region iteration was actually aborted. So now _finger 2621 // points to the address of the object we last scanned. If we 2622 // leave it there, when we restart this task, we will rescan 2623 // the object. It is easy to avoid this. We move the finger by 2624 // enough to point to the next possible object header. 2625 assert(_finger < _region_limit, "invariant"); 2626 HeapWord* const new_finger = _finger + ((oop)_finger)->size(); 2627 // Check if bitmap iteration was aborted while scanning the last object 2628 if (new_finger >= _region_limit) { 2629 giveup_current_region(); 2630 } else { 2631 move_finger_to(new_finger); 2632 } 2633 } 2634 } 2635 // At this point we have either completed iterating over the 2636 // region we were holding on to, or we have aborted. 2637 2638 // We then partially drain the local queue and the global stack. 2639 // (Do we really need this?) 2640 drain_local_queue(true); 2641 drain_global_stack(true); 2642 2643 // Read the note on the claim_region() method on why it might 2644 // return NULL with potentially more regions available for 2645 // claiming and why we have to check out_of_regions() to determine 2646 // whether we're done or not. 2647 while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) { 2648 // We are going to try to claim a new region. We should have 2649 // given up on the previous one. 2650 // Separated the asserts so that we know which one fires. 2651 assert(_curr_region == NULL, "invariant"); 2652 assert(_finger == NULL, "invariant"); 2653 assert(_region_limit == NULL, "invariant"); 2654 HeapRegion* claimed_region = _cm->claim_region(_worker_id); 2655 if (claimed_region != NULL) { 2656 // Yes, we managed to claim one 2657 setup_for_region(claimed_region); 2658 assert(_curr_region == claimed_region, "invariant"); 2659 } 2660 // It is important to call the regular clock here. It might take 2661 // a while to claim a region if, for example, we hit a large 2662 // block of empty regions. So we need to call the regular clock 2663 // method once round the loop to make sure it's called 2664 // frequently enough. 2665 regular_clock_call(); 2666 } 2667 2668 if (!has_aborted() && _curr_region == NULL) { 2669 assert(_cm->out_of_regions(), 2670 "at this point we should be out of regions"); 2671 } 2672 } while ( _curr_region != NULL && !has_aborted()); 2673 2674 if (!has_aborted()) { 2675 // We cannot check whether the global stack is empty, since other 2676 // tasks might be pushing objects to it concurrently. 2677 assert(_cm->out_of_regions(), 2678 "at this point we should be out of regions"); 2679 // Try to reduce the number of available SATB buffers so that 2680 // remark has less work to do. 2681 drain_satb_buffers(); 2682 } 2683 2684 // Since we've done everything else, we can now totally drain the 2685 // local queue and global stack. 2686 drain_local_queue(false); 2687 drain_global_stack(false); 2688 2689 // Attempt at work stealing from other task's queues. 2690 if (do_stealing && !has_aborted()) { 2691 // We have not aborted. This means that we have finished all that 2692 // we could. Let's try to do some stealing... 2693 2694 // We cannot check whether the global stack is empty, since other 2695 // tasks might be pushing objects to it concurrently. 2696 assert(_cm->out_of_regions() && _task_queue->size() == 0, 2697 "only way to reach here"); 2698 while (!has_aborted()) { 2699 G1TaskQueueEntry entry; 2700 if (_cm->try_stealing(_worker_id, &_hash_seed, entry)) { 2701 scan_task_entry(entry); 2702 2703 // And since we're towards the end, let's totally drain the 2704 // local queue and global stack. 2705 drain_local_queue(false); 2706 drain_global_stack(false); 2707 } else { 2708 break; 2709 } 2710 } 2711 } 2712 2713 // We still haven't aborted. Now, let's try to get into the 2714 // termination protocol. 2715 if (do_termination && !has_aborted()) { 2716 // We cannot check whether the global stack is empty, since other 2717 // tasks might be concurrently pushing objects on it. 2718 // Separated the asserts so that we know which one fires. 2719 assert(_cm->out_of_regions(), "only way to reach here"); 2720 assert(_task_queue->size() == 0, "only way to reach here"); 2721 _termination_start_time_ms = os::elapsedVTime() * 1000.0; 2722 2723 // The G1CMTask class also extends the TerminatorTerminator class, 2724 // hence its should_exit_termination() method will also decide 2725 // whether to exit the termination protocol or not. 2726 bool finished = (is_serial || 2727 _cm->terminator()->offer_termination(this)); 2728 double termination_end_time_ms = os::elapsedVTime() * 1000.0; 2729 _termination_time_ms += 2730 termination_end_time_ms - _termination_start_time_ms; 2731 2732 if (finished) { 2733 // We're all done. 2734 2735 if (_worker_id == 0) { 2736 // Let's allow task 0 to do this 2737 if (_concurrent) { 2738 assert(_cm->concurrent_marking_in_progress(), "invariant"); 2739 // We need to set this to false before the next 2740 // safepoint. This way we ensure that the marking phase 2741 // doesn't observe any more heap expansions. 2742 _cm->clear_concurrent_marking_in_progress(); 2743 } 2744 } 2745 2746 // We can now guarantee that the global stack is empty, since 2747 // all other tasks have finished. We separated the guarantees so 2748 // that, if a condition is false, we can immediately find out 2749 // which one. 2750 guarantee(_cm->out_of_regions(), "only way to reach here"); 2751 guarantee(_cm->mark_stack_empty(), "only way to reach here"); 2752 guarantee(_task_queue->size() == 0, "only way to reach here"); 2753 guarantee(!_cm->has_overflown(), "only way to reach here"); 2754 } else { 2755 // Apparently there's more work to do. Let's abort this task. It 2756 // will restart it and we can hopefully find more things to do. 2757 set_has_aborted(); 2758 } 2759 } 2760 2761 // Mainly for debugging purposes to make sure that a pointer to the 2762 // closure which was statically allocated in this frame doesn't 2763 // escape it by accident. 2764 set_cm_oop_closure(NULL); 2765 double end_time_ms = os::elapsedVTime() * 1000.0; 2766 double elapsed_time_ms = end_time_ms - _start_time_ms; 2767 // Update the step history. 2768 _step_times_ms.add(elapsed_time_ms); 2769 2770 if (has_aborted()) { 2771 // The task was aborted for some reason. 2772 if (_has_timed_out) { 2773 double diff_ms = elapsed_time_ms - _time_target_ms; 2774 // Keep statistics of how well we did with respect to hitting 2775 // our target only if we actually timed out (if we aborted for 2776 // other reasons, then the results might get skewed). 2777 _marking_step_diffs_ms.add(diff_ms); 2778 } 2779 2780 if (_cm->has_overflown()) { 2781 // This is the interesting one. We aborted because a global 2782 // overflow was raised. This means we have to restart the 2783 // marking phase and start iterating over regions. However, in 2784 // order to do this we have to make sure that all tasks stop 2785 // what they are doing and re-initialize in a safe manner. We 2786 // will achieve this with the use of two barrier sync points. 2787 2788 if (!is_serial) { 2789 // We only need to enter the sync barrier if being called 2790 // from a parallel context 2791 _cm->enter_first_sync_barrier(_worker_id); 2792 2793 // When we exit this sync barrier we know that all tasks have 2794 // stopped doing marking work. So, it's now safe to 2795 // re-initialize our data structures. At the end of this method, 2796 // task 0 will clear the global data structures. 2797 } 2798 2799 // We clear the local state of this task... 2800 clear_region_fields(); 2801 2802 if (!is_serial) { 2803 // ...and enter the second barrier. 2804 _cm->enter_second_sync_barrier(_worker_id); 2805 } 2806 // At this point, if we're during the concurrent phase of 2807 // marking, everything has been re-initialized and we're 2808 // ready to restart. 2809 } 2810 } 2811 } 2812 2813 G1CMTask::G1CMTask(uint worker_id, G1ConcurrentMark* cm, G1CMTaskQueue* task_queue) : 2814 _objArray_processor(this), 2815 _worker_id(worker_id), 2816 _g1h(G1CollectedHeap::heap()), 2817 _cm(cm), 2818 _next_mark_bitmap(NULL), 2819 _task_queue(task_queue), 2820 _calls(0), 2821 _time_target_ms(0.0), 2822 _start_time_ms(0.0), 2823 _cm_oop_closure(NULL), 2824 _curr_region(NULL), 2825 _finger(NULL), 2826 _region_limit(NULL), 2827 _words_scanned(0), 2828 _words_scanned_limit(0), 2829 _real_words_scanned_limit(0), 2830 _refs_reached(0), 2831 _refs_reached_limit(0), 2832 _real_refs_reached_limit(0), 2833 _hash_seed(17), 2834 _has_aborted(false), 2835 _has_timed_out(false), 2836 _draining_satb_buffers(false), 2837 _step_times_ms(), 2838 _elapsed_time_ms(0.0), 2839 _termination_time_ms(0.0), 2840 _termination_start_time_ms(0.0), 2841 _concurrent(false), 2842 _marking_step_diffs_ms() 2843 { 2844 guarantee(task_queue != NULL, "invariant"); 2845 2846 _marking_step_diffs_ms.add(0.5); 2847 } 2848 2849 // These are formatting macros that are used below to ensure 2850 // consistent formatting. The *_H_* versions are used to format the 2851 // header for a particular value and they should be kept consistent 2852 // with the corresponding macro. Also note that most of the macros add 2853 // the necessary white space (as a prefix) which makes them a bit 2854 // easier to compose. 2855 2856 // All the output lines are prefixed with this string to be able to 2857 // identify them easily in a large log file. 2858 #define G1PPRL_LINE_PREFIX "###" 2859 2860 #define G1PPRL_ADDR_BASE_FORMAT " " PTR_FORMAT "-" PTR_FORMAT 2861 #ifdef _LP64 2862 #define G1PPRL_ADDR_BASE_H_FORMAT " %37s" 2863 #else // _LP64 2864 #define G1PPRL_ADDR_BASE_H_FORMAT " %21s" 2865 #endif // _LP64 2866 2867 // For per-region info 2868 #define G1PPRL_TYPE_FORMAT " %-4s" 2869 #define G1PPRL_TYPE_H_FORMAT " %4s" 2870 #define G1PPRL_BYTE_FORMAT " " SIZE_FORMAT_W(9) 2871 #define G1PPRL_BYTE_H_FORMAT " %9s" 2872 #define G1PPRL_DOUBLE_FORMAT " %14.1f" 2873 #define G1PPRL_DOUBLE_H_FORMAT " %14s" 2874 2875 // For summary info 2876 #define G1PPRL_SUM_ADDR_FORMAT(tag) " " tag ":" G1PPRL_ADDR_BASE_FORMAT 2877 #define G1PPRL_SUM_BYTE_FORMAT(tag) " " tag ": " SIZE_FORMAT 2878 #define G1PPRL_SUM_MB_FORMAT(tag) " " tag ": %1.2f MB" 2879 #define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag) " / %1.2f %%" 2880 2881 G1PrintRegionLivenessInfoClosure::G1PrintRegionLivenessInfoClosure(const char* phase_name) : 2882 _total_used_bytes(0), _total_capacity_bytes(0), 2883 _total_prev_live_bytes(0), _total_next_live_bytes(0), 2884 _total_remset_bytes(0), _total_strong_code_roots_bytes(0) 2885 { 2886 G1CollectedHeap* g1h = G1CollectedHeap::heap(); 2887 MemRegion g1_reserved = g1h->g1_reserved(); 2888 double now = os::elapsedTime(); 2889 2890 // Print the header of the output. 2891 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now); 2892 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX" HEAP" 2893 G1PPRL_SUM_ADDR_FORMAT("reserved") 2894 G1PPRL_SUM_BYTE_FORMAT("region-size"), 2895 p2i(g1_reserved.start()), p2i(g1_reserved.end()), 2896 HeapRegion::GrainBytes); 2897 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX); 2898 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX 2899 G1PPRL_TYPE_H_FORMAT 2900 G1PPRL_ADDR_BASE_H_FORMAT 2901 G1PPRL_BYTE_H_FORMAT 2902 G1PPRL_BYTE_H_FORMAT 2903 G1PPRL_BYTE_H_FORMAT 2904 G1PPRL_DOUBLE_H_FORMAT 2905 G1PPRL_BYTE_H_FORMAT 2906 G1PPRL_BYTE_H_FORMAT, 2907 "type", "address-range", 2908 "used", "prev-live", "next-live", "gc-eff", 2909 "remset", "code-roots"); 2910 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX 2911 G1PPRL_TYPE_H_FORMAT 2912 G1PPRL_ADDR_BASE_H_FORMAT 2913 G1PPRL_BYTE_H_FORMAT 2914 G1PPRL_BYTE_H_FORMAT 2915 G1PPRL_BYTE_H_FORMAT 2916 G1PPRL_DOUBLE_H_FORMAT 2917 G1PPRL_BYTE_H_FORMAT 2918 G1PPRL_BYTE_H_FORMAT, 2919 "", "", 2920 "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)", 2921 "(bytes)", "(bytes)"); 2922 } 2923 2924 bool G1PrintRegionLivenessInfoClosure::do_heap_region(HeapRegion* r) { 2925 const char* type = r->get_type_str(); 2926 HeapWord* bottom = r->bottom(); 2927 HeapWord* end = r->end(); 2928 size_t capacity_bytes = r->capacity(); 2929 size_t used_bytes = r->used(); 2930 size_t prev_live_bytes = r->live_bytes(); 2931 size_t next_live_bytes = r->next_live_bytes(); 2932 double gc_eff = r->gc_efficiency(); 2933 size_t remset_bytes = r->rem_set()->mem_size(); 2934 size_t strong_code_roots_bytes = r->rem_set()->strong_code_roots_mem_size(); 2935 2936 _total_used_bytes += used_bytes; 2937 _total_capacity_bytes += capacity_bytes; 2938 _total_prev_live_bytes += prev_live_bytes; 2939 _total_next_live_bytes += next_live_bytes; 2940 _total_remset_bytes += remset_bytes; 2941 _total_strong_code_roots_bytes += strong_code_roots_bytes; 2942 2943 // Print a line for this particular region. 2944 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX 2945 G1PPRL_TYPE_FORMAT 2946 G1PPRL_ADDR_BASE_FORMAT 2947 G1PPRL_BYTE_FORMAT 2948 G1PPRL_BYTE_FORMAT 2949 G1PPRL_BYTE_FORMAT 2950 G1PPRL_DOUBLE_FORMAT 2951 G1PPRL_BYTE_FORMAT 2952 G1PPRL_BYTE_FORMAT, 2953 type, p2i(bottom), p2i(end), 2954 used_bytes, prev_live_bytes, next_live_bytes, gc_eff, 2955 remset_bytes, strong_code_roots_bytes); 2956 2957 return false; 2958 } 2959 2960 G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() { 2961 // add static memory usages to remembered set sizes 2962 _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size(); 2963 // Print the footer of the output. 2964 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX); 2965 log_trace(gc, liveness)(G1PPRL_LINE_PREFIX 2966 " SUMMARY" 2967 G1PPRL_SUM_MB_FORMAT("capacity") 2968 G1PPRL_SUM_MB_PERC_FORMAT("used") 2969 G1PPRL_SUM_MB_PERC_FORMAT("prev-live") 2970 G1PPRL_SUM_MB_PERC_FORMAT("next-live") 2971 G1PPRL_SUM_MB_FORMAT("remset") 2972 G1PPRL_SUM_MB_FORMAT("code-roots"), 2973 bytes_to_mb(_total_capacity_bytes), 2974 bytes_to_mb(_total_used_bytes), 2975 percent_of(_total_used_bytes, _total_capacity_bytes), 2976 bytes_to_mb(_total_prev_live_bytes), 2977 percent_of(_total_prev_live_bytes, _total_capacity_bytes), 2978 bytes_to_mb(_total_next_live_bytes), 2979 percent_of(_total_next_live_bytes, _total_capacity_bytes), 2980 bytes_to_mb(_total_remset_bytes), 2981 bytes_to_mb(_total_strong_code_roots_bytes)); 2982 }