1 /*
   2  * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/metadataOnStackMark.hpp"
  27 #include "classfile/stringTable.hpp"
  28 #include "code/codeCache.hpp"
  29 #include "code/icBuffer.hpp"
  30 #include "gc/g1/bufferingOopClosure.hpp"
  31 #include "gc/g1/concurrentG1Refine.hpp"
  32 #include "gc/g1/concurrentG1RefineThread.hpp"
  33 #include "gc/g1/concurrentMarkThread.inline.hpp"
  34 #include "gc/g1/g1Allocator.inline.hpp"
  35 #include "gc/g1/g1CollectedHeap.inline.hpp"
  36 #include "gc/g1/g1CollectorPolicy.hpp"
  37 #include "gc/g1/g1CollectorState.hpp"
  38 #include "gc/g1/g1ErgoVerbose.hpp"
  39 #include "gc/g1/g1EvacFailure.hpp"
  40 #include "gc/g1/g1GCPhaseTimes.hpp"
  41 #include "gc/g1/g1Log.hpp"
  42 #include "gc/g1/g1MarkSweep.hpp"
  43 #include "gc/g1/g1OopClosures.inline.hpp"
  44 #include "gc/g1/g1ParScanThreadState.inline.hpp"
  45 #include "gc/g1/g1RegionToSpaceMapper.hpp"
  46 #include "gc/g1/g1RemSet.inline.hpp"
  47 #include "gc/g1/g1RootClosures.hpp"
  48 #include "gc/g1/g1RootProcessor.hpp"
  49 #include "gc/g1/g1StringDedup.hpp"
  50 #include "gc/g1/g1YCTypes.hpp"
  51 #include "gc/g1/heapRegion.inline.hpp"
  52 #include "gc/g1/heapRegionRemSet.hpp"
  53 #include "gc/g1/heapRegionSet.inline.hpp"
  54 #include "gc/g1/suspendibleThreadSet.hpp"
  55 #include "gc/g1/vm_operations_g1.hpp"
  56 #include "gc/shared/gcHeapSummary.hpp"
  57 #include "gc/shared/gcId.hpp"
  58 #include "gc/shared/gcLocker.inline.hpp"
  59 #include "gc/shared/gcTimer.hpp"
  60 #include "gc/shared/gcTrace.hpp"
  61 #include "gc/shared/gcTraceTime.hpp"
  62 #include "gc/shared/generationSpec.hpp"
  63 #include "gc/shared/isGCActiveMark.hpp"
  64 #include "gc/shared/referenceProcessor.hpp"
  65 #include "gc/shared/taskqueue.inline.hpp"
  66 #include "memory/allocation.hpp"
  67 #include "memory/iterator.hpp"
  68 #include "oops/oop.inline.hpp"
  69 #include "runtime/atomic.inline.hpp"
  70 #include "runtime/init.hpp"
  71 #include "runtime/orderAccess.inline.hpp"
  72 #include "runtime/vmThread.hpp"
  73 #include "utilities/globalDefinitions.hpp"
  74 #include "utilities/stack.inline.hpp"
  75 
  76 size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0;
  77 
  78 // INVARIANTS/NOTES
  79 //
  80 // All allocation activity covered by the G1CollectedHeap interface is
  81 // serialized by acquiring the HeapLock.  This happens in mem_allocate
  82 // and allocate_new_tlab, which are the "entry" points to the
  83 // allocation code from the rest of the JVM.  (Note that this does not
  84 // apply to TLAB allocation, which is not part of this interface: it
  85 // is done by clients of this interface.)
  86 
  87 // Local to this file.
  88 
  89 class RefineCardTableEntryClosure: public CardTableEntryClosure {
  90   bool _concurrent;
  91 public:
  92   RefineCardTableEntryClosure() : _concurrent(true) { }
  93 
  94   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
  95     bool oops_into_cset = G1CollectedHeap::heap()->g1_rem_set()->refine_card(card_ptr, worker_i, false);
  96     // This path is executed by the concurrent refine or mutator threads,
  97     // concurrently, and so we do not care if card_ptr contains references
  98     // that point into the collection set.
  99     assert(!oops_into_cset, "should be");
 100 
 101     if (_concurrent && SuspendibleThreadSet::should_yield()) {
 102       // Caller will actually yield.
 103       return false;
 104     }
 105     // Otherwise, we finished successfully; return true.
 106     return true;
 107   }
 108 
 109   void set_concurrent(bool b) { _concurrent = b; }
 110 };
 111 
 112 
 113 class RedirtyLoggedCardTableEntryClosure : public CardTableEntryClosure {
 114  private:
 115   size_t _num_processed;
 116 
 117  public:
 118   RedirtyLoggedCardTableEntryClosure() : CardTableEntryClosure(), _num_processed(0) { }
 119 
 120   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
 121     *card_ptr = CardTableModRefBS::dirty_card_val();
 122     _num_processed++;
 123     return true;
 124   }
 125 
 126   size_t num_processed() const { return _num_processed; }
 127 };
 128 
 129 
 130 void G1RegionMappingChangedListener::reset_from_card_cache(uint start_idx, size_t num_regions) {
 131   HeapRegionRemSet::invalidate_from_card_cache(start_idx, num_regions);
 132 }
 133 
 134 void G1RegionMappingChangedListener::on_commit(uint start_idx, size_t num_regions, bool zero_filled) {
 135   // The from card cache is not the memory that is actually committed. So we cannot
 136   // take advantage of the zero_filled parameter.
 137   reset_from_card_cache(start_idx, num_regions);
 138 }
 139 
 140 void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr)
 141 {
 142   // Claim the right to put the region on the dirty cards region list
 143   // by installing a self pointer.
 144   HeapRegion* next = hr->get_next_dirty_cards_region();
 145   if (next == NULL) {
 146     HeapRegion* res = (HeapRegion*)
 147       Atomic::cmpxchg_ptr(hr, hr->next_dirty_cards_region_addr(),
 148                           NULL);
 149     if (res == NULL) {
 150       HeapRegion* head;
 151       do {
 152         // Put the region to the dirty cards region list.
 153         head = _dirty_cards_region_list;
 154         next = (HeapRegion*)
 155           Atomic::cmpxchg_ptr(hr, &_dirty_cards_region_list, head);
 156         if (next == head) {
 157           assert(hr->get_next_dirty_cards_region() == hr,
 158                  "hr->get_next_dirty_cards_region() != hr");
 159           if (next == NULL) {
 160             // The last region in the list points to itself.
 161             hr->set_next_dirty_cards_region(hr);
 162           } else {
 163             hr->set_next_dirty_cards_region(next);
 164           }
 165         }
 166       } while (next != head);
 167     }
 168   }
 169 }
 170 
 171 HeapRegion* G1CollectedHeap::pop_dirty_cards_region()
 172 {
 173   HeapRegion* head;
 174   HeapRegion* hr;
 175   do {
 176     head = _dirty_cards_region_list;
 177     if (head == NULL) {
 178       return NULL;
 179     }
 180     HeapRegion* new_head = head->get_next_dirty_cards_region();
 181     if (head == new_head) {
 182       // The last region.
 183       new_head = NULL;
 184     }
 185     hr = (HeapRegion*)Atomic::cmpxchg_ptr(new_head, &_dirty_cards_region_list,
 186                                           head);
 187   } while (hr != head);
 188   assert(hr != NULL, "invariant");
 189   hr->set_next_dirty_cards_region(NULL);
 190   return hr;
 191 }
 192 
 193 // Returns true if the reference points to an object that
 194 // can move in an incremental collection.
 195 bool G1CollectedHeap::is_scavengable(const void* p) {
 196   HeapRegion* hr = heap_region_containing(p);
 197   return !hr->is_pinned();
 198 }
 199 
 200 // Private methods.
 201 
 202 HeapRegion*
 203 G1CollectedHeap::new_region_try_secondary_free_list(bool is_old) {
 204   MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
 205   while (!_secondary_free_list.is_empty() || free_regions_coming()) {
 206     if (!_secondary_free_list.is_empty()) {
 207       if (G1ConcRegionFreeingVerbose) {
 208         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "
 209                                "secondary_free_list has %u entries",
 210                                _secondary_free_list.length());
 211       }
 212       // It looks as if there are free regions available on the
 213       // secondary_free_list. Let's move them to the free_list and try
 214       // again to allocate from it.
 215       append_secondary_free_list();
 216 
 217       assert(_hrm.num_free_regions() > 0, "if the secondary_free_list was not "
 218              "empty we should have moved at least one entry to the free_list");
 219       HeapRegion* res = _hrm.allocate_free_region(is_old);
 220       if (G1ConcRegionFreeingVerbose) {
 221         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "
 222                                "allocated " HR_FORMAT " from secondary_free_list",
 223                                HR_FORMAT_PARAMS(res));
 224       }
 225       return res;
 226     }
 227 
 228     // Wait here until we get notified either when (a) there are no
 229     // more free regions coming or (b) some regions have been moved on
 230     // the secondary_free_list.
 231     SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag);
 232   }
 233 
 234   if (G1ConcRegionFreeingVerbose) {
 235     gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "
 236                            "could not allocate from secondary_free_list");
 237   }
 238   return NULL;
 239 }
 240 
 241 HeapRegion* G1CollectedHeap::new_region(size_t word_size, bool is_old, bool do_expand) {
 242   assert(!is_humongous(word_size) || word_size <= HeapRegion::GrainWords,
 243          "the only time we use this to allocate a humongous region is "
 244          "when we are allocating a single humongous region");
 245 
 246   HeapRegion* res;
 247   if (G1StressConcRegionFreeing) {
 248     if (!_secondary_free_list.is_empty()) {
 249       if (G1ConcRegionFreeingVerbose) {
 250         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "
 251                                "forced to look at the secondary_free_list");
 252       }
 253       res = new_region_try_secondary_free_list(is_old);
 254       if (res != NULL) {
 255         return res;
 256       }
 257     }
 258   }
 259 
 260   res = _hrm.allocate_free_region(is_old);
 261 
 262   if (res == NULL) {
 263     if (G1ConcRegionFreeingVerbose) {
 264       gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "
 265                              "res == NULL, trying the secondary_free_list");
 266     }
 267     res = new_region_try_secondary_free_list(is_old);
 268   }
 269   if (res == NULL && do_expand && _expand_heap_after_alloc_failure) {
 270     // Currently, only attempts to allocate GC alloc regions set
 271     // do_expand to true. So, we should only reach here during a
 272     // safepoint. If this assumption changes we might have to
 273     // reconsider the use of _expand_heap_after_alloc_failure.
 274     assert(SafepointSynchronize::is_at_safepoint(), "invariant");
 275 
 276     ergo_verbose1(ErgoHeapSizing,
 277                   "attempt heap expansion",
 278                   ergo_format_reason("region allocation request failed")
 279                   ergo_format_byte("allocation request"),
 280                   word_size * HeapWordSize);
 281     if (expand(word_size * HeapWordSize)) {
 282       // Given that expand() succeeded in expanding the heap, and we
 283       // always expand the heap by an amount aligned to the heap
 284       // region size, the free list should in theory not be empty.
 285       // In either case allocate_free_region() will check for NULL.
 286       res = _hrm.allocate_free_region(is_old);
 287     } else {
 288       _expand_heap_after_alloc_failure = false;
 289     }
 290   }
 291   return res;
 292 }
 293 
 294 HeapWord*
 295 G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first,
 296                                                            uint num_regions,
 297                                                            size_t word_size,
 298                                                            AllocationContext_t context) {
 299   assert(first != G1_NO_HRM_INDEX, "pre-condition");
 300   assert(is_humongous(word_size), "word_size should be humongous");
 301   assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");
 302 
 303   // Index of last region in the series + 1.
 304   uint last = first + num_regions;
 305 
 306   // We need to initialize the region(s) we just discovered. This is
 307   // a bit tricky given that it can happen concurrently with
 308   // refinement threads refining cards on these regions and
 309   // potentially wanting to refine the BOT as they are scanning
 310   // those cards (this can happen shortly after a cleanup; see CR
 311   // 6991377). So we have to set up the region(s) carefully and in
 312   // a specific order.
 313 
 314   // The word size sum of all the regions we will allocate.
 315   size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
 316   assert(word_size <= word_size_sum, "sanity");
 317 
 318   // This will be the "starts humongous" region.
 319   HeapRegion* first_hr = region_at(first);
 320   // The header of the new object will be placed at the bottom of
 321   // the first region.
 322   HeapWord* new_obj = first_hr->bottom();
 323   // This will be the new top of the new object.
 324   HeapWord* obj_top = new_obj + word_size;
 325 
 326   // First, we need to zero the header of the space that we will be
 327   // allocating. When we update top further down, some refinement
 328   // threads might try to scan the region. By zeroing the header we
 329   // ensure that any thread that will try to scan the region will
 330   // come across the zero klass word and bail out.
 331   //
 332   // NOTE: It would not have been correct to have used
 333   // CollectedHeap::fill_with_object() and make the space look like
 334   // an int array. The thread that is doing the allocation will
 335   // later update the object header to a potentially different array
 336   // type and, for a very short period of time, the klass and length
 337   // fields will be inconsistent. This could cause a refinement
 338   // thread to calculate the object size incorrectly.
 339   Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
 340 
 341   // We will set up the first region as "starts humongous". This
 342   // will also update the BOT covering all the regions to reflect
 343   // that there is a single object that starts at the bottom of the
 344   // first region.
 345   first_hr->set_starts_humongous(obj_top);
 346   first_hr->set_allocation_context(context);
 347   // Then, if there are any, we will set up the "continues
 348   // humongous" regions.
 349   HeapRegion* hr = NULL;
 350   for (uint i = first + 1; i < last; ++i) {
 351     hr = region_at(i);
 352     hr->set_continues_humongous(first_hr);
 353     hr->set_allocation_context(context);
 354   }
 355 
 356   // Up to this point no concurrent thread would have been able to
 357   // do any scanning on any region in this series. All the top
 358   // fields still point to bottom, so the intersection between
 359   // [bottom,top] and [card_start,card_end] will be empty. Before we
 360   // update the top fields, we'll do a storestore to make sure that
 361   // no thread sees the update to top before the zeroing of the
 362   // object header and the BOT initialization.
 363   OrderAccess::storestore();
 364 
 365   // Now that the BOT and the object header have been initialized,
 366   // we can update top of the "starts humongous" region.
 367   first_hr->set_top(MIN2(first_hr->end(), obj_top));
 368   if (_hr_printer.is_active()) {
 369     _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, first_hr->top());
 370   }
 371 
 372   // Now, we will update the top fields of the "continues humongous"
 373   // regions.
 374   hr = NULL;
 375   for (uint i = first + 1; i < last; ++i) {
 376     hr = region_at(i);
 377     if ((i + 1) == last) {
 378       // last continues humongous region
 379       assert(hr->bottom() < obj_top && obj_top <= hr->end(),
 380              "new_top should fall on this region");
 381       hr->set_top(obj_top);
 382       _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, obj_top);
 383     } else {
 384       // not last one
 385       assert(obj_top > hr->end(), "obj_top should be above this region");
 386       hr->set_top(hr->end());
 387       _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end());
 388     }
 389   }
 390   // If we have continues humongous regions (hr != NULL), its top should
 391   // match obj_top.
 392   assert(hr == NULL || (hr->top() == obj_top), "sanity");
 393   check_bitmaps("Humongous Region Allocation", first_hr);
 394 
 395   increase_used(word_size * HeapWordSize);
 396 
 397   for (uint i = first; i < last; ++i) {
 398     _humongous_set.add(region_at(i));
 399   }
 400 
 401   return new_obj;
 402 }
 403 
 404 // If could fit into free regions w/o expansion, try.
 405 // Otherwise, if can expand, do so.
 406 // Otherwise, if using ex regions might help, try with ex given back.
 407 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) {
 408   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
 409 
 410   verify_region_sets_optional();
 411 
 412   uint first = G1_NO_HRM_INDEX;
 413   uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords);
 414 
 415   if (obj_regions == 1) {
 416     // Only one region to allocate, try to use a fast path by directly allocating
 417     // from the free lists. Do not try to expand here, we will potentially do that
 418     // later.
 419     HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */);
 420     if (hr != NULL) {
 421       first = hr->hrm_index();
 422     }
 423   } else {
 424     // We can't allocate humongous regions spanning more than one region while
 425     // cleanupComplete() is running, since some of the regions we find to be
 426     // empty might not yet be added to the free list. It is not straightforward
 427     // to know in which list they are on so that we can remove them. We only
 428     // need to do this if we need to allocate more than one region to satisfy the
 429     // current humongous allocation request. If we are only allocating one region
 430     // we use the one-region region allocation code (see above), that already
 431     // potentially waits for regions from the secondary free list.
 432     wait_while_free_regions_coming();
 433     append_secondary_free_list_if_not_empty_with_lock();
 434 
 435     // Policy: Try only empty regions (i.e. already committed first). Maybe we
 436     // are lucky enough to find some.
 437     first = _hrm.find_contiguous_only_empty(obj_regions);
 438     if (first != G1_NO_HRM_INDEX) {
 439       _hrm.allocate_free_regions_starting_at(first, obj_regions);
 440     }
 441   }
 442 
 443   if (first == G1_NO_HRM_INDEX) {
 444     // Policy: We could not find enough regions for the humongous object in the
 445     // free list. Look through the heap to find a mix of free and uncommitted regions.
 446     // If so, try expansion.
 447     first = _hrm.find_contiguous_empty_or_unavailable(obj_regions);
 448     if (first != G1_NO_HRM_INDEX) {
 449       // We found something. Make sure these regions are committed, i.e. expand
 450       // the heap. Alternatively we could do a defragmentation GC.
 451       ergo_verbose1(ErgoHeapSizing,
 452                     "attempt heap expansion",
 453                     ergo_format_reason("humongous allocation request failed")
 454                     ergo_format_byte("allocation request"),
 455                     word_size * HeapWordSize);
 456 
 457       _hrm.expand_at(first, obj_regions);
 458       g1_policy()->record_new_heap_size(num_regions());
 459 
 460 #ifdef ASSERT
 461       for (uint i = first; i < first + obj_regions; ++i) {
 462         HeapRegion* hr = region_at(i);
 463         assert(hr->is_free(), "sanity");
 464         assert(hr->is_empty(), "sanity");
 465         assert(is_on_master_free_list(hr), "sanity");
 466       }
 467 #endif
 468       _hrm.allocate_free_regions_starting_at(first, obj_regions);
 469     } else {
 470       // Policy: Potentially trigger a defragmentation GC.
 471     }
 472   }
 473 
 474   HeapWord* result = NULL;
 475   if (first != G1_NO_HRM_INDEX) {
 476     result = humongous_obj_allocate_initialize_regions(first, obj_regions,
 477                                                        word_size, context);
 478     assert(result != NULL, "it should always return a valid result");
 479 
 480     // A successful humongous object allocation changes the used space
 481     // information of the old generation so we need to recalculate the
 482     // sizes and update the jstat counters here.
 483     g1mm()->update_sizes();
 484   }
 485 
 486   verify_region_sets_optional();
 487 
 488   return result;
 489 }
 490 
 491 HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) {
 492   assert_heap_not_locked_and_not_at_safepoint();
 493   assert(!is_humongous(word_size), "we do not allow humongous TLABs");
 494 
 495   uint dummy_gc_count_before;
 496   uint dummy_gclocker_retry_count = 0;
 497   return attempt_allocation(word_size, &dummy_gc_count_before, &dummy_gclocker_retry_count);
 498 }
 499 
 500 HeapWord*
 501 G1CollectedHeap::mem_allocate(size_t word_size,
 502                               bool*  gc_overhead_limit_was_exceeded) {
 503   assert_heap_not_locked_and_not_at_safepoint();
 504 
 505   // Loop until the allocation is satisfied, or unsatisfied after GC.
 506   for (uint try_count = 1, gclocker_retry_count = 0; /* we'll return */; try_count += 1) {
 507     uint gc_count_before;
 508 
 509     HeapWord* result = NULL;
 510     if (!is_humongous(word_size)) {
 511       result = attempt_allocation(word_size, &gc_count_before, &gclocker_retry_count);
 512     } else {
 513       result = attempt_allocation_humongous(word_size, &gc_count_before, &gclocker_retry_count);
 514     }
 515     if (result != NULL) {
 516       return result;
 517     }
 518 
 519     // Create the garbage collection operation...
 520     VM_G1CollectForAllocation op(gc_count_before, word_size);
 521     op.set_allocation_context(AllocationContext::current());
 522 
 523     // ...and get the VM thread to execute it.
 524     VMThread::execute(&op);
 525 
 526     if (op.prologue_succeeded() && op.pause_succeeded()) {
 527       // If the operation was successful we'll return the result even
 528       // if it is NULL. If the allocation attempt failed immediately
 529       // after a Full GC, it's unlikely we'll be able to allocate now.
 530       HeapWord* result = op.result();
 531       if (result != NULL && !is_humongous(word_size)) {
 532         // Allocations that take place on VM operations do not do any
 533         // card dirtying and we have to do it here. We only have to do
 534         // this for non-humongous allocations, though.
 535         dirty_young_block(result, word_size);
 536       }
 537       return result;
 538     } else {
 539       if (gclocker_retry_count > GCLockerRetryAllocationCount) {
 540         return NULL;
 541       }
 542       assert(op.result() == NULL,
 543              "the result should be NULL if the VM op did not succeed");
 544     }
 545 
 546     // Give a warning if we seem to be looping forever.
 547     if ((QueuedAllocationWarningCount > 0) &&
 548         (try_count % QueuedAllocationWarningCount == 0)) {
 549       warning("G1CollectedHeap::mem_allocate retries %d times", try_count);
 550     }
 551   }
 552 
 553   ShouldNotReachHere();
 554   return NULL;
 555 }
 556 
 557 HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size,
 558                                                    AllocationContext_t context,
 559                                                    uint* gc_count_before_ret,
 560                                                    uint* gclocker_retry_count_ret) {
 561   // Make sure you read the note in attempt_allocation_humongous().
 562 
 563   assert_heap_not_locked_and_not_at_safepoint();
 564   assert(!is_humongous(word_size), "attempt_allocation_slow() should not "
 565          "be called for humongous allocation requests");
 566 
 567   // We should only get here after the first-level allocation attempt
 568   // (attempt_allocation()) failed to allocate.
 569 
 570   // We will loop until a) we manage to successfully perform the
 571   // allocation or b) we successfully schedule a collection which
 572   // fails to perform the allocation. b) is the only case when we'll
 573   // return NULL.
 574   HeapWord* result = NULL;
 575   for (int try_count = 1; /* we'll return */; try_count += 1) {
 576     bool should_try_gc;
 577     uint gc_count_before;
 578 
 579     {
 580       MutexLockerEx x(Heap_lock);
 581       result = _allocator->attempt_allocation_locked(word_size, context);
 582       if (result != NULL) {
 583         return result;
 584       }
 585 
 586       if (GC_locker::is_active_and_needs_gc()) {
 587         if (g1_policy()->can_expand_young_list()) {
 588           // No need for an ergo verbose message here,
 589           // can_expand_young_list() does this when it returns true.
 590           result = _allocator->attempt_allocation_force(word_size, context);
 591           if (result != NULL) {
 592             return result;
 593           }
 594         }
 595         should_try_gc = false;
 596       } else {
 597         // The GCLocker may not be active but the GCLocker initiated
 598         // GC may not yet have been performed (GCLocker::needs_gc()
 599         // returns true). In this case we do not try this GC and
 600         // wait until the GCLocker initiated GC is performed, and
 601         // then retry the allocation.
 602         if (GC_locker::needs_gc()) {
 603           should_try_gc = false;
 604         } else {
 605           // Read the GC count while still holding the Heap_lock.
 606           gc_count_before = total_collections();
 607           should_try_gc = true;
 608         }
 609       }
 610     }
 611 
 612     if (should_try_gc) {
 613       bool succeeded;
 614       result = do_collection_pause(word_size, gc_count_before, &succeeded,
 615                                    GCCause::_g1_inc_collection_pause);
 616       if (result != NULL) {
 617         assert(succeeded, "only way to get back a non-NULL result");
 618         return result;
 619       }
 620 
 621       if (succeeded) {
 622         // If we get here we successfully scheduled a collection which
 623         // failed to allocate. No point in trying to allocate
 624         // further. We'll just return NULL.
 625         MutexLockerEx x(Heap_lock);
 626         *gc_count_before_ret = total_collections();
 627         return NULL;
 628       }
 629     } else {
 630       if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) {
 631         MutexLockerEx x(Heap_lock);
 632         *gc_count_before_ret = total_collections();
 633         return NULL;
 634       }
 635       // The GCLocker is either active or the GCLocker initiated
 636       // GC has not yet been performed. Stall until it is and
 637       // then retry the allocation.
 638       GC_locker::stall_until_clear();
 639       (*gclocker_retry_count_ret) += 1;
 640     }
 641 
 642     // We can reach here if we were unsuccessful in scheduling a
 643     // collection (because another thread beat us to it) or if we were
 644     // stalled due to the GC locker. In either can we should retry the
 645     // allocation attempt in case another thread successfully
 646     // performed a collection and reclaimed enough space. We do the
 647     // first attempt (without holding the Heap_lock) here and the
 648     // follow-on attempt will be at the start of the next loop
 649     // iteration (after taking the Heap_lock).
 650     result = _allocator->attempt_allocation(word_size, context);
 651     if (result != NULL) {
 652       return result;
 653     }
 654 
 655     // Give a warning if we seem to be looping forever.
 656     if ((QueuedAllocationWarningCount > 0) &&
 657         (try_count % QueuedAllocationWarningCount == 0)) {
 658       warning("G1CollectedHeap::attempt_allocation_slow() "
 659               "retries %d times", try_count);
 660     }
 661   }
 662 
 663   ShouldNotReachHere();
 664   return NULL;
 665 }
 666 
 667 void G1CollectedHeap::begin_archive_alloc_range() {
 668   assert_at_safepoint(true /* should_be_vm_thread */);
 669   if (_archive_allocator == NULL) {
 670     _archive_allocator = G1ArchiveAllocator::create_allocator(this);
 671   }
 672 }
 673 
 674 bool G1CollectedHeap::is_archive_alloc_too_large(size_t word_size) {
 675   // Allocations in archive regions cannot be of a size that would be considered
 676   // humongous even for a minimum-sized region, because G1 region sizes/boundaries
 677   // may be different at archive-restore time.
 678   return word_size >= humongous_threshold_for(HeapRegion::min_region_size_in_words());
 679 }
 680 
 681 HeapWord* G1CollectedHeap::archive_mem_allocate(size_t word_size) {
 682   assert_at_safepoint(true /* should_be_vm_thread */);
 683   assert(_archive_allocator != NULL, "_archive_allocator not initialized");
 684   if (is_archive_alloc_too_large(word_size)) {
 685     return NULL;
 686   }
 687   return _archive_allocator->archive_mem_allocate(word_size);
 688 }
 689 
 690 void G1CollectedHeap::end_archive_alloc_range(GrowableArray<MemRegion>* ranges,
 691                                               size_t end_alignment_in_bytes) {
 692   assert_at_safepoint(true /* should_be_vm_thread */);
 693   assert(_archive_allocator != NULL, "_archive_allocator not initialized");
 694 
 695   // Call complete_archive to do the real work, filling in the MemRegion
 696   // array with the archive regions.
 697   _archive_allocator->complete_archive(ranges, end_alignment_in_bytes);
 698   delete _archive_allocator;
 699   _archive_allocator = NULL;
 700 }
 701 
 702 bool G1CollectedHeap::check_archive_addresses(MemRegion* ranges, size_t count) {
 703   assert(ranges != NULL, "MemRegion array NULL");
 704   assert(count != 0, "No MemRegions provided");
 705   MemRegion reserved = _hrm.reserved();
 706   for (size_t i = 0; i < count; i++) {
 707     if (!reserved.contains(ranges[i].start()) || !reserved.contains(ranges[i].last())) {
 708       return false;
 709     }
 710   }
 711   return true;
 712 }
 713 
 714 bool G1CollectedHeap::alloc_archive_regions(MemRegion* ranges, size_t count) {
 715   assert(!is_init_completed(), "Expect to be called at JVM init time");
 716   assert(ranges != NULL, "MemRegion array NULL");
 717   assert(count != 0, "No MemRegions provided");
 718   MutexLockerEx x(Heap_lock);
 719 
 720   MemRegion reserved = _hrm.reserved();
 721   HeapWord* prev_last_addr = NULL;
 722   HeapRegion* prev_last_region = NULL;
 723 
 724   // Temporarily disable pretouching of heap pages. This interface is used
 725   // when mmap'ing archived heap data in, so pre-touching is wasted.
 726   FlagSetting fs(AlwaysPreTouch, false);
 727 
 728   // Enable archive object checking in G1MarkSweep. We have to let it know
 729   // about each archive range, so that objects in those ranges aren't marked.
 730   G1MarkSweep::enable_archive_object_check();
 731 
 732   // For each specified MemRegion range, allocate the corresponding G1
 733   // regions and mark them as archive regions. We expect the ranges in
 734   // ascending starting address order, without overlap.
 735   for (size_t i = 0; i < count; i++) {
 736     MemRegion curr_range = ranges[i];
 737     HeapWord* start_address = curr_range.start();
 738     size_t word_size = curr_range.word_size();
 739     HeapWord* last_address = curr_range.last();
 740     size_t commits = 0;
 741 
 742     guarantee(reserved.contains(start_address) && reserved.contains(last_address),
 743               "MemRegion outside of heap [" PTR_FORMAT ", " PTR_FORMAT "]",
 744               p2i(start_address), p2i(last_address));
 745     guarantee(start_address > prev_last_addr,
 746               "Ranges not in ascending order: " PTR_FORMAT " <= " PTR_FORMAT ,
 747               p2i(start_address), p2i(prev_last_addr));
 748     prev_last_addr = last_address;
 749 
 750     // Check for ranges that start in the same G1 region in which the previous
 751     // range ended, and adjust the start address so we don't try to allocate
 752     // the same region again. If the current range is entirely within that
 753     // region, skip it, just adjusting the recorded top.
 754     HeapRegion* start_region = _hrm.addr_to_region(start_address);
 755     if ((prev_last_region != NULL) && (start_region == prev_last_region)) {
 756       start_address = start_region->end();
 757       if (start_address > last_address) {
 758         increase_used(word_size * HeapWordSize);
 759         start_region->set_top(last_address + 1);
 760         continue;
 761       }
 762       start_region->set_top(start_address);
 763       curr_range = MemRegion(start_address, last_address + 1);
 764       start_region = _hrm.addr_to_region(start_address);
 765     }
 766 
 767     // Perform the actual region allocation, exiting if it fails.
 768     // Then note how much new space we have allocated.
 769     if (!_hrm.allocate_containing_regions(curr_range, &commits)) {
 770       return false;
 771     }
 772     increase_used(word_size * HeapWordSize);
 773     if (commits != 0) {
 774       ergo_verbose1(ErgoHeapSizing,
 775                     "attempt heap expansion",
 776                     ergo_format_reason("allocate archive regions")
 777                     ergo_format_byte("total size"),
 778                     HeapRegion::GrainWords * HeapWordSize * commits);
 779     }
 780 
 781     // Mark each G1 region touched by the range as archive, add it to the old set,
 782     // and set the allocation context and top.
 783     HeapRegion* curr_region = _hrm.addr_to_region(start_address);
 784     HeapRegion* last_region = _hrm.addr_to_region(last_address);
 785     prev_last_region = last_region;
 786 
 787     while (curr_region != NULL) {
 788       assert(curr_region->is_empty() && !curr_region->is_pinned(),
 789              "Region already in use (index %u)", curr_region->hrm_index());
 790       _hr_printer.alloc(curr_region, G1HRPrinter::Archive);
 791       curr_region->set_allocation_context(AllocationContext::system());
 792       curr_region->set_archive();
 793       _old_set.add(curr_region);
 794       if (curr_region != last_region) {
 795         curr_region->set_top(curr_region->end());
 796         curr_region = _hrm.next_region_in_heap(curr_region);
 797       } else {
 798         curr_region->set_top(last_address + 1);
 799         curr_region = NULL;
 800       }
 801     }
 802 
 803     // Notify mark-sweep of the archive range.
 804     G1MarkSweep::set_range_archive(curr_range, true);
 805   }
 806   return true;
 807 }
 808 
 809 void G1CollectedHeap::fill_archive_regions(MemRegion* ranges, size_t count) {
 810   assert(!is_init_completed(), "Expect to be called at JVM init time");
 811   assert(ranges != NULL, "MemRegion array NULL");
 812   assert(count != 0, "No MemRegions provided");
 813   MemRegion reserved = _hrm.reserved();
 814   HeapWord *prev_last_addr = NULL;
 815   HeapRegion* prev_last_region = NULL;
 816 
 817   // For each MemRegion, create filler objects, if needed, in the G1 regions
 818   // that contain the address range. The address range actually within the
 819   // MemRegion will not be modified. That is assumed to have been initialized
 820   // elsewhere, probably via an mmap of archived heap data.
 821   MutexLockerEx x(Heap_lock);
 822   for (size_t i = 0; i < count; i++) {
 823     HeapWord* start_address = ranges[i].start();
 824     HeapWord* last_address = ranges[i].last();
 825 
 826     assert(reserved.contains(start_address) && reserved.contains(last_address),
 827            "MemRegion outside of heap [" PTR_FORMAT ", " PTR_FORMAT "]",
 828            p2i(start_address), p2i(last_address));
 829     assert(start_address > prev_last_addr,
 830            "Ranges not in ascending order: " PTR_FORMAT " <= " PTR_FORMAT ,
 831            p2i(start_address), p2i(prev_last_addr));
 832 
 833     HeapRegion* start_region = _hrm.addr_to_region(start_address);
 834     HeapRegion* last_region = _hrm.addr_to_region(last_address);
 835     HeapWord* bottom_address = start_region->bottom();
 836 
 837     // Check for a range beginning in the same region in which the
 838     // previous one ended.
 839     if (start_region == prev_last_region) {
 840       bottom_address = prev_last_addr + 1;
 841     }
 842 
 843     // Verify that the regions were all marked as archive regions by
 844     // alloc_archive_regions.
 845     HeapRegion* curr_region = start_region;
 846     while (curr_region != NULL) {
 847       guarantee(curr_region->is_archive(),
 848                 "Expected archive region at index %u", curr_region->hrm_index());
 849       if (curr_region != last_region) {
 850         curr_region = _hrm.next_region_in_heap(curr_region);
 851       } else {
 852         curr_region = NULL;
 853       }
 854     }
 855 
 856     prev_last_addr = last_address;
 857     prev_last_region = last_region;
 858 
 859     // Fill the memory below the allocated range with dummy object(s),
 860     // if the region bottom does not match the range start, or if the previous
 861     // range ended within the same G1 region, and there is a gap.
 862     if (start_address != bottom_address) {
 863       size_t fill_size = pointer_delta(start_address, bottom_address);
 864       G1CollectedHeap::fill_with_objects(bottom_address, fill_size);
 865       increase_used(fill_size * HeapWordSize);
 866     }
 867   }
 868 }
 869 
 870 inline HeapWord* G1CollectedHeap::attempt_allocation(size_t word_size,
 871                                                      uint* gc_count_before_ret,
 872                                                      uint* gclocker_retry_count_ret) {
 873   assert_heap_not_locked_and_not_at_safepoint();
 874   assert(!is_humongous(word_size), "attempt_allocation() should not "
 875          "be called for humongous allocation requests");
 876 
 877   AllocationContext_t context = AllocationContext::current();
 878   HeapWord* result = _allocator->attempt_allocation(word_size, context);
 879 
 880   if (result == NULL) {
 881     result = attempt_allocation_slow(word_size,
 882                                      context,
 883                                      gc_count_before_ret,
 884                                      gclocker_retry_count_ret);
 885   }
 886   assert_heap_not_locked();
 887   if (result != NULL) {
 888     dirty_young_block(result, word_size);
 889   }
 890   return result;
 891 }
 892 
 893 void G1CollectedHeap::dealloc_archive_regions(MemRegion* ranges, size_t count) {
 894   assert(!is_init_completed(), "Expect to be called at JVM init time");
 895   assert(ranges != NULL, "MemRegion array NULL");
 896   assert(count != 0, "No MemRegions provided");
 897   MemRegion reserved = _hrm.reserved();
 898   HeapWord* prev_last_addr = NULL;
 899   HeapRegion* prev_last_region = NULL;
 900   size_t size_used = 0;
 901   size_t uncommitted_regions = 0;
 902 
 903   // For each Memregion, free the G1 regions that constitute it, and
 904   // notify mark-sweep that the range is no longer to be considered 'archive.'
 905   MutexLockerEx x(Heap_lock);
 906   for (size_t i = 0; i < count; i++) {
 907     HeapWord* start_address = ranges[i].start();
 908     HeapWord* last_address = ranges[i].last();
 909 
 910     assert(reserved.contains(start_address) && reserved.contains(last_address),
 911            "MemRegion outside of heap [" PTR_FORMAT ", " PTR_FORMAT "]",
 912            p2i(start_address), p2i(last_address));
 913     assert(start_address > prev_last_addr,
 914            "Ranges not in ascending order: " PTR_FORMAT " <= " PTR_FORMAT ,
 915            p2i(start_address), p2i(prev_last_addr));
 916     size_used += ranges[i].byte_size();
 917     prev_last_addr = last_address;
 918 
 919     HeapRegion* start_region = _hrm.addr_to_region(start_address);
 920     HeapRegion* last_region = _hrm.addr_to_region(last_address);
 921 
 922     // Check for ranges that start in the same G1 region in which the previous
 923     // range ended, and adjust the start address so we don't try to free
 924     // the same region again. If the current range is entirely within that
 925     // region, skip it.
 926     if (start_region == prev_last_region) {
 927       start_address = start_region->end();
 928       if (start_address > last_address) {
 929         continue;
 930       }
 931       start_region = _hrm.addr_to_region(start_address);
 932     }
 933     prev_last_region = last_region;
 934 
 935     // After verifying that each region was marked as an archive region by
 936     // alloc_archive_regions, set it free and empty and uncommit it.
 937     HeapRegion* curr_region = start_region;
 938     while (curr_region != NULL) {
 939       guarantee(curr_region->is_archive(),
 940                 "Expected archive region at index %u", curr_region->hrm_index());
 941       uint curr_index = curr_region->hrm_index();
 942       _old_set.remove(curr_region);
 943       curr_region->set_free();
 944       curr_region->set_top(curr_region->bottom());
 945       if (curr_region != last_region) {
 946         curr_region = _hrm.next_region_in_heap(curr_region);
 947       } else {
 948         curr_region = NULL;
 949       }
 950       _hrm.shrink_at(curr_index, 1);
 951       uncommitted_regions++;
 952     }
 953 
 954     // Notify mark-sweep that this is no longer an archive range.
 955     G1MarkSweep::set_range_archive(ranges[i], false);
 956   }
 957 
 958   if (uncommitted_regions != 0) {
 959     ergo_verbose1(ErgoHeapSizing,
 960                   "attempt heap shrinking",
 961                   ergo_format_reason("uncommitted archive regions")
 962                   ergo_format_byte("total size"),
 963                   HeapRegion::GrainWords * HeapWordSize * uncommitted_regions);
 964   }
 965   decrease_used(size_used);
 966 }
 967 
 968 HeapWord* G1CollectedHeap::attempt_allocation_humongous(size_t word_size,
 969                                                         uint* gc_count_before_ret,
 970                                                         uint* gclocker_retry_count_ret) {
 971   // The structure of this method has a lot of similarities to
 972   // attempt_allocation_slow(). The reason these two were not merged
 973   // into a single one is that such a method would require several "if
 974   // allocation is not humongous do this, otherwise do that"
 975   // conditional paths which would obscure its flow. In fact, an early
 976   // version of this code did use a unified method which was harder to
 977   // follow and, as a result, it had subtle bugs that were hard to
 978   // track down. So keeping these two methods separate allows each to
 979   // be more readable. It will be good to keep these two in sync as
 980   // much as possible.
 981 
 982   assert_heap_not_locked_and_not_at_safepoint();
 983   assert(is_humongous(word_size), "attempt_allocation_humongous() "
 984          "should only be called for humongous allocations");
 985 
 986   // Humongous objects can exhaust the heap quickly, so we should check if we
 987   // need to start a marking cycle at each humongous object allocation. We do
 988   // the check before we do the actual allocation. The reason for doing it
 989   // before the allocation is that we avoid having to keep track of the newly
 990   // allocated memory while we do a GC.
 991   if (g1_policy()->need_to_start_conc_mark("concurrent humongous allocation",
 992                                            word_size)) {
 993     collect(GCCause::_g1_humongous_allocation);
 994   }
 995 
 996   // We will loop until a) we manage to successfully perform the
 997   // allocation or b) we successfully schedule a collection which
 998   // fails to perform the allocation. b) is the only case when we'll
 999   // return NULL.
1000   HeapWord* result = NULL;
1001   for (int try_count = 1; /* we'll return */; try_count += 1) {
1002     bool should_try_gc;
1003     uint gc_count_before;
1004 
1005     {
1006       MutexLockerEx x(Heap_lock);
1007 
1008       // Given that humongous objects are not allocated in young
1009       // regions, we'll first try to do the allocation without doing a
1010       // collection hoping that there's enough space in the heap.
1011       result = humongous_obj_allocate(word_size, AllocationContext::current());
1012       if (result != NULL) {
1013         return result;
1014       }
1015 
1016       if (GC_locker::is_active_and_needs_gc()) {
1017         should_try_gc = false;
1018       } else {
1019          // The GCLocker may not be active but the GCLocker initiated
1020         // GC may not yet have been performed (GCLocker::needs_gc()
1021         // returns true). In this case we do not try this GC and
1022         // wait until the GCLocker initiated GC is performed, and
1023         // then retry the allocation.
1024         if (GC_locker::needs_gc()) {
1025           should_try_gc = false;
1026         } else {
1027           // Read the GC count while still holding the Heap_lock.
1028           gc_count_before = total_collections();
1029           should_try_gc = true;
1030         }
1031       }
1032     }
1033 
1034     if (should_try_gc) {
1035       // If we failed to allocate the humongous object, we should try to
1036       // do a collection pause (if we're allowed) in case it reclaims
1037       // enough space for the allocation to succeed after the pause.
1038 
1039       bool succeeded;
1040       result = do_collection_pause(word_size, gc_count_before, &succeeded,
1041                                    GCCause::_g1_humongous_allocation);
1042       if (result != NULL) {
1043         assert(succeeded, "only way to get back a non-NULL result");
1044         return result;
1045       }
1046 
1047       if (succeeded) {
1048         // If we get here we successfully scheduled a collection which
1049         // failed to allocate. No point in trying to allocate
1050         // further. We'll just return NULL.
1051         MutexLockerEx x(Heap_lock);
1052         *gc_count_before_ret = total_collections();
1053         return NULL;
1054       }
1055     } else {
1056       if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) {
1057         MutexLockerEx x(Heap_lock);
1058         *gc_count_before_ret = total_collections();
1059         return NULL;
1060       }
1061       // The GCLocker is either active or the GCLocker initiated
1062       // GC has not yet been performed. Stall until it is and
1063       // then retry the allocation.
1064       GC_locker::stall_until_clear();
1065       (*gclocker_retry_count_ret) += 1;
1066     }
1067 
1068     // We can reach here if we were unsuccessful in scheduling a
1069     // collection (because another thread beat us to it) or if we were
1070     // stalled due to the GC locker. In either can we should retry the
1071     // allocation attempt in case another thread successfully
1072     // performed a collection and reclaimed enough space.  Give a
1073     // warning if we seem to be looping forever.
1074 
1075     if ((QueuedAllocationWarningCount > 0) &&
1076         (try_count % QueuedAllocationWarningCount == 0)) {
1077       warning("G1CollectedHeap::attempt_allocation_humongous() "
1078               "retries %d times", try_count);
1079     }
1080   }
1081 
1082   ShouldNotReachHere();
1083   return NULL;
1084 }
1085 
1086 HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size,
1087                                                            AllocationContext_t context,
1088                                                            bool expect_null_mutator_alloc_region) {
1089   assert_at_safepoint(true /* should_be_vm_thread */);
1090   assert(!_allocator->has_mutator_alloc_region(context) || !expect_null_mutator_alloc_region,
1091          "the current alloc region was unexpectedly found to be non-NULL");
1092 
1093   if (!is_humongous(word_size)) {
1094     return _allocator->attempt_allocation_locked(word_size, context);
1095   } else {
1096     HeapWord* result = humongous_obj_allocate(word_size, context);
1097     if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) {
1098       collector_state()->set_initiate_conc_mark_if_possible(true);
1099     }
1100     return result;
1101   }
1102 
1103   ShouldNotReachHere();
1104 }
1105 
1106 class PostMCRemSetClearClosure: public HeapRegionClosure {
1107   G1CollectedHeap* _g1h;
1108   ModRefBarrierSet* _mr_bs;
1109 public:
1110   PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) :
1111     _g1h(g1h), _mr_bs(mr_bs) {}
1112 
1113   bool doHeapRegion(HeapRegion* r) {
1114     HeapRegionRemSet* hrrs = r->rem_set();
1115 
1116     _g1h->reset_gc_time_stamps(r);
1117 
1118     if (r->is_continues_humongous()) {
1119       // We'll assert that the strong code root list and RSet is empty
1120       assert(hrrs->strong_code_roots_list_length() == 0, "sanity");
1121       assert(hrrs->occupied() == 0, "RSet should be empty");
1122     } else {
1123       hrrs->clear();
1124     }
1125     // You might think here that we could clear just the cards
1126     // corresponding to the used region.  But no: if we leave a dirty card
1127     // in a region we might allocate into, then it would prevent that card
1128     // from being enqueued, and cause it to be missed.
1129     // Re: the performance cost: we shouldn't be doing full GC anyway!
1130     _mr_bs->clear(MemRegion(r->bottom(), r->end()));
1131 
1132     return false;
1133   }
1134 };
1135 
1136 void G1CollectedHeap::clear_rsets_post_compaction() {
1137   PostMCRemSetClearClosure rs_clear(this, g1_barrier_set());
1138   heap_region_iterate(&rs_clear);
1139 }
1140 
1141 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
1142   G1CollectedHeap*   _g1h;
1143   UpdateRSOopClosure _cl;
1144 public:
1145   RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, uint worker_i = 0) :
1146     _cl(g1->g1_rem_set(), worker_i),
1147     _g1h(g1)
1148   { }
1149 
1150   bool doHeapRegion(HeapRegion* r) {
1151     if (!r->is_continues_humongous()) {
1152       _cl.set_from(r);
1153       r->oop_iterate(&_cl);
1154     }
1155     return false;
1156   }
1157 };
1158 
1159 class ParRebuildRSTask: public AbstractGangTask {
1160   G1CollectedHeap* _g1;
1161   HeapRegionClaimer _hrclaimer;
1162 
1163 public:
1164   ParRebuildRSTask(G1CollectedHeap* g1) :
1165       AbstractGangTask("ParRebuildRSTask"), _g1(g1), _hrclaimer(g1->workers()->active_workers()) {}
1166 
1167   void work(uint worker_id) {
1168     RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id);
1169     _g1->heap_region_par_iterate(&rebuild_rs, worker_id, &_hrclaimer);
1170   }
1171 };
1172 
1173 class PostCompactionPrinterClosure: public HeapRegionClosure {
1174 private:
1175   G1HRPrinter* _hr_printer;
1176 public:
1177   bool doHeapRegion(HeapRegion* hr) {
1178     assert(!hr->is_young(), "not expecting to find young regions");
1179     if (hr->is_free()) {
1180       // We only generate output for non-empty regions.
1181     } else if (hr->is_starts_humongous()) {
1182       _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous);
1183     } else if (hr->is_continues_humongous()) {
1184       _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous);
1185     } else if (hr->is_archive()) {
1186       _hr_printer->post_compaction(hr, G1HRPrinter::Archive);
1187     } else if (hr->is_old()) {
1188       _hr_printer->post_compaction(hr, G1HRPrinter::Old);
1189     } else {
1190       ShouldNotReachHere();
1191     }
1192     return false;
1193   }
1194 
1195   PostCompactionPrinterClosure(G1HRPrinter* hr_printer)
1196     : _hr_printer(hr_printer) { }
1197 };
1198 
1199 void G1CollectedHeap::print_hrm_post_compaction() {
1200   PostCompactionPrinterClosure cl(hr_printer());
1201   heap_region_iterate(&cl);
1202 }
1203 
1204 bool G1CollectedHeap::do_full_collection(bool explicit_gc,
1205                                          bool clear_all_soft_refs) {
1206   assert_at_safepoint(true /* should_be_vm_thread */);
1207 
1208   if (GC_locker::check_active_before_gc()) {
1209     return false;
1210   }
1211 
1212   STWGCTimer* gc_timer = G1MarkSweep::gc_timer();
1213   gc_timer->register_gc_start();
1214 
1215   SerialOldTracer* gc_tracer = G1MarkSweep::gc_tracer();
1216   GCIdMark gc_id_mark;
1217   gc_tracer->report_gc_start(gc_cause(), gc_timer->gc_start());
1218 
1219   SvcGCMarker sgcm(SvcGCMarker::FULL);
1220   ResourceMark rm;
1221 
1222   G1Log::update_level();
1223   print_heap_before_gc();
1224   trace_heap_before_gc(gc_tracer);
1225 
1226   size_t metadata_prev_used = MetaspaceAux::used_bytes();
1227 
1228   verify_region_sets_optional();
1229 
1230   const bool do_clear_all_soft_refs = clear_all_soft_refs ||
1231                            collector_policy()->should_clear_all_soft_refs();
1232 
1233   ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy());
1234 
1235   {
1236     IsGCActiveMark x;
1237 
1238     // Timing
1239     assert(!GCCause::is_user_requested_gc(gc_cause()) || explicit_gc, "invariant");
1240     TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty);
1241 
1242     {
1243       GCTraceTime t(GCCauseString("Full GC", gc_cause()), G1Log::fine(), true, NULL);
1244       TraceCollectorStats tcs(g1mm()->full_collection_counters());
1245       TraceMemoryManagerStats tms(true /* fullGC */, gc_cause());
1246 
1247       g1_policy()->record_full_collection_start();
1248 
1249       // Note: When we have a more flexible GC logging framework that
1250       // allows us to add optional attributes to a GC log record we
1251       // could consider timing and reporting how long we wait in the
1252       // following two methods.
1253       wait_while_free_regions_coming();
1254       // If we start the compaction before the CM threads finish
1255       // scanning the root regions we might trip them over as we'll
1256       // be moving objects / updating references. So let's wait until
1257       // they are done. By telling them to abort, they should complete
1258       // early.
1259       _cm->root_regions()->abort();
1260       _cm->root_regions()->wait_until_scan_finished();
1261       append_secondary_free_list_if_not_empty_with_lock();
1262 
1263       gc_prologue(true);
1264       increment_total_collections(true /* full gc */);
1265       increment_old_marking_cycles_started();
1266 
1267       assert(used() == recalculate_used(), "Should be equal");
1268 
1269       verify_before_gc();
1270 
1271       check_bitmaps("Full GC Start");
1272       pre_full_gc_dump(gc_timer);
1273 
1274 #if defined(COMPILER2) || INCLUDE_JVMCI
1275       DerivedPointerTable::clear();
1276 #endif
1277 
1278       // Disable discovery and empty the discovered lists
1279       // for the CM ref processor.
1280       ref_processor_cm()->disable_discovery();
1281       ref_processor_cm()->abandon_partial_discovery();
1282       ref_processor_cm()->verify_no_references_recorded();
1283 
1284       // Abandon current iterations of concurrent marking and concurrent
1285       // refinement, if any are in progress. We have to do this before
1286       // wait_until_scan_finished() below.
1287       concurrent_mark()->abort();
1288 
1289       // Make sure we'll choose a new allocation region afterwards.
1290       _allocator->release_mutator_alloc_region();
1291       _allocator->abandon_gc_alloc_regions();
1292       g1_rem_set()->cleanupHRRS();
1293 
1294       // We should call this after we retire any currently active alloc
1295       // regions so that all the ALLOC / RETIRE events are generated
1296       // before the start GC event.
1297       _hr_printer.start_gc(true /* full */, (size_t) total_collections());
1298 
1299       // We may have added regions to the current incremental collection
1300       // set between the last GC or pause and now. We need to clear the
1301       // incremental collection set and then start rebuilding it afresh
1302       // after this full GC.
1303       abandon_collection_set(g1_policy()->inc_cset_head());
1304       g1_policy()->clear_incremental_cset();
1305       g1_policy()->stop_incremental_cset_building();
1306 
1307       tear_down_region_sets(false /* free_list_only */);
1308       collector_state()->set_gcs_are_young(true);
1309 
1310       // See the comments in g1CollectedHeap.hpp and
1311       // G1CollectedHeap::ref_processing_init() about
1312       // how reference processing currently works in G1.
1313 
1314       // Temporarily make discovery by the STW ref processor single threaded (non-MT).
1315       ReferenceProcessorMTDiscoveryMutator stw_rp_disc_ser(ref_processor_stw(), false);
1316 
1317       // Temporarily clear the STW ref processor's _is_alive_non_header field.
1318       ReferenceProcessorIsAliveMutator stw_rp_is_alive_null(ref_processor_stw(), NULL);
1319 
1320       ref_processor_stw()->enable_discovery();
1321       ref_processor_stw()->setup_policy(do_clear_all_soft_refs);
1322 
1323       // Do collection work
1324       {
1325         HandleMark hm;  // Discard invalid handles created during gc
1326         G1MarkSweep::invoke_at_safepoint(ref_processor_stw(), do_clear_all_soft_refs);
1327       }
1328 
1329       assert(num_free_regions() == 0, "we should not have added any free regions");
1330       rebuild_region_sets(false /* free_list_only */);
1331 
1332       // Enqueue any discovered reference objects that have
1333       // not been removed from the discovered lists.
1334       ref_processor_stw()->enqueue_discovered_references();
1335 
1336 #if defined(COMPILER2) || INCLUDE_JVMCI
1337       DerivedPointerTable::update_pointers();
1338 #endif
1339 
1340       MemoryService::track_memory_usage();
1341 
1342       assert(!ref_processor_stw()->discovery_enabled(), "Postcondition");
1343       ref_processor_stw()->verify_no_references_recorded();
1344 
1345       // Delete metaspaces for unloaded class loaders and clean up loader_data graph
1346       ClassLoaderDataGraph::purge();
1347       MetaspaceAux::verify_metrics();
1348 
1349       // Note: since we've just done a full GC, concurrent
1350       // marking is no longer active. Therefore we need not
1351       // re-enable reference discovery for the CM ref processor.
1352       // That will be done at the start of the next marking cycle.
1353       assert(!ref_processor_cm()->discovery_enabled(), "Postcondition");
1354       ref_processor_cm()->verify_no_references_recorded();
1355 
1356       reset_gc_time_stamp();
1357       // Since everything potentially moved, we will clear all remembered
1358       // sets, and clear all cards.  Later we will rebuild remembered
1359       // sets. We will also reset the GC time stamps of the regions.
1360       clear_rsets_post_compaction();
1361       check_gc_time_stamps();
1362 
1363       resize_if_necessary_after_full_collection();
1364 
1365       if (_hr_printer.is_active()) {
1366         // We should do this after we potentially resize the heap so
1367         // that all the COMMIT / UNCOMMIT events are generated before
1368         // the end GC event.
1369 
1370         print_hrm_post_compaction();
1371         _hr_printer.end_gc(true /* full */, (size_t) total_collections());
1372       }
1373 
1374       G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
1375       if (hot_card_cache->use_cache()) {
1376         hot_card_cache->reset_card_counts();
1377         hot_card_cache->reset_hot_cache();
1378       }
1379 
1380       // Rebuild remembered sets of all regions.
1381       uint n_workers =
1382         AdaptiveSizePolicy::calc_active_workers(workers()->total_workers(),
1383                                                 workers()->active_workers(),
1384                                                 Threads::number_of_non_daemon_threads());
1385       workers()->set_active_workers(n_workers);
1386 
1387       ParRebuildRSTask rebuild_rs_task(this);
1388       workers()->run_task(&rebuild_rs_task);
1389 
1390       // Rebuild the strong code root lists for each region
1391       rebuild_strong_code_roots();
1392 
1393       if (true) { // FIXME
1394         MetaspaceGC::compute_new_size();
1395       }
1396 
1397 #ifdef TRACESPINNING
1398       ParallelTaskTerminator::print_termination_counts();
1399 #endif
1400 
1401       // Discard all rset updates
1402       JavaThread::dirty_card_queue_set().abandon_logs();
1403       assert(dirty_card_queue_set().completed_buffers_num() == 0, "DCQS should be empty");
1404 
1405       _young_list->reset_sampled_info();
1406       // At this point there should be no regions in the
1407       // entire heap tagged as young.
1408       assert(check_young_list_empty(true /* check_heap */),
1409              "young list should be empty at this point");
1410 
1411       // Update the number of full collections that have been completed.
1412       increment_old_marking_cycles_completed(false /* concurrent */);
1413 
1414       _hrm.verify_optional();
1415       verify_region_sets_optional();
1416 
1417       verify_after_gc();
1418 
1419       // Clear the previous marking bitmap, if needed for bitmap verification.
1420       // Note we cannot do this when we clear the next marking bitmap in
1421       // ConcurrentMark::abort() above since VerifyDuringGC verifies the
1422       // objects marked during a full GC against the previous bitmap.
1423       // But we need to clear it before calling check_bitmaps below since
1424       // the full GC has compacted objects and updated TAMS but not updated
1425       // the prev bitmap.
1426       if (G1VerifyBitmaps) {
1427         ((CMBitMap*) concurrent_mark()->prevMarkBitMap())->clearAll();
1428       }
1429       check_bitmaps("Full GC End");
1430 
1431       // Start a new incremental collection set for the next pause
1432       assert(g1_policy()->collection_set() == NULL, "must be");
1433       g1_policy()->start_incremental_cset_building();
1434 
1435       clear_cset_fast_test();
1436 
1437       _allocator->init_mutator_alloc_region();
1438 
1439       g1_policy()->record_full_collection_end();
1440 
1441       if (G1Log::fine()) {
1442         g1_policy()->print_heap_transition();
1443       }
1444 
1445       // We must call G1MonitoringSupport::update_sizes() in the same scoping level
1446       // as an active TraceMemoryManagerStats object (i.e. before the destructor for the
1447       // TraceMemoryManagerStats is called) so that the G1 memory pools are updated
1448       // before any GC notifications are raised.
1449       g1mm()->update_sizes();
1450 
1451       gc_epilogue(true);
1452     }
1453 
1454     if (G1Log::finer()) {
1455       g1_policy()->print_detailed_heap_transition(true /* full */);
1456     }
1457 
1458     print_heap_after_gc();
1459     trace_heap_after_gc(gc_tracer);
1460 
1461     post_full_gc_dump(gc_timer);
1462 
1463     gc_timer->register_gc_end();
1464     gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions());
1465   }
1466 
1467   return true;
1468 }
1469 
1470 void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {
1471   // Currently, there is no facility in the do_full_collection(bool) API to notify
1472   // the caller that the collection did not succeed (e.g., because it was locked
1473   // out by the GC locker). So, right now, we'll ignore the return value.
1474   bool dummy = do_full_collection(true,                /* explicit_gc */
1475                                   clear_all_soft_refs);
1476 }
1477 
1478 // This code is mostly copied from CardGeneration::compute_new_size.
1479 void
1480 G1CollectedHeap::
1481 resize_if_necessary_after_full_collection() {
1482   // Include bytes that will be pre-allocated to support collections, as "used".
1483   const size_t used_after_gc = used();
1484   const size_t capacity_after_gc = capacity();
1485   const size_t free_after_gc = capacity_after_gc - used_after_gc;
1486 
1487   // This is enforced in arguments.cpp.
1488   assert(MinHeapFreeRatio <= MaxHeapFreeRatio,
1489          "otherwise the code below doesn't make sense");
1490 
1491   // We don't have floating point command-line arguments
1492   const double minimum_free_percentage = (double) MinHeapFreeRatio / 100.0;
1493   const double maximum_used_percentage = 1.0 - minimum_free_percentage;
1494   const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100.0;
1495   const double minimum_used_percentage = 1.0 - maximum_free_percentage;
1496 
1497   const size_t min_heap_size = collector_policy()->min_heap_byte_size();
1498   const size_t max_heap_size = collector_policy()->max_heap_byte_size();
1499 
1500   // We have to be careful here as these two calculations can overflow
1501   // 32-bit size_t's.
1502   double used_after_gc_d = (double) used_after_gc;
1503   double minimum_desired_capacity_d = used_after_gc_d / maximum_used_percentage;
1504   double maximum_desired_capacity_d = used_after_gc_d / minimum_used_percentage;
1505 
1506   // Let's make sure that they are both under the max heap size, which
1507   // by default will make them fit into a size_t.
1508   double desired_capacity_upper_bound = (double) max_heap_size;
1509   minimum_desired_capacity_d = MIN2(minimum_desired_capacity_d,
1510                                     desired_capacity_upper_bound);
1511   maximum_desired_capacity_d = MIN2(maximum_desired_capacity_d,
1512                                     desired_capacity_upper_bound);
1513 
1514   // We can now safely turn them into size_t's.
1515   size_t minimum_desired_capacity = (size_t) minimum_desired_capacity_d;
1516   size_t maximum_desired_capacity = (size_t) maximum_desired_capacity_d;
1517 
1518   // This assert only makes sense here, before we adjust them
1519   // with respect to the min and max heap size.
1520   assert(minimum_desired_capacity <= maximum_desired_capacity,
1521          "minimum_desired_capacity = " SIZE_FORMAT ", "
1522          "maximum_desired_capacity = " SIZE_FORMAT,
1523          minimum_desired_capacity, maximum_desired_capacity);
1524 
1525   // Should not be greater than the heap max size. No need to adjust
1526   // it with respect to the heap min size as it's a lower bound (i.e.,
1527   // we'll try to make the capacity larger than it, not smaller).
1528   minimum_desired_capacity = MIN2(minimum_desired_capacity, max_heap_size);
1529   // Should not be less than the heap min size. No need to adjust it
1530   // with respect to the heap max size as it's an upper bound (i.e.,
1531   // we'll try to make the capacity smaller than it, not greater).
1532   maximum_desired_capacity =  MAX2(maximum_desired_capacity, min_heap_size);
1533 
1534   if (capacity_after_gc < minimum_desired_capacity) {
1535     // Don't expand unless it's significant
1536     size_t expand_bytes = minimum_desired_capacity - capacity_after_gc;
1537     ergo_verbose4(ErgoHeapSizing,
1538                   "attempt heap expansion",
1539                   ergo_format_reason("capacity lower than "
1540                                      "min desired capacity after Full GC")
1541                   ergo_format_byte("capacity")
1542                   ergo_format_byte("occupancy")
1543                   ergo_format_byte_perc("min desired capacity"),
1544                   capacity_after_gc, used_after_gc,
1545                   minimum_desired_capacity, (double) MinHeapFreeRatio);
1546     expand(expand_bytes);
1547 
1548     // No expansion, now see if we want to shrink
1549   } else if (capacity_after_gc > maximum_desired_capacity) {
1550     // Capacity too large, compute shrinking size
1551     size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity;
1552     ergo_verbose4(ErgoHeapSizing,
1553                   "attempt heap shrinking",
1554                   ergo_format_reason("capacity higher than "
1555                                      "max desired capacity after Full GC")
1556                   ergo_format_byte("capacity")
1557                   ergo_format_byte("occupancy")
1558                   ergo_format_byte_perc("max desired capacity"),
1559                   capacity_after_gc, used_after_gc,
1560                   maximum_desired_capacity, (double) MaxHeapFreeRatio);
1561     shrink(shrink_bytes);
1562   }
1563 }
1564 
1565 HeapWord* G1CollectedHeap::satisfy_failed_allocation_helper(size_t word_size,
1566                                                             AllocationContext_t context,
1567                                                             bool do_gc,
1568                                                             bool clear_all_soft_refs,
1569                                                             bool expect_null_mutator_alloc_region,
1570                                                             bool* gc_succeeded) {
1571   *gc_succeeded = true;
1572   // Let's attempt the allocation first.
1573   HeapWord* result =
1574     attempt_allocation_at_safepoint(word_size,
1575                                     context,
1576                                     expect_null_mutator_alloc_region);
1577   if (result != NULL) {
1578     assert(*gc_succeeded, "sanity");
1579     return result;
1580   }
1581 
1582   // In a G1 heap, we're supposed to keep allocation from failing by
1583   // incremental pauses.  Therefore, at least for now, we'll favor
1584   // expansion over collection.  (This might change in the future if we can
1585   // do something smarter than full collection to satisfy a failed alloc.)
1586   result = expand_and_allocate(word_size, context);
1587   if (result != NULL) {
1588     assert(*gc_succeeded, "sanity");
1589     return result;
1590   }
1591 
1592   if (do_gc) {
1593     // Expansion didn't work, we'll try to do a Full GC.
1594     *gc_succeeded = do_full_collection(false, /* explicit_gc */
1595                                        clear_all_soft_refs);
1596   }
1597 
1598   return NULL;
1599 }
1600 
1601 HeapWord* G1CollectedHeap::satisfy_failed_allocation(size_t word_size,
1602                                                      AllocationContext_t context,
1603                                                      bool* succeeded) {
1604   assert_at_safepoint(true /* should_be_vm_thread */);
1605 
1606   // Attempts to allocate followed by Full GC.
1607   HeapWord* result =
1608     satisfy_failed_allocation_helper(word_size,
1609                                      context,
1610                                      true,  /* do_gc */
1611                                      false, /* clear_all_soft_refs */
1612                                      false, /* expect_null_mutator_alloc_region */
1613                                      succeeded);
1614 
1615   if (result != NULL || !*succeeded) {
1616     return result;
1617   }
1618 
1619   // Attempts to allocate followed by Full GC that will collect all soft references.
1620   result = satisfy_failed_allocation_helper(word_size,
1621                                             context,
1622                                             true, /* do_gc */
1623                                             true, /* clear_all_soft_refs */
1624                                             true, /* expect_null_mutator_alloc_region */
1625                                             succeeded);
1626 
1627   if (result != NULL || !*succeeded) {
1628     return result;
1629   }
1630 
1631   // Attempts to allocate, no GC
1632   result = satisfy_failed_allocation_helper(word_size,
1633                                             context,
1634                                             false, /* do_gc */
1635                                             false, /* clear_all_soft_refs */
1636                                             true,  /* expect_null_mutator_alloc_region */
1637                                             succeeded);
1638 
1639   if (result != NULL) {
1640     assert(*succeeded, "sanity");
1641     return result;
1642   }
1643 
1644   assert(!collector_policy()->should_clear_all_soft_refs(),
1645          "Flag should have been handled and cleared prior to this point");
1646 
1647   // What else?  We might try synchronous finalization later.  If the total
1648   // space available is large enough for the allocation, then a more
1649   // complete compaction phase than we've tried so far might be
1650   // appropriate.
1651   assert(*succeeded, "sanity");
1652   return NULL;
1653 }
1654 
1655 // Attempting to expand the heap sufficiently
1656 // to support an allocation of the given "word_size".  If
1657 // successful, perform the allocation and return the address of the
1658 // allocated block, or else "NULL".
1659 
1660 HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size, AllocationContext_t context) {
1661   assert_at_safepoint(true /* should_be_vm_thread */);
1662 
1663   verify_region_sets_optional();
1664 
1665   size_t expand_bytes = MAX2(word_size * HeapWordSize, MinHeapDeltaBytes);
1666   ergo_verbose1(ErgoHeapSizing,
1667                 "attempt heap expansion",
1668                 ergo_format_reason("allocation request failed")
1669                 ergo_format_byte("allocation request"),
1670                 word_size * HeapWordSize);
1671   if (expand(expand_bytes)) {
1672     _hrm.verify_optional();
1673     verify_region_sets_optional();
1674     return attempt_allocation_at_safepoint(word_size,
1675                                            context,
1676                                            false /* expect_null_mutator_alloc_region */);
1677   }
1678   return NULL;
1679 }
1680 
1681 bool G1CollectedHeap::expand(size_t expand_bytes, double* expand_time_ms) {
1682   size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes);
1683   aligned_expand_bytes = align_size_up(aligned_expand_bytes,
1684                                        HeapRegion::GrainBytes);
1685   ergo_verbose2(ErgoHeapSizing,
1686                 "expand the heap",
1687                 ergo_format_byte("requested expansion amount")
1688                 ergo_format_byte("attempted expansion amount"),
1689                 expand_bytes, aligned_expand_bytes);
1690 
1691   if (is_maximal_no_gc()) {
1692     ergo_verbose0(ErgoHeapSizing,
1693                       "did not expand the heap",
1694                       ergo_format_reason("heap already fully expanded"));
1695     return false;
1696   }
1697 
1698   double expand_heap_start_time_sec = os::elapsedTime();
1699   uint regions_to_expand = (uint)(aligned_expand_bytes / HeapRegion::GrainBytes);
1700   assert(regions_to_expand > 0, "Must expand by at least one region");
1701 
1702   uint expanded_by = _hrm.expand_by(regions_to_expand);
1703   if (expand_time_ms != NULL) {
1704     *expand_time_ms = (os::elapsedTime() - expand_heap_start_time_sec) * MILLIUNITS;
1705   }
1706 
1707   if (expanded_by > 0) {
1708     size_t actual_expand_bytes = expanded_by * HeapRegion::GrainBytes;
1709     assert(actual_expand_bytes <= aligned_expand_bytes, "post-condition");
1710     g1_policy()->record_new_heap_size(num_regions());
1711   } else {
1712     ergo_verbose0(ErgoHeapSizing,
1713                   "did not expand the heap",
1714                   ergo_format_reason("heap expansion operation failed"));
1715     // The expansion of the virtual storage space was unsuccessful.
1716     // Let's see if it was because we ran out of swap.
1717     if (G1ExitOnExpansionFailure &&
1718         _hrm.available() >= regions_to_expand) {
1719       // We had head room...
1720       vm_exit_out_of_memory(aligned_expand_bytes, OOM_MMAP_ERROR, "G1 heap expansion");
1721     }
1722   }
1723   return regions_to_expand > 0;
1724 }
1725 
1726 void G1CollectedHeap::shrink_helper(size_t shrink_bytes) {
1727   size_t aligned_shrink_bytes =
1728     ReservedSpace::page_align_size_down(shrink_bytes);
1729   aligned_shrink_bytes = align_size_down(aligned_shrink_bytes,
1730                                          HeapRegion::GrainBytes);
1731   uint num_regions_to_remove = (uint)(shrink_bytes / HeapRegion::GrainBytes);
1732 
1733   uint num_regions_removed = _hrm.shrink_by(num_regions_to_remove);
1734   size_t shrunk_bytes = num_regions_removed * HeapRegion::GrainBytes;
1735 
1736   ergo_verbose3(ErgoHeapSizing,
1737                 "shrink the heap",
1738                 ergo_format_byte("requested shrinking amount")
1739                 ergo_format_byte("aligned shrinking amount")
1740                 ergo_format_byte("attempted shrinking amount"),
1741                 shrink_bytes, aligned_shrink_bytes, shrunk_bytes);
1742   if (num_regions_removed > 0) {
1743     g1_policy()->record_new_heap_size(num_regions());
1744   } else {
1745     ergo_verbose0(ErgoHeapSizing,
1746                   "did not shrink the heap",
1747                   ergo_format_reason("heap shrinking operation failed"));
1748   }
1749 }
1750 
1751 void G1CollectedHeap::shrink(size_t shrink_bytes) {
1752   verify_region_sets_optional();
1753 
1754   // We should only reach here at the end of a Full GC which means we
1755   // should not not be holding to any GC alloc regions. The method
1756   // below will make sure of that and do any remaining clean up.
1757   _allocator->abandon_gc_alloc_regions();
1758 
1759   // Instead of tearing down / rebuilding the free lists here, we
1760   // could instead use the remove_all_pending() method on free_list to
1761   // remove only the ones that we need to remove.
1762   tear_down_region_sets(true /* free_list_only */);
1763   shrink_helper(shrink_bytes);
1764   rebuild_region_sets(true /* free_list_only */);
1765 
1766   _hrm.verify_optional();
1767   verify_region_sets_optional();
1768 }
1769 
1770 // Public methods.
1771 
1772 G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) :
1773   CollectedHeap(),
1774   _g1_policy(policy_),
1775   _dirty_card_queue_set(false),
1776   _is_alive_closure_cm(this),
1777   _is_alive_closure_stw(this),
1778   _ref_processor_cm(NULL),
1779   _ref_processor_stw(NULL),
1780   _bot_shared(NULL),
1781   _cg1r(NULL),
1782   _g1mm(NULL),
1783   _refine_cte_cl(NULL),
1784   _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()),
1785   _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()),
1786   _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()),
1787   _humongous_reclaim_candidates(),
1788   _has_humongous_reclaim_candidates(false),
1789   _archive_allocator(NULL),
1790   _free_regions_coming(false),
1791   _young_list(new YoungList(this)),
1792   _gc_time_stamp(0),
1793   _summary_bytes_used(0),
1794   _survivor_evac_stats(YoungPLABSize, PLABWeight),
1795   _old_evac_stats(OldPLABSize, PLABWeight),
1796   _expand_heap_after_alloc_failure(true),
1797   _old_marking_cycles_started(0),
1798   _old_marking_cycles_completed(0),
1799   _heap_summary_sent(false),
1800   _in_cset_fast_test(),
1801   _dirty_cards_region_list(NULL),
1802   _worker_cset_start_region(NULL),
1803   _worker_cset_start_region_time_stamp(NULL),
1804   _gc_timer_stw(new (ResourceObj::C_HEAP, mtGC) STWGCTimer()),
1805   _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
1806   _gc_tracer_stw(new (ResourceObj::C_HEAP, mtGC) G1NewTracer()),
1807   _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) G1OldTracer()) {
1808 
1809   _workers = new WorkGang("GC Thread", ParallelGCThreads,
1810                           /* are_GC_task_threads */true,
1811                           /* are_ConcurrentGC_threads */false);
1812   _workers->initialize_workers();
1813 
1814   _allocator = G1Allocator::create_allocator(this);
1815   _humongous_object_threshold_in_words = humongous_threshold_for(HeapRegion::GrainWords);
1816 
1817   // Override the default _filler_array_max_size so that no humongous filler
1818   // objects are created.
1819   _filler_array_max_size = _humongous_object_threshold_in_words;
1820 
1821   uint n_queues = ParallelGCThreads;
1822   _task_queues = new RefToScanQueueSet(n_queues);
1823 
1824   uint n_rem_sets = HeapRegionRemSet::num_par_rem_sets();
1825   assert(n_rem_sets > 0, "Invariant.");
1826 
1827   _worker_cset_start_region = NEW_C_HEAP_ARRAY(HeapRegion*, n_queues, mtGC);
1828   _worker_cset_start_region_time_stamp = NEW_C_HEAP_ARRAY(uint, n_queues, mtGC);
1829   _evacuation_failed_info_array = NEW_C_HEAP_ARRAY(EvacuationFailedInfo, n_queues, mtGC);
1830 
1831   for (uint i = 0; i < n_queues; i++) {
1832     RefToScanQueue* q = new RefToScanQueue();
1833     q->initialize();
1834     _task_queues->register_queue(i, q);
1835     ::new (&_evacuation_failed_info_array[i]) EvacuationFailedInfo();
1836   }
1837   clear_cset_start_regions();
1838 
1839   // Initialize the G1EvacuationFailureALot counters and flags.
1840   NOT_PRODUCT(reset_evacuation_should_fail();)
1841 
1842   guarantee(_task_queues != NULL, "task_queues allocation failure.");
1843 }
1844 
1845 G1RegionToSpaceMapper* G1CollectedHeap::create_aux_memory_mapper(const char* description,
1846                                                                  size_t size,
1847                                                                  size_t translation_factor) {
1848   size_t preferred_page_size = os::page_size_for_region_unaligned(size, 1);
1849   // Allocate a new reserved space, preferring to use large pages.
1850   ReservedSpace rs(size, preferred_page_size);
1851   G1RegionToSpaceMapper* result  =
1852     G1RegionToSpaceMapper::create_mapper(rs,
1853                                          size,
1854                                          rs.alignment(),
1855                                          HeapRegion::GrainBytes,
1856                                          translation_factor,
1857                                          mtGC);
1858   if (TracePageSizes) {
1859     gclog_or_tty->print_cr("G1 '%s': pg_sz=" SIZE_FORMAT " base=" PTR_FORMAT " size=" SIZE_FORMAT " alignment=" SIZE_FORMAT " reqsize=" SIZE_FORMAT,
1860                            description, preferred_page_size, p2i(rs.base()), rs.size(), rs.alignment(), size);
1861   }
1862   return result;
1863 }
1864 
1865 jint G1CollectedHeap::initialize() {
1866   CollectedHeap::pre_initialize();
1867   os::enable_vtime();
1868 
1869   G1Log::init();
1870 
1871   // Necessary to satisfy locking discipline assertions.
1872 
1873   MutexLocker x(Heap_lock);
1874 
1875   // We have to initialize the printer before committing the heap, as
1876   // it will be used then.
1877   _hr_printer.set_active(G1PrintHeapRegions);
1878 
1879   // While there are no constraints in the GC code that HeapWordSize
1880   // be any particular value, there are multiple other areas in the
1881   // system which believe this to be true (e.g. oop->object_size in some
1882   // cases incorrectly returns the size in wordSize units rather than
1883   // HeapWordSize).
1884   guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize");
1885 
1886   size_t init_byte_size = collector_policy()->initial_heap_byte_size();
1887   size_t max_byte_size = collector_policy()->max_heap_byte_size();
1888   size_t heap_alignment = collector_policy()->heap_alignment();
1889 
1890   // Ensure that the sizes are properly aligned.
1891   Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap");
1892   Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap");
1893   Universe::check_alignment(max_byte_size, heap_alignment, "g1 heap");
1894 
1895   _refine_cte_cl = new RefineCardTableEntryClosure();
1896 
1897   jint ecode = JNI_OK;
1898   _cg1r = ConcurrentG1Refine::create(this, _refine_cte_cl, &ecode);
1899   if (_cg1r == NULL) {
1900     return ecode;
1901   }
1902 
1903   // Reserve the maximum.
1904 
1905   // When compressed oops are enabled, the preferred heap base
1906   // is calculated by subtracting the requested size from the
1907   // 32Gb boundary and using the result as the base address for
1908   // heap reservation. If the requested size is not aligned to
1909   // HeapRegion::GrainBytes (i.e. the alignment that is passed
1910   // into the ReservedHeapSpace constructor) then the actual
1911   // base of the reserved heap may end up differing from the
1912   // address that was requested (i.e. the preferred heap base).
1913   // If this happens then we could end up using a non-optimal
1914   // compressed oops mode.
1915 
1916   ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size,
1917                                                  heap_alignment);
1918 
1919   initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*)(heap_rs.base() + heap_rs.size()));
1920 
1921   // Create the barrier set for the entire reserved region.
1922   G1SATBCardTableLoggingModRefBS* bs
1923     = new G1SATBCardTableLoggingModRefBS(reserved_region());
1924   bs->initialize();
1925   assert(bs->is_a(BarrierSet::G1SATBCTLogging), "sanity");
1926   set_barrier_set(bs);
1927 
1928   // Also create a G1 rem set.
1929   _g1_rem_set = new G1RemSet(this, g1_barrier_set());
1930 
1931   // Carve out the G1 part of the heap.
1932 
1933   ReservedSpace g1_rs = heap_rs.first_part(max_byte_size);
1934   size_t page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
1935   G1RegionToSpaceMapper* heap_storage =
1936     G1RegionToSpaceMapper::create_mapper(g1_rs,
1937                                          g1_rs.size(),
1938                                          page_size,
1939                                          HeapRegion::GrainBytes,
1940                                          1,
1941                                          mtJavaHeap);
1942   os::trace_page_sizes("G1 Heap", collector_policy()->min_heap_byte_size(),
1943                        max_byte_size, page_size,
1944                        heap_rs.base(),
1945                        heap_rs.size());
1946   heap_storage->set_mapping_changed_listener(&_listener);
1947 
1948   // Create storage for the BOT, card table, card counts table (hot card cache) and the bitmaps.
1949   G1RegionToSpaceMapper* bot_storage =
1950     create_aux_memory_mapper("Block offset table",
1951                              G1BlockOffsetSharedArray::compute_size(g1_rs.size() / HeapWordSize),
1952                              G1BlockOffsetSharedArray::heap_map_factor());
1953 
1954   ReservedSpace cardtable_rs(G1SATBCardTableLoggingModRefBS::compute_size(g1_rs.size() / HeapWordSize));
1955   G1RegionToSpaceMapper* cardtable_storage =
1956     create_aux_memory_mapper("Card table",
1957                              G1SATBCardTableLoggingModRefBS::compute_size(g1_rs.size() / HeapWordSize),
1958                              G1SATBCardTableLoggingModRefBS::heap_map_factor());
1959 
1960   G1RegionToSpaceMapper* card_counts_storage =
1961     create_aux_memory_mapper("Card counts table",
1962                              G1CardCounts::compute_size(g1_rs.size() / HeapWordSize),
1963                              G1CardCounts::heap_map_factor());
1964 
1965   size_t bitmap_size = CMBitMap::compute_size(g1_rs.size());
1966   G1RegionToSpaceMapper* prev_bitmap_storage =
1967     create_aux_memory_mapper("Prev Bitmap", bitmap_size, CMBitMap::heap_map_factor());
1968   G1RegionToSpaceMapper* next_bitmap_storage =
1969     create_aux_memory_mapper("Next Bitmap", bitmap_size, CMBitMap::heap_map_factor());
1970 
1971   _hrm.initialize(heap_storage, prev_bitmap_storage, next_bitmap_storage, bot_storage, cardtable_storage, card_counts_storage);
1972   g1_barrier_set()->initialize(cardtable_storage);
1973    // Do later initialization work for concurrent refinement.
1974   _cg1r->init(card_counts_storage);
1975 
1976   // 6843694 - ensure that the maximum region index can fit
1977   // in the remembered set structures.
1978   const uint max_region_idx = (1U << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1;
1979   guarantee((max_regions() - 1) <= max_region_idx, "too many regions");
1980 
1981   size_t max_cards_per_region = ((size_t)1 << (sizeof(CardIdx_t)*BitsPerByte-1)) - 1;
1982   guarantee(HeapRegion::CardsPerRegion > 0, "make sure it's initialized");
1983   guarantee(HeapRegion::CardsPerRegion < max_cards_per_region,
1984             "too many cards per region");
1985 
1986   FreeRegionList::set_unrealistically_long_length(max_regions() + 1);
1987 
1988   _bot_shared = new G1BlockOffsetSharedArray(reserved_region(), bot_storage);
1989 
1990   {
1991     HeapWord* start = _hrm.reserved().start();
1992     HeapWord* end = _hrm.reserved().end();
1993     size_t granularity = HeapRegion::GrainBytes;
1994 
1995     _in_cset_fast_test.initialize(start, end, granularity);
1996     _humongous_reclaim_candidates.initialize(start, end, granularity);
1997   }
1998 
1999   // Create the ConcurrentMark data structure and thread.
2000   // (Must do this late, so that "max_regions" is defined.)
2001   _cm = new ConcurrentMark(this, prev_bitmap_storage, next_bitmap_storage);
2002   if (_cm == NULL || !_cm->completed_initialization()) {
2003     vm_shutdown_during_initialization("Could not create/initialize ConcurrentMark");
2004     return JNI_ENOMEM;
2005   }
2006   _cmThread = _cm->cmThread();
2007 
2008   // Initialize the from_card cache structure of HeapRegionRemSet.
2009   HeapRegionRemSet::init_heap(max_regions());
2010 
2011   // Now expand into the initial heap size.
2012   if (!expand(init_byte_size)) {
2013     vm_shutdown_during_initialization("Failed to allocate initial heap.");
2014     return JNI_ENOMEM;
2015   }
2016 
2017   // Perform any initialization actions delegated to the policy.
2018   g1_policy()->init();
2019 
2020   JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,
2021                                                SATB_Q_FL_lock,
2022                                                G1SATBProcessCompletedThreshold,
2023                                                Shared_SATB_Q_lock);
2024 
2025   JavaThread::dirty_card_queue_set().initialize(_refine_cte_cl,
2026                                                 DirtyCardQ_CBL_mon,
2027                                                 DirtyCardQ_FL_lock,
2028                                                 concurrent_g1_refine()->yellow_zone(),
2029                                                 concurrent_g1_refine()->red_zone(),
2030                                                 Shared_DirtyCardQ_lock);
2031 
2032   dirty_card_queue_set().initialize(NULL, // Should never be called by the Java code
2033                                     DirtyCardQ_CBL_mon,
2034                                     DirtyCardQ_FL_lock,
2035                                     -1, // never trigger processing
2036                                     -1, // no limit on length
2037                                     Shared_DirtyCardQ_lock,
2038                                     &JavaThread::dirty_card_queue_set());
2039 
2040   // Here we allocate the dummy HeapRegion that is required by the
2041   // G1AllocRegion class.
2042   HeapRegion* dummy_region = _hrm.get_dummy_region();
2043 
2044   // We'll re-use the same region whether the alloc region will
2045   // require BOT updates or not and, if it doesn't, then a non-young
2046   // region will complain that it cannot support allocations without
2047   // BOT updates. So we'll tag the dummy region as eden to avoid that.
2048   dummy_region->set_eden();
2049   // Make sure it's full.
2050   dummy_region->set_top(dummy_region->end());
2051   G1AllocRegion::setup(this, dummy_region);
2052 
2053   _allocator->init_mutator_alloc_region();
2054 
2055   // Do create of the monitoring and management support so that
2056   // values in the heap have been properly initialized.
2057   _g1mm = new G1MonitoringSupport(this);
2058 
2059   G1StringDedup::initialize();
2060 
2061   _preserved_objs = NEW_C_HEAP_ARRAY(OopAndMarkOopStack, ParallelGCThreads, mtGC);
2062   for (uint i = 0; i < ParallelGCThreads; i++) {
2063     new (&_preserved_objs[i]) OopAndMarkOopStack();
2064   }
2065 
2066   return JNI_OK;
2067 }
2068 
2069 void G1CollectedHeap::stop() {
2070   // Stop all concurrent threads. We do this to make sure these threads
2071   // do not continue to execute and access resources (e.g. gclog_or_tty)
2072   // that are destroyed during shutdown.
2073   _cg1r->stop();
2074   _cmThread->stop();
2075   if (G1StringDedup::is_enabled()) {
2076     G1StringDedup::stop();
2077   }
2078 }
2079 
2080 size_t G1CollectedHeap::conservative_max_heap_alignment() {
2081   return HeapRegion::max_region_size();
2082 }
2083 
2084 void G1CollectedHeap::post_initialize() {
2085   CollectedHeap::post_initialize();
2086   ref_processing_init();
2087 }
2088 
2089 void G1CollectedHeap::ref_processing_init() {
2090   // Reference processing in G1 currently works as follows:
2091   //
2092   // * There are two reference processor instances. One is
2093   //   used to record and process discovered references
2094   //   during concurrent marking; the other is used to
2095   //   record and process references during STW pauses
2096   //   (both full and incremental).
2097   // * Both ref processors need to 'span' the entire heap as
2098   //   the regions in the collection set may be dotted around.
2099   //
2100   // * For the concurrent marking ref processor:
2101   //   * Reference discovery is enabled at initial marking.
2102   //   * Reference discovery is disabled and the discovered
2103   //     references processed etc during remarking.
2104   //   * Reference discovery is MT (see below).
2105   //   * Reference discovery requires a barrier (see below).
2106   //   * Reference processing may or may not be MT
2107   //     (depending on the value of ParallelRefProcEnabled
2108   //     and ParallelGCThreads).
2109   //   * A full GC disables reference discovery by the CM
2110   //     ref processor and abandons any entries on it's
2111   //     discovered lists.
2112   //
2113   // * For the STW processor:
2114   //   * Non MT discovery is enabled at the start of a full GC.
2115   //   * Processing and enqueueing during a full GC is non-MT.
2116   //   * During a full GC, references are processed after marking.
2117   //
2118   //   * Discovery (may or may not be MT) is enabled at the start
2119   //     of an incremental evacuation pause.
2120   //   * References are processed near the end of a STW evacuation pause.
2121   //   * For both types of GC:
2122   //     * Discovery is atomic - i.e. not concurrent.
2123   //     * Reference discovery will not need a barrier.
2124 
2125   MemRegion mr = reserved_region();
2126 
2127   // Concurrent Mark ref processor
2128   _ref_processor_cm =
2129     new ReferenceProcessor(mr,    // span
2130                            ParallelRefProcEnabled && (ParallelGCThreads > 1),
2131                                 // mt processing
2132                            ParallelGCThreads,
2133                                 // degree of mt processing
2134                            (ParallelGCThreads > 1) || (ConcGCThreads > 1),
2135                                 // mt discovery
2136                            MAX2(ParallelGCThreads, ConcGCThreads),
2137                                 // degree of mt discovery
2138                            false,
2139                                 // Reference discovery is not atomic
2140                            &_is_alive_closure_cm);
2141                                 // is alive closure
2142                                 // (for efficiency/performance)
2143 
2144   // STW ref processor
2145   _ref_processor_stw =
2146     new ReferenceProcessor(mr,    // span
2147                            ParallelRefProcEnabled && (ParallelGCThreads > 1),
2148                                 // mt processing
2149                            ParallelGCThreads,
2150                                 // degree of mt processing
2151                            (ParallelGCThreads > 1),
2152                                 // mt discovery
2153                            ParallelGCThreads,
2154                                 // degree of mt discovery
2155                            true,
2156                                 // Reference discovery is atomic
2157                            &_is_alive_closure_stw);
2158                                 // is alive closure
2159                                 // (for efficiency/performance)
2160 }
2161 
2162 CollectorPolicy* G1CollectedHeap::collector_policy() const {
2163   return g1_policy();
2164 }
2165 
2166 size_t G1CollectedHeap::capacity() const {
2167   return _hrm.length() * HeapRegion::GrainBytes;
2168 }
2169 
2170 void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) {
2171   hr->reset_gc_time_stamp();
2172 }
2173 
2174 #ifndef PRODUCT
2175 
2176 class CheckGCTimeStampsHRClosure : public HeapRegionClosure {
2177 private:
2178   unsigned _gc_time_stamp;
2179   bool _failures;
2180 
2181 public:
2182   CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) :
2183     _gc_time_stamp(gc_time_stamp), _failures(false) { }
2184 
2185   virtual bool doHeapRegion(HeapRegion* hr) {
2186     unsigned region_gc_time_stamp = hr->get_gc_time_stamp();
2187     if (_gc_time_stamp != region_gc_time_stamp) {
2188       gclog_or_tty->print_cr("Region " HR_FORMAT " has GC time stamp = %d, "
2189                              "expected %d", HR_FORMAT_PARAMS(hr),
2190                              region_gc_time_stamp, _gc_time_stamp);
2191       _failures = true;
2192     }
2193     return false;
2194   }
2195 
2196   bool failures() { return _failures; }
2197 };
2198 
2199 void G1CollectedHeap::check_gc_time_stamps() {
2200   CheckGCTimeStampsHRClosure cl(_gc_time_stamp);
2201   heap_region_iterate(&cl);
2202   guarantee(!cl.failures(), "all GC time stamps should have been reset");
2203 }
2204 #endif // PRODUCT
2205 
2206 void G1CollectedHeap::iterate_hcc_closure(CardTableEntryClosure* cl, uint worker_i) {
2207   _cg1r->hot_card_cache()->drain(cl, worker_i);
2208 }
2209 
2210 void G1CollectedHeap::iterate_dirty_card_closure(CardTableEntryClosure* cl, uint worker_i) {
2211   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
2212   size_t n_completed_buffers = 0;
2213   while (dcqs.apply_closure_to_completed_buffer(cl, worker_i, 0, true)) {
2214     n_completed_buffers++;
2215   }
2216   g1_policy()->phase_times()->record_thread_work_item(G1GCPhaseTimes::UpdateRS, worker_i, n_completed_buffers);
2217   dcqs.clear_n_completed_buffers();
2218   assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!");
2219 }
2220 
2221 // Computes the sum of the storage used by the various regions.
2222 size_t G1CollectedHeap::used() const {
2223   size_t result = _summary_bytes_used + _allocator->used_in_alloc_regions();
2224   if (_archive_allocator != NULL) {
2225     result += _archive_allocator->used();
2226   }
2227   return result;
2228 }
2229 
2230 size_t G1CollectedHeap::used_unlocked() const {
2231   return _summary_bytes_used;
2232 }
2233 
2234 class SumUsedClosure: public HeapRegionClosure {
2235   size_t _used;
2236 public:
2237   SumUsedClosure() : _used(0) {}
2238   bool doHeapRegion(HeapRegion* r) {
2239     _used += r->used();
2240     return false;
2241   }
2242   size_t result() { return _used; }
2243 };
2244 
2245 size_t G1CollectedHeap::recalculate_used() const {
2246   double recalculate_used_start = os::elapsedTime();
2247 
2248   SumUsedClosure blk;
2249   heap_region_iterate(&blk);
2250 
2251   g1_policy()->phase_times()->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0);
2252   return blk.result();
2253 }
2254 
2255 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {
2256   switch (cause) {
2257     case GCCause::_gc_locker:               return GCLockerInvokesConcurrent;
2258     case GCCause::_java_lang_system_gc:     return ExplicitGCInvokesConcurrent;
2259     case GCCause::_dcmd_gc_run:             return ExplicitGCInvokesConcurrent;
2260     case GCCause::_g1_humongous_allocation: return true;
2261     case GCCause::_update_allocation_context_stats_inc: return true;
2262     case GCCause::_wb_conc_mark:            return true;
2263     default:                                return false;
2264   }
2265 }
2266 
2267 #ifndef PRODUCT
2268 void G1CollectedHeap::allocate_dummy_regions() {
2269   // Let's fill up most of the region
2270   size_t word_size = HeapRegion::GrainWords - 1024;
2271   // And as a result the region we'll allocate will be humongous.
2272   guarantee(is_humongous(word_size), "sanity");
2273 
2274   // _filler_array_max_size is set to humongous object threshold
2275   // but temporarily change it to use CollectedHeap::fill_with_object().
2276   SizeTFlagSetting fs(_filler_array_max_size, word_size);
2277 
2278   for (uintx i = 0; i < G1DummyRegionsPerGC; ++i) {
2279     // Let's use the existing mechanism for the allocation
2280     HeapWord* dummy_obj = humongous_obj_allocate(word_size,
2281                                                  AllocationContext::system());
2282     if (dummy_obj != NULL) {
2283       MemRegion mr(dummy_obj, word_size);
2284       CollectedHeap::fill_with_object(mr);
2285     } else {
2286       // If we can't allocate once, we probably cannot allocate
2287       // again. Let's get out of the loop.
2288       break;
2289     }
2290   }
2291 }
2292 #endif // !PRODUCT
2293 
2294 void G1CollectedHeap::increment_old_marking_cycles_started() {
2295   assert(_old_marking_cycles_started == _old_marking_cycles_completed ||
2296          _old_marking_cycles_started == _old_marking_cycles_completed + 1,
2297          "Wrong marking cycle count (started: %d, completed: %d)",
2298          _old_marking_cycles_started, _old_marking_cycles_completed);
2299 
2300   _old_marking_cycles_started++;
2301 }
2302 
2303 void G1CollectedHeap::increment_old_marking_cycles_completed(bool concurrent) {
2304   MonitorLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag);
2305 
2306   // We assume that if concurrent == true, then the caller is a
2307   // concurrent thread that was joined the Suspendible Thread
2308   // Set. If there's ever a cheap way to check this, we should add an
2309   // assert here.
2310 
2311   // Given that this method is called at the end of a Full GC or of a
2312   // concurrent cycle, and those can be nested (i.e., a Full GC can
2313   // interrupt a concurrent cycle), the number of full collections
2314   // completed should be either one (in the case where there was no
2315   // nesting) or two (when a Full GC interrupted a concurrent cycle)
2316   // behind the number of full collections started.
2317 
2318   // This is the case for the inner caller, i.e. a Full GC.
2319   assert(concurrent ||
2320          (_old_marking_cycles_started == _old_marking_cycles_completed + 1) ||
2321          (_old_marking_cycles_started == _old_marking_cycles_completed + 2),
2322          "for inner caller (Full GC): _old_marking_cycles_started = %u "
2323          "is inconsistent with _old_marking_cycles_completed = %u",
2324          _old_marking_cycles_started, _old_marking_cycles_completed);
2325 
2326   // This is the case for the outer caller, i.e. the concurrent cycle.
2327   assert(!concurrent ||
2328          (_old_marking_cycles_started == _old_marking_cycles_completed + 1),
2329          "for outer caller (concurrent cycle): "
2330          "_old_marking_cycles_started = %u "
2331          "is inconsistent with _old_marking_cycles_completed = %u",
2332          _old_marking_cycles_started, _old_marking_cycles_completed);
2333 
2334   _old_marking_cycles_completed += 1;
2335 
2336   // We need to clear the "in_progress" flag in the CM thread before
2337   // we wake up any waiters (especially when ExplicitInvokesConcurrent
2338   // is set) so that if a waiter requests another System.gc() it doesn't
2339   // incorrectly see that a marking cycle is still in progress.
2340   if (concurrent) {
2341     _cmThread->set_idle();
2342   }
2343 
2344   // This notify_all() will ensure that a thread that called
2345   // System.gc() with (with ExplicitGCInvokesConcurrent set or not)
2346   // and it's waiting for a full GC to finish will be woken up. It is
2347   // waiting in VM_G1IncCollectionPause::doit_epilogue().
2348   FullGCCount_lock->notify_all();
2349 }
2350 
2351 void G1CollectedHeap::register_concurrent_cycle_start(const Ticks& start_time) {
2352   GCIdMarkAndRestore conc_gc_id_mark;
2353   collector_state()->set_concurrent_cycle_started(true);
2354   _gc_timer_cm->register_gc_start(start_time);
2355 
2356   _gc_tracer_cm->report_gc_start(gc_cause(), _gc_timer_cm->gc_start());
2357   trace_heap_before_gc(_gc_tracer_cm);
2358   _cmThread->set_gc_id(GCId::current());
2359 }
2360 
2361 void G1CollectedHeap::register_concurrent_cycle_end() {
2362   if (collector_state()->concurrent_cycle_started()) {
2363     GCIdMarkAndRestore conc_gc_id_mark(_cmThread->gc_id());
2364     if (_cm->has_aborted()) {
2365       _gc_tracer_cm->report_concurrent_mode_failure();
2366     }
2367 
2368     _gc_timer_cm->register_gc_end();
2369     _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
2370 
2371     // Clear state variables to prepare for the next concurrent cycle.
2372      collector_state()->set_concurrent_cycle_started(false);
2373     _heap_summary_sent = false;
2374   }
2375 }
2376 
2377 void G1CollectedHeap::trace_heap_after_concurrent_cycle() {
2378   if (collector_state()->concurrent_cycle_started()) {
2379     // This function can be called when:
2380     //  the cleanup pause is run
2381     //  the concurrent cycle is aborted before the cleanup pause.
2382     //  the concurrent cycle is aborted after the cleanup pause,
2383     //   but before the concurrent cycle end has been registered.
2384     // Make sure that we only send the heap information once.
2385     if (!_heap_summary_sent) {
2386       GCIdMarkAndRestore conc_gc_id_mark(_cmThread->gc_id());
2387       trace_heap_after_gc(_gc_tracer_cm);
2388       _heap_summary_sent = true;
2389     }
2390   }
2391 }
2392 
2393 void G1CollectedHeap::collect(GCCause::Cause cause) {
2394   assert_heap_not_locked();
2395 
2396   uint gc_count_before;
2397   uint old_marking_count_before;
2398   uint full_gc_count_before;
2399   bool retry_gc;
2400 
2401   do {
2402     retry_gc = false;
2403 
2404     {
2405       MutexLocker ml(Heap_lock);
2406 
2407       // Read the GC count while holding the Heap_lock
2408       gc_count_before = total_collections();
2409       full_gc_count_before = total_full_collections();
2410       old_marking_count_before = _old_marking_cycles_started;
2411     }
2412 
2413     if (should_do_concurrent_full_gc(cause)) {
2414       // Schedule an initial-mark evacuation pause that will start a
2415       // concurrent cycle. We're setting word_size to 0 which means that
2416       // we are not requesting a post-GC allocation.
2417       VM_G1IncCollectionPause op(gc_count_before,
2418                                  0,     /* word_size */
2419                                  true,  /* should_initiate_conc_mark */
2420                                  g1_policy()->max_pause_time_ms(),
2421                                  cause);
2422       op.set_allocation_context(AllocationContext::current());
2423 
2424       VMThread::execute(&op);
2425       if (!op.pause_succeeded()) {
2426         if (old_marking_count_before == _old_marking_cycles_started) {
2427           retry_gc = op.should_retry_gc();
2428         } else {
2429           // A Full GC happened while we were trying to schedule the
2430           // initial-mark GC. No point in starting a new cycle given
2431           // that the whole heap was collected anyway.
2432         }
2433 
2434         if (retry_gc) {
2435           if (GC_locker::is_active_and_needs_gc()) {
2436             GC_locker::stall_until_clear();
2437           }
2438         }
2439       }
2440     } else {
2441       if (cause == GCCause::_gc_locker || cause == GCCause::_wb_young_gc
2442           DEBUG_ONLY(|| cause == GCCause::_scavenge_alot)) {
2443 
2444         // Schedule a standard evacuation pause. We're setting word_size
2445         // to 0 which means that we are not requesting a post-GC allocation.
2446         VM_G1IncCollectionPause op(gc_count_before,
2447                                    0,     /* word_size */
2448                                    false, /* should_initiate_conc_mark */
2449                                    g1_policy()->max_pause_time_ms(),
2450                                    cause);
2451         VMThread::execute(&op);
2452       } else {
2453         // Schedule a Full GC.
2454         VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);
2455         VMThread::execute(&op);
2456       }
2457     }
2458   } while (retry_gc);
2459 }
2460 
2461 bool G1CollectedHeap::is_in(const void* p) const {
2462   if (_hrm.reserved().contains(p)) {
2463     // Given that we know that p is in the reserved space,
2464     // heap_region_containing() should successfully
2465     // return the containing region.
2466     HeapRegion* hr = heap_region_containing(p);
2467     return hr->is_in(p);
2468   } else {
2469     return false;
2470   }
2471 }
2472 
2473 #ifdef ASSERT
2474 bool G1CollectedHeap::is_in_exact(const void* p) const {
2475   bool contains = reserved_region().contains(p);
2476   bool available = _hrm.is_available(addr_to_region((HeapWord*)p));
2477   if (contains && available) {
2478     return true;
2479   } else {
2480     return false;
2481   }
2482 }
2483 #endif
2484 
2485 bool G1CollectedHeap::obj_in_cs(oop obj) {
2486   HeapRegion* r = _hrm.addr_to_region((HeapWord*) obj);
2487   return r != NULL && r->in_collection_set();
2488 }
2489 
2490 // Iteration functions.
2491 
2492 // Applies an ExtendedOopClosure onto all references of objects within a HeapRegion.
2493 
2494 class IterateOopClosureRegionClosure: public HeapRegionClosure {
2495   ExtendedOopClosure* _cl;
2496 public:
2497   IterateOopClosureRegionClosure(ExtendedOopClosure* cl) : _cl(cl) {}
2498   bool doHeapRegion(HeapRegion* r) {
2499     if (!r->is_continues_humongous()) {
2500       r->oop_iterate(_cl);
2501     }
2502     return false;
2503   }
2504 };
2505 
2506 // Iterates an ObjectClosure over all objects within a HeapRegion.
2507 
2508 class IterateObjectClosureRegionClosure: public HeapRegionClosure {
2509   ObjectClosure* _cl;
2510 public:
2511   IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}
2512   bool doHeapRegion(HeapRegion* r) {
2513     if (!r->is_continues_humongous()) {
2514       r->object_iterate(_cl);
2515     }
2516     return false;
2517   }
2518 };
2519 
2520 void G1CollectedHeap::object_iterate(ObjectClosure* cl) {
2521   IterateObjectClosureRegionClosure blk(cl);
2522   heap_region_iterate(&blk);
2523 }
2524 
2525 void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const {
2526   _hrm.iterate(cl);
2527 }
2528 
2529 void
2530 G1CollectedHeap::heap_region_par_iterate(HeapRegionClosure* cl,
2531                                          uint worker_id,
2532                                          HeapRegionClaimer *hrclaimer,
2533                                          bool concurrent) const {
2534   _hrm.par_iterate(cl, worker_id, hrclaimer, concurrent);
2535 }
2536 
2537 // Clear the cached CSet starting regions and (more importantly)
2538 // the time stamps. Called when we reset the GC time stamp.
2539 void G1CollectedHeap::clear_cset_start_regions() {
2540   assert(_worker_cset_start_region != NULL, "sanity");
2541   assert(_worker_cset_start_region_time_stamp != NULL, "sanity");
2542 
2543   for (uint i = 0; i < ParallelGCThreads; i++) {
2544     _worker_cset_start_region[i] = NULL;
2545     _worker_cset_start_region_time_stamp[i] = 0;
2546   }
2547 }
2548 
2549 // Given the id of a worker, obtain or calculate a suitable
2550 // starting region for iterating over the current collection set.
2551 HeapRegion* G1CollectedHeap::start_cset_region_for_worker(uint worker_i) {
2552   assert(get_gc_time_stamp() > 0, "should have been updated by now");
2553 
2554   HeapRegion* result = NULL;
2555   unsigned gc_time_stamp = get_gc_time_stamp();
2556 
2557   if (_worker_cset_start_region_time_stamp[worker_i] == gc_time_stamp) {
2558     // Cached starting region for current worker was set
2559     // during the current pause - so it's valid.
2560     // Note: the cached starting heap region may be NULL
2561     // (when the collection set is empty).
2562     result = _worker_cset_start_region[worker_i];
2563     assert(result == NULL || result->in_collection_set(), "sanity");
2564     return result;
2565   }
2566 
2567   // The cached entry was not valid so let's calculate
2568   // a suitable starting heap region for this worker.
2569 
2570   // We want the parallel threads to start their collection
2571   // set iteration at different collection set regions to
2572   // avoid contention.
2573   // If we have:
2574   //          n collection set regions
2575   //          p threads
2576   // Then thread t will start at region floor ((t * n) / p)
2577 
2578   result = g1_policy()->collection_set();
2579   uint cs_size = g1_policy()->cset_region_length();
2580   uint active_workers = workers()->active_workers();
2581 
2582   uint end_ind   = (cs_size * worker_i) / active_workers;
2583   uint start_ind = 0;
2584 
2585   if (worker_i > 0 &&
2586       _worker_cset_start_region_time_stamp[worker_i - 1] == gc_time_stamp) {
2587     // Previous workers starting region is valid
2588     // so let's iterate from there
2589     start_ind = (cs_size * (worker_i - 1)) / active_workers;
2590     result = _worker_cset_start_region[worker_i - 1];
2591   }
2592 
2593   for (uint i = start_ind; i < end_ind; i++) {
2594     result = result->next_in_collection_set();
2595   }
2596 
2597   // Note: the calculated starting heap region may be NULL
2598   // (when the collection set is empty).
2599   assert(result == NULL || result->in_collection_set(), "sanity");
2600   assert(_worker_cset_start_region_time_stamp[worker_i] != gc_time_stamp,
2601          "should be updated only once per pause");
2602   _worker_cset_start_region[worker_i] = result;
2603   OrderAccess::storestore();
2604   _worker_cset_start_region_time_stamp[worker_i] = gc_time_stamp;
2605   return result;
2606 }
2607 
2608 void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) {
2609   HeapRegion* r = g1_policy()->collection_set();
2610   while (r != NULL) {
2611     HeapRegion* next = r->next_in_collection_set();
2612     if (cl->doHeapRegion(r)) {
2613       cl->incomplete();
2614       return;
2615     }
2616     r = next;
2617   }
2618 }
2619 
2620 void G1CollectedHeap::collection_set_iterate_from(HeapRegion* r,
2621                                                   HeapRegionClosure *cl) {
2622   if (r == NULL) {
2623     // The CSet is empty so there's nothing to do.
2624     return;
2625   }
2626 
2627   assert(r->in_collection_set(),
2628          "Start region must be a member of the collection set.");
2629   HeapRegion* cur = r;
2630   while (cur != NULL) {
2631     HeapRegion* next = cur->next_in_collection_set();
2632     if (cl->doHeapRegion(cur) && false) {
2633       cl->incomplete();
2634       return;
2635     }
2636     cur = next;
2637   }
2638   cur = g1_policy()->collection_set();
2639   while (cur != r) {
2640     HeapRegion* next = cur->next_in_collection_set();
2641     if (cl->doHeapRegion(cur) && false) {
2642       cl->incomplete();
2643       return;
2644     }
2645     cur = next;
2646   }
2647 }
2648 
2649 HeapRegion* G1CollectedHeap::next_compaction_region(const HeapRegion* from) const {
2650   HeapRegion* result = _hrm.next_region_in_heap(from);
2651   while (result != NULL && result->is_pinned()) {
2652     result = _hrm.next_region_in_heap(result);
2653   }
2654   return result;
2655 }
2656 
2657 HeapWord* G1CollectedHeap::block_start(const void* addr) const {
2658   HeapRegion* hr = heap_region_containing(addr);
2659   return hr->block_start(addr);
2660 }
2661 
2662 size_t G1CollectedHeap::block_size(const HeapWord* addr) const {
2663   HeapRegion* hr = heap_region_containing(addr);
2664   return hr->block_size(addr);
2665 }
2666 
2667 bool G1CollectedHeap::block_is_obj(const HeapWord* addr) const {
2668   HeapRegion* hr = heap_region_containing(addr);
2669   return hr->block_is_obj(addr);
2670 }
2671 
2672 bool G1CollectedHeap::supports_tlab_allocation() const {
2673   return true;
2674 }
2675 
2676 size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const {
2677   return (_g1_policy->young_list_target_length() - young_list()->survivor_length()) * HeapRegion::GrainBytes;
2678 }
2679 
2680 size_t G1CollectedHeap::tlab_used(Thread* ignored) const {
2681   return young_list()->eden_used_bytes();
2682 }
2683 
2684 // For G1 TLABs should not contain humongous objects, so the maximum TLAB size
2685 // must be equal to the humongous object limit.
2686 size_t G1CollectedHeap::max_tlab_size() const {
2687   return align_size_down(_humongous_object_threshold_in_words, MinObjAlignment);
2688 }
2689 
2690 size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const {
2691   AllocationContext_t context = AllocationContext::current();
2692   return _allocator->unsafe_max_tlab_alloc(context);
2693 }
2694 
2695 size_t G1CollectedHeap::max_capacity() const {
2696   return _hrm.reserved().byte_size();
2697 }
2698 
2699 jlong G1CollectedHeap::millis_since_last_gc() {
2700   // assert(false, "NYI");
2701   return 0;
2702 }
2703 
2704 void G1CollectedHeap::prepare_for_verify() {
2705   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
2706     ensure_parsability(false);
2707   }
2708   g1_rem_set()->prepare_for_verify();
2709 }
2710 
2711 bool G1CollectedHeap::allocated_since_marking(oop obj, HeapRegion* hr,
2712                                               VerifyOption vo) {
2713   switch (vo) {
2714   case VerifyOption_G1UsePrevMarking:
2715     return hr->obj_allocated_since_prev_marking(obj);
2716   case VerifyOption_G1UseNextMarking:
2717     return hr->obj_allocated_since_next_marking(obj);
2718   case VerifyOption_G1UseMarkWord:
2719     return false;
2720   default:
2721     ShouldNotReachHere();
2722   }
2723   return false; // keep some compilers happy
2724 }
2725 
2726 HeapWord* G1CollectedHeap::top_at_mark_start(HeapRegion* hr, VerifyOption vo) {
2727   switch (vo) {
2728   case VerifyOption_G1UsePrevMarking: return hr->prev_top_at_mark_start();
2729   case VerifyOption_G1UseNextMarking: return hr->next_top_at_mark_start();
2730   case VerifyOption_G1UseMarkWord:    return NULL;
2731   default:                            ShouldNotReachHere();
2732   }
2733   return NULL; // keep some compilers happy
2734 }
2735 
2736 bool G1CollectedHeap::is_marked(oop obj, VerifyOption vo) {
2737   switch (vo) {
2738   case VerifyOption_G1UsePrevMarking: return isMarkedPrev(obj);
2739   case VerifyOption_G1UseNextMarking: return isMarkedNext(obj);
2740   case VerifyOption_G1UseMarkWord:    return obj->is_gc_marked();
2741   default:                            ShouldNotReachHere();
2742   }
2743   return false; // keep some compilers happy
2744 }
2745 
2746 const char* G1CollectedHeap::top_at_mark_start_str(VerifyOption vo) {
2747   switch (vo) {
2748   case VerifyOption_G1UsePrevMarking: return "PTAMS";
2749   case VerifyOption_G1UseNextMarking: return "NTAMS";
2750   case VerifyOption_G1UseMarkWord:    return "NONE";
2751   default:                            ShouldNotReachHere();
2752   }
2753   return NULL; // keep some compilers happy
2754 }
2755 
2756 class VerifyRootsClosure: public OopClosure {
2757 private:
2758   G1CollectedHeap* _g1h;
2759   VerifyOption     _vo;
2760   bool             _failures;
2761 public:
2762   // _vo == UsePrevMarking -> use "prev" marking information,
2763   // _vo == UseNextMarking -> use "next" marking information,
2764   // _vo == UseMarkWord    -> use mark word from object header.
2765   VerifyRootsClosure(VerifyOption vo) :
2766     _g1h(G1CollectedHeap::heap()),
2767     _vo(vo),
2768     _failures(false) { }
2769 
2770   bool failures() { return _failures; }
2771 
2772   template <class T> void do_oop_nv(T* p) {
2773     T heap_oop = oopDesc::load_heap_oop(p);
2774     if (!oopDesc::is_null(heap_oop)) {
2775       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
2776       if (_g1h->is_obj_dead_cond(obj, _vo)) {
2777         gclog_or_tty->print_cr("Root location " PTR_FORMAT " "
2778                                "points to dead obj " PTR_FORMAT, p2i(p), p2i(obj));
2779         if (_vo == VerifyOption_G1UseMarkWord) {
2780           gclog_or_tty->print_cr("  Mark word: " INTPTR_FORMAT, (intptr_t)obj->mark());
2781         }
2782         obj->print_on(gclog_or_tty);
2783         _failures = true;
2784       }
2785     }
2786   }
2787 
2788   void do_oop(oop* p)       { do_oop_nv(p); }
2789   void do_oop(narrowOop* p) { do_oop_nv(p); }
2790 };
2791 
2792 class G1VerifyCodeRootOopClosure: public OopClosure {
2793   G1CollectedHeap* _g1h;
2794   OopClosure* _root_cl;
2795   nmethod* _nm;
2796   VerifyOption _vo;
2797   bool _failures;
2798 
2799   template <class T> void do_oop_work(T* p) {
2800     // First verify that this root is live
2801     _root_cl->do_oop(p);
2802 
2803     if (!G1VerifyHeapRegionCodeRoots) {
2804       // We're not verifying the code roots attached to heap region.
2805       return;
2806     }
2807 
2808     // Don't check the code roots during marking verification in a full GC
2809     if (_vo == VerifyOption_G1UseMarkWord) {
2810       return;
2811     }
2812 
2813     // Now verify that the current nmethod (which contains p) is
2814     // in the code root list of the heap region containing the
2815     // object referenced by p.
2816 
2817     T heap_oop = oopDesc::load_heap_oop(p);
2818     if (!oopDesc::is_null(heap_oop)) {
2819       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
2820 
2821       // Now fetch the region containing the object
2822       HeapRegion* hr = _g1h->heap_region_containing(obj);
2823       HeapRegionRemSet* hrrs = hr->rem_set();
2824       // Verify that the strong code root list for this region
2825       // contains the nmethod
2826       if (!hrrs->strong_code_roots_list_contains(_nm)) {
2827         gclog_or_tty->print_cr("Code root location " PTR_FORMAT " "
2828                                "from nmethod " PTR_FORMAT " not in strong "
2829                                "code roots for region [" PTR_FORMAT "," PTR_FORMAT ")",
2830                                p2i(p), p2i(_nm), p2i(hr->bottom()), p2i(hr->end()));
2831         _failures = true;
2832       }
2833     }
2834   }
2835 
2836 public:
2837   G1VerifyCodeRootOopClosure(G1CollectedHeap* g1h, OopClosure* root_cl, VerifyOption vo):
2838     _g1h(g1h), _root_cl(root_cl), _vo(vo), _nm(NULL), _failures(false) {}
2839 
2840   void do_oop(oop* p) { do_oop_work(p); }
2841   void do_oop(narrowOop* p) { do_oop_work(p); }
2842 
2843   void set_nmethod(nmethod* nm) { _nm = nm; }
2844   bool failures() { return _failures; }
2845 };
2846 
2847 class G1VerifyCodeRootBlobClosure: public CodeBlobClosure {
2848   G1VerifyCodeRootOopClosure* _oop_cl;
2849 
2850 public:
2851   G1VerifyCodeRootBlobClosure(G1VerifyCodeRootOopClosure* oop_cl):
2852     _oop_cl(oop_cl) {}
2853 
2854   void do_code_blob(CodeBlob* cb) {
2855     nmethod* nm = cb->as_nmethod_or_null();
2856     if (nm != NULL) {
2857       _oop_cl->set_nmethod(nm);
2858       nm->oops_do(_oop_cl);
2859     }
2860   }
2861 };
2862 
2863 class YoungRefCounterClosure : public OopClosure {
2864   G1CollectedHeap* _g1h;
2865   int              _count;
2866  public:
2867   YoungRefCounterClosure(G1CollectedHeap* g1h) : _g1h(g1h), _count(0) {}
2868   void do_oop(oop* p)       { if (_g1h->is_in_young(*p)) { _count++; } }
2869   void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2870 
2871   int count() { return _count; }
2872   void reset_count() { _count = 0; };
2873 };
2874 
2875 class VerifyKlassClosure: public KlassClosure {
2876   YoungRefCounterClosure _young_ref_counter_closure;
2877   OopClosure *_oop_closure;
2878  public:
2879   VerifyKlassClosure(G1CollectedHeap* g1h, OopClosure* cl) : _young_ref_counter_closure(g1h), _oop_closure(cl) {}
2880   void do_klass(Klass* k) {
2881     k->oops_do(_oop_closure);
2882 
2883     _young_ref_counter_closure.reset_count();
2884     k->oops_do(&_young_ref_counter_closure);
2885     if (_young_ref_counter_closure.count() > 0) {
2886       guarantee(k->has_modified_oops(), "Klass " PTR_FORMAT ", has young refs but is not dirty.", p2i(k));
2887     }
2888   }
2889 };
2890 
2891 class VerifyLivenessOopClosure: public OopClosure {
2892   G1CollectedHeap* _g1h;
2893   VerifyOption _vo;
2894 public:
2895   VerifyLivenessOopClosure(G1CollectedHeap* g1h, VerifyOption vo):
2896     _g1h(g1h), _vo(vo)
2897   { }
2898   void do_oop(narrowOop *p) { do_oop_work(p); }
2899   void do_oop(      oop *p) { do_oop_work(p); }
2900 
2901   template <class T> void do_oop_work(T *p) {
2902     oop obj = oopDesc::load_decode_heap_oop(p);
2903     guarantee(obj == NULL || !_g1h->is_obj_dead_cond(obj, _vo),
2904               "Dead object referenced by a not dead object");
2905   }
2906 };
2907 
2908 class VerifyObjsInRegionClosure: public ObjectClosure {
2909 private:
2910   G1CollectedHeap* _g1h;
2911   size_t _live_bytes;
2912   HeapRegion *_hr;
2913   VerifyOption _vo;
2914 public:
2915   // _vo == UsePrevMarking -> use "prev" marking information,
2916   // _vo == UseNextMarking -> use "next" marking information,
2917   // _vo == UseMarkWord    -> use mark word from object header.
2918   VerifyObjsInRegionClosure(HeapRegion *hr, VerifyOption vo)
2919     : _live_bytes(0), _hr(hr), _vo(vo) {
2920     _g1h = G1CollectedHeap::heap();
2921   }
2922   void do_object(oop o) {
2923     VerifyLivenessOopClosure isLive(_g1h, _vo);
2924     assert(o != NULL, "Huh?");
2925     if (!_g1h->is_obj_dead_cond(o, _vo)) {
2926       // If the object is alive according to the mark word,
2927       // then verify that the marking information agrees.
2928       // Note we can't verify the contra-positive of the
2929       // above: if the object is dead (according to the mark
2930       // word), it may not be marked, or may have been marked
2931       // but has since became dead, or may have been allocated
2932       // since the last marking.
2933       if (_vo == VerifyOption_G1UseMarkWord) {
2934         guarantee(!_g1h->is_obj_dead(o), "mark word and concurrent mark mismatch");
2935       }
2936 
2937       o->oop_iterate_no_header(&isLive);
2938       if (!_hr->obj_allocated_since_prev_marking(o)) {
2939         size_t obj_size = o->size();    // Make sure we don't overflow
2940         _live_bytes += (obj_size * HeapWordSize);
2941       }
2942     }
2943   }
2944   size_t live_bytes() { return _live_bytes; }
2945 };
2946 
2947 class VerifyArchiveOopClosure: public OopClosure {
2948 public:
2949   VerifyArchiveOopClosure(HeapRegion *hr) { }
2950   void do_oop(narrowOop *p) { do_oop_work(p); }
2951   void do_oop(      oop *p) { do_oop_work(p); }
2952 
2953   template <class T> void do_oop_work(T *p) {
2954     oop obj = oopDesc::load_decode_heap_oop(p);
2955     guarantee(obj == NULL || G1MarkSweep::in_archive_range(obj),
2956               "Archive object at " PTR_FORMAT " references a non-archive object at " PTR_FORMAT,
2957               p2i(p), p2i(obj));
2958   }
2959 };
2960 
2961 class VerifyArchiveRegionClosure: public ObjectClosure {
2962 public:
2963   VerifyArchiveRegionClosure(HeapRegion *hr) { }
2964   // Verify that all object pointers are to archive regions.
2965   void do_object(oop o) {
2966     VerifyArchiveOopClosure checkOop(NULL);
2967     assert(o != NULL, "Should not be here for NULL oops");
2968     o->oop_iterate_no_header(&checkOop);
2969   }
2970 };
2971 
2972 class VerifyRegionClosure: public HeapRegionClosure {
2973 private:
2974   bool             _par;
2975   VerifyOption     _vo;
2976   bool             _failures;
2977 public:
2978   // _vo == UsePrevMarking -> use "prev" marking information,
2979   // _vo == UseNextMarking -> use "next" marking information,
2980   // _vo == UseMarkWord    -> use mark word from object header.
2981   VerifyRegionClosure(bool par, VerifyOption vo)
2982     : _par(par),
2983       _vo(vo),
2984       _failures(false) {}
2985 
2986   bool failures() {
2987     return _failures;
2988   }
2989 
2990   bool doHeapRegion(HeapRegion* r) {
2991     // For archive regions, verify there are no heap pointers to
2992     // non-pinned regions. For all others, verify liveness info.
2993     if (r->is_archive()) {
2994       VerifyArchiveRegionClosure verify_oop_pointers(r);
2995       r->object_iterate(&verify_oop_pointers);
2996       return true;
2997     }
2998     if (!r->is_continues_humongous()) {
2999       bool failures = false;
3000       r->verify(_vo, &failures);
3001       if (failures) {
3002         _failures = true;
3003       } else if (!r->is_starts_humongous()) {
3004         VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo);
3005         r->object_iterate(&not_dead_yet_cl);
3006         if (_vo != VerifyOption_G1UseNextMarking) {
3007           if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
3008             gclog_or_tty->print_cr("[" PTR_FORMAT "," PTR_FORMAT "] "
3009                                    "max_live_bytes " SIZE_FORMAT " "
3010                                    "< calculated " SIZE_FORMAT,
3011                                    p2i(r->bottom()), p2i(r->end()),
3012                                    r->max_live_bytes(),
3013                                  not_dead_yet_cl.live_bytes());
3014             _failures = true;
3015           }
3016         } else {
3017           // When vo == UseNextMarking we cannot currently do a sanity
3018           // check on the live bytes as the calculation has not been
3019           // finalized yet.
3020         }
3021       }
3022     }
3023     return false; // stop the region iteration if we hit a failure
3024   }
3025 };
3026 
3027 // This is the task used for parallel verification of the heap regions
3028 
3029 class G1ParVerifyTask: public AbstractGangTask {
3030 private:
3031   G1CollectedHeap*  _g1h;
3032   VerifyOption      _vo;
3033   bool              _failures;
3034   HeapRegionClaimer _hrclaimer;
3035 
3036 public:
3037   // _vo == UsePrevMarking -> use "prev" marking information,
3038   // _vo == UseNextMarking -> use "next" marking information,
3039   // _vo == UseMarkWord    -> use mark word from object header.
3040   G1ParVerifyTask(G1CollectedHeap* g1h, VerifyOption vo) :
3041       AbstractGangTask("Parallel verify task"),
3042       _g1h(g1h),
3043       _vo(vo),
3044       _failures(false),
3045       _hrclaimer(g1h->workers()->active_workers()) {}
3046 
3047   bool failures() {
3048     return _failures;
3049   }
3050 
3051   void work(uint worker_id) {
3052     HandleMark hm;
3053     VerifyRegionClosure blk(true, _vo);
3054     _g1h->heap_region_par_iterate(&blk, worker_id, &_hrclaimer);
3055     if (blk.failures()) {
3056       _failures = true;
3057     }
3058   }
3059 };
3060 
3061 void G1CollectedHeap::verify(bool silent, VerifyOption vo) {
3062   if (SafepointSynchronize::is_at_safepoint()) {
3063     assert(Thread::current()->is_VM_thread(),
3064            "Expected to be executed serially by the VM thread at this point");
3065 
3066     if (!silent) { gclog_or_tty->print("Roots "); }
3067     VerifyRootsClosure rootsCl(vo);
3068     VerifyKlassClosure klassCl(this, &rootsCl);
3069     CLDToKlassAndOopClosure cldCl(&klassCl, &rootsCl, false);
3070 
3071     // We apply the relevant closures to all the oops in the
3072     // system dictionary, class loader data graph, the string table
3073     // and the nmethods in the code cache.
3074     G1VerifyCodeRootOopClosure codeRootsCl(this, &rootsCl, vo);
3075     G1VerifyCodeRootBlobClosure blobsCl(&codeRootsCl);
3076 
3077     {
3078       G1RootProcessor root_processor(this, 1);
3079       root_processor.process_all_roots(&rootsCl,
3080                                        &cldCl,
3081                                        &blobsCl);
3082     }
3083 
3084     bool failures = rootsCl.failures() || codeRootsCl.failures();
3085 
3086     if (vo != VerifyOption_G1UseMarkWord) {
3087       // If we're verifying during a full GC then the region sets
3088       // will have been torn down at the start of the GC. Therefore
3089       // verifying the region sets will fail. So we only verify
3090       // the region sets when not in a full GC.
3091       if (!silent) { gclog_or_tty->print("HeapRegionSets "); }
3092       verify_region_sets();
3093     }
3094 
3095     if (!silent) { gclog_or_tty->print("HeapRegions "); }
3096     if (GCParallelVerificationEnabled && ParallelGCThreads > 1) {
3097 
3098       G1ParVerifyTask task(this, vo);
3099       workers()->run_task(&task);
3100       if (task.failures()) {
3101         failures = true;
3102       }
3103 
3104     } else {
3105       VerifyRegionClosure blk(false, vo);
3106       heap_region_iterate(&blk);
3107       if (blk.failures()) {
3108         failures = true;
3109       }
3110     }
3111 
3112     if (G1StringDedup::is_enabled()) {
3113       if (!silent) gclog_or_tty->print("StrDedup ");
3114       G1StringDedup::verify();
3115     }
3116 
3117     if (failures) {
3118       gclog_or_tty->print_cr("Heap:");
3119       // It helps to have the per-region information in the output to
3120       // help us track down what went wrong. This is why we call
3121       // print_extended_on() instead of print_on().
3122       print_extended_on(gclog_or_tty);
3123       gclog_or_tty->cr();
3124       gclog_or_tty->flush();
3125     }
3126     guarantee(!failures, "there should not have been any failures");
3127   } else {
3128     if (!silent) {
3129       gclog_or_tty->print("(SKIPPING Roots, HeapRegionSets, HeapRegions, RemSet");
3130       if (G1StringDedup::is_enabled()) {
3131         gclog_or_tty->print(", StrDedup");
3132       }
3133       gclog_or_tty->print(") ");
3134     }
3135   }
3136 }
3137 
3138 void G1CollectedHeap::verify(bool silent) {
3139   verify(silent, VerifyOption_G1UsePrevMarking);
3140 }
3141 
3142 double G1CollectedHeap::verify(bool guard, const char* msg) {
3143   double verify_time_ms = 0.0;
3144 
3145   if (guard && total_collections() >= VerifyGCStartAt) {
3146     double verify_start = os::elapsedTime();
3147     HandleMark hm;  // Discard invalid handles created during verification
3148     prepare_for_verify();
3149     Universe::verify(VerifyOption_G1UsePrevMarking, msg);
3150     verify_time_ms = (os::elapsedTime() - verify_start) * 1000;
3151   }
3152 
3153   return verify_time_ms;
3154 }
3155 
3156 void G1CollectedHeap::verify_before_gc() {
3157   double verify_time_ms = verify(VerifyBeforeGC, " VerifyBeforeGC:");
3158   g1_policy()->phase_times()->record_verify_before_time_ms(verify_time_ms);
3159 }
3160 
3161 void G1CollectedHeap::verify_after_gc() {
3162   double verify_time_ms = verify(VerifyAfterGC, " VerifyAfterGC:");
3163   g1_policy()->phase_times()->record_verify_after_time_ms(verify_time_ms);
3164 }
3165 
3166 class PrintRegionClosure: public HeapRegionClosure {
3167   outputStream* _st;
3168 public:
3169   PrintRegionClosure(outputStream* st) : _st(st) {}
3170   bool doHeapRegion(HeapRegion* r) {
3171     r->print_on(_st);
3172     return false;
3173   }
3174 };
3175 
3176 bool G1CollectedHeap::is_obj_dead_cond(const oop obj,
3177                                        const HeapRegion* hr,
3178                                        const VerifyOption vo) const {
3179   switch (vo) {
3180   case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj, hr);
3181   case VerifyOption_G1UseNextMarking: return is_obj_ill(obj, hr);
3182   case VerifyOption_G1UseMarkWord:    return !obj->is_gc_marked() && !hr->is_archive();
3183   default:                            ShouldNotReachHere();
3184   }
3185   return false; // keep some compilers happy
3186 }
3187 
3188 bool G1CollectedHeap::is_obj_dead_cond(const oop obj,
3189                                        const VerifyOption vo) const {
3190   switch (vo) {
3191   case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj);
3192   case VerifyOption_G1UseNextMarking: return is_obj_ill(obj);
3193   case VerifyOption_G1UseMarkWord: {
3194     HeapRegion* hr = _hrm.addr_to_region((HeapWord*)obj);
3195     return !obj->is_gc_marked() && !hr->is_archive();
3196   }
3197   default:                            ShouldNotReachHere();
3198   }
3199   return false; // keep some compilers happy
3200 }
3201 
3202 void G1CollectedHeap::print_on(outputStream* st) const {
3203   st->print(" %-20s", "garbage-first heap");
3204   st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K",
3205             capacity()/K, used_unlocked()/K);
3206   st->print(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
3207             p2i(_hrm.reserved().start()),
3208             p2i(_hrm.reserved().start() + _hrm.length() + HeapRegion::GrainWords),
3209             p2i(_hrm.reserved().end()));
3210   st->cr();
3211   st->print("  region size " SIZE_FORMAT "K, ", HeapRegion::GrainBytes / K);
3212   uint young_regions = _young_list->length();
3213   st->print("%u young (" SIZE_FORMAT "K), ", young_regions,
3214             (size_t) young_regions * HeapRegion::GrainBytes / K);
3215   uint survivor_regions = g1_policy()->recorded_survivor_regions();
3216   st->print("%u survivors (" SIZE_FORMAT "K)", survivor_regions,
3217             (size_t) survivor_regions * HeapRegion::GrainBytes / K);
3218   st->cr();
3219   MetaspaceAux::print_on(st);
3220 }
3221 
3222 void G1CollectedHeap::print_extended_on(outputStream* st) const {
3223   print_on(st);
3224 
3225   // Print the per-region information.
3226   st->cr();
3227   st->print_cr("Heap Regions: (E=young(eden), S=young(survivor), O=old, "
3228                "HS=humongous(starts), HC=humongous(continues), "
3229                "CS=collection set, F=free, A=archive, TS=gc time stamp, "
3230                "PTAMS=previous top-at-mark-start, "
3231                "NTAMS=next top-at-mark-start)");
3232   PrintRegionClosure blk(st);
3233   heap_region_iterate(&blk);
3234 }
3235 
3236 void G1CollectedHeap::print_on_error(outputStream* st) const {
3237   this->CollectedHeap::print_on_error(st);
3238 
3239   if (_cm != NULL) {
3240     st->cr();
3241     _cm->print_on_error(st);
3242   }
3243 }
3244 
3245 void G1CollectedHeap::print_gc_threads_on(outputStream* st) const {
3246   workers()->print_worker_threads_on(st);
3247   _cmThread->print_on(st);
3248   st->cr();
3249   _cm->print_worker_threads_on(st);
3250   _cg1r->print_worker_threads_on(st);
3251   if (G1StringDedup::is_enabled()) {
3252     G1StringDedup::print_worker_threads_on(st);
3253   }
3254 }
3255 
3256 void G1CollectedHeap::gc_threads_do(ThreadClosure* tc) const {
3257   workers()->threads_do(tc);
3258   tc->do_thread(_cmThread);
3259   _cg1r->threads_do(tc);
3260   if (G1StringDedup::is_enabled()) {
3261     G1StringDedup::threads_do(tc);
3262   }
3263 }
3264 
3265 void G1CollectedHeap::print_tracing_info() const {
3266   // We'll overload this to mean "trace GC pause statistics."
3267   if (TraceYoungGenTime || TraceOldGenTime) {
3268     // The "G1CollectorPolicy" is keeping track of these stats, so delegate
3269     // to that.
3270     g1_policy()->print_tracing_info();
3271   }
3272   if (G1SummarizeRSetStats) {
3273     g1_rem_set()->print_summary_info();
3274   }
3275   if (G1SummarizeConcMark) {
3276     concurrent_mark()->print_summary_info();
3277   }
3278   g1_policy()->print_yg_surv_rate_info();
3279 }
3280 
3281 #ifndef PRODUCT
3282 // Helpful for debugging RSet issues.
3283 
3284 class PrintRSetsClosure : public HeapRegionClosure {
3285 private:
3286   const char* _msg;
3287   size_t _occupied_sum;
3288 
3289 public:
3290   bool doHeapRegion(HeapRegion* r) {
3291     HeapRegionRemSet* hrrs = r->rem_set();
3292     size_t occupied = hrrs->occupied();
3293     _occupied_sum += occupied;
3294 
3295     gclog_or_tty->print_cr("Printing RSet for region " HR_FORMAT,
3296                            HR_FORMAT_PARAMS(r));
3297     if (occupied == 0) {
3298       gclog_or_tty->print_cr("  RSet is empty");
3299     } else {
3300       hrrs->print();
3301     }
3302     gclog_or_tty->print_cr("----------");
3303     return false;
3304   }
3305 
3306   PrintRSetsClosure(const char* msg) : _msg(msg), _occupied_sum(0) {
3307     gclog_or_tty->cr();
3308     gclog_or_tty->print_cr("========================================");
3309     gclog_or_tty->print_cr("%s", msg);
3310     gclog_or_tty->cr();
3311   }
3312 
3313   ~PrintRSetsClosure() {
3314     gclog_or_tty->print_cr("Occupied Sum: " SIZE_FORMAT, _occupied_sum);
3315     gclog_or_tty->print_cr("========================================");
3316     gclog_or_tty->cr();
3317   }
3318 };
3319 
3320 void G1CollectedHeap::print_cset_rsets() {
3321   PrintRSetsClosure cl("Printing CSet RSets");
3322   collection_set_iterate(&cl);
3323 }
3324 
3325 void G1CollectedHeap::print_all_rsets() {
3326   PrintRSetsClosure cl("Printing All RSets");;
3327   heap_region_iterate(&cl);
3328 }
3329 #endif // PRODUCT
3330 
3331 G1HeapSummary G1CollectedHeap::create_g1_heap_summary() {
3332   YoungList* young_list = heap()->young_list();
3333 
3334   size_t eden_used_bytes = young_list->eden_used_bytes();
3335   size_t survivor_used_bytes = young_list->survivor_used_bytes();
3336 
3337   size_t eden_capacity_bytes =
3338     (g1_policy()->young_list_target_length() * HeapRegion::GrainBytes) - survivor_used_bytes;
3339 
3340   VirtualSpaceSummary heap_summary = create_heap_space_summary();
3341   return G1HeapSummary(heap_summary, used(), eden_used_bytes, eden_capacity_bytes, survivor_used_bytes);
3342 }
3343 
3344 G1EvacSummary G1CollectedHeap::create_g1_evac_summary(G1EvacStats* stats) {
3345   return G1EvacSummary(stats->allocated(), stats->wasted(), stats->undo_wasted(),
3346                        stats->unused(), stats->used(), stats->region_end_waste(),
3347                        stats->regions_filled(), stats->direct_allocated(),
3348                        stats->failure_used(), stats->failure_waste());
3349 }
3350 
3351 void G1CollectedHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) {
3352   const G1HeapSummary& heap_summary = create_g1_heap_summary();
3353   gc_tracer->report_gc_heap_summary(when, heap_summary);
3354 
3355   const MetaspaceSummary& metaspace_summary = create_metaspace_summary();
3356   gc_tracer->report_metaspace_summary(when, metaspace_summary);
3357 }
3358 
3359 
3360 G1CollectedHeap* G1CollectedHeap::heap() {
3361   CollectedHeap* heap = Universe::heap();
3362   assert(heap != NULL, "Uninitialized access to G1CollectedHeap::heap()");
3363   assert(heap->kind() == CollectedHeap::G1CollectedHeap, "Not a G1CollectedHeap");
3364   return (G1CollectedHeap*)heap;
3365 }
3366 
3367 void G1CollectedHeap::gc_prologue(bool full /* Ignored */) {
3368   // always_do_update_barrier = false;
3369   assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");
3370   // Fill TLAB's and such
3371   accumulate_statistics_all_tlabs();
3372   ensure_parsability(true);
3373 
3374   if (G1SummarizeRSetStats && (G1SummarizeRSetStatsPeriod > 0) &&
3375       (total_collections() % G1SummarizeRSetStatsPeriod == 0)) {
3376     g1_rem_set()->print_periodic_summary_info("Before GC RS summary");
3377   }
3378 }
3379 
3380 void G1CollectedHeap::gc_epilogue(bool full) {
3381 
3382   if (G1SummarizeRSetStats &&
3383       (G1SummarizeRSetStatsPeriod > 0) &&
3384       // we are at the end of the GC. Total collections has already been increased.
3385       ((total_collections() - 1) % G1SummarizeRSetStatsPeriod == 0)) {
3386     g1_rem_set()->print_periodic_summary_info("After GC RS summary");
3387   }
3388 
3389   // FIXME: what is this about?
3390   // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled"
3391   // is set.
3392 #if defined(COMPILER2) || INCLUDE_JVMCI
3393   assert(DerivedPointerTable::is_empty(), "derived pointer present");
3394 #endif
3395   // always_do_update_barrier = true;
3396 
3397   resize_all_tlabs();
3398   allocation_context_stats().update(full);
3399 
3400   // We have just completed a GC. Update the soft reference
3401   // policy with the new heap occupancy
3402   Universe::update_heap_info_at_gc();
3403 }
3404 
3405 HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size,
3406                                                uint gc_count_before,
3407                                                bool* succeeded,
3408                                                GCCause::Cause gc_cause) {
3409   assert_heap_not_locked_and_not_at_safepoint();
3410   g1_policy()->record_stop_world_start();
3411   VM_G1IncCollectionPause op(gc_count_before,
3412                              word_size,
3413                              false, /* should_initiate_conc_mark */
3414                              g1_policy()->max_pause_time_ms(),
3415                              gc_cause);
3416 
3417   op.set_allocation_context(AllocationContext::current());
3418   VMThread::execute(&op);
3419 
3420   HeapWord* result = op.result();
3421   bool ret_succeeded = op.prologue_succeeded() && op.pause_succeeded();
3422   assert(result == NULL || ret_succeeded,
3423          "the result should be NULL if the VM did not succeed");
3424   *succeeded = ret_succeeded;
3425 
3426   assert_heap_not_locked();
3427   return result;
3428 }
3429 
3430 void
3431 G1CollectedHeap::doConcurrentMark() {
3432   MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
3433   if (!_cmThread->in_progress()) {
3434     _cmThread->set_started();
3435     CGC_lock->notify();
3436   }
3437 }
3438 
3439 size_t G1CollectedHeap::pending_card_num() {
3440   size_t extra_cards = 0;
3441   JavaThread *curr = Threads::first();
3442   while (curr != NULL) {
3443     DirtyCardQueue& dcq = curr->dirty_card_queue();
3444     extra_cards += dcq.size();
3445     curr = curr->next();
3446   }
3447   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
3448   size_t buffer_size = dcqs.buffer_size();
3449   size_t buffer_num = dcqs.completed_buffers_num();
3450 
3451   // PtrQueueSet::buffer_size() and PtrQueue:size() return sizes
3452   // in bytes - not the number of 'entries'. We need to convert
3453   // into a number of cards.
3454   return (buffer_size * buffer_num + extra_cards) / oopSize;
3455 }
3456 
3457 class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure {
3458  private:
3459   size_t _total_humongous;
3460   size_t _candidate_humongous;
3461 
3462   DirtyCardQueue _dcq;
3463 
3464   // We don't nominate objects with many remembered set entries, on
3465   // the assumption that such objects are likely still live.
3466   bool is_remset_small(HeapRegion* region) const {
3467     HeapRegionRemSet* const rset = region->rem_set();
3468     return G1EagerReclaimHumongousObjectsWithStaleRefs
3469       ? rset->occupancy_less_or_equal_than(G1RSetSparseRegionEntries)
3470       : rset->is_empty();
3471   }
3472 
3473   bool is_typeArray_region(HeapRegion* region) const {
3474     return oop(region->bottom())->is_typeArray();
3475   }
3476 
3477   bool humongous_region_is_candidate(G1CollectedHeap* heap, HeapRegion* region) const {
3478     assert(region->is_starts_humongous(), "Must start a humongous object");
3479 
3480     // Candidate selection must satisfy the following constraints
3481     // while concurrent marking is in progress:
3482     //
3483     // * In order to maintain SATB invariants, an object must not be
3484     // reclaimed if it was allocated before the start of marking and
3485     // has not had its references scanned.  Such an object must have
3486     // its references (including type metadata) scanned to ensure no
3487     // live objects are missed by the marking process.  Objects
3488     // allocated after the start of concurrent marking don't need to
3489     // be scanned.
3490     //
3491     // * An object must not be reclaimed if it is on the concurrent
3492     // mark stack.  Objects allocated after the start of concurrent
3493     // marking are never pushed on the mark stack.
3494     //
3495     // Nominating only objects allocated after the start of concurrent
3496     // marking is sufficient to meet both constraints.  This may miss
3497     // some objects that satisfy the constraints, but the marking data
3498     // structures don't support efficiently performing the needed
3499     // additional tests or scrubbing of the mark stack.
3500     //
3501     // However, we presently only nominate is_typeArray() objects.
3502     // A humongous object containing references induces remembered
3503     // set entries on other regions.  In order to reclaim such an
3504     // object, those remembered sets would need to be cleaned up.
3505     //
3506     // We also treat is_typeArray() objects specially, allowing them
3507     // to be reclaimed even if allocated before the start of
3508     // concurrent mark.  For this we rely on mark stack insertion to
3509     // exclude is_typeArray() objects, preventing reclaiming an object
3510     // that is in the mark stack.  We also rely on the metadata for
3511     // such objects to be built-in and so ensured to be kept live.
3512     // Frequent allocation and drop of large binary blobs is an
3513     // important use case for eager reclaim, and this special handling
3514     // may reduce needed headroom.
3515 
3516     return is_typeArray_region(region) && is_remset_small(region);
3517   }
3518 
3519  public:
3520   RegisterHumongousWithInCSetFastTestClosure()
3521   : _total_humongous(0),
3522     _candidate_humongous(0),
3523     _dcq(&JavaThread::dirty_card_queue_set()) {
3524   }
3525 
3526   virtual bool doHeapRegion(HeapRegion* r) {
3527     if (!r->is_starts_humongous()) {
3528       return false;
3529     }
3530     G1CollectedHeap* g1h = G1CollectedHeap::heap();
3531 
3532     bool is_candidate = humongous_region_is_candidate(g1h, r);
3533     uint rindex = r->hrm_index();
3534     g1h->set_humongous_reclaim_candidate(rindex, is_candidate);
3535     if (is_candidate) {
3536       _candidate_humongous++;
3537       g1h->register_humongous_region_with_cset(rindex);
3538       // Is_candidate already filters out humongous object with large remembered sets.
3539       // If we have a humongous object with a few remembered sets, we simply flush these
3540       // remembered set entries into the DCQS. That will result in automatic
3541       // re-evaluation of their remembered set entries during the following evacuation
3542       // phase.
3543       if (!r->rem_set()->is_empty()) {
3544         guarantee(r->rem_set()->occupancy_less_or_equal_than(G1RSetSparseRegionEntries),
3545                   "Found a not-small remembered set here. This is inconsistent with previous assumptions.");
3546         G1SATBCardTableLoggingModRefBS* bs = g1h->g1_barrier_set();
3547         HeapRegionRemSetIterator hrrs(r->rem_set());
3548         size_t card_index;
3549         while (hrrs.has_next(card_index)) {
3550           jbyte* card_ptr = (jbyte*)bs->byte_for_index(card_index);
3551           // The remembered set might contain references to already freed
3552           // regions. Filter out such entries to avoid failing card table
3553           // verification.
3554           if (g1h->is_in_closed_subset(bs->addr_for(card_ptr))) {
3555             if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
3556               *card_ptr = CardTableModRefBS::dirty_card_val();
3557               _dcq.enqueue(card_ptr);
3558             }
3559           }
3560         }
3561         assert(hrrs.n_yielded() == r->rem_set()->occupied(),
3562                "Remembered set hash maps out of sync, cur: " SIZE_FORMAT " entries, next: " SIZE_FORMAT " entries",
3563                hrrs.n_yielded(), r->rem_set()->occupied());
3564         r->rem_set()->clear_locked();
3565       }
3566       assert(r->rem_set()->is_empty(), "At this point any humongous candidate remembered set must be empty.");
3567     }
3568     _total_humongous++;
3569 
3570     return false;
3571   }
3572 
3573   size_t total_humongous() const { return _total_humongous; }
3574   size_t candidate_humongous() const { return _candidate_humongous; }
3575 
3576   void flush_rem_set_entries() { _dcq.flush(); }
3577 };
3578 
3579 void G1CollectedHeap::register_humongous_regions_with_cset() {
3580   if (!G1EagerReclaimHumongousObjects) {
3581     g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0.0, 0, 0);
3582     return;
3583   }
3584   double time = os::elapsed_counter();
3585 
3586   // Collect reclaim candidate information and register candidates with cset.
3587   RegisterHumongousWithInCSetFastTestClosure cl;
3588   heap_region_iterate(&cl);
3589 
3590   time = ((double)(os::elapsed_counter() - time) / os::elapsed_frequency()) * 1000.0;
3591   g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(time,
3592                                                                   cl.total_humongous(),
3593                                                                   cl.candidate_humongous());
3594   _has_humongous_reclaim_candidates = cl.candidate_humongous() > 0;
3595 
3596   // Finally flush all remembered set entries to re-check into the global DCQS.
3597   cl.flush_rem_set_entries();
3598 }
3599 
3600 #ifdef ASSERT
3601 class VerifyCSetClosure: public HeapRegionClosure {
3602 public:
3603   bool doHeapRegion(HeapRegion* hr) {
3604     // Here we check that the CSet region's RSet is ready for parallel
3605     // iteration. The fields that we'll verify are only manipulated
3606     // when the region is part of a CSet and is collected. Afterwards,
3607     // we reset these fields when we clear the region's RSet (when the
3608     // region is freed) so they are ready when the region is
3609     // re-allocated. The only exception to this is if there's an
3610     // evacuation failure and instead of freeing the region we leave
3611     // it in the heap. In that case, we reset these fields during
3612     // evacuation failure handling.
3613     guarantee(hr->rem_set()->verify_ready_for_par_iteration(), "verification");
3614 
3615     // Here's a good place to add any other checks we'd like to
3616     // perform on CSet regions.
3617     return false;
3618   }
3619 };
3620 #endif // ASSERT
3621 
3622 uint G1CollectedHeap::num_task_queues() const {
3623   return _task_queues->size();
3624 }
3625 
3626 #if TASKQUEUE_STATS
3627 void G1CollectedHeap::print_taskqueue_stats_hdr(outputStream* const st) {
3628   st->print_raw_cr("GC Task Stats");
3629   st->print_raw("thr "); TaskQueueStats::print_header(1, st); st->cr();
3630   st->print_raw("--- "); TaskQueueStats::print_header(2, st); st->cr();
3631 }
3632 
3633 void G1CollectedHeap::print_taskqueue_stats(outputStream* const st) const {
3634   print_taskqueue_stats_hdr(st);
3635 
3636   TaskQueueStats totals;
3637   const uint n = num_task_queues();
3638   for (uint i = 0; i < n; ++i) {
3639     st->print("%3u ", i); task_queue(i)->stats.print(st); st->cr();
3640     totals += task_queue(i)->stats;
3641   }
3642   st->print_raw("tot "); totals.print(st); st->cr();
3643 
3644   DEBUG_ONLY(totals.verify());
3645 }
3646 
3647 void G1CollectedHeap::reset_taskqueue_stats() {
3648   const uint n = num_task_queues();
3649   for (uint i = 0; i < n; ++i) {
3650     task_queue(i)->stats.reset();
3651   }
3652 }
3653 #endif // TASKQUEUE_STATS
3654 
3655 void G1CollectedHeap::log_gc_header() {
3656   if (!G1Log::fine()) {
3657     return;
3658   }
3659 
3660   gclog_or_tty->gclog_stamp();
3661 
3662   GCCauseString gc_cause_str = GCCauseString("GC pause", gc_cause())
3663     .append(collector_state()->gcs_are_young() ? "(young)" : "(mixed)")
3664     .append(collector_state()->during_initial_mark_pause() ? " (initial-mark)" : "");
3665 
3666   gclog_or_tty->print("[%s", (const char*)gc_cause_str);
3667 }
3668 
3669 void G1CollectedHeap::log_gc_footer(double pause_time_sec) {
3670   if (!G1Log::fine()) {
3671     return;
3672   }
3673 
3674   if (G1Log::finer()) {
3675     if (evacuation_failed()) {
3676       gclog_or_tty->print(" (to-space exhausted)");
3677     }
3678     gclog_or_tty->print_cr(", %3.7f secs]", pause_time_sec);
3679     g1_policy()->print_phases(pause_time_sec);
3680     g1_policy()->print_detailed_heap_transition();
3681   } else {
3682     if (evacuation_failed()) {
3683       gclog_or_tty->print("--");
3684     }
3685     g1_policy()->print_heap_transition();
3686     gclog_or_tty->print_cr(", %3.7f secs]", pause_time_sec);
3687   }
3688   gclog_or_tty->flush();
3689 }
3690 
3691 void G1CollectedHeap::wait_for_root_region_scanning() {
3692   double scan_wait_start = os::elapsedTime();
3693   // We have to wait until the CM threads finish scanning the
3694   // root regions as it's the only way to ensure that all the
3695   // objects on them have been correctly scanned before we start
3696   // moving them during the GC.
3697   bool waited = _cm->root_regions()->wait_until_scan_finished();
3698   double wait_time_ms = 0.0;
3699   if (waited) {
3700     double scan_wait_end = os::elapsedTime();
3701     wait_time_ms = (scan_wait_end - scan_wait_start) * 1000.0;
3702   }
3703   g1_policy()->phase_times()->record_root_region_scan_wait_time(wait_time_ms);
3704 }
3705 
3706 bool
3707 G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ms) {
3708   assert_at_safepoint(true /* should_be_vm_thread */);
3709   guarantee(!is_gc_active(), "collection is not reentrant");
3710 
3711   if (GC_locker::check_active_before_gc()) {
3712     return false;
3713   }
3714 
3715   _gc_timer_stw->register_gc_start();
3716 
3717   GCIdMark gc_id_mark;
3718   _gc_tracer_stw->report_gc_start(gc_cause(), _gc_timer_stw->gc_start());
3719 
3720   SvcGCMarker sgcm(SvcGCMarker::MINOR);
3721   ResourceMark rm;
3722 
3723   wait_for_root_region_scanning();
3724 
3725   G1Log::update_level();
3726   print_heap_before_gc();
3727   trace_heap_before_gc(_gc_tracer_stw);
3728 
3729   verify_region_sets_optional();
3730   verify_dirty_young_regions();
3731 
3732   // This call will decide whether this pause is an initial-mark
3733   // pause. If it is, during_initial_mark_pause() will return true
3734   // for the duration of this pause.
3735   g1_policy()->decide_on_conc_mark_initiation();
3736 
3737   // We do not allow initial-mark to be piggy-backed on a mixed GC.
3738   assert(!collector_state()->during_initial_mark_pause() ||
3739           collector_state()->gcs_are_young(), "sanity");
3740 
3741   // We also do not allow mixed GCs during marking.
3742   assert(!collector_state()->mark_in_progress() || collector_state()->gcs_are_young(), "sanity");
3743 
3744   // Record whether this pause is an initial mark. When the current
3745   // thread has completed its logging output and it's safe to signal
3746   // the CM thread, the flag's value in the policy has been reset.
3747   bool should_start_conc_mark = collector_state()->during_initial_mark_pause();
3748 
3749   // Inner scope for scope based logging, timers, and stats collection
3750   {
3751     EvacuationInfo evacuation_info;
3752 
3753     if (collector_state()->during_initial_mark_pause()) {
3754       // We are about to start a marking cycle, so we increment the
3755       // full collection counter.
3756       increment_old_marking_cycles_started();
3757       register_concurrent_cycle_start(_gc_timer_stw->gc_start());
3758     }
3759 
3760     _gc_tracer_stw->report_yc_type(collector_state()->yc_type());
3761 
3762     TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty);
3763 
3764     uint active_workers = AdaptiveSizePolicy::calc_active_workers(workers()->total_workers(),
3765                                                                   workers()->active_workers(),
3766                                                                   Threads::number_of_non_daemon_threads());
3767     workers()->set_active_workers(active_workers);
3768 
3769     double pause_start_sec = os::elapsedTime();
3770     g1_policy()->note_gc_start(active_workers);
3771     log_gc_header();
3772 
3773     TraceCollectorStats tcs(g1mm()->incremental_collection_counters());
3774     TraceMemoryManagerStats tms(false /* fullGC */, gc_cause());
3775 
3776     // If the secondary_free_list is not empty, append it to the
3777     // free_list. No need to wait for the cleanup operation to finish;
3778     // the region allocation code will check the secondary_free_list
3779     // and wait if necessary. If the G1StressConcRegionFreeing flag is
3780     // set, skip this step so that the region allocation code has to
3781     // get entries from the secondary_free_list.
3782     if (!G1StressConcRegionFreeing) {
3783       append_secondary_free_list_if_not_empty_with_lock();
3784     }
3785 
3786     assert(check_young_list_well_formed(), "young list should be well formed");
3787 
3788     // Don't dynamically change the number of GC threads this early.  A value of
3789     // 0 is used to indicate serial work.  When parallel work is done,
3790     // it will be set.
3791 
3792     { // Call to jvmpi::post_class_unload_events must occur outside of active GC
3793       IsGCActiveMark x;
3794 
3795       gc_prologue(false);
3796       increment_total_collections(false /* full gc */);
3797       increment_gc_time_stamp();
3798 
3799       verify_before_gc();
3800 
3801       check_bitmaps("GC Start");
3802 
3803 #if defined(COMPILER2) || INCLUDE_JVMCI
3804       DerivedPointerTable::clear();
3805 #endif
3806 
3807       // Please see comment in g1CollectedHeap.hpp and
3808       // G1CollectedHeap::ref_processing_init() to see how
3809       // reference processing currently works in G1.
3810 
3811       // Enable discovery in the STW reference processor
3812       ref_processor_stw()->enable_discovery();
3813 
3814       {
3815         // We want to temporarily turn off discovery by the
3816         // CM ref processor, if necessary, and turn it back on
3817         // on again later if we do. Using a scoped
3818         // NoRefDiscovery object will do this.
3819         NoRefDiscovery no_cm_discovery(ref_processor_cm());
3820 
3821         // Forget the current alloc region (we might even choose it to be part
3822         // of the collection set!).
3823         _allocator->release_mutator_alloc_region();
3824 
3825         // We should call this after we retire the mutator alloc
3826         // region(s) so that all the ALLOC / RETIRE events are generated
3827         // before the start GC event.
3828         _hr_printer.start_gc(false /* full */, (size_t) total_collections());
3829 
3830         // This timing is only used by the ergonomics to handle our pause target.
3831         // It is unclear why this should not include the full pause. We will
3832         // investigate this in CR 7178365.
3833         //
3834         // Preserving the old comment here if that helps the investigation:
3835         //
3836         // The elapsed time induced by the start time below deliberately elides
3837         // the possible verification above.
3838         double sample_start_time_sec = os::elapsedTime();
3839 
3840         g1_policy()->record_collection_pause_start(sample_start_time_sec);
3841 
3842         if (collector_state()->during_initial_mark_pause()) {
3843           concurrent_mark()->checkpointRootsInitialPre();
3844         }
3845 
3846         double time_remaining_ms = g1_policy()->finalize_young_cset_part(target_pause_time_ms);
3847         g1_policy()->finalize_old_cset_part(time_remaining_ms);
3848 
3849         evacuation_info.set_collectionset_regions(g1_policy()->cset_region_length());
3850 
3851         // Make sure the remembered sets are up to date. This needs to be
3852         // done before register_humongous_regions_with_cset(), because the
3853         // remembered sets are used there to choose eager reclaim candidates.
3854         // If the remembered sets are not up to date we might miss some
3855         // entries that need to be handled.
3856         g1_rem_set()->cleanupHRRS();
3857 
3858         register_humongous_regions_with_cset();
3859 
3860         assert(check_cset_fast_test(), "Inconsistency in the InCSetState table.");
3861 
3862         _cm->note_start_of_gc();
3863         // We call this after finalize_cset() to
3864         // ensure that the CSet has been finalized.
3865         _cm->verify_no_cset_oops();
3866 
3867         if (_hr_printer.is_active()) {
3868           HeapRegion* hr = g1_policy()->collection_set();
3869           while (hr != NULL) {
3870             _hr_printer.cset(hr);
3871             hr = hr->next_in_collection_set();
3872           }
3873         }
3874 
3875 #ifdef ASSERT
3876         VerifyCSetClosure cl;
3877         collection_set_iterate(&cl);
3878 #endif // ASSERT
3879 
3880         // Initialize the GC alloc regions.
3881         _allocator->init_gc_alloc_regions(evacuation_info);
3882 
3883         G1ParScanThreadStateSet per_thread_states(this, workers()->active_workers(), g1_policy()->young_cset_region_length());
3884         pre_evacuate_collection_set();
3885 
3886         // Actually do the work...
3887         evacuate_collection_set(evacuation_info, &per_thread_states);
3888 
3889         post_evacuate_collection_set(evacuation_info, &per_thread_states);
3890 
3891         const size_t* surviving_young_words = per_thread_states.surviving_young_words();
3892         free_collection_set(g1_policy()->collection_set(), evacuation_info, surviving_young_words);
3893 
3894         eagerly_reclaim_humongous_regions();
3895 
3896         g1_policy()->clear_collection_set();
3897 
3898         // Start a new incremental collection set for the next pause.
3899         g1_policy()->start_incremental_cset_building();
3900 
3901         clear_cset_fast_test();
3902 
3903         _young_list->reset_sampled_info();
3904 
3905         // Don't check the whole heap at this point as the
3906         // GC alloc regions from this pause have been tagged
3907         // as survivors and moved on to the survivor list.
3908         // Survivor regions will fail the !is_young() check.
3909         assert(check_young_list_empty(false /* check_heap */),
3910           "young list should be empty");
3911 
3912         g1_policy()->record_survivor_regions(_young_list->survivor_length(),
3913                                              _young_list->first_survivor_region(),
3914                                              _young_list->last_survivor_region());
3915 
3916         _young_list->reset_auxilary_lists();
3917 
3918         if (evacuation_failed()) {
3919           set_used(recalculate_used());
3920           if (_archive_allocator != NULL) {
3921             _archive_allocator->clear_used();
3922           }
3923           for (uint i = 0; i < ParallelGCThreads; i++) {
3924             if (_evacuation_failed_info_array[i].has_failed()) {
3925               _gc_tracer_stw->report_evacuation_failed(_evacuation_failed_info_array[i]);
3926             }
3927           }
3928         } else {
3929           // The "used" of the the collection set have already been subtracted
3930           // when they were freed.  Add in the bytes evacuated.
3931           increase_used(g1_policy()->bytes_copied_during_gc());
3932         }
3933 
3934         if (collector_state()->during_initial_mark_pause()) {
3935           // We have to do this before we notify the CM threads that
3936           // they can start working to make sure that all the
3937           // appropriate initialization is done on the CM object.
3938           concurrent_mark()->checkpointRootsInitialPost();
3939           collector_state()->set_mark_in_progress(true);
3940           // Note that we don't actually trigger the CM thread at
3941           // this point. We do that later when we're sure that
3942           // the current thread has completed its logging output.
3943         }
3944 
3945         allocate_dummy_regions();
3946 
3947         _allocator->init_mutator_alloc_region();
3948 
3949         {
3950           size_t expand_bytes = g1_policy()->expansion_amount();
3951           if (expand_bytes > 0) {
3952             size_t bytes_before = capacity();
3953             // No need for an ergo verbose message here,
3954             // expansion_amount() does this when it returns a value > 0.
3955             double expand_ms;
3956             if (!expand(expand_bytes, &expand_ms)) {
3957               // We failed to expand the heap. Cannot do anything about it.
3958             }
3959             g1_policy()->phase_times()->record_expand_heap_time(expand_ms);
3960           }
3961         }
3962 
3963         // We redo the verification but now wrt to the new CSet which
3964         // has just got initialized after the previous CSet was freed.
3965         _cm->verify_no_cset_oops();
3966         _cm->note_end_of_gc();
3967 
3968         // This timing is only used by the ergonomics to handle our pause target.
3969         // It is unclear why this should not include the full pause. We will
3970         // investigate this in CR 7178365.
3971         double sample_end_time_sec = os::elapsedTime();
3972         double pause_time_ms = (sample_end_time_sec - sample_start_time_sec) * MILLIUNITS;
3973         size_t total_cards_scanned = per_thread_states.total_cards_scanned();
3974         g1_policy()->record_collection_pause_end(pause_time_ms, total_cards_scanned);
3975 
3976         evacuation_info.set_collectionset_used_before(g1_policy()->collection_set_bytes_used_before());
3977         evacuation_info.set_bytes_copied(g1_policy()->bytes_copied_during_gc());
3978 
3979         MemoryService::track_memory_usage();
3980 
3981         // In prepare_for_verify() below we'll need to scan the deferred
3982         // update buffers to bring the RSets up-to-date if
3983         // G1HRRSFlushLogBuffersOnVerify has been set. While scanning
3984         // the update buffers we'll probably need to scan cards on the
3985         // regions we just allocated to (i.e., the GC alloc
3986         // regions). However, during the last GC we called
3987         // set_saved_mark() on all the GC alloc regions, so card
3988         // scanning might skip the [saved_mark_word()...top()] area of
3989         // those regions (i.e., the area we allocated objects into
3990         // during the last GC). But it shouldn't. Given that
3991         // saved_mark_word() is conditional on whether the GC time stamp
3992         // on the region is current or not, by incrementing the GC time
3993         // stamp here we invalidate all the GC time stamps on all the
3994         // regions and saved_mark_word() will simply return top() for
3995         // all the regions. This is a nicer way of ensuring this rather
3996         // than iterating over the regions and fixing them. In fact, the
3997         // GC time stamp increment here also ensures that
3998         // saved_mark_word() will return top() between pauses, i.e.,
3999         // during concurrent refinement. So we don't need the
4000         // is_gc_active() check to decided which top to use when
4001         // scanning cards (see CR 7039627).
4002         increment_gc_time_stamp();
4003 
4004         verify_after_gc();
4005         check_bitmaps("GC End");
4006 
4007         assert(!ref_processor_stw()->discovery_enabled(), "Postcondition");
4008         ref_processor_stw()->verify_no_references_recorded();
4009 
4010         // CM reference discovery will be re-enabled if necessary.
4011       }
4012 
4013       // We should do this after we potentially expand the heap so
4014       // that all the COMMIT events are generated before the end GC
4015       // event, and after we retire the GC alloc regions so that all
4016       // RETIRE events are generated before the end GC event.
4017       _hr_printer.end_gc(false /* full */, (size_t) total_collections());
4018 
4019 #ifdef TRACESPINNING
4020       ParallelTaskTerminator::print_termination_counts();
4021 #endif
4022 
4023       gc_epilogue(false);
4024     }
4025 
4026     // Print the remainder of the GC log output.
4027     log_gc_footer(os::elapsedTime() - pause_start_sec);
4028 
4029     // It is not yet to safe to tell the concurrent mark to
4030     // start as we have some optional output below. We don't want the
4031     // output from the concurrent mark thread interfering with this
4032     // logging output either.
4033 
4034     _hrm.verify_optional();
4035     verify_region_sets_optional();
4036 
4037     TASKQUEUE_STATS_ONLY(if (PrintTaskqueue) print_taskqueue_stats());
4038     TASKQUEUE_STATS_ONLY(reset_taskqueue_stats());
4039 
4040     print_heap_after_gc();
4041     trace_heap_after_gc(_gc_tracer_stw);
4042 
4043     // We must call G1MonitoringSupport::update_sizes() in the same scoping level
4044     // as an active TraceMemoryManagerStats object (i.e. before the destructor for the
4045     // TraceMemoryManagerStats is called) so that the G1 memory pools are updated
4046     // before any GC notifications are raised.
4047     g1mm()->update_sizes();
4048 
4049     _gc_tracer_stw->report_evacuation_info(&evacuation_info);
4050     _gc_tracer_stw->report_tenuring_threshold(_g1_policy->tenuring_threshold());
4051     _gc_timer_stw->register_gc_end();
4052     _gc_tracer_stw->report_gc_end(_gc_timer_stw->gc_end(), _gc_timer_stw->time_partitions());
4053   }
4054   // It should now be safe to tell the concurrent mark thread to start
4055   // without its logging output interfering with the logging output
4056   // that came from the pause.
4057 
4058   if (should_start_conc_mark) {
4059     // CAUTION: after the doConcurrentMark() call below,
4060     // the concurrent marking thread(s) could be running
4061     // concurrently with us. Make sure that anything after
4062     // this point does not assume that we are the only GC thread
4063     // running. Note: of course, the actual marking work will
4064     // not start until the safepoint itself is released in
4065     // SuspendibleThreadSet::desynchronize().
4066     doConcurrentMark();
4067   }
4068 
4069   return true;
4070 }
4071 
4072 void G1CollectedHeap::remove_self_forwarding_pointers() {
4073   double remove_self_forwards_start = os::elapsedTime();
4074 
4075   G1ParRemoveSelfForwardPtrsTask rsfp_task;
4076   workers()->run_task(&rsfp_task);
4077 
4078   // Now restore saved marks, if any.
4079   for (uint i = 0; i < ParallelGCThreads; i++) {
4080     OopAndMarkOopStack& cur = _preserved_objs[i];
4081     while (!cur.is_empty()) {
4082       OopAndMarkOop elem = cur.pop();
4083       elem.set_mark();
4084     }
4085     cur.clear(true);
4086   }
4087 
4088   g1_policy()->phase_times()->record_evac_fail_remove_self_forwards((os::elapsedTime() - remove_self_forwards_start) * 1000.0);
4089 }
4090 
4091 void G1CollectedHeap::preserve_mark_during_evac_failure(uint worker_id, oop obj, markOop m) {
4092   if (!_evacuation_failed) {
4093     _evacuation_failed = true;
4094   }
4095 
4096   _evacuation_failed_info_array[worker_id].register_copy_failure(obj->size());
4097 
4098   // We want to call the "for_promotion_failure" version only in the
4099   // case of a promotion failure.
4100   if (m->must_be_preserved_for_promotion_failure(obj)) {
4101     OopAndMarkOop elem(obj, m);
4102     _preserved_objs[worker_id].push(elem);
4103   }
4104 }
4105 
4106 class G1ParEvacuateFollowersClosure : public VoidClosure {
4107 private:
4108   double _start_term;
4109   double _term_time;
4110   size_t _term_attempts;
4111 
4112   void start_term_time() { _term_attempts++; _start_term = os::elapsedTime(); }
4113   void end_term_time() { _term_time += os::elapsedTime() - _start_term; }
4114 protected:
4115   G1CollectedHeap*              _g1h;
4116   G1ParScanThreadState*         _par_scan_state;
4117   RefToScanQueueSet*            _queues;
4118   ParallelTaskTerminator*       _terminator;
4119 
4120   G1ParScanThreadState*   par_scan_state() { return _par_scan_state; }
4121   RefToScanQueueSet*      queues()         { return _queues; }
4122   ParallelTaskTerminator* terminator()     { return _terminator; }
4123 
4124 public:
4125   G1ParEvacuateFollowersClosure(G1CollectedHeap* g1h,
4126                                 G1ParScanThreadState* par_scan_state,
4127                                 RefToScanQueueSet* queues,
4128                                 ParallelTaskTerminator* terminator)
4129     : _g1h(g1h), _par_scan_state(par_scan_state),
4130       _queues(queues), _terminator(terminator),
4131       _start_term(0.0), _term_time(0.0), _term_attempts(0) {}
4132 
4133   void do_void();
4134 
4135   double term_time() const { return _term_time; }
4136   size_t term_attempts() const { return _term_attempts; }
4137 
4138 private:
4139   inline bool offer_termination();
4140 };
4141 
4142 bool G1ParEvacuateFollowersClosure::offer_termination() {
4143   G1ParScanThreadState* const pss = par_scan_state();
4144   start_term_time();
4145   const bool res = terminator()->offer_termination();
4146   end_term_time();
4147   return res;
4148 }
4149 
4150 void G1ParEvacuateFollowersClosure::do_void() {
4151   G1ParScanThreadState* const pss = par_scan_state();
4152   pss->trim_queue();
4153   do {
4154     pss->steal_and_trim_queue(queues());
4155   } while (!offer_termination());
4156 }
4157 
4158 class G1ParTask : public AbstractGangTask {
4159 protected:
4160   G1CollectedHeap*         _g1h;
4161   G1ParScanThreadStateSet* _pss;
4162   RefToScanQueueSet*       _queues;
4163   G1RootProcessor*         _root_processor;
4164   ParallelTaskTerminator   _terminator;
4165   uint                     _n_workers;
4166 
4167 public:
4168   G1ParTask(G1CollectedHeap* g1h, G1ParScanThreadStateSet* per_thread_states, RefToScanQueueSet *task_queues, G1RootProcessor* root_processor, uint n_workers)
4169     : AbstractGangTask("G1 collection"),
4170       _g1h(g1h),
4171       _pss(per_thread_states),
4172       _queues(task_queues),
4173       _root_processor(root_processor),
4174       _terminator(n_workers, _queues),
4175       _n_workers(n_workers)
4176   {}
4177 
4178   void work(uint worker_id) {
4179     if (worker_id >= _n_workers) return;  // no work needed this round
4180 
4181     double start_sec = os::elapsedTime();
4182     _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::GCWorkerStart, worker_id, start_sec);
4183 
4184     {
4185       ResourceMark rm;
4186       HandleMark   hm;
4187 
4188       ReferenceProcessor*             rp = _g1h->ref_processor_stw();
4189 
4190       G1ParScanThreadState*           pss = _pss->state_for_worker(worker_id);
4191       pss->set_ref_processor(rp);
4192 
4193       double start_strong_roots_sec = os::elapsedTime();
4194 
4195       _root_processor->evacuate_roots(pss->closures(), worker_id);
4196 
4197       G1ParPushHeapRSClosure push_heap_rs_cl(_g1h, pss);
4198 
4199       // We pass a weak code blobs closure to the remembered set scanning because we want to avoid
4200       // treating the nmethods visited to act as roots for concurrent marking.
4201       // We only want to make sure that the oops in the nmethods are adjusted with regard to the
4202       // objects copied by the current evacuation.
4203       size_t cards_scanned = _g1h->g1_rem_set()->oops_into_collection_set_do(&push_heap_rs_cl,
4204                                                                              pss->closures()->weak_codeblobs(),
4205                                                                              worker_id);
4206 
4207       _pss->add_cards_scanned(worker_id, cards_scanned);
4208 
4209       double strong_roots_sec = os::elapsedTime() - start_strong_roots_sec;
4210 
4211       double term_sec = 0.0;
4212       size_t evac_term_attempts = 0;
4213       {
4214         double start = os::elapsedTime();
4215         G1ParEvacuateFollowersClosure evac(_g1h, pss, _queues, &_terminator);
4216         evac.do_void();
4217 
4218         evac_term_attempts = evac.term_attempts();
4219         term_sec = evac.term_time();
4220         double elapsed_sec = os::elapsedTime() - start;
4221         _g1h->g1_policy()->phase_times()->add_time_secs(G1GCPhaseTimes::ObjCopy, worker_id, elapsed_sec - term_sec);
4222         _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::Termination, worker_id, term_sec);
4223         _g1h->g1_policy()->phase_times()->record_thread_work_item(G1GCPhaseTimes::Termination, worker_id, evac_term_attempts);
4224       }
4225 
4226       assert(pss->queue_is_empty(), "should be empty");
4227 
4228       if (PrintTerminationStats) {
4229         MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
4230         size_t lab_waste;
4231         size_t lab_undo_waste;
4232         pss->waste(lab_waste, lab_undo_waste);
4233         _g1h->print_termination_stats(gclog_or_tty,
4234                                       worker_id,
4235                                       (os::elapsedTime() - start_sec) * 1000.0,   /* elapsed time */
4236                                       strong_roots_sec * 1000.0,                  /* strong roots time */
4237                                       term_sec * 1000.0,                          /* evac term time */
4238                                       evac_term_attempts,                         /* evac term attempts */
4239                                       lab_waste,                                  /* alloc buffer waste */
4240                                       lab_undo_waste                              /* undo waste */
4241                                       );
4242       }
4243 
4244       // Close the inner scope so that the ResourceMark and HandleMark
4245       // destructors are executed here and are included as part of the
4246       // "GC Worker Time".
4247     }
4248     _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::GCWorkerEnd, worker_id, os::elapsedTime());
4249   }
4250 };
4251 
4252 void G1CollectedHeap::print_termination_stats_hdr(outputStream* const st) {
4253   st->print_raw_cr("GC Termination Stats");
4254   st->print_raw_cr("     elapsed  --strong roots-- -------termination------- ------waste (KiB)------");
4255   st->print_raw_cr("thr     ms        ms      %        ms      %    attempts  total   alloc    undo");
4256   st->print_raw_cr("--- --------- --------- ------ --------- ------ -------- ------- ------- -------");
4257 }
4258 
4259 void G1CollectedHeap::print_termination_stats(outputStream* const st,
4260                                               uint worker_id,
4261                                               double elapsed_ms,
4262                                               double strong_roots_ms,
4263                                               double term_ms,
4264                                               size_t term_attempts,
4265                                               size_t alloc_buffer_waste,
4266                                               size_t undo_waste) const {
4267   st->print_cr("%3d %9.2f %9.2f %6.2f "
4268                "%9.2f %6.2f " SIZE_FORMAT_W(8) " "
4269                SIZE_FORMAT_W(7) " " SIZE_FORMAT_W(7) " " SIZE_FORMAT_W(7),
4270                worker_id, elapsed_ms, strong_roots_ms, strong_roots_ms * 100 / elapsed_ms,
4271                term_ms, term_ms * 100 / elapsed_ms, term_attempts,
4272                (alloc_buffer_waste + undo_waste) * HeapWordSize / K,
4273                alloc_buffer_waste * HeapWordSize / K,
4274                undo_waste * HeapWordSize / K);
4275 }
4276 
4277 class G1StringSymbolTableUnlinkTask : public AbstractGangTask {
4278 private:
4279   BoolObjectClosure* _is_alive;
4280   int _initial_string_table_size;
4281   int _initial_symbol_table_size;
4282 
4283   bool  _process_strings;
4284   int _strings_processed;
4285   int _strings_removed;
4286 
4287   bool  _process_symbols;
4288   int _symbols_processed;
4289   int _symbols_removed;
4290 
4291 public:
4292   G1StringSymbolTableUnlinkTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) :
4293     AbstractGangTask("String/Symbol Unlinking"),
4294     _is_alive(is_alive),
4295     _process_strings(process_strings), _strings_processed(0), _strings_removed(0),
4296     _process_symbols(process_symbols), _symbols_processed(0), _symbols_removed(0) {
4297 
4298     _initial_string_table_size = StringTable::the_table()->table_size();
4299     _initial_symbol_table_size = SymbolTable::the_table()->table_size();
4300     if (process_strings) {
4301       StringTable::clear_parallel_claimed_index();
4302     }
4303     if (process_symbols) {
4304       SymbolTable::clear_parallel_claimed_index();
4305     }
4306   }
4307 
4308   ~G1StringSymbolTableUnlinkTask() {
4309     guarantee(!_process_strings || StringTable::parallel_claimed_index() >= _initial_string_table_size,
4310               "claim value %d after unlink less than initial string table size %d",
4311               StringTable::parallel_claimed_index(), _initial_string_table_size);
4312     guarantee(!_process_symbols || SymbolTable::parallel_claimed_index() >= _initial_symbol_table_size,
4313               "claim value %d after unlink less than initial symbol table size %d",
4314               SymbolTable::parallel_claimed_index(), _initial_symbol_table_size);
4315 
4316     if (G1TraceStringSymbolTableScrubbing) {
4317       gclog_or_tty->print_cr("Cleaned string and symbol table, "
4318                              "strings: " SIZE_FORMAT " processed, " SIZE_FORMAT " removed, "
4319                              "symbols: " SIZE_FORMAT " processed, " SIZE_FORMAT " removed",
4320                              strings_processed(), strings_removed(),
4321                              symbols_processed(), symbols_removed());
4322     }
4323   }
4324 
4325   void work(uint worker_id) {
4326     int strings_processed = 0;
4327     int strings_removed = 0;
4328     int symbols_processed = 0;
4329     int symbols_removed = 0;
4330     if (_process_strings) {
4331       StringTable::possibly_parallel_unlink(_is_alive, &strings_processed, &strings_removed);
4332       Atomic::add(strings_processed, &_strings_processed);
4333       Atomic::add(strings_removed, &_strings_removed);
4334     }
4335     if (_process_symbols) {
4336       SymbolTable::possibly_parallel_unlink(&symbols_processed, &symbols_removed);
4337       Atomic::add(symbols_processed, &_symbols_processed);
4338       Atomic::add(symbols_removed, &_symbols_removed);
4339     }
4340   }
4341 
4342   size_t strings_processed() const { return (size_t)_strings_processed; }
4343   size_t strings_removed()   const { return (size_t)_strings_removed; }
4344 
4345   size_t symbols_processed() const { return (size_t)_symbols_processed; }
4346   size_t symbols_removed()   const { return (size_t)_symbols_removed; }
4347 };
4348 
4349 class G1CodeCacheUnloadingTask VALUE_OBJ_CLASS_SPEC {
4350 private:
4351   static Monitor* _lock;
4352 
4353   BoolObjectClosure* const _is_alive;
4354   const bool               _unloading_occurred;
4355   const uint               _num_workers;
4356 
4357   // Variables used to claim nmethods.
4358   nmethod* _first_nmethod;
4359   volatile nmethod* _claimed_nmethod;
4360 
4361   // The list of nmethods that need to be processed by the second pass.
4362   volatile nmethod* _postponed_list;
4363   volatile uint     _num_entered_barrier;
4364 
4365  public:
4366   G1CodeCacheUnloadingTask(uint num_workers, BoolObjectClosure* is_alive, bool unloading_occurred) :
4367       _is_alive(is_alive),
4368       _unloading_occurred(unloading_occurred),
4369       _num_workers(num_workers),
4370       _first_nmethod(NULL),
4371       _claimed_nmethod(NULL),
4372       _postponed_list(NULL),
4373       _num_entered_barrier(0)
4374   {
4375     nmethod::increase_unloading_clock();
4376     // Get first alive nmethod
4377     NMethodIterator iter = NMethodIterator();
4378     if(iter.next_alive()) {
4379       _first_nmethod = iter.method();
4380     }
4381     _claimed_nmethod = (volatile nmethod*)_first_nmethod;
4382   }
4383 
4384   ~G1CodeCacheUnloadingTask() {
4385     CodeCache::verify_clean_inline_caches();
4386 
4387     CodeCache::set_needs_cache_clean(false);
4388     guarantee(CodeCache::scavenge_root_nmethods() == NULL, "Must be");
4389 
4390     CodeCache::verify_icholder_relocations();
4391   }
4392 
4393  private:
4394   void add_to_postponed_list(nmethod* nm) {
4395       nmethod* old;
4396       do {
4397         old = (nmethod*)_postponed_list;
4398         nm->set_unloading_next(old);
4399       } while ((nmethod*)Atomic::cmpxchg_ptr(nm, &_postponed_list, old) != old);
4400   }
4401 
4402   void clean_nmethod(nmethod* nm) {
4403     bool postponed = nm->do_unloading_parallel(_is_alive, _unloading_occurred);
4404 
4405     if (postponed) {
4406       // This nmethod referred to an nmethod that has not been cleaned/unloaded yet.
4407       add_to_postponed_list(nm);
4408     }
4409 
4410     // Mark that this thread has been cleaned/unloaded.
4411     // After this call, it will be safe to ask if this nmethod was unloaded or not.
4412     nm->set_unloading_clock(nmethod::global_unloading_clock());
4413   }
4414 
4415   void clean_nmethod_postponed(nmethod* nm) {
4416     nm->do_unloading_parallel_postponed(_is_alive, _unloading_occurred);
4417   }
4418 
4419   static const int MaxClaimNmethods = 16;
4420 
4421   void claim_nmethods(nmethod** claimed_nmethods, int *num_claimed_nmethods) {
4422     nmethod* first;
4423     NMethodIterator last;
4424 
4425     do {
4426       *num_claimed_nmethods = 0;
4427 
4428       first = (nmethod*)_claimed_nmethod;
4429       last = NMethodIterator(first);
4430 
4431       if (first != NULL) {
4432 
4433         for (int i = 0; i < MaxClaimNmethods; i++) {
4434           if (!last.next_alive()) {
4435             break;
4436           }
4437           claimed_nmethods[i] = last.method();
4438           (*num_claimed_nmethods)++;
4439         }
4440       }
4441 
4442     } while ((nmethod*)Atomic::cmpxchg_ptr(last.method(), &_claimed_nmethod, first) != first);
4443   }
4444 
4445   nmethod* claim_postponed_nmethod() {
4446     nmethod* claim;
4447     nmethod* next;
4448 
4449     do {
4450       claim = (nmethod*)_postponed_list;
4451       if (claim == NULL) {
4452         return NULL;
4453       }
4454 
4455       next = claim->unloading_next();
4456 
4457     } while ((nmethod*)Atomic::cmpxchg_ptr(next, &_postponed_list, claim) != claim);
4458 
4459     return claim;
4460   }
4461 
4462  public:
4463   // Mark that we're done with the first pass of nmethod cleaning.
4464   void barrier_mark(uint worker_id) {
4465     MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag);
4466     _num_entered_barrier++;
4467     if (_num_entered_barrier == _num_workers) {
4468       ml.notify_all();
4469     }
4470   }
4471 
4472   // See if we have to wait for the other workers to
4473   // finish their first-pass nmethod cleaning work.
4474   void barrier_wait(uint worker_id) {
4475     if (_num_entered_barrier < _num_workers) {
4476       MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag);
4477       while (_num_entered_barrier < _num_workers) {
4478           ml.wait(Mutex::_no_safepoint_check_flag, 0, false);
4479       }
4480     }
4481   }
4482 
4483   // Cleaning and unloading of nmethods. Some work has to be postponed
4484   // to the second pass, when we know which nmethods survive.
4485   void work_first_pass(uint worker_id) {
4486     // The first nmethods is claimed by the first worker.
4487     if (worker_id == 0 && _first_nmethod != NULL) {
4488       clean_nmethod(_first_nmethod);
4489       _first_nmethod = NULL;
4490     }
4491 
4492     int num_claimed_nmethods;
4493     nmethod* claimed_nmethods[MaxClaimNmethods];
4494 
4495     while (true) {
4496       claim_nmethods(claimed_nmethods, &num_claimed_nmethods);
4497 
4498       if (num_claimed_nmethods == 0) {
4499         break;
4500       }
4501 
4502       for (int i = 0; i < num_claimed_nmethods; i++) {
4503         clean_nmethod(claimed_nmethods[i]);
4504       }
4505     }
4506   }
4507 
4508   void work_second_pass(uint worker_id) {
4509     nmethod* nm;
4510     // Take care of postponed nmethods.
4511     while ((nm = claim_postponed_nmethod()) != NULL) {
4512       clean_nmethod_postponed(nm);
4513     }
4514   }
4515 };
4516 
4517 Monitor* G1CodeCacheUnloadingTask::_lock = new Monitor(Mutex::leaf, "Code Cache Unload lock", false, Monitor::_safepoint_check_never);
4518 
4519 class G1KlassCleaningTask : public StackObj {
4520   BoolObjectClosure*                      _is_alive;
4521   volatile jint                           _clean_klass_tree_claimed;
4522   ClassLoaderDataGraphKlassIteratorAtomic _klass_iterator;
4523 
4524  public:
4525   G1KlassCleaningTask(BoolObjectClosure* is_alive) :
4526       _is_alive(is_alive),
4527       _clean_klass_tree_claimed(0),
4528       _klass_iterator() {
4529   }
4530 
4531  private:
4532   bool claim_clean_klass_tree_task() {
4533     if (_clean_klass_tree_claimed) {
4534       return false;
4535     }
4536 
4537     return Atomic::cmpxchg(1, (jint*)&_clean_klass_tree_claimed, 0) == 0;
4538   }
4539 
4540   InstanceKlass* claim_next_klass() {
4541     Klass* klass;
4542     do {
4543       klass =_klass_iterator.next_klass();
4544     } while (klass != NULL && !klass->is_instance_klass());
4545 
4546     // this can be null so don't call InstanceKlass::cast
4547     return static_cast<InstanceKlass*>(klass);
4548   }
4549 
4550 public:
4551 
4552   void clean_klass(InstanceKlass* ik) {
4553     ik->clean_weak_instanceklass_links(_is_alive);
4554   }
4555 
4556   void work() {
4557     ResourceMark rm;
4558 
4559     // One worker will clean the subklass/sibling klass tree.
4560     if (claim_clean_klass_tree_task()) {
4561       Klass::clean_subklass_tree(_is_alive);
4562     }
4563 
4564     // All workers will help cleaning the classes,
4565     InstanceKlass* klass;
4566     while ((klass = claim_next_klass()) != NULL) {
4567       clean_klass(klass);
4568     }
4569   }
4570 };
4571 
4572 // To minimize the remark pause times, the tasks below are done in parallel.
4573 class G1ParallelCleaningTask : public AbstractGangTask {
4574 private:
4575   G1StringSymbolTableUnlinkTask _string_symbol_task;
4576   G1CodeCacheUnloadingTask      _code_cache_task;
4577   G1KlassCleaningTask           _klass_cleaning_task;
4578 
4579 public:
4580   // The constructor is run in the VMThread.
4581   G1ParallelCleaningTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols, uint num_workers, bool unloading_occurred) :
4582       AbstractGangTask("Parallel Cleaning"),
4583       _string_symbol_task(is_alive, process_strings, process_symbols),
4584       _code_cache_task(num_workers, is_alive, unloading_occurred),
4585       _klass_cleaning_task(is_alive) {
4586   }
4587 
4588   // The parallel work done by all worker threads.
4589   void work(uint worker_id) {
4590     // Do first pass of code cache cleaning.
4591     _code_cache_task.work_first_pass(worker_id);
4592 
4593     // Let the threads mark that the first pass is done.
4594     _code_cache_task.barrier_mark(worker_id);
4595 
4596     // Clean the Strings and Symbols.
4597     _string_symbol_task.work(worker_id);
4598 
4599     // Wait for all workers to finish the first code cache cleaning pass.
4600     _code_cache_task.barrier_wait(worker_id);
4601 
4602     // Do the second code cache cleaning work, which realize on
4603     // the liveness information gathered during the first pass.
4604     _code_cache_task.work_second_pass(worker_id);
4605 
4606     // Clean all klasses that were not unloaded.
4607     _klass_cleaning_task.work();
4608   }
4609 };
4610 
4611 
4612 void G1CollectedHeap::parallel_cleaning(BoolObjectClosure* is_alive,
4613                                         bool process_strings,
4614                                         bool process_symbols,
4615                                         bool class_unloading_occurred) {
4616   uint n_workers = workers()->active_workers();
4617 
4618   G1ParallelCleaningTask g1_unlink_task(is_alive, process_strings, process_symbols,
4619                                         n_workers, class_unloading_occurred);
4620   workers()->run_task(&g1_unlink_task);
4621 }
4622 
4623 void G1CollectedHeap::unlink_string_and_symbol_table(BoolObjectClosure* is_alive,
4624                                                      bool process_strings, bool process_symbols) {
4625   {
4626     G1StringSymbolTableUnlinkTask g1_unlink_task(is_alive, process_strings, process_symbols);
4627     workers()->run_task(&g1_unlink_task);
4628   }
4629 
4630   if (G1StringDedup::is_enabled()) {
4631     G1StringDedup::unlink(is_alive);
4632   }
4633 }
4634 
4635 class G1RedirtyLoggedCardsTask : public AbstractGangTask {
4636  private:
4637   DirtyCardQueueSet* _queue;
4638  public:
4639   G1RedirtyLoggedCardsTask(DirtyCardQueueSet* queue) : AbstractGangTask("Redirty Cards"), _queue(queue) { }
4640 
4641   virtual void work(uint worker_id) {
4642     G1GCPhaseTimes* phase_times = G1CollectedHeap::heap()->g1_policy()->phase_times();
4643     G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::RedirtyCards, worker_id);
4644 
4645     RedirtyLoggedCardTableEntryClosure cl;
4646     _queue->par_apply_closure_to_all_completed_buffers(&cl);
4647 
4648     phase_times->record_thread_work_item(G1GCPhaseTimes::RedirtyCards, worker_id, cl.num_processed());
4649   }
4650 };
4651 
4652 void G1CollectedHeap::redirty_logged_cards() {
4653   double redirty_logged_cards_start = os::elapsedTime();
4654 
4655   G1RedirtyLoggedCardsTask redirty_task(&dirty_card_queue_set());
4656   dirty_card_queue_set().reset_for_par_iteration();
4657   workers()->run_task(&redirty_task);
4658 
4659   DirtyCardQueueSet& dcq = JavaThread::dirty_card_queue_set();
4660   dcq.merge_bufferlists(&dirty_card_queue_set());
4661   assert(dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed");
4662 
4663   g1_policy()->phase_times()->record_redirty_logged_cards_time_ms((os::elapsedTime() - redirty_logged_cards_start) * 1000.0);
4664 }
4665 
4666 // Weak Reference Processing support
4667 
4668 // An always "is_alive" closure that is used to preserve referents.
4669 // If the object is non-null then it's alive.  Used in the preservation
4670 // of referent objects that are pointed to by reference objects
4671 // discovered by the CM ref processor.
4672 class G1AlwaysAliveClosure: public BoolObjectClosure {
4673   G1CollectedHeap* _g1;
4674 public:
4675   G1AlwaysAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
4676   bool do_object_b(oop p) {
4677     if (p != NULL) {
4678       return true;
4679     }
4680     return false;
4681   }
4682 };
4683 
4684 bool G1STWIsAliveClosure::do_object_b(oop p) {
4685   // An object is reachable if it is outside the collection set,
4686   // or is inside and copied.
4687   return !_g1->is_in_cset(p) || p->is_forwarded();
4688 }
4689 
4690 // Non Copying Keep Alive closure
4691 class G1KeepAliveClosure: public OopClosure {
4692   G1CollectedHeap* _g1;
4693 public:
4694   G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
4695   void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
4696   void do_oop(oop* p) {
4697     oop obj = *p;
4698     assert(obj != NULL, "the caller should have filtered out NULL values");
4699 
4700     const InCSetState cset_state = _g1->in_cset_state(obj);
4701     if (!cset_state.is_in_cset_or_humongous()) {
4702       return;
4703     }
4704     if (cset_state.is_in_cset()) {
4705       assert( obj->is_forwarded(), "invariant" );
4706       *p = obj->forwardee();
4707     } else {
4708       assert(!obj->is_forwarded(), "invariant" );
4709       assert(cset_state.is_humongous(),
4710              "Only allowed InCSet state is IsHumongous, but is %d", cset_state.value());
4711       _g1->set_humongous_is_live(obj);
4712     }
4713   }
4714 };
4715 
4716 // Copying Keep Alive closure - can be called from both
4717 // serial and parallel code as long as different worker
4718 // threads utilize different G1ParScanThreadState instances
4719 // and different queues.
4720 
4721 class G1CopyingKeepAliveClosure: public OopClosure {
4722   G1CollectedHeap*         _g1h;
4723   OopClosure*              _copy_non_heap_obj_cl;
4724   G1ParScanThreadState*    _par_scan_state;
4725 
4726 public:
4727   G1CopyingKeepAliveClosure(G1CollectedHeap* g1h,
4728                             OopClosure* non_heap_obj_cl,
4729                             G1ParScanThreadState* pss):
4730     _g1h(g1h),
4731     _copy_non_heap_obj_cl(non_heap_obj_cl),
4732     _par_scan_state(pss)
4733   {}
4734 
4735   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
4736   virtual void do_oop(      oop* p) { do_oop_work(p); }
4737 
4738   template <class T> void do_oop_work(T* p) {
4739     oop obj = oopDesc::load_decode_heap_oop(p);
4740 
4741     if (_g1h->is_in_cset_or_humongous(obj)) {
4742       // If the referent object has been forwarded (either copied
4743       // to a new location or to itself in the event of an
4744       // evacuation failure) then we need to update the reference
4745       // field and, if both reference and referent are in the G1
4746       // heap, update the RSet for the referent.
4747       //
4748       // If the referent has not been forwarded then we have to keep
4749       // it alive by policy. Therefore we have copy the referent.
4750       //
4751       // If the reference field is in the G1 heap then we can push
4752       // on the PSS queue. When the queue is drained (after each
4753       // phase of reference processing) the object and it's followers
4754       // will be copied, the reference field set to point to the
4755       // new location, and the RSet updated. Otherwise we need to
4756       // use the the non-heap or metadata closures directly to copy
4757       // the referent object and update the pointer, while avoiding
4758       // updating the RSet.
4759 
4760       if (_g1h->is_in_g1_reserved(p)) {
4761         _par_scan_state->push_on_queue(p);
4762       } else {
4763         assert(!Metaspace::contains((const void*)p),
4764                "Unexpectedly found a pointer from metadata: " PTR_FORMAT, p2i(p));
4765         _copy_non_heap_obj_cl->do_oop(p);
4766       }
4767     }
4768   }
4769 };
4770 
4771 // Serial drain queue closure. Called as the 'complete_gc'
4772 // closure for each discovered list in some of the
4773 // reference processing phases.
4774 
4775 class G1STWDrainQueueClosure: public VoidClosure {
4776 protected:
4777   G1CollectedHeap* _g1h;
4778   G1ParScanThreadState* _par_scan_state;
4779 
4780   G1ParScanThreadState*   par_scan_state() { return _par_scan_state; }
4781 
4782 public:
4783   G1STWDrainQueueClosure(G1CollectedHeap* g1h, G1ParScanThreadState* pss) :
4784     _g1h(g1h),
4785     _par_scan_state(pss)
4786   { }
4787 
4788   void do_void() {
4789     G1ParScanThreadState* const pss = par_scan_state();
4790     pss->trim_queue();
4791   }
4792 };
4793 
4794 // Parallel Reference Processing closures
4795 
4796 // Implementation of AbstractRefProcTaskExecutor for parallel reference
4797 // processing during G1 evacuation pauses.
4798 
4799 class G1STWRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
4800 private:
4801   G1CollectedHeap*          _g1h;
4802   G1ParScanThreadStateSet*  _pss;
4803   RefToScanQueueSet*        _queues;
4804   WorkGang*                 _workers;
4805   uint                      _active_workers;
4806 
4807 public:
4808   G1STWRefProcTaskExecutor(G1CollectedHeap* g1h,
4809                            G1ParScanThreadStateSet* per_thread_states,
4810                            WorkGang* workers,
4811                            RefToScanQueueSet *task_queues,
4812                            uint n_workers) :
4813     _g1h(g1h),
4814     _pss(per_thread_states),
4815     _queues(task_queues),
4816     _workers(workers),
4817     _active_workers(n_workers)
4818   {
4819     assert(n_workers > 0, "shouldn't call this otherwise");
4820   }
4821 
4822   // Executes the given task using concurrent marking worker threads.
4823   virtual void execute(ProcessTask& task);
4824   virtual void execute(EnqueueTask& task);
4825 };
4826 
4827 // Gang task for possibly parallel reference processing
4828 
4829 class G1STWRefProcTaskProxy: public AbstractGangTask {
4830   typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
4831   ProcessTask&     _proc_task;
4832   G1CollectedHeap* _g1h;
4833   G1ParScanThreadStateSet* _pss;
4834   RefToScanQueueSet* _task_queues;
4835   ParallelTaskTerminator* _terminator;
4836 
4837 public:
4838   G1STWRefProcTaskProxy(ProcessTask& proc_task,
4839                         G1CollectedHeap* g1h,
4840                         G1ParScanThreadStateSet* per_thread_states,
4841                         RefToScanQueueSet *task_queues,
4842                         ParallelTaskTerminator* terminator) :
4843     AbstractGangTask("Process reference objects in parallel"),
4844     _proc_task(proc_task),
4845     _g1h(g1h),
4846     _pss(per_thread_states),
4847     _task_queues(task_queues),
4848     _terminator(terminator)
4849   {}
4850 
4851   virtual void work(uint worker_id) {
4852     // The reference processing task executed by a single worker.
4853     ResourceMark rm;
4854     HandleMark   hm;
4855 
4856     G1STWIsAliveClosure is_alive(_g1h);
4857 
4858     G1ParScanThreadState*          pss = _pss->state_for_worker(worker_id);
4859     pss->set_ref_processor(NULL);
4860 
4861     // Keep alive closure.
4862     G1CopyingKeepAliveClosure keep_alive(_g1h, pss->closures()->raw_strong_oops(), pss);
4863 
4864     // Complete GC closure
4865     G1ParEvacuateFollowersClosure drain_queue(_g1h, pss, _task_queues, _terminator);
4866 
4867     // Call the reference processing task's work routine.
4868     _proc_task.work(worker_id, is_alive, keep_alive, drain_queue);
4869 
4870     // Note we cannot assert that the refs array is empty here as not all
4871     // of the processing tasks (specifically phase2 - pp2_work) execute
4872     // the complete_gc closure (which ordinarily would drain the queue) so
4873     // the queue may not be empty.
4874   }
4875 };
4876 
4877 // Driver routine for parallel reference processing.
4878 // Creates an instance of the ref processing gang
4879 // task and has the worker threads execute it.
4880 void G1STWRefProcTaskExecutor::execute(ProcessTask& proc_task) {
4881   assert(_workers != NULL, "Need parallel worker threads.");
4882 
4883   ParallelTaskTerminator terminator(_active_workers, _queues);
4884   G1STWRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _pss, _queues, &terminator);
4885 
4886   _workers->run_task(&proc_task_proxy);
4887 }
4888 
4889 // Gang task for parallel reference enqueueing.
4890 
4891 class G1STWRefEnqueueTaskProxy: public AbstractGangTask {
4892   typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
4893   EnqueueTask& _enq_task;
4894 
4895 public:
4896   G1STWRefEnqueueTaskProxy(EnqueueTask& enq_task) :
4897     AbstractGangTask("Enqueue reference objects in parallel"),
4898     _enq_task(enq_task)
4899   { }
4900 
4901   virtual void work(uint worker_id) {
4902     _enq_task.work(worker_id);
4903   }
4904 };
4905 
4906 // Driver routine for parallel reference enqueueing.
4907 // Creates an instance of the ref enqueueing gang
4908 // task and has the worker threads execute it.
4909 
4910 void G1STWRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
4911   assert(_workers != NULL, "Need parallel worker threads.");
4912 
4913   G1STWRefEnqueueTaskProxy enq_task_proxy(enq_task);
4914 
4915   _workers->run_task(&enq_task_proxy);
4916 }
4917 
4918 // End of weak reference support closures
4919 
4920 // Abstract task used to preserve (i.e. copy) any referent objects
4921 // that are in the collection set and are pointed to by reference
4922 // objects discovered by the CM ref processor.
4923 
4924 class G1ParPreserveCMReferentsTask: public AbstractGangTask {
4925 protected:
4926   G1CollectedHeap*         _g1h;
4927   G1ParScanThreadStateSet* _pss;
4928   RefToScanQueueSet*       _queues;
4929   ParallelTaskTerminator   _terminator;
4930   uint                     _n_workers;
4931 
4932 public:
4933   G1ParPreserveCMReferentsTask(G1CollectedHeap* g1h, G1ParScanThreadStateSet* per_thread_states, int workers, RefToScanQueueSet *task_queues) :
4934     AbstractGangTask("ParPreserveCMReferents"),
4935     _g1h(g1h),
4936     _pss(per_thread_states),
4937     _queues(task_queues),
4938     _terminator(workers, _queues),
4939     _n_workers(workers)
4940   { }
4941 
4942   void work(uint worker_id) {
4943     ResourceMark rm;
4944     HandleMark   hm;
4945 
4946     G1ParScanThreadState*          pss = _pss->state_for_worker(worker_id);
4947     pss->set_ref_processor(NULL);
4948     assert(pss->queue_is_empty(), "both queue and overflow should be empty");
4949 
4950     // Is alive closure
4951     G1AlwaysAliveClosure always_alive(_g1h);
4952 
4953     // Copying keep alive closure. Applied to referent objects that need
4954     // to be copied.
4955     G1CopyingKeepAliveClosure keep_alive(_g1h, pss->closures()->raw_strong_oops(), pss);
4956 
4957     ReferenceProcessor* rp = _g1h->ref_processor_cm();
4958 
4959     uint limit = ReferenceProcessor::number_of_subclasses_of_ref() * rp->max_num_q();
4960     uint stride = MIN2(MAX2(_n_workers, 1U), limit);
4961 
4962     // limit is set using max_num_q() - which was set using ParallelGCThreads.
4963     // So this must be true - but assert just in case someone decides to
4964     // change the worker ids.
4965     assert(worker_id < limit, "sanity");
4966     assert(!rp->discovery_is_atomic(), "check this code");
4967 
4968     // Select discovered lists [i, i+stride, i+2*stride,...,limit)
4969     for (uint idx = worker_id; idx < limit; idx += stride) {
4970       DiscoveredList& ref_list = rp->discovered_refs()[idx];
4971 
4972       DiscoveredListIterator iter(ref_list, &keep_alive, &always_alive);
4973       while (iter.has_next()) {
4974         // Since discovery is not atomic for the CM ref processor, we
4975         // can see some null referent objects.
4976         iter.load_ptrs(DEBUG_ONLY(true));
4977         oop ref = iter.obj();
4978 
4979         // This will filter nulls.
4980         if (iter.is_referent_alive()) {
4981           iter.make_referent_alive();
4982         }
4983         iter.move_to_next();
4984       }
4985     }
4986 
4987     // Drain the queue - which may cause stealing
4988     G1ParEvacuateFollowersClosure drain_queue(_g1h, pss, _queues, &_terminator);
4989     drain_queue.do_void();
4990     // Allocation buffers were retired at the end of G1ParEvacuateFollowersClosure
4991     assert(pss->queue_is_empty(), "should be");
4992   }
4993 };
4994 
4995 // Weak Reference processing during an evacuation pause (part 1).
4996 void G1CollectedHeap::process_discovered_references(G1ParScanThreadStateSet* per_thread_states) {
4997   double ref_proc_start = os::elapsedTime();
4998 
4999   ReferenceProcessor* rp = _ref_processor_stw;
5000   assert(rp->discovery_enabled(), "should have been enabled");
5001 
5002   // Any reference objects, in the collection set, that were 'discovered'
5003   // by the CM ref processor should have already been copied (either by
5004   // applying the external root copy closure to the discovered lists, or
5005   // by following an RSet entry).
5006   //
5007   // But some of the referents, that are in the collection set, that these
5008   // reference objects point to may not have been copied: the STW ref
5009   // processor would have seen that the reference object had already
5010   // been 'discovered' and would have skipped discovering the reference,
5011   // but would not have treated the reference object as a regular oop.
5012   // As a result the copy closure would not have been applied to the
5013   // referent object.
5014   //
5015   // We need to explicitly copy these referent objects - the references
5016   // will be processed at the end of remarking.
5017   //
5018   // We also need to do this copying before we process the reference
5019   // objects discovered by the STW ref processor in case one of these
5020   // referents points to another object which is also referenced by an
5021   // object discovered by the STW ref processor.
5022 
5023   uint no_of_gc_workers = workers()->active_workers();
5024 
5025   G1ParPreserveCMReferentsTask keep_cm_referents(this,
5026                                                  per_thread_states,
5027                                                  no_of_gc_workers,
5028                                                  _task_queues);
5029 
5030   workers()->run_task(&keep_cm_referents);
5031 
5032   // Closure to test whether a referent is alive.
5033   G1STWIsAliveClosure is_alive(this);
5034 
5035   // Even when parallel reference processing is enabled, the processing
5036   // of JNI refs is serial and performed serially by the current thread
5037   // rather than by a worker. The following PSS will be used for processing
5038   // JNI refs.
5039 
5040   // Use only a single queue for this PSS.
5041   G1ParScanThreadState*          pss = per_thread_states->state_for_worker(0);
5042   pss->set_ref_processor(NULL);
5043   assert(pss->queue_is_empty(), "pre-condition");
5044 
5045   // Keep alive closure.
5046   G1CopyingKeepAliveClosure keep_alive(this, pss->closures()->raw_strong_oops(), pss);
5047 
5048   // Serial Complete GC closure
5049   G1STWDrainQueueClosure drain_queue(this, pss);
5050 
5051   // Setup the soft refs policy...
5052   rp->setup_policy(false);
5053 
5054   ReferenceProcessorStats stats;
5055   if (!rp->processing_is_mt()) {
5056     // Serial reference processing...
5057     stats = rp->process_discovered_references(&is_alive,
5058                                               &keep_alive,
5059                                               &drain_queue,
5060                                               NULL,
5061                                               _gc_timer_stw);
5062   } else {
5063     // Parallel reference processing
5064     assert(rp->num_q() == no_of_gc_workers, "sanity");
5065     assert(no_of_gc_workers <= rp->max_num_q(), "sanity");
5066 
5067     G1STWRefProcTaskExecutor par_task_executor(this, per_thread_states, workers(), _task_queues, no_of_gc_workers);
5068     stats = rp->process_discovered_references(&is_alive,
5069                                               &keep_alive,
5070                                               &drain_queue,
5071                                               &par_task_executor,
5072                                               _gc_timer_stw);
5073   }
5074 
5075   _gc_tracer_stw->report_gc_reference_stats(stats);
5076 
5077   // We have completed copying any necessary live referent objects.
5078   assert(pss->queue_is_empty(), "both queue and overflow should be empty");
5079 
5080   double ref_proc_time = os::elapsedTime() - ref_proc_start;
5081   g1_policy()->phase_times()->record_ref_proc_time(ref_proc_time * 1000.0);
5082 }
5083 
5084 // Weak Reference processing during an evacuation pause (part 2).
5085 void G1CollectedHeap::enqueue_discovered_references(G1ParScanThreadStateSet* per_thread_states) {
5086   double ref_enq_start = os::elapsedTime();
5087 
5088   ReferenceProcessor* rp = _ref_processor_stw;
5089   assert(!rp->discovery_enabled(), "should have been disabled as part of processing");
5090 
5091   // Now enqueue any remaining on the discovered lists on to
5092   // the pending list.
5093   if (!rp->processing_is_mt()) {
5094     // Serial reference processing...
5095     rp->enqueue_discovered_references();
5096   } else {
5097     // Parallel reference enqueueing
5098 
5099     uint n_workers = workers()->active_workers();
5100 
5101     assert(rp->num_q() == n_workers, "sanity");
5102     assert(n_workers <= rp->max_num_q(), "sanity");
5103 
5104     G1STWRefProcTaskExecutor par_task_executor(this, per_thread_states, workers(), _task_queues, n_workers);
5105     rp->enqueue_discovered_references(&par_task_executor);
5106   }
5107 
5108   rp->verify_no_references_recorded();
5109   assert(!rp->discovery_enabled(), "should have been disabled");
5110 
5111   // FIXME
5112   // CM's reference processing also cleans up the string and symbol tables.
5113   // Should we do that here also? We could, but it is a serial operation
5114   // and could significantly increase the pause time.
5115 
5116   double ref_enq_time = os::elapsedTime() - ref_enq_start;
5117   g1_policy()->phase_times()->record_ref_enq_time(ref_enq_time * 1000.0);
5118 }
5119 
5120 void G1CollectedHeap::pre_evacuate_collection_set() {
5121   _expand_heap_after_alloc_failure = true;
5122   _evacuation_failed = false;
5123 
5124   // Disable the hot card cache.
5125   G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
5126   hot_card_cache->reset_hot_cache_claimed_index();
5127   hot_card_cache->set_use_cache(false);
5128 
5129 }
5130 
5131 void G1CollectedHeap::evacuate_collection_set(EvacuationInfo& evacuation_info, G1ParScanThreadStateSet* per_thread_states) {
5132   g1_rem_set()->prepare_for_oops_into_collection_set_do();
5133 
5134   // Should G1EvacuationFailureALot be in effect for this GC?
5135   NOT_PRODUCT(set_evacuation_failure_alot_for_current_gc();)
5136 
5137   assert(dirty_card_queue_set().completed_buffers_num() == 0, "Should be empty");
5138   double start_par_time_sec = os::elapsedTime();
5139   double end_par_time_sec;
5140 
5141   {
5142     const uint n_workers = workers()->active_workers();
5143     G1RootProcessor root_processor(this, n_workers);
5144     G1ParTask g1_par_task(this, per_thread_states, _task_queues, &root_processor, n_workers);
5145     // InitialMark needs claim bits to keep track of the marked-through CLDs.
5146     if (collector_state()->during_initial_mark_pause()) {
5147       ClassLoaderDataGraph::clear_claimed_marks();
5148     }
5149 
5150     // The individual threads will set their evac-failure closures.
5151     if (PrintTerminationStats) {
5152       print_termination_stats_hdr(gclog_or_tty);
5153     }
5154 
5155     workers()->run_task(&g1_par_task);
5156     end_par_time_sec = os::elapsedTime();
5157 
5158     // Closing the inner scope will execute the destructor
5159     // for the G1RootProcessor object. We record the current
5160     // elapsed time before closing the scope so that time
5161     // taken for the destructor is NOT included in the
5162     // reported parallel time.
5163   }
5164 
5165   G1GCPhaseTimes* phase_times = g1_policy()->phase_times();
5166 
5167   double par_time_ms = (end_par_time_sec - start_par_time_sec) * 1000.0;
5168   phase_times->record_par_time(par_time_ms);
5169 
5170   double code_root_fixup_time_ms =
5171         (os::elapsedTime() - end_par_time_sec) * 1000.0;
5172   phase_times->record_code_root_fixup_time(code_root_fixup_time_ms);
5173 
5174   // Process any discovered reference objects - we have
5175   // to do this _before_ we retire the GC alloc regions
5176   // as we may have to copy some 'reachable' referent
5177   // objects (and their reachable sub-graphs) that were
5178   // not copied during the pause.
5179   process_discovered_references(per_thread_states);
5180 
5181   if (G1StringDedup::is_enabled()) {
5182     double fixup_start = os::elapsedTime();
5183 
5184     G1STWIsAliveClosure is_alive(this);
5185     G1KeepAliveClosure keep_alive(this);
5186     G1StringDedup::unlink_or_oops_do(&is_alive, &keep_alive, true, phase_times);
5187 
5188     double fixup_time_ms = (os::elapsedTime() - fixup_start) * 1000.0;
5189     phase_times->record_string_dedup_fixup_time(fixup_time_ms);
5190   }
5191 
5192   g1_rem_set()->cleanup_after_oops_into_collection_set_do();
5193 
5194   if (evacuation_failed()) {
5195     remove_self_forwarding_pointers();
5196 
5197     // Reset the G1EvacuationFailureALot counters and flags
5198     // Note: the values are reset only when an actual
5199     // evacuation failure occurs.
5200     NOT_PRODUCT(reset_evacuation_should_fail();)
5201   }
5202 
5203   // Enqueue any remaining references remaining on the STW
5204   // reference processor's discovered lists. We need to do
5205   // this after the card table is cleaned (and verified) as
5206   // the act of enqueueing entries on to the pending list
5207   // will log these updates (and dirty their associated
5208   // cards). We need these updates logged to update any
5209   // RSets.
5210   enqueue_discovered_references(per_thread_states);
5211 }
5212 
5213 void G1CollectedHeap::post_evacuate_collection_set(EvacuationInfo& evacuation_info, G1ParScanThreadStateSet* per_thread_states) {
5214   _allocator->release_gc_alloc_regions(evacuation_info);
5215 
5216   per_thread_states->flush();
5217 
5218   record_obj_copy_mem_stats();
5219 
5220   _survivor_evac_stats.adjust_desired_plab_sz();
5221   _old_evac_stats.adjust_desired_plab_sz();
5222 
5223   // Reset and re-enable the hot card cache.
5224   // Note the counts for the cards in the regions in the
5225   // collection set are reset when the collection set is freed.
5226   G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
5227   hot_card_cache->reset_hot_cache();
5228   hot_card_cache->set_use_cache(true);
5229 
5230   purge_code_root_memory();
5231 
5232   redirty_logged_cards();
5233 #if defined(COMPILER2) || INCLUDE_JVMCI
5234   DerivedPointerTable::update_pointers();
5235 #endif
5236 }
5237 
5238 void G1CollectedHeap::record_obj_copy_mem_stats() {
5239   _gc_tracer_stw->report_evacuation_statistics(create_g1_evac_summary(&_survivor_evac_stats),
5240                                                create_g1_evac_summary(&_old_evac_stats));
5241 }
5242 
5243 void G1CollectedHeap::free_region(HeapRegion* hr,
5244                                   FreeRegionList* free_list,
5245                                   bool par,
5246                                   bool locked) {
5247   assert(!hr->is_free(), "the region should not be free");
5248   assert(!hr->is_empty(), "the region should not be empty");
5249   assert(_hrm.is_available(hr->hrm_index()), "region should be committed");
5250   assert(free_list != NULL, "pre-condition");
5251 
5252   if (G1VerifyBitmaps) {
5253     MemRegion mr(hr->bottom(), hr->end());
5254     concurrent_mark()->clearRangePrevBitmap(mr);
5255   }
5256 
5257   // Clear the card counts for this region.
5258   // Note: we only need to do this if the region is not young
5259   // (since we don't refine cards in young regions).
5260   if (!hr->is_young()) {
5261     _cg1r->hot_card_cache()->reset_card_counts(hr);
5262   }
5263   hr->hr_clear(par, true /* clear_space */, locked /* locked */);
5264   free_list->add_ordered(hr);
5265 }
5266 
5267 void G1CollectedHeap::free_humongous_region(HeapRegion* hr,
5268                                             FreeRegionList* free_list,
5269                                             bool par) {
5270   assert(hr->is_humongous(), "this is only for humongous regions");
5271   assert(free_list != NULL, "pre-condition");
5272   hr->clear_humongous();
5273   free_region(hr, free_list, par);
5274 }
5275 
5276 void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed,
5277                                            const HeapRegionSetCount& humongous_regions_removed) {
5278   if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) {
5279     MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag);
5280     _old_set.bulk_remove(old_regions_removed);
5281     _humongous_set.bulk_remove(humongous_regions_removed);
5282   }
5283 
5284 }
5285 
5286 void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) {
5287   assert(list != NULL, "list can't be null");
5288   if (!list->is_empty()) {
5289     MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag);
5290     _hrm.insert_list_into_free_list(list);
5291   }
5292 }
5293 
5294 void G1CollectedHeap::decrement_summary_bytes(size_t bytes) {
5295   decrease_used(bytes);
5296 }
5297 
5298 class G1ParCleanupCTTask : public AbstractGangTask {
5299   G1SATBCardTableModRefBS* _ct_bs;
5300   G1CollectedHeap* _g1h;
5301   HeapRegion* volatile _su_head;
5302 public:
5303   G1ParCleanupCTTask(G1SATBCardTableModRefBS* ct_bs,
5304                      G1CollectedHeap* g1h) :
5305     AbstractGangTask("G1 Par Cleanup CT Task"),
5306     _ct_bs(ct_bs), _g1h(g1h) { }
5307 
5308   void work(uint worker_id) {
5309     HeapRegion* r;
5310     while (r = _g1h->pop_dirty_cards_region()) {
5311       clear_cards(r);
5312     }
5313   }
5314 
5315   void clear_cards(HeapRegion* r) {
5316     // Cards of the survivors should have already been dirtied.
5317     if (!r->is_survivor()) {
5318       _ct_bs->clear(MemRegion(r->bottom(), r->end()));
5319     }
5320   }
5321 };
5322 
5323 #ifndef PRODUCT
5324 class G1VerifyCardTableCleanup: public HeapRegionClosure {
5325   G1CollectedHeap* _g1h;
5326   G1SATBCardTableModRefBS* _ct_bs;
5327 public:
5328   G1VerifyCardTableCleanup(G1CollectedHeap* g1h, G1SATBCardTableModRefBS* ct_bs)
5329     : _g1h(g1h), _ct_bs(ct_bs) { }
5330   virtual bool doHeapRegion(HeapRegion* r) {
5331     if (r->is_survivor()) {
5332       _g1h->verify_dirty_region(r);
5333     } else {
5334       _g1h->verify_not_dirty_region(r);
5335     }
5336     return false;
5337   }
5338 };
5339 
5340 void G1CollectedHeap::verify_not_dirty_region(HeapRegion* hr) {
5341   // All of the region should be clean.
5342   G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
5343   MemRegion mr(hr->bottom(), hr->end());
5344   ct_bs->verify_not_dirty_region(mr);
5345 }
5346 
5347 void G1CollectedHeap::verify_dirty_region(HeapRegion* hr) {
5348   // We cannot guarantee that [bottom(),end()] is dirty.  Threads
5349   // dirty allocated blocks as they allocate them. The thread that
5350   // retires each region and replaces it with a new one will do a
5351   // maximal allocation to fill in [pre_dummy_top(),end()] but will
5352   // not dirty that area (one less thing to have to do while holding
5353   // a lock). So we can only verify that [bottom(),pre_dummy_top()]
5354   // is dirty.
5355   G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
5356   MemRegion mr(hr->bottom(), hr->pre_dummy_top());
5357   if (hr->is_young()) {
5358     ct_bs->verify_g1_young_region(mr);
5359   } else {
5360     ct_bs->verify_dirty_region(mr);
5361   }
5362 }
5363 
5364 void G1CollectedHeap::verify_dirty_young_list(HeapRegion* head) {
5365   G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
5366   for (HeapRegion* hr = head; hr != NULL; hr = hr->get_next_young_region()) {
5367     verify_dirty_region(hr);
5368   }
5369 }
5370 
5371 void G1CollectedHeap::verify_dirty_young_regions() {
5372   verify_dirty_young_list(_young_list->first_region());
5373 }
5374 
5375 bool G1CollectedHeap::verify_no_bits_over_tams(const char* bitmap_name, CMBitMapRO* bitmap,
5376                                                HeapWord* tams, HeapWord* end) {
5377   guarantee(tams <= end,
5378             "tams: " PTR_FORMAT " end: " PTR_FORMAT, p2i(tams), p2i(end));
5379   HeapWord* result = bitmap->getNextMarkedWordAddress(tams, end);
5380   if (result < end) {
5381     gclog_or_tty->cr();
5382     gclog_or_tty->print_cr("## wrong marked address on %s bitmap: " PTR_FORMAT,
5383                            bitmap_name, p2i(result));
5384     gclog_or_tty->print_cr("## %s tams: " PTR_FORMAT " end: " PTR_FORMAT,
5385                            bitmap_name, p2i(tams), p2i(end));
5386     return false;
5387   }
5388   return true;
5389 }
5390 
5391 bool G1CollectedHeap::verify_bitmaps(const char* caller, HeapRegion* hr) {
5392   CMBitMapRO* prev_bitmap = concurrent_mark()->prevMarkBitMap();
5393   CMBitMapRO* next_bitmap = (CMBitMapRO*) concurrent_mark()->nextMarkBitMap();
5394 
5395   HeapWord* bottom = hr->bottom();
5396   HeapWord* ptams  = hr->prev_top_at_mark_start();
5397   HeapWord* ntams  = hr->next_top_at_mark_start();
5398   HeapWord* end    = hr->end();
5399 
5400   bool res_p = verify_no_bits_over_tams("prev", prev_bitmap, ptams, end);
5401 
5402   bool res_n = true;
5403   // We reset mark_in_progress() before we reset _cmThread->in_progress() and in this window
5404   // we do the clearing of the next bitmap concurrently. Thus, we can not verify the bitmap
5405   // if we happen to be in that state.
5406   if (collector_state()->mark_in_progress() || !_cmThread->in_progress()) {
5407     res_n = verify_no_bits_over_tams("next", next_bitmap, ntams, end);
5408   }
5409   if (!res_p || !res_n) {
5410     gclog_or_tty->print_cr("#### Bitmap verification failed for " HR_FORMAT,
5411                            HR_FORMAT_PARAMS(hr));
5412     gclog_or_tty->print_cr("#### Caller: %s", caller);
5413     return false;
5414   }
5415   return true;
5416 }
5417 
5418 void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) {
5419   if (!G1VerifyBitmaps) return;
5420 
5421   guarantee(verify_bitmaps(caller, hr), "bitmap verification");
5422 }
5423 
5424 class G1VerifyBitmapClosure : public HeapRegionClosure {
5425 private:
5426   const char* _caller;
5427   G1CollectedHeap* _g1h;
5428   bool _failures;
5429 
5430 public:
5431   G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) :
5432     _caller(caller), _g1h(g1h), _failures(false) { }
5433 
5434   bool failures() { return _failures; }
5435 
5436   virtual bool doHeapRegion(HeapRegion* hr) {
5437     bool result = _g1h->verify_bitmaps(_caller, hr);
5438     if (!result) {
5439       _failures = true;
5440     }
5441     return false;
5442   }
5443 };
5444 
5445 void G1CollectedHeap::check_bitmaps(const char* caller) {
5446   if (!G1VerifyBitmaps) return;
5447 
5448   G1VerifyBitmapClosure cl(caller, this);
5449   heap_region_iterate(&cl);
5450   guarantee(!cl.failures(), "bitmap verification");
5451 }
5452 
5453 class G1CheckCSetFastTableClosure : public HeapRegionClosure {
5454  private:
5455   bool _failures;
5456  public:
5457   G1CheckCSetFastTableClosure() : HeapRegionClosure(), _failures(false) { }
5458 
5459   virtual bool doHeapRegion(HeapRegion* hr) {
5460     uint i = hr->hrm_index();
5461     InCSetState cset_state = (InCSetState) G1CollectedHeap::heap()->_in_cset_fast_test.get_by_index(i);
5462     if (hr->is_humongous()) {
5463       if (hr->in_collection_set()) {
5464         gclog_or_tty->print_cr("\n## humongous region %u in CSet", i);
5465         _failures = true;
5466         return true;
5467       }
5468       if (cset_state.is_in_cset()) {
5469         gclog_or_tty->print_cr("\n## inconsistent cset state %d for humongous region %u", cset_state.value(), i);
5470         _failures = true;
5471         return true;
5472       }
5473       if (hr->is_continues_humongous() && cset_state.is_humongous()) {
5474         gclog_or_tty->print_cr("\n## inconsistent cset state %d for continues humongous region %u", cset_state.value(), i);
5475         _failures = true;
5476         return true;
5477       }
5478     } else {
5479       if (cset_state.is_humongous()) {
5480         gclog_or_tty->print_cr("\n## inconsistent cset state %d for non-humongous region %u", cset_state.value(), i);
5481         _failures = true;
5482         return true;
5483       }
5484       if (hr->in_collection_set() != cset_state.is_in_cset()) {
5485         gclog_or_tty->print_cr("\n## in CSet %d / cset state %d inconsistency for region %u",
5486                                hr->in_collection_set(), cset_state.value(), i);
5487         _failures = true;
5488         return true;
5489       }
5490       if (cset_state.is_in_cset()) {
5491         if (hr->is_young() != (cset_state.is_young())) {
5492           gclog_or_tty->print_cr("\n## is_young %d / cset state %d inconsistency for region %u",
5493                                  hr->is_young(), cset_state.value(), i);
5494           _failures = true;
5495           return true;
5496         }
5497         if (hr->is_old() != (cset_state.is_old())) {
5498           gclog_or_tty->print_cr("\n## is_old %d / cset state %d inconsistency for region %u",
5499                                  hr->is_old(), cset_state.value(), i);
5500           _failures = true;
5501           return true;
5502         }
5503       }
5504     }
5505     return false;
5506   }
5507 
5508   bool failures() const { return _failures; }
5509 };
5510 
5511 bool G1CollectedHeap::check_cset_fast_test() {
5512   G1CheckCSetFastTableClosure cl;
5513   _hrm.iterate(&cl);
5514   return !cl.failures();
5515 }
5516 #endif // PRODUCT
5517 
5518 void G1CollectedHeap::cleanUpCardTable() {
5519   G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
5520   double start = os::elapsedTime();
5521 
5522   {
5523     // Iterate over the dirty cards region list.
5524     G1ParCleanupCTTask cleanup_task(ct_bs, this);
5525 
5526     workers()->run_task(&cleanup_task);
5527 #ifndef PRODUCT
5528     if (G1VerifyCTCleanup || VerifyAfterGC) {
5529       G1VerifyCardTableCleanup cleanup_verifier(this, ct_bs);
5530       heap_region_iterate(&cleanup_verifier);
5531     }
5532 #endif
5533   }
5534 
5535   double elapsed = os::elapsedTime() - start;
5536   g1_policy()->phase_times()->record_clear_ct_time(elapsed * 1000.0);
5537 }
5538 
5539 void G1CollectedHeap::free_collection_set(HeapRegion* cs_head, EvacuationInfo& evacuation_info, const size_t* surviving_young_words) {
5540   size_t pre_used = 0;
5541   FreeRegionList local_free_list("Local List for CSet Freeing");
5542 
5543   double young_time_ms     = 0.0;
5544   double non_young_time_ms = 0.0;
5545 
5546   // Since the collection set is a superset of the the young list,
5547   // all we need to do to clear the young list is clear its
5548   // head and length, and unlink any young regions in the code below
5549   _young_list->clear();
5550 
5551   G1CollectorPolicy* policy = g1_policy();
5552 
5553   double start_sec = os::elapsedTime();
5554   bool non_young = true;
5555 
5556   HeapRegion* cur = cs_head;
5557   int age_bound = -1;
5558   size_t rs_lengths = 0;
5559 
5560   while (cur != NULL) {
5561     assert(!is_on_master_free_list(cur), "sanity");
5562     if (non_young) {
5563       if (cur->is_young()) {
5564         double end_sec = os::elapsedTime();
5565         double elapsed_ms = (end_sec - start_sec) * 1000.0;
5566         non_young_time_ms += elapsed_ms;
5567 
5568         start_sec = os::elapsedTime();
5569         non_young = false;
5570       }
5571     } else {
5572       if (!cur->is_young()) {
5573         double end_sec = os::elapsedTime();
5574         double elapsed_ms = (end_sec - start_sec) * 1000.0;
5575         young_time_ms += elapsed_ms;
5576 
5577         start_sec = os::elapsedTime();
5578         non_young = true;
5579       }
5580     }
5581 
5582     rs_lengths += cur->rem_set()->occupied_locked();
5583 
5584     HeapRegion* next = cur->next_in_collection_set();
5585     assert(cur->in_collection_set(), "bad CS");
5586     cur->set_next_in_collection_set(NULL);
5587     clear_in_cset(cur);
5588 
5589     if (cur->is_young()) {
5590       int index = cur->young_index_in_cset();
5591       assert(index != -1, "invariant");
5592       assert((uint) index < policy->young_cset_region_length(), "invariant");
5593       size_t words_survived = surviving_young_words[index];
5594       cur->record_surv_words_in_group(words_survived);
5595 
5596       // At this point the we have 'popped' cur from the collection set
5597       // (linked via next_in_collection_set()) but it is still in the
5598       // young list (linked via next_young_region()). Clear the
5599       // _next_young_region field.
5600       cur->set_next_young_region(NULL);
5601     } else {
5602       int index = cur->young_index_in_cset();
5603       assert(index == -1, "invariant");
5604     }
5605 
5606     assert( (cur->is_young() && cur->young_index_in_cset() > -1) ||
5607             (!cur->is_young() && cur->young_index_in_cset() == -1),
5608             "invariant" );
5609 
5610     if (!cur->evacuation_failed()) {
5611       MemRegion used_mr = cur->used_region();
5612 
5613       // And the region is empty.
5614       assert(!used_mr.is_empty(), "Should not have empty regions in a CS.");
5615       pre_used += cur->used();
5616       free_region(cur, &local_free_list, false /* par */, true /* locked */);
5617     } else {
5618       cur->uninstall_surv_rate_group();
5619       if (cur->is_young()) {
5620         cur->set_young_index_in_cset(-1);
5621       }
5622       cur->set_evacuation_failed(false);
5623       // The region is now considered to be old.
5624       cur->set_old();
5625       // Do some allocation statistics accounting. Regions that failed evacuation
5626       // are always made old, so there is no need to update anything in the young
5627       // gen statistics, but we need to update old gen statistics.
5628       size_t used_words = cur->marked_bytes() / HeapWordSize;
5629       _old_evac_stats.add_failure_used_and_waste(used_words, HeapRegion::GrainWords - used_words);
5630       _old_set.add(cur);
5631       evacuation_info.increment_collectionset_used_after(cur->used());
5632     }
5633     cur = next;
5634   }
5635 
5636   evacuation_info.set_regions_freed(local_free_list.length());
5637   policy->record_max_rs_lengths(rs_lengths);
5638   policy->cset_regions_freed();
5639 
5640   double end_sec = os::elapsedTime();
5641   double elapsed_ms = (end_sec - start_sec) * 1000.0;
5642 
5643   if (non_young) {
5644     non_young_time_ms += elapsed_ms;
5645   } else {
5646     young_time_ms += elapsed_ms;
5647   }
5648 
5649   prepend_to_freelist(&local_free_list);
5650   decrement_summary_bytes(pre_used);
5651   policy->phase_times()->record_young_free_cset_time_ms(young_time_ms);
5652   policy->phase_times()->record_non_young_free_cset_time_ms(non_young_time_ms);
5653 }
5654 
5655 class G1FreeHumongousRegionClosure : public HeapRegionClosure {
5656  private:
5657   FreeRegionList* _free_region_list;
5658   HeapRegionSet* _proxy_set;
5659   HeapRegionSetCount _humongous_regions_removed;
5660   size_t _freed_bytes;
5661  public:
5662 
5663   G1FreeHumongousRegionClosure(FreeRegionList* free_region_list) :
5664     _free_region_list(free_region_list), _humongous_regions_removed(), _freed_bytes(0) {
5665   }
5666 
5667   virtual bool doHeapRegion(HeapRegion* r) {
5668     if (!r->is_starts_humongous()) {
5669       return false;
5670     }
5671 
5672     G1CollectedHeap* g1h = G1CollectedHeap::heap();
5673 
5674     oop obj = (oop)r->bottom();
5675     CMBitMap* next_bitmap = g1h->concurrent_mark()->nextMarkBitMap();
5676 
5677     // The following checks whether the humongous object is live are sufficient.
5678     // The main additional check (in addition to having a reference from the roots
5679     // or the young gen) is whether the humongous object has a remembered set entry.
5680     //
5681     // A humongous object cannot be live if there is no remembered set for it
5682     // because:
5683     // - there can be no references from within humongous starts regions referencing
5684     // the object because we never allocate other objects into them.
5685     // (I.e. there are no intra-region references that may be missed by the
5686     // remembered set)
5687     // - as soon there is a remembered set entry to the humongous starts region
5688     // (i.e. it has "escaped" to an old object) this remembered set entry will stay
5689     // until the end of a concurrent mark.
5690     //
5691     // It is not required to check whether the object has been found dead by marking
5692     // or not, in fact it would prevent reclamation within a concurrent cycle, as
5693     // all objects allocated during that time are considered live.
5694     // SATB marking is even more conservative than the remembered set.
5695     // So if at this point in the collection there is no remembered set entry,
5696     // nobody has a reference to it.
5697     // At the start of collection we flush all refinement logs, and remembered sets
5698     // are completely up-to-date wrt to references to the humongous object.
5699     //
5700     // Other implementation considerations:
5701     // - never consider object arrays at this time because they would pose
5702     // considerable effort for cleaning up the the remembered sets. This is
5703     // required because stale remembered sets might reference locations that
5704     // are currently allocated into.
5705     uint region_idx = r->hrm_index();
5706     if (!g1h->is_humongous_reclaim_candidate(region_idx) ||
5707         !r->rem_set()->is_empty()) {
5708 
5709       if (G1TraceEagerReclaimHumongousObjects) {
5710         gclog_or_tty->print_cr("Live humongous region %u object size " SIZE_FORMAT " start " PTR_FORMAT "  with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5711                                region_idx,
5712                                (size_t)obj->size() * HeapWordSize,
5713                                p2i(r->bottom()),
5714                                r->rem_set()->occupied(),
5715                                r->rem_set()->strong_code_roots_list_length(),
5716                                next_bitmap->isMarked(r->bottom()),
5717                                g1h->is_humongous_reclaim_candidate(region_idx),
5718                                obj->is_typeArray()
5719                               );
5720       }
5721 
5722       return false;
5723     }
5724 
5725     guarantee(obj->is_typeArray(),
5726               "Only eagerly reclaiming type arrays is supported, but the object "
5727               PTR_FORMAT " is not.", p2i(r->bottom()));
5728 
5729     if (G1TraceEagerReclaimHumongousObjects) {
5730       gclog_or_tty->print_cr("Dead humongous region %u object size " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5731                              region_idx,
5732                              (size_t)obj->size() * HeapWordSize,
5733                              p2i(r->bottom()),
5734                              r->rem_set()->occupied(),
5735                              r->rem_set()->strong_code_roots_list_length(),
5736                              next_bitmap->isMarked(r->bottom()),
5737                              g1h->is_humongous_reclaim_candidate(region_idx),
5738                              obj->is_typeArray()
5739                             );
5740     }
5741     // Need to clear mark bit of the humongous object if already set.
5742     if (next_bitmap->isMarked(r->bottom())) {
5743       next_bitmap->clear(r->bottom());
5744     }
5745     do {
5746       HeapRegion* next = g1h->next_region_in_humongous(r);
5747       _freed_bytes += r->used();
5748       r->set_containing_set(NULL);
5749       _humongous_regions_removed.increment(1u, r->capacity());
5750       g1h->free_humongous_region(r, _free_region_list, false);
5751       r = next;
5752     } while (r != NULL);
5753 
5754     return false;
5755   }
5756 
5757   HeapRegionSetCount& humongous_free_count() {
5758     return _humongous_regions_removed;
5759   }
5760 
5761   size_t bytes_freed() const {
5762     return _freed_bytes;
5763   }
5764 
5765   size_t humongous_reclaimed() const {
5766     return _humongous_regions_removed.length();
5767   }
5768 };
5769 
5770 void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
5771   assert_at_safepoint(true);
5772 
5773   if (!G1EagerReclaimHumongousObjects ||
5774       (!_has_humongous_reclaim_candidates && !G1TraceEagerReclaimHumongousObjects)) {
5775     g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms(0.0, 0);
5776     return;
5777   }
5778 
5779   double start_time = os::elapsedTime();
5780 
5781   FreeRegionList local_cleanup_list("Local Humongous Cleanup List");
5782 
5783   G1FreeHumongousRegionClosure cl(&local_cleanup_list);
5784   heap_region_iterate(&cl);
5785 
5786   HeapRegionSetCount empty_set;
5787   remove_from_old_sets(empty_set, cl.humongous_free_count());
5788 
5789   G1HRPrinter* hrp = hr_printer();
5790   if (hrp->is_active()) {
5791     FreeRegionListIterator iter(&local_cleanup_list);
5792     while (iter.more_available()) {
5793       HeapRegion* hr = iter.get_next();
5794       hrp->cleanup(hr);
5795     }
5796   }
5797 
5798   prepend_to_freelist(&local_cleanup_list);
5799   decrement_summary_bytes(cl.bytes_freed());
5800 
5801   g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms((os::elapsedTime() - start_time) * 1000.0,
5802                                                                     cl.humongous_reclaimed());
5803 }
5804 
5805 // This routine is similar to the above but does not record
5806 // any policy statistics or update free lists; we are abandoning
5807 // the current incremental collection set in preparation of a
5808 // full collection. After the full GC we will start to build up
5809 // the incremental collection set again.
5810 // This is only called when we're doing a full collection
5811 // and is immediately followed by the tearing down of the young list.
5812 
5813 void G1CollectedHeap::abandon_collection_set(HeapRegion* cs_head) {
5814   HeapRegion* cur = cs_head;
5815 
5816   while (cur != NULL) {
5817     HeapRegion* next = cur->next_in_collection_set();
5818     assert(cur->in_collection_set(), "bad CS");
5819     cur->set_next_in_collection_set(NULL);
5820     clear_in_cset(cur);
5821     cur->set_young_index_in_cset(-1);
5822     cur = next;
5823   }
5824 }
5825 
5826 void G1CollectedHeap::set_free_regions_coming() {
5827   if (G1ConcRegionFreeingVerbose) {
5828     gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : "
5829                            "setting free regions coming");
5830   }
5831 
5832   assert(!free_regions_coming(), "pre-condition");
5833   _free_regions_coming = true;
5834 }
5835 
5836 void G1CollectedHeap::reset_free_regions_coming() {
5837   assert(free_regions_coming(), "pre-condition");
5838 
5839   {
5840     MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
5841     _free_regions_coming = false;
5842     SecondaryFreeList_lock->notify_all();
5843   }
5844 
5845   if (G1ConcRegionFreeingVerbose) {
5846     gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : "
5847                            "reset free regions coming");
5848   }
5849 }
5850 
5851 void G1CollectedHeap::wait_while_free_regions_coming() {
5852   // Most of the time we won't have to wait, so let's do a quick test
5853   // first before we take the lock.
5854   if (!free_regions_coming()) {
5855     return;
5856   }
5857 
5858   if (G1ConcRegionFreeingVerbose) {
5859     gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : "
5860                            "waiting for free regions");
5861   }
5862 
5863   {
5864     MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
5865     while (free_regions_coming()) {
5866       SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag);
5867     }
5868   }
5869 
5870   if (G1ConcRegionFreeingVerbose) {
5871     gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : "
5872                            "done waiting for free regions");
5873   }
5874 }
5875 
5876 bool G1CollectedHeap::is_old_gc_alloc_region(HeapRegion* hr) {
5877   return _allocator->is_retained_old_region(hr);
5878 }
5879 
5880 void G1CollectedHeap::set_region_short_lived_locked(HeapRegion* hr) {
5881   _young_list->push_region(hr);
5882 }
5883 
5884 class NoYoungRegionsClosure: public HeapRegionClosure {
5885 private:
5886   bool _success;
5887 public:
5888   NoYoungRegionsClosure() : _success(true) { }
5889   bool doHeapRegion(HeapRegion* r) {
5890     if (r->is_young()) {
5891       gclog_or_tty->print_cr("Region [" PTR_FORMAT ", " PTR_FORMAT ") tagged as young",
5892                              p2i(r->bottom()), p2i(r->end()));
5893       _success = false;
5894     }
5895     return false;
5896   }
5897   bool success() { return _success; }
5898 };
5899 
5900 bool G1CollectedHeap::check_young_list_empty(bool check_heap, bool check_sample) {
5901   bool ret = _young_list->check_list_empty(check_sample);
5902 
5903   if (check_heap) {
5904     NoYoungRegionsClosure closure;
5905     heap_region_iterate(&closure);
5906     ret = ret && closure.success();
5907   }
5908 
5909   return ret;
5910 }
5911 
5912 class TearDownRegionSetsClosure : public HeapRegionClosure {
5913 private:
5914   HeapRegionSet *_old_set;
5915 
5916 public:
5917   TearDownRegionSetsClosure(HeapRegionSet* old_set) : _old_set(old_set) { }
5918 
5919   bool doHeapRegion(HeapRegion* r) {
5920     if (r->is_old()) {
5921       _old_set->remove(r);
5922     } else {
5923       // We ignore free regions, we'll empty the free list afterwards.
5924       // We ignore young regions, we'll empty the young list afterwards.
5925       // We ignore humongous regions, we're not tearing down the
5926       // humongous regions set.
5927       assert(r->is_free() || r->is_young() || r->is_humongous(),
5928              "it cannot be another type");
5929     }
5930     return false;
5931   }
5932 
5933   ~TearDownRegionSetsClosure() {
5934     assert(_old_set->is_empty(), "post-condition");
5935   }
5936 };
5937 
5938 void G1CollectedHeap::tear_down_region_sets(bool free_list_only) {
5939   assert_at_safepoint(true /* should_be_vm_thread */);
5940 
5941   if (!free_list_only) {
5942     TearDownRegionSetsClosure cl(&_old_set);
5943     heap_region_iterate(&cl);
5944 
5945     // Note that emptying the _young_list is postponed and instead done as
5946     // the first step when rebuilding the regions sets again. The reason for
5947     // this is that during a full GC string deduplication needs to know if
5948     // a collected region was young or old when the full GC was initiated.
5949   }
5950   _hrm.remove_all_free_regions();
5951 }
5952 
5953 void G1CollectedHeap::increase_used(size_t bytes) {
5954   _summary_bytes_used += bytes;
5955 }
5956 
5957 void G1CollectedHeap::decrease_used(size_t bytes) {
5958   assert(_summary_bytes_used >= bytes,
5959          "invariant: _summary_bytes_used: " SIZE_FORMAT " should be >= bytes: " SIZE_FORMAT,
5960          _summary_bytes_used, bytes);
5961   _summary_bytes_used -= bytes;
5962 }
5963 
5964 void G1CollectedHeap::set_used(size_t bytes) {
5965   _summary_bytes_used = bytes;
5966 }
5967 
5968 class RebuildRegionSetsClosure : public HeapRegionClosure {
5969 private:
5970   bool            _free_list_only;
5971   HeapRegionSet*   _old_set;
5972   HeapRegionManager*   _hrm;
5973   size_t          _total_used;
5974 
5975 public:
5976   RebuildRegionSetsClosure(bool free_list_only,
5977                            HeapRegionSet* old_set, HeapRegionManager* hrm) :
5978     _free_list_only(free_list_only),
5979     _old_set(old_set), _hrm(hrm), _total_used(0) {
5980     assert(_hrm->num_free_regions() == 0, "pre-condition");
5981     if (!free_list_only) {
5982       assert(_old_set->is_empty(), "pre-condition");
5983     }
5984   }
5985 
5986   bool doHeapRegion(HeapRegion* r) {
5987     if (r->is_empty()) {
5988       // Add free regions to the free list
5989       r->set_free();
5990       r->set_allocation_context(AllocationContext::system());
5991       _hrm->insert_into_free_list(r);
5992     } else if (!_free_list_only) {
5993       assert(!r->is_young(), "we should not come across young regions");
5994 
5995       if (r->is_humongous()) {
5996         // We ignore humongous regions. We left the humongous set unchanged.
5997       } else {
5998         // Objects that were compacted would have ended up on regions
5999         // that were previously old or free.  Archive regions (which are
6000         // old) will not have been touched.
6001         assert(r->is_free() || r->is_old(), "invariant");
6002         // We now consider them old, so register as such. Leave
6003         // archive regions set that way, however, while still adding
6004         // them to the old set.
6005         if (!r->is_archive()) {
6006           r->set_old();
6007         }
6008         _old_set->add(r);
6009       }
6010       _total_used += r->used();
6011     }
6012 
6013     return false;
6014   }
6015 
6016   size_t total_used() {
6017     return _total_used;
6018   }
6019 };
6020 
6021 void G1CollectedHeap::rebuild_region_sets(bool free_list_only) {
6022   assert_at_safepoint(true /* should_be_vm_thread */);
6023 
6024   if (!free_list_only) {
6025     _young_list->empty_list();
6026   }
6027 
6028   RebuildRegionSetsClosure cl(free_list_only, &_old_set, &_hrm);
6029   heap_region_iterate(&cl);
6030 
6031   if (!free_list_only) {
6032     set_used(cl.total_used());
6033     if (_archive_allocator != NULL) {
6034       _archive_allocator->clear_used();
6035     }
6036   }
6037   assert(used_unlocked() == recalculate_used(),
6038          "inconsistent used_unlocked(), "
6039          "value: " SIZE_FORMAT " recalculated: " SIZE_FORMAT,
6040          used_unlocked(), recalculate_used());
6041 }
6042 
6043 void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) {
6044   _refine_cte_cl->set_concurrent(concurrent);
6045 }
6046 
6047 bool G1CollectedHeap::is_in_closed_subset(const void* p) const {
6048   HeapRegion* hr = heap_region_containing(p);
6049   return hr->is_in(p);
6050 }
6051 
6052 // Methods for the mutator alloc region
6053 
6054 HeapRegion* G1CollectedHeap::new_mutator_alloc_region(size_t word_size,
6055                                                       bool force) {
6056   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
6057   assert(!force || g1_policy()->can_expand_young_list(),
6058          "if force is true we should be able to expand the young list");
6059   bool young_list_full = g1_policy()->is_young_list_full();
6060   if (force || !young_list_full) {
6061     HeapRegion* new_alloc_region = new_region(word_size,
6062                                               false /* is_old */,
6063                                               false /* do_expand */);
6064     if (new_alloc_region != NULL) {
6065       set_region_short_lived_locked(new_alloc_region);
6066       _hr_printer.alloc(new_alloc_region, G1HRPrinter::Eden, young_list_full);
6067       check_bitmaps("Mutator Region Allocation", new_alloc_region);
6068       return new_alloc_region;
6069     }
6070   }
6071   return NULL;
6072 }
6073 
6074 void G1CollectedHeap::retire_mutator_alloc_region(HeapRegion* alloc_region,
6075                                                   size_t allocated_bytes) {
6076   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
6077   assert(alloc_region->is_eden(), "all mutator alloc regions should be eden");
6078 
6079   g1_policy()->add_region_to_incremental_cset_lhs(alloc_region);
6080   increase_used(allocated_bytes);
6081   _hr_printer.retire(alloc_region);
6082   // We update the eden sizes here, when the region is retired,
6083   // instead of when it's allocated, since this is the point that its
6084   // used space has been recored in _summary_bytes_used.
6085   g1mm()->update_eden_size();
6086 }
6087 
6088 // Methods for the GC alloc regions
6089 
6090 HeapRegion* G1CollectedHeap::new_gc_alloc_region(size_t word_size,
6091                                                  uint count,
6092                                                  InCSetState dest) {
6093   assert(FreeList_lock->owned_by_self(), "pre-condition");
6094 
6095   if (count < g1_policy()->max_regions(dest)) {
6096     const bool is_survivor = (dest.is_young());
6097     HeapRegion* new_alloc_region = new_region(word_size,
6098                                               !is_survivor,
6099                                               true /* do_expand */);
6100     if (new_alloc_region != NULL) {
6101       // We really only need to do this for old regions given that we
6102       // should never scan survivors. But it doesn't hurt to do it
6103       // for survivors too.
6104       new_alloc_region->record_timestamp();
6105       if (is_survivor) {
6106         new_alloc_region->set_survivor();
6107         _hr_printer.alloc(new_alloc_region, G1HRPrinter::Survivor);
6108         check_bitmaps("Survivor Region Allocation", new_alloc_region);
6109       } else {
6110         new_alloc_region->set_old();
6111         _hr_printer.alloc(new_alloc_region, G1HRPrinter::Old);
6112         check_bitmaps("Old Region Allocation", new_alloc_region);
6113       }
6114       bool during_im = collector_state()->during_initial_mark_pause();
6115       new_alloc_region->note_start_of_copying(during_im);
6116       return new_alloc_region;
6117     }
6118   }
6119   return NULL;
6120 }
6121 
6122 void G1CollectedHeap::retire_gc_alloc_region(HeapRegion* alloc_region,
6123                                              size_t allocated_bytes,
6124                                              InCSetState dest) {
6125   bool during_im = collector_state()->during_initial_mark_pause();
6126   alloc_region->note_end_of_copying(during_im);
6127   g1_policy()->record_bytes_copied_during_gc(allocated_bytes);
6128   if (dest.is_young()) {
6129     young_list()->add_survivor_region(alloc_region);
6130   } else {
6131     _old_set.add(alloc_region);
6132   }
6133   _hr_printer.retire(alloc_region);
6134 }
6135 
6136 HeapRegion* G1CollectedHeap::alloc_highest_free_region() {
6137   bool expanded = false;
6138   uint index = _hrm.find_highest_free(&expanded);
6139 
6140   if (index != G1_NO_HRM_INDEX) {
6141     if (expanded) {
6142       ergo_verbose1(ErgoHeapSizing,
6143                     "attempt heap expansion",
6144                     ergo_format_reason("requested address range outside heap bounds")
6145                     ergo_format_byte("region size"),
6146                     HeapRegion::GrainWords * HeapWordSize);
6147     }
6148     _hrm.allocate_free_regions_starting_at(index, 1);
6149     return region_at(index);
6150   }
6151   return NULL;
6152 }
6153 
6154 // Heap region set verification
6155 
6156 class VerifyRegionListsClosure : public HeapRegionClosure {
6157 private:
6158   HeapRegionSet*   _old_set;
6159   HeapRegionSet*   _humongous_set;
6160   HeapRegionManager*   _hrm;
6161 
6162 public:
6163   HeapRegionSetCount _old_count;
6164   HeapRegionSetCount _humongous_count;
6165   HeapRegionSetCount _free_count;
6166 
6167   VerifyRegionListsClosure(HeapRegionSet* old_set,
6168                            HeapRegionSet* humongous_set,
6169                            HeapRegionManager* hrm) :
6170     _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm),
6171     _old_count(), _humongous_count(), _free_count(){ }
6172 
6173   bool doHeapRegion(HeapRegion* hr) {
6174     if (hr->is_young()) {
6175       // TODO
6176     } else if (hr->is_humongous()) {
6177       assert(hr->containing_set() == _humongous_set, "Heap region %u is humongous but not in humongous set.", hr->hrm_index());
6178       _humongous_count.increment(1u, hr->capacity());
6179     } else if (hr->is_empty()) {
6180       assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index());
6181       _free_count.increment(1u, hr->capacity());
6182     } else if (hr->is_old()) {
6183       assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index());
6184       _old_count.increment(1u, hr->capacity());
6185     } else {
6186       // There are no other valid region types. Check for one invalid
6187       // one we can identify: pinned without old or humongous set.
6188       assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index());
6189       ShouldNotReachHere();
6190     }
6191     return false;
6192   }
6193 
6194   void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) {
6195     guarantee(old_set->length() == _old_count.length(), "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length());
6196     guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), "Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6197               old_set->total_capacity_bytes(), _old_count.capacity());
6198 
6199     guarantee(humongous_set->length() == _humongous_count.length(), "Hum set count mismatch. Expected %u, actual %u.", humongous_set->length(), _humongous_count.length());
6200     guarantee(humongous_set->total_capacity_bytes() == _humongous_count.capacity(), "Hum set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6201               humongous_set->total_capacity_bytes(), _humongous_count.capacity());
6202 
6203     guarantee(free_list->num_free_regions() == _free_count.length(), "Free list count mismatch. Expected %u, actual %u.", free_list->num_free_regions(), _free_count.length());
6204     guarantee(free_list->total_capacity_bytes() == _free_count.capacity(), "Free list capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6205               free_list->total_capacity_bytes(), _free_count.capacity());
6206   }
6207 };
6208 
6209 void G1CollectedHeap::verify_region_sets() {
6210   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
6211 
6212   // First, check the explicit lists.
6213   _hrm.verify();
6214   {
6215     // Given that a concurrent operation might be adding regions to
6216     // the secondary free list we have to take the lock before
6217     // verifying it.
6218     MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
6219     _secondary_free_list.verify_list();
6220   }
6221 
6222   // If a concurrent region freeing operation is in progress it will
6223   // be difficult to correctly attributed any free regions we come
6224   // across to the correct free list given that they might belong to
6225   // one of several (free_list, secondary_free_list, any local lists,
6226   // etc.). So, if that's the case we will skip the rest of the
6227   // verification operation. Alternatively, waiting for the concurrent
6228   // operation to complete will have a non-trivial effect on the GC's
6229   // operation (no concurrent operation will last longer than the
6230   // interval between two calls to verification) and it might hide
6231   // any issues that we would like to catch during testing.
6232   if (free_regions_coming()) {
6233     return;
6234   }
6235 
6236   // Make sure we append the secondary_free_list on the free_list so
6237   // that all free regions we will come across can be safely
6238   // attributed to the free_list.
6239   append_secondary_free_list_if_not_empty_with_lock();
6240 
6241   // Finally, make sure that the region accounting in the lists is
6242   // consistent with what we see in the heap.
6243 
6244   VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_hrm);
6245   heap_region_iterate(&cl);
6246   cl.verify_counts(&_old_set, &_humongous_set, &_hrm);
6247 }
6248 
6249 // Optimized nmethod scanning
6250 
6251 class RegisterNMethodOopClosure: public OopClosure {
6252   G1CollectedHeap* _g1h;
6253   nmethod* _nm;
6254 
6255   template <class T> void do_oop_work(T* p) {
6256     T heap_oop = oopDesc::load_heap_oop(p);
6257     if (!oopDesc::is_null(heap_oop)) {
6258       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
6259       HeapRegion* hr = _g1h->heap_region_containing(obj);
6260       assert(!hr->is_continues_humongous(),
6261              "trying to add code root " PTR_FORMAT " in continuation of humongous region " HR_FORMAT
6262              " starting at " HR_FORMAT,
6263              p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()));
6264 
6265       // HeapRegion::add_strong_code_root_locked() avoids adding duplicate entries.
6266       hr->add_strong_code_root_locked(_nm);
6267     }
6268   }
6269 
6270 public:
6271   RegisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
6272     _g1h(g1h), _nm(nm) {}
6273 
6274   void do_oop(oop* p)       { do_oop_work(p); }
6275   void do_oop(narrowOop* p) { do_oop_work(p); }
6276 };
6277 
6278 class UnregisterNMethodOopClosure: public OopClosure {
6279   G1CollectedHeap* _g1h;
6280   nmethod* _nm;
6281 
6282   template <class T> void do_oop_work(T* p) {
6283     T heap_oop = oopDesc::load_heap_oop(p);
6284     if (!oopDesc::is_null(heap_oop)) {
6285       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
6286       HeapRegion* hr = _g1h->heap_region_containing(obj);
6287       assert(!hr->is_continues_humongous(),
6288              "trying to remove code root " PTR_FORMAT " in continuation of humongous region " HR_FORMAT
6289              " starting at " HR_FORMAT,
6290              p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()));
6291 
6292       hr->remove_strong_code_root(_nm);
6293     }
6294   }
6295 
6296 public:
6297   UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
6298     _g1h(g1h), _nm(nm) {}
6299 
6300   void do_oop(oop* p)       { do_oop_work(p); }
6301   void do_oop(narrowOop* p) { do_oop_work(p); }
6302 };
6303 
6304 void G1CollectedHeap::register_nmethod(nmethod* nm) {
6305   CollectedHeap::register_nmethod(nm);
6306 
6307   guarantee(nm != NULL, "sanity");
6308   RegisterNMethodOopClosure reg_cl(this, nm);
6309   nm->oops_do(&reg_cl);
6310 }
6311 
6312 void G1CollectedHeap::unregister_nmethod(nmethod* nm) {
6313   CollectedHeap::unregister_nmethod(nm);
6314 
6315   guarantee(nm != NULL, "sanity");
6316   UnregisterNMethodOopClosure reg_cl(this, nm);
6317   nm->oops_do(&reg_cl, true);
6318 }
6319 
6320 void G1CollectedHeap::purge_code_root_memory() {
6321   double purge_start = os::elapsedTime();
6322   G1CodeRootSet::purge();
6323   double purge_time_ms = (os::elapsedTime() - purge_start) * 1000.0;
6324   g1_policy()->phase_times()->record_strong_code_root_purge_time(purge_time_ms);
6325 }
6326 
6327 class RebuildStrongCodeRootClosure: public CodeBlobClosure {
6328   G1CollectedHeap* _g1h;
6329 
6330 public:
6331   RebuildStrongCodeRootClosure(G1CollectedHeap* g1h) :
6332     _g1h(g1h) {}
6333 
6334   void do_code_blob(CodeBlob* cb) {
6335     nmethod* nm = (cb != NULL) ? cb->as_nmethod_or_null() : NULL;
6336     if (nm == NULL) {
6337       return;
6338     }
6339 
6340     if (ScavengeRootsInCode) {
6341       _g1h->register_nmethod(nm);
6342     }
6343   }
6344 };
6345 
6346 void G1CollectedHeap::rebuild_strong_code_roots() {
6347   RebuildStrongCodeRootClosure blob_cl(this);
6348   CodeCache::blobs_do(&blob_cl);
6349 }