rev 9312 : 8139424: SIGSEGV, Problematic frame: # V [libjvm.so+0xd0c0cc] void InstanceKlass::oop_oop_iterate_oop_maps_specialized<true,oopDesc*,MarkAndPushClosure>
Summary: The crash was caused by a faulty eager humongous reclaim. The reason for reclaiming a live obejct was that the call to cleanupHRRS was done after dirtying cards and clearing the remembered sets for the humongous object. This could lead to one or many cards being missed.
Reviewed-by:

   1 /*
   2  * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/metadataOnStackMark.hpp"
  27 #include "classfile/stringTable.hpp"
  28 #include "code/codeCache.hpp"
  29 #include "code/icBuffer.hpp"
  30 #include "gc/g1/bufferingOopClosure.hpp"
  31 #include "gc/g1/concurrentG1Refine.hpp"
  32 #include "gc/g1/concurrentG1RefineThread.hpp"
  33 #include "gc/g1/concurrentMarkThread.inline.hpp"
  34 #include "gc/g1/g1Allocator.inline.hpp"
  35 #include "gc/g1/g1CollectedHeap.inline.hpp"
  36 #include "gc/g1/g1CollectorPolicy.hpp"
  37 #include "gc/g1/g1CollectorState.hpp"
  38 #include "gc/g1/g1ErgoVerbose.hpp"
  39 #include "gc/g1/g1EvacFailure.hpp"
  40 #include "gc/g1/g1GCPhaseTimes.hpp"
  41 #include "gc/g1/g1Log.hpp"
  42 #include "gc/g1/g1MarkSweep.hpp"
  43 #include "gc/g1/g1OopClosures.inline.hpp"
  44 #include "gc/g1/g1ParScanThreadState.inline.hpp"
  45 #include "gc/g1/g1RegionToSpaceMapper.hpp"
  46 #include "gc/g1/g1RemSet.inline.hpp"
  47 #include "gc/g1/g1RootClosures.hpp"
  48 #include "gc/g1/g1RootProcessor.hpp"
  49 #include "gc/g1/g1StringDedup.hpp"
  50 #include "gc/g1/g1YCTypes.hpp"
  51 #include "gc/g1/heapRegion.inline.hpp"
  52 #include "gc/g1/heapRegionRemSet.hpp"
  53 #include "gc/g1/heapRegionSet.inline.hpp"
  54 #include "gc/g1/suspendibleThreadSet.hpp"
  55 #include "gc/g1/vm_operations_g1.hpp"
  56 #include "gc/shared/gcHeapSummary.hpp"
  57 #include "gc/shared/gcId.hpp"
  58 #include "gc/shared/gcLocker.inline.hpp"
  59 #include "gc/shared/gcTimer.hpp"
  60 #include "gc/shared/gcTrace.hpp"
  61 #include "gc/shared/gcTraceTime.hpp"
  62 #include "gc/shared/generationSpec.hpp"
  63 #include "gc/shared/isGCActiveMark.hpp"
  64 #include "gc/shared/referenceProcessor.hpp"
  65 #include "gc/shared/taskqueue.inline.hpp"
  66 #include "memory/allocation.hpp"
  67 #include "memory/iterator.hpp"
  68 #include "oops/oop.inline.hpp"
  69 #include "runtime/atomic.inline.hpp"
  70 #include "runtime/init.hpp"
  71 #include "runtime/orderAccess.inline.hpp"
  72 #include "runtime/vmThread.hpp"
  73 #include "utilities/globalDefinitions.hpp"
  74 #include "utilities/stack.inline.hpp"
  75 
  76 size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0;
  77 
  78 // INVARIANTS/NOTES
  79 //
  80 // All allocation activity covered by the G1CollectedHeap interface is
  81 // serialized by acquiring the HeapLock.  This happens in mem_allocate
  82 // and allocate_new_tlab, which are the "entry" points to the
  83 // allocation code from the rest of the JVM.  (Note that this does not
  84 // apply to TLAB allocation, which is not part of this interface: it
  85 // is done by clients of this interface.)
  86 
  87 // Local to this file.
  88 
  89 class RefineCardTableEntryClosure: public CardTableEntryClosure {
  90   bool _concurrent;
  91 public:
  92   RefineCardTableEntryClosure() : _concurrent(true) { }
  93 
  94   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
  95     bool oops_into_cset = G1CollectedHeap::heap()->g1_rem_set()->refine_card(card_ptr, worker_i, false);
  96     // This path is executed by the concurrent refine or mutator threads,
  97     // concurrently, and so we do not care if card_ptr contains references
  98     // that point into the collection set.
  99     assert(!oops_into_cset, "should be");
 100 
 101     if (_concurrent && SuspendibleThreadSet::should_yield()) {
 102       // Caller will actually yield.
 103       return false;
 104     }
 105     // Otherwise, we finished successfully; return true.
 106     return true;
 107   }
 108 
 109   void set_concurrent(bool b) { _concurrent = b; }
 110 };
 111 
 112 
 113 class RedirtyLoggedCardTableEntryClosure : public CardTableEntryClosure {
 114  private:
 115   size_t _num_processed;
 116 
 117  public:
 118   RedirtyLoggedCardTableEntryClosure() : CardTableEntryClosure(), _num_processed(0) { }
 119 
 120   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
 121     *card_ptr = CardTableModRefBS::dirty_card_val();
 122     _num_processed++;
 123     return true;
 124   }
 125 
 126   size_t num_processed() const { return _num_processed; }
 127 };
 128 
 129 
 130 void G1RegionMappingChangedListener::reset_from_card_cache(uint start_idx, size_t num_regions) {
 131   HeapRegionRemSet::invalidate_from_card_cache(start_idx, num_regions);
 132 }
 133 
 134 void G1RegionMappingChangedListener::on_commit(uint start_idx, size_t num_regions, bool zero_filled) {
 135   // The from card cache is not the memory that is actually committed. So we cannot
 136   // take advantage of the zero_filled parameter.
 137   reset_from_card_cache(start_idx, num_regions);
 138 }
 139 
 140 void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr)
 141 {
 142   // Claim the right to put the region on the dirty cards region list
 143   // by installing a self pointer.
 144   HeapRegion* next = hr->get_next_dirty_cards_region();
 145   if (next == NULL) {
 146     HeapRegion* res = (HeapRegion*)
 147       Atomic::cmpxchg_ptr(hr, hr->next_dirty_cards_region_addr(),
 148                           NULL);
 149     if (res == NULL) {
 150       HeapRegion* head;
 151       do {
 152         // Put the region to the dirty cards region list.
 153         head = _dirty_cards_region_list;
 154         next = (HeapRegion*)
 155           Atomic::cmpxchg_ptr(hr, &_dirty_cards_region_list, head);
 156         if (next == head) {
 157           assert(hr->get_next_dirty_cards_region() == hr,
 158                  "hr->get_next_dirty_cards_region() != hr");
 159           if (next == NULL) {
 160             // The last region in the list points to itself.
 161             hr->set_next_dirty_cards_region(hr);
 162           } else {
 163             hr->set_next_dirty_cards_region(next);
 164           }
 165         }
 166       } while (next != head);
 167     }
 168   }
 169 }
 170 
 171 HeapRegion* G1CollectedHeap::pop_dirty_cards_region()
 172 {
 173   HeapRegion* head;
 174   HeapRegion* hr;
 175   do {
 176     head = _dirty_cards_region_list;
 177     if (head == NULL) {
 178       return NULL;
 179     }
 180     HeapRegion* new_head = head->get_next_dirty_cards_region();
 181     if (head == new_head) {
 182       // The last region.
 183       new_head = NULL;
 184     }
 185     hr = (HeapRegion*)Atomic::cmpxchg_ptr(new_head, &_dirty_cards_region_list,
 186                                           head);
 187   } while (hr != head);
 188   assert(hr != NULL, "invariant");
 189   hr->set_next_dirty_cards_region(NULL);
 190   return hr;
 191 }
 192 
 193 // Returns true if the reference points to an object that
 194 // can move in an incremental collection.
 195 bool G1CollectedHeap::is_scavengable(const void* p) {
 196   HeapRegion* hr = heap_region_containing(p);
 197   return !hr->is_pinned();
 198 }
 199 
 200 // Private methods.
 201 
 202 HeapRegion*
 203 G1CollectedHeap::new_region_try_secondary_free_list(bool is_old) {
 204   MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
 205   while (!_secondary_free_list.is_empty() || free_regions_coming()) {
 206     if (!_secondary_free_list.is_empty()) {
 207       if (G1ConcRegionFreeingVerbose) {
 208         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "
 209                                "secondary_free_list has %u entries",
 210                                _secondary_free_list.length());
 211       }
 212       // It looks as if there are free regions available on the
 213       // secondary_free_list. Let's move them to the free_list and try
 214       // again to allocate from it.
 215       append_secondary_free_list();
 216 
 217       assert(_hrm.num_free_regions() > 0, "if the secondary_free_list was not "
 218              "empty we should have moved at least one entry to the free_list");
 219       HeapRegion* res = _hrm.allocate_free_region(is_old);
 220       if (G1ConcRegionFreeingVerbose) {
 221         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "
 222                                "allocated " HR_FORMAT " from secondary_free_list",
 223                                HR_FORMAT_PARAMS(res));
 224       }
 225       return res;
 226     }
 227 
 228     // Wait here until we get notified either when (a) there are no
 229     // more free regions coming or (b) some regions have been moved on
 230     // the secondary_free_list.
 231     SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag);
 232   }
 233 
 234   if (G1ConcRegionFreeingVerbose) {
 235     gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "
 236                            "could not allocate from secondary_free_list");
 237   }
 238   return NULL;
 239 }
 240 
 241 HeapRegion* G1CollectedHeap::new_region(size_t word_size, bool is_old, bool do_expand) {
 242   assert(!is_humongous(word_size) || word_size <= HeapRegion::GrainWords,
 243          "the only time we use this to allocate a humongous region is "
 244          "when we are allocating a single humongous region");
 245 
 246   HeapRegion* res;
 247   if (G1StressConcRegionFreeing) {
 248     if (!_secondary_free_list.is_empty()) {
 249       if (G1ConcRegionFreeingVerbose) {
 250         gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "
 251                                "forced to look at the secondary_free_list");
 252       }
 253       res = new_region_try_secondary_free_list(is_old);
 254       if (res != NULL) {
 255         return res;
 256       }
 257     }
 258   }
 259 
 260   res = _hrm.allocate_free_region(is_old);
 261 
 262   if (res == NULL) {
 263     if (G1ConcRegionFreeingVerbose) {
 264       gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : "
 265                              "res == NULL, trying the secondary_free_list");
 266     }
 267     res = new_region_try_secondary_free_list(is_old);
 268   }
 269   if (res == NULL && do_expand && _expand_heap_after_alloc_failure) {
 270     // Currently, only attempts to allocate GC alloc regions set
 271     // do_expand to true. So, we should only reach here during a
 272     // safepoint. If this assumption changes we might have to
 273     // reconsider the use of _expand_heap_after_alloc_failure.
 274     assert(SafepointSynchronize::is_at_safepoint(), "invariant");
 275 
 276     ergo_verbose1(ErgoHeapSizing,
 277                   "attempt heap expansion",
 278                   ergo_format_reason("region allocation request failed")
 279                   ergo_format_byte("allocation request"),
 280                   word_size * HeapWordSize);
 281     if (expand(word_size * HeapWordSize)) {
 282       // Given that expand() succeeded in expanding the heap, and we
 283       // always expand the heap by an amount aligned to the heap
 284       // region size, the free list should in theory not be empty.
 285       // In either case allocate_free_region() will check for NULL.
 286       res = _hrm.allocate_free_region(is_old);
 287     } else {
 288       _expand_heap_after_alloc_failure = false;
 289     }
 290   }
 291   return res;
 292 }
 293 
 294 HeapWord*
 295 G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first,
 296                                                            uint num_regions,
 297                                                            size_t word_size,
 298                                                            AllocationContext_t context) {
 299   assert(first != G1_NO_HRM_INDEX, "pre-condition");
 300   assert(is_humongous(word_size), "word_size should be humongous");
 301   assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");
 302 
 303   // Index of last region in the series + 1.
 304   uint last = first + num_regions;
 305 
 306   // We need to initialize the region(s) we just discovered. This is
 307   // a bit tricky given that it can happen concurrently with
 308   // refinement threads refining cards on these regions and
 309   // potentially wanting to refine the BOT as they are scanning
 310   // those cards (this can happen shortly after a cleanup; see CR
 311   // 6991377). So we have to set up the region(s) carefully and in
 312   // a specific order.
 313 
 314   // The word size sum of all the regions we will allocate.
 315   size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
 316   assert(word_size <= word_size_sum, "sanity");
 317 
 318   // This will be the "starts humongous" region.
 319   HeapRegion* first_hr = region_at(first);
 320   // The header of the new object will be placed at the bottom of
 321   // the first region.
 322   HeapWord* new_obj = first_hr->bottom();
 323   // This will be the new top of the new object.
 324   HeapWord* obj_top = new_obj + word_size;
 325 
 326   // First, we need to zero the header of the space that we will be
 327   // allocating. When we update top further down, some refinement
 328   // threads might try to scan the region. By zeroing the header we
 329   // ensure that any thread that will try to scan the region will
 330   // come across the zero klass word and bail out.
 331   //
 332   // NOTE: It would not have been correct to have used
 333   // CollectedHeap::fill_with_object() and make the space look like
 334   // an int array. The thread that is doing the allocation will
 335   // later update the object header to a potentially different array
 336   // type and, for a very short period of time, the klass and length
 337   // fields will be inconsistent. This could cause a refinement
 338   // thread to calculate the object size incorrectly.
 339   Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
 340 
 341   // We will set up the first region as "starts humongous". This
 342   // will also update the BOT covering all the regions to reflect
 343   // that there is a single object that starts at the bottom of the
 344   // first region.
 345   first_hr->set_starts_humongous(obj_top);
 346   first_hr->set_allocation_context(context);
 347   // Then, if there are any, we will set up the "continues
 348   // humongous" regions.
 349   HeapRegion* hr = NULL;
 350   for (uint i = first + 1; i < last; ++i) {
 351     hr = region_at(i);
 352     hr->set_continues_humongous(first_hr);
 353     hr->set_allocation_context(context);
 354   }
 355 
 356   // Up to this point no concurrent thread would have been able to
 357   // do any scanning on any region in this series. All the top
 358   // fields still point to bottom, so the intersection between
 359   // [bottom,top] and [card_start,card_end] will be empty. Before we
 360   // update the top fields, we'll do a storestore to make sure that
 361   // no thread sees the update to top before the zeroing of the
 362   // object header and the BOT initialization.
 363   OrderAccess::storestore();
 364 
 365   // Now that the BOT and the object header have been initialized,
 366   // we can update top of the "starts humongous" region.
 367   first_hr->set_top(MIN2(first_hr->end(), obj_top));
 368   if (_hr_printer.is_active()) {
 369     _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, first_hr->top());
 370   }
 371 
 372   // Now, we will update the top fields of the "continues humongous"
 373   // regions.
 374   hr = NULL;
 375   for (uint i = first + 1; i < last; ++i) {
 376     hr = region_at(i);
 377     if ((i + 1) == last) {
 378       // last continues humongous region
 379       assert(hr->bottom() < obj_top && obj_top <= hr->end(),
 380              "new_top should fall on this region");
 381       hr->set_top(obj_top);
 382       _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, obj_top);
 383     } else {
 384       // not last one
 385       assert(obj_top > hr->end(), "obj_top should be above this region");
 386       hr->set_top(hr->end());
 387       _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end());
 388     }
 389   }
 390   // If we have continues humongous regions (hr != NULL), its top should
 391   // match obj_top.
 392   assert(hr == NULL || (hr->top() == obj_top), "sanity");
 393   check_bitmaps("Humongous Region Allocation", first_hr);
 394 
 395   increase_used(word_size * HeapWordSize);
 396 
 397   for (uint i = first; i < last; ++i) {
 398     _humongous_set.add(region_at(i));
 399   }
 400 
 401   return new_obj;
 402 }
 403 
 404 // If could fit into free regions w/o expansion, try.
 405 // Otherwise, if can expand, do so.
 406 // Otherwise, if using ex regions might help, try with ex given back.
 407 HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) {
 408   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
 409 
 410   verify_region_sets_optional();
 411 
 412   uint first = G1_NO_HRM_INDEX;
 413   uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords);
 414 
 415   if (obj_regions == 1) {
 416     // Only one region to allocate, try to use a fast path by directly allocating
 417     // from the free lists. Do not try to expand here, we will potentially do that
 418     // later.
 419     HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */);
 420     if (hr != NULL) {
 421       first = hr->hrm_index();
 422     }
 423   } else {
 424     // We can't allocate humongous regions spanning more than one region while
 425     // cleanupComplete() is running, since some of the regions we find to be
 426     // empty might not yet be added to the free list. It is not straightforward
 427     // to know in which list they are on so that we can remove them. We only
 428     // need to do this if we need to allocate more than one region to satisfy the
 429     // current humongous allocation request. If we are only allocating one region
 430     // we use the one-region region allocation code (see above), that already
 431     // potentially waits for regions from the secondary free list.
 432     wait_while_free_regions_coming();
 433     append_secondary_free_list_if_not_empty_with_lock();
 434 
 435     // Policy: Try only empty regions (i.e. already committed first). Maybe we
 436     // are lucky enough to find some.
 437     first = _hrm.find_contiguous_only_empty(obj_regions);
 438     if (first != G1_NO_HRM_INDEX) {
 439       _hrm.allocate_free_regions_starting_at(first, obj_regions);
 440     }
 441   }
 442 
 443   if (first == G1_NO_HRM_INDEX) {
 444     // Policy: We could not find enough regions for the humongous object in the
 445     // free list. Look through the heap to find a mix of free and uncommitted regions.
 446     // If so, try expansion.
 447     first = _hrm.find_contiguous_empty_or_unavailable(obj_regions);
 448     if (first != G1_NO_HRM_INDEX) {
 449       // We found something. Make sure these regions are committed, i.e. expand
 450       // the heap. Alternatively we could do a defragmentation GC.
 451       ergo_verbose1(ErgoHeapSizing,
 452                     "attempt heap expansion",
 453                     ergo_format_reason("humongous allocation request failed")
 454                     ergo_format_byte("allocation request"),
 455                     word_size * HeapWordSize);
 456 
 457       _hrm.expand_at(first, obj_regions);
 458       g1_policy()->record_new_heap_size(num_regions());
 459 
 460 #ifdef ASSERT
 461       for (uint i = first; i < first + obj_regions; ++i) {
 462         HeapRegion* hr = region_at(i);
 463         assert(hr->is_free(), "sanity");
 464         assert(hr->is_empty(), "sanity");
 465         assert(is_on_master_free_list(hr), "sanity");
 466       }
 467 #endif
 468       _hrm.allocate_free_regions_starting_at(first, obj_regions);
 469     } else {
 470       // Policy: Potentially trigger a defragmentation GC.
 471     }
 472   }
 473 
 474   HeapWord* result = NULL;
 475   if (first != G1_NO_HRM_INDEX) {
 476     result = humongous_obj_allocate_initialize_regions(first, obj_regions,
 477                                                        word_size, context);
 478     assert(result != NULL, "it should always return a valid result");
 479 
 480     // A successful humongous object allocation changes the used space
 481     // information of the old generation so we need to recalculate the
 482     // sizes and update the jstat counters here.
 483     g1mm()->update_sizes();
 484   }
 485 
 486   verify_region_sets_optional();
 487 
 488   return result;
 489 }
 490 
 491 HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) {
 492   assert_heap_not_locked_and_not_at_safepoint();
 493   assert(!is_humongous(word_size), "we do not allow humongous TLABs");
 494 
 495   uint dummy_gc_count_before;
 496   uint dummy_gclocker_retry_count = 0;
 497   return attempt_allocation(word_size, &dummy_gc_count_before, &dummy_gclocker_retry_count);
 498 }
 499 
 500 HeapWord*
 501 G1CollectedHeap::mem_allocate(size_t word_size,
 502                               bool*  gc_overhead_limit_was_exceeded) {
 503   assert_heap_not_locked_and_not_at_safepoint();
 504 
 505   // Loop until the allocation is satisfied, or unsatisfied after GC.
 506   for (uint try_count = 1, gclocker_retry_count = 0; /* we'll return */; try_count += 1) {
 507     uint gc_count_before;
 508 
 509     HeapWord* result = NULL;
 510     if (!is_humongous(word_size)) {
 511       result = attempt_allocation(word_size, &gc_count_before, &gclocker_retry_count);
 512     } else {
 513       result = attempt_allocation_humongous(word_size, &gc_count_before, &gclocker_retry_count);
 514     }
 515     if (result != NULL) {
 516       return result;
 517     }
 518 
 519     // Create the garbage collection operation...
 520     VM_G1CollectForAllocation op(gc_count_before, word_size);
 521     op.set_allocation_context(AllocationContext::current());
 522 
 523     // ...and get the VM thread to execute it.
 524     VMThread::execute(&op);
 525 
 526     if (op.prologue_succeeded() && op.pause_succeeded()) {
 527       // If the operation was successful we'll return the result even
 528       // if it is NULL. If the allocation attempt failed immediately
 529       // after a Full GC, it's unlikely we'll be able to allocate now.
 530       HeapWord* result = op.result();
 531       if (result != NULL && !is_humongous(word_size)) {
 532         // Allocations that take place on VM operations do not do any
 533         // card dirtying and we have to do it here. We only have to do
 534         // this for non-humongous allocations, though.
 535         dirty_young_block(result, word_size);
 536       }
 537       return result;
 538     } else {
 539       if (gclocker_retry_count > GCLockerRetryAllocationCount) {
 540         return NULL;
 541       }
 542       assert(op.result() == NULL,
 543              "the result should be NULL if the VM op did not succeed");
 544     }
 545 
 546     // Give a warning if we seem to be looping forever.
 547     if ((QueuedAllocationWarningCount > 0) &&
 548         (try_count % QueuedAllocationWarningCount == 0)) {
 549       warning("G1CollectedHeap::mem_allocate retries %d times", try_count);
 550     }
 551   }
 552 
 553   ShouldNotReachHere();
 554   return NULL;
 555 }
 556 
 557 HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size,
 558                                                    AllocationContext_t context,
 559                                                    uint* gc_count_before_ret,
 560                                                    uint* gclocker_retry_count_ret) {
 561   // Make sure you read the note in attempt_allocation_humongous().
 562 
 563   assert_heap_not_locked_and_not_at_safepoint();
 564   assert(!is_humongous(word_size), "attempt_allocation_slow() should not "
 565          "be called for humongous allocation requests");
 566 
 567   // We should only get here after the first-level allocation attempt
 568   // (attempt_allocation()) failed to allocate.
 569 
 570   // We will loop until a) we manage to successfully perform the
 571   // allocation or b) we successfully schedule a collection which
 572   // fails to perform the allocation. b) is the only case when we'll
 573   // return NULL.
 574   HeapWord* result = NULL;
 575   for (int try_count = 1; /* we'll return */; try_count += 1) {
 576     bool should_try_gc;
 577     uint gc_count_before;
 578 
 579     {
 580       MutexLockerEx x(Heap_lock);
 581       result = _allocator->attempt_allocation_locked(word_size, context);
 582       if (result != NULL) {
 583         return result;
 584       }
 585 
 586       if (GC_locker::is_active_and_needs_gc()) {
 587         if (g1_policy()->can_expand_young_list()) {
 588           // No need for an ergo verbose message here,
 589           // can_expand_young_list() does this when it returns true.
 590           result = _allocator->attempt_allocation_force(word_size, context);
 591           if (result != NULL) {
 592             return result;
 593           }
 594         }
 595         should_try_gc = false;
 596       } else {
 597         // The GCLocker may not be active but the GCLocker initiated
 598         // GC may not yet have been performed (GCLocker::needs_gc()
 599         // returns true). In this case we do not try this GC and
 600         // wait until the GCLocker initiated GC is performed, and
 601         // then retry the allocation.
 602         if (GC_locker::needs_gc()) {
 603           should_try_gc = false;
 604         } else {
 605           // Read the GC count while still holding the Heap_lock.
 606           gc_count_before = total_collections();
 607           should_try_gc = true;
 608         }
 609       }
 610     }
 611 
 612     if (should_try_gc) {
 613       bool succeeded;
 614       result = do_collection_pause(word_size, gc_count_before, &succeeded,
 615                                    GCCause::_g1_inc_collection_pause);
 616       if (result != NULL) {
 617         assert(succeeded, "only way to get back a non-NULL result");
 618         return result;
 619       }
 620 
 621       if (succeeded) {
 622         // If we get here we successfully scheduled a collection which
 623         // failed to allocate. No point in trying to allocate
 624         // further. We'll just return NULL.
 625         MutexLockerEx x(Heap_lock);
 626         *gc_count_before_ret = total_collections();
 627         return NULL;
 628       }
 629     } else {
 630       if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) {
 631         MutexLockerEx x(Heap_lock);
 632         *gc_count_before_ret = total_collections();
 633         return NULL;
 634       }
 635       // The GCLocker is either active or the GCLocker initiated
 636       // GC has not yet been performed. Stall until it is and
 637       // then retry the allocation.
 638       GC_locker::stall_until_clear();
 639       (*gclocker_retry_count_ret) += 1;
 640     }
 641 
 642     // We can reach here if we were unsuccessful in scheduling a
 643     // collection (because another thread beat us to it) or if we were
 644     // stalled due to the GC locker. In either can we should retry the
 645     // allocation attempt in case another thread successfully
 646     // performed a collection and reclaimed enough space. We do the
 647     // first attempt (without holding the Heap_lock) here and the
 648     // follow-on attempt will be at the start of the next loop
 649     // iteration (after taking the Heap_lock).
 650     result = _allocator->attempt_allocation(word_size, context);
 651     if (result != NULL) {
 652       return result;
 653     }
 654 
 655     // Give a warning if we seem to be looping forever.
 656     if ((QueuedAllocationWarningCount > 0) &&
 657         (try_count % QueuedAllocationWarningCount == 0)) {
 658       warning("G1CollectedHeap::attempt_allocation_slow() "
 659               "retries %d times", try_count);
 660     }
 661   }
 662 
 663   ShouldNotReachHere();
 664   return NULL;
 665 }
 666 
 667 void G1CollectedHeap::begin_archive_alloc_range() {
 668   assert_at_safepoint(true /* should_be_vm_thread */);
 669   if (_archive_allocator == NULL) {
 670     _archive_allocator = G1ArchiveAllocator::create_allocator(this);
 671   }
 672 }
 673 
 674 bool G1CollectedHeap::is_archive_alloc_too_large(size_t word_size) {
 675   // Allocations in archive regions cannot be of a size that would be considered
 676   // humongous even for a minimum-sized region, because G1 region sizes/boundaries
 677   // may be different at archive-restore time.
 678   return word_size >= humongous_threshold_for(HeapRegion::min_region_size_in_words());
 679 }
 680 
 681 HeapWord* G1CollectedHeap::archive_mem_allocate(size_t word_size) {
 682   assert_at_safepoint(true /* should_be_vm_thread */);
 683   assert(_archive_allocator != NULL, "_archive_allocator not initialized");
 684   if (is_archive_alloc_too_large(word_size)) {
 685     return NULL;
 686   }
 687   return _archive_allocator->archive_mem_allocate(word_size);
 688 }
 689 
 690 void G1CollectedHeap::end_archive_alloc_range(GrowableArray<MemRegion>* ranges,
 691                                               size_t end_alignment_in_bytes) {
 692   assert_at_safepoint(true /* should_be_vm_thread */);
 693   assert(_archive_allocator != NULL, "_archive_allocator not initialized");
 694 
 695   // Call complete_archive to do the real work, filling in the MemRegion
 696   // array with the archive regions.
 697   _archive_allocator->complete_archive(ranges, end_alignment_in_bytes);
 698   delete _archive_allocator;
 699   _archive_allocator = NULL;
 700 }
 701 
 702 bool G1CollectedHeap::check_archive_addresses(MemRegion* ranges, size_t count) {
 703   assert(ranges != NULL, "MemRegion array NULL");
 704   assert(count != 0, "No MemRegions provided");
 705   MemRegion reserved = _hrm.reserved();
 706   for (size_t i = 0; i < count; i++) {
 707     if (!reserved.contains(ranges[i].start()) || !reserved.contains(ranges[i].last())) {
 708       return false;
 709     }
 710   }
 711   return true;
 712 }
 713 
 714 bool G1CollectedHeap::alloc_archive_regions(MemRegion* ranges, size_t count) {
 715   assert(!is_init_completed(), "Expect to be called at JVM init time");
 716   assert(ranges != NULL, "MemRegion array NULL");
 717   assert(count != 0, "No MemRegions provided");
 718   MutexLockerEx x(Heap_lock);
 719 
 720   MemRegion reserved = _hrm.reserved();
 721   HeapWord* prev_last_addr = NULL;
 722   HeapRegion* prev_last_region = NULL;
 723 
 724   // Temporarily disable pretouching of heap pages. This interface is used
 725   // when mmap'ing archived heap data in, so pre-touching is wasted.
 726   FlagSetting fs(AlwaysPreTouch, false);
 727 
 728   // Enable archive object checking in G1MarkSweep. We have to let it know
 729   // about each archive range, so that objects in those ranges aren't marked.
 730   G1MarkSweep::enable_archive_object_check();
 731 
 732   // For each specified MemRegion range, allocate the corresponding G1
 733   // regions and mark them as archive regions. We expect the ranges in
 734   // ascending starting address order, without overlap.
 735   for (size_t i = 0; i < count; i++) {
 736     MemRegion curr_range = ranges[i];
 737     HeapWord* start_address = curr_range.start();
 738     size_t word_size = curr_range.word_size();
 739     HeapWord* last_address = curr_range.last();
 740     size_t commits = 0;
 741 
 742     guarantee(reserved.contains(start_address) && reserved.contains(last_address),
 743               "MemRegion outside of heap [" PTR_FORMAT ", " PTR_FORMAT "]",
 744               p2i(start_address), p2i(last_address));
 745     guarantee(start_address > prev_last_addr,
 746               "Ranges not in ascending order: " PTR_FORMAT " <= " PTR_FORMAT ,
 747               p2i(start_address), p2i(prev_last_addr));
 748     prev_last_addr = last_address;
 749 
 750     // Check for ranges that start in the same G1 region in which the previous
 751     // range ended, and adjust the start address so we don't try to allocate
 752     // the same region again. If the current range is entirely within that
 753     // region, skip it, just adjusting the recorded top.
 754     HeapRegion* start_region = _hrm.addr_to_region(start_address);
 755     if ((prev_last_region != NULL) && (start_region == prev_last_region)) {
 756       start_address = start_region->end();
 757       if (start_address > last_address) {
 758         increase_used(word_size * HeapWordSize);
 759         start_region->set_top(last_address + 1);
 760         continue;
 761       }
 762       start_region->set_top(start_address);
 763       curr_range = MemRegion(start_address, last_address + 1);
 764       start_region = _hrm.addr_to_region(start_address);
 765     }
 766 
 767     // Perform the actual region allocation, exiting if it fails.
 768     // Then note how much new space we have allocated.
 769     if (!_hrm.allocate_containing_regions(curr_range, &commits)) {
 770       return false;
 771     }
 772     increase_used(word_size * HeapWordSize);
 773     if (commits != 0) {
 774       ergo_verbose1(ErgoHeapSizing,
 775                     "attempt heap expansion",
 776                     ergo_format_reason("allocate archive regions")
 777                     ergo_format_byte("total size"),
 778                     HeapRegion::GrainWords * HeapWordSize * commits);
 779     }
 780 
 781     // Mark each G1 region touched by the range as archive, add it to the old set,
 782     // and set the allocation context and top.
 783     HeapRegion* curr_region = _hrm.addr_to_region(start_address);
 784     HeapRegion* last_region = _hrm.addr_to_region(last_address);
 785     prev_last_region = last_region;
 786 
 787     while (curr_region != NULL) {
 788       assert(curr_region->is_empty() && !curr_region->is_pinned(),
 789              "Region already in use (index %u)", curr_region->hrm_index());
 790       _hr_printer.alloc(curr_region, G1HRPrinter::Archive);
 791       curr_region->set_allocation_context(AllocationContext::system());
 792       curr_region->set_archive();
 793       _old_set.add(curr_region);
 794       if (curr_region != last_region) {
 795         curr_region->set_top(curr_region->end());
 796         curr_region = _hrm.next_region_in_heap(curr_region);
 797       } else {
 798         curr_region->set_top(last_address + 1);
 799         curr_region = NULL;
 800       }
 801     }
 802 
 803     // Notify mark-sweep of the archive range.
 804     G1MarkSweep::set_range_archive(curr_range, true);
 805   }
 806   return true;
 807 }
 808 
 809 void G1CollectedHeap::fill_archive_regions(MemRegion* ranges, size_t count) {
 810   assert(!is_init_completed(), "Expect to be called at JVM init time");
 811   assert(ranges != NULL, "MemRegion array NULL");
 812   assert(count != 0, "No MemRegions provided");
 813   MemRegion reserved = _hrm.reserved();
 814   HeapWord *prev_last_addr = NULL;
 815   HeapRegion* prev_last_region = NULL;
 816 
 817   // For each MemRegion, create filler objects, if needed, in the G1 regions
 818   // that contain the address range. The address range actually within the
 819   // MemRegion will not be modified. That is assumed to have been initialized
 820   // elsewhere, probably via an mmap of archived heap data.
 821   MutexLockerEx x(Heap_lock);
 822   for (size_t i = 0; i < count; i++) {
 823     HeapWord* start_address = ranges[i].start();
 824     HeapWord* last_address = ranges[i].last();
 825 
 826     assert(reserved.contains(start_address) && reserved.contains(last_address),
 827            "MemRegion outside of heap [" PTR_FORMAT ", " PTR_FORMAT "]",
 828            p2i(start_address), p2i(last_address));
 829     assert(start_address > prev_last_addr,
 830            "Ranges not in ascending order: " PTR_FORMAT " <= " PTR_FORMAT ,
 831            p2i(start_address), p2i(prev_last_addr));
 832 
 833     HeapRegion* start_region = _hrm.addr_to_region(start_address);
 834     HeapRegion* last_region = _hrm.addr_to_region(last_address);
 835     HeapWord* bottom_address = start_region->bottom();
 836 
 837     // Check for a range beginning in the same region in which the
 838     // previous one ended.
 839     if (start_region == prev_last_region) {
 840       bottom_address = prev_last_addr + 1;
 841     }
 842 
 843     // Verify that the regions were all marked as archive regions by
 844     // alloc_archive_regions.
 845     HeapRegion* curr_region = start_region;
 846     while (curr_region != NULL) {
 847       guarantee(curr_region->is_archive(),
 848                 "Expected archive region at index %u", curr_region->hrm_index());
 849       if (curr_region != last_region) {
 850         curr_region = _hrm.next_region_in_heap(curr_region);
 851       } else {
 852         curr_region = NULL;
 853       }
 854     }
 855 
 856     prev_last_addr = last_address;
 857     prev_last_region = last_region;
 858 
 859     // Fill the memory below the allocated range with dummy object(s),
 860     // if the region bottom does not match the range start, or if the previous
 861     // range ended within the same G1 region, and there is a gap.
 862     if (start_address != bottom_address) {
 863       size_t fill_size = pointer_delta(start_address, bottom_address);
 864       G1CollectedHeap::fill_with_objects(bottom_address, fill_size);
 865       increase_used(fill_size * HeapWordSize);
 866     }
 867   }
 868 }
 869 
 870 inline HeapWord* G1CollectedHeap::attempt_allocation(size_t word_size,
 871                                                      uint* gc_count_before_ret,
 872                                                      uint* gclocker_retry_count_ret) {
 873   assert_heap_not_locked_and_not_at_safepoint();
 874   assert(!is_humongous(word_size), "attempt_allocation() should not "
 875          "be called for humongous allocation requests");
 876 
 877   AllocationContext_t context = AllocationContext::current();
 878   HeapWord* result = _allocator->attempt_allocation(word_size, context);
 879 
 880   if (result == NULL) {
 881     result = attempt_allocation_slow(word_size,
 882                                      context,
 883                                      gc_count_before_ret,
 884                                      gclocker_retry_count_ret);
 885   }
 886   assert_heap_not_locked();
 887   if (result != NULL) {
 888     dirty_young_block(result, word_size);
 889   }
 890   return result;
 891 }
 892 
 893 void G1CollectedHeap::dealloc_archive_regions(MemRegion* ranges, size_t count) {
 894   assert(!is_init_completed(), "Expect to be called at JVM init time");
 895   assert(ranges != NULL, "MemRegion array NULL");
 896   assert(count != 0, "No MemRegions provided");
 897   MemRegion reserved = _hrm.reserved();
 898   HeapWord* prev_last_addr = NULL;
 899   HeapRegion* prev_last_region = NULL;
 900   size_t size_used = 0;
 901   size_t uncommitted_regions = 0;
 902 
 903   // For each Memregion, free the G1 regions that constitute it, and
 904   // notify mark-sweep that the range is no longer to be considered 'archive.'
 905   MutexLockerEx x(Heap_lock);
 906   for (size_t i = 0; i < count; i++) {
 907     HeapWord* start_address = ranges[i].start();
 908     HeapWord* last_address = ranges[i].last();
 909 
 910     assert(reserved.contains(start_address) && reserved.contains(last_address),
 911            "MemRegion outside of heap [" PTR_FORMAT ", " PTR_FORMAT "]",
 912            p2i(start_address), p2i(last_address));
 913     assert(start_address > prev_last_addr,
 914            "Ranges not in ascending order: " PTR_FORMAT " <= " PTR_FORMAT ,
 915            p2i(start_address), p2i(prev_last_addr));
 916     size_used += ranges[i].byte_size();
 917     prev_last_addr = last_address;
 918 
 919     HeapRegion* start_region = _hrm.addr_to_region(start_address);
 920     HeapRegion* last_region = _hrm.addr_to_region(last_address);
 921 
 922     // Check for ranges that start in the same G1 region in which the previous
 923     // range ended, and adjust the start address so we don't try to free
 924     // the same region again. If the current range is entirely within that
 925     // region, skip it.
 926     if (start_region == prev_last_region) {
 927       start_address = start_region->end();
 928       if (start_address > last_address) {
 929         continue;
 930       }
 931       start_region = _hrm.addr_to_region(start_address);
 932     }
 933     prev_last_region = last_region;
 934 
 935     // After verifying that each region was marked as an archive region by
 936     // alloc_archive_regions, set it free and empty and uncommit it.
 937     HeapRegion* curr_region = start_region;
 938     while (curr_region != NULL) {
 939       guarantee(curr_region->is_archive(),
 940                 "Expected archive region at index %u", curr_region->hrm_index());
 941       uint curr_index = curr_region->hrm_index();
 942       _old_set.remove(curr_region);
 943       curr_region->set_free();
 944       curr_region->set_top(curr_region->bottom());
 945       if (curr_region != last_region) {
 946         curr_region = _hrm.next_region_in_heap(curr_region);
 947       } else {
 948         curr_region = NULL;
 949       }
 950       _hrm.shrink_at(curr_index, 1);
 951       uncommitted_regions++;
 952     }
 953 
 954     // Notify mark-sweep that this is no longer an archive range.
 955     G1MarkSweep::set_range_archive(ranges[i], false);
 956   }
 957 
 958   if (uncommitted_regions != 0) {
 959     ergo_verbose1(ErgoHeapSizing,
 960                   "attempt heap shrinking",
 961                   ergo_format_reason("uncommitted archive regions")
 962                   ergo_format_byte("total size"),
 963                   HeapRegion::GrainWords * HeapWordSize * uncommitted_regions);
 964   }
 965   decrease_used(size_used);
 966 }
 967 
 968 HeapWord* G1CollectedHeap::attempt_allocation_humongous(size_t word_size,
 969                                                         uint* gc_count_before_ret,
 970                                                         uint* gclocker_retry_count_ret) {
 971   // The structure of this method has a lot of similarities to
 972   // attempt_allocation_slow(). The reason these two were not merged
 973   // into a single one is that such a method would require several "if
 974   // allocation is not humongous do this, otherwise do that"
 975   // conditional paths which would obscure its flow. In fact, an early
 976   // version of this code did use a unified method which was harder to
 977   // follow and, as a result, it had subtle bugs that were hard to
 978   // track down. So keeping these two methods separate allows each to
 979   // be more readable. It will be good to keep these two in sync as
 980   // much as possible.
 981 
 982   assert_heap_not_locked_and_not_at_safepoint();
 983   assert(is_humongous(word_size), "attempt_allocation_humongous() "
 984          "should only be called for humongous allocations");
 985 
 986   // Humongous objects can exhaust the heap quickly, so we should check if we
 987   // need to start a marking cycle at each humongous object allocation. We do
 988   // the check before we do the actual allocation. The reason for doing it
 989   // before the allocation is that we avoid having to keep track of the newly
 990   // allocated memory while we do a GC.
 991   if (g1_policy()->need_to_start_conc_mark("concurrent humongous allocation",
 992                                            word_size)) {
 993     collect(GCCause::_g1_humongous_allocation);
 994   }
 995 
 996   // We will loop until a) we manage to successfully perform the
 997   // allocation or b) we successfully schedule a collection which
 998   // fails to perform the allocation. b) is the only case when we'll
 999   // return NULL.
1000   HeapWord* result = NULL;
1001   for (int try_count = 1; /* we'll return */; try_count += 1) {
1002     bool should_try_gc;
1003     uint gc_count_before;
1004 
1005     {
1006       MutexLockerEx x(Heap_lock);
1007 
1008       // Given that humongous objects are not allocated in young
1009       // regions, we'll first try to do the allocation without doing a
1010       // collection hoping that there's enough space in the heap.
1011       result = humongous_obj_allocate(word_size, AllocationContext::current());
1012       if (result != NULL) {
1013         return result;
1014       }
1015 
1016       if (GC_locker::is_active_and_needs_gc()) {
1017         should_try_gc = false;
1018       } else {
1019          // The GCLocker may not be active but the GCLocker initiated
1020         // GC may not yet have been performed (GCLocker::needs_gc()
1021         // returns true). In this case we do not try this GC and
1022         // wait until the GCLocker initiated GC is performed, and
1023         // then retry the allocation.
1024         if (GC_locker::needs_gc()) {
1025           should_try_gc = false;
1026         } else {
1027           // Read the GC count while still holding the Heap_lock.
1028           gc_count_before = total_collections();
1029           should_try_gc = true;
1030         }
1031       }
1032     }
1033 
1034     if (should_try_gc) {
1035       // If we failed to allocate the humongous object, we should try to
1036       // do a collection pause (if we're allowed) in case it reclaims
1037       // enough space for the allocation to succeed after the pause.
1038 
1039       bool succeeded;
1040       result = do_collection_pause(word_size, gc_count_before, &succeeded,
1041                                    GCCause::_g1_humongous_allocation);
1042       if (result != NULL) {
1043         assert(succeeded, "only way to get back a non-NULL result");
1044         return result;
1045       }
1046 
1047       if (succeeded) {
1048         // If we get here we successfully scheduled a collection which
1049         // failed to allocate. No point in trying to allocate
1050         // further. We'll just return NULL.
1051         MutexLockerEx x(Heap_lock);
1052         *gc_count_before_ret = total_collections();
1053         return NULL;
1054       }
1055     } else {
1056       if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) {
1057         MutexLockerEx x(Heap_lock);
1058         *gc_count_before_ret = total_collections();
1059         return NULL;
1060       }
1061       // The GCLocker is either active or the GCLocker initiated
1062       // GC has not yet been performed. Stall until it is and
1063       // then retry the allocation.
1064       GC_locker::stall_until_clear();
1065       (*gclocker_retry_count_ret) += 1;
1066     }
1067 
1068     // We can reach here if we were unsuccessful in scheduling a
1069     // collection (because another thread beat us to it) or if we were
1070     // stalled due to the GC locker. In either can we should retry the
1071     // allocation attempt in case another thread successfully
1072     // performed a collection and reclaimed enough space.  Give a
1073     // warning if we seem to be looping forever.
1074 
1075     if ((QueuedAllocationWarningCount > 0) &&
1076         (try_count % QueuedAllocationWarningCount == 0)) {
1077       warning("G1CollectedHeap::attempt_allocation_humongous() "
1078               "retries %d times", try_count);
1079     }
1080   }
1081 
1082   ShouldNotReachHere();
1083   return NULL;
1084 }
1085 
1086 HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size,
1087                                                            AllocationContext_t context,
1088                                                            bool expect_null_mutator_alloc_region) {
1089   assert_at_safepoint(true /* should_be_vm_thread */);
1090   assert(!_allocator->has_mutator_alloc_region(context) || !expect_null_mutator_alloc_region,
1091          "the current alloc region was unexpectedly found to be non-NULL");
1092 
1093   if (!is_humongous(word_size)) {
1094     return _allocator->attempt_allocation_locked(word_size, context);
1095   } else {
1096     HeapWord* result = humongous_obj_allocate(word_size, context);
1097     if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) {
1098       collector_state()->set_initiate_conc_mark_if_possible(true);
1099     }
1100     return result;
1101   }
1102 
1103   ShouldNotReachHere();
1104 }
1105 
1106 class PostMCRemSetClearClosure: public HeapRegionClosure {
1107   G1CollectedHeap* _g1h;
1108   ModRefBarrierSet* _mr_bs;
1109 public:
1110   PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) :
1111     _g1h(g1h), _mr_bs(mr_bs) {}
1112 
1113   bool doHeapRegion(HeapRegion* r) {
1114     HeapRegionRemSet* hrrs = r->rem_set();
1115 
1116     _g1h->reset_gc_time_stamps(r);
1117 
1118     if (r->is_continues_humongous()) {
1119       // We'll assert that the strong code root list and RSet is empty
1120       assert(hrrs->strong_code_roots_list_length() == 0, "sanity");
1121       assert(hrrs->occupied() == 0, "RSet should be empty");
1122     } else {
1123       hrrs->clear();
1124     }
1125     // You might think here that we could clear just the cards
1126     // corresponding to the used region.  But no: if we leave a dirty card
1127     // in a region we might allocate into, then it would prevent that card
1128     // from being enqueued, and cause it to be missed.
1129     // Re: the performance cost: we shouldn't be doing full GC anyway!
1130     _mr_bs->clear(MemRegion(r->bottom(), r->end()));
1131 
1132     return false;
1133   }
1134 };
1135 
1136 void G1CollectedHeap::clear_rsets_post_compaction() {
1137   PostMCRemSetClearClosure rs_clear(this, g1_barrier_set());
1138   heap_region_iterate(&rs_clear);
1139 }
1140 
1141 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
1142   G1CollectedHeap*   _g1h;
1143   UpdateRSOopClosure _cl;
1144 public:
1145   RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, uint worker_i = 0) :
1146     _cl(g1->g1_rem_set(), worker_i),
1147     _g1h(g1)
1148   { }
1149 
1150   bool doHeapRegion(HeapRegion* r) {
1151     if (!r->is_continues_humongous()) {
1152       _cl.set_from(r);
1153       r->oop_iterate(&_cl);
1154     }
1155     return false;
1156   }
1157 };
1158 
1159 class ParRebuildRSTask: public AbstractGangTask {
1160   G1CollectedHeap* _g1;
1161   HeapRegionClaimer _hrclaimer;
1162 
1163 public:
1164   ParRebuildRSTask(G1CollectedHeap* g1) :
1165       AbstractGangTask("ParRebuildRSTask"), _g1(g1), _hrclaimer(g1->workers()->active_workers()) {}
1166 
1167   void work(uint worker_id) {
1168     RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id);
1169     _g1->heap_region_par_iterate(&rebuild_rs, worker_id, &_hrclaimer);
1170   }
1171 };
1172 
1173 class PostCompactionPrinterClosure: public HeapRegionClosure {
1174 private:
1175   G1HRPrinter* _hr_printer;
1176 public:
1177   bool doHeapRegion(HeapRegion* hr) {
1178     assert(!hr->is_young(), "not expecting to find young regions");
1179     if (hr->is_free()) {
1180       // We only generate output for non-empty regions.
1181     } else if (hr->is_starts_humongous()) {
1182       _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous);
1183     } else if (hr->is_continues_humongous()) {
1184       _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous);
1185     } else if (hr->is_archive()) {
1186       _hr_printer->post_compaction(hr, G1HRPrinter::Archive);
1187     } else if (hr->is_old()) {
1188       _hr_printer->post_compaction(hr, G1HRPrinter::Old);
1189     } else {
1190       ShouldNotReachHere();
1191     }
1192     return false;
1193   }
1194 
1195   PostCompactionPrinterClosure(G1HRPrinter* hr_printer)
1196     : _hr_printer(hr_printer) { }
1197 };
1198 
1199 void G1CollectedHeap::print_hrm_post_compaction() {
1200   PostCompactionPrinterClosure cl(hr_printer());
1201   heap_region_iterate(&cl);
1202 }
1203 
1204 bool G1CollectedHeap::do_collection(bool explicit_gc,
1205                                     bool clear_all_soft_refs,
1206                                     size_t word_size) {
1207   assert_at_safepoint(true /* should_be_vm_thread */);
1208 
1209   if (GC_locker::check_active_before_gc()) {
1210     return false;
1211   }
1212 
1213   STWGCTimer* gc_timer = G1MarkSweep::gc_timer();
1214   gc_timer->register_gc_start();
1215 
1216   SerialOldTracer* gc_tracer = G1MarkSweep::gc_tracer();
1217   GCIdMark gc_id_mark;
1218   gc_tracer->report_gc_start(gc_cause(), gc_timer->gc_start());
1219 
1220   SvcGCMarker sgcm(SvcGCMarker::FULL);
1221   ResourceMark rm;
1222 
1223   G1Log::update_level();
1224   print_heap_before_gc();
1225   trace_heap_before_gc(gc_tracer);
1226 
1227   size_t metadata_prev_used = MetaspaceAux::used_bytes();
1228 
1229   verify_region_sets_optional();
1230 
1231   const bool do_clear_all_soft_refs = clear_all_soft_refs ||
1232                            collector_policy()->should_clear_all_soft_refs();
1233 
1234   ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy());
1235 
1236   {
1237     IsGCActiveMark x;
1238 
1239     // Timing
1240     assert(!GCCause::is_user_requested_gc(gc_cause()) || explicit_gc, "invariant");
1241     TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty);
1242 
1243     {
1244       GCTraceTime t(GCCauseString("Full GC", gc_cause()), G1Log::fine(), true, NULL);
1245       TraceCollectorStats tcs(g1mm()->full_collection_counters());
1246       TraceMemoryManagerStats tms(true /* fullGC */, gc_cause());
1247 
1248       g1_policy()->record_full_collection_start();
1249 
1250       // Note: When we have a more flexible GC logging framework that
1251       // allows us to add optional attributes to a GC log record we
1252       // could consider timing and reporting how long we wait in the
1253       // following two methods.
1254       wait_while_free_regions_coming();
1255       // If we start the compaction before the CM threads finish
1256       // scanning the root regions we might trip them over as we'll
1257       // be moving objects / updating references. So let's wait until
1258       // they are done. By telling them to abort, they should complete
1259       // early.
1260       _cm->root_regions()->abort();
1261       _cm->root_regions()->wait_until_scan_finished();
1262       append_secondary_free_list_if_not_empty_with_lock();
1263 
1264       gc_prologue(true);
1265       increment_total_collections(true /* full gc */);
1266       increment_old_marking_cycles_started();
1267 
1268       assert(used() == recalculate_used(), "Should be equal");
1269 
1270       verify_before_gc();
1271 
1272       check_bitmaps("Full GC Start");
1273       pre_full_gc_dump(gc_timer);
1274 
1275 #if defined(COMPILER2) || INCLUDE_JVMCI
1276       DerivedPointerTable::clear();
1277 #endif
1278 
1279       // Disable discovery and empty the discovered lists
1280       // for the CM ref processor.
1281       ref_processor_cm()->disable_discovery();
1282       ref_processor_cm()->abandon_partial_discovery();
1283       ref_processor_cm()->verify_no_references_recorded();
1284 
1285       // Abandon current iterations of concurrent marking and concurrent
1286       // refinement, if any are in progress. We have to do this before
1287       // wait_until_scan_finished() below.
1288       concurrent_mark()->abort();
1289 
1290       // Make sure we'll choose a new allocation region afterwards.
1291       _allocator->release_mutator_alloc_region();
1292       _allocator->abandon_gc_alloc_regions();
1293       g1_rem_set()->cleanupHRRS();
1294 
1295       // We should call this after we retire any currently active alloc
1296       // regions so that all the ALLOC / RETIRE events are generated
1297       // before the start GC event.
1298       _hr_printer.start_gc(true /* full */, (size_t) total_collections());
1299 
1300       // We may have added regions to the current incremental collection
1301       // set between the last GC or pause and now. We need to clear the
1302       // incremental collection set and then start rebuilding it afresh
1303       // after this full GC.
1304       abandon_collection_set(g1_policy()->inc_cset_head());
1305       g1_policy()->clear_incremental_cset();
1306       g1_policy()->stop_incremental_cset_building();
1307 
1308       tear_down_region_sets(false /* free_list_only */);
1309       collector_state()->set_gcs_are_young(true);
1310 
1311       // See the comments in g1CollectedHeap.hpp and
1312       // G1CollectedHeap::ref_processing_init() about
1313       // how reference processing currently works in G1.
1314 
1315       // Temporarily make discovery by the STW ref processor single threaded (non-MT).
1316       ReferenceProcessorMTDiscoveryMutator stw_rp_disc_ser(ref_processor_stw(), false);
1317 
1318       // Temporarily clear the STW ref processor's _is_alive_non_header field.
1319       ReferenceProcessorIsAliveMutator stw_rp_is_alive_null(ref_processor_stw(), NULL);
1320 
1321       ref_processor_stw()->enable_discovery();
1322       ref_processor_stw()->setup_policy(do_clear_all_soft_refs);
1323 
1324       // Do collection work
1325       {
1326         HandleMark hm;  // Discard invalid handles created during gc
1327         G1MarkSweep::invoke_at_safepoint(ref_processor_stw(), do_clear_all_soft_refs);
1328       }
1329 
1330       assert(num_free_regions() == 0, "we should not have added any free regions");
1331       rebuild_region_sets(false /* free_list_only */);
1332 
1333       // Enqueue any discovered reference objects that have
1334       // not been removed from the discovered lists.
1335       ref_processor_stw()->enqueue_discovered_references();
1336 
1337 #if defined(COMPILER2) || INCLUDE_JVMCI
1338       DerivedPointerTable::update_pointers();
1339 #endif
1340 
1341       MemoryService::track_memory_usage();
1342 
1343       assert(!ref_processor_stw()->discovery_enabled(), "Postcondition");
1344       ref_processor_stw()->verify_no_references_recorded();
1345 
1346       // Delete metaspaces for unloaded class loaders and clean up loader_data graph
1347       ClassLoaderDataGraph::purge();
1348       MetaspaceAux::verify_metrics();
1349 
1350       // Note: since we've just done a full GC, concurrent
1351       // marking is no longer active. Therefore we need not
1352       // re-enable reference discovery for the CM ref processor.
1353       // That will be done at the start of the next marking cycle.
1354       assert(!ref_processor_cm()->discovery_enabled(), "Postcondition");
1355       ref_processor_cm()->verify_no_references_recorded();
1356 
1357       reset_gc_time_stamp();
1358       // Since everything potentially moved, we will clear all remembered
1359       // sets, and clear all cards.  Later we will rebuild remembered
1360       // sets. We will also reset the GC time stamps of the regions.
1361       clear_rsets_post_compaction();
1362       check_gc_time_stamps();
1363 
1364       // Resize the heap if necessary.
1365       resize_if_necessary_after_full_collection(explicit_gc ? 0 : word_size);
1366 
1367       if (_hr_printer.is_active()) {
1368         // We should do this after we potentially resize the heap so
1369         // that all the COMMIT / UNCOMMIT events are generated before
1370         // the end GC event.
1371 
1372         print_hrm_post_compaction();
1373         _hr_printer.end_gc(true /* full */, (size_t) total_collections());
1374       }
1375 
1376       G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
1377       if (hot_card_cache->use_cache()) {
1378         hot_card_cache->reset_card_counts();
1379         hot_card_cache->reset_hot_cache();
1380       }
1381 
1382       // Rebuild remembered sets of all regions.
1383       uint n_workers =
1384         AdaptiveSizePolicy::calc_active_workers(workers()->total_workers(),
1385                                                 workers()->active_workers(),
1386                                                 Threads::number_of_non_daemon_threads());
1387       workers()->set_active_workers(n_workers);
1388 
1389       ParRebuildRSTask rebuild_rs_task(this);
1390       workers()->run_task(&rebuild_rs_task);
1391 
1392       // Rebuild the strong code root lists for each region
1393       rebuild_strong_code_roots();
1394 
1395       if (true) { // FIXME
1396         MetaspaceGC::compute_new_size();
1397       }
1398 
1399 #ifdef TRACESPINNING
1400       ParallelTaskTerminator::print_termination_counts();
1401 #endif
1402 
1403       // Discard all rset updates
1404       JavaThread::dirty_card_queue_set().abandon_logs();
1405       assert(dirty_card_queue_set().completed_buffers_num() == 0, "DCQS should be empty");
1406 
1407       _young_list->reset_sampled_info();
1408       // At this point there should be no regions in the
1409       // entire heap tagged as young.
1410       assert(check_young_list_empty(true /* check_heap */),
1411              "young list should be empty at this point");
1412 
1413       // Update the number of full collections that have been completed.
1414       increment_old_marking_cycles_completed(false /* concurrent */);
1415 
1416       _hrm.verify_optional();
1417       verify_region_sets_optional();
1418 
1419       verify_after_gc();
1420 
1421       // Clear the previous marking bitmap, if needed for bitmap verification.
1422       // Note we cannot do this when we clear the next marking bitmap in
1423       // ConcurrentMark::abort() above since VerifyDuringGC verifies the
1424       // objects marked during a full GC against the previous bitmap.
1425       // But we need to clear it before calling check_bitmaps below since
1426       // the full GC has compacted objects and updated TAMS but not updated
1427       // the prev bitmap.
1428       if (G1VerifyBitmaps) {
1429         ((CMBitMap*) concurrent_mark()->prevMarkBitMap())->clearAll();
1430       }
1431       check_bitmaps("Full GC End");
1432 
1433       // Start a new incremental collection set for the next pause
1434       assert(g1_policy()->collection_set() == NULL, "must be");
1435       g1_policy()->start_incremental_cset_building();
1436 
1437       clear_cset_fast_test();
1438 
1439       _allocator->init_mutator_alloc_region();
1440 
1441       g1_policy()->record_full_collection_end();
1442 
1443       if (G1Log::fine()) {
1444         g1_policy()->print_heap_transition();
1445       }
1446 
1447       // We must call G1MonitoringSupport::update_sizes() in the same scoping level
1448       // as an active TraceMemoryManagerStats object (i.e. before the destructor for the
1449       // TraceMemoryManagerStats is called) so that the G1 memory pools are updated
1450       // before any GC notifications are raised.
1451       g1mm()->update_sizes();
1452 
1453       gc_epilogue(true);
1454     }
1455 
1456     if (G1Log::finer()) {
1457       g1_policy()->print_detailed_heap_transition(true /* full */);
1458     }
1459 
1460     print_heap_after_gc();
1461     trace_heap_after_gc(gc_tracer);
1462 
1463     post_full_gc_dump(gc_timer);
1464 
1465     gc_timer->register_gc_end();
1466     gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions());
1467   }
1468 
1469   return true;
1470 }
1471 
1472 void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {
1473   // do_collection() will return whether it succeeded in performing
1474   // the GC. Currently, there is no facility on the
1475   // do_full_collection() API to notify the caller than the collection
1476   // did not succeed (e.g., because it was locked out by the GC
1477   // locker). So, right now, we'll ignore the return value.
1478   bool dummy = do_collection(true,                /* explicit_gc */
1479                              clear_all_soft_refs,
1480                              0                    /* word_size */);
1481 }
1482 
1483 // This code is mostly copied from TenuredGeneration.
1484 void
1485 G1CollectedHeap::
1486 resize_if_necessary_after_full_collection(size_t word_size) {
1487   // Include the current allocation, if any, and bytes that will be
1488   // pre-allocated to support collections, as "used".
1489   const size_t used_after_gc = used();
1490   const size_t capacity_after_gc = capacity();
1491   const size_t free_after_gc = capacity_after_gc - used_after_gc;
1492 
1493   // This is enforced in arguments.cpp.
1494   assert(MinHeapFreeRatio <= MaxHeapFreeRatio,
1495          "otherwise the code below doesn't make sense");
1496 
1497   // We don't have floating point command-line arguments
1498   const double minimum_free_percentage = (double) MinHeapFreeRatio / 100.0;
1499   const double maximum_used_percentage = 1.0 - minimum_free_percentage;
1500   const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100.0;
1501   const double minimum_used_percentage = 1.0 - maximum_free_percentage;
1502 
1503   const size_t min_heap_size = collector_policy()->min_heap_byte_size();
1504   const size_t max_heap_size = collector_policy()->max_heap_byte_size();
1505 
1506   // We have to be careful here as these two calculations can overflow
1507   // 32-bit size_t's.
1508   double used_after_gc_d = (double) used_after_gc;
1509   double minimum_desired_capacity_d = used_after_gc_d / maximum_used_percentage;
1510   double maximum_desired_capacity_d = used_after_gc_d / minimum_used_percentage;
1511 
1512   // Let's make sure that they are both under the max heap size, which
1513   // by default will make them fit into a size_t.
1514   double desired_capacity_upper_bound = (double) max_heap_size;
1515   minimum_desired_capacity_d = MIN2(minimum_desired_capacity_d,
1516                                     desired_capacity_upper_bound);
1517   maximum_desired_capacity_d = MIN2(maximum_desired_capacity_d,
1518                                     desired_capacity_upper_bound);
1519 
1520   // We can now safely turn them into size_t's.
1521   size_t minimum_desired_capacity = (size_t) minimum_desired_capacity_d;
1522   size_t maximum_desired_capacity = (size_t) maximum_desired_capacity_d;
1523 
1524   // This assert only makes sense here, before we adjust them
1525   // with respect to the min and max heap size.
1526   assert(minimum_desired_capacity <= maximum_desired_capacity,
1527          "minimum_desired_capacity = " SIZE_FORMAT ", "
1528          "maximum_desired_capacity = " SIZE_FORMAT,
1529          minimum_desired_capacity, maximum_desired_capacity);
1530 
1531   // Should not be greater than the heap max size. No need to adjust
1532   // it with respect to the heap min size as it's a lower bound (i.e.,
1533   // we'll try to make the capacity larger than it, not smaller).
1534   minimum_desired_capacity = MIN2(minimum_desired_capacity, max_heap_size);
1535   // Should not be less than the heap min size. No need to adjust it
1536   // with respect to the heap max size as it's an upper bound (i.e.,
1537   // we'll try to make the capacity smaller than it, not greater).
1538   maximum_desired_capacity =  MAX2(maximum_desired_capacity, min_heap_size);
1539 
1540   if (capacity_after_gc < minimum_desired_capacity) {
1541     // Don't expand unless it's significant
1542     size_t expand_bytes = minimum_desired_capacity - capacity_after_gc;
1543     ergo_verbose4(ErgoHeapSizing,
1544                   "attempt heap expansion",
1545                   ergo_format_reason("capacity lower than "
1546                                      "min desired capacity after Full GC")
1547                   ergo_format_byte("capacity")
1548                   ergo_format_byte("occupancy")
1549                   ergo_format_byte_perc("min desired capacity"),
1550                   capacity_after_gc, used_after_gc,
1551                   minimum_desired_capacity, (double) MinHeapFreeRatio);
1552     expand(expand_bytes);
1553 
1554     // No expansion, now see if we want to shrink
1555   } else if (capacity_after_gc > maximum_desired_capacity) {
1556     // Capacity too large, compute shrinking size
1557     size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity;
1558     ergo_verbose4(ErgoHeapSizing,
1559                   "attempt heap shrinking",
1560                   ergo_format_reason("capacity higher than "
1561                                      "max desired capacity after Full GC")
1562                   ergo_format_byte("capacity")
1563                   ergo_format_byte("occupancy")
1564                   ergo_format_byte_perc("max desired capacity"),
1565                   capacity_after_gc, used_after_gc,
1566                   maximum_desired_capacity, (double) MaxHeapFreeRatio);
1567     shrink(shrink_bytes);
1568   }
1569 }
1570 
1571 HeapWord* G1CollectedHeap::satisfy_failed_allocation_helper(size_t word_size,
1572                                                             AllocationContext_t context,
1573                                                             bool do_gc,
1574                                                             bool clear_all_soft_refs,
1575                                                             bool expect_null_mutator_alloc_region,
1576                                                             bool* gc_succeeded) {
1577   *gc_succeeded = true;
1578   // Let's attempt the allocation first.
1579   HeapWord* result =
1580     attempt_allocation_at_safepoint(word_size,
1581                                     context,
1582                                     expect_null_mutator_alloc_region);
1583   if (result != NULL) {
1584     assert(*gc_succeeded, "sanity");
1585     return result;
1586   }
1587 
1588   // In a G1 heap, we're supposed to keep allocation from failing by
1589   // incremental pauses.  Therefore, at least for now, we'll favor
1590   // expansion over collection.  (This might change in the future if we can
1591   // do something smarter than full collection to satisfy a failed alloc.)
1592   result = expand_and_allocate(word_size, context);
1593   if (result != NULL) {
1594     assert(*gc_succeeded, "sanity");
1595     return result;
1596   }
1597 
1598   if (do_gc) {
1599     // Expansion didn't work, we'll try to do a Full GC.
1600     *gc_succeeded = do_collection(false, /* explicit_gc */
1601                                   clear_all_soft_refs,
1602                                   word_size);
1603   }
1604 
1605   return NULL;
1606 }
1607 
1608 HeapWord* G1CollectedHeap::satisfy_failed_allocation(size_t word_size,
1609                                                      AllocationContext_t context,
1610                                                      bool* succeeded) {
1611   assert_at_safepoint(true /* should_be_vm_thread */);
1612 
1613   // Attempts to allocate followed by Full GC.
1614   HeapWord* result =
1615     satisfy_failed_allocation_helper(word_size,
1616                                      context,
1617                                      true,  /* do_gc */
1618                                      false, /* clear_all_soft_refs */
1619                                      false, /* expect_null_mutator_alloc_region */
1620                                      succeeded);
1621 
1622   if (result != NULL || !*succeeded) {
1623     return result;
1624   }
1625 
1626   // Attempts to allocate followed by Full GC that will collect all soft references.
1627   result = satisfy_failed_allocation_helper(word_size,
1628                                             context,
1629                                             true, /* do_gc */
1630                                             true, /* clear_all_soft_refs */
1631                                             true, /* expect_null_mutator_alloc_region */
1632                                             succeeded);
1633 
1634   if (result != NULL || !*succeeded) {
1635     return result;
1636   }
1637 
1638   // Attempts to allocate, no GC
1639   result = satisfy_failed_allocation_helper(word_size,
1640                                             context,
1641                                             false, /* do_gc */
1642                                             false, /* clear_all_soft_refs */
1643                                             true,  /* expect_null_mutator_alloc_region */
1644                                             succeeded);
1645 
1646   if (result != NULL) {
1647     assert(*succeeded, "sanity");
1648     return result;
1649   }
1650 
1651   assert(!collector_policy()->should_clear_all_soft_refs(),
1652          "Flag should have been handled and cleared prior to this point");
1653 
1654   // What else?  We might try synchronous finalization later.  If the total
1655   // space available is large enough for the allocation, then a more
1656   // complete compaction phase than we've tried so far might be
1657   // appropriate.
1658   assert(*succeeded, "sanity");
1659   return NULL;
1660 }
1661 
1662 // Attempting to expand the heap sufficiently
1663 // to support an allocation of the given "word_size".  If
1664 // successful, perform the allocation and return the address of the
1665 // allocated block, or else "NULL".
1666 
1667 HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size, AllocationContext_t context) {
1668   assert_at_safepoint(true /* should_be_vm_thread */);
1669 
1670   verify_region_sets_optional();
1671 
1672   size_t expand_bytes = MAX2(word_size * HeapWordSize, MinHeapDeltaBytes);
1673   ergo_verbose1(ErgoHeapSizing,
1674                 "attempt heap expansion",
1675                 ergo_format_reason("allocation request failed")
1676                 ergo_format_byte("allocation request"),
1677                 word_size * HeapWordSize);
1678   if (expand(expand_bytes)) {
1679     _hrm.verify_optional();
1680     verify_region_sets_optional();
1681     return attempt_allocation_at_safepoint(word_size,
1682                                            context,
1683                                            false /* expect_null_mutator_alloc_region */);
1684   }
1685   return NULL;
1686 }
1687 
1688 bool G1CollectedHeap::expand(size_t expand_bytes, double* expand_time_ms) {
1689   size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes);
1690   aligned_expand_bytes = align_size_up(aligned_expand_bytes,
1691                                        HeapRegion::GrainBytes);
1692   ergo_verbose2(ErgoHeapSizing,
1693                 "expand the heap",
1694                 ergo_format_byte("requested expansion amount")
1695                 ergo_format_byte("attempted expansion amount"),
1696                 expand_bytes, aligned_expand_bytes);
1697 
1698   if (is_maximal_no_gc()) {
1699     ergo_verbose0(ErgoHeapSizing,
1700                       "did not expand the heap",
1701                       ergo_format_reason("heap already fully expanded"));
1702     return false;
1703   }
1704 
1705   double expand_heap_start_time_sec = os::elapsedTime();
1706   uint regions_to_expand = (uint)(aligned_expand_bytes / HeapRegion::GrainBytes);
1707   assert(regions_to_expand > 0, "Must expand by at least one region");
1708 
1709   uint expanded_by = _hrm.expand_by(regions_to_expand);
1710   if (expand_time_ms != NULL) {
1711     *expand_time_ms = (os::elapsedTime() - expand_heap_start_time_sec) * MILLIUNITS;
1712   }
1713 
1714   if (expanded_by > 0) {
1715     size_t actual_expand_bytes = expanded_by * HeapRegion::GrainBytes;
1716     assert(actual_expand_bytes <= aligned_expand_bytes, "post-condition");
1717     g1_policy()->record_new_heap_size(num_regions());
1718   } else {
1719     ergo_verbose0(ErgoHeapSizing,
1720                   "did not expand the heap",
1721                   ergo_format_reason("heap expansion operation failed"));
1722     // The expansion of the virtual storage space was unsuccessful.
1723     // Let's see if it was because we ran out of swap.
1724     if (G1ExitOnExpansionFailure &&
1725         _hrm.available() >= regions_to_expand) {
1726       // We had head room...
1727       vm_exit_out_of_memory(aligned_expand_bytes, OOM_MMAP_ERROR, "G1 heap expansion");
1728     }
1729   }
1730   return regions_to_expand > 0;
1731 }
1732 
1733 void G1CollectedHeap::shrink_helper(size_t shrink_bytes) {
1734   size_t aligned_shrink_bytes =
1735     ReservedSpace::page_align_size_down(shrink_bytes);
1736   aligned_shrink_bytes = align_size_down(aligned_shrink_bytes,
1737                                          HeapRegion::GrainBytes);
1738   uint num_regions_to_remove = (uint)(shrink_bytes / HeapRegion::GrainBytes);
1739 
1740   uint num_regions_removed = _hrm.shrink_by(num_regions_to_remove);
1741   size_t shrunk_bytes = num_regions_removed * HeapRegion::GrainBytes;
1742 
1743   ergo_verbose3(ErgoHeapSizing,
1744                 "shrink the heap",
1745                 ergo_format_byte("requested shrinking amount")
1746                 ergo_format_byte("aligned shrinking amount")
1747                 ergo_format_byte("attempted shrinking amount"),
1748                 shrink_bytes, aligned_shrink_bytes, shrunk_bytes);
1749   if (num_regions_removed > 0) {
1750     g1_policy()->record_new_heap_size(num_regions());
1751   } else {
1752     ergo_verbose0(ErgoHeapSizing,
1753                   "did not shrink the heap",
1754                   ergo_format_reason("heap shrinking operation failed"));
1755   }
1756 }
1757 
1758 void G1CollectedHeap::shrink(size_t shrink_bytes) {
1759   verify_region_sets_optional();
1760 
1761   // We should only reach here at the end of a Full GC which means we
1762   // should not not be holding to any GC alloc regions. The method
1763   // below will make sure of that and do any remaining clean up.
1764   _allocator->abandon_gc_alloc_regions();
1765 
1766   // Instead of tearing down / rebuilding the free lists here, we
1767   // could instead use the remove_all_pending() method on free_list to
1768   // remove only the ones that we need to remove.
1769   tear_down_region_sets(true /* free_list_only */);
1770   shrink_helper(shrink_bytes);
1771   rebuild_region_sets(true /* free_list_only */);
1772 
1773   _hrm.verify_optional();
1774   verify_region_sets_optional();
1775 }
1776 
1777 // Public methods.
1778 
1779 G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) :
1780   CollectedHeap(),
1781   _g1_policy(policy_),
1782   _dirty_card_queue_set(false),
1783   _is_alive_closure_cm(this),
1784   _is_alive_closure_stw(this),
1785   _ref_processor_cm(NULL),
1786   _ref_processor_stw(NULL),
1787   _bot_shared(NULL),
1788   _cg1r(NULL),
1789   _g1mm(NULL),
1790   _refine_cte_cl(NULL),
1791   _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()),
1792   _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()),
1793   _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()),
1794   _humongous_reclaim_candidates(),
1795   _has_humongous_reclaim_candidates(false),
1796   _archive_allocator(NULL),
1797   _free_regions_coming(false),
1798   _young_list(new YoungList(this)),
1799   _gc_time_stamp(0),
1800   _summary_bytes_used(0),
1801   _survivor_evac_stats(YoungPLABSize, PLABWeight),
1802   _old_evac_stats(OldPLABSize, PLABWeight),
1803   _expand_heap_after_alloc_failure(true),
1804   _old_marking_cycles_started(0),
1805   _old_marking_cycles_completed(0),
1806   _heap_summary_sent(false),
1807   _in_cset_fast_test(),
1808   _dirty_cards_region_list(NULL),
1809   _worker_cset_start_region(NULL),
1810   _worker_cset_start_region_time_stamp(NULL),
1811   _gc_timer_stw(new (ResourceObj::C_HEAP, mtGC) STWGCTimer()),
1812   _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
1813   _gc_tracer_stw(new (ResourceObj::C_HEAP, mtGC) G1NewTracer()),
1814   _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) G1OldTracer()) {
1815 
1816   _workers = new WorkGang("GC Thread", ParallelGCThreads,
1817                           /* are_GC_task_threads */true,
1818                           /* are_ConcurrentGC_threads */false);
1819   _workers->initialize_workers();
1820 
1821   _allocator = G1Allocator::create_allocator(this);
1822   _humongous_object_threshold_in_words = humongous_threshold_for(HeapRegion::GrainWords);
1823 
1824   // Override the default _filler_array_max_size so that no humongous filler
1825   // objects are created.
1826   _filler_array_max_size = _humongous_object_threshold_in_words;
1827 
1828   uint n_queues = ParallelGCThreads;
1829   _task_queues = new RefToScanQueueSet(n_queues);
1830 
1831   uint n_rem_sets = HeapRegionRemSet::num_par_rem_sets();
1832   assert(n_rem_sets > 0, "Invariant.");
1833 
1834   _worker_cset_start_region = NEW_C_HEAP_ARRAY(HeapRegion*, n_queues, mtGC);
1835   _worker_cset_start_region_time_stamp = NEW_C_HEAP_ARRAY(uint, n_queues, mtGC);
1836   _evacuation_failed_info_array = NEW_C_HEAP_ARRAY(EvacuationFailedInfo, n_queues, mtGC);
1837 
1838   for (uint i = 0; i < n_queues; i++) {
1839     RefToScanQueue* q = new RefToScanQueue();
1840     q->initialize();
1841     _task_queues->register_queue(i, q);
1842     ::new (&_evacuation_failed_info_array[i]) EvacuationFailedInfo();
1843   }
1844   clear_cset_start_regions();
1845 
1846   // Initialize the G1EvacuationFailureALot counters and flags.
1847   NOT_PRODUCT(reset_evacuation_should_fail();)
1848 
1849   guarantee(_task_queues != NULL, "task_queues allocation failure.");
1850 }
1851 
1852 G1RegionToSpaceMapper* G1CollectedHeap::create_aux_memory_mapper(const char* description,
1853                                                                  size_t size,
1854                                                                  size_t translation_factor) {
1855   size_t preferred_page_size = os::page_size_for_region_unaligned(size, 1);
1856   // Allocate a new reserved space, preferring to use large pages.
1857   ReservedSpace rs(size, preferred_page_size);
1858   G1RegionToSpaceMapper* result  =
1859     G1RegionToSpaceMapper::create_mapper(rs,
1860                                          size,
1861                                          rs.alignment(),
1862                                          HeapRegion::GrainBytes,
1863                                          translation_factor,
1864                                          mtGC);
1865   if (TracePageSizes) {
1866     gclog_or_tty->print_cr("G1 '%s': pg_sz=" SIZE_FORMAT " base=" PTR_FORMAT " size=" SIZE_FORMAT " alignment=" SIZE_FORMAT " reqsize=" SIZE_FORMAT,
1867                            description, preferred_page_size, p2i(rs.base()), rs.size(), rs.alignment(), size);
1868   }
1869   return result;
1870 }
1871 
1872 jint G1CollectedHeap::initialize() {
1873   CollectedHeap::pre_initialize();
1874   os::enable_vtime();
1875 
1876   G1Log::init();
1877 
1878   // Necessary to satisfy locking discipline assertions.
1879 
1880   MutexLocker x(Heap_lock);
1881 
1882   // We have to initialize the printer before committing the heap, as
1883   // it will be used then.
1884   _hr_printer.set_active(G1PrintHeapRegions);
1885 
1886   // While there are no constraints in the GC code that HeapWordSize
1887   // be any particular value, there are multiple other areas in the
1888   // system which believe this to be true (e.g. oop->object_size in some
1889   // cases incorrectly returns the size in wordSize units rather than
1890   // HeapWordSize).
1891   guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize");
1892 
1893   size_t init_byte_size = collector_policy()->initial_heap_byte_size();
1894   size_t max_byte_size = collector_policy()->max_heap_byte_size();
1895   size_t heap_alignment = collector_policy()->heap_alignment();
1896 
1897   // Ensure that the sizes are properly aligned.
1898   Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap");
1899   Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap");
1900   Universe::check_alignment(max_byte_size, heap_alignment, "g1 heap");
1901 
1902   _refine_cte_cl = new RefineCardTableEntryClosure();
1903 
1904   jint ecode = JNI_OK;
1905   _cg1r = ConcurrentG1Refine::create(this, _refine_cte_cl, &ecode);
1906   if (_cg1r == NULL) {
1907     return ecode;
1908   }
1909 
1910   // Reserve the maximum.
1911 
1912   // When compressed oops are enabled, the preferred heap base
1913   // is calculated by subtracting the requested size from the
1914   // 32Gb boundary and using the result as the base address for
1915   // heap reservation. If the requested size is not aligned to
1916   // HeapRegion::GrainBytes (i.e. the alignment that is passed
1917   // into the ReservedHeapSpace constructor) then the actual
1918   // base of the reserved heap may end up differing from the
1919   // address that was requested (i.e. the preferred heap base).
1920   // If this happens then we could end up using a non-optimal
1921   // compressed oops mode.
1922 
1923   ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size,
1924                                                  heap_alignment);
1925 
1926   initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*)(heap_rs.base() + heap_rs.size()));
1927 
1928   // Create the barrier set for the entire reserved region.
1929   G1SATBCardTableLoggingModRefBS* bs
1930     = new G1SATBCardTableLoggingModRefBS(reserved_region());
1931   bs->initialize();
1932   assert(bs->is_a(BarrierSet::G1SATBCTLogging), "sanity");
1933   set_barrier_set(bs);
1934 
1935   // Also create a G1 rem set.
1936   _g1_rem_set = new G1RemSet(this, g1_barrier_set());
1937 
1938   // Carve out the G1 part of the heap.
1939 
1940   ReservedSpace g1_rs = heap_rs.first_part(max_byte_size);
1941   size_t page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
1942   G1RegionToSpaceMapper* heap_storage =
1943     G1RegionToSpaceMapper::create_mapper(g1_rs,
1944                                          g1_rs.size(),
1945                                          page_size,
1946                                          HeapRegion::GrainBytes,
1947                                          1,
1948                                          mtJavaHeap);
1949   os::trace_page_sizes("G1 Heap", collector_policy()->min_heap_byte_size(),
1950                        max_byte_size, page_size,
1951                        heap_rs.base(),
1952                        heap_rs.size());
1953   heap_storage->set_mapping_changed_listener(&_listener);
1954 
1955   // Create storage for the BOT, card table, card counts table (hot card cache) and the bitmaps.
1956   G1RegionToSpaceMapper* bot_storage =
1957     create_aux_memory_mapper("Block offset table",
1958                              G1BlockOffsetSharedArray::compute_size(g1_rs.size() / HeapWordSize),
1959                              G1BlockOffsetSharedArray::heap_map_factor());
1960 
1961   ReservedSpace cardtable_rs(G1SATBCardTableLoggingModRefBS::compute_size(g1_rs.size() / HeapWordSize));
1962   G1RegionToSpaceMapper* cardtable_storage =
1963     create_aux_memory_mapper("Card table",
1964                              G1SATBCardTableLoggingModRefBS::compute_size(g1_rs.size() / HeapWordSize),
1965                              G1SATBCardTableLoggingModRefBS::heap_map_factor());
1966 
1967   G1RegionToSpaceMapper* card_counts_storage =
1968     create_aux_memory_mapper("Card counts table",
1969                              G1CardCounts::compute_size(g1_rs.size() / HeapWordSize),
1970                              G1CardCounts::heap_map_factor());
1971 
1972   size_t bitmap_size = CMBitMap::compute_size(g1_rs.size());
1973   G1RegionToSpaceMapper* prev_bitmap_storage =
1974     create_aux_memory_mapper("Prev Bitmap", bitmap_size, CMBitMap::heap_map_factor());
1975   G1RegionToSpaceMapper* next_bitmap_storage =
1976     create_aux_memory_mapper("Next Bitmap", bitmap_size, CMBitMap::heap_map_factor());
1977 
1978   _hrm.initialize(heap_storage, prev_bitmap_storage, next_bitmap_storage, bot_storage, cardtable_storage, card_counts_storage);
1979   g1_barrier_set()->initialize(cardtable_storage);
1980    // Do later initialization work for concurrent refinement.
1981   _cg1r->init(card_counts_storage);
1982 
1983   // 6843694 - ensure that the maximum region index can fit
1984   // in the remembered set structures.
1985   const uint max_region_idx = (1U << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1;
1986   guarantee((max_regions() - 1) <= max_region_idx, "too many regions");
1987 
1988   size_t max_cards_per_region = ((size_t)1 << (sizeof(CardIdx_t)*BitsPerByte-1)) - 1;
1989   guarantee(HeapRegion::CardsPerRegion > 0, "make sure it's initialized");
1990   guarantee(HeapRegion::CardsPerRegion < max_cards_per_region,
1991             "too many cards per region");
1992 
1993   FreeRegionList::set_unrealistically_long_length(max_regions() + 1);
1994 
1995   _bot_shared = new G1BlockOffsetSharedArray(reserved_region(), bot_storage);
1996 
1997   {
1998     HeapWord* start = _hrm.reserved().start();
1999     HeapWord* end = _hrm.reserved().end();
2000     size_t granularity = HeapRegion::GrainBytes;
2001 
2002     _in_cset_fast_test.initialize(start, end, granularity);
2003     _humongous_reclaim_candidates.initialize(start, end, granularity);
2004   }
2005 
2006   // Create the ConcurrentMark data structure and thread.
2007   // (Must do this late, so that "max_regions" is defined.)
2008   _cm = new ConcurrentMark(this, prev_bitmap_storage, next_bitmap_storage);
2009   if (_cm == NULL || !_cm->completed_initialization()) {
2010     vm_shutdown_during_initialization("Could not create/initialize ConcurrentMark");
2011     return JNI_ENOMEM;
2012   }
2013   _cmThread = _cm->cmThread();
2014 
2015   // Initialize the from_card cache structure of HeapRegionRemSet.
2016   HeapRegionRemSet::init_heap(max_regions());
2017 
2018   // Now expand into the initial heap size.
2019   if (!expand(init_byte_size)) {
2020     vm_shutdown_during_initialization("Failed to allocate initial heap.");
2021     return JNI_ENOMEM;
2022   }
2023 
2024   // Perform any initialization actions delegated to the policy.
2025   g1_policy()->init();
2026 
2027   JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,
2028                                                SATB_Q_FL_lock,
2029                                                G1SATBProcessCompletedThreshold,
2030                                                Shared_SATB_Q_lock);
2031 
2032   JavaThread::dirty_card_queue_set().initialize(_refine_cte_cl,
2033                                                 DirtyCardQ_CBL_mon,
2034                                                 DirtyCardQ_FL_lock,
2035                                                 concurrent_g1_refine()->yellow_zone(),
2036                                                 concurrent_g1_refine()->red_zone(),
2037                                                 Shared_DirtyCardQ_lock);
2038 
2039   dirty_card_queue_set().initialize(NULL, // Should never be called by the Java code
2040                                     DirtyCardQ_CBL_mon,
2041                                     DirtyCardQ_FL_lock,
2042                                     -1, // never trigger processing
2043                                     -1, // no limit on length
2044                                     Shared_DirtyCardQ_lock,
2045                                     &JavaThread::dirty_card_queue_set());
2046 
2047   // Here we allocate the dummy HeapRegion that is required by the
2048   // G1AllocRegion class.
2049   HeapRegion* dummy_region = _hrm.get_dummy_region();
2050 
2051   // We'll re-use the same region whether the alloc region will
2052   // require BOT updates or not and, if it doesn't, then a non-young
2053   // region will complain that it cannot support allocations without
2054   // BOT updates. So we'll tag the dummy region as eden to avoid that.
2055   dummy_region->set_eden();
2056   // Make sure it's full.
2057   dummy_region->set_top(dummy_region->end());
2058   G1AllocRegion::setup(this, dummy_region);
2059 
2060   _allocator->init_mutator_alloc_region();
2061 
2062   // Do create of the monitoring and management support so that
2063   // values in the heap have been properly initialized.
2064   _g1mm = new G1MonitoringSupport(this);
2065 
2066   G1StringDedup::initialize();
2067 
2068   _preserved_objs = NEW_C_HEAP_ARRAY(OopAndMarkOopStack, ParallelGCThreads, mtGC);
2069   for (uint i = 0; i < ParallelGCThreads; i++) {
2070     new (&_preserved_objs[i]) OopAndMarkOopStack();
2071   }
2072 
2073   return JNI_OK;
2074 }
2075 
2076 void G1CollectedHeap::stop() {
2077   // Stop all concurrent threads. We do this to make sure these threads
2078   // do not continue to execute and access resources (e.g. gclog_or_tty)
2079   // that are destroyed during shutdown.
2080   _cg1r->stop();
2081   _cmThread->stop();
2082   if (G1StringDedup::is_enabled()) {
2083     G1StringDedup::stop();
2084   }
2085 }
2086 
2087 size_t G1CollectedHeap::conservative_max_heap_alignment() {
2088   return HeapRegion::max_region_size();
2089 }
2090 
2091 void G1CollectedHeap::post_initialize() {
2092   CollectedHeap::post_initialize();
2093   ref_processing_init();
2094 }
2095 
2096 void G1CollectedHeap::ref_processing_init() {
2097   // Reference processing in G1 currently works as follows:
2098   //
2099   // * There are two reference processor instances. One is
2100   //   used to record and process discovered references
2101   //   during concurrent marking; the other is used to
2102   //   record and process references during STW pauses
2103   //   (both full and incremental).
2104   // * Both ref processors need to 'span' the entire heap as
2105   //   the regions in the collection set may be dotted around.
2106   //
2107   // * For the concurrent marking ref processor:
2108   //   * Reference discovery is enabled at initial marking.
2109   //   * Reference discovery is disabled and the discovered
2110   //     references processed etc during remarking.
2111   //   * Reference discovery is MT (see below).
2112   //   * Reference discovery requires a barrier (see below).
2113   //   * Reference processing may or may not be MT
2114   //     (depending on the value of ParallelRefProcEnabled
2115   //     and ParallelGCThreads).
2116   //   * A full GC disables reference discovery by the CM
2117   //     ref processor and abandons any entries on it's
2118   //     discovered lists.
2119   //
2120   // * For the STW processor:
2121   //   * Non MT discovery is enabled at the start of a full GC.
2122   //   * Processing and enqueueing during a full GC is non-MT.
2123   //   * During a full GC, references are processed after marking.
2124   //
2125   //   * Discovery (may or may not be MT) is enabled at the start
2126   //     of an incremental evacuation pause.
2127   //   * References are processed near the end of a STW evacuation pause.
2128   //   * For both types of GC:
2129   //     * Discovery is atomic - i.e. not concurrent.
2130   //     * Reference discovery will not need a barrier.
2131 
2132   MemRegion mr = reserved_region();
2133 
2134   // Concurrent Mark ref processor
2135   _ref_processor_cm =
2136     new ReferenceProcessor(mr,    // span
2137                            ParallelRefProcEnabled && (ParallelGCThreads > 1),
2138                                 // mt processing
2139                            ParallelGCThreads,
2140                                 // degree of mt processing
2141                            (ParallelGCThreads > 1) || (ConcGCThreads > 1),
2142                                 // mt discovery
2143                            MAX2(ParallelGCThreads, ConcGCThreads),
2144                                 // degree of mt discovery
2145                            false,
2146                                 // Reference discovery is not atomic
2147                            &_is_alive_closure_cm);
2148                                 // is alive closure
2149                                 // (for efficiency/performance)
2150 
2151   // STW ref processor
2152   _ref_processor_stw =
2153     new ReferenceProcessor(mr,    // span
2154                            ParallelRefProcEnabled && (ParallelGCThreads > 1),
2155                                 // mt processing
2156                            ParallelGCThreads,
2157                                 // degree of mt processing
2158                            (ParallelGCThreads > 1),
2159                                 // mt discovery
2160                            ParallelGCThreads,
2161                                 // degree of mt discovery
2162                            true,
2163                                 // Reference discovery is atomic
2164                            &_is_alive_closure_stw);
2165                                 // is alive closure
2166                                 // (for efficiency/performance)
2167 }
2168 
2169 CollectorPolicy* G1CollectedHeap::collector_policy() const {
2170   return g1_policy();
2171 }
2172 
2173 size_t G1CollectedHeap::capacity() const {
2174   return _hrm.length() * HeapRegion::GrainBytes;
2175 }
2176 
2177 void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) {
2178   hr->reset_gc_time_stamp();
2179 }
2180 
2181 #ifndef PRODUCT
2182 
2183 class CheckGCTimeStampsHRClosure : public HeapRegionClosure {
2184 private:
2185   unsigned _gc_time_stamp;
2186   bool _failures;
2187 
2188 public:
2189   CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) :
2190     _gc_time_stamp(gc_time_stamp), _failures(false) { }
2191 
2192   virtual bool doHeapRegion(HeapRegion* hr) {
2193     unsigned region_gc_time_stamp = hr->get_gc_time_stamp();
2194     if (_gc_time_stamp != region_gc_time_stamp) {
2195       gclog_or_tty->print_cr("Region " HR_FORMAT " has GC time stamp = %d, "
2196                              "expected %d", HR_FORMAT_PARAMS(hr),
2197                              region_gc_time_stamp, _gc_time_stamp);
2198       _failures = true;
2199     }
2200     return false;
2201   }
2202 
2203   bool failures() { return _failures; }
2204 };
2205 
2206 void G1CollectedHeap::check_gc_time_stamps() {
2207   CheckGCTimeStampsHRClosure cl(_gc_time_stamp);
2208   heap_region_iterate(&cl);
2209   guarantee(!cl.failures(), "all GC time stamps should have been reset");
2210 }
2211 #endif // PRODUCT
2212 
2213 void G1CollectedHeap::iterate_hcc_closure(CardTableEntryClosure* cl, uint worker_i) {
2214   _cg1r->hot_card_cache()->drain(cl, worker_i);
2215 }
2216 
2217 void G1CollectedHeap::iterate_dirty_card_closure(CardTableEntryClosure* cl, uint worker_i) {
2218   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
2219   size_t n_completed_buffers = 0;
2220   while (dcqs.apply_closure_to_completed_buffer(cl, worker_i, 0, true)) {
2221     n_completed_buffers++;
2222   }
2223   g1_policy()->phase_times()->record_thread_work_item(G1GCPhaseTimes::UpdateRS, worker_i, n_completed_buffers);
2224   dcqs.clear_n_completed_buffers();
2225   assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!");
2226 }
2227 
2228 // Computes the sum of the storage used by the various regions.
2229 size_t G1CollectedHeap::used() const {
2230   size_t result = _summary_bytes_used + _allocator->used_in_alloc_regions();
2231   if (_archive_allocator != NULL) {
2232     result += _archive_allocator->used();
2233   }
2234   return result;
2235 }
2236 
2237 size_t G1CollectedHeap::used_unlocked() const {
2238   return _summary_bytes_used;
2239 }
2240 
2241 class SumUsedClosure: public HeapRegionClosure {
2242   size_t _used;
2243 public:
2244   SumUsedClosure() : _used(0) {}
2245   bool doHeapRegion(HeapRegion* r) {
2246     _used += r->used();
2247     return false;
2248   }
2249   size_t result() { return _used; }
2250 };
2251 
2252 size_t G1CollectedHeap::recalculate_used() const {
2253   double recalculate_used_start = os::elapsedTime();
2254 
2255   SumUsedClosure blk;
2256   heap_region_iterate(&blk);
2257 
2258   g1_policy()->phase_times()->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0);
2259   return blk.result();
2260 }
2261 
2262 bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) {
2263   switch (cause) {
2264     case GCCause::_gc_locker:               return GCLockerInvokesConcurrent;
2265     case GCCause::_java_lang_system_gc:     return ExplicitGCInvokesConcurrent;
2266     case GCCause::_dcmd_gc_run:             return ExplicitGCInvokesConcurrent;
2267     case GCCause::_g1_humongous_allocation: return true;
2268     case GCCause::_update_allocation_context_stats_inc: return true;
2269     case GCCause::_wb_conc_mark:            return true;
2270     default:                                return false;
2271   }
2272 }
2273 
2274 #ifndef PRODUCT
2275 void G1CollectedHeap::allocate_dummy_regions() {
2276   // Let's fill up most of the region
2277   size_t word_size = HeapRegion::GrainWords - 1024;
2278   // And as a result the region we'll allocate will be humongous.
2279   guarantee(is_humongous(word_size), "sanity");
2280 
2281   for (uintx i = 0; i < G1DummyRegionsPerGC; ++i) {
2282     // Let's use the existing mechanism for the allocation
2283     HeapWord* dummy_obj = humongous_obj_allocate(word_size,
2284                                                  AllocationContext::system());
2285     if (dummy_obj != NULL) {
2286       MemRegion mr(dummy_obj, word_size);
2287       CollectedHeap::fill_with_object(mr);
2288     } else {
2289       // If we can't allocate once, we probably cannot allocate
2290       // again. Let's get out of the loop.
2291       break;
2292     }
2293   }
2294 }
2295 #endif // !PRODUCT
2296 
2297 void G1CollectedHeap::increment_old_marking_cycles_started() {
2298   assert(_old_marking_cycles_started == _old_marking_cycles_completed ||
2299          _old_marking_cycles_started == _old_marking_cycles_completed + 1,
2300          "Wrong marking cycle count (started: %d, completed: %d)",
2301          _old_marking_cycles_started, _old_marking_cycles_completed);
2302 
2303   _old_marking_cycles_started++;
2304 }
2305 
2306 void G1CollectedHeap::increment_old_marking_cycles_completed(bool concurrent) {
2307   MonitorLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag);
2308 
2309   // We assume that if concurrent == true, then the caller is a
2310   // concurrent thread that was joined the Suspendible Thread
2311   // Set. If there's ever a cheap way to check this, we should add an
2312   // assert here.
2313 
2314   // Given that this method is called at the end of a Full GC or of a
2315   // concurrent cycle, and those can be nested (i.e., a Full GC can
2316   // interrupt a concurrent cycle), the number of full collections
2317   // completed should be either one (in the case where there was no
2318   // nesting) or two (when a Full GC interrupted a concurrent cycle)
2319   // behind the number of full collections started.
2320 
2321   // This is the case for the inner caller, i.e. a Full GC.
2322   assert(concurrent ||
2323          (_old_marking_cycles_started == _old_marking_cycles_completed + 1) ||
2324          (_old_marking_cycles_started == _old_marking_cycles_completed + 2),
2325          "for inner caller (Full GC): _old_marking_cycles_started = %u "
2326          "is inconsistent with _old_marking_cycles_completed = %u",
2327          _old_marking_cycles_started, _old_marking_cycles_completed);
2328 
2329   // This is the case for the outer caller, i.e. the concurrent cycle.
2330   assert(!concurrent ||
2331          (_old_marking_cycles_started == _old_marking_cycles_completed + 1),
2332          "for outer caller (concurrent cycle): "
2333          "_old_marking_cycles_started = %u "
2334          "is inconsistent with _old_marking_cycles_completed = %u",
2335          _old_marking_cycles_started, _old_marking_cycles_completed);
2336 
2337   _old_marking_cycles_completed += 1;
2338 
2339   // We need to clear the "in_progress" flag in the CM thread before
2340   // we wake up any waiters (especially when ExplicitInvokesConcurrent
2341   // is set) so that if a waiter requests another System.gc() it doesn't
2342   // incorrectly see that a marking cycle is still in progress.
2343   if (concurrent) {
2344     _cmThread->set_idle();
2345   }
2346 
2347   // This notify_all() will ensure that a thread that called
2348   // System.gc() with (with ExplicitGCInvokesConcurrent set or not)
2349   // and it's waiting for a full GC to finish will be woken up. It is
2350   // waiting in VM_G1IncCollectionPause::doit_epilogue().
2351   FullGCCount_lock->notify_all();
2352 }
2353 
2354 void G1CollectedHeap::register_concurrent_cycle_start(const Ticks& start_time) {
2355   GCIdMarkAndRestore conc_gc_id_mark;
2356   collector_state()->set_concurrent_cycle_started(true);
2357   _gc_timer_cm->register_gc_start(start_time);
2358 
2359   _gc_tracer_cm->report_gc_start(gc_cause(), _gc_timer_cm->gc_start());
2360   trace_heap_before_gc(_gc_tracer_cm);
2361   _cmThread->set_gc_id(GCId::current());
2362 }
2363 
2364 void G1CollectedHeap::register_concurrent_cycle_end() {
2365   if (collector_state()->concurrent_cycle_started()) {
2366     GCIdMarkAndRestore conc_gc_id_mark(_cmThread->gc_id());
2367     if (_cm->has_aborted()) {
2368       _gc_tracer_cm->report_concurrent_mode_failure();
2369     }
2370 
2371     _gc_timer_cm->register_gc_end();
2372     _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions());
2373 
2374     // Clear state variables to prepare for the next concurrent cycle.
2375      collector_state()->set_concurrent_cycle_started(false);
2376     _heap_summary_sent = false;
2377   }
2378 }
2379 
2380 void G1CollectedHeap::trace_heap_after_concurrent_cycle() {
2381   if (collector_state()->concurrent_cycle_started()) {
2382     // This function can be called when:
2383     //  the cleanup pause is run
2384     //  the concurrent cycle is aborted before the cleanup pause.
2385     //  the concurrent cycle is aborted after the cleanup pause,
2386     //   but before the concurrent cycle end has been registered.
2387     // Make sure that we only send the heap information once.
2388     if (!_heap_summary_sent) {
2389       GCIdMarkAndRestore conc_gc_id_mark(_cmThread->gc_id());
2390       trace_heap_after_gc(_gc_tracer_cm);
2391       _heap_summary_sent = true;
2392     }
2393   }
2394 }
2395 
2396 void G1CollectedHeap::collect(GCCause::Cause cause) {
2397   assert_heap_not_locked();
2398 
2399   uint gc_count_before;
2400   uint old_marking_count_before;
2401   uint full_gc_count_before;
2402   bool retry_gc;
2403 
2404   do {
2405     retry_gc = false;
2406 
2407     {
2408       MutexLocker ml(Heap_lock);
2409 
2410       // Read the GC count while holding the Heap_lock
2411       gc_count_before = total_collections();
2412       full_gc_count_before = total_full_collections();
2413       old_marking_count_before = _old_marking_cycles_started;
2414     }
2415 
2416     if (should_do_concurrent_full_gc(cause)) {
2417       // Schedule an initial-mark evacuation pause that will start a
2418       // concurrent cycle. We're setting word_size to 0 which means that
2419       // we are not requesting a post-GC allocation.
2420       VM_G1IncCollectionPause op(gc_count_before,
2421                                  0,     /* word_size */
2422                                  true,  /* should_initiate_conc_mark */
2423                                  g1_policy()->max_pause_time_ms(),
2424                                  cause);
2425       op.set_allocation_context(AllocationContext::current());
2426 
2427       VMThread::execute(&op);
2428       if (!op.pause_succeeded()) {
2429         if (old_marking_count_before == _old_marking_cycles_started) {
2430           retry_gc = op.should_retry_gc();
2431         } else {
2432           // A Full GC happened while we were trying to schedule the
2433           // initial-mark GC. No point in starting a new cycle given
2434           // that the whole heap was collected anyway.
2435         }
2436 
2437         if (retry_gc) {
2438           if (GC_locker::is_active_and_needs_gc()) {
2439             GC_locker::stall_until_clear();
2440           }
2441         }
2442       }
2443     } else {
2444       if (cause == GCCause::_gc_locker || cause == GCCause::_wb_young_gc
2445           DEBUG_ONLY(|| cause == GCCause::_scavenge_alot)) {
2446 
2447         // Schedule a standard evacuation pause. We're setting word_size
2448         // to 0 which means that we are not requesting a post-GC allocation.
2449         VM_G1IncCollectionPause op(gc_count_before,
2450                                    0,     /* word_size */
2451                                    false, /* should_initiate_conc_mark */
2452                                    g1_policy()->max_pause_time_ms(),
2453                                    cause);
2454         VMThread::execute(&op);
2455       } else {
2456         // Schedule a Full GC.
2457         VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause);
2458         VMThread::execute(&op);
2459       }
2460     }
2461   } while (retry_gc);
2462 }
2463 
2464 bool G1CollectedHeap::is_in(const void* p) const {
2465   if (_hrm.reserved().contains(p)) {
2466     // Given that we know that p is in the reserved space,
2467     // heap_region_containing() should successfully
2468     // return the containing region.
2469     HeapRegion* hr = heap_region_containing(p);
2470     return hr->is_in(p);
2471   } else {
2472     return false;
2473   }
2474 }
2475 
2476 #ifdef ASSERT
2477 bool G1CollectedHeap::is_in_exact(const void* p) const {
2478   bool contains = reserved_region().contains(p);
2479   bool available = _hrm.is_available(addr_to_region((HeapWord*)p));
2480   if (contains && available) {
2481     return true;
2482   } else {
2483     return false;
2484   }
2485 }
2486 #endif
2487 
2488 bool G1CollectedHeap::obj_in_cs(oop obj) {
2489   HeapRegion* r = _hrm.addr_to_region((HeapWord*) obj);
2490   return r != NULL && r->in_collection_set();
2491 }
2492 
2493 // Iteration functions.
2494 
2495 // Applies an ExtendedOopClosure onto all references of objects within a HeapRegion.
2496 
2497 class IterateOopClosureRegionClosure: public HeapRegionClosure {
2498   ExtendedOopClosure* _cl;
2499 public:
2500   IterateOopClosureRegionClosure(ExtendedOopClosure* cl) : _cl(cl) {}
2501   bool doHeapRegion(HeapRegion* r) {
2502     if (!r->is_continues_humongous()) {
2503       r->oop_iterate(_cl);
2504     }
2505     return false;
2506   }
2507 };
2508 
2509 // Iterates an ObjectClosure over all objects within a HeapRegion.
2510 
2511 class IterateObjectClosureRegionClosure: public HeapRegionClosure {
2512   ObjectClosure* _cl;
2513 public:
2514   IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}
2515   bool doHeapRegion(HeapRegion* r) {
2516     if (!r->is_continues_humongous()) {
2517       r->object_iterate(_cl);
2518     }
2519     return false;
2520   }
2521 };
2522 
2523 void G1CollectedHeap::object_iterate(ObjectClosure* cl) {
2524   IterateObjectClosureRegionClosure blk(cl);
2525   heap_region_iterate(&blk);
2526 }
2527 
2528 void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const {
2529   _hrm.iterate(cl);
2530 }
2531 
2532 void
2533 G1CollectedHeap::heap_region_par_iterate(HeapRegionClosure* cl,
2534                                          uint worker_id,
2535                                          HeapRegionClaimer *hrclaimer,
2536                                          bool concurrent) const {
2537   _hrm.par_iterate(cl, worker_id, hrclaimer, concurrent);
2538 }
2539 
2540 // Clear the cached CSet starting regions and (more importantly)
2541 // the time stamps. Called when we reset the GC time stamp.
2542 void G1CollectedHeap::clear_cset_start_regions() {
2543   assert(_worker_cset_start_region != NULL, "sanity");
2544   assert(_worker_cset_start_region_time_stamp != NULL, "sanity");
2545 
2546   for (uint i = 0; i < ParallelGCThreads; i++) {
2547     _worker_cset_start_region[i] = NULL;
2548     _worker_cset_start_region_time_stamp[i] = 0;
2549   }
2550 }
2551 
2552 // Given the id of a worker, obtain or calculate a suitable
2553 // starting region for iterating over the current collection set.
2554 HeapRegion* G1CollectedHeap::start_cset_region_for_worker(uint worker_i) {
2555   assert(get_gc_time_stamp() > 0, "should have been updated by now");
2556 
2557   HeapRegion* result = NULL;
2558   unsigned gc_time_stamp = get_gc_time_stamp();
2559 
2560   if (_worker_cset_start_region_time_stamp[worker_i] == gc_time_stamp) {
2561     // Cached starting region for current worker was set
2562     // during the current pause - so it's valid.
2563     // Note: the cached starting heap region may be NULL
2564     // (when the collection set is empty).
2565     result = _worker_cset_start_region[worker_i];
2566     assert(result == NULL || result->in_collection_set(), "sanity");
2567     return result;
2568   }
2569 
2570   // The cached entry was not valid so let's calculate
2571   // a suitable starting heap region for this worker.
2572 
2573   // We want the parallel threads to start their collection
2574   // set iteration at different collection set regions to
2575   // avoid contention.
2576   // If we have:
2577   //          n collection set regions
2578   //          p threads
2579   // Then thread t will start at region floor ((t * n) / p)
2580 
2581   result = g1_policy()->collection_set();
2582   uint cs_size = g1_policy()->cset_region_length();
2583   uint active_workers = workers()->active_workers();
2584 
2585   uint end_ind   = (cs_size * worker_i) / active_workers;
2586   uint start_ind = 0;
2587 
2588   if (worker_i > 0 &&
2589       _worker_cset_start_region_time_stamp[worker_i - 1] == gc_time_stamp) {
2590     // Previous workers starting region is valid
2591     // so let's iterate from there
2592     start_ind = (cs_size * (worker_i - 1)) / active_workers;
2593     result = _worker_cset_start_region[worker_i - 1];
2594   }
2595 
2596   for (uint i = start_ind; i < end_ind; i++) {
2597     result = result->next_in_collection_set();
2598   }
2599 
2600   // Note: the calculated starting heap region may be NULL
2601   // (when the collection set is empty).
2602   assert(result == NULL || result->in_collection_set(), "sanity");
2603   assert(_worker_cset_start_region_time_stamp[worker_i] != gc_time_stamp,
2604          "should be updated only once per pause");
2605   _worker_cset_start_region[worker_i] = result;
2606   OrderAccess::storestore();
2607   _worker_cset_start_region_time_stamp[worker_i] = gc_time_stamp;
2608   return result;
2609 }
2610 
2611 void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) {
2612   HeapRegion* r = g1_policy()->collection_set();
2613   while (r != NULL) {
2614     HeapRegion* next = r->next_in_collection_set();
2615     if (cl->doHeapRegion(r)) {
2616       cl->incomplete();
2617       return;
2618     }
2619     r = next;
2620   }
2621 }
2622 
2623 void G1CollectedHeap::collection_set_iterate_from(HeapRegion* r,
2624                                                   HeapRegionClosure *cl) {
2625   if (r == NULL) {
2626     // The CSet is empty so there's nothing to do.
2627     return;
2628   }
2629 
2630   assert(r->in_collection_set(),
2631          "Start region must be a member of the collection set.");
2632   HeapRegion* cur = r;
2633   while (cur != NULL) {
2634     HeapRegion* next = cur->next_in_collection_set();
2635     if (cl->doHeapRegion(cur) && false) {
2636       cl->incomplete();
2637       return;
2638     }
2639     cur = next;
2640   }
2641   cur = g1_policy()->collection_set();
2642   while (cur != r) {
2643     HeapRegion* next = cur->next_in_collection_set();
2644     if (cl->doHeapRegion(cur) && false) {
2645       cl->incomplete();
2646       return;
2647     }
2648     cur = next;
2649   }
2650 }
2651 
2652 HeapRegion* G1CollectedHeap::next_compaction_region(const HeapRegion* from) const {
2653   HeapRegion* result = _hrm.next_region_in_heap(from);
2654   while (result != NULL && result->is_pinned()) {
2655     result = _hrm.next_region_in_heap(result);
2656   }
2657   return result;
2658 }
2659 
2660 HeapWord* G1CollectedHeap::block_start(const void* addr) const {
2661   HeapRegion* hr = heap_region_containing(addr);
2662   return hr->block_start(addr);
2663 }
2664 
2665 size_t G1CollectedHeap::block_size(const HeapWord* addr) const {
2666   HeapRegion* hr = heap_region_containing(addr);
2667   return hr->block_size(addr);
2668 }
2669 
2670 bool G1CollectedHeap::block_is_obj(const HeapWord* addr) const {
2671   HeapRegion* hr = heap_region_containing(addr);
2672   return hr->block_is_obj(addr);
2673 }
2674 
2675 bool G1CollectedHeap::supports_tlab_allocation() const {
2676   return true;
2677 }
2678 
2679 size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const {
2680   return (_g1_policy->young_list_target_length() - young_list()->survivor_length()) * HeapRegion::GrainBytes;
2681 }
2682 
2683 size_t G1CollectedHeap::tlab_used(Thread* ignored) const {
2684   return young_list()->eden_used_bytes();
2685 }
2686 
2687 // For G1 TLABs should not contain humongous objects, so the maximum TLAB size
2688 // must be equal to the humongous object limit.
2689 size_t G1CollectedHeap::max_tlab_size() const {
2690   return align_size_down(_humongous_object_threshold_in_words, MinObjAlignment);
2691 }
2692 
2693 size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const {
2694   AllocationContext_t context = AllocationContext::current();
2695   return _allocator->unsafe_max_tlab_alloc(context);
2696 }
2697 
2698 size_t G1CollectedHeap::max_capacity() const {
2699   return _hrm.reserved().byte_size();
2700 }
2701 
2702 jlong G1CollectedHeap::millis_since_last_gc() {
2703   // assert(false, "NYI");
2704   return 0;
2705 }
2706 
2707 void G1CollectedHeap::prepare_for_verify() {
2708   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
2709     ensure_parsability(false);
2710   }
2711   g1_rem_set()->prepare_for_verify();
2712 }
2713 
2714 bool G1CollectedHeap::allocated_since_marking(oop obj, HeapRegion* hr,
2715                                               VerifyOption vo) {
2716   switch (vo) {
2717   case VerifyOption_G1UsePrevMarking:
2718     return hr->obj_allocated_since_prev_marking(obj);
2719   case VerifyOption_G1UseNextMarking:
2720     return hr->obj_allocated_since_next_marking(obj);
2721   case VerifyOption_G1UseMarkWord:
2722     return false;
2723   default:
2724     ShouldNotReachHere();
2725   }
2726   return false; // keep some compilers happy
2727 }
2728 
2729 HeapWord* G1CollectedHeap::top_at_mark_start(HeapRegion* hr, VerifyOption vo) {
2730   switch (vo) {
2731   case VerifyOption_G1UsePrevMarking: return hr->prev_top_at_mark_start();
2732   case VerifyOption_G1UseNextMarking: return hr->next_top_at_mark_start();
2733   case VerifyOption_G1UseMarkWord:    return NULL;
2734   default:                            ShouldNotReachHere();
2735   }
2736   return NULL; // keep some compilers happy
2737 }
2738 
2739 bool G1CollectedHeap::is_marked(oop obj, VerifyOption vo) {
2740   switch (vo) {
2741   case VerifyOption_G1UsePrevMarking: return isMarkedPrev(obj);
2742   case VerifyOption_G1UseNextMarking: return isMarkedNext(obj);
2743   case VerifyOption_G1UseMarkWord:    return obj->is_gc_marked();
2744   default:                            ShouldNotReachHere();
2745   }
2746   return false; // keep some compilers happy
2747 }
2748 
2749 const char* G1CollectedHeap::top_at_mark_start_str(VerifyOption vo) {
2750   switch (vo) {
2751   case VerifyOption_G1UsePrevMarking: return "PTAMS";
2752   case VerifyOption_G1UseNextMarking: return "NTAMS";
2753   case VerifyOption_G1UseMarkWord:    return "NONE";
2754   default:                            ShouldNotReachHere();
2755   }
2756   return NULL; // keep some compilers happy
2757 }
2758 
2759 class VerifyRootsClosure: public OopClosure {
2760 private:
2761   G1CollectedHeap* _g1h;
2762   VerifyOption     _vo;
2763   bool             _failures;
2764 public:
2765   // _vo == UsePrevMarking -> use "prev" marking information,
2766   // _vo == UseNextMarking -> use "next" marking information,
2767   // _vo == UseMarkWord    -> use mark word from object header.
2768   VerifyRootsClosure(VerifyOption vo) :
2769     _g1h(G1CollectedHeap::heap()),
2770     _vo(vo),
2771     _failures(false) { }
2772 
2773   bool failures() { return _failures; }
2774 
2775   template <class T> void do_oop_nv(T* p) {
2776     T heap_oop = oopDesc::load_heap_oop(p);
2777     if (!oopDesc::is_null(heap_oop)) {
2778       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
2779       if (_g1h->is_obj_dead_cond(obj, _vo)) {
2780         gclog_or_tty->print_cr("Root location " PTR_FORMAT " "
2781                                "points to dead obj " PTR_FORMAT, p2i(p), p2i(obj));
2782         if (_vo == VerifyOption_G1UseMarkWord) {
2783           gclog_or_tty->print_cr("  Mark word: " INTPTR_FORMAT, (intptr_t)obj->mark());
2784         }
2785         obj->print_on(gclog_or_tty);
2786         _failures = true;
2787       }
2788     }
2789   }
2790 
2791   void do_oop(oop* p)       { do_oop_nv(p); }
2792   void do_oop(narrowOop* p) { do_oop_nv(p); }
2793 };
2794 
2795 class G1VerifyCodeRootOopClosure: public OopClosure {
2796   G1CollectedHeap* _g1h;
2797   OopClosure* _root_cl;
2798   nmethod* _nm;
2799   VerifyOption _vo;
2800   bool _failures;
2801 
2802   template <class T> void do_oop_work(T* p) {
2803     // First verify that this root is live
2804     _root_cl->do_oop(p);
2805 
2806     if (!G1VerifyHeapRegionCodeRoots) {
2807       // We're not verifying the code roots attached to heap region.
2808       return;
2809     }
2810 
2811     // Don't check the code roots during marking verification in a full GC
2812     if (_vo == VerifyOption_G1UseMarkWord) {
2813       return;
2814     }
2815 
2816     // Now verify that the current nmethod (which contains p) is
2817     // in the code root list of the heap region containing the
2818     // object referenced by p.
2819 
2820     T heap_oop = oopDesc::load_heap_oop(p);
2821     if (!oopDesc::is_null(heap_oop)) {
2822       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
2823 
2824       // Now fetch the region containing the object
2825       HeapRegion* hr = _g1h->heap_region_containing(obj);
2826       HeapRegionRemSet* hrrs = hr->rem_set();
2827       // Verify that the strong code root list for this region
2828       // contains the nmethod
2829       if (!hrrs->strong_code_roots_list_contains(_nm)) {
2830         gclog_or_tty->print_cr("Code root location " PTR_FORMAT " "
2831                                "from nmethod " PTR_FORMAT " not in strong "
2832                                "code roots for region [" PTR_FORMAT "," PTR_FORMAT ")",
2833                                p2i(p), p2i(_nm), p2i(hr->bottom()), p2i(hr->end()));
2834         _failures = true;
2835       }
2836     }
2837   }
2838 
2839 public:
2840   G1VerifyCodeRootOopClosure(G1CollectedHeap* g1h, OopClosure* root_cl, VerifyOption vo):
2841     _g1h(g1h), _root_cl(root_cl), _vo(vo), _nm(NULL), _failures(false) {}
2842 
2843   void do_oop(oop* p) { do_oop_work(p); }
2844   void do_oop(narrowOop* p) { do_oop_work(p); }
2845 
2846   void set_nmethod(nmethod* nm) { _nm = nm; }
2847   bool failures() { return _failures; }
2848 };
2849 
2850 class G1VerifyCodeRootBlobClosure: public CodeBlobClosure {
2851   G1VerifyCodeRootOopClosure* _oop_cl;
2852 
2853 public:
2854   G1VerifyCodeRootBlobClosure(G1VerifyCodeRootOopClosure* oop_cl):
2855     _oop_cl(oop_cl) {}
2856 
2857   void do_code_blob(CodeBlob* cb) {
2858     nmethod* nm = cb->as_nmethod_or_null();
2859     if (nm != NULL) {
2860       _oop_cl->set_nmethod(nm);
2861       nm->oops_do(_oop_cl);
2862     }
2863   }
2864 };
2865 
2866 class YoungRefCounterClosure : public OopClosure {
2867   G1CollectedHeap* _g1h;
2868   int              _count;
2869  public:
2870   YoungRefCounterClosure(G1CollectedHeap* g1h) : _g1h(g1h), _count(0) {}
2871   void do_oop(oop* p)       { if (_g1h->is_in_young(*p)) { _count++; } }
2872   void do_oop(narrowOop* p) { ShouldNotReachHere(); }
2873 
2874   int count() { return _count; }
2875   void reset_count() { _count = 0; };
2876 };
2877 
2878 class VerifyKlassClosure: public KlassClosure {
2879   YoungRefCounterClosure _young_ref_counter_closure;
2880   OopClosure *_oop_closure;
2881  public:
2882   VerifyKlassClosure(G1CollectedHeap* g1h, OopClosure* cl) : _young_ref_counter_closure(g1h), _oop_closure(cl) {}
2883   void do_klass(Klass* k) {
2884     k->oops_do(_oop_closure);
2885 
2886     _young_ref_counter_closure.reset_count();
2887     k->oops_do(&_young_ref_counter_closure);
2888     if (_young_ref_counter_closure.count() > 0) {
2889       guarantee(k->has_modified_oops(), "Klass " PTR_FORMAT ", has young refs but is not dirty.", p2i(k));
2890     }
2891   }
2892 };
2893 
2894 class VerifyLivenessOopClosure: public OopClosure {
2895   G1CollectedHeap* _g1h;
2896   VerifyOption _vo;
2897 public:
2898   VerifyLivenessOopClosure(G1CollectedHeap* g1h, VerifyOption vo):
2899     _g1h(g1h), _vo(vo)
2900   { }
2901   void do_oop(narrowOop *p) { do_oop_work(p); }
2902   void do_oop(      oop *p) { do_oop_work(p); }
2903 
2904   template <class T> void do_oop_work(T *p) {
2905     oop obj = oopDesc::load_decode_heap_oop(p);
2906     guarantee(obj == NULL || !_g1h->is_obj_dead_cond(obj, _vo),
2907               "Dead object referenced by a not dead object");
2908   }
2909 };
2910 
2911 class VerifyObjsInRegionClosure: public ObjectClosure {
2912 private:
2913   G1CollectedHeap* _g1h;
2914   size_t _live_bytes;
2915   HeapRegion *_hr;
2916   VerifyOption _vo;
2917 public:
2918   // _vo == UsePrevMarking -> use "prev" marking information,
2919   // _vo == UseNextMarking -> use "next" marking information,
2920   // _vo == UseMarkWord    -> use mark word from object header.
2921   VerifyObjsInRegionClosure(HeapRegion *hr, VerifyOption vo)
2922     : _live_bytes(0), _hr(hr), _vo(vo) {
2923     _g1h = G1CollectedHeap::heap();
2924   }
2925   void do_object(oop o) {
2926     VerifyLivenessOopClosure isLive(_g1h, _vo);
2927     assert(o != NULL, "Huh?");
2928     if (!_g1h->is_obj_dead_cond(o, _vo)) {
2929       // If the object is alive according to the mark word,
2930       // then verify that the marking information agrees.
2931       // Note we can't verify the contra-positive of the
2932       // above: if the object is dead (according to the mark
2933       // word), it may not be marked, or may have been marked
2934       // but has since became dead, or may have been allocated
2935       // since the last marking.
2936       if (_vo == VerifyOption_G1UseMarkWord) {
2937         guarantee(!_g1h->is_obj_dead(o), "mark word and concurrent mark mismatch");
2938       }
2939 
2940       o->oop_iterate_no_header(&isLive);
2941       if (!_hr->obj_allocated_since_prev_marking(o)) {
2942         size_t obj_size = o->size();    // Make sure we don't overflow
2943         _live_bytes += (obj_size * HeapWordSize);
2944       }
2945     }
2946   }
2947   size_t live_bytes() { return _live_bytes; }
2948 };
2949 
2950 class VerifyArchiveOopClosure: public OopClosure {
2951 public:
2952   VerifyArchiveOopClosure(HeapRegion *hr) { }
2953   void do_oop(narrowOop *p) { do_oop_work(p); }
2954   void do_oop(      oop *p) { do_oop_work(p); }
2955 
2956   template <class T> void do_oop_work(T *p) {
2957     oop obj = oopDesc::load_decode_heap_oop(p);
2958     guarantee(obj == NULL || G1MarkSweep::in_archive_range(obj),
2959               "Archive object at " PTR_FORMAT " references a non-archive object at " PTR_FORMAT,
2960               p2i(p), p2i(obj));
2961   }
2962 };
2963 
2964 class VerifyArchiveRegionClosure: public ObjectClosure {
2965 public:
2966   VerifyArchiveRegionClosure(HeapRegion *hr) { }
2967   // Verify that all object pointers are to archive regions.
2968   void do_object(oop o) {
2969     VerifyArchiveOopClosure checkOop(NULL);
2970     assert(o != NULL, "Should not be here for NULL oops");
2971     o->oop_iterate_no_header(&checkOop);
2972   }
2973 };
2974 
2975 class VerifyRegionClosure: public HeapRegionClosure {
2976 private:
2977   bool             _par;
2978   VerifyOption     _vo;
2979   bool             _failures;
2980 public:
2981   // _vo == UsePrevMarking -> use "prev" marking information,
2982   // _vo == UseNextMarking -> use "next" marking information,
2983   // _vo == UseMarkWord    -> use mark word from object header.
2984   VerifyRegionClosure(bool par, VerifyOption vo)
2985     : _par(par),
2986       _vo(vo),
2987       _failures(false) {}
2988 
2989   bool failures() {
2990     return _failures;
2991   }
2992 
2993   bool doHeapRegion(HeapRegion* r) {
2994     // For archive regions, verify there are no heap pointers to
2995     // non-pinned regions. For all others, verify liveness info.
2996     if (r->is_archive()) {
2997       VerifyArchiveRegionClosure verify_oop_pointers(r);
2998       r->object_iterate(&verify_oop_pointers);
2999       return true;
3000     }
3001     if (!r->is_continues_humongous()) {
3002       bool failures = false;
3003       r->verify(_vo, &failures);
3004       if (failures) {
3005         _failures = true;
3006       } else if (!r->is_starts_humongous()) {
3007         VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo);
3008         r->object_iterate(&not_dead_yet_cl);
3009         if (_vo != VerifyOption_G1UseNextMarking) {
3010           if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) {
3011             gclog_or_tty->print_cr("[" PTR_FORMAT "," PTR_FORMAT "] "
3012                                    "max_live_bytes " SIZE_FORMAT " "
3013                                    "< calculated " SIZE_FORMAT,
3014                                    p2i(r->bottom()), p2i(r->end()),
3015                                    r->max_live_bytes(),
3016                                  not_dead_yet_cl.live_bytes());
3017             _failures = true;
3018           }
3019         } else {
3020           // When vo == UseNextMarking we cannot currently do a sanity
3021           // check on the live bytes as the calculation has not been
3022           // finalized yet.
3023         }
3024       }
3025     }
3026     return false; // stop the region iteration if we hit a failure
3027   }
3028 };
3029 
3030 // This is the task used for parallel verification of the heap regions
3031 
3032 class G1ParVerifyTask: public AbstractGangTask {
3033 private:
3034   G1CollectedHeap*  _g1h;
3035   VerifyOption      _vo;
3036   bool              _failures;
3037   HeapRegionClaimer _hrclaimer;
3038 
3039 public:
3040   // _vo == UsePrevMarking -> use "prev" marking information,
3041   // _vo == UseNextMarking -> use "next" marking information,
3042   // _vo == UseMarkWord    -> use mark word from object header.
3043   G1ParVerifyTask(G1CollectedHeap* g1h, VerifyOption vo) :
3044       AbstractGangTask("Parallel verify task"),
3045       _g1h(g1h),
3046       _vo(vo),
3047       _failures(false),
3048       _hrclaimer(g1h->workers()->active_workers()) {}
3049 
3050   bool failures() {
3051     return _failures;
3052   }
3053 
3054   void work(uint worker_id) {
3055     HandleMark hm;
3056     VerifyRegionClosure blk(true, _vo);
3057     _g1h->heap_region_par_iterate(&blk, worker_id, &_hrclaimer);
3058     if (blk.failures()) {
3059       _failures = true;
3060     }
3061   }
3062 };
3063 
3064 void G1CollectedHeap::verify(bool silent, VerifyOption vo) {
3065   if (SafepointSynchronize::is_at_safepoint()) {
3066     assert(Thread::current()->is_VM_thread(),
3067            "Expected to be executed serially by the VM thread at this point");
3068 
3069     if (!silent) { gclog_or_tty->print("Roots "); }
3070     VerifyRootsClosure rootsCl(vo);
3071     VerifyKlassClosure klassCl(this, &rootsCl);
3072     CLDToKlassAndOopClosure cldCl(&klassCl, &rootsCl, false);
3073 
3074     // We apply the relevant closures to all the oops in the
3075     // system dictionary, class loader data graph, the string table
3076     // and the nmethods in the code cache.
3077     G1VerifyCodeRootOopClosure codeRootsCl(this, &rootsCl, vo);
3078     G1VerifyCodeRootBlobClosure blobsCl(&codeRootsCl);
3079 
3080     {
3081       G1RootProcessor root_processor(this, 1);
3082       root_processor.process_all_roots(&rootsCl,
3083                                        &cldCl,
3084                                        &blobsCl);
3085     }
3086 
3087     bool failures = rootsCl.failures() || codeRootsCl.failures();
3088 
3089     if (vo != VerifyOption_G1UseMarkWord) {
3090       // If we're verifying during a full GC then the region sets
3091       // will have been torn down at the start of the GC. Therefore
3092       // verifying the region sets will fail. So we only verify
3093       // the region sets when not in a full GC.
3094       if (!silent) { gclog_or_tty->print("HeapRegionSets "); }
3095       verify_region_sets();
3096     }
3097 
3098     if (!silent) { gclog_or_tty->print("HeapRegions "); }
3099     if (GCParallelVerificationEnabled && ParallelGCThreads > 1) {
3100 
3101       G1ParVerifyTask task(this, vo);
3102       workers()->run_task(&task);
3103       if (task.failures()) {
3104         failures = true;
3105       }
3106 
3107     } else {
3108       VerifyRegionClosure blk(false, vo);
3109       heap_region_iterate(&blk);
3110       if (blk.failures()) {
3111         failures = true;
3112       }
3113     }
3114 
3115     if (G1StringDedup::is_enabled()) {
3116       if (!silent) gclog_or_tty->print("StrDedup ");
3117       G1StringDedup::verify();
3118     }
3119 
3120     if (failures) {
3121       gclog_or_tty->print_cr("Heap:");
3122       // It helps to have the per-region information in the output to
3123       // help us track down what went wrong. This is why we call
3124       // print_extended_on() instead of print_on().
3125       print_extended_on(gclog_or_tty);
3126       gclog_or_tty->cr();
3127       gclog_or_tty->flush();
3128     }
3129     guarantee(!failures, "there should not have been any failures");
3130   } else {
3131     if (!silent) {
3132       gclog_or_tty->print("(SKIPPING Roots, HeapRegionSets, HeapRegions, RemSet");
3133       if (G1StringDedup::is_enabled()) {
3134         gclog_or_tty->print(", StrDedup");
3135       }
3136       gclog_or_tty->print(") ");
3137     }
3138   }
3139 }
3140 
3141 void G1CollectedHeap::verify(bool silent) {
3142   verify(silent, VerifyOption_G1UsePrevMarking);
3143 }
3144 
3145 double G1CollectedHeap::verify(bool guard, const char* msg) {
3146   double verify_time_ms = 0.0;
3147 
3148   if (guard && total_collections() >= VerifyGCStartAt) {
3149     double verify_start = os::elapsedTime();
3150     HandleMark hm;  // Discard invalid handles created during verification
3151     prepare_for_verify();
3152     Universe::verify(VerifyOption_G1UsePrevMarking, msg);
3153     verify_time_ms = (os::elapsedTime() - verify_start) * 1000;
3154   }
3155 
3156   return verify_time_ms;
3157 }
3158 
3159 void G1CollectedHeap::verify_before_gc() {
3160   double verify_time_ms = verify(VerifyBeforeGC, " VerifyBeforeGC:");
3161   g1_policy()->phase_times()->record_verify_before_time_ms(verify_time_ms);
3162 }
3163 
3164 void G1CollectedHeap::verify_after_gc() {
3165   double verify_time_ms = verify(VerifyAfterGC, " VerifyAfterGC:");
3166   g1_policy()->phase_times()->record_verify_after_time_ms(verify_time_ms);
3167 }
3168 
3169 class PrintRegionClosure: public HeapRegionClosure {
3170   outputStream* _st;
3171 public:
3172   PrintRegionClosure(outputStream* st) : _st(st) {}
3173   bool doHeapRegion(HeapRegion* r) {
3174     r->print_on(_st);
3175     return false;
3176   }
3177 };
3178 
3179 bool G1CollectedHeap::is_obj_dead_cond(const oop obj,
3180                                        const HeapRegion* hr,
3181                                        const VerifyOption vo) const {
3182   switch (vo) {
3183   case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj, hr);
3184   case VerifyOption_G1UseNextMarking: return is_obj_ill(obj, hr);
3185   case VerifyOption_G1UseMarkWord:    return !obj->is_gc_marked() && !hr->is_archive();
3186   default:                            ShouldNotReachHere();
3187   }
3188   return false; // keep some compilers happy
3189 }
3190 
3191 bool G1CollectedHeap::is_obj_dead_cond(const oop obj,
3192                                        const VerifyOption vo) const {
3193   switch (vo) {
3194   case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj);
3195   case VerifyOption_G1UseNextMarking: return is_obj_ill(obj);
3196   case VerifyOption_G1UseMarkWord: {
3197     HeapRegion* hr = _hrm.addr_to_region((HeapWord*)obj);
3198     return !obj->is_gc_marked() && !hr->is_archive();
3199   }
3200   default:                            ShouldNotReachHere();
3201   }
3202   return false; // keep some compilers happy
3203 }
3204 
3205 void G1CollectedHeap::print_on(outputStream* st) const {
3206   st->print(" %-20s", "garbage-first heap");
3207   st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K",
3208             capacity()/K, used_unlocked()/K);
3209   st->print(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
3210             p2i(_hrm.reserved().start()),
3211             p2i(_hrm.reserved().start() + _hrm.length() + HeapRegion::GrainWords),
3212             p2i(_hrm.reserved().end()));
3213   st->cr();
3214   st->print("  region size " SIZE_FORMAT "K, ", HeapRegion::GrainBytes / K);
3215   uint young_regions = _young_list->length();
3216   st->print("%u young (" SIZE_FORMAT "K), ", young_regions,
3217             (size_t) young_regions * HeapRegion::GrainBytes / K);
3218   uint survivor_regions = g1_policy()->recorded_survivor_regions();
3219   st->print("%u survivors (" SIZE_FORMAT "K)", survivor_regions,
3220             (size_t) survivor_regions * HeapRegion::GrainBytes / K);
3221   st->cr();
3222   MetaspaceAux::print_on(st);
3223 }
3224 
3225 void G1CollectedHeap::print_extended_on(outputStream* st) const {
3226   print_on(st);
3227 
3228   // Print the per-region information.
3229   st->cr();
3230   st->print_cr("Heap Regions: (E=young(eden), S=young(survivor), O=old, "
3231                "HS=humongous(starts), HC=humongous(continues), "
3232                "CS=collection set, F=free, A=archive, TS=gc time stamp, "
3233                "PTAMS=previous top-at-mark-start, "
3234                "NTAMS=next top-at-mark-start)");
3235   PrintRegionClosure blk(st);
3236   heap_region_iterate(&blk);
3237 }
3238 
3239 void G1CollectedHeap::print_on_error(outputStream* st) const {
3240   this->CollectedHeap::print_on_error(st);
3241 
3242   if (_cm != NULL) {
3243     st->cr();
3244     _cm->print_on_error(st);
3245   }
3246 }
3247 
3248 void G1CollectedHeap::print_gc_threads_on(outputStream* st) const {
3249   workers()->print_worker_threads_on(st);
3250   _cmThread->print_on(st);
3251   st->cr();
3252   _cm->print_worker_threads_on(st);
3253   _cg1r->print_worker_threads_on(st);
3254   if (G1StringDedup::is_enabled()) {
3255     G1StringDedup::print_worker_threads_on(st);
3256   }
3257 }
3258 
3259 void G1CollectedHeap::gc_threads_do(ThreadClosure* tc) const {
3260   workers()->threads_do(tc);
3261   tc->do_thread(_cmThread);
3262   _cg1r->threads_do(tc);
3263   if (G1StringDedup::is_enabled()) {
3264     G1StringDedup::threads_do(tc);
3265   }
3266 }
3267 
3268 void G1CollectedHeap::print_tracing_info() const {
3269   // We'll overload this to mean "trace GC pause statistics."
3270   if (TraceYoungGenTime || TraceOldGenTime) {
3271     // The "G1CollectorPolicy" is keeping track of these stats, so delegate
3272     // to that.
3273     g1_policy()->print_tracing_info();
3274   }
3275   if (G1SummarizeRSetStats) {
3276     g1_rem_set()->print_summary_info();
3277   }
3278   if (G1SummarizeConcMark) {
3279     concurrent_mark()->print_summary_info();
3280   }
3281   g1_policy()->print_yg_surv_rate_info();
3282 }
3283 
3284 #ifndef PRODUCT
3285 // Helpful for debugging RSet issues.
3286 
3287 class PrintRSetsClosure : public HeapRegionClosure {
3288 private:
3289   const char* _msg;
3290   size_t _occupied_sum;
3291 
3292 public:
3293   bool doHeapRegion(HeapRegion* r) {
3294     HeapRegionRemSet* hrrs = r->rem_set();
3295     size_t occupied = hrrs->occupied();
3296     _occupied_sum += occupied;
3297 
3298     gclog_or_tty->print_cr("Printing RSet for region " HR_FORMAT,
3299                            HR_FORMAT_PARAMS(r));
3300     if (occupied == 0) {
3301       gclog_or_tty->print_cr("  RSet is empty");
3302     } else {
3303       hrrs->print();
3304     }
3305     gclog_or_tty->print_cr("----------");
3306     return false;
3307   }
3308 
3309   PrintRSetsClosure(const char* msg) : _msg(msg), _occupied_sum(0) {
3310     gclog_or_tty->cr();
3311     gclog_or_tty->print_cr("========================================");
3312     gclog_or_tty->print_cr("%s", msg);
3313     gclog_or_tty->cr();
3314   }
3315 
3316   ~PrintRSetsClosure() {
3317     gclog_or_tty->print_cr("Occupied Sum: " SIZE_FORMAT, _occupied_sum);
3318     gclog_or_tty->print_cr("========================================");
3319     gclog_or_tty->cr();
3320   }
3321 };
3322 
3323 void G1CollectedHeap::print_cset_rsets() {
3324   PrintRSetsClosure cl("Printing CSet RSets");
3325   collection_set_iterate(&cl);
3326 }
3327 
3328 void G1CollectedHeap::print_all_rsets() {
3329   PrintRSetsClosure cl("Printing All RSets");;
3330   heap_region_iterate(&cl);
3331 }
3332 #endif // PRODUCT
3333 
3334 G1HeapSummary G1CollectedHeap::create_g1_heap_summary() {
3335   YoungList* young_list = heap()->young_list();
3336 
3337   size_t eden_used_bytes = young_list->eden_used_bytes();
3338   size_t survivor_used_bytes = young_list->survivor_used_bytes();
3339 
3340   size_t eden_capacity_bytes =
3341     (g1_policy()->young_list_target_length() * HeapRegion::GrainBytes) - survivor_used_bytes;
3342 
3343   VirtualSpaceSummary heap_summary = create_heap_space_summary();
3344   return G1HeapSummary(heap_summary, used(), eden_used_bytes, eden_capacity_bytes, survivor_used_bytes);
3345 }
3346 
3347 G1EvacSummary G1CollectedHeap::create_g1_evac_summary(G1EvacStats* stats) {
3348   return G1EvacSummary(stats->allocated(), stats->wasted(), stats->undo_wasted(),
3349                        stats->unused(), stats->used(), stats->region_end_waste(),
3350                        stats->regions_filled(), stats->direct_allocated(),
3351                        stats->failure_used(), stats->failure_waste());
3352 }
3353 
3354 void G1CollectedHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) {
3355   const G1HeapSummary& heap_summary = create_g1_heap_summary();
3356   gc_tracer->report_gc_heap_summary(when, heap_summary);
3357 
3358   const MetaspaceSummary& metaspace_summary = create_metaspace_summary();
3359   gc_tracer->report_metaspace_summary(when, metaspace_summary);
3360 }
3361 
3362 
3363 G1CollectedHeap* G1CollectedHeap::heap() {
3364   CollectedHeap* heap = Universe::heap();
3365   assert(heap != NULL, "Uninitialized access to G1CollectedHeap::heap()");
3366   assert(heap->kind() == CollectedHeap::G1CollectedHeap, "Not a G1CollectedHeap");
3367   return (G1CollectedHeap*)heap;
3368 }
3369 
3370 void G1CollectedHeap::gc_prologue(bool full /* Ignored */) {
3371   // always_do_update_barrier = false;
3372   assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");
3373   // Fill TLAB's and such
3374   accumulate_statistics_all_tlabs();
3375   ensure_parsability(true);
3376 
3377   if (G1SummarizeRSetStats && (G1SummarizeRSetStatsPeriod > 0) &&
3378       (total_collections() % G1SummarizeRSetStatsPeriod == 0)) {
3379     g1_rem_set()->print_periodic_summary_info("Before GC RS summary");
3380   }
3381 }
3382 
3383 void G1CollectedHeap::gc_epilogue(bool full) {
3384 
3385   if (G1SummarizeRSetStats &&
3386       (G1SummarizeRSetStatsPeriod > 0) &&
3387       // we are at the end of the GC. Total collections has already been increased.
3388       ((total_collections() - 1) % G1SummarizeRSetStatsPeriod == 0)) {
3389     g1_rem_set()->print_periodic_summary_info("After GC RS summary");
3390   }
3391 
3392   // FIXME: what is this about?
3393   // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled"
3394   // is set.
3395 #if defined(COMPILER2) || INCLUDE_JVMCI
3396   assert(DerivedPointerTable::is_empty(), "derived pointer present");
3397 #endif
3398   // always_do_update_barrier = true;
3399 
3400   resize_all_tlabs();
3401   allocation_context_stats().update(full);
3402 
3403   // We have just completed a GC. Update the soft reference
3404   // policy with the new heap occupancy
3405   Universe::update_heap_info_at_gc();
3406 }
3407 
3408 HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size,
3409                                                uint gc_count_before,
3410                                                bool* succeeded,
3411                                                GCCause::Cause gc_cause) {
3412   assert_heap_not_locked_and_not_at_safepoint();
3413   g1_policy()->record_stop_world_start();
3414   VM_G1IncCollectionPause op(gc_count_before,
3415                              word_size,
3416                              false, /* should_initiate_conc_mark */
3417                              g1_policy()->max_pause_time_ms(),
3418                              gc_cause);
3419 
3420   op.set_allocation_context(AllocationContext::current());
3421   VMThread::execute(&op);
3422 
3423   HeapWord* result = op.result();
3424   bool ret_succeeded = op.prologue_succeeded() && op.pause_succeeded();
3425   assert(result == NULL || ret_succeeded,
3426          "the result should be NULL if the VM did not succeed");
3427   *succeeded = ret_succeeded;
3428 
3429   assert_heap_not_locked();
3430   return result;
3431 }
3432 
3433 void
3434 G1CollectedHeap::doConcurrentMark() {
3435   MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
3436   if (!_cmThread->in_progress()) {
3437     _cmThread->set_started();
3438     CGC_lock->notify();
3439   }
3440 }
3441 
3442 size_t G1CollectedHeap::pending_card_num() {
3443   size_t extra_cards = 0;
3444   JavaThread *curr = Threads::first();
3445   while (curr != NULL) {
3446     DirtyCardQueue& dcq = curr->dirty_card_queue();
3447     extra_cards += dcq.size();
3448     curr = curr->next();
3449   }
3450   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
3451   size_t buffer_size = dcqs.buffer_size();
3452   size_t buffer_num = dcqs.completed_buffers_num();
3453 
3454   // PtrQueueSet::buffer_size() and PtrQueue:size() return sizes
3455   // in bytes - not the number of 'entries'. We need to convert
3456   // into a number of cards.
3457   return (buffer_size * buffer_num + extra_cards) / oopSize;
3458 }
3459 
3460 class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure {
3461  private:
3462   size_t _total_humongous;
3463   size_t _candidate_humongous;
3464 
3465   DirtyCardQueue _dcq;
3466 
3467   // We don't nominate objects with many remembered set entries, on
3468   // the assumption that such objects are likely still live.
3469   bool is_remset_small(HeapRegion* region) const {
3470     HeapRegionRemSet* const rset = region->rem_set();
3471     return G1EagerReclaimHumongousObjectsWithStaleRefs
3472       ? rset->occupancy_less_or_equal_than(G1RSetSparseRegionEntries)
3473       : rset->is_empty();
3474   }
3475 
3476   bool is_typeArray_region(HeapRegion* region) const {
3477     return oop(region->bottom())->is_typeArray();
3478   }
3479 
3480   bool humongous_region_is_candidate(G1CollectedHeap* heap, HeapRegion* region) const {
3481     assert(region->is_starts_humongous(), "Must start a humongous object");
3482 
3483     // Candidate selection must satisfy the following constraints
3484     // while concurrent marking is in progress:
3485     //
3486     // * In order to maintain SATB invariants, an object must not be
3487     // reclaimed if it was allocated before the start of marking and
3488     // has not had its references scanned.  Such an object must have
3489     // its references (including type metadata) scanned to ensure no
3490     // live objects are missed by the marking process.  Objects
3491     // allocated after the start of concurrent marking don't need to
3492     // be scanned.
3493     //
3494     // * An object must not be reclaimed if it is on the concurrent
3495     // mark stack.  Objects allocated after the start of concurrent
3496     // marking are never pushed on the mark stack.
3497     //
3498     // Nominating only objects allocated after the start of concurrent
3499     // marking is sufficient to meet both constraints.  This may miss
3500     // some objects that satisfy the constraints, but the marking data
3501     // structures don't support efficiently performing the needed
3502     // additional tests or scrubbing of the mark stack.
3503     //
3504     // However, we presently only nominate is_typeArray() objects.
3505     // A humongous object containing references induces remembered
3506     // set entries on other regions.  In order to reclaim such an
3507     // object, those remembered sets would need to be cleaned up.
3508     //
3509     // We also treat is_typeArray() objects specially, allowing them
3510     // to be reclaimed even if allocated before the start of
3511     // concurrent mark.  For this we rely on mark stack insertion to
3512     // exclude is_typeArray() objects, preventing reclaiming an object
3513     // that is in the mark stack.  We also rely on the metadata for
3514     // such objects to be built-in and so ensured to be kept live.
3515     // Frequent allocation and drop of large binary blobs is an
3516     // important use case for eager reclaim, and this special handling
3517     // may reduce needed headroom.
3518 
3519     return is_typeArray_region(region) && is_remset_small(region);
3520   }
3521 
3522  public:
3523   RegisterHumongousWithInCSetFastTestClosure()
3524   : _total_humongous(0),
3525     _candidate_humongous(0),
3526     _dcq(&JavaThread::dirty_card_queue_set()) {
3527   }
3528 
3529   virtual bool doHeapRegion(HeapRegion* r) {
3530     if (!r->is_starts_humongous()) {
3531       return false;
3532     }
3533     G1CollectedHeap* g1h = G1CollectedHeap::heap();
3534 
3535     bool is_candidate = humongous_region_is_candidate(g1h, r);
3536     uint rindex = r->hrm_index();
3537     g1h->set_humongous_reclaim_candidate(rindex, is_candidate);
3538     if (is_candidate) {
3539       _candidate_humongous++;
3540       g1h->register_humongous_region_with_cset(rindex);
3541       // Is_candidate already filters out humongous object with large remembered sets.
3542       // If we have a humongous object with a few remembered sets, we simply flush these
3543       // remembered set entries into the DCQS. That will result in automatic
3544       // re-evaluation of their remembered set entries during the following evacuation
3545       // phase.
3546       if (!r->rem_set()->is_empty()) {
3547         guarantee(r->rem_set()->occupancy_less_or_equal_than(G1RSetSparseRegionEntries),
3548                   "Found a not-small remembered set here. This is inconsistent with previous assumptions.");
3549         G1SATBCardTableLoggingModRefBS* bs = g1h->g1_barrier_set();
3550         HeapRegionRemSetIterator hrrs(r->rem_set());
3551         size_t card_index;
3552         while (hrrs.has_next(card_index)) {
3553           jbyte* card_ptr = (jbyte*)bs->byte_for_index(card_index);
3554           // The remembered set might contain references to already freed
3555           // regions. Filter out such entries to avoid failing card table
3556           // verification.
3557           if (g1h->is_in_closed_subset(bs->addr_for(card_ptr))) {
3558             if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
3559               *card_ptr = CardTableModRefBS::dirty_card_val();
3560               _dcq.enqueue(card_ptr);
3561             }
3562           }
3563         }



3564         r->rem_set()->clear_locked();
3565       }
3566       assert(r->rem_set()->is_empty(), "At this point any humongous candidate remembered set must be empty.");
3567     }
3568     _total_humongous++;
3569 
3570     return false;
3571   }
3572 
3573   size_t total_humongous() const { return _total_humongous; }
3574   size_t candidate_humongous() const { return _candidate_humongous; }
3575 
3576   void flush_rem_set_entries() { _dcq.flush(); }
3577 };
3578 
3579 void G1CollectedHeap::register_humongous_regions_with_cset() {
3580   if (!G1EagerReclaimHumongousObjects) {
3581     g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0.0, 0, 0);
3582     return;
3583   }
3584   double time = os::elapsed_counter();
3585 
3586   // Collect reclaim candidate information and register candidates with cset.
3587   RegisterHumongousWithInCSetFastTestClosure cl;
3588   heap_region_iterate(&cl);
3589 
3590   time = ((double)(os::elapsed_counter() - time) / os::elapsed_frequency()) * 1000.0;
3591   g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(time,
3592                                                                   cl.total_humongous(),
3593                                                                   cl.candidate_humongous());
3594   _has_humongous_reclaim_candidates = cl.candidate_humongous() > 0;
3595 
3596   // Finally flush all remembered set entries to re-check into the global DCQS.
3597   cl.flush_rem_set_entries();
3598 }
3599 
3600 #ifdef ASSERT
3601 class VerifyCSetClosure: public HeapRegionClosure {
3602 public:
3603   bool doHeapRegion(HeapRegion* hr) {
3604     // Here we check that the CSet region's RSet is ready for parallel
3605     // iteration. The fields that we'll verify are only manipulated
3606     // when the region is part of a CSet and is collected. Afterwards,
3607     // we reset these fields when we clear the region's RSet (when the
3608     // region is freed) so they are ready when the region is
3609     // re-allocated. The only exception to this is if there's an
3610     // evacuation failure and instead of freeing the region we leave
3611     // it in the heap. In that case, we reset these fields during
3612     // evacuation failure handling.
3613     guarantee(hr->rem_set()->verify_ready_for_par_iteration(), "verification");
3614 
3615     // Here's a good place to add any other checks we'd like to
3616     // perform on CSet regions.
3617     return false;
3618   }
3619 };
3620 #endif // ASSERT
3621 
3622 uint G1CollectedHeap::num_task_queues() const {
3623   return _task_queues->size();
3624 }
3625 
3626 #if TASKQUEUE_STATS
3627 void G1CollectedHeap::print_taskqueue_stats_hdr(outputStream* const st) {
3628   st->print_raw_cr("GC Task Stats");
3629   st->print_raw("thr "); TaskQueueStats::print_header(1, st); st->cr();
3630   st->print_raw("--- "); TaskQueueStats::print_header(2, st); st->cr();
3631 }
3632 
3633 void G1CollectedHeap::print_taskqueue_stats(outputStream* const st) const {
3634   print_taskqueue_stats_hdr(st);
3635 
3636   TaskQueueStats totals;
3637   const uint n = num_task_queues();
3638   for (uint i = 0; i < n; ++i) {
3639     st->print("%3u ", i); task_queue(i)->stats.print(st); st->cr();
3640     totals += task_queue(i)->stats;
3641   }
3642   st->print_raw("tot "); totals.print(st); st->cr();
3643 
3644   DEBUG_ONLY(totals.verify());
3645 }
3646 
3647 void G1CollectedHeap::reset_taskqueue_stats() {
3648   const uint n = num_task_queues();
3649   for (uint i = 0; i < n; ++i) {
3650     task_queue(i)->stats.reset();
3651   }
3652 }
3653 #endif // TASKQUEUE_STATS
3654 
3655 void G1CollectedHeap::log_gc_header() {
3656   if (!G1Log::fine()) {
3657     return;
3658   }
3659 
3660   gclog_or_tty->gclog_stamp();
3661 
3662   GCCauseString gc_cause_str = GCCauseString("GC pause", gc_cause())
3663     .append(collector_state()->gcs_are_young() ? "(young)" : "(mixed)")
3664     .append(collector_state()->during_initial_mark_pause() ? " (initial-mark)" : "");
3665 
3666   gclog_or_tty->print("[%s", (const char*)gc_cause_str);
3667 }
3668 
3669 void G1CollectedHeap::log_gc_footer(double pause_time_sec) {
3670   if (!G1Log::fine()) {
3671     return;
3672   }
3673 
3674   if (G1Log::finer()) {
3675     if (evacuation_failed()) {
3676       gclog_or_tty->print(" (to-space exhausted)");
3677     }
3678     gclog_or_tty->print_cr(", %3.7f secs]", pause_time_sec);
3679     g1_policy()->print_phases(pause_time_sec);
3680     g1_policy()->print_detailed_heap_transition();
3681   } else {
3682     if (evacuation_failed()) {
3683       gclog_or_tty->print("--");
3684     }
3685     g1_policy()->print_heap_transition();
3686     gclog_or_tty->print_cr(", %3.7f secs]", pause_time_sec);
3687   }
3688   gclog_or_tty->flush();
3689 }
3690 
3691 void G1CollectedHeap::wait_for_root_region_scanning() {
3692   double scan_wait_start = os::elapsedTime();
3693   // We have to wait until the CM threads finish scanning the
3694   // root regions as it's the only way to ensure that all the
3695   // objects on them have been correctly scanned before we start
3696   // moving them during the GC.
3697   bool waited = _cm->root_regions()->wait_until_scan_finished();
3698   double wait_time_ms = 0.0;
3699   if (waited) {
3700     double scan_wait_end = os::elapsedTime();
3701     wait_time_ms = (scan_wait_end - scan_wait_start) * 1000.0;
3702   }
3703   g1_policy()->phase_times()->record_root_region_scan_wait_time(wait_time_ms);
3704 }
3705 
3706 bool
3707 G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ms) {
3708   assert_at_safepoint(true /* should_be_vm_thread */);
3709   guarantee(!is_gc_active(), "collection is not reentrant");
3710 
3711   if (GC_locker::check_active_before_gc()) {
3712     return false;
3713   }
3714 
3715   _gc_timer_stw->register_gc_start();
3716 
3717   GCIdMark gc_id_mark;
3718   _gc_tracer_stw->report_gc_start(gc_cause(), _gc_timer_stw->gc_start());
3719 
3720   SvcGCMarker sgcm(SvcGCMarker::MINOR);
3721   ResourceMark rm;
3722 
3723   wait_for_root_region_scanning();
3724 
3725   G1Log::update_level();
3726   print_heap_before_gc();
3727   trace_heap_before_gc(_gc_tracer_stw);
3728 
3729   verify_region_sets_optional();
3730   verify_dirty_young_regions();
3731 
3732   // This call will decide whether this pause is an initial-mark
3733   // pause. If it is, during_initial_mark_pause() will return true
3734   // for the duration of this pause.
3735   g1_policy()->decide_on_conc_mark_initiation();
3736 
3737   // We do not allow initial-mark to be piggy-backed on a mixed GC.
3738   assert(!collector_state()->during_initial_mark_pause() ||
3739           collector_state()->gcs_are_young(), "sanity");
3740 
3741   // We also do not allow mixed GCs during marking.
3742   assert(!collector_state()->mark_in_progress() || collector_state()->gcs_are_young(), "sanity");
3743 
3744   // Record whether this pause is an initial mark. When the current
3745   // thread has completed its logging output and it's safe to signal
3746   // the CM thread, the flag's value in the policy has been reset.
3747   bool should_start_conc_mark = collector_state()->during_initial_mark_pause();
3748 
3749   // Inner scope for scope based logging, timers, and stats collection
3750   {
3751     EvacuationInfo evacuation_info;
3752 
3753     if (collector_state()->during_initial_mark_pause()) {
3754       // We are about to start a marking cycle, so we increment the
3755       // full collection counter.
3756       increment_old_marking_cycles_started();
3757       register_concurrent_cycle_start(_gc_timer_stw->gc_start());
3758     }
3759 
3760     _gc_tracer_stw->report_yc_type(collector_state()->yc_type());
3761 
3762     TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty);
3763 
3764     uint active_workers = AdaptiveSizePolicy::calc_active_workers(workers()->total_workers(),
3765                                                                   workers()->active_workers(),
3766                                                                   Threads::number_of_non_daemon_threads());
3767     workers()->set_active_workers(active_workers);
3768 
3769     double pause_start_sec = os::elapsedTime();
3770     g1_policy()->note_gc_start(active_workers);
3771     log_gc_header();
3772 
3773     TraceCollectorStats tcs(g1mm()->incremental_collection_counters());
3774     TraceMemoryManagerStats tms(false /* fullGC */, gc_cause());
3775 
3776     // If the secondary_free_list is not empty, append it to the
3777     // free_list. No need to wait for the cleanup operation to finish;
3778     // the region allocation code will check the secondary_free_list
3779     // and wait if necessary. If the G1StressConcRegionFreeing flag is
3780     // set, skip this step so that the region allocation code has to
3781     // get entries from the secondary_free_list.
3782     if (!G1StressConcRegionFreeing) {
3783       append_secondary_free_list_if_not_empty_with_lock();
3784     }
3785 
3786     assert(check_young_list_well_formed(), "young list should be well formed");
3787 
3788     // Don't dynamically change the number of GC threads this early.  A value of
3789     // 0 is used to indicate serial work.  When parallel work is done,
3790     // it will be set.
3791 
3792     { // Call to jvmpi::post_class_unload_events must occur outside of active GC
3793       IsGCActiveMark x;
3794 
3795       gc_prologue(false);
3796       increment_total_collections(false /* full gc */);
3797       increment_gc_time_stamp();
3798 
3799       verify_before_gc();
3800 
3801       check_bitmaps("GC Start");
3802 
3803 #if defined(COMPILER2) || INCLUDE_JVMCI
3804       DerivedPointerTable::clear();
3805 #endif
3806 
3807       // Please see comment in g1CollectedHeap.hpp and
3808       // G1CollectedHeap::ref_processing_init() to see how
3809       // reference processing currently works in G1.
3810 
3811       // Enable discovery in the STW reference processor
3812       ref_processor_stw()->enable_discovery();
3813 
3814       {
3815         // We want to temporarily turn off discovery by the
3816         // CM ref processor, if necessary, and turn it back on
3817         // on again later if we do. Using a scoped
3818         // NoRefDiscovery object will do this.
3819         NoRefDiscovery no_cm_discovery(ref_processor_cm());
3820 
3821         // Forget the current alloc region (we might even choose it to be part
3822         // of the collection set!).
3823         _allocator->release_mutator_alloc_region();
3824 
3825         // We should call this after we retire the mutator alloc
3826         // region(s) so that all the ALLOC / RETIRE events are generated
3827         // before the start GC event.
3828         _hr_printer.start_gc(false /* full */, (size_t) total_collections());
3829 
3830         // This timing is only used by the ergonomics to handle our pause target.
3831         // It is unclear why this should not include the full pause. We will
3832         // investigate this in CR 7178365.
3833         //
3834         // Preserving the old comment here if that helps the investigation:
3835         //
3836         // The elapsed time induced by the start time below deliberately elides
3837         // the possible verification above.
3838         double sample_start_time_sec = os::elapsedTime();
3839 
3840         g1_policy()->record_collection_pause_start(sample_start_time_sec);
3841 
3842         if (collector_state()->during_initial_mark_pause()) {
3843           concurrent_mark()->checkpointRootsInitialPre();
3844         }
3845 
3846         double time_remaining_ms = g1_policy()->finalize_young_cset_part(target_pause_time_ms);
3847         g1_policy()->finalize_old_cset_part(time_remaining_ms);
3848 
3849         evacuation_info.set_collectionset_regions(g1_policy()->cset_region_length());





3850 
3851         register_humongous_regions_with_cset();
3852 
3853         assert(check_cset_fast_test(), "Inconsistency in the InCSetState table.");
3854 
3855         _cm->note_start_of_gc();
3856         // We call this after finalize_cset() to
3857         // ensure that the CSet has been finalized.
3858         _cm->verify_no_cset_oops();
3859 
3860         if (_hr_printer.is_active()) {
3861           HeapRegion* hr = g1_policy()->collection_set();
3862           while (hr != NULL) {
3863             _hr_printer.cset(hr);
3864             hr = hr->next_in_collection_set();
3865           }
3866         }
3867 
3868 #ifdef ASSERT
3869         VerifyCSetClosure cl;
3870         collection_set_iterate(&cl);
3871 #endif // ASSERT
3872 
3873         // Initialize the GC alloc regions.
3874         _allocator->init_gc_alloc_regions(evacuation_info);
3875 
3876         G1ParScanThreadStateSet per_thread_states(this, workers()->active_workers(), g1_policy()->young_cset_region_length());
3877         pre_evacuate_collection_set();
3878 
3879         // Actually do the work...
3880         evacuate_collection_set(evacuation_info, &per_thread_states);
3881 
3882         post_evacuate_collection_set(evacuation_info, &per_thread_states);
3883 
3884         const size_t* surviving_young_words = per_thread_states.surviving_young_words();
3885         free_collection_set(g1_policy()->collection_set(), evacuation_info, surviving_young_words);
3886 
3887         eagerly_reclaim_humongous_regions();
3888 
3889         g1_policy()->clear_collection_set();
3890 
3891         // Start a new incremental collection set for the next pause.
3892         g1_policy()->start_incremental_cset_building();
3893 
3894         clear_cset_fast_test();
3895 
3896         _young_list->reset_sampled_info();
3897 
3898         // Don't check the whole heap at this point as the
3899         // GC alloc regions from this pause have been tagged
3900         // as survivors and moved on to the survivor list.
3901         // Survivor regions will fail the !is_young() check.
3902         assert(check_young_list_empty(false /* check_heap */),
3903           "young list should be empty");
3904 
3905         g1_policy()->record_survivor_regions(_young_list->survivor_length(),
3906                                              _young_list->first_survivor_region(),
3907                                              _young_list->last_survivor_region());
3908 
3909         _young_list->reset_auxilary_lists();
3910 
3911         if (evacuation_failed()) {
3912           set_used(recalculate_used());
3913           if (_archive_allocator != NULL) {
3914             _archive_allocator->clear_used();
3915           }
3916           for (uint i = 0; i < ParallelGCThreads; i++) {
3917             if (_evacuation_failed_info_array[i].has_failed()) {
3918               _gc_tracer_stw->report_evacuation_failed(_evacuation_failed_info_array[i]);
3919             }
3920           }
3921         } else {
3922           // The "used" of the the collection set have already been subtracted
3923           // when they were freed.  Add in the bytes evacuated.
3924           increase_used(g1_policy()->bytes_copied_during_gc());
3925         }
3926 
3927         if (collector_state()->during_initial_mark_pause()) {
3928           // We have to do this before we notify the CM threads that
3929           // they can start working to make sure that all the
3930           // appropriate initialization is done on the CM object.
3931           concurrent_mark()->checkpointRootsInitialPost();
3932           collector_state()->set_mark_in_progress(true);
3933           // Note that we don't actually trigger the CM thread at
3934           // this point. We do that later when we're sure that
3935           // the current thread has completed its logging output.
3936         }
3937 
3938         allocate_dummy_regions();
3939 
3940         _allocator->init_mutator_alloc_region();
3941 
3942         {
3943           size_t expand_bytes = g1_policy()->expansion_amount();
3944           if (expand_bytes > 0) {
3945             size_t bytes_before = capacity();
3946             // No need for an ergo verbose message here,
3947             // expansion_amount() does this when it returns a value > 0.
3948             double expand_ms;
3949             if (!expand(expand_bytes, &expand_ms)) {
3950               // We failed to expand the heap. Cannot do anything about it.
3951             }
3952             g1_policy()->phase_times()->record_expand_heap_time(expand_ms);
3953           }
3954         }
3955 
3956         // We redo the verification but now wrt to the new CSet which
3957         // has just got initialized after the previous CSet was freed.
3958         _cm->verify_no_cset_oops();
3959         _cm->note_end_of_gc();
3960 
3961         // This timing is only used by the ergonomics to handle our pause target.
3962         // It is unclear why this should not include the full pause. We will
3963         // investigate this in CR 7178365.
3964         double sample_end_time_sec = os::elapsedTime();
3965         double pause_time_ms = (sample_end_time_sec - sample_start_time_sec) * MILLIUNITS;
3966         size_t total_cards_scanned = per_thread_states.total_cards_scanned();
3967         g1_policy()->record_collection_pause_end(pause_time_ms, total_cards_scanned);
3968 
3969         evacuation_info.set_collectionset_used_before(g1_policy()->collection_set_bytes_used_before());
3970         evacuation_info.set_bytes_copied(g1_policy()->bytes_copied_during_gc());
3971 
3972         MemoryService::track_memory_usage();
3973 
3974         // In prepare_for_verify() below we'll need to scan the deferred
3975         // update buffers to bring the RSets up-to-date if
3976         // G1HRRSFlushLogBuffersOnVerify has been set. While scanning
3977         // the update buffers we'll probably need to scan cards on the
3978         // regions we just allocated to (i.e., the GC alloc
3979         // regions). However, during the last GC we called
3980         // set_saved_mark() on all the GC alloc regions, so card
3981         // scanning might skip the [saved_mark_word()...top()] area of
3982         // those regions (i.e., the area we allocated objects into
3983         // during the last GC). But it shouldn't. Given that
3984         // saved_mark_word() is conditional on whether the GC time stamp
3985         // on the region is current or not, by incrementing the GC time
3986         // stamp here we invalidate all the GC time stamps on all the
3987         // regions and saved_mark_word() will simply return top() for
3988         // all the regions. This is a nicer way of ensuring this rather
3989         // than iterating over the regions and fixing them. In fact, the
3990         // GC time stamp increment here also ensures that
3991         // saved_mark_word() will return top() between pauses, i.e.,
3992         // during concurrent refinement. So we don't need the
3993         // is_gc_active() check to decided which top to use when
3994         // scanning cards (see CR 7039627).
3995         increment_gc_time_stamp();
3996 
3997         verify_after_gc();
3998         check_bitmaps("GC End");
3999 
4000         assert(!ref_processor_stw()->discovery_enabled(), "Postcondition");
4001         ref_processor_stw()->verify_no_references_recorded();
4002 
4003         // CM reference discovery will be re-enabled if necessary.
4004       }
4005 
4006       // We should do this after we potentially expand the heap so
4007       // that all the COMMIT events are generated before the end GC
4008       // event, and after we retire the GC alloc regions so that all
4009       // RETIRE events are generated before the end GC event.
4010       _hr_printer.end_gc(false /* full */, (size_t) total_collections());
4011 
4012 #ifdef TRACESPINNING
4013       ParallelTaskTerminator::print_termination_counts();
4014 #endif
4015 
4016       gc_epilogue(false);
4017     }
4018 
4019     // Print the remainder of the GC log output.
4020     log_gc_footer(os::elapsedTime() - pause_start_sec);
4021 
4022     // It is not yet to safe to tell the concurrent mark to
4023     // start as we have some optional output below. We don't want the
4024     // output from the concurrent mark thread interfering with this
4025     // logging output either.
4026 
4027     _hrm.verify_optional();
4028     verify_region_sets_optional();
4029 
4030     TASKQUEUE_STATS_ONLY(if (PrintTaskqueue) print_taskqueue_stats());
4031     TASKQUEUE_STATS_ONLY(reset_taskqueue_stats());
4032 
4033     print_heap_after_gc();
4034     trace_heap_after_gc(_gc_tracer_stw);
4035 
4036     // We must call G1MonitoringSupport::update_sizes() in the same scoping level
4037     // as an active TraceMemoryManagerStats object (i.e. before the destructor for the
4038     // TraceMemoryManagerStats is called) so that the G1 memory pools are updated
4039     // before any GC notifications are raised.
4040     g1mm()->update_sizes();
4041 
4042     _gc_tracer_stw->report_evacuation_info(&evacuation_info);
4043     _gc_tracer_stw->report_tenuring_threshold(_g1_policy->tenuring_threshold());
4044     _gc_timer_stw->register_gc_end();
4045     _gc_tracer_stw->report_gc_end(_gc_timer_stw->gc_end(), _gc_timer_stw->time_partitions());
4046   }
4047   // It should now be safe to tell the concurrent mark thread to start
4048   // without its logging output interfering with the logging output
4049   // that came from the pause.
4050 
4051   if (should_start_conc_mark) {
4052     // CAUTION: after the doConcurrentMark() call below,
4053     // the concurrent marking thread(s) could be running
4054     // concurrently with us. Make sure that anything after
4055     // this point does not assume that we are the only GC thread
4056     // running. Note: of course, the actual marking work will
4057     // not start until the safepoint itself is released in
4058     // SuspendibleThreadSet::desynchronize().
4059     doConcurrentMark();
4060   }
4061 
4062   return true;
4063 }
4064 
4065 void G1CollectedHeap::remove_self_forwarding_pointers() {
4066   double remove_self_forwards_start = os::elapsedTime();
4067 
4068   G1ParRemoveSelfForwardPtrsTask rsfp_task;
4069   workers()->run_task(&rsfp_task);
4070 
4071   // Now restore saved marks, if any.
4072   for (uint i = 0; i < ParallelGCThreads; i++) {
4073     OopAndMarkOopStack& cur = _preserved_objs[i];
4074     while (!cur.is_empty()) {
4075       OopAndMarkOop elem = cur.pop();
4076       elem.set_mark();
4077     }
4078     cur.clear(true);
4079   }
4080 
4081   g1_policy()->phase_times()->record_evac_fail_remove_self_forwards((os::elapsedTime() - remove_self_forwards_start) * 1000.0);
4082 }
4083 
4084 void G1CollectedHeap::preserve_mark_during_evac_failure(uint worker_id, oop obj, markOop m) {
4085   if (!_evacuation_failed) {
4086     _evacuation_failed = true;
4087   }
4088 
4089   _evacuation_failed_info_array[worker_id].register_copy_failure(obj->size());
4090 
4091   // We want to call the "for_promotion_failure" version only in the
4092   // case of a promotion failure.
4093   if (m->must_be_preserved_for_promotion_failure(obj)) {
4094     OopAndMarkOop elem(obj, m);
4095     _preserved_objs[worker_id].push(elem);
4096   }
4097 }
4098 
4099 class G1ParEvacuateFollowersClosure : public VoidClosure {
4100 private:
4101   double _start_term;
4102   double _term_time;
4103   size_t _term_attempts;
4104 
4105   void start_term_time() { _term_attempts++; _start_term = os::elapsedTime(); }
4106   void end_term_time() { _term_time += os::elapsedTime() - _start_term; }
4107 protected:
4108   G1CollectedHeap*              _g1h;
4109   G1ParScanThreadState*         _par_scan_state;
4110   RefToScanQueueSet*            _queues;
4111   ParallelTaskTerminator*       _terminator;
4112 
4113   G1ParScanThreadState*   par_scan_state() { return _par_scan_state; }
4114   RefToScanQueueSet*      queues()         { return _queues; }
4115   ParallelTaskTerminator* terminator()     { return _terminator; }
4116 
4117 public:
4118   G1ParEvacuateFollowersClosure(G1CollectedHeap* g1h,
4119                                 G1ParScanThreadState* par_scan_state,
4120                                 RefToScanQueueSet* queues,
4121                                 ParallelTaskTerminator* terminator)
4122     : _g1h(g1h), _par_scan_state(par_scan_state),
4123       _queues(queues), _terminator(terminator),
4124       _start_term(0.0), _term_time(0.0), _term_attempts(0) {}
4125 
4126   void do_void();
4127 
4128   double term_time() const { return _term_time; }
4129   size_t term_attempts() const { return _term_attempts; }
4130 
4131 private:
4132   inline bool offer_termination();
4133 };
4134 
4135 bool G1ParEvacuateFollowersClosure::offer_termination() {
4136   G1ParScanThreadState* const pss = par_scan_state();
4137   start_term_time();
4138   const bool res = terminator()->offer_termination();
4139   end_term_time();
4140   return res;
4141 }
4142 
4143 void G1ParEvacuateFollowersClosure::do_void() {
4144   G1ParScanThreadState* const pss = par_scan_state();
4145   pss->trim_queue();
4146   do {
4147     pss->steal_and_trim_queue(queues());
4148   } while (!offer_termination());
4149 }
4150 
4151 class G1ParTask : public AbstractGangTask {
4152 protected:
4153   G1CollectedHeap*         _g1h;
4154   G1ParScanThreadStateSet* _pss;
4155   RefToScanQueueSet*       _queues;
4156   G1RootProcessor*         _root_processor;
4157   ParallelTaskTerminator   _terminator;
4158   uint                     _n_workers;
4159 
4160 public:
4161   G1ParTask(G1CollectedHeap* g1h, G1ParScanThreadStateSet* per_thread_states, RefToScanQueueSet *task_queues, G1RootProcessor* root_processor, uint n_workers)
4162     : AbstractGangTask("G1 collection"),
4163       _g1h(g1h),
4164       _pss(per_thread_states),
4165       _queues(task_queues),
4166       _root_processor(root_processor),
4167       _terminator(n_workers, _queues),
4168       _n_workers(n_workers)
4169   {}
4170 
4171   void work(uint worker_id) {
4172     if (worker_id >= _n_workers) return;  // no work needed this round
4173 
4174     double start_sec = os::elapsedTime();
4175     _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::GCWorkerStart, worker_id, start_sec);
4176 
4177     {
4178       ResourceMark rm;
4179       HandleMark   hm;
4180 
4181       ReferenceProcessor*             rp = _g1h->ref_processor_stw();
4182 
4183       G1ParScanThreadState*           pss = _pss->state_for_worker(worker_id);
4184       pss->set_ref_processor(rp);
4185 
4186       double start_strong_roots_sec = os::elapsedTime();
4187 
4188       _root_processor->evacuate_roots(pss->closures(), worker_id);
4189 
4190       G1ParPushHeapRSClosure push_heap_rs_cl(_g1h, pss);
4191 
4192       // We pass a weak code blobs closure to the remembered set scanning because we want to avoid
4193       // treating the nmethods visited to act as roots for concurrent marking.
4194       // We only want to make sure that the oops in the nmethods are adjusted with regard to the
4195       // objects copied by the current evacuation.
4196       size_t cards_scanned = _g1h->g1_rem_set()->oops_into_collection_set_do(&push_heap_rs_cl,
4197                                                                              pss->closures()->weak_codeblobs(),
4198                                                                              worker_id);
4199 
4200       _pss->add_cards_scanned(worker_id, cards_scanned);
4201 
4202       double strong_roots_sec = os::elapsedTime() - start_strong_roots_sec;
4203 
4204       double term_sec = 0.0;
4205       size_t evac_term_attempts = 0;
4206       {
4207         double start = os::elapsedTime();
4208         G1ParEvacuateFollowersClosure evac(_g1h, pss, _queues, &_terminator);
4209         evac.do_void();
4210 
4211         evac_term_attempts = evac.term_attempts();
4212         term_sec = evac.term_time();
4213         double elapsed_sec = os::elapsedTime() - start;
4214         _g1h->g1_policy()->phase_times()->add_time_secs(G1GCPhaseTimes::ObjCopy, worker_id, elapsed_sec - term_sec);
4215         _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::Termination, worker_id, term_sec);
4216         _g1h->g1_policy()->phase_times()->record_thread_work_item(G1GCPhaseTimes::Termination, worker_id, evac_term_attempts);
4217       }
4218 
4219       assert(pss->queue_is_empty(), "should be empty");
4220 
4221       if (PrintTerminationStats) {
4222         MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
4223         size_t lab_waste;
4224         size_t lab_undo_waste;
4225         pss->waste(lab_waste, lab_undo_waste);
4226         _g1h->print_termination_stats(gclog_or_tty,
4227                                       worker_id,
4228                                       (os::elapsedTime() - start_sec) * 1000.0,   /* elapsed time */
4229                                       strong_roots_sec * 1000.0,                  /* strong roots time */
4230                                       term_sec * 1000.0,                          /* evac term time */
4231                                       evac_term_attempts,                         /* evac term attempts */
4232                                       lab_waste,                                  /* alloc buffer waste */
4233                                       lab_undo_waste                              /* undo waste */
4234                                       );
4235       }
4236 
4237       // Close the inner scope so that the ResourceMark and HandleMark
4238       // destructors are executed here and are included as part of the
4239       // "GC Worker Time".
4240     }
4241     _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::GCWorkerEnd, worker_id, os::elapsedTime());
4242   }
4243 };
4244 
4245 void G1CollectedHeap::print_termination_stats_hdr(outputStream* const st) {
4246   st->print_raw_cr("GC Termination Stats");
4247   st->print_raw_cr("     elapsed  --strong roots-- -------termination------- ------waste (KiB)------");
4248   st->print_raw_cr("thr     ms        ms      %        ms      %    attempts  total   alloc    undo");
4249   st->print_raw_cr("--- --------- --------- ------ --------- ------ -------- ------- ------- -------");
4250 }
4251 
4252 void G1CollectedHeap::print_termination_stats(outputStream* const st,
4253                                               uint worker_id,
4254                                               double elapsed_ms,
4255                                               double strong_roots_ms,
4256                                               double term_ms,
4257                                               size_t term_attempts,
4258                                               size_t alloc_buffer_waste,
4259                                               size_t undo_waste) const {
4260   st->print_cr("%3d %9.2f %9.2f %6.2f "
4261                "%9.2f %6.2f " SIZE_FORMAT_W(8) " "
4262                SIZE_FORMAT_W(7) " " SIZE_FORMAT_W(7) " " SIZE_FORMAT_W(7),
4263                worker_id, elapsed_ms, strong_roots_ms, strong_roots_ms * 100 / elapsed_ms,
4264                term_ms, term_ms * 100 / elapsed_ms, term_attempts,
4265                (alloc_buffer_waste + undo_waste) * HeapWordSize / K,
4266                alloc_buffer_waste * HeapWordSize / K,
4267                undo_waste * HeapWordSize / K);
4268 }
4269 
4270 class G1StringSymbolTableUnlinkTask : public AbstractGangTask {
4271 private:
4272   BoolObjectClosure* _is_alive;
4273   int _initial_string_table_size;
4274   int _initial_symbol_table_size;
4275 
4276   bool  _process_strings;
4277   int _strings_processed;
4278   int _strings_removed;
4279 
4280   bool  _process_symbols;
4281   int _symbols_processed;
4282   int _symbols_removed;
4283 
4284 public:
4285   G1StringSymbolTableUnlinkTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) :
4286     AbstractGangTask("String/Symbol Unlinking"),
4287     _is_alive(is_alive),
4288     _process_strings(process_strings), _strings_processed(0), _strings_removed(0),
4289     _process_symbols(process_symbols), _symbols_processed(0), _symbols_removed(0) {
4290 
4291     _initial_string_table_size = StringTable::the_table()->table_size();
4292     _initial_symbol_table_size = SymbolTable::the_table()->table_size();
4293     if (process_strings) {
4294       StringTable::clear_parallel_claimed_index();
4295     }
4296     if (process_symbols) {
4297       SymbolTable::clear_parallel_claimed_index();
4298     }
4299   }
4300 
4301   ~G1StringSymbolTableUnlinkTask() {
4302     guarantee(!_process_strings || StringTable::parallel_claimed_index() >= _initial_string_table_size,
4303               "claim value %d after unlink less than initial string table size %d",
4304               StringTable::parallel_claimed_index(), _initial_string_table_size);
4305     guarantee(!_process_symbols || SymbolTable::parallel_claimed_index() >= _initial_symbol_table_size,
4306               "claim value %d after unlink less than initial symbol table size %d",
4307               SymbolTable::parallel_claimed_index(), _initial_symbol_table_size);
4308 
4309     if (G1TraceStringSymbolTableScrubbing) {
4310       gclog_or_tty->print_cr("Cleaned string and symbol table, "
4311                              "strings: " SIZE_FORMAT " processed, " SIZE_FORMAT " removed, "
4312                              "symbols: " SIZE_FORMAT " processed, " SIZE_FORMAT " removed",
4313                              strings_processed(), strings_removed(),
4314                              symbols_processed(), symbols_removed());
4315     }
4316   }
4317 
4318   void work(uint worker_id) {
4319     int strings_processed = 0;
4320     int strings_removed = 0;
4321     int symbols_processed = 0;
4322     int symbols_removed = 0;
4323     if (_process_strings) {
4324       StringTable::possibly_parallel_unlink(_is_alive, &strings_processed, &strings_removed);
4325       Atomic::add(strings_processed, &_strings_processed);
4326       Atomic::add(strings_removed, &_strings_removed);
4327     }
4328     if (_process_symbols) {
4329       SymbolTable::possibly_parallel_unlink(&symbols_processed, &symbols_removed);
4330       Atomic::add(symbols_processed, &_symbols_processed);
4331       Atomic::add(symbols_removed, &_symbols_removed);
4332     }
4333   }
4334 
4335   size_t strings_processed() const { return (size_t)_strings_processed; }
4336   size_t strings_removed()   const { return (size_t)_strings_removed; }
4337 
4338   size_t symbols_processed() const { return (size_t)_symbols_processed; }
4339   size_t symbols_removed()   const { return (size_t)_symbols_removed; }
4340 };
4341 
4342 class G1CodeCacheUnloadingTask VALUE_OBJ_CLASS_SPEC {
4343 private:
4344   static Monitor* _lock;
4345 
4346   BoolObjectClosure* const _is_alive;
4347   const bool               _unloading_occurred;
4348   const uint               _num_workers;
4349 
4350   // Variables used to claim nmethods.
4351   nmethod* _first_nmethod;
4352   volatile nmethod* _claimed_nmethod;
4353 
4354   // The list of nmethods that need to be processed by the second pass.
4355   volatile nmethod* _postponed_list;
4356   volatile uint     _num_entered_barrier;
4357 
4358  public:
4359   G1CodeCacheUnloadingTask(uint num_workers, BoolObjectClosure* is_alive, bool unloading_occurred) :
4360       _is_alive(is_alive),
4361       _unloading_occurred(unloading_occurred),
4362       _num_workers(num_workers),
4363       _first_nmethod(NULL),
4364       _claimed_nmethod(NULL),
4365       _postponed_list(NULL),
4366       _num_entered_barrier(0)
4367   {
4368     nmethod::increase_unloading_clock();
4369     // Get first alive nmethod
4370     NMethodIterator iter = NMethodIterator();
4371     if(iter.next_alive()) {
4372       _first_nmethod = iter.method();
4373     }
4374     _claimed_nmethod = (volatile nmethod*)_first_nmethod;
4375   }
4376 
4377   ~G1CodeCacheUnloadingTask() {
4378     CodeCache::verify_clean_inline_caches();
4379 
4380     CodeCache::set_needs_cache_clean(false);
4381     guarantee(CodeCache::scavenge_root_nmethods() == NULL, "Must be");
4382 
4383     CodeCache::verify_icholder_relocations();
4384   }
4385 
4386  private:
4387   void add_to_postponed_list(nmethod* nm) {
4388       nmethod* old;
4389       do {
4390         old = (nmethod*)_postponed_list;
4391         nm->set_unloading_next(old);
4392       } while ((nmethod*)Atomic::cmpxchg_ptr(nm, &_postponed_list, old) != old);
4393   }
4394 
4395   void clean_nmethod(nmethod* nm) {
4396     bool postponed = nm->do_unloading_parallel(_is_alive, _unloading_occurred);
4397 
4398     if (postponed) {
4399       // This nmethod referred to an nmethod that has not been cleaned/unloaded yet.
4400       add_to_postponed_list(nm);
4401     }
4402 
4403     // Mark that this thread has been cleaned/unloaded.
4404     // After this call, it will be safe to ask if this nmethod was unloaded or not.
4405     nm->set_unloading_clock(nmethod::global_unloading_clock());
4406   }
4407 
4408   void clean_nmethod_postponed(nmethod* nm) {
4409     nm->do_unloading_parallel_postponed(_is_alive, _unloading_occurred);
4410   }
4411 
4412   static const int MaxClaimNmethods = 16;
4413 
4414   void claim_nmethods(nmethod** claimed_nmethods, int *num_claimed_nmethods) {
4415     nmethod* first;
4416     NMethodIterator last;
4417 
4418     do {
4419       *num_claimed_nmethods = 0;
4420 
4421       first = (nmethod*)_claimed_nmethod;
4422       last = NMethodIterator(first);
4423 
4424       if (first != NULL) {
4425 
4426         for (int i = 0; i < MaxClaimNmethods; i++) {
4427           if (!last.next_alive()) {
4428             break;
4429           }
4430           claimed_nmethods[i] = last.method();
4431           (*num_claimed_nmethods)++;
4432         }
4433       }
4434 
4435     } while ((nmethod*)Atomic::cmpxchg_ptr(last.method(), &_claimed_nmethod, first) != first);
4436   }
4437 
4438   nmethod* claim_postponed_nmethod() {
4439     nmethod* claim;
4440     nmethod* next;
4441 
4442     do {
4443       claim = (nmethod*)_postponed_list;
4444       if (claim == NULL) {
4445         return NULL;
4446       }
4447 
4448       next = claim->unloading_next();
4449 
4450     } while ((nmethod*)Atomic::cmpxchg_ptr(next, &_postponed_list, claim) != claim);
4451 
4452     return claim;
4453   }
4454 
4455  public:
4456   // Mark that we're done with the first pass of nmethod cleaning.
4457   void barrier_mark(uint worker_id) {
4458     MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag);
4459     _num_entered_barrier++;
4460     if (_num_entered_barrier == _num_workers) {
4461       ml.notify_all();
4462     }
4463   }
4464 
4465   // See if we have to wait for the other workers to
4466   // finish their first-pass nmethod cleaning work.
4467   void barrier_wait(uint worker_id) {
4468     if (_num_entered_barrier < _num_workers) {
4469       MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag);
4470       while (_num_entered_barrier < _num_workers) {
4471           ml.wait(Mutex::_no_safepoint_check_flag, 0, false);
4472       }
4473     }
4474   }
4475 
4476   // Cleaning and unloading of nmethods. Some work has to be postponed
4477   // to the second pass, when we know which nmethods survive.
4478   void work_first_pass(uint worker_id) {
4479     // The first nmethods is claimed by the first worker.
4480     if (worker_id == 0 && _first_nmethod != NULL) {
4481       clean_nmethod(_first_nmethod);
4482       _first_nmethod = NULL;
4483     }
4484 
4485     int num_claimed_nmethods;
4486     nmethod* claimed_nmethods[MaxClaimNmethods];
4487 
4488     while (true) {
4489       claim_nmethods(claimed_nmethods, &num_claimed_nmethods);
4490 
4491       if (num_claimed_nmethods == 0) {
4492         break;
4493       }
4494 
4495       for (int i = 0; i < num_claimed_nmethods; i++) {
4496         clean_nmethod(claimed_nmethods[i]);
4497       }
4498     }
4499   }
4500 
4501   void work_second_pass(uint worker_id) {
4502     nmethod* nm;
4503     // Take care of postponed nmethods.
4504     while ((nm = claim_postponed_nmethod()) != NULL) {
4505       clean_nmethod_postponed(nm);
4506     }
4507   }
4508 };
4509 
4510 Monitor* G1CodeCacheUnloadingTask::_lock = new Monitor(Mutex::leaf, "Code Cache Unload lock", false, Monitor::_safepoint_check_never);
4511 
4512 class G1KlassCleaningTask : public StackObj {
4513   BoolObjectClosure*                      _is_alive;
4514   volatile jint                           _clean_klass_tree_claimed;
4515   ClassLoaderDataGraphKlassIteratorAtomic _klass_iterator;
4516 
4517  public:
4518   G1KlassCleaningTask(BoolObjectClosure* is_alive) :
4519       _is_alive(is_alive),
4520       _clean_klass_tree_claimed(0),
4521       _klass_iterator() {
4522   }
4523 
4524  private:
4525   bool claim_clean_klass_tree_task() {
4526     if (_clean_klass_tree_claimed) {
4527       return false;
4528     }
4529 
4530     return Atomic::cmpxchg(1, (jint*)&_clean_klass_tree_claimed, 0) == 0;
4531   }
4532 
4533   InstanceKlass* claim_next_klass() {
4534     Klass* klass;
4535     do {
4536       klass =_klass_iterator.next_klass();
4537     } while (klass != NULL && !klass->is_instance_klass());
4538 
4539     // this can be null so don't call InstanceKlass::cast
4540     return static_cast<InstanceKlass*>(klass);
4541   }
4542 
4543 public:
4544 
4545   void clean_klass(InstanceKlass* ik) {
4546     ik->clean_weak_instanceklass_links(_is_alive);
4547   }
4548 
4549   void work() {
4550     ResourceMark rm;
4551 
4552     // One worker will clean the subklass/sibling klass tree.
4553     if (claim_clean_klass_tree_task()) {
4554       Klass::clean_subklass_tree(_is_alive);
4555     }
4556 
4557     // All workers will help cleaning the classes,
4558     InstanceKlass* klass;
4559     while ((klass = claim_next_klass()) != NULL) {
4560       clean_klass(klass);
4561     }
4562   }
4563 };
4564 
4565 // To minimize the remark pause times, the tasks below are done in parallel.
4566 class G1ParallelCleaningTask : public AbstractGangTask {
4567 private:
4568   G1StringSymbolTableUnlinkTask _string_symbol_task;
4569   G1CodeCacheUnloadingTask      _code_cache_task;
4570   G1KlassCleaningTask           _klass_cleaning_task;
4571 
4572 public:
4573   // The constructor is run in the VMThread.
4574   G1ParallelCleaningTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols, uint num_workers, bool unloading_occurred) :
4575       AbstractGangTask("Parallel Cleaning"),
4576       _string_symbol_task(is_alive, process_strings, process_symbols),
4577       _code_cache_task(num_workers, is_alive, unloading_occurred),
4578       _klass_cleaning_task(is_alive) {
4579   }
4580 
4581   // The parallel work done by all worker threads.
4582   void work(uint worker_id) {
4583     // Do first pass of code cache cleaning.
4584     _code_cache_task.work_first_pass(worker_id);
4585 
4586     // Let the threads mark that the first pass is done.
4587     _code_cache_task.barrier_mark(worker_id);
4588 
4589     // Clean the Strings and Symbols.
4590     _string_symbol_task.work(worker_id);
4591 
4592     // Wait for all workers to finish the first code cache cleaning pass.
4593     _code_cache_task.barrier_wait(worker_id);
4594 
4595     // Do the second code cache cleaning work, which realize on
4596     // the liveness information gathered during the first pass.
4597     _code_cache_task.work_second_pass(worker_id);
4598 
4599     // Clean all klasses that were not unloaded.
4600     _klass_cleaning_task.work();
4601   }
4602 };
4603 
4604 
4605 void G1CollectedHeap::parallel_cleaning(BoolObjectClosure* is_alive,
4606                                         bool process_strings,
4607                                         bool process_symbols,
4608                                         bool class_unloading_occurred) {
4609   uint n_workers = workers()->active_workers();
4610 
4611   G1ParallelCleaningTask g1_unlink_task(is_alive, process_strings, process_symbols,
4612                                         n_workers, class_unloading_occurred);
4613   workers()->run_task(&g1_unlink_task);
4614 }
4615 
4616 void G1CollectedHeap::unlink_string_and_symbol_table(BoolObjectClosure* is_alive,
4617                                                      bool process_strings, bool process_symbols) {
4618   {
4619     G1StringSymbolTableUnlinkTask g1_unlink_task(is_alive, process_strings, process_symbols);
4620     workers()->run_task(&g1_unlink_task);
4621   }
4622 
4623   if (G1StringDedup::is_enabled()) {
4624     G1StringDedup::unlink(is_alive);
4625   }
4626 }
4627 
4628 class G1RedirtyLoggedCardsTask : public AbstractGangTask {
4629  private:
4630   DirtyCardQueueSet* _queue;
4631  public:
4632   G1RedirtyLoggedCardsTask(DirtyCardQueueSet* queue) : AbstractGangTask("Redirty Cards"), _queue(queue) { }
4633 
4634   virtual void work(uint worker_id) {
4635     G1GCPhaseTimes* phase_times = G1CollectedHeap::heap()->g1_policy()->phase_times();
4636     G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::RedirtyCards, worker_id);
4637 
4638     RedirtyLoggedCardTableEntryClosure cl;
4639     _queue->par_apply_closure_to_all_completed_buffers(&cl);
4640 
4641     phase_times->record_thread_work_item(G1GCPhaseTimes::RedirtyCards, worker_id, cl.num_processed());
4642   }
4643 };
4644 
4645 void G1CollectedHeap::redirty_logged_cards() {
4646   double redirty_logged_cards_start = os::elapsedTime();
4647 
4648   G1RedirtyLoggedCardsTask redirty_task(&dirty_card_queue_set());
4649   dirty_card_queue_set().reset_for_par_iteration();
4650   workers()->run_task(&redirty_task);
4651 
4652   DirtyCardQueueSet& dcq = JavaThread::dirty_card_queue_set();
4653   dcq.merge_bufferlists(&dirty_card_queue_set());
4654   assert(dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed");
4655 
4656   g1_policy()->phase_times()->record_redirty_logged_cards_time_ms((os::elapsedTime() - redirty_logged_cards_start) * 1000.0);
4657 }
4658 
4659 // Weak Reference Processing support
4660 
4661 // An always "is_alive" closure that is used to preserve referents.
4662 // If the object is non-null then it's alive.  Used in the preservation
4663 // of referent objects that are pointed to by reference objects
4664 // discovered by the CM ref processor.
4665 class G1AlwaysAliveClosure: public BoolObjectClosure {
4666   G1CollectedHeap* _g1;
4667 public:
4668   G1AlwaysAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
4669   bool do_object_b(oop p) {
4670     if (p != NULL) {
4671       return true;
4672     }
4673     return false;
4674   }
4675 };
4676 
4677 bool G1STWIsAliveClosure::do_object_b(oop p) {
4678   // An object is reachable if it is outside the collection set,
4679   // or is inside and copied.
4680   return !_g1->is_in_cset(p) || p->is_forwarded();
4681 }
4682 
4683 // Non Copying Keep Alive closure
4684 class G1KeepAliveClosure: public OopClosure {
4685   G1CollectedHeap* _g1;
4686 public:
4687   G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
4688   void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
4689   void do_oop(oop* p) {
4690     oop obj = *p;
4691     assert(obj != NULL, "the caller should have filtered out NULL values");
4692 
4693     const InCSetState cset_state = _g1->in_cset_state(obj);
4694     if (!cset_state.is_in_cset_or_humongous()) {
4695       return;
4696     }
4697     if (cset_state.is_in_cset()) {
4698       assert( obj->is_forwarded(), "invariant" );
4699       *p = obj->forwardee();
4700     } else {
4701       assert(!obj->is_forwarded(), "invariant" );
4702       assert(cset_state.is_humongous(),
4703              "Only allowed InCSet state is IsHumongous, but is %d", cset_state.value());
4704       _g1->set_humongous_is_live(obj);
4705     }
4706   }
4707 };
4708 
4709 // Copying Keep Alive closure - can be called from both
4710 // serial and parallel code as long as different worker
4711 // threads utilize different G1ParScanThreadState instances
4712 // and different queues.
4713 
4714 class G1CopyingKeepAliveClosure: public OopClosure {
4715   G1CollectedHeap*         _g1h;
4716   OopClosure*              _copy_non_heap_obj_cl;
4717   G1ParScanThreadState*    _par_scan_state;
4718 
4719 public:
4720   G1CopyingKeepAliveClosure(G1CollectedHeap* g1h,
4721                             OopClosure* non_heap_obj_cl,
4722                             G1ParScanThreadState* pss):
4723     _g1h(g1h),
4724     _copy_non_heap_obj_cl(non_heap_obj_cl),
4725     _par_scan_state(pss)
4726   {}
4727 
4728   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
4729   virtual void do_oop(      oop* p) { do_oop_work(p); }
4730 
4731   template <class T> void do_oop_work(T* p) {
4732     oop obj = oopDesc::load_decode_heap_oop(p);
4733 
4734     if (_g1h->is_in_cset_or_humongous(obj)) {
4735       // If the referent object has been forwarded (either copied
4736       // to a new location or to itself in the event of an
4737       // evacuation failure) then we need to update the reference
4738       // field and, if both reference and referent are in the G1
4739       // heap, update the RSet for the referent.
4740       //
4741       // If the referent has not been forwarded then we have to keep
4742       // it alive by policy. Therefore we have copy the referent.
4743       //
4744       // If the reference field is in the G1 heap then we can push
4745       // on the PSS queue. When the queue is drained (after each
4746       // phase of reference processing) the object and it's followers
4747       // will be copied, the reference field set to point to the
4748       // new location, and the RSet updated. Otherwise we need to
4749       // use the the non-heap or metadata closures directly to copy
4750       // the referent object and update the pointer, while avoiding
4751       // updating the RSet.
4752 
4753       if (_g1h->is_in_g1_reserved(p)) {
4754         _par_scan_state->push_on_queue(p);
4755       } else {
4756         assert(!Metaspace::contains((const void*)p),
4757                "Unexpectedly found a pointer from metadata: " PTR_FORMAT, p2i(p));
4758         _copy_non_heap_obj_cl->do_oop(p);
4759       }
4760     }
4761   }
4762 };
4763 
4764 // Serial drain queue closure. Called as the 'complete_gc'
4765 // closure for each discovered list in some of the
4766 // reference processing phases.
4767 
4768 class G1STWDrainQueueClosure: public VoidClosure {
4769 protected:
4770   G1CollectedHeap* _g1h;
4771   G1ParScanThreadState* _par_scan_state;
4772 
4773   G1ParScanThreadState*   par_scan_state() { return _par_scan_state; }
4774 
4775 public:
4776   G1STWDrainQueueClosure(G1CollectedHeap* g1h, G1ParScanThreadState* pss) :
4777     _g1h(g1h),
4778     _par_scan_state(pss)
4779   { }
4780 
4781   void do_void() {
4782     G1ParScanThreadState* const pss = par_scan_state();
4783     pss->trim_queue();
4784   }
4785 };
4786 
4787 // Parallel Reference Processing closures
4788 
4789 // Implementation of AbstractRefProcTaskExecutor for parallel reference
4790 // processing during G1 evacuation pauses.
4791 
4792 class G1STWRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
4793 private:
4794   G1CollectedHeap*          _g1h;
4795   G1ParScanThreadStateSet*  _pss;
4796   RefToScanQueueSet*        _queues;
4797   WorkGang*                 _workers;
4798   uint                      _active_workers;
4799 
4800 public:
4801   G1STWRefProcTaskExecutor(G1CollectedHeap* g1h,
4802                            G1ParScanThreadStateSet* per_thread_states,
4803                            WorkGang* workers,
4804                            RefToScanQueueSet *task_queues,
4805                            uint n_workers) :
4806     _g1h(g1h),
4807     _pss(per_thread_states),
4808     _queues(task_queues),
4809     _workers(workers),
4810     _active_workers(n_workers)
4811   {
4812     assert(n_workers > 0, "shouldn't call this otherwise");
4813   }
4814 
4815   // Executes the given task using concurrent marking worker threads.
4816   virtual void execute(ProcessTask& task);
4817   virtual void execute(EnqueueTask& task);
4818 };
4819 
4820 // Gang task for possibly parallel reference processing
4821 
4822 class G1STWRefProcTaskProxy: public AbstractGangTask {
4823   typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
4824   ProcessTask&     _proc_task;
4825   G1CollectedHeap* _g1h;
4826   G1ParScanThreadStateSet* _pss;
4827   RefToScanQueueSet* _task_queues;
4828   ParallelTaskTerminator* _terminator;
4829 
4830 public:
4831   G1STWRefProcTaskProxy(ProcessTask& proc_task,
4832                         G1CollectedHeap* g1h,
4833                         G1ParScanThreadStateSet* per_thread_states,
4834                         RefToScanQueueSet *task_queues,
4835                         ParallelTaskTerminator* terminator) :
4836     AbstractGangTask("Process reference objects in parallel"),
4837     _proc_task(proc_task),
4838     _g1h(g1h),
4839     _pss(per_thread_states),
4840     _task_queues(task_queues),
4841     _terminator(terminator)
4842   {}
4843 
4844   virtual void work(uint worker_id) {
4845     // The reference processing task executed by a single worker.
4846     ResourceMark rm;
4847     HandleMark   hm;
4848 
4849     G1STWIsAliveClosure is_alive(_g1h);
4850 
4851     G1ParScanThreadState*          pss = _pss->state_for_worker(worker_id);
4852     pss->set_ref_processor(NULL);
4853 
4854     // Keep alive closure.
4855     G1CopyingKeepAliveClosure keep_alive(_g1h, pss->closures()->raw_strong_oops(), pss);
4856 
4857     // Complete GC closure
4858     G1ParEvacuateFollowersClosure drain_queue(_g1h, pss, _task_queues, _terminator);
4859 
4860     // Call the reference processing task's work routine.
4861     _proc_task.work(worker_id, is_alive, keep_alive, drain_queue);
4862 
4863     // Note we cannot assert that the refs array is empty here as not all
4864     // of the processing tasks (specifically phase2 - pp2_work) execute
4865     // the complete_gc closure (which ordinarily would drain the queue) so
4866     // the queue may not be empty.
4867   }
4868 };
4869 
4870 // Driver routine for parallel reference processing.
4871 // Creates an instance of the ref processing gang
4872 // task and has the worker threads execute it.
4873 void G1STWRefProcTaskExecutor::execute(ProcessTask& proc_task) {
4874   assert(_workers != NULL, "Need parallel worker threads.");
4875 
4876   ParallelTaskTerminator terminator(_active_workers, _queues);
4877   G1STWRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _pss, _queues, &terminator);
4878 
4879   _workers->run_task(&proc_task_proxy);
4880 }
4881 
4882 // Gang task for parallel reference enqueueing.
4883 
4884 class G1STWRefEnqueueTaskProxy: public AbstractGangTask {
4885   typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
4886   EnqueueTask& _enq_task;
4887 
4888 public:
4889   G1STWRefEnqueueTaskProxy(EnqueueTask& enq_task) :
4890     AbstractGangTask("Enqueue reference objects in parallel"),
4891     _enq_task(enq_task)
4892   { }
4893 
4894   virtual void work(uint worker_id) {
4895     _enq_task.work(worker_id);
4896   }
4897 };
4898 
4899 // Driver routine for parallel reference enqueueing.
4900 // Creates an instance of the ref enqueueing gang
4901 // task and has the worker threads execute it.
4902 
4903 void G1STWRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
4904   assert(_workers != NULL, "Need parallel worker threads.");
4905 
4906   G1STWRefEnqueueTaskProxy enq_task_proxy(enq_task);
4907 
4908   _workers->run_task(&enq_task_proxy);
4909 }
4910 
4911 // End of weak reference support closures
4912 
4913 // Abstract task used to preserve (i.e. copy) any referent objects
4914 // that are in the collection set and are pointed to by reference
4915 // objects discovered by the CM ref processor.
4916 
4917 class G1ParPreserveCMReferentsTask: public AbstractGangTask {
4918 protected:
4919   G1CollectedHeap*         _g1h;
4920   G1ParScanThreadStateSet* _pss;
4921   RefToScanQueueSet*       _queues;
4922   ParallelTaskTerminator   _terminator;
4923   uint                     _n_workers;
4924 
4925 public:
4926   G1ParPreserveCMReferentsTask(G1CollectedHeap* g1h, G1ParScanThreadStateSet* per_thread_states, int workers, RefToScanQueueSet *task_queues) :
4927     AbstractGangTask("ParPreserveCMReferents"),
4928     _g1h(g1h),
4929     _pss(per_thread_states),
4930     _queues(task_queues),
4931     _terminator(workers, _queues),
4932     _n_workers(workers)
4933   { }
4934 
4935   void work(uint worker_id) {
4936     ResourceMark rm;
4937     HandleMark   hm;
4938 
4939     G1ParScanThreadState*          pss = _pss->state_for_worker(worker_id);
4940     pss->set_ref_processor(NULL);
4941     assert(pss->queue_is_empty(), "both queue and overflow should be empty");
4942 
4943     // Is alive closure
4944     G1AlwaysAliveClosure always_alive(_g1h);
4945 
4946     // Copying keep alive closure. Applied to referent objects that need
4947     // to be copied.
4948     G1CopyingKeepAliveClosure keep_alive(_g1h, pss->closures()->raw_strong_oops(), pss);
4949 
4950     ReferenceProcessor* rp = _g1h->ref_processor_cm();
4951 
4952     uint limit = ReferenceProcessor::number_of_subclasses_of_ref() * rp->max_num_q();
4953     uint stride = MIN2(MAX2(_n_workers, 1U), limit);
4954 
4955     // limit is set using max_num_q() - which was set using ParallelGCThreads.
4956     // So this must be true - but assert just in case someone decides to
4957     // change the worker ids.
4958     assert(worker_id < limit, "sanity");
4959     assert(!rp->discovery_is_atomic(), "check this code");
4960 
4961     // Select discovered lists [i, i+stride, i+2*stride,...,limit)
4962     for (uint idx = worker_id; idx < limit; idx += stride) {
4963       DiscoveredList& ref_list = rp->discovered_refs()[idx];
4964 
4965       DiscoveredListIterator iter(ref_list, &keep_alive, &always_alive);
4966       while (iter.has_next()) {
4967         // Since discovery is not atomic for the CM ref processor, we
4968         // can see some null referent objects.
4969         iter.load_ptrs(DEBUG_ONLY(true));
4970         oop ref = iter.obj();
4971 
4972         // This will filter nulls.
4973         if (iter.is_referent_alive()) {
4974           iter.make_referent_alive();
4975         }
4976         iter.move_to_next();
4977       }
4978     }
4979 
4980     // Drain the queue - which may cause stealing
4981     G1ParEvacuateFollowersClosure drain_queue(_g1h, pss, _queues, &_terminator);
4982     drain_queue.do_void();
4983     // Allocation buffers were retired at the end of G1ParEvacuateFollowersClosure
4984     assert(pss->queue_is_empty(), "should be");
4985   }
4986 };
4987 
4988 // Weak Reference processing during an evacuation pause (part 1).
4989 void G1CollectedHeap::process_discovered_references(G1ParScanThreadStateSet* per_thread_states) {
4990   double ref_proc_start = os::elapsedTime();
4991 
4992   ReferenceProcessor* rp = _ref_processor_stw;
4993   assert(rp->discovery_enabled(), "should have been enabled");
4994 
4995   // Any reference objects, in the collection set, that were 'discovered'
4996   // by the CM ref processor should have already been copied (either by
4997   // applying the external root copy closure to the discovered lists, or
4998   // by following an RSet entry).
4999   //
5000   // But some of the referents, that are in the collection set, that these
5001   // reference objects point to may not have been copied: the STW ref
5002   // processor would have seen that the reference object had already
5003   // been 'discovered' and would have skipped discovering the reference,
5004   // but would not have treated the reference object as a regular oop.
5005   // As a result the copy closure would not have been applied to the
5006   // referent object.
5007   //
5008   // We need to explicitly copy these referent objects - the references
5009   // will be processed at the end of remarking.
5010   //
5011   // We also need to do this copying before we process the reference
5012   // objects discovered by the STW ref processor in case one of these
5013   // referents points to another object which is also referenced by an
5014   // object discovered by the STW ref processor.
5015 
5016   uint no_of_gc_workers = workers()->active_workers();
5017 
5018   G1ParPreserveCMReferentsTask keep_cm_referents(this,
5019                                                  per_thread_states,
5020                                                  no_of_gc_workers,
5021                                                  _task_queues);
5022 
5023   workers()->run_task(&keep_cm_referents);
5024 
5025   // Closure to test whether a referent is alive.
5026   G1STWIsAliveClosure is_alive(this);
5027 
5028   // Even when parallel reference processing is enabled, the processing
5029   // of JNI refs is serial and performed serially by the current thread
5030   // rather than by a worker. The following PSS will be used for processing
5031   // JNI refs.
5032 
5033   // Use only a single queue for this PSS.
5034   G1ParScanThreadState*          pss = per_thread_states->state_for_worker(0);
5035   pss->set_ref_processor(NULL);
5036   assert(pss->queue_is_empty(), "pre-condition");
5037 
5038   // Keep alive closure.
5039   G1CopyingKeepAliveClosure keep_alive(this, pss->closures()->raw_strong_oops(), pss);
5040 
5041   // Serial Complete GC closure
5042   G1STWDrainQueueClosure drain_queue(this, pss);
5043 
5044   // Setup the soft refs policy...
5045   rp->setup_policy(false);
5046 
5047   ReferenceProcessorStats stats;
5048   if (!rp->processing_is_mt()) {
5049     // Serial reference processing...
5050     stats = rp->process_discovered_references(&is_alive,
5051                                               &keep_alive,
5052                                               &drain_queue,
5053                                               NULL,
5054                                               _gc_timer_stw);
5055   } else {
5056     // Parallel reference processing
5057     assert(rp->num_q() == no_of_gc_workers, "sanity");
5058     assert(no_of_gc_workers <= rp->max_num_q(), "sanity");
5059 
5060     G1STWRefProcTaskExecutor par_task_executor(this, per_thread_states, workers(), _task_queues, no_of_gc_workers);
5061     stats = rp->process_discovered_references(&is_alive,
5062                                               &keep_alive,
5063                                               &drain_queue,
5064                                               &par_task_executor,
5065                                               _gc_timer_stw);
5066   }
5067 
5068   _gc_tracer_stw->report_gc_reference_stats(stats);
5069 
5070   // We have completed copying any necessary live referent objects.
5071   assert(pss->queue_is_empty(), "both queue and overflow should be empty");
5072 
5073   double ref_proc_time = os::elapsedTime() - ref_proc_start;
5074   g1_policy()->phase_times()->record_ref_proc_time(ref_proc_time * 1000.0);
5075 }
5076 
5077 // Weak Reference processing during an evacuation pause (part 2).
5078 void G1CollectedHeap::enqueue_discovered_references(G1ParScanThreadStateSet* per_thread_states) {
5079   double ref_enq_start = os::elapsedTime();
5080 
5081   ReferenceProcessor* rp = _ref_processor_stw;
5082   assert(!rp->discovery_enabled(), "should have been disabled as part of processing");
5083 
5084   // Now enqueue any remaining on the discovered lists on to
5085   // the pending list.
5086   if (!rp->processing_is_mt()) {
5087     // Serial reference processing...
5088     rp->enqueue_discovered_references();
5089   } else {
5090     // Parallel reference enqueueing
5091 
5092     uint n_workers = workers()->active_workers();
5093 
5094     assert(rp->num_q() == n_workers, "sanity");
5095     assert(n_workers <= rp->max_num_q(), "sanity");
5096 
5097     G1STWRefProcTaskExecutor par_task_executor(this, per_thread_states, workers(), _task_queues, n_workers);
5098     rp->enqueue_discovered_references(&par_task_executor);
5099   }
5100 
5101   rp->verify_no_references_recorded();
5102   assert(!rp->discovery_enabled(), "should have been disabled");
5103 
5104   // FIXME
5105   // CM's reference processing also cleans up the string and symbol tables.
5106   // Should we do that here also? We could, but it is a serial operation
5107   // and could significantly increase the pause time.
5108 
5109   double ref_enq_time = os::elapsedTime() - ref_enq_start;
5110   g1_policy()->phase_times()->record_ref_enq_time(ref_enq_time * 1000.0);
5111 }
5112 
5113 void G1CollectedHeap::pre_evacuate_collection_set() {
5114   _expand_heap_after_alloc_failure = true;
5115   _evacuation_failed = false;
5116 
5117   // Disable the hot card cache.
5118   G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
5119   hot_card_cache->reset_hot_cache_claimed_index();
5120   hot_card_cache->set_use_cache(false);
5121 
5122 }
5123 
5124 void G1CollectedHeap::evacuate_collection_set(EvacuationInfo& evacuation_info, G1ParScanThreadStateSet* per_thread_states) {
5125   g1_rem_set()->prepare_for_oops_into_collection_set_do();
5126 
5127   // Should G1EvacuationFailureALot be in effect for this GC?
5128   NOT_PRODUCT(set_evacuation_failure_alot_for_current_gc();)
5129 
5130   assert(dirty_card_queue_set().completed_buffers_num() == 0, "Should be empty");
5131   double start_par_time_sec = os::elapsedTime();
5132   double end_par_time_sec;
5133 
5134   {
5135     const uint n_workers = workers()->active_workers();
5136     G1RootProcessor root_processor(this, n_workers);
5137     G1ParTask g1_par_task(this, per_thread_states, _task_queues, &root_processor, n_workers);
5138     // InitialMark needs claim bits to keep track of the marked-through CLDs.
5139     if (collector_state()->during_initial_mark_pause()) {
5140       ClassLoaderDataGraph::clear_claimed_marks();
5141     }
5142 
5143     // The individual threads will set their evac-failure closures.
5144     if (PrintTerminationStats) {
5145       print_termination_stats_hdr(gclog_or_tty);
5146     }
5147 
5148     workers()->run_task(&g1_par_task);
5149     end_par_time_sec = os::elapsedTime();
5150 
5151     // Closing the inner scope will execute the destructor
5152     // for the G1RootProcessor object. We record the current
5153     // elapsed time before closing the scope so that time
5154     // taken for the destructor is NOT included in the
5155     // reported parallel time.
5156   }
5157 
5158   G1GCPhaseTimes* phase_times = g1_policy()->phase_times();
5159 
5160   double par_time_ms = (end_par_time_sec - start_par_time_sec) * 1000.0;
5161   phase_times->record_par_time(par_time_ms);
5162 
5163   double code_root_fixup_time_ms =
5164         (os::elapsedTime() - end_par_time_sec) * 1000.0;
5165   phase_times->record_code_root_fixup_time(code_root_fixup_time_ms);
5166 
5167   // Process any discovered reference objects - we have
5168   // to do this _before_ we retire the GC alloc regions
5169   // as we may have to copy some 'reachable' referent
5170   // objects (and their reachable sub-graphs) that were
5171   // not copied during the pause.
5172   process_discovered_references(per_thread_states);
5173 
5174   if (G1StringDedup::is_enabled()) {
5175     double fixup_start = os::elapsedTime();
5176 
5177     G1STWIsAliveClosure is_alive(this);
5178     G1KeepAliveClosure keep_alive(this);
5179     G1StringDedup::unlink_or_oops_do(&is_alive, &keep_alive, true, phase_times);
5180 
5181     double fixup_time_ms = (os::elapsedTime() - fixup_start) * 1000.0;
5182     phase_times->record_string_dedup_fixup_time(fixup_time_ms);
5183   }
5184 
5185   g1_rem_set()->cleanup_after_oops_into_collection_set_do();
5186 
5187   if (evacuation_failed()) {
5188     remove_self_forwarding_pointers();
5189 
5190     // Reset the G1EvacuationFailureALot counters and flags
5191     // Note: the values are reset only when an actual
5192     // evacuation failure occurs.
5193     NOT_PRODUCT(reset_evacuation_should_fail();)
5194   }
5195 
5196   // Enqueue any remaining references remaining on the STW
5197   // reference processor's discovered lists. We need to do
5198   // this after the card table is cleaned (and verified) as
5199   // the act of enqueueing entries on to the pending list
5200   // will log these updates (and dirty their associated
5201   // cards). We need these updates logged to update any
5202   // RSets.
5203   enqueue_discovered_references(per_thread_states);
5204 }
5205 
5206 void G1CollectedHeap::post_evacuate_collection_set(EvacuationInfo& evacuation_info, G1ParScanThreadStateSet* per_thread_states) {
5207   _allocator->release_gc_alloc_regions(evacuation_info);
5208 
5209   per_thread_states->flush();
5210 
5211   record_obj_copy_mem_stats();
5212 
5213   _survivor_evac_stats.adjust_desired_plab_sz();
5214   _old_evac_stats.adjust_desired_plab_sz();
5215 
5216   // Reset and re-enable the hot card cache.
5217   // Note the counts for the cards in the regions in the
5218   // collection set are reset when the collection set is freed.
5219   G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache();
5220   hot_card_cache->reset_hot_cache();
5221   hot_card_cache->set_use_cache(true);
5222 
5223   purge_code_root_memory();
5224 
5225   redirty_logged_cards();
5226 #if defined(COMPILER2) || INCLUDE_JVMCI
5227   DerivedPointerTable::update_pointers();
5228 #endif
5229 }
5230 
5231 void G1CollectedHeap::record_obj_copy_mem_stats() {
5232   _gc_tracer_stw->report_evacuation_statistics(create_g1_evac_summary(&_survivor_evac_stats),
5233                                                create_g1_evac_summary(&_old_evac_stats));
5234 }
5235 
5236 void G1CollectedHeap::free_region(HeapRegion* hr,
5237                                   FreeRegionList* free_list,
5238                                   bool par,
5239                                   bool locked) {
5240   assert(!hr->is_free(), "the region should not be free");
5241   assert(!hr->is_empty(), "the region should not be empty");
5242   assert(_hrm.is_available(hr->hrm_index()), "region should be committed");
5243   assert(free_list != NULL, "pre-condition");
5244 
5245   if (G1VerifyBitmaps) {
5246     MemRegion mr(hr->bottom(), hr->end());
5247     concurrent_mark()->clearRangePrevBitmap(mr);
5248   }
5249 
5250   // Clear the card counts for this region.
5251   // Note: we only need to do this if the region is not young
5252   // (since we don't refine cards in young regions).
5253   if (!hr->is_young()) {
5254     _cg1r->hot_card_cache()->reset_card_counts(hr);
5255   }
5256   hr->hr_clear(par, true /* clear_space */, locked /* locked */);
5257   free_list->add_ordered(hr);
5258 }
5259 
5260 void G1CollectedHeap::free_humongous_region(HeapRegion* hr,
5261                                             FreeRegionList* free_list,
5262                                             bool par) {
5263   assert(hr->is_humongous(), "this is only for humongous regions");
5264   assert(free_list != NULL, "pre-condition");
5265   hr->clear_humongous();
5266   free_region(hr, free_list, par);
5267 }
5268 
5269 void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed,
5270                                            const HeapRegionSetCount& humongous_regions_removed) {
5271   if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) {
5272     MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag);
5273     _old_set.bulk_remove(old_regions_removed);
5274     _humongous_set.bulk_remove(humongous_regions_removed);
5275   }
5276 
5277 }
5278 
5279 void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) {
5280   assert(list != NULL, "list can't be null");
5281   if (!list->is_empty()) {
5282     MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag);
5283     _hrm.insert_list_into_free_list(list);
5284   }
5285 }
5286 
5287 void G1CollectedHeap::decrement_summary_bytes(size_t bytes) {
5288   decrease_used(bytes);
5289 }
5290 
5291 class G1ParCleanupCTTask : public AbstractGangTask {
5292   G1SATBCardTableModRefBS* _ct_bs;
5293   G1CollectedHeap* _g1h;
5294   HeapRegion* volatile _su_head;
5295 public:
5296   G1ParCleanupCTTask(G1SATBCardTableModRefBS* ct_bs,
5297                      G1CollectedHeap* g1h) :
5298     AbstractGangTask("G1 Par Cleanup CT Task"),
5299     _ct_bs(ct_bs), _g1h(g1h) { }
5300 
5301   void work(uint worker_id) {
5302     HeapRegion* r;
5303     while (r = _g1h->pop_dirty_cards_region()) {
5304       clear_cards(r);
5305     }
5306   }
5307 
5308   void clear_cards(HeapRegion* r) {
5309     // Cards of the survivors should have already been dirtied.
5310     if (!r->is_survivor()) {
5311       _ct_bs->clear(MemRegion(r->bottom(), r->end()));
5312     }
5313   }
5314 };
5315 
5316 #ifndef PRODUCT
5317 class G1VerifyCardTableCleanup: public HeapRegionClosure {
5318   G1CollectedHeap* _g1h;
5319   G1SATBCardTableModRefBS* _ct_bs;
5320 public:
5321   G1VerifyCardTableCleanup(G1CollectedHeap* g1h, G1SATBCardTableModRefBS* ct_bs)
5322     : _g1h(g1h), _ct_bs(ct_bs) { }
5323   virtual bool doHeapRegion(HeapRegion* r) {
5324     if (r->is_survivor()) {
5325       _g1h->verify_dirty_region(r);
5326     } else {
5327       _g1h->verify_not_dirty_region(r);
5328     }
5329     return false;
5330   }
5331 };
5332 
5333 void G1CollectedHeap::verify_not_dirty_region(HeapRegion* hr) {
5334   // All of the region should be clean.
5335   G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
5336   MemRegion mr(hr->bottom(), hr->end());
5337   ct_bs->verify_not_dirty_region(mr);
5338 }
5339 
5340 void G1CollectedHeap::verify_dirty_region(HeapRegion* hr) {
5341   // We cannot guarantee that [bottom(),end()] is dirty.  Threads
5342   // dirty allocated blocks as they allocate them. The thread that
5343   // retires each region and replaces it with a new one will do a
5344   // maximal allocation to fill in [pre_dummy_top(),end()] but will
5345   // not dirty that area (one less thing to have to do while holding
5346   // a lock). So we can only verify that [bottom(),pre_dummy_top()]
5347   // is dirty.
5348   G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
5349   MemRegion mr(hr->bottom(), hr->pre_dummy_top());
5350   if (hr->is_young()) {
5351     ct_bs->verify_g1_young_region(mr);
5352   } else {
5353     ct_bs->verify_dirty_region(mr);
5354   }
5355 }
5356 
5357 void G1CollectedHeap::verify_dirty_young_list(HeapRegion* head) {
5358   G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
5359   for (HeapRegion* hr = head; hr != NULL; hr = hr->get_next_young_region()) {
5360     verify_dirty_region(hr);
5361   }
5362 }
5363 
5364 void G1CollectedHeap::verify_dirty_young_regions() {
5365   verify_dirty_young_list(_young_list->first_region());
5366 }
5367 
5368 bool G1CollectedHeap::verify_no_bits_over_tams(const char* bitmap_name, CMBitMapRO* bitmap,
5369                                                HeapWord* tams, HeapWord* end) {
5370   guarantee(tams <= end,
5371             "tams: " PTR_FORMAT " end: " PTR_FORMAT, p2i(tams), p2i(end));
5372   HeapWord* result = bitmap->getNextMarkedWordAddress(tams, end);
5373   if (result < end) {
5374     gclog_or_tty->cr();
5375     gclog_or_tty->print_cr("## wrong marked address on %s bitmap: " PTR_FORMAT,
5376                            bitmap_name, p2i(result));
5377     gclog_or_tty->print_cr("## %s tams: " PTR_FORMAT " end: " PTR_FORMAT,
5378                            bitmap_name, p2i(tams), p2i(end));
5379     return false;
5380   }
5381   return true;
5382 }
5383 
5384 bool G1CollectedHeap::verify_bitmaps(const char* caller, HeapRegion* hr) {
5385   CMBitMapRO* prev_bitmap = concurrent_mark()->prevMarkBitMap();
5386   CMBitMapRO* next_bitmap = (CMBitMapRO*) concurrent_mark()->nextMarkBitMap();
5387 
5388   HeapWord* bottom = hr->bottom();
5389   HeapWord* ptams  = hr->prev_top_at_mark_start();
5390   HeapWord* ntams  = hr->next_top_at_mark_start();
5391   HeapWord* end    = hr->end();
5392 
5393   bool res_p = verify_no_bits_over_tams("prev", prev_bitmap, ptams, end);
5394 
5395   bool res_n = true;
5396   // We reset mark_in_progress() before we reset _cmThread->in_progress() and in this window
5397   // we do the clearing of the next bitmap concurrently. Thus, we can not verify the bitmap
5398   // if we happen to be in that state.
5399   if (collector_state()->mark_in_progress() || !_cmThread->in_progress()) {
5400     res_n = verify_no_bits_over_tams("next", next_bitmap, ntams, end);
5401   }
5402   if (!res_p || !res_n) {
5403     gclog_or_tty->print_cr("#### Bitmap verification failed for " HR_FORMAT,
5404                            HR_FORMAT_PARAMS(hr));
5405     gclog_or_tty->print_cr("#### Caller: %s", caller);
5406     return false;
5407   }
5408   return true;
5409 }
5410 
5411 void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) {
5412   if (!G1VerifyBitmaps) return;
5413 
5414   guarantee(verify_bitmaps(caller, hr), "bitmap verification");
5415 }
5416 
5417 class G1VerifyBitmapClosure : public HeapRegionClosure {
5418 private:
5419   const char* _caller;
5420   G1CollectedHeap* _g1h;
5421   bool _failures;
5422 
5423 public:
5424   G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) :
5425     _caller(caller), _g1h(g1h), _failures(false) { }
5426 
5427   bool failures() { return _failures; }
5428 
5429   virtual bool doHeapRegion(HeapRegion* hr) {
5430     bool result = _g1h->verify_bitmaps(_caller, hr);
5431     if (!result) {
5432       _failures = true;
5433     }
5434     return false;
5435   }
5436 };
5437 
5438 void G1CollectedHeap::check_bitmaps(const char* caller) {
5439   if (!G1VerifyBitmaps) return;
5440 
5441   G1VerifyBitmapClosure cl(caller, this);
5442   heap_region_iterate(&cl);
5443   guarantee(!cl.failures(), "bitmap verification");
5444 }
5445 
5446 class G1CheckCSetFastTableClosure : public HeapRegionClosure {
5447  private:
5448   bool _failures;
5449  public:
5450   G1CheckCSetFastTableClosure() : HeapRegionClosure(), _failures(false) { }
5451 
5452   virtual bool doHeapRegion(HeapRegion* hr) {
5453     uint i = hr->hrm_index();
5454     InCSetState cset_state = (InCSetState) G1CollectedHeap::heap()->_in_cset_fast_test.get_by_index(i);
5455     if (hr->is_humongous()) {
5456       if (hr->in_collection_set()) {
5457         gclog_or_tty->print_cr("\n## humongous region %u in CSet", i);
5458         _failures = true;
5459         return true;
5460       }
5461       if (cset_state.is_in_cset()) {
5462         gclog_or_tty->print_cr("\n## inconsistent cset state %d for humongous region %u", cset_state.value(), i);
5463         _failures = true;
5464         return true;
5465       }
5466       if (hr->is_continues_humongous() && cset_state.is_humongous()) {
5467         gclog_or_tty->print_cr("\n## inconsistent cset state %d for continues humongous region %u", cset_state.value(), i);
5468         _failures = true;
5469         return true;
5470       }
5471     } else {
5472       if (cset_state.is_humongous()) {
5473         gclog_or_tty->print_cr("\n## inconsistent cset state %d for non-humongous region %u", cset_state.value(), i);
5474         _failures = true;
5475         return true;
5476       }
5477       if (hr->in_collection_set() != cset_state.is_in_cset()) {
5478         gclog_or_tty->print_cr("\n## in CSet %d / cset state %d inconsistency for region %u",
5479                                hr->in_collection_set(), cset_state.value(), i);
5480         _failures = true;
5481         return true;
5482       }
5483       if (cset_state.is_in_cset()) {
5484         if (hr->is_young() != (cset_state.is_young())) {
5485           gclog_or_tty->print_cr("\n## is_young %d / cset state %d inconsistency for region %u",
5486                                  hr->is_young(), cset_state.value(), i);
5487           _failures = true;
5488           return true;
5489         }
5490         if (hr->is_old() != (cset_state.is_old())) {
5491           gclog_or_tty->print_cr("\n## is_old %d / cset state %d inconsistency for region %u",
5492                                  hr->is_old(), cset_state.value(), i);
5493           _failures = true;
5494           return true;
5495         }
5496       }
5497     }
5498     return false;
5499   }
5500 
5501   bool failures() const { return _failures; }
5502 };
5503 
5504 bool G1CollectedHeap::check_cset_fast_test() {
5505   G1CheckCSetFastTableClosure cl;
5506   _hrm.iterate(&cl);
5507   return !cl.failures();
5508 }
5509 #endif // PRODUCT
5510 
5511 void G1CollectedHeap::cleanUpCardTable() {
5512   G1SATBCardTableModRefBS* ct_bs = g1_barrier_set();
5513   double start = os::elapsedTime();
5514 
5515   {
5516     // Iterate over the dirty cards region list.
5517     G1ParCleanupCTTask cleanup_task(ct_bs, this);
5518 
5519     workers()->run_task(&cleanup_task);
5520 #ifndef PRODUCT
5521     if (G1VerifyCTCleanup || VerifyAfterGC) {
5522       G1VerifyCardTableCleanup cleanup_verifier(this, ct_bs);
5523       heap_region_iterate(&cleanup_verifier);
5524     }
5525 #endif
5526   }
5527 
5528   double elapsed = os::elapsedTime() - start;
5529   g1_policy()->phase_times()->record_clear_ct_time(elapsed * 1000.0);
5530 }
5531 
5532 void G1CollectedHeap::free_collection_set(HeapRegion* cs_head, EvacuationInfo& evacuation_info, const size_t* surviving_young_words) {
5533   size_t pre_used = 0;
5534   FreeRegionList local_free_list("Local List for CSet Freeing");
5535 
5536   double young_time_ms     = 0.0;
5537   double non_young_time_ms = 0.0;
5538 
5539   // Since the collection set is a superset of the the young list,
5540   // all we need to do to clear the young list is clear its
5541   // head and length, and unlink any young regions in the code below
5542   _young_list->clear();
5543 
5544   G1CollectorPolicy* policy = g1_policy();
5545 
5546   double start_sec = os::elapsedTime();
5547   bool non_young = true;
5548 
5549   HeapRegion* cur = cs_head;
5550   int age_bound = -1;
5551   size_t rs_lengths = 0;
5552 
5553   while (cur != NULL) {
5554     assert(!is_on_master_free_list(cur), "sanity");
5555     if (non_young) {
5556       if (cur->is_young()) {
5557         double end_sec = os::elapsedTime();
5558         double elapsed_ms = (end_sec - start_sec) * 1000.0;
5559         non_young_time_ms += elapsed_ms;
5560 
5561         start_sec = os::elapsedTime();
5562         non_young = false;
5563       }
5564     } else {
5565       if (!cur->is_young()) {
5566         double end_sec = os::elapsedTime();
5567         double elapsed_ms = (end_sec - start_sec) * 1000.0;
5568         young_time_ms += elapsed_ms;
5569 
5570         start_sec = os::elapsedTime();
5571         non_young = true;
5572       }
5573     }
5574 
5575     rs_lengths += cur->rem_set()->occupied_locked();
5576 
5577     HeapRegion* next = cur->next_in_collection_set();
5578     assert(cur->in_collection_set(), "bad CS");
5579     cur->set_next_in_collection_set(NULL);
5580     clear_in_cset(cur);
5581 
5582     if (cur->is_young()) {
5583       int index = cur->young_index_in_cset();
5584       assert(index != -1, "invariant");
5585       assert((uint) index < policy->young_cset_region_length(), "invariant");
5586       size_t words_survived = surviving_young_words[index];
5587       cur->record_surv_words_in_group(words_survived);
5588 
5589       // At this point the we have 'popped' cur from the collection set
5590       // (linked via next_in_collection_set()) but it is still in the
5591       // young list (linked via next_young_region()). Clear the
5592       // _next_young_region field.
5593       cur->set_next_young_region(NULL);
5594     } else {
5595       int index = cur->young_index_in_cset();
5596       assert(index == -1, "invariant");
5597     }
5598 
5599     assert( (cur->is_young() && cur->young_index_in_cset() > -1) ||
5600             (!cur->is_young() && cur->young_index_in_cset() == -1),
5601             "invariant" );
5602 
5603     if (!cur->evacuation_failed()) {
5604       MemRegion used_mr = cur->used_region();
5605 
5606       // And the region is empty.
5607       assert(!used_mr.is_empty(), "Should not have empty regions in a CS.");
5608       pre_used += cur->used();
5609       free_region(cur, &local_free_list, false /* par */, true /* locked */);
5610     } else {
5611       cur->uninstall_surv_rate_group();
5612       if (cur->is_young()) {
5613         cur->set_young_index_in_cset(-1);
5614       }
5615       cur->set_evacuation_failed(false);
5616       // The region is now considered to be old.
5617       cur->set_old();
5618       // Do some allocation statistics accounting. Regions that failed evacuation
5619       // are always made old, so there is no need to update anything in the young
5620       // gen statistics, but we need to update old gen statistics.
5621       size_t used_words = cur->marked_bytes() / HeapWordSize;
5622       _old_evac_stats.add_failure_used_and_waste(used_words, HeapRegion::GrainWords - used_words);
5623       _old_set.add(cur);
5624       evacuation_info.increment_collectionset_used_after(cur->used());
5625     }
5626     cur = next;
5627   }
5628 
5629   evacuation_info.set_regions_freed(local_free_list.length());
5630   policy->record_max_rs_lengths(rs_lengths);
5631   policy->cset_regions_freed();
5632 
5633   double end_sec = os::elapsedTime();
5634   double elapsed_ms = (end_sec - start_sec) * 1000.0;
5635 
5636   if (non_young) {
5637     non_young_time_ms += elapsed_ms;
5638   } else {
5639     young_time_ms += elapsed_ms;
5640   }
5641 
5642   prepend_to_freelist(&local_free_list);
5643   decrement_summary_bytes(pre_used);
5644   policy->phase_times()->record_young_free_cset_time_ms(young_time_ms);
5645   policy->phase_times()->record_non_young_free_cset_time_ms(non_young_time_ms);
5646 }
5647 
5648 class G1FreeHumongousRegionClosure : public HeapRegionClosure {
5649  private:
5650   FreeRegionList* _free_region_list;
5651   HeapRegionSet* _proxy_set;
5652   HeapRegionSetCount _humongous_regions_removed;
5653   size_t _freed_bytes;
5654  public:
5655 
5656   G1FreeHumongousRegionClosure(FreeRegionList* free_region_list) :
5657     _free_region_list(free_region_list), _humongous_regions_removed(), _freed_bytes(0) {
5658   }
5659 
5660   virtual bool doHeapRegion(HeapRegion* r) {
5661     if (!r->is_starts_humongous()) {
5662       return false;
5663     }
5664 
5665     G1CollectedHeap* g1h = G1CollectedHeap::heap();
5666 
5667     oop obj = (oop)r->bottom();
5668     CMBitMap* next_bitmap = g1h->concurrent_mark()->nextMarkBitMap();
5669 
5670     // The following checks whether the humongous object is live are sufficient.
5671     // The main additional check (in addition to having a reference from the roots
5672     // or the young gen) is whether the humongous object has a remembered set entry.
5673     //
5674     // A humongous object cannot be live if there is no remembered set for it
5675     // because:
5676     // - there can be no references from within humongous starts regions referencing
5677     // the object because we never allocate other objects into them.
5678     // (I.e. there are no intra-region references that may be missed by the
5679     // remembered set)
5680     // - as soon there is a remembered set entry to the humongous starts region
5681     // (i.e. it has "escaped" to an old object) this remembered set entry will stay
5682     // until the end of a concurrent mark.
5683     //
5684     // It is not required to check whether the object has been found dead by marking
5685     // or not, in fact it would prevent reclamation within a concurrent cycle, as
5686     // all objects allocated during that time are considered live.
5687     // SATB marking is even more conservative than the remembered set.
5688     // So if at this point in the collection there is no remembered set entry,
5689     // nobody has a reference to it.
5690     // At the start of collection we flush all refinement logs, and remembered sets
5691     // are completely up-to-date wrt to references to the humongous object.
5692     //
5693     // Other implementation considerations:
5694     // - never consider object arrays at this time because they would pose
5695     // considerable effort for cleaning up the the remembered sets. This is
5696     // required because stale remembered sets might reference locations that
5697     // are currently allocated into.
5698     uint region_idx = r->hrm_index();
5699     if (!g1h->is_humongous_reclaim_candidate(region_idx) ||
5700         !r->rem_set()->is_empty()) {
5701 
5702       if (G1TraceEagerReclaimHumongousObjects) {
5703         gclog_or_tty->print_cr("Live humongous region %u object size " SIZE_FORMAT " start " PTR_FORMAT "  with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5704                                region_idx,
5705                                (size_t)obj->size() * HeapWordSize,
5706                                p2i(r->bottom()),
5707                                r->rem_set()->occupied(),
5708                                r->rem_set()->strong_code_roots_list_length(),
5709                                next_bitmap->isMarked(r->bottom()),
5710                                g1h->is_humongous_reclaim_candidate(region_idx),
5711                                obj->is_typeArray()
5712                               );
5713       }
5714 
5715       return false;
5716     }
5717 
5718     guarantee(obj->is_typeArray(),
5719               "Only eagerly reclaiming type arrays is supported, but the object "
5720               PTR_FORMAT " is not.", p2i(r->bottom()));
5721 
5722     if (G1TraceEagerReclaimHumongousObjects) {
5723       gclog_or_tty->print_cr("Dead humongous region %u object size " SIZE_FORMAT " start " PTR_FORMAT " with remset " SIZE_FORMAT " code roots " SIZE_FORMAT " is marked %d reclaim candidate %d type array %d",
5724                              region_idx,
5725                              (size_t)obj->size() * HeapWordSize,
5726                              p2i(r->bottom()),
5727                              r->rem_set()->occupied(),
5728                              r->rem_set()->strong_code_roots_list_length(),
5729                              next_bitmap->isMarked(r->bottom()),
5730                              g1h->is_humongous_reclaim_candidate(region_idx),
5731                              obj->is_typeArray()
5732                             );
5733     }
5734     // Need to clear mark bit of the humongous object if already set.
5735     if (next_bitmap->isMarked(r->bottom())) {
5736       next_bitmap->clear(r->bottom());
5737     }
5738     do {
5739       HeapRegion* next = g1h->next_region_in_humongous(r);
5740       _freed_bytes += r->used();
5741       r->set_containing_set(NULL);
5742       _humongous_regions_removed.increment(1u, r->capacity());
5743       g1h->free_humongous_region(r, _free_region_list, false);
5744       r = next;
5745     } while (r != NULL);
5746 
5747     return false;
5748   }
5749 
5750   HeapRegionSetCount& humongous_free_count() {
5751     return _humongous_regions_removed;
5752   }
5753 
5754   size_t bytes_freed() const {
5755     return _freed_bytes;
5756   }
5757 
5758   size_t humongous_reclaimed() const {
5759     return _humongous_regions_removed.length();
5760   }
5761 };
5762 
5763 void G1CollectedHeap::eagerly_reclaim_humongous_regions() {
5764   assert_at_safepoint(true);
5765 
5766   if (!G1EagerReclaimHumongousObjects ||
5767       (!_has_humongous_reclaim_candidates && !G1TraceEagerReclaimHumongousObjects)) {
5768     g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms(0.0, 0);
5769     return;
5770   }
5771 
5772   double start_time = os::elapsedTime();
5773 
5774   FreeRegionList local_cleanup_list("Local Humongous Cleanup List");
5775 
5776   G1FreeHumongousRegionClosure cl(&local_cleanup_list);
5777   heap_region_iterate(&cl);
5778 
5779   HeapRegionSetCount empty_set;
5780   remove_from_old_sets(empty_set, cl.humongous_free_count());
5781 
5782   G1HRPrinter* hrp = hr_printer();
5783   if (hrp->is_active()) {
5784     FreeRegionListIterator iter(&local_cleanup_list);
5785     while (iter.more_available()) {
5786       HeapRegion* hr = iter.get_next();
5787       hrp->cleanup(hr);
5788     }
5789   }
5790 
5791   prepend_to_freelist(&local_cleanup_list);
5792   decrement_summary_bytes(cl.bytes_freed());
5793 
5794   g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms((os::elapsedTime() - start_time) * 1000.0,
5795                                                                     cl.humongous_reclaimed());
5796 }
5797 
5798 // This routine is similar to the above but does not record
5799 // any policy statistics or update free lists; we are abandoning
5800 // the current incremental collection set in preparation of a
5801 // full collection. After the full GC we will start to build up
5802 // the incremental collection set again.
5803 // This is only called when we're doing a full collection
5804 // and is immediately followed by the tearing down of the young list.
5805 
5806 void G1CollectedHeap::abandon_collection_set(HeapRegion* cs_head) {
5807   HeapRegion* cur = cs_head;
5808 
5809   while (cur != NULL) {
5810     HeapRegion* next = cur->next_in_collection_set();
5811     assert(cur->in_collection_set(), "bad CS");
5812     cur->set_next_in_collection_set(NULL);
5813     clear_in_cset(cur);
5814     cur->set_young_index_in_cset(-1);
5815     cur = next;
5816   }
5817 }
5818 
5819 void G1CollectedHeap::set_free_regions_coming() {
5820   if (G1ConcRegionFreeingVerbose) {
5821     gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : "
5822                            "setting free regions coming");
5823   }
5824 
5825   assert(!free_regions_coming(), "pre-condition");
5826   _free_regions_coming = true;
5827 }
5828 
5829 void G1CollectedHeap::reset_free_regions_coming() {
5830   assert(free_regions_coming(), "pre-condition");
5831 
5832   {
5833     MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
5834     _free_regions_coming = false;
5835     SecondaryFreeList_lock->notify_all();
5836   }
5837 
5838   if (G1ConcRegionFreeingVerbose) {
5839     gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : "
5840                            "reset free regions coming");
5841   }
5842 }
5843 
5844 void G1CollectedHeap::wait_while_free_regions_coming() {
5845   // Most of the time we won't have to wait, so let's do a quick test
5846   // first before we take the lock.
5847   if (!free_regions_coming()) {
5848     return;
5849   }
5850 
5851   if (G1ConcRegionFreeingVerbose) {
5852     gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : "
5853                            "waiting for free regions");
5854   }
5855 
5856   {
5857     MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
5858     while (free_regions_coming()) {
5859       SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag);
5860     }
5861   }
5862 
5863   if (G1ConcRegionFreeingVerbose) {
5864     gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : "
5865                            "done waiting for free regions");
5866   }
5867 }
5868 
5869 bool G1CollectedHeap::is_old_gc_alloc_region(HeapRegion* hr) {
5870   return _allocator->is_retained_old_region(hr);
5871 }
5872 
5873 void G1CollectedHeap::set_region_short_lived_locked(HeapRegion* hr) {
5874   _young_list->push_region(hr);
5875 }
5876 
5877 class NoYoungRegionsClosure: public HeapRegionClosure {
5878 private:
5879   bool _success;
5880 public:
5881   NoYoungRegionsClosure() : _success(true) { }
5882   bool doHeapRegion(HeapRegion* r) {
5883     if (r->is_young()) {
5884       gclog_or_tty->print_cr("Region [" PTR_FORMAT ", " PTR_FORMAT ") tagged as young",
5885                              p2i(r->bottom()), p2i(r->end()));
5886       _success = false;
5887     }
5888     return false;
5889   }
5890   bool success() { return _success; }
5891 };
5892 
5893 bool G1CollectedHeap::check_young_list_empty(bool check_heap, bool check_sample) {
5894   bool ret = _young_list->check_list_empty(check_sample);
5895 
5896   if (check_heap) {
5897     NoYoungRegionsClosure closure;
5898     heap_region_iterate(&closure);
5899     ret = ret && closure.success();
5900   }
5901 
5902   return ret;
5903 }
5904 
5905 class TearDownRegionSetsClosure : public HeapRegionClosure {
5906 private:
5907   HeapRegionSet *_old_set;
5908 
5909 public:
5910   TearDownRegionSetsClosure(HeapRegionSet* old_set) : _old_set(old_set) { }
5911 
5912   bool doHeapRegion(HeapRegion* r) {
5913     if (r->is_old()) {
5914       _old_set->remove(r);
5915     } else {
5916       // We ignore free regions, we'll empty the free list afterwards.
5917       // We ignore young regions, we'll empty the young list afterwards.
5918       // We ignore humongous regions, we're not tearing down the
5919       // humongous regions set.
5920       assert(r->is_free() || r->is_young() || r->is_humongous(),
5921              "it cannot be another type");
5922     }
5923     return false;
5924   }
5925 
5926   ~TearDownRegionSetsClosure() {
5927     assert(_old_set->is_empty(), "post-condition");
5928   }
5929 };
5930 
5931 void G1CollectedHeap::tear_down_region_sets(bool free_list_only) {
5932   assert_at_safepoint(true /* should_be_vm_thread */);
5933 
5934   if (!free_list_only) {
5935     TearDownRegionSetsClosure cl(&_old_set);
5936     heap_region_iterate(&cl);
5937 
5938     // Note that emptying the _young_list is postponed and instead done as
5939     // the first step when rebuilding the regions sets again. The reason for
5940     // this is that during a full GC string deduplication needs to know if
5941     // a collected region was young or old when the full GC was initiated.
5942   }
5943   _hrm.remove_all_free_regions();
5944 }
5945 
5946 void G1CollectedHeap::increase_used(size_t bytes) {
5947   _summary_bytes_used += bytes;
5948 }
5949 
5950 void G1CollectedHeap::decrease_used(size_t bytes) {
5951   assert(_summary_bytes_used >= bytes,
5952          "invariant: _summary_bytes_used: " SIZE_FORMAT " should be >= bytes: " SIZE_FORMAT,
5953          _summary_bytes_used, bytes);
5954   _summary_bytes_used -= bytes;
5955 }
5956 
5957 void G1CollectedHeap::set_used(size_t bytes) {
5958   _summary_bytes_used = bytes;
5959 }
5960 
5961 class RebuildRegionSetsClosure : public HeapRegionClosure {
5962 private:
5963   bool            _free_list_only;
5964   HeapRegionSet*   _old_set;
5965   HeapRegionManager*   _hrm;
5966   size_t          _total_used;
5967 
5968 public:
5969   RebuildRegionSetsClosure(bool free_list_only,
5970                            HeapRegionSet* old_set, HeapRegionManager* hrm) :
5971     _free_list_only(free_list_only),
5972     _old_set(old_set), _hrm(hrm), _total_used(0) {
5973     assert(_hrm->num_free_regions() == 0, "pre-condition");
5974     if (!free_list_only) {
5975       assert(_old_set->is_empty(), "pre-condition");
5976     }
5977   }
5978 
5979   bool doHeapRegion(HeapRegion* r) {
5980     if (r->is_empty()) {
5981       // Add free regions to the free list
5982       r->set_free();
5983       r->set_allocation_context(AllocationContext::system());
5984       _hrm->insert_into_free_list(r);
5985     } else if (!_free_list_only) {
5986       assert(!r->is_young(), "we should not come across young regions");
5987 
5988       if (r->is_humongous()) {
5989         // We ignore humongous regions. We left the humongous set unchanged.
5990       } else {
5991         // Objects that were compacted would have ended up on regions
5992         // that were previously old or free.  Archive regions (which are
5993         // old) will not have been touched.
5994         assert(r->is_free() || r->is_old(), "invariant");
5995         // We now consider them old, so register as such. Leave
5996         // archive regions set that way, however, while still adding
5997         // them to the old set.
5998         if (!r->is_archive()) {
5999           r->set_old();
6000         }
6001         _old_set->add(r);
6002       }
6003       _total_used += r->used();
6004     }
6005 
6006     return false;
6007   }
6008 
6009   size_t total_used() {
6010     return _total_used;
6011   }
6012 };
6013 
6014 void G1CollectedHeap::rebuild_region_sets(bool free_list_only) {
6015   assert_at_safepoint(true /* should_be_vm_thread */);
6016 
6017   if (!free_list_only) {
6018     _young_list->empty_list();
6019   }
6020 
6021   RebuildRegionSetsClosure cl(free_list_only, &_old_set, &_hrm);
6022   heap_region_iterate(&cl);
6023 
6024   if (!free_list_only) {
6025     set_used(cl.total_used());
6026     if (_archive_allocator != NULL) {
6027       _archive_allocator->clear_used();
6028     }
6029   }
6030   assert(used_unlocked() == recalculate_used(),
6031          "inconsistent used_unlocked(), "
6032          "value: " SIZE_FORMAT " recalculated: " SIZE_FORMAT,
6033          used_unlocked(), recalculate_used());
6034 }
6035 
6036 void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) {
6037   _refine_cte_cl->set_concurrent(concurrent);
6038 }
6039 
6040 bool G1CollectedHeap::is_in_closed_subset(const void* p) const {
6041   HeapRegion* hr = heap_region_containing(p);
6042   return hr->is_in(p);
6043 }
6044 
6045 // Methods for the mutator alloc region
6046 
6047 HeapRegion* G1CollectedHeap::new_mutator_alloc_region(size_t word_size,
6048                                                       bool force) {
6049   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
6050   assert(!force || g1_policy()->can_expand_young_list(),
6051          "if force is true we should be able to expand the young list");
6052   bool young_list_full = g1_policy()->is_young_list_full();
6053   if (force || !young_list_full) {
6054     HeapRegion* new_alloc_region = new_region(word_size,
6055                                               false /* is_old */,
6056                                               false /* do_expand */);
6057     if (new_alloc_region != NULL) {
6058       set_region_short_lived_locked(new_alloc_region);
6059       _hr_printer.alloc(new_alloc_region, G1HRPrinter::Eden, young_list_full);
6060       check_bitmaps("Mutator Region Allocation", new_alloc_region);
6061       return new_alloc_region;
6062     }
6063   }
6064   return NULL;
6065 }
6066 
6067 void G1CollectedHeap::retire_mutator_alloc_region(HeapRegion* alloc_region,
6068                                                   size_t allocated_bytes) {
6069   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
6070   assert(alloc_region->is_eden(), "all mutator alloc regions should be eden");
6071 
6072   g1_policy()->add_region_to_incremental_cset_lhs(alloc_region);
6073   increase_used(allocated_bytes);
6074   _hr_printer.retire(alloc_region);
6075   // We update the eden sizes here, when the region is retired,
6076   // instead of when it's allocated, since this is the point that its
6077   // used space has been recored in _summary_bytes_used.
6078   g1mm()->update_eden_size();
6079 }
6080 
6081 // Methods for the GC alloc regions
6082 
6083 HeapRegion* G1CollectedHeap::new_gc_alloc_region(size_t word_size,
6084                                                  uint count,
6085                                                  InCSetState dest) {
6086   assert(FreeList_lock->owned_by_self(), "pre-condition");
6087 
6088   if (count < g1_policy()->max_regions(dest)) {
6089     const bool is_survivor = (dest.is_young());
6090     HeapRegion* new_alloc_region = new_region(word_size,
6091                                               !is_survivor,
6092                                               true /* do_expand */);
6093     if (new_alloc_region != NULL) {
6094       // We really only need to do this for old regions given that we
6095       // should never scan survivors. But it doesn't hurt to do it
6096       // for survivors too.
6097       new_alloc_region->record_timestamp();
6098       if (is_survivor) {
6099         new_alloc_region->set_survivor();
6100         _hr_printer.alloc(new_alloc_region, G1HRPrinter::Survivor);
6101         check_bitmaps("Survivor Region Allocation", new_alloc_region);
6102       } else {
6103         new_alloc_region->set_old();
6104         _hr_printer.alloc(new_alloc_region, G1HRPrinter::Old);
6105         check_bitmaps("Old Region Allocation", new_alloc_region);
6106       }
6107       bool during_im = collector_state()->during_initial_mark_pause();
6108       new_alloc_region->note_start_of_copying(during_im);
6109       return new_alloc_region;
6110     }
6111   }
6112   return NULL;
6113 }
6114 
6115 void G1CollectedHeap::retire_gc_alloc_region(HeapRegion* alloc_region,
6116                                              size_t allocated_bytes,
6117                                              InCSetState dest) {
6118   bool during_im = collector_state()->during_initial_mark_pause();
6119   alloc_region->note_end_of_copying(during_im);
6120   g1_policy()->record_bytes_copied_during_gc(allocated_bytes);
6121   if (dest.is_young()) {
6122     young_list()->add_survivor_region(alloc_region);
6123   } else {
6124     _old_set.add(alloc_region);
6125   }
6126   _hr_printer.retire(alloc_region);
6127 }
6128 
6129 HeapRegion* G1CollectedHeap::alloc_highest_free_region() {
6130   bool expanded = false;
6131   uint index = _hrm.find_highest_free(&expanded);
6132 
6133   if (index != G1_NO_HRM_INDEX) {
6134     if (expanded) {
6135       ergo_verbose1(ErgoHeapSizing,
6136                     "attempt heap expansion",
6137                     ergo_format_reason("requested address range outside heap bounds")
6138                     ergo_format_byte("region size"),
6139                     HeapRegion::GrainWords * HeapWordSize);
6140     }
6141     _hrm.allocate_free_regions_starting_at(index, 1);
6142     return region_at(index);
6143   }
6144   return NULL;
6145 }
6146 
6147 // Heap region set verification
6148 
6149 class VerifyRegionListsClosure : public HeapRegionClosure {
6150 private:
6151   HeapRegionSet*   _old_set;
6152   HeapRegionSet*   _humongous_set;
6153   HeapRegionManager*   _hrm;
6154 
6155 public:
6156   HeapRegionSetCount _old_count;
6157   HeapRegionSetCount _humongous_count;
6158   HeapRegionSetCount _free_count;
6159 
6160   VerifyRegionListsClosure(HeapRegionSet* old_set,
6161                            HeapRegionSet* humongous_set,
6162                            HeapRegionManager* hrm) :
6163     _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm),
6164     _old_count(), _humongous_count(), _free_count(){ }
6165 
6166   bool doHeapRegion(HeapRegion* hr) {
6167     if (hr->is_young()) {
6168       // TODO
6169     } else if (hr->is_humongous()) {
6170       assert(hr->containing_set() == _humongous_set, "Heap region %u is humongous but not in humongous set.", hr->hrm_index());
6171       _humongous_count.increment(1u, hr->capacity());
6172     } else if (hr->is_empty()) {
6173       assert(_hrm->is_free(hr), "Heap region %u is empty but not on the free list.", hr->hrm_index());
6174       _free_count.increment(1u, hr->capacity());
6175     } else if (hr->is_old()) {
6176       assert(hr->containing_set() == _old_set, "Heap region %u is old but not in the old set.", hr->hrm_index());
6177       _old_count.increment(1u, hr->capacity());
6178     } else {
6179       // There are no other valid region types. Check for one invalid
6180       // one we can identify: pinned without old or humongous set.
6181       assert(!hr->is_pinned(), "Heap region %u is pinned but not old (archive) or humongous.", hr->hrm_index());
6182       ShouldNotReachHere();
6183     }
6184     return false;
6185   }
6186 
6187   void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) {
6188     guarantee(old_set->length() == _old_count.length(), "Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length());
6189     guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), "Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6190               old_set->total_capacity_bytes(), _old_count.capacity());
6191 
6192     guarantee(humongous_set->length() == _humongous_count.length(), "Hum set count mismatch. Expected %u, actual %u.", humongous_set->length(), _humongous_count.length());
6193     guarantee(humongous_set->total_capacity_bytes() == _humongous_count.capacity(), "Hum set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6194               humongous_set->total_capacity_bytes(), _humongous_count.capacity());
6195 
6196     guarantee(free_list->num_free_regions() == _free_count.length(), "Free list count mismatch. Expected %u, actual %u.", free_list->num_free_regions(), _free_count.length());
6197     guarantee(free_list->total_capacity_bytes() == _free_count.capacity(), "Free list capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT,
6198               free_list->total_capacity_bytes(), _free_count.capacity());
6199   }
6200 };
6201 
6202 void G1CollectedHeap::verify_region_sets() {
6203   assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
6204 
6205   // First, check the explicit lists.
6206   _hrm.verify();
6207   {
6208     // Given that a concurrent operation might be adding regions to
6209     // the secondary free list we have to take the lock before
6210     // verifying it.
6211     MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
6212     _secondary_free_list.verify_list();
6213   }
6214 
6215   // If a concurrent region freeing operation is in progress it will
6216   // be difficult to correctly attributed any free regions we come
6217   // across to the correct free list given that they might belong to
6218   // one of several (free_list, secondary_free_list, any local lists,
6219   // etc.). So, if that's the case we will skip the rest of the
6220   // verification operation. Alternatively, waiting for the concurrent
6221   // operation to complete will have a non-trivial effect on the GC's
6222   // operation (no concurrent operation will last longer than the
6223   // interval between two calls to verification) and it might hide
6224   // any issues that we would like to catch during testing.
6225   if (free_regions_coming()) {
6226     return;
6227   }
6228 
6229   // Make sure we append the secondary_free_list on the free_list so
6230   // that all free regions we will come across can be safely
6231   // attributed to the free_list.
6232   append_secondary_free_list_if_not_empty_with_lock();
6233 
6234   // Finally, make sure that the region accounting in the lists is
6235   // consistent with what we see in the heap.
6236 
6237   VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_hrm);
6238   heap_region_iterate(&cl);
6239   cl.verify_counts(&_old_set, &_humongous_set, &_hrm);
6240 }
6241 
6242 // Optimized nmethod scanning
6243 
6244 class RegisterNMethodOopClosure: public OopClosure {
6245   G1CollectedHeap* _g1h;
6246   nmethod* _nm;
6247 
6248   template <class T> void do_oop_work(T* p) {
6249     T heap_oop = oopDesc::load_heap_oop(p);
6250     if (!oopDesc::is_null(heap_oop)) {
6251       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
6252       HeapRegion* hr = _g1h->heap_region_containing(obj);
6253       assert(!hr->is_continues_humongous(),
6254              "trying to add code root " PTR_FORMAT " in continuation of humongous region " HR_FORMAT
6255              " starting at " HR_FORMAT,
6256              p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()));
6257 
6258       // HeapRegion::add_strong_code_root_locked() avoids adding duplicate entries.
6259       hr->add_strong_code_root_locked(_nm);
6260     }
6261   }
6262 
6263 public:
6264   RegisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
6265     _g1h(g1h), _nm(nm) {}
6266 
6267   void do_oop(oop* p)       { do_oop_work(p); }
6268   void do_oop(narrowOop* p) { do_oop_work(p); }
6269 };
6270 
6271 class UnregisterNMethodOopClosure: public OopClosure {
6272   G1CollectedHeap* _g1h;
6273   nmethod* _nm;
6274 
6275   template <class T> void do_oop_work(T* p) {
6276     T heap_oop = oopDesc::load_heap_oop(p);
6277     if (!oopDesc::is_null(heap_oop)) {
6278       oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
6279       HeapRegion* hr = _g1h->heap_region_containing(obj);
6280       assert(!hr->is_continues_humongous(),
6281              "trying to remove code root " PTR_FORMAT " in continuation of humongous region " HR_FORMAT
6282              " starting at " HR_FORMAT,
6283              p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()));
6284 
6285       hr->remove_strong_code_root(_nm);
6286     }
6287   }
6288 
6289 public:
6290   UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
6291     _g1h(g1h), _nm(nm) {}
6292 
6293   void do_oop(oop* p)       { do_oop_work(p); }
6294   void do_oop(narrowOop* p) { do_oop_work(p); }
6295 };
6296 
6297 void G1CollectedHeap::register_nmethod(nmethod* nm) {
6298   CollectedHeap::register_nmethod(nm);
6299 
6300   guarantee(nm != NULL, "sanity");
6301   RegisterNMethodOopClosure reg_cl(this, nm);
6302   nm->oops_do(&reg_cl);
6303 }
6304 
6305 void G1CollectedHeap::unregister_nmethod(nmethod* nm) {
6306   CollectedHeap::unregister_nmethod(nm);
6307 
6308   guarantee(nm != NULL, "sanity");
6309   UnregisterNMethodOopClosure reg_cl(this, nm);
6310   nm->oops_do(&reg_cl, true);
6311 }
6312 
6313 void G1CollectedHeap::purge_code_root_memory() {
6314   double purge_start = os::elapsedTime();
6315   G1CodeRootSet::purge();
6316   double purge_time_ms = (os::elapsedTime() - purge_start) * 1000.0;
6317   g1_policy()->phase_times()->record_strong_code_root_purge_time(purge_time_ms);
6318 }
6319 
6320 class RebuildStrongCodeRootClosure: public CodeBlobClosure {
6321   G1CollectedHeap* _g1h;
6322 
6323 public:
6324   RebuildStrongCodeRootClosure(G1CollectedHeap* g1h) :
6325     _g1h(g1h) {}
6326 
6327   void do_code_blob(CodeBlob* cb) {
6328     nmethod* nm = (cb != NULL) ? cb->as_nmethod_or_null() : NULL;
6329     if (nm == NULL) {
6330       return;
6331     }
6332 
6333     if (ScavengeRootsInCode) {
6334       _g1h->register_nmethod(nm);
6335     }
6336   }
6337 };
6338 
6339 void G1CollectedHeap::rebuild_strong_code_roots() {
6340   RebuildStrongCodeRootClosure blob_cl(this);
6341   CodeCache::blobs_do(&blob_cl);
6342 }
--- EOF ---