1 /*
   2  * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.hpp"
  26 
  27 #include "gc/shared/gcTimer.hpp"
  28 #include "gc/shared/gcTraceTime.inline.hpp"
  29 #include "gc/shared/memAllocator.hpp"
  30 #include "gc/shared/parallelCleaning.hpp"
  31 #include "gc/shared/plab.hpp"
  32 
  33 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
  34 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  35 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"
  36 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  37 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  38 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  39 #include "gc/shenandoah/shenandoahControlThread.hpp"
  40 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  41 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  42 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  43 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  44 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  45 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  46 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  47 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  48 #include "gc/shenandoah/shenandoahMetrics.hpp"
  49 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  50 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  51 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  52 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
  53 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  54 #include "gc/shenandoah/shenandoahUtils.hpp"
  55 #include "gc/shenandoah/shenandoahVerifier.hpp"
  56 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  57 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  58 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  59 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  60 #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp"
  61 #include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp"
  62 #include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp"
  63 #include "gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp"
  64 #include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp"
  65 #include "gc/shenandoah/heuristics/shenandoahTraversalHeuristics.hpp"
  66 
  67 #include "memory/metaspace.hpp"
  68 #include "runtime/interfaceSupport.inline.hpp"
  69 #include "runtime/safepointMechanism.hpp"
  70 #include "runtime/vmThread.hpp"
  71 #include "services/mallocTracker.hpp"
  72 
  73 ShenandoahUpdateRefsClosure::ShenandoahUpdateRefsClosure() : _heap(ShenandoahHeap::heap()) {}
  74 
  75 #ifdef ASSERT
  76 template <class T>
  77 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
  78   T o = RawAccess<>::oop_load(p);
  79   if (! CompressedOops::is_null(o)) {
  80     oop obj = CompressedOops::decode_not_null(o);
  81     shenandoah_assert_not_forwarded(p, obj);
  82   }
  83 }
  84 
  85 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
  86 void ShenandoahAssertToSpaceClosure::do_oop(oop* p)       { do_oop_work(p); }
  87 #endif
  88 
  89 class ShenandoahPretouchHeapTask : public AbstractGangTask {
  90 private:
  91   ShenandoahRegionIterator _regions;
  92   const size_t _page_size;
  93 public:
  94   ShenandoahPretouchHeapTask(size_t page_size) :
  95     AbstractGangTask("Shenandoah Pretouch Heap"),
  96     _page_size(page_size) {}
  97 
  98   virtual void work(uint worker_id) {
  99     ShenandoahHeapRegion* r = _regions.next();
 100     while (r != NULL) {
 101       os::pretouch_memory(r->bottom(), r->end(), _page_size);
 102       r = _regions.next();
 103     }
 104   }
 105 };
 106 
 107 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
 108 private:
 109   ShenandoahRegionIterator _regions;
 110   char* _bitmap_base;
 111   const size_t _bitmap_size;
 112   const size_t _page_size;
 113 public:
 114   ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
 115     AbstractGangTask("Shenandoah Pretouch Bitmap"),
 116     _bitmap_base(bitmap_base),
 117     _bitmap_size(bitmap_size),
 118     _page_size(page_size) {}
 119 
 120   virtual void work(uint worker_id) {
 121     ShenandoahHeapRegion* r = _regions.next();
 122     while (r != NULL) {
 123       size_t start = r->region_number()       * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 124       size_t end   = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 125       assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
 126 
 127       os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
 128 
 129       r = _regions.next();
 130     }
 131   }
 132 };
 133 
 134 jint ShenandoahHeap::initialize() {
 135   ShenandoahBrooksPointer::initial_checks();
 136 
 137   initialize_heuristics();
 138 
 139   //
 140   // Figure out heap sizing
 141   //
 142 
 143   size_t init_byte_size = collector_policy()->initial_heap_byte_size();
 144   size_t max_byte_size  = collector_policy()->max_heap_byte_size();
 145   size_t heap_alignment = collector_policy()->heap_alignment();
 146 
 147   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 148 
 149   if (ShenandoahAlwaysPreTouch) {
 150     // Enabled pre-touch means the entire heap is committed right away.
 151     init_byte_size = max_byte_size;
 152   }
 153 
 154   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 155   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 156 
 157   _num_regions = ShenandoahHeapRegion::region_count();
 158 
 159   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 160   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 161   assert(num_committed_regions <= _num_regions, "sanity");
 162 
 163   _initial_size = num_committed_regions * reg_size_bytes;
 164   _committed = _initial_size;
 165 
 166   size_t heap_page_size   = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 167   size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 168 
 169   //
 170   // Reserve and commit memory for heap
 171   //
 172 
 173   ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 174   initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*) (heap_rs.base() + heap_rs.size()));
 175   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 176 
 177   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 178          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 179 
 180   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 181   os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 182                             "Cannot commit heap memory");
 183 
 184   //
 185   // Reserve and commit memory for bitmap(s)
 186   //
 187 
 188   _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
 189   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 190 
 191   size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
 192 
 193   guarantee(bitmap_bytes_per_region != 0,
 194             "Bitmap bytes per region should not be zero");
 195   guarantee(is_power_of_2(bitmap_bytes_per_region),
 196             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 197 
 198   if (bitmap_page_size > bitmap_bytes_per_region) {
 199     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 200     _bitmap_bytes_per_slice = bitmap_page_size;
 201   } else {
 202     _bitmap_regions_per_slice = 1;
 203     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 204   }
 205 
 206   guarantee(_bitmap_regions_per_slice >= 1,
 207             "Should have at least one region per slice: " SIZE_FORMAT,
 208             _bitmap_regions_per_slice);
 209 
 210   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 211             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 212             _bitmap_bytes_per_slice, bitmap_page_size);
 213 
 214   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 215   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 216   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 217 
 218   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 219                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 220   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 221   os::commit_memory_or_exit((char *)_bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 222                             "Cannot commit bitmap memory");
 223 
 224   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 225 
 226   if (ShenandoahVerify) {
 227     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 228     os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 229                               "Cannot commit verification bitmap memory");
 230     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 231     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 232     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 233     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 234   }
 235 
 236   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 237   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
 238   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 239   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 240   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 241 
 242   //
 243   // Create regions and region sets
 244   //
 245 
 246   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 247   _free_set = new ShenandoahFreeSet(this, _num_regions);
 248   _collection_set = new ShenandoahCollectionSet(this, (HeapWord*)sh_rs.base());
 249 
 250   {
 251     ShenandoahHeapLocker locker(lock());
 252     for (size_t i = 0; i < _num_regions; i++) {
 253       ShenandoahHeapRegion* r = new ShenandoahHeapRegion(
 254               this,
 255               (HeapWord*) sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i,
 256               ShenandoahHeapRegion::region_size_words(),
 257               i,
 258               i < num_committed_regions
 259       );
 260 
 261       _marking_context->initialize_top_at_mark_start(r);
 262       _regions[i] = r;
 263       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 264     }
 265 
 266     // Initialize to complete
 267     _marking_context->mark_complete();
 268 
 269     _free_set->rebuild();
 270   }
 271 
 272   if (ShenandoahAlwaysPreTouch) {
 273     assert(!AlwaysPreTouch, "Should have been overridden");
 274 
 275     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 276     // before initialize() below zeroes it with initializing thread. For any given region,
 277     // we touch the region and the corresponding bitmaps from the same thread.
 278     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 279 
 280     size_t pretouch_heap_page_size = heap_page_size;
 281     size_t pretouch_bitmap_page_size = bitmap_page_size;
 282 
 283 #ifdef LINUX
 284     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 285     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 286     // them into huge one. Therefore, we need to pretouch with smaller pages.
 287     if (UseTransparentHugePages) {
 288       pretouch_heap_page_size = (size_t)os::vm_page_size();
 289       pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 290     }
 291 #endif
 292 
 293     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 294     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 295 
 296     log_info(gc, init)("Pretouch bitmap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 297                        _num_regions, pretouch_bitmap_page_size);
 298     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, pretouch_bitmap_page_size);
 299     _workers->run_task(&bcl);
 300 
 301     log_info(gc, init)("Pretouch heap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 302                        _num_regions, pretouch_heap_page_size);
 303     ShenandoahPretouchHeapTask hcl(pretouch_heap_page_size);
 304     _workers->run_task(&hcl);
 305   }
 306 
 307   //
 308   // Initialize the rest of GC subsystems
 309   //
 310 
 311   _liveness_cache = NEW_C_HEAP_ARRAY(jushort*, _max_workers, mtGC);
 312   for (uint worker = 0; worker < _max_workers; worker++) {
 313     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(jushort, _num_regions, mtGC);
 314     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(jushort));
 315   }
 316 
 317   // The call below uses stuff (the SATB* things) that are in G1, but probably
 318   // belong into a shared location.
 319   ShenandoahBarrierSet::satb_mark_queue_set().initialize(this,
 320                                                          SATB_Q_CBL_mon,
 321                                                          20 /* G1SATBProcessCompletedThreshold */,
 322                                                          60 /* G1SATBBufferEnqueueingThresholdPercent */);
 323 
 324   _monitoring_support = new ShenandoahMonitoringSupport(this);
 325   _phase_timings = new ShenandoahPhaseTimings();
 326   ShenandoahStringDedup::initialize();
 327   ShenandoahCodeRoots::initialize();
 328 
 329   if (ShenandoahAllocationTrace) {
 330     _alloc_tracker = new ShenandoahAllocTracker();
 331   }
 332 
 333   if (ShenandoahPacing) {
 334     _pacer = new ShenandoahPacer(this);
 335     _pacer->setup_for_idle();
 336   } else {
 337     _pacer = NULL;
 338   }
 339 
 340   _traversal_gc = heuristics()->can_do_traversal_gc() ?
 341                   new ShenandoahTraversalGC(this, _num_regions) :
 342                   NULL;
 343 
 344   _control_thread = new ShenandoahControlThread();
 345 
 346   log_info(gc, init)("Initialize Shenandoah heap with initial size " SIZE_FORMAT "%s",
 347                      byte_size_in_proper_unit(_initial_size), proper_unit_for_byte_size(_initial_size));
 348 
 349   log_info(gc, init)("Safepointing mechanism: %s",
 350                      SafepointMechanism::uses_thread_local_poll() ? "thread-local poll" :
 351                      (SafepointMechanism::uses_global_page_poll() ? "global-page poll" : "unknown"));
 352 
 353   return JNI_OK;
 354 }
 355 
 356 void ShenandoahHeap::initialize_heuristics() {
 357   if (ShenandoahGCHeuristics != NULL) {
 358     if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) {
 359       _heuristics = new ShenandoahAggressiveHeuristics();
 360     } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) {
 361       _heuristics = new ShenandoahStaticHeuristics();
 362     } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) {
 363       _heuristics = new ShenandoahAdaptiveHeuristics();
 364     } else if (strcmp(ShenandoahGCHeuristics, "passive") == 0) {
 365       _heuristics = new ShenandoahPassiveHeuristics();
 366     } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) {
 367       _heuristics = new ShenandoahCompactHeuristics();
 368     } else if (strcmp(ShenandoahGCHeuristics, "traversal") == 0) {
 369       _heuristics = new ShenandoahTraversalHeuristics();
 370     } else {
 371       vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option");
 372     }
 373 
 374     if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 375       vm_exit_during_initialization(
 376               err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 377                       _heuristics->name()));
 378     }
 379     if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 380       vm_exit_during_initialization(
 381               err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 382                       _heuristics->name()));
 383     }
 384 
 385     if (ShenandoahStoreValEnqueueBarrier && ShenandoahStoreValReadBarrier) {
 386       vm_exit_during_initialization("Cannot use both ShenandoahStoreValEnqueueBarrier and ShenandoahStoreValReadBarrier");
 387     }
 388     log_info(gc, init)("Shenandoah heuristics: %s",
 389                        _heuristics->name());
 390   } else {
 391       ShouldNotReachHere();
 392   }
 393 
 394 }
 395 
 396 #ifdef _MSC_VER
 397 #pragma warning( push )
 398 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 399 #endif
 400 
 401 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 402   CollectedHeap(),
 403   _initial_size(0),
 404   _used(0),
 405   _committed(0),
 406   _bytes_allocated_since_gc_start(0),
 407   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 408   _workers(NULL),
 409   _safepoint_workers(NULL),
 410   _num_regions(0),
 411   _regions(NULL),
 412   _update_refs_iterator(this),
 413   _control_thread(NULL),
 414   _shenandoah_policy(policy),
 415   _heuristics(NULL),
 416   _free_set(NULL),
 417   _scm(new ShenandoahConcurrentMark()),
 418   _traversal_gc(NULL),
 419   _full_gc(new ShenandoahMarkCompact()),
 420   _pacer(NULL),
 421   _verifier(NULL),
 422   _alloc_tracker(NULL),
 423   _phase_timings(NULL),
 424   _monitoring_support(NULL),
 425   _memory_pool(NULL),
 426   _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
 427   _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
 428   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
 429   _soft_ref_policy(),
 430   _ref_processor(NULL),
 431   _marking_context(NULL),
 432   _bitmap_size(0),
 433   _bitmap_regions_per_slice(0),
 434   _bitmap_bytes_per_slice(0),
 435   _liveness_cache(NULL),
 436   _collection_set(NULL)
 437 {
 438   log_info(gc, init)("GC threads: " UINT32_FORMAT " parallel, " UINT32_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads);
 439   log_info(gc, init)("Reference processing: %s", ParallelRefProcEnabled ? "parallel" : "serial");
 440 
 441   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 442 
 443   _max_workers = MAX2(_max_workers, 1U);
 444   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
 445                             /* are_GC_task_threads */true,
 446                             /* are_ConcurrentGC_threads */false);
 447   if (_workers == NULL) {
 448     vm_exit_during_initialization("Failed necessary allocation.");
 449   } else {
 450     _workers->initialize_workers();
 451   }
 452 
 453   if (ShenandoahParallelSafepointThreads > 1) {
 454     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
 455                                                 ShenandoahParallelSafepointThreads,
 456                                                 false, false);
 457     _safepoint_workers->initialize_workers();
 458   }
 459 }
 460 
 461 #ifdef _MSC_VER
 462 #pragma warning( pop )
 463 #endif
 464 
 465 class ShenandoahResetBitmapTask : public AbstractGangTask {
 466 private:
 467   ShenandoahRegionIterator _regions;
 468 
 469 public:
 470   ShenandoahResetBitmapTask() :
 471     AbstractGangTask("Parallel Reset Bitmap Task") {}
 472 
 473   void work(uint worker_id) {
 474     ShenandoahHeapRegion* region = _regions.next();
 475     ShenandoahHeap* heap = ShenandoahHeap::heap();
 476     ShenandoahMarkingContext* const ctx = heap->marking_context();
 477     while (region != NULL) {
 478       if (heap->is_bitmap_slice_committed(region)) {
 479         ctx->clear_bitmap(region);
 480       }
 481       region = _regions.next();
 482     }
 483   }
 484 };
 485 
 486 void ShenandoahHeap::reset_mark_bitmap() {
 487   assert_gc_workers(_workers->active_workers());
 488   mark_incomplete_marking_context();
 489 
 490   ShenandoahResetBitmapTask task;
 491   _workers->run_task(&task);
 492 }
 493 
 494 void ShenandoahHeap::print_on(outputStream* st) const {
 495   st->print_cr("Shenandoah Heap");
 496   st->print_cr(" " SIZE_FORMAT "K total, " SIZE_FORMAT "K committed, " SIZE_FORMAT "K used",
 497                capacity() / K, committed() / K, used() / K);
 498   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"K regions",
 499                num_regions(), ShenandoahHeapRegion::region_size_bytes() / K);
 500 
 501   st->print("Status: ");
 502   if (has_forwarded_objects())               st->print("has forwarded objects, ");
 503   if (is_concurrent_mark_in_progress())      st->print("marking, ");
 504   if (is_evacuation_in_progress())           st->print("evacuating, ");
 505   if (is_update_refs_in_progress())          st->print("updating refs, ");
 506   if (is_concurrent_traversal_in_progress()) st->print("traversal, ");
 507   if (is_degenerated_gc_in_progress())       st->print("degenerated gc, ");
 508   if (is_full_gc_in_progress())              st->print("full gc, ");
 509   if (is_full_gc_move_in_progress())         st->print("full gc move, ");
 510 
 511   if (cancelled_gc()) {
 512     st->print("cancelled");
 513   } else {
 514     st->print("not cancelled");
 515   }
 516   st->cr();
 517 
 518   st->print_cr("Reserved region:");
 519   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 520                p2i(reserved_region().start()),
 521                p2i(reserved_region().end()));
 522 
 523   st->cr();
 524   MetaspaceUtils::print_on(st);
 525 
 526   if (Verbose) {
 527     print_heap_regions_on(st);
 528   }
 529 }
 530 
 531 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 532 public:
 533   void do_thread(Thread* thread) {
 534     assert(thread != NULL, "Sanity");
 535     assert(thread->is_Worker_thread(), "Only worker thread expected");
 536     ShenandoahThreadLocalData::initialize_gclab(thread);
 537   }
 538 };
 539 
 540 void ShenandoahHeap::post_initialize() {
 541   CollectedHeap::post_initialize();
 542   MutexLocker ml(Threads_lock);
 543 
 544   ShenandoahInitWorkerGCLABClosure init_gclabs;
 545   _workers->threads_do(&init_gclabs);
 546 
 547   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 548   // Now, we will let WorkGang to initialize gclab when new worker is created.
 549   _workers->set_initialize_gclab();
 550 
 551   _scm->initialize(_max_workers);
 552   _full_gc->initialize(_gc_timer);
 553 
 554   ref_processing_init();
 555 
 556   _heuristics->initialize();
 557 }
 558 
 559 size_t ShenandoahHeap::used() const {
 560   return OrderAccess::load_acquire(&_used);
 561 }
 562 
 563 size_t ShenandoahHeap::committed() const {
 564   OrderAccess::acquire();
 565   return _committed;
 566 }
 567 
 568 void ShenandoahHeap::increase_committed(size_t bytes) {
 569   assert_heaplock_or_safepoint();
 570   _committed += bytes;
 571 }
 572 
 573 void ShenandoahHeap::decrease_committed(size_t bytes) {
 574   assert_heaplock_or_safepoint();
 575   _committed -= bytes;
 576 }
 577 
 578 void ShenandoahHeap::increase_used(size_t bytes) {
 579   Atomic::add(bytes, &_used);
 580 }
 581 
 582 void ShenandoahHeap::set_used(size_t bytes) {
 583   OrderAccess::release_store_fence(&_used, bytes);
 584 }
 585 
 586 void ShenandoahHeap::decrease_used(size_t bytes) {
 587   assert(used() >= bytes, "never decrease heap size by more than we've left");
 588   Atomic::sub(bytes, &_used);
 589 }
 590 
 591 void ShenandoahHeap::increase_allocated(size_t bytes) {
 592   Atomic::add(bytes, &_bytes_allocated_since_gc_start);
 593 }
 594 
 595 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 596   size_t bytes = words * HeapWordSize;
 597   if (!waste) {
 598     increase_used(bytes);
 599   }
 600   increase_allocated(bytes);
 601   if (ShenandoahPacing) {
 602     control_thread()->pacing_notify_alloc(words);
 603     if (waste) {
 604       pacer()->claim_for_alloc(words, true);
 605     }
 606   }
 607 }
 608 
 609 size_t ShenandoahHeap::capacity() const {
 610   return num_regions() * ShenandoahHeapRegion::region_size_bytes();
 611 }
 612 
 613 size_t ShenandoahHeap::max_capacity() const {
 614   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 615 }
 616 
 617 size_t ShenandoahHeap::initial_capacity() const {
 618   return _initial_size;
 619 }
 620 
 621 bool ShenandoahHeap::is_in(const void* p) const {
 622   HeapWord* heap_base = (HeapWord*) base();
 623   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 624   return p >= heap_base && p < last_region_end;
 625 }
 626 
 627 void ShenandoahHeap::op_uncommit(double shrink_before) {
 628   assert (ShenandoahUncommit, "should be enabled");
 629 
 630   size_t count = 0;
 631   for (size_t i = 0; i < num_regions(); i++) {
 632     ShenandoahHeapRegion* r = get_region(i);
 633     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 634       ShenandoahHeapLocker locker(lock());
 635       if (r->is_empty_committed()) {
 636         r->make_uncommitted();
 637         count++;
 638       }
 639     }
 640     SpinPause(); // allow allocators to take the lock
 641   }
 642 
 643   if (count > 0) {
 644     log_info(gc)("Uncommitted " SIZE_FORMAT "M. Heap: " SIZE_FORMAT "M reserved, " SIZE_FORMAT "M committed, " SIZE_FORMAT "M used",
 645                  count * ShenandoahHeapRegion::region_size_bytes() / M, capacity() / M, committed() / M, used() / M);
 646     control_thread()->notify_heap_changed();
 647   }
 648 }
 649 
 650 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 651   // New object should fit the GCLAB size
 652   size_t min_size = MAX2(size, PLAB::min_size());
 653 
 654   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 655   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 656   new_size = MIN2(new_size, PLAB::max_size());
 657   new_size = MAX2(new_size, PLAB::min_size());
 658 
 659   // Record new heuristic value even if we take any shortcut. This captures
 660   // the case when moderately-sized objects always take a shortcut. At some point,
 661   // heuristics should catch up with them.
 662   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 663 
 664   if (new_size < size) {
 665     // New size still does not fit the object. Fall back to shared allocation.
 666     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 667     return NULL;
 668   }
 669 
 670   // Retire current GCLAB, and allocate a new one.
 671   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 672   gclab->retire();
 673 
 674   size_t actual_size = 0;
 675   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 676   if (gclab_buf == NULL) {
 677     return NULL;
 678   }
 679 
 680   assert (size <= actual_size, "allocation should fit");
 681 
 682   if (ZeroTLAB) {
 683     // ..and clear it.
 684     Copy::zero_to_words(gclab_buf, actual_size);
 685   } else {
 686     // ...and zap just allocated object.
 687 #ifdef ASSERT
 688     // Skip mangling the space corresponding to the object header to
 689     // ensure that the returned space is not considered parsable by
 690     // any concurrent GC thread.
 691     size_t hdr_size = oopDesc::header_size();
 692     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 693 #endif // ASSERT
 694   }
 695   gclab->set_buf(gclab_buf, actual_size);
 696   return gclab->allocate(size);
 697 }
 698 
 699 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 700                                             size_t requested_size,
 701                                             size_t* actual_size) {
 702   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 703   HeapWord* res = allocate_memory(req);
 704   if (res != NULL) {
 705     *actual_size = req.actual_size();
 706   } else {
 707     *actual_size = 0;
 708   }
 709   return res;
 710 }
 711 
 712 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 713                                              size_t word_size,
 714                                              size_t* actual_size) {
 715   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 716   HeapWord* res = allocate_memory(req);
 717   if (res != NULL) {
 718     *actual_size = req.actual_size();
 719   } else {
 720     *actual_size = 0;
 721   }
 722   return res;
 723 }
 724 
 725 ShenandoahHeap* ShenandoahHeap::heap() {
 726   CollectedHeap* heap = Universe::heap();
 727   assert(heap != NULL, "Unitialized access to ShenandoahHeap::heap()");
 728   assert(heap->kind() == CollectedHeap::Shenandoah, "not a shenandoah heap");
 729   return (ShenandoahHeap*) heap;
 730 }
 731 
 732 ShenandoahHeap* ShenandoahHeap::heap_no_check() {
 733   CollectedHeap* heap = Universe::heap();
 734   return (ShenandoahHeap*) heap;
 735 }
 736 
 737 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 738   ShenandoahAllocTrace trace_alloc(req.size(), req.type());
 739 
 740   intptr_t pacer_epoch = 0;
 741   bool in_new_region = false;
 742   HeapWord* result = NULL;
 743 
 744   if (req.is_mutator_alloc()) {
 745     if (ShenandoahPacing) {
 746       pacer()->pace_for_alloc(req.size());
 747       pacer_epoch = pacer()->epoch();
 748     }
 749 
 750     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 751       result = allocate_memory_under_lock(req, in_new_region);
 752     }
 753 
 754     // Allocation failed, block until control thread reacted, then retry allocation.
 755     //
 756     // It might happen that one of the threads requesting allocation would unblock
 757     // way later after GC happened, only to fail the second allocation, because
 758     // other threads have already depleted the free storage. In this case, a better
 759     // strategy is to try again, as long as GC makes progress.
 760     //
 761     // Then, we need to make sure the allocation was retried after at least one
 762     // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
 763 
 764     size_t tries = 0;
 765 
 766     while (result == NULL && _progress_last_gc.is_set()) {
 767       tries++;
 768       control_thread()->handle_alloc_failure(req.size());
 769       result = allocate_memory_under_lock(req, in_new_region);
 770     }
 771 
 772     while (result == NULL && tries <= ShenandoahFullGCThreshold) {
 773       tries++;
 774       control_thread()->handle_alloc_failure(req.size());
 775       result = allocate_memory_under_lock(req, in_new_region);
 776     }
 777 
 778   } else {
 779     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 780     result = allocate_memory_under_lock(req, in_new_region);
 781     // Do not call handle_alloc_failure() here, because we cannot block.
 782     // The allocation failure would be handled by the WB slowpath with handle_alloc_failure_evac().
 783   }
 784 
 785   if (in_new_region) {
 786     control_thread()->notify_heap_changed();
 787   }
 788 
 789   if (result != NULL) {
 790     size_t requested = req.size();
 791     size_t actual = req.actual_size();
 792 
 793     assert (req.is_lab_alloc() || (requested == actual),
 794             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 795             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 796 
 797     if (req.is_mutator_alloc()) {
 798       notify_mutator_alloc_words(actual, false);
 799 
 800       // If we requested more than we were granted, give the rest back to pacer.
 801       // This only matters if we are in the same pacing epoch: do not try to unpace
 802       // over the budget for the other phase.
 803       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 804         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 805       }
 806     } else {
 807       increase_used(actual*HeapWordSize);
 808     }
 809   }
 810 
 811   return result;
 812 }
 813 
 814 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 815   ShenandoahHeapLocker locker(lock());
 816   return _free_set->allocate(req, in_new_region);
 817 }
 818 
 819 class ShenandoahMemAllocator : public MemAllocator {
 820 private:
 821   MemAllocator& _initializer;
 822 public:
 823   ShenandoahMemAllocator(MemAllocator& initializer, Klass* klass, size_t word_size, Thread* thread) :
 824   MemAllocator(klass, word_size + ShenandoahBrooksPointer::word_size(), thread),
 825     _initializer(initializer) {}
 826 
 827 protected:
 828   virtual HeapWord* mem_allocate(Allocation& allocation) const {
 829     HeapWord* result = MemAllocator::mem_allocate(allocation);
 830     // Initialize brooks-pointer
 831     if (result != NULL) {
 832       result += ShenandoahBrooksPointer::word_size();
 833       ShenandoahBrooksPointer::initialize(oop(result));
 834       assert(! ShenandoahHeap::heap()->in_collection_set(result), "never allocate in targetted region");
 835     }
 836     return result;
 837   }
 838 
 839   virtual oop initialize(HeapWord* mem) const {
 840      return _initializer.initialize(mem);
 841   }
 842 };
 843 
 844 oop ShenandoahHeap::obj_allocate(Klass* klass, int size, TRAPS) {
 845   ObjAllocator initializer(klass, size, THREAD);
 846   ShenandoahMemAllocator allocator(initializer, klass, size, THREAD);
 847   return allocator.allocate();
 848 }
 849 
 850 oop ShenandoahHeap::array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS) {
 851   ObjArrayAllocator initializer(klass, size, length, do_zero, THREAD);
 852   ShenandoahMemAllocator allocator(initializer, klass, size, THREAD);
 853   return allocator.allocate();
 854 }
 855 
 856 oop ShenandoahHeap::class_allocate(Klass* klass, int size, TRAPS) {
 857   ClassAllocator initializer(klass, size, THREAD);
 858   ShenandoahMemAllocator allocator(initializer, klass, size, THREAD);
 859   return allocator.allocate();
 860 }
 861 
 862 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 863                                         bool*  gc_overhead_limit_was_exceeded) {
 864   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 865   return allocate_memory(req);
 866 }
 867 
 868 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 869                                                              size_t size,
 870                                                              Metaspace::MetadataType mdtype) {
 871   MetaWord* result;
 872 
 873   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 874   if (heuristics()->can_unload_classes()) {
 875     ShenandoahHeuristics* h = heuristics();
 876     h->record_metaspace_oom();
 877   }
 878 
 879   // Expand and retry allocation
 880   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 881   if (result != NULL) {
 882     return result;
 883   }
 884 
 885   // Start full GC
 886   collect(GCCause::_metadata_GC_clear_soft_refs);
 887 
 888   // Retry allocation
 889   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 890   if (result != NULL) {
 891     return result;
 892   }
 893 
 894   // Expand and retry allocation
 895   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 896   if (result != NULL) {
 897     return result;
 898   }
 899 
 900   // Out of memory
 901   return NULL;
 902 }
 903 
 904 void ShenandoahHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap) {
 905   HeapWord* obj = tlab_post_allocation_setup(start);
 906   CollectedHeap::fill_with_object(obj, end);
 907 }
 908 
 909 size_t ShenandoahHeap::min_dummy_object_size() const {
 910   return CollectedHeap::min_dummy_object_size() + ShenandoahBrooksPointer::word_size();
 911 }
 912 
 913 class ShenandoahEvacuateUpdateRootsClosure: public BasicOopIterateClosure {
 914 private:
 915   ShenandoahHeap* _heap;
 916   Thread* _thread;
 917 public:
 918   ShenandoahEvacuateUpdateRootsClosure() :
 919     _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
 920   }
 921 
 922 private:
 923   template <class T>
 924   void do_oop_work(T* p) {
 925     assert(_heap->is_evacuation_in_progress(), "Only do this when evacuation is in progress");
 926 
 927     T o = RawAccess<>::oop_load(p);
 928     if (! CompressedOops::is_null(o)) {
 929       oop obj = CompressedOops::decode_not_null(o);
 930       if (_heap->in_collection_set(obj)) {
 931         shenandoah_assert_marked(p, obj);
 932         oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 933         if (oopDesc::equals_raw(resolved, obj)) {
 934           resolved = _heap->evacuate_object(obj, _thread);
 935         }
 936         RawAccess<IS_NOT_NULL>::oop_store(p, resolved);
 937       }
 938     }
 939   }
 940 
 941 public:
 942   void do_oop(oop* p) {
 943     do_oop_work(p);
 944   }
 945   void do_oop(narrowOop* p) {
 946     do_oop_work(p);
 947   }
 948 };
 949 
 950 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
 951 private:
 952   ShenandoahHeap* const _heap;
 953   Thread* const _thread;
 954 public:
 955   ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
 956     _heap(heap), _thread(Thread::current()) {}
 957 
 958   void do_object(oop p) {
 959     shenandoah_assert_marked(NULL, p);
 960     if (oopDesc::equals_raw(p, ShenandoahBarrierSet::resolve_forwarded_not_null(p))) {
 961       _heap->evacuate_object(p, _thread);
 962     }
 963   }
 964 };
 965 
 966 class ShenandoahEvacuationTask : public AbstractGangTask {
 967 private:
 968   ShenandoahHeap* const _sh;
 969   ShenandoahCollectionSet* const _cs;
 970   bool _concurrent;
 971 public:
 972   ShenandoahEvacuationTask(ShenandoahHeap* sh,
 973                            ShenandoahCollectionSet* cs,
 974                            bool concurrent) :
 975     AbstractGangTask("Parallel Evacuation Task"),
 976     _sh(sh),
 977     _cs(cs),
 978     _concurrent(concurrent)
 979   {}
 980 
 981   void work(uint worker_id) {
 982     if (_concurrent) {
 983       ShenandoahConcurrentWorkerSession worker_session(worker_id);
 984       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 985       ShenandoahEvacOOMScope oom_evac_scope;
 986       do_work();
 987     } else {
 988       ShenandoahParallelWorkerSession worker_session(worker_id);
 989       ShenandoahEvacOOMScope oom_evac_scope;
 990       do_work();
 991     }
 992   }
 993 
 994 private:
 995   void do_work() {
 996     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
 997     ShenandoahHeapRegion* r;
 998     while ((r =_cs->claim_next()) != NULL) {
 999       assert(r->has_live(), "all-garbage regions are reclaimed early");
1000       _sh->marked_object_iterate(r, &cl);
1001 
1002       if (ShenandoahPacing) {
1003         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
1004       }
1005 
1006       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
1007         break;
1008       }
1009     }
1010   }
1011 };
1012 
1013 void ShenandoahHeap::trash_cset_regions() {
1014   ShenandoahHeapLocker locker(lock());
1015 
1016   ShenandoahCollectionSet* set = collection_set();
1017   ShenandoahHeapRegion* r;
1018   set->clear_current_index();
1019   while ((r = set->next()) != NULL) {
1020     r->make_trash();
1021   }
1022   collection_set()->clear();
1023 }
1024 
1025 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1026   st->print_cr("Heap Regions:");
1027   st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
1028   st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
1029   st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start (previous, next)");
1030   st->print_cr("SN=alloc sequence numbers (first mutator, last mutator, first gc, last gc)");
1031 
1032   for (size_t i = 0; i < num_regions(); i++) {
1033     get_region(i)->print_on(st);
1034   }
1035 }
1036 
1037 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1038   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1039 
1040   oop humongous_obj = oop(start->bottom() + ShenandoahBrooksPointer::word_size());
1041   size_t size = humongous_obj->size() + ShenandoahBrooksPointer::word_size();
1042   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1043   size_t index = start->region_number() + required_regions - 1;
1044 
1045   assert(!start->has_live(), "liveness must be zero");
1046 
1047   for(size_t i = 0; i < required_regions; i++) {
1048     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1049     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1050     ShenandoahHeapRegion* region = get_region(index --);
1051 
1052     assert(region->is_humongous(), "expect correct humongous start or continuation");
1053     assert(!region->is_cset(), "Humongous region should not be in collection set");
1054 
1055     region->make_trash_immediate();
1056   }
1057 }
1058 
1059 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1060 public:
1061   void do_thread(Thread* thread) {
1062     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1063     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1064     gclab->retire();
1065   }
1066 };
1067 
1068 void ShenandoahHeap::make_parsable(bool retire_tlabs) {
1069   if (UseTLAB) {
1070     CollectedHeap::ensure_parsability(retire_tlabs);
1071   }
1072   ShenandoahRetireGCLABClosure cl;
1073   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1074     cl.do_thread(t);
1075   }
1076   workers()->threads_do(&cl);
1077 }
1078 
1079 void ShenandoahHeap::resize_tlabs() {
1080   CollectedHeap::resize_all_tlabs();
1081 }
1082 
1083 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1084 private:
1085   ShenandoahRootEvacuator* _rp;
1086 
1087 public:
1088   ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1089     AbstractGangTask("Shenandoah evacuate and update roots"),
1090     _rp(rp) {}
1091 
1092   void work(uint worker_id) {
1093     ShenandoahParallelWorkerSession worker_session(worker_id);
1094     ShenandoahEvacOOMScope oom_evac_scope;
1095     ShenandoahEvacuateUpdateRootsClosure cl;
1096 
1097     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1098     _rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
1099   }
1100 };
1101 
1102 void ShenandoahHeap::evacuate_and_update_roots() {
1103 #if defined(COMPILER2) || INCLUDE_JVMCI
1104   DerivedPointerTable::clear();
1105 #endif
1106   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1107 
1108   {
1109     ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
1110     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1111     workers()->run_task(&roots_task);
1112   }
1113 
1114 #if defined(COMPILER2) || INCLUDE_JVMCI
1115   DerivedPointerTable::update_pointers();
1116 #endif
1117 }
1118 
1119 void ShenandoahHeap::roots_iterate(OopClosure* cl) {
1120   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1121 
1122   CodeBlobToOopClosure blobsCl(cl, false);
1123   CLDToOopClosure cldCl(cl, ClassLoaderData::_claim_strong);
1124 
1125   ShenandoahRootProcessor rp(this, 1, ShenandoahPhaseTimings::_num_phases);
1126   rp.process_all_roots(cl, NULL, &cldCl, &blobsCl, NULL, 0);
1127 }
1128 
1129 // Returns size in bytes
1130 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1131   if (ShenandoahElasticTLAB) {
1132     // With Elastic TLABs, return the max allowed size, and let the allocation path
1133     // figure out the safe size for current allocation.
1134     return ShenandoahHeapRegion::max_tlab_size_bytes();
1135   } else {
1136     return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1137   }
1138 }
1139 
1140 size_t ShenandoahHeap::max_tlab_size() const {
1141   // Returns size in words
1142   return ShenandoahHeapRegion::max_tlab_size_words();
1143 }
1144 
1145 class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
1146 public:
1147   void do_thread(Thread* thread) {
1148     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1149     gclab->retire();
1150     if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1151       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1152     }
1153   }
1154 };
1155 
1156 void ShenandoahHeap::retire_and_reset_gclabs() {
1157   ShenandoahRetireAndResetGCLABClosure cl;
1158   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1159     cl.do_thread(t);
1160   }
1161   workers()->threads_do(&cl);
1162 }
1163 
1164 void ShenandoahHeap::collect(GCCause::Cause cause) {
1165   control_thread()->request_gc(cause);
1166 }
1167 
1168 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1169   //assert(false, "Shouldn't need to do full collections");
1170 }
1171 
1172 CollectorPolicy* ShenandoahHeap::collector_policy() const {
1173   return _shenandoah_policy;
1174 }
1175 
1176 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1177   Space* sp = heap_region_containing(addr);
1178   if (sp != NULL) {
1179     return sp->block_start(addr);
1180   }
1181   return NULL;
1182 }
1183 
1184 size_t ShenandoahHeap::block_size(const HeapWord* addr) const {
1185   Space* sp = heap_region_containing(addr);
1186   assert(sp != NULL, "block_size of address outside of heap");
1187   return sp->block_size(addr);
1188 }
1189 
1190 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1191   Space* sp = heap_region_containing(addr);
1192   return sp->block_is_obj(addr);
1193 }
1194 
1195 jlong ShenandoahHeap::millis_since_last_gc() {
1196   double v = heuristics()->time_since_last_gc() * 1000;
1197   assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1198   return (jlong)v;
1199 }
1200 
1201 void ShenandoahHeap::prepare_for_verify() {
1202   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1203     make_parsable(false);
1204   }
1205 }
1206 
1207 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1208   workers()->print_worker_threads_on(st);
1209   if (ShenandoahStringDedup::is_enabled()) {
1210     ShenandoahStringDedup::print_worker_threads_on(st);
1211   }
1212 }
1213 
1214 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1215   workers()->threads_do(tcl);
1216   _safepoint_workers->threads_do(tcl);
1217   if (ShenandoahStringDedup::is_enabled()) {
1218     ShenandoahStringDedup::threads_do(tcl);
1219   }
1220 }
1221 
1222 void ShenandoahHeap::print_tracing_info() const {
1223   LogTarget(Info, gc, stats) lt;
1224   if (lt.is_enabled()) {
1225     ResourceMark rm;
1226     LogStream ls(lt);
1227 
1228     phase_timings()->print_on(&ls);
1229 
1230     ls.cr();
1231     ls.cr();
1232 
1233     shenandoah_policy()->print_gc_stats(&ls);
1234 
1235     ls.cr();
1236     ls.cr();
1237 
1238     if (ShenandoahPacing) {
1239       pacer()->print_on(&ls);
1240     }
1241 
1242     ls.cr();
1243     ls.cr();
1244 
1245     if (ShenandoahAllocationTrace) {
1246       assert(alloc_tracker() != NULL, "Must be");
1247       alloc_tracker()->print_on(&ls);
1248     } else {
1249       ls.print_cr("  Allocation tracing is disabled, use -XX:+ShenandoahAllocationTrace to enable.");
1250     }
1251   }
1252 }
1253 
1254 void ShenandoahHeap::verify(VerifyOption vo) {
1255   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1256     if (ShenandoahVerify) {
1257       verifier()->verify_generic(vo);
1258     } else {
1259       // TODO: Consider allocating verification bitmaps on demand,
1260       // and turn this on unconditionally.
1261     }
1262   }
1263 }
1264 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1265   return _free_set->capacity();
1266 }
1267 
1268 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1269 private:
1270   MarkBitMap* _bitmap;
1271   Stack<oop,mtGC>* _oop_stack;
1272 
1273   template <class T>
1274   void do_oop_work(T* p) {
1275     T o = RawAccess<>::oop_load(p);
1276     if (!CompressedOops::is_null(o)) {
1277       oop obj = CompressedOops::decode_not_null(o);
1278       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1279       assert(oopDesc::is_oop(obj), "must be a valid oop");
1280       if (!_bitmap->is_marked((HeapWord*) obj)) {
1281         _bitmap->mark((HeapWord*) obj);
1282         _oop_stack->push(obj);
1283       }
1284     }
1285   }
1286 public:
1287   ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1288     _bitmap(bitmap), _oop_stack(oop_stack) {}
1289   void do_oop(oop* p)       { do_oop_work(p); }
1290   void do_oop(narrowOop* p) { do_oop_work(p); }
1291 };
1292 
1293 /*
1294  * This is public API, used in preparation of object_iterate().
1295  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1296  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1297  * control, we call SH::make_tlabs_parsable().
1298  */
1299 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1300   // No-op.
1301 }
1302 
1303 /*
1304  * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1305  *
1306  * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1307  * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1308  * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1309  * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1310  * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1311  * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1312  * wiped the bitmap in preparation for next marking).
1313  *
1314  * For all those reasons, we implement object iteration as a single marking traversal, reporting
1315  * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1316  * is allowed to report dead objects, but is not required to do so.
1317  */
1318 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1319   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1320   if (!os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1321     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1322     return;
1323   }
1324 
1325   // Reset bitmap
1326   _aux_bit_map.clear();
1327 
1328   Stack<oop,mtGC> oop_stack;
1329 
1330   // First, we process all GC roots. This populates the work stack with initial objects.
1331   ShenandoahRootProcessor rp(this, 1, ShenandoahPhaseTimings::_num_phases);
1332   ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1333   CLDToOopClosure clds(&oops, ClassLoaderData::_claim_none);
1334   CodeBlobToOopClosure blobs(&oops, false);
1335   rp.process_all_roots(&oops, &oops, &clds, &blobs, NULL, 0);
1336 
1337   // Work through the oop stack to traverse heap.
1338   while (! oop_stack.is_empty()) {
1339     oop obj = oop_stack.pop();
1340     assert(oopDesc::is_oop(obj), "must be a valid oop");
1341     cl->do_object(obj);
1342     obj->oop_iterate(&oops);
1343   }
1344 
1345   assert(oop_stack.is_empty(), "should be empty");
1346 
1347   if (!os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1348     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1349   }
1350 }
1351 
1352 void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) {
1353   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1354   object_iterate(cl);
1355 }
1356 
1357 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1358   for (size_t i = 0; i < num_regions(); i++) {
1359     ShenandoahHeapRegion* current = get_region(i);
1360     blk->heap_region_do(current);
1361   }
1362 }
1363 
1364 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1365 private:
1366   ShenandoahHeap* const _heap;
1367   ShenandoahHeapRegionClosure* const _blk;
1368 
1369   DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
1370   volatile size_t _index;
1371   DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
1372 
1373 public:
1374   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1375           AbstractGangTask("Parallel Region Task"),
1376           _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1377 
1378   void work(uint worker_id) {
1379     size_t stride = ShenandoahParallelRegionStride;
1380 
1381     size_t max = _heap->num_regions();
1382     while (_index < max) {
1383       size_t cur = Atomic::add(stride, &_index) - stride;
1384       size_t start = cur;
1385       size_t end = MIN2(cur + stride, max);
1386       if (start >= max) break;
1387 
1388       for (size_t i = cur; i < end; i++) {
1389         ShenandoahHeapRegion* current = _heap->get_region(i);
1390         _blk->heap_region_do(current);
1391       }
1392     }
1393   }
1394 };
1395 
1396 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1397   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1398   if (num_regions() > ShenandoahParallelRegionStride) {
1399     ShenandoahParallelHeapRegionTask task(blk);
1400     workers()->run_task(&task);
1401   } else {
1402     heap_region_iterate(blk);
1403   }
1404 }
1405 
1406 class ShenandoahClearLivenessClosure : public ShenandoahHeapRegionClosure {
1407 private:
1408   ShenandoahMarkingContext* const _ctx;
1409 public:
1410   ShenandoahClearLivenessClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1411 
1412   void heap_region_do(ShenandoahHeapRegion* r) {
1413     if (r->is_active()) {
1414       r->clear_live_data();
1415       _ctx->capture_top_at_mark_start(r);
1416     } else {
1417       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->region_number());
1418       assert(_ctx->top_at_mark_start(r) == r->top(),
1419              "Region " SIZE_FORMAT " should already have correct TAMS", r->region_number());
1420     }
1421   }
1422 
1423   bool is_thread_safe() { return true; }
1424 };
1425 
1426 void ShenandoahHeap::op_init_mark() {
1427   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1428   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1429 
1430   assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1431   assert(!marking_context()->is_complete(), "should not be complete");
1432 
1433   if (ShenandoahVerify) {
1434     verifier()->verify_before_concmark();
1435   }
1436 
1437   if (VerifyBeforeGC) {
1438     Universe::verify();
1439   }
1440 
1441   set_concurrent_mark_in_progress(true);
1442   // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1443   {
1444     ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
1445     make_parsable(true);
1446   }
1447 
1448   {
1449     ShenandoahGCPhase phase(ShenandoahPhaseTimings::clear_liveness);
1450     ShenandoahClearLivenessClosure clc;
1451     parallel_heap_region_iterate(&clc);
1452   }
1453 
1454   // Make above changes visible to worker threads
1455   OrderAccess::fence();
1456 
1457   concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1458 
1459   if (UseTLAB) {
1460     ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
1461     resize_tlabs();
1462   }
1463 
1464   if (ShenandoahPacing) {
1465     pacer()->setup_for_mark();
1466   }
1467 }
1468 
1469 void ShenandoahHeap::op_mark() {
1470   concurrent_mark()->mark_from_roots();
1471 }
1472 
1473 class ShenandoahCompleteLivenessClosure : public ShenandoahHeapRegionClosure {
1474 private:
1475   ShenandoahMarkingContext* const _ctx;
1476 public:
1477   ShenandoahCompleteLivenessClosure() : _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
1478 
1479   void heap_region_do(ShenandoahHeapRegion* r) {
1480     if (r->is_active()) {
1481       HeapWord *tams = _ctx->top_at_mark_start(r);
1482       HeapWord *top = r->top();
1483       if (top > tams) {
1484         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1485       }
1486     } else {
1487       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->region_number());
1488       assert(_ctx->top_at_mark_start(r) == r->top(),
1489              "Region " SIZE_FORMAT " should have correct TAMS", r->region_number());
1490     }
1491   }
1492 
1493   bool is_thread_safe() { return true; }
1494 };
1495 
1496 void ShenandoahHeap::op_final_mark() {
1497   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1498 
1499   // It is critical that we
1500   // evacuate roots right after finishing marking, so that we don't
1501   // get unmarked objects in the roots.
1502 
1503   if (!cancelled_gc()) {
1504     concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1505 
1506     if (has_forwarded_objects()) {
1507       concurrent_mark()->update_roots(ShenandoahPhaseTimings::update_roots);
1508     }
1509 
1510     stop_concurrent_marking();
1511 
1512     {
1513       ShenandoahGCPhase phase(ShenandoahPhaseTimings::complete_liveness);
1514 
1515       // All allocations past TAMS are implicitly live, adjust the region data.
1516       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1517       ShenandoahCompleteLivenessClosure cl;
1518       parallel_heap_region_iterate(&cl);
1519     }
1520 
1521     {
1522       ShenandoahGCPhase prepare_evac(ShenandoahPhaseTimings::prepare_evac);
1523 
1524       make_parsable(true);
1525 
1526       trash_cset_regions();
1527 
1528       {
1529         ShenandoahHeapLocker locker(lock());
1530         _collection_set->clear();
1531         _free_set->clear();
1532 
1533         heuristics()->choose_collection_set(_collection_set);
1534 
1535         _free_set->rebuild();
1536       }
1537     }
1538 
1539     // If collection set has candidates, start evacuation.
1540     // Otherwise, bypass the rest of the cycle.
1541     if (!collection_set()->is_empty()) {
1542       ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1543 
1544       if (ShenandoahVerify) {
1545         verifier()->verify_before_evacuation();
1546       }
1547 
1548       set_evacuation_in_progress(true);
1549       // From here on, we need to update references.
1550       set_has_forwarded_objects(true);
1551 
1552       evacuate_and_update_roots();
1553 
1554       if (ShenandoahPacing) {
1555         pacer()->setup_for_evac();
1556       }
1557     } else {
1558       if (ShenandoahVerify) {
1559         verifier()->verify_after_concmark();
1560       }
1561 
1562       if (VerifyAfterGC) {
1563         Universe::verify();
1564       }
1565     }
1566 
1567   } else {
1568     concurrent_mark()->cancel();
1569     stop_concurrent_marking();
1570 
1571     if (process_references()) {
1572       // Abandon reference processing right away: pre-cleaning must have failed.
1573       ReferenceProcessor *rp = ref_processor();
1574       rp->disable_discovery();
1575       rp->abandon_partial_discovery();
1576       rp->verify_no_references_recorded();
1577     }
1578   }
1579 }
1580 
1581 void ShenandoahHeap::op_final_evac() {
1582   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1583 
1584   set_evacuation_in_progress(false);
1585 
1586   retire_and_reset_gclabs();
1587 
1588   if (ShenandoahVerify) {
1589     verifier()->verify_after_evacuation();
1590   }
1591 
1592   if (VerifyAfterGC) {
1593     Universe::verify();
1594   }
1595 }
1596 
1597 void ShenandoahHeap::op_conc_evac() {
1598   ShenandoahEvacuationTask task(this, _collection_set, true);
1599   workers()->run_task(&task);
1600 }
1601 
1602 void ShenandoahHeap::op_stw_evac() {
1603   ShenandoahEvacuationTask task(this, _collection_set, false);
1604   workers()->run_task(&task);
1605 }
1606 
1607 void ShenandoahHeap::op_updaterefs() {
1608   update_heap_references(true);
1609 }
1610 
1611 void ShenandoahHeap::op_cleanup() {
1612   free_set()->recycle_trash();
1613 }
1614 
1615 void ShenandoahHeap::op_reset() {
1616   reset_mark_bitmap();
1617 }
1618 
1619 void ShenandoahHeap::op_preclean() {
1620   concurrent_mark()->preclean_weak_refs();
1621 }
1622 
1623 void ShenandoahHeap::op_init_traversal() {
1624   traversal_gc()->init_traversal_collection();
1625 }
1626 
1627 void ShenandoahHeap::op_traversal() {
1628   traversal_gc()->concurrent_traversal_collection();
1629 }
1630 
1631 void ShenandoahHeap::op_final_traversal() {
1632   traversal_gc()->final_traversal_collection();
1633 }
1634 
1635 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1636   ShenandoahMetricsSnapshot metrics;
1637   metrics.snap_before();
1638 
1639   full_gc()->do_it(cause);
1640   if (UseTLAB) {
1641     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
1642     resize_all_tlabs();
1643   }
1644 
1645   metrics.snap_after();
1646   metrics.print();
1647 
1648   if (metrics.is_good_progress("Full GC")) {
1649     _progress_last_gc.set();
1650   } else {
1651     // Nothing to do. Tell the allocation path that we have failed to make
1652     // progress, and it can finally fail.
1653     _progress_last_gc.unset();
1654   }
1655 }
1656 
1657 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1658   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1659   // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1660   // some phase, we have to upgrade the Degenerate GC to Full GC.
1661 
1662   clear_cancelled_gc();
1663 
1664   ShenandoahMetricsSnapshot metrics;
1665   metrics.snap_before();
1666 
1667   switch (point) {
1668     case _degenerated_traversal:
1669       {
1670         // Drop the collection set. Note: this leaves some already forwarded objects
1671         // behind, which may be problematic, see comments for ShenandoahEvacAssist
1672         // workarounds in ShenandoahTraversalHeuristics.
1673 
1674         ShenandoahHeapLocker locker(lock());
1675         collection_set()->clear_current_index();
1676         for (size_t i = 0; i < collection_set()->count(); i++) {
1677           ShenandoahHeapRegion* r = collection_set()->next();
1678           r->make_regular_bypass();
1679         }
1680         collection_set()->clear();
1681       }
1682       op_final_traversal();
1683       op_cleanup();
1684       return;
1685 
1686     // The cases below form the Duff's-like device: it describes the actual GC cycle,
1687     // but enters it at different points, depending on which concurrent phase had
1688     // degenerated.
1689 
1690     case _degenerated_outside_cycle:
1691       // We have degenerated from outside the cycle, which means something is bad with
1692       // the heap, most probably heavy humongous fragmentation, or we are very low on free
1693       // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1694       // we can do the most aggressive degen cycle, which includes processing references and
1695       // class unloading, unless those features are explicitly disabled.
1696       //
1697       // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1698       // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1699       set_process_references(heuristics()->can_process_references());
1700       set_unload_classes(heuristics()->can_unload_classes());
1701 
1702       if (heuristics()->can_do_traversal_gc()) {
1703         // Not possible to degenerate from here, upgrade to Full GC right away.
1704         cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1705         op_degenerated_fail();
1706         return;
1707       }
1708 
1709       op_reset();
1710 
1711       op_init_mark();
1712       if (cancelled_gc()) {
1713         op_degenerated_fail();
1714         return;
1715       }
1716 
1717     case _degenerated_mark:
1718       op_final_mark();
1719       if (cancelled_gc()) {
1720         op_degenerated_fail();
1721         return;
1722       }
1723 
1724       op_cleanup();
1725 
1726     case _degenerated_evac:
1727       // If heuristics thinks we should do the cycle, this flag would be set,
1728       // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1729       if (is_evacuation_in_progress()) {
1730 
1731         // Degeneration under oom-evac protocol might have left some objects in
1732         // collection set un-evacuated. Restart evacuation from the beginning to
1733         // capture all objects. For all the objects that are already evacuated,
1734         // it would be a simple check, which is supposed to be fast. This is also
1735         // safe to do even without degeneration, as CSet iterator is at beginning
1736         // in preparation for evacuation anyway.
1737         collection_set()->clear_current_index();
1738 
1739         op_stw_evac();
1740         if (cancelled_gc()) {
1741           op_degenerated_fail();
1742           return;
1743         }
1744       }
1745 
1746       // If heuristics thinks we should do the cycle, this flag would be set,
1747       // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
1748       if (has_forwarded_objects()) {
1749         op_init_updaterefs();
1750         if (cancelled_gc()) {
1751           op_degenerated_fail();
1752           return;
1753         }
1754       }
1755 
1756     case _degenerated_updaterefs:
1757       if (has_forwarded_objects()) {
1758         op_final_updaterefs();
1759         if (cancelled_gc()) {
1760           op_degenerated_fail();
1761           return;
1762         }
1763       }
1764 
1765       op_cleanup();
1766       break;
1767 
1768     default:
1769       ShouldNotReachHere();
1770   }
1771 
1772   if (ShenandoahVerify) {
1773     verifier()->verify_after_degenerated();
1774   }
1775 
1776   if (VerifyAfterGC) {
1777     Universe::verify();
1778   }
1779 
1780   metrics.snap_after();
1781   metrics.print();
1782 
1783   // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
1784   // because that probably means the heap is overloaded and/or fragmented.
1785   if (!metrics.is_good_progress("Degenerated GC")) {
1786     _progress_last_gc.unset();
1787     cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1788     op_degenerated_futile();
1789   } else {
1790     _progress_last_gc.set();
1791   }
1792 }
1793 
1794 void ShenandoahHeap::op_degenerated_fail() {
1795   log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
1796   shenandoah_policy()->record_degenerated_upgrade_to_full();
1797   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
1798 }
1799 
1800 void ShenandoahHeap::op_degenerated_futile() {
1801   shenandoah_policy()->record_degenerated_upgrade_to_full();
1802   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
1803 }
1804 
1805 void ShenandoahHeap::stop_concurrent_marking() {
1806   assert(is_concurrent_mark_in_progress(), "How else could we get here?");
1807   if (!cancelled_gc()) {
1808     // If we needed to update refs, and concurrent marking has been cancelled,
1809     // we need to finish updating references.
1810     set_has_forwarded_objects(false);
1811     mark_complete_marking_context();
1812   }
1813   set_concurrent_mark_in_progress(false);
1814 }
1815 
1816 void ShenandoahHeap::force_satb_flush_all_threads() {
1817   if (!is_concurrent_mark_in_progress() && !is_concurrent_traversal_in_progress()) {
1818     // No need to flush SATBs
1819     return;
1820   }
1821 
1822   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1823     ShenandoahThreadLocalData::set_force_satb_flush(t, true);
1824   }
1825   // The threads are not "acquiring" their thread-local data, but it does not
1826   // hurt to "release" the updates here anyway.
1827   OrderAccess::fence();
1828 }
1829 
1830 void ShenandoahHeap::set_gc_state_all_threads(char state) {
1831   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1832     ShenandoahThreadLocalData::set_gc_state(t, state);
1833   }
1834 }
1835 
1836 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
1837   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
1838   _gc_state.set_cond(mask, value);
1839   set_gc_state_all_threads(_gc_state.raw_value());
1840 }
1841 
1842 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
1843   set_gc_state_mask(MARKING, in_progress);
1844   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1845 }
1846 
1847 void ShenandoahHeap::set_concurrent_traversal_in_progress(bool in_progress) {
1848    set_gc_state_mask(TRAVERSAL | HAS_FORWARDED, in_progress);
1849    ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1850 }
1851 
1852 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
1853   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
1854   set_gc_state_mask(EVACUATION, in_progress);
1855 }
1856 
1857 HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
1858   // Initialize Brooks pointer for the next object
1859   HeapWord* result = obj + ShenandoahBrooksPointer::word_size();
1860   ShenandoahBrooksPointer::initialize(oop(result));
1861   return result;
1862 }
1863 
1864 ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
1865   _mark_context(ShenandoahHeap::heap()->marking_context()) {
1866 }
1867 
1868 ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() :
1869   _mark_context(ShenandoahHeap::heap()->marking_context()) {
1870 }
1871 
1872 bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
1873   if (CompressedOops::is_null(obj)) {
1874     return false;
1875   }
1876   obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1877   shenandoah_assert_not_forwarded_if(NULL, obj, ShenandoahHeap::heap()->is_concurrent_mark_in_progress() || ShenandoahHeap::heap()->is_concurrent_traversal_in_progress());
1878   return _mark_context->is_marked(obj);
1879 }
1880 
1881 bool ShenandoahIsAliveClosure::do_object_b(oop obj) {
1882   if (CompressedOops::is_null(obj)) {
1883     return false;
1884   }
1885   shenandoah_assert_not_forwarded(NULL, obj);
1886   return _mark_context->is_marked(obj);
1887 }
1888 
1889 void ShenandoahHeap::ref_processing_init() {
1890   assert(_max_workers > 0, "Sanity");
1891 
1892   _ref_processor =
1893     new ReferenceProcessor(&_subject_to_discovery,  // is_subject_to_discovery
1894                            ParallelRefProcEnabled,  // MT processing
1895                            _max_workers,            // Degree of MT processing
1896                            true,                    // MT discovery
1897                            _max_workers,            // Degree of MT discovery
1898                            false,                   // Reference discovery is not atomic
1899                            NULL,                    // No closure, should be installed before use
1900                            true);                   // Scale worker threads
1901 
1902   shenandoah_assert_rp_isalive_not_installed();
1903 }
1904 
1905 GCTracer* ShenandoahHeap::tracer() {
1906   return shenandoah_policy()->tracer();
1907 }
1908 
1909 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
1910   return _free_set->used();
1911 }
1912 
1913 bool ShenandoahHeap::try_cancel_gc() {
1914   while (true) {
1915     jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
1916     if (prev == CANCELLABLE) return true;
1917     else if (prev == CANCELLED) return false;
1918     assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
1919     assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
1920     {
1921       // We need to provide a safepoint here, otherwise we might
1922       // spin forever if a SP is pending.
1923       ThreadBlockInVM sp(JavaThread::current());
1924       SpinPause();
1925     }
1926   }
1927 }
1928 
1929 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
1930   if (try_cancel_gc()) {
1931     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
1932     log_info(gc)("%s", msg.buffer());
1933     Events::log(Thread::current(), "%s", msg.buffer());
1934   }
1935 }
1936 
1937 uint ShenandoahHeap::max_workers() {
1938   return _max_workers;
1939 }
1940 
1941 void ShenandoahHeap::stop() {
1942   // The shutdown sequence should be able to terminate when GC is running.
1943 
1944   // Step 0. Notify policy to disable event recording.
1945   _shenandoah_policy->record_shutdown();
1946 
1947   // Step 1. Notify control thread that we are in shutdown.
1948   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
1949   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
1950   control_thread()->prepare_for_graceful_shutdown();
1951 
1952   // Step 2. Notify GC workers that we are cancelling GC.
1953   cancel_gc(GCCause::_shenandoah_stop_vm);
1954 
1955   // Step 3. Wait until GC worker exits normally.
1956   control_thread()->stop();
1957 
1958   // Step 4. Stop String Dedup thread if it is active
1959   if (ShenandoahStringDedup::is_enabled()) {
1960     ShenandoahStringDedup::stop();
1961   }
1962 }
1963 
1964 void ShenandoahHeap::unload_classes_and_cleanup_tables(bool full_gc) {
1965   assert(heuristics()->can_unload_classes(), "Class unloading should be enabled");
1966 
1967   ShenandoahGCPhase root_phase(full_gc ?
1968                                ShenandoahPhaseTimings::full_gc_purge :
1969                                ShenandoahPhaseTimings::purge);
1970 
1971   ShenandoahIsAliveSelector alive;
1972   BoolObjectClosure* is_alive = alive.is_alive_closure();
1973 
1974   bool purged_class;
1975 
1976   // Unload classes and purge SystemDictionary.
1977   {
1978     ShenandoahGCPhase phase(full_gc ?
1979                             ShenandoahPhaseTimings::full_gc_purge_class_unload :
1980                             ShenandoahPhaseTimings::purge_class_unload);
1981     purged_class = SystemDictionary::do_unloading(gc_timer());
1982   }
1983 
1984   {
1985     ShenandoahGCPhase phase(full_gc ?
1986                             ShenandoahPhaseTimings::full_gc_purge_par :
1987                             ShenandoahPhaseTimings::purge_par);
1988     uint active = _workers->active_workers();
1989     ParallelCleaningTask unlink_task(is_alive, active, purged_class, true);
1990     _workers->run_task(&unlink_task);
1991   }
1992 
1993   {
1994     ShenandoahGCPhase phase(full_gc ?
1995                       ShenandoahPhaseTimings::full_gc_purge_cldg :
1996                       ShenandoahPhaseTimings::purge_cldg);
1997     ClassLoaderDataGraph::purge();
1998   }
1999 }
2000 
2001 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2002   set_gc_state_mask(HAS_FORWARDED, cond);
2003 }
2004 
2005 void ShenandoahHeap::set_process_references(bool pr) {
2006   _process_references.set_cond(pr);
2007 }
2008 
2009 void ShenandoahHeap::set_unload_classes(bool uc) {
2010   _unload_classes.set_cond(uc);
2011 }
2012 
2013 bool ShenandoahHeap::process_references() const {
2014   return _process_references.is_set();
2015 }
2016 
2017 bool ShenandoahHeap::unload_classes() const {
2018   return _unload_classes.is_set();
2019 }
2020 
2021 address ShenandoahHeap::in_cset_fast_test_addr() {
2022   ShenandoahHeap* heap = ShenandoahHeap::heap();
2023   assert(heap->collection_set() != NULL, "Sanity");
2024   return (address) heap->collection_set()->biased_map_address();
2025 }
2026 
2027 address ShenandoahHeap::cancelled_gc_addr() {
2028   return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
2029 }
2030 
2031 address ShenandoahHeap::gc_state_addr() {
2032   return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
2033 }
2034 
2035 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
2036   return OrderAccess::load_acquire(&_bytes_allocated_since_gc_start);
2037 }
2038 
2039 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2040   OrderAccess::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
2041 }
2042 
2043 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2044   _degenerated_gc_in_progress.set_cond(in_progress);
2045 }
2046 
2047 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2048   _full_gc_in_progress.set_cond(in_progress);
2049 }
2050 
2051 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2052   assert (is_full_gc_in_progress(), "should be");
2053   _full_gc_move_in_progress.set_cond(in_progress);
2054 }
2055 
2056 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2057   set_gc_state_mask(UPDATEREFS, in_progress);
2058 }
2059 
2060 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2061   ShenandoahCodeRoots::add_nmethod(nm);
2062 }
2063 
2064 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2065   ShenandoahCodeRoots::remove_nmethod(nm);
2066 }
2067 
2068 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2069   o = ShenandoahBarrierSet::barrier_set()->write_barrier(o);
2070   ShenandoahHeapLocker locker(lock());
2071   heap_region_containing(o)->make_pinned();
2072   return o;
2073 }
2074 
2075 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2076   o = ShenandoahBarrierSet::barrier_set()->read_barrier(o);
2077   ShenandoahHeapLocker locker(lock());
2078   heap_region_containing(o)->make_unpinned();
2079 }
2080 
2081 GCTimer* ShenandoahHeap::gc_timer() const {
2082   return _gc_timer;
2083 }
2084 
2085 #ifdef ASSERT
2086 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2087   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2088 
2089   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2090     if (UseDynamicNumberOfGCThreads ||
2091         (FLAG_IS_DEFAULT(ParallelGCThreads) && ForceDynamicNumberOfGCThreads)) {
2092       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2093     } else {
2094       // Use ParallelGCThreads inside safepoints
2095       assert(nworkers == ParallelGCThreads, "Use ParalleGCThreads within safepoints");
2096     }
2097   } else {
2098     if (UseDynamicNumberOfGCThreads ||
2099         (FLAG_IS_DEFAULT(ConcGCThreads) && ForceDynamicNumberOfGCThreads)) {
2100       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2101     } else {
2102       // Use ConcGCThreads outside safepoints
2103       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2104     }
2105   }
2106 }
2107 #endif
2108 
2109 ShenandoahVerifier* ShenandoahHeap::verifier() {
2110   guarantee(ShenandoahVerify, "Should be enabled");
2111   assert (_verifier != NULL, "sanity");
2112   return _verifier;
2113 }
2114 
2115 template<class T>
2116 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2117 private:
2118   T cl;
2119   ShenandoahHeap* _heap;
2120   ShenandoahRegionIterator* _regions;
2121   bool _concurrent;
2122 public:
2123   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2124     AbstractGangTask("Concurrent Update References Task"),
2125     cl(T()),
2126     _heap(ShenandoahHeap::heap()),
2127     _regions(regions),
2128     _concurrent(concurrent) {
2129   }
2130 
2131   void work(uint worker_id) {
2132     if (_concurrent) {
2133       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2134       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2135       do_work();
2136     } else {
2137       ShenandoahParallelWorkerSession worker_session(worker_id);
2138       do_work();
2139     }
2140   }
2141 
2142 private:
2143   void do_work() {
2144     ShenandoahHeapRegion* r = _regions->next();
2145     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2146     while (r != NULL) {
2147       HeapWord* top_at_start_ur = r->concurrent_iteration_safe_limit();
2148       assert (top_at_start_ur >= r->bottom(), "sanity");
2149       if (r->is_active() && !r->is_cset()) {
2150         _heap->marked_object_oop_iterate(r, &cl, top_at_start_ur);
2151       }
2152       if (ShenandoahPacing) {
2153         _heap->pacer()->report_updaterefs(pointer_delta(top_at_start_ur, r->bottom()));
2154       }
2155       if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2156         return;
2157       }
2158       r = _regions->next();
2159     }
2160   }
2161 };
2162 
2163 void ShenandoahHeap::update_heap_references(bool concurrent) {
2164   ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2165   workers()->run_task(&task);
2166 }
2167 
2168 void ShenandoahHeap::op_init_updaterefs() {
2169   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2170 
2171   set_evacuation_in_progress(false);
2172 
2173   retire_and_reset_gclabs();
2174 
2175   if (ShenandoahVerify) {
2176     verifier()->verify_before_updaterefs();
2177   }
2178 
2179   set_update_refs_in_progress(true);
2180   make_parsable(true);
2181   for (uint i = 0; i < num_regions(); i++) {
2182     ShenandoahHeapRegion* r = get_region(i);
2183     r->set_concurrent_iteration_safe_limit(r->top());
2184   }
2185 
2186   // Reset iterator.
2187   _update_refs_iterator.reset();
2188 
2189   if (ShenandoahPacing) {
2190     pacer()->setup_for_updaterefs();
2191   }
2192 }
2193 
2194 void ShenandoahHeap::op_final_updaterefs() {
2195   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2196 
2197   // Check if there is left-over work, and finish it
2198   if (_update_refs_iterator.has_next()) {
2199     ShenandoahGCPhase final_work(ShenandoahPhaseTimings::final_update_refs_finish_work);
2200 
2201     // Finish updating references where we left off.
2202     clear_cancelled_gc();
2203     update_heap_references(false);
2204   }
2205 
2206   // Clear cancelled GC, if set. On cancellation path, the block before would handle
2207   // everything. On degenerated paths, cancelled gc would not be set anyway.
2208   if (cancelled_gc()) {
2209     clear_cancelled_gc();
2210   }
2211   assert(!cancelled_gc(), "Should have been done right before");
2212 
2213   concurrent_mark()->update_roots(is_degenerated_gc_in_progress() ?
2214                                  ShenandoahPhaseTimings::degen_gc_update_roots:
2215                                  ShenandoahPhaseTimings::final_update_refs_roots);
2216 
2217   ShenandoahGCPhase final_update_refs(ShenandoahPhaseTimings::final_update_refs_recycle);
2218 
2219   trash_cset_regions();
2220   set_has_forwarded_objects(false);
2221   set_update_refs_in_progress(false);
2222 
2223   if (ShenandoahVerify) {
2224     verifier()->verify_after_updaterefs();
2225   }
2226 
2227   if (VerifyAfterGC) {
2228     Universe::verify();
2229   }
2230 
2231   {
2232     ShenandoahHeapLocker locker(lock());
2233     _free_set->rebuild();
2234   }
2235 }
2236 
2237 #ifdef ASSERT
2238 void ShenandoahHeap::assert_heaplock_owned_by_current_thread() {
2239   _lock.assert_owned_by_current_thread();
2240 }
2241 
2242 void ShenandoahHeap::assert_heaplock_not_owned_by_current_thread() {
2243   _lock.assert_not_owned_by_current_thread();
2244 }
2245 
2246 void ShenandoahHeap::assert_heaplock_or_safepoint() {
2247   _lock.assert_owned_by_current_thread_or_safepoint();
2248 }
2249 #endif
2250 
2251 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2252   print_on(st);
2253   print_heap_regions_on(st);
2254 }
2255 
2256 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2257   size_t slice = r->region_number() / _bitmap_regions_per_slice;
2258 
2259   size_t regions_from = _bitmap_regions_per_slice * slice;
2260   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2261   for (size_t g = regions_from; g < regions_to; g++) {
2262     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2263     if (skip_self && g == r->region_number()) continue;
2264     if (get_region(g)->is_committed()) {
2265       return true;
2266     }
2267   }
2268   return false;
2269 }
2270 
2271 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2272   assert_heaplock_owned_by_current_thread();
2273 
2274   if (is_bitmap_slice_committed(r, true)) {
2275     // Some other region from the group is already committed, meaning the bitmap
2276     // slice is already committed, we exit right away.
2277     return true;
2278   }
2279 
2280   // Commit the bitmap slice:
2281   size_t slice = r->region_number() / _bitmap_regions_per_slice;
2282   size_t off = _bitmap_bytes_per_slice * slice;
2283   size_t len = _bitmap_bytes_per_slice;
2284   if (!os::commit_memory((char*)_bitmap_region.start() + off, len, false)) {
2285     return false;
2286   }
2287   return true;
2288 }
2289 
2290 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2291   assert_heaplock_owned_by_current_thread();
2292 
2293   if (is_bitmap_slice_committed(r, true)) {
2294     // Some other region from the group is still committed, meaning the bitmap
2295     // slice is should stay committed, exit right away.
2296     return true;
2297   }
2298 
2299   // Uncommit the bitmap slice:
2300   size_t slice = r->region_number() / _bitmap_regions_per_slice;
2301   size_t off = _bitmap_bytes_per_slice * slice;
2302   size_t len = _bitmap_bytes_per_slice;
2303   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2304     return false;
2305   }
2306   return true;
2307 }
2308 
2309 void ShenandoahHeap::safepoint_synchronize_begin() {
2310   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2311     SuspendibleThreadSet::synchronize();
2312   }
2313 }
2314 
2315 void ShenandoahHeap::safepoint_synchronize_end() {
2316   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2317     SuspendibleThreadSet::desynchronize();
2318   }
2319 }
2320 
2321 void ShenandoahHeap::vmop_entry_init_mark() {
2322   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2323   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2324   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark_gross);
2325 
2326   try_inject_alloc_failure();
2327   VM_ShenandoahInitMark op;
2328   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2329 }
2330 
2331 void ShenandoahHeap::vmop_entry_final_mark() {
2332   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2333   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2334   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark_gross);
2335 
2336   try_inject_alloc_failure();
2337   VM_ShenandoahFinalMarkStartEvac op;
2338   VMThread::execute(&op); // jump to entry_final_mark under safepoint
2339 }
2340 
2341 void ShenandoahHeap::vmop_entry_final_evac() {
2342   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2343   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2344   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac_gross);
2345 
2346   VM_ShenandoahFinalEvac op;
2347   VMThread::execute(&op); // jump to entry_final_evac under safepoint
2348 }
2349 
2350 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2351   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2352   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2353   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_gross);
2354 
2355   try_inject_alloc_failure();
2356   VM_ShenandoahInitUpdateRefs op;
2357   VMThread::execute(&op);
2358 }
2359 
2360 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2361   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2362   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2363   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_gross);
2364 
2365   try_inject_alloc_failure();
2366   VM_ShenandoahFinalUpdateRefs op;
2367   VMThread::execute(&op);
2368 }
2369 
2370 void ShenandoahHeap::vmop_entry_init_traversal() {
2371   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2372   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2373   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc_gross);
2374 
2375   try_inject_alloc_failure();
2376   VM_ShenandoahInitTraversalGC op;
2377   VMThread::execute(&op);
2378 }
2379 
2380 void ShenandoahHeap::vmop_entry_final_traversal() {
2381   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2382   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2383   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc_gross);
2384 
2385   try_inject_alloc_failure();
2386   VM_ShenandoahFinalTraversalGC op;
2387   VMThread::execute(&op);
2388 }
2389 
2390 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2391   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2392   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2393   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_gross);
2394 
2395   try_inject_alloc_failure();
2396   VM_ShenandoahFullGC op(cause);
2397   VMThread::execute(&op);
2398 }
2399 
2400 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2401   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2402   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2403   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_gross);
2404 
2405   VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2406   VMThread::execute(&degenerated_gc);
2407 }
2408 
2409 void ShenandoahHeap::entry_init_mark() {
2410   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2411   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark);
2412   const char* msg = init_mark_event_message();
2413   GCTraceTime(Info, gc) time(msg, gc_timer());
2414   EventMark em("%s", msg);
2415 
2416   ShenandoahWorkerScope scope(workers(),
2417                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2418                               "init marking");
2419 
2420   op_init_mark();
2421 }
2422 
2423 void ShenandoahHeap::entry_final_mark() {
2424   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2425   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark);
2426   const char* msg = final_mark_event_message();
2427   GCTraceTime(Info, gc) time(msg, gc_timer());
2428   EventMark em("%s", msg);
2429 
2430   ShenandoahWorkerScope scope(workers(),
2431                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2432                               "final marking");
2433 
2434   op_final_mark();
2435 }
2436 
2437 void ShenandoahHeap::entry_final_evac() {
2438   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2439   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac);
2440   static const char* msg = "Pause Final Evac";
2441   GCTraceTime(Info, gc) time(msg, gc_timer());
2442   EventMark em("%s", msg);
2443 
2444   op_final_evac();
2445 }
2446 
2447 void ShenandoahHeap::entry_init_updaterefs() {
2448   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2449   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs);
2450 
2451   static const char* msg = "Pause Init Update Refs";
2452   GCTraceTime(Info, gc) time(msg, gc_timer());
2453   EventMark em("%s", msg);
2454 
2455   // No workers used in this phase, no setup required
2456 
2457   op_init_updaterefs();
2458 }
2459 
2460 void ShenandoahHeap::entry_final_updaterefs() {
2461   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2462   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs);
2463 
2464   static const char* msg = "Pause Final Update Refs";
2465   GCTraceTime(Info, gc) time(msg, gc_timer());
2466   EventMark em("%s", msg);
2467 
2468   ShenandoahWorkerScope scope(workers(),
2469                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2470                               "final reference update");
2471 
2472   op_final_updaterefs();
2473 }
2474 
2475 void ShenandoahHeap::entry_init_traversal() {
2476   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2477   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc);
2478 
2479   static const char* msg = "Pause Init Traversal";
2480   GCTraceTime(Info, gc) time(msg, gc_timer());
2481   EventMark em("%s", msg);
2482 
2483   ShenandoahWorkerScope scope(workers(),
2484                               ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(),
2485                               "init traversal");
2486 
2487   op_init_traversal();
2488 }
2489 
2490 void ShenandoahHeap::entry_final_traversal() {
2491   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2492   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc);
2493 
2494   static const char* msg = "Pause Final Traversal";
2495   GCTraceTime(Info, gc) time(msg, gc_timer());
2496   EventMark em("%s", msg);
2497 
2498   ShenandoahWorkerScope scope(workers(),
2499                               ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(),
2500                               "final traversal");
2501 
2502   op_final_traversal();
2503 }
2504 
2505 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2506   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2507   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc);
2508 
2509   static const char* msg = "Pause Full";
2510   GCTraceTime(Info, gc) time(msg, gc_timer(), cause, true);
2511   EventMark em("%s", msg);
2512 
2513   ShenandoahWorkerScope scope(workers(),
2514                               ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2515                               "full gc");
2516 
2517   op_full(cause);
2518 }
2519 
2520 void ShenandoahHeap::entry_degenerated(int point) {
2521   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2522   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc);
2523 
2524   ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2525   const char* msg = degen_event_message(dpoint);
2526   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2527   EventMark em("%s", msg);
2528 
2529   ShenandoahWorkerScope scope(workers(),
2530                               ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2531                               "stw degenerated gc");
2532 
2533   set_degenerated_gc_in_progress(true);
2534   op_degenerated(dpoint);
2535   set_degenerated_gc_in_progress(false);
2536 }
2537 
2538 void ShenandoahHeap::entry_mark() {
2539   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2540 
2541   const char* msg = conc_mark_event_message();
2542   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2543   EventMark em("%s", msg);
2544 
2545   ShenandoahWorkerScope scope(workers(),
2546                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2547                               "concurrent marking");
2548 
2549   try_inject_alloc_failure();
2550   op_mark();
2551 }
2552 
2553 void ShenandoahHeap::entry_evac() {
2554   ShenandoahGCPhase conc_evac_phase(ShenandoahPhaseTimings::conc_evac);
2555   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2556 
2557   static const char* msg = "Concurrent evacuation";
2558   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2559   EventMark em("%s", msg);
2560 
2561   ShenandoahWorkerScope scope(workers(),
2562                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2563                               "concurrent evacuation");
2564 
2565   try_inject_alloc_failure();
2566   op_conc_evac();
2567 }
2568 
2569 void ShenandoahHeap::entry_updaterefs() {
2570   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2571 
2572   static const char* msg = "Concurrent update references";
2573   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2574   EventMark em("%s", msg);
2575 
2576   ShenandoahWorkerScope scope(workers(),
2577                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2578                               "concurrent reference update");
2579 
2580   try_inject_alloc_failure();
2581   op_updaterefs();
2582 }
2583 void ShenandoahHeap::entry_cleanup() {
2584   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
2585 
2586   static const char* msg = "Concurrent cleanup";
2587   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2588   EventMark em("%s", msg);
2589 
2590   // This phase does not use workers, no need for setup
2591 
2592   try_inject_alloc_failure();
2593   op_cleanup();
2594 }
2595 
2596 void ShenandoahHeap::entry_reset() {
2597   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_reset);
2598 
2599   static const char* msg = "Concurrent reset";
2600   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2601   EventMark em("%s", msg);
2602 
2603   ShenandoahWorkerScope scope(workers(),
2604                               ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2605                               "concurrent reset");
2606 
2607   try_inject_alloc_failure();
2608   op_reset();
2609 }
2610 
2611 void ShenandoahHeap::entry_preclean() {
2612   if (ShenandoahPreclean && process_references()) {
2613     static const char* msg = "Concurrent precleaning";
2614     GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2615     EventMark em("%s", msg);
2616 
2617     ShenandoahGCPhase conc_preclean(ShenandoahPhaseTimings::conc_preclean);
2618 
2619     ShenandoahWorkerScope scope(workers(),
2620                                 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2621                                 "concurrent preclean",
2622                                 /* check_workers = */ false);
2623 
2624     try_inject_alloc_failure();
2625     op_preclean();
2626   }
2627 }
2628 
2629 void ShenandoahHeap::entry_traversal() {
2630   static const char* msg = "Concurrent traversal";
2631   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2632   EventMark em("%s", msg);
2633 
2634   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2635 
2636   ShenandoahWorkerScope scope(workers(),
2637                               ShenandoahWorkerPolicy::calc_workers_for_conc_traversal(),
2638                               "concurrent traversal");
2639 
2640   try_inject_alloc_failure();
2641   op_traversal();
2642 }
2643 
2644 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2645   static const char *msg = "Concurrent uncommit";
2646   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2647   EventMark em("%s", msg);
2648 
2649   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_uncommit);
2650 
2651   op_uncommit(shrink_before);
2652 }
2653 
2654 void ShenandoahHeap::try_inject_alloc_failure() {
2655   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2656     _inject_alloc_failure.set();
2657     os::naked_short_sleep(1);
2658     if (cancelled_gc()) {
2659       log_info(gc)("Allocation failure was successfully injected");
2660     }
2661   }
2662 }
2663 
2664 bool ShenandoahHeap::should_inject_alloc_failure() {
2665   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2666 }
2667 
2668 void ShenandoahHeap::initialize_serviceability() {
2669   _memory_pool = new ShenandoahMemoryPool(this);
2670   _cycle_memory_manager.add_pool(_memory_pool);
2671   _stw_memory_manager.add_pool(_memory_pool);
2672 }
2673 
2674 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2675   GrowableArray<GCMemoryManager*> memory_managers(2);
2676   memory_managers.append(&_cycle_memory_manager);
2677   memory_managers.append(&_stw_memory_manager);
2678   return memory_managers;
2679 }
2680 
2681 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2682   GrowableArray<MemoryPool*> memory_pools(1);
2683   memory_pools.append(_memory_pool);
2684   return memory_pools;
2685 }
2686 
2687 MemoryUsage ShenandoahHeap::memory_usage() {
2688   return _memory_pool->get_memory_usage();
2689 }
2690 
2691 void ShenandoahHeap::enter_evacuation() {
2692   _oom_evac_handler.enter_evacuation();
2693 }
2694 
2695 void ShenandoahHeap::leave_evacuation() {
2696   _oom_evac_handler.leave_evacuation();
2697 }
2698 
2699 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2700   _heap(ShenandoahHeap::heap()),
2701   _index(0) {}
2702 
2703 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2704   _heap(heap),
2705   _index(0) {}
2706 
2707 void ShenandoahRegionIterator::reset() {
2708   _index = 0;
2709 }
2710 
2711 bool ShenandoahRegionIterator::has_next() const {
2712   return _index < _heap->num_regions();
2713 }
2714 
2715 char ShenandoahHeap::gc_state() const {
2716   return _gc_state.raw_value();
2717 }
2718 
2719 void ShenandoahHeap::deduplicate_string(oop str) {
2720   assert(java_lang_String::is_instance(str), "invariant");
2721 
2722   if (ShenandoahStringDedup::is_enabled()) {
2723     ShenandoahStringDedup::deduplicate(str);
2724   }
2725 }
2726 
2727 const char* ShenandoahHeap::init_mark_event_message() const {
2728   bool update_refs = has_forwarded_objects();
2729   bool proc_refs = process_references();
2730   bool unload_cls = unload_classes();
2731 
2732   if (update_refs && proc_refs && unload_cls) {
2733     return "Pause Init Mark (update refs) (process weakrefs) (unload classes)";
2734   } else if (update_refs && proc_refs) {
2735     return "Pause Init Mark (update refs) (process weakrefs)";
2736   } else if (update_refs && unload_cls) {
2737     return "Pause Init Mark (update refs) (unload classes)";
2738   } else if (proc_refs && unload_cls) {
2739     return "Pause Init Mark (process weakrefs) (unload classes)";
2740   } else if (update_refs) {
2741     return "Pause Init Mark (update refs)";
2742   } else if (proc_refs) {
2743     return "Pause Init Mark (process weakrefs)";
2744   } else if (unload_cls) {
2745     return "Pause Init Mark (unload classes)";
2746   } else {
2747     return "Pause Init Mark";
2748   }
2749 }
2750 
2751 const char* ShenandoahHeap::final_mark_event_message() const {
2752   bool update_refs = has_forwarded_objects();
2753   bool proc_refs = process_references();
2754   bool unload_cls = unload_classes();
2755 
2756   if (update_refs && proc_refs && unload_cls) {
2757     return "Pause Final Mark (update refs) (process weakrefs) (unload classes)";
2758   } else if (update_refs && proc_refs) {
2759     return "Pause Final Mark (update refs) (process weakrefs)";
2760   } else if (update_refs && unload_cls) {
2761     return "Pause Final Mark (update refs) (unload classes)";
2762   } else if (proc_refs && unload_cls) {
2763     return "Pause Final Mark (process weakrefs) (unload classes)";
2764   } else if (update_refs) {
2765     return "Pause Final Mark (update refs)";
2766   } else if (proc_refs) {
2767     return "Pause Final Mark (process weakrefs)";
2768   } else if (unload_cls) {
2769     return "Pause Final Mark (unload classes)";
2770   } else {
2771     return "Pause Final Mark";
2772   }
2773 }
2774 
2775 const char* ShenandoahHeap::conc_mark_event_message() const {
2776   bool update_refs = has_forwarded_objects();
2777   bool proc_refs = process_references();
2778   bool unload_cls = unload_classes();
2779 
2780   if (update_refs && proc_refs && unload_cls) {
2781     return "Concurrent marking (update refs) (process weakrefs) (unload classes)";
2782   } else if (update_refs && proc_refs) {
2783     return "Concurrent marking (update refs) (process weakrefs)";
2784   } else if (update_refs && unload_cls) {
2785     return "Concurrent marking (update refs) (unload classes)";
2786   } else if (proc_refs && unload_cls) {
2787     return "Concurrent marking (process weakrefs) (unload classes)";
2788   } else if (update_refs) {
2789     return "Concurrent marking (update refs)";
2790   } else if (proc_refs) {
2791     return "Concurrent marking (process weakrefs)";
2792   } else if (unload_cls) {
2793     return "Concurrent marking (unload classes)";
2794   } else {
2795     return "Concurrent marking";
2796   }
2797 }
2798 
2799 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
2800   switch (point) {
2801     case _degenerated_unset:
2802       return "Pause Degenerated GC (<UNSET>)";
2803     case _degenerated_traversal:
2804       return "Pause Degenerated GC (Traversal)";
2805     case _degenerated_outside_cycle:
2806       return "Pause Degenerated GC (Outside of Cycle)";
2807     case _degenerated_mark:
2808       return "Pause Degenerated GC (Mark)";
2809     case _degenerated_evac:
2810       return "Pause Degenerated GC (Evacuation)";
2811     case _degenerated_updaterefs:
2812       return "Pause Degenerated GC (Update Refs)";
2813     default:
2814       ShouldNotReachHere();
2815       return "ERROR";
2816   }
2817 }
2818 
2819 jushort* ShenandoahHeap::get_liveness_cache(uint worker_id) {
2820 #ifdef ASSERT
2821   assert(_liveness_cache != NULL, "sanity");
2822   assert(worker_id < _max_workers, "sanity");
2823   for (uint i = 0; i < num_regions(); i++) {
2824     assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
2825   }
2826 #endif
2827   return _liveness_cache[worker_id];
2828 }
2829 
2830 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
2831   assert(worker_id < _max_workers, "sanity");
2832   assert(_liveness_cache != NULL, "sanity");
2833   jushort* ld = _liveness_cache[worker_id];
2834   for (uint i = 0; i < num_regions(); i++) {
2835     ShenandoahHeapRegion* r = get_region(i);
2836     jushort live = ld[i];
2837     if (live > 0) {
2838       r->increase_live_data_gc_words(live);
2839       ld[i] = 0;
2840     }
2841   }
2842 }
2843 
2844 size_t ShenandoahHeap::obj_size(oop obj) const {
2845   return CollectedHeap::obj_size(obj) + ShenandoahBrooksPointer::word_size();
2846 }
2847 
2848 ptrdiff_t ShenandoahHeap::cell_header_size() const {
2849   return ShenandoahBrooksPointer::byte_size();
2850 }
2851 
2852 BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
2853   return ShenandoahHeap::heap()->has_forwarded_objects() ? reinterpret_cast<BoolObjectClosure*>(&_fwd_alive_cl)
2854                                                          : reinterpret_cast<BoolObjectClosure*>(&_alive_cl);
2855 }