1 /*
   2  * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.hpp"
  26 
  27 #include "gc/shared/gcTimer.hpp"
  28 #include "gc/shared/gcTraceTime.inline.hpp"
  29 #include "gc/shared/memAllocator.hpp"
  30 #include "gc/shared/parallelCleaning.hpp"
  31 #include "gc/shared/plab.hpp"
  32 
  33 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
  34 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  35 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"
  36 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  37 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  38 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  39 #include "gc/shenandoah/shenandoahControlThread.hpp"
  40 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  41 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  42 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  43 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  44 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  45 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  46 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  47 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  48 #include "gc/shenandoah/shenandoahMetrics.hpp"
  49 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  50 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  51 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  52 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
  53 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  54 #include "gc/shenandoah/shenandoahUtils.hpp"
  55 #include "gc/shenandoah/shenandoahVerifier.hpp"
  56 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  57 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  58 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  59 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  60 #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp"
  61 #include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp"
  62 #include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp"
  63 #include "gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp"
  64 #include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp"
  65 #include "gc/shenandoah/heuristics/shenandoahTraversalHeuristics.hpp"
  66 
  67 #include "memory/metaspace.hpp"
  68 #include "runtime/interfaceSupport.inline.hpp"
  69 #include "runtime/safepointMechanism.hpp"
  70 #include "runtime/vmThread.hpp"
  71 #include "services/mallocTracker.hpp"
  72 
  73 ShenandoahUpdateRefsClosure::ShenandoahUpdateRefsClosure() : _heap(ShenandoahHeap::heap()) {}
  74 
  75 #ifdef ASSERT
  76 template <class T>
  77 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
  78   T o = RawAccess<>::oop_load(p);
  79   if (! CompressedOops::is_null(o)) {
  80     oop obj = CompressedOops::decode_not_null(o);
  81     shenandoah_assert_not_forwarded(p, obj);
  82   }
  83 }
  84 
  85 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
  86 void ShenandoahAssertToSpaceClosure::do_oop(oop* p)       { do_oop_work(p); }
  87 #endif
  88 
  89 class ShenandoahPretouchHeapTask : public AbstractGangTask {
  90 private:
  91   ShenandoahRegionIterator _regions;
  92   const size_t _page_size;
  93 public:
  94   ShenandoahPretouchHeapTask(size_t page_size) :
  95     AbstractGangTask("Shenandoah Pretouch Heap"),
  96     _page_size(page_size) {}
  97 
  98   virtual void work(uint worker_id) {
  99     ShenandoahHeapRegion* r = _regions.next();
 100     while (r != NULL) {
 101       os::pretouch_memory(r->bottom(), r->end(), _page_size);
 102       r = _regions.next();
 103     }
 104   }
 105 };
 106 
 107 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
 108 private:
 109   ShenandoahRegionIterator _regions;
 110   char* _bitmap_base;
 111   const size_t _bitmap_size;
 112   const size_t _page_size;
 113 public:
 114   ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
 115     AbstractGangTask("Shenandoah Pretouch Bitmap"),
 116     _bitmap_base(bitmap_base),
 117     _bitmap_size(bitmap_size),
 118     _page_size(page_size) {}
 119 
 120   virtual void work(uint worker_id) {
 121     ShenandoahHeapRegion* r = _regions.next();
 122     while (r != NULL) {
 123       size_t start = r->region_number()       * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 124       size_t end   = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 125       assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
 126 
 127       os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
 128 
 129       r = _regions.next();
 130     }
 131   }
 132 };
 133 
 134 jint ShenandoahHeap::initialize() {
 135   ShenandoahBrooksPointer::initial_checks();
 136 
 137   initialize_heuristics();
 138 
 139   //
 140   // Figure out heap sizing
 141   //
 142 
 143   size_t init_byte_size = collector_policy()->initial_heap_byte_size();
 144   size_t max_byte_size  = collector_policy()->max_heap_byte_size();
 145   size_t heap_alignment = collector_policy()->heap_alignment();
 146 
 147   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 148 
 149   if (ShenandoahAlwaysPreTouch) {
 150     // Enabled pre-touch means the entire heap is committed right away.
 151     init_byte_size = max_byte_size;
 152   }
 153 
 154   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 155   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 156 
 157   _num_regions = ShenandoahHeapRegion::region_count();
 158 
 159   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 160   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 161   assert(num_committed_regions <= _num_regions, "sanity");
 162 
 163   _initial_size = num_committed_regions * reg_size_bytes;
 164   _committed = _initial_size;
 165 
 166   size_t heap_page_size   = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 167   size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 168 
 169   //
 170   // Reserve and commit memory for heap
 171   //
 172 
 173   ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 174   initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*) (heap_rs.base() + heap_rs.size()));
 175   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 176   _heap_region_special = heap_rs.special();
 177 
 178   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 179          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 180 
 181   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 182   if (!_heap_region_special) {
 183     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 184                               "Cannot commit heap memory");
 185   }
 186 
 187   //
 188   // Reserve and commit memory for bitmap(s)
 189   //
 190 
 191   _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
 192   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 193 
 194   size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
 195 
 196   guarantee(bitmap_bytes_per_region != 0,
 197             "Bitmap bytes per region should not be zero");
 198   guarantee(is_power_of_2(bitmap_bytes_per_region),
 199             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 200 
 201   if (bitmap_page_size > bitmap_bytes_per_region) {
 202     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 203     _bitmap_bytes_per_slice = bitmap_page_size;
 204   } else {
 205     _bitmap_regions_per_slice = 1;
 206     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 207   }
 208 
 209   guarantee(_bitmap_regions_per_slice >= 1,
 210             "Should have at least one region per slice: " SIZE_FORMAT,
 211             _bitmap_regions_per_slice);
 212 
 213   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 214             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 215             _bitmap_bytes_per_slice, bitmap_page_size);
 216 
 217   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 218   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 219   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 220   _bitmap_region_special = bitmap.special();
 221 
 222   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 223                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 224   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 225   if (!_bitmap_region_special) {
 226     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 227                               "Cannot commit bitmap memory");
 228   }
 229 
 230   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 231 
 232   if (ShenandoahVerify) {
 233     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 234     if (!verify_bitmap.special()) {
 235       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 236                                 "Cannot commit verification bitmap memory");
 237     }
 238     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 239     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 240     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 241     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 242   }
 243 
 244   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 245   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
 246   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 247   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 248   _aux_bitmap_region_special = aux_bitmap.special();
 249   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 250 
 251   //
 252   // Create regions and region sets
 253   //
 254 
 255   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 256   _free_set = new ShenandoahFreeSet(this, _num_regions);
 257   _collection_set = new ShenandoahCollectionSet(this, (HeapWord*)sh_rs.base());
 258 
 259   {
 260     ShenandoahHeapLocker locker(lock());
 261     for (size_t i = 0; i < _num_regions; i++) {
 262       ShenandoahHeapRegion* r = new ShenandoahHeapRegion(
 263               this,
 264               (HeapWord*) sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i,
 265               ShenandoahHeapRegion::region_size_words(),
 266               i,
 267               i < num_committed_regions
 268       );
 269 
 270       _marking_context->initialize_top_at_mark_start(r);
 271       _regions[i] = r;
 272       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 273     }
 274 
 275     // Initialize to complete
 276     _marking_context->mark_complete();
 277 
 278     _free_set->rebuild();
 279   }
 280 
 281   if (ShenandoahAlwaysPreTouch) {
 282     assert(!AlwaysPreTouch, "Should have been overridden");
 283 
 284     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 285     // before initialize() below zeroes it with initializing thread. For any given region,
 286     // we touch the region and the corresponding bitmaps from the same thread.
 287     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 288 
 289     size_t pretouch_heap_page_size = heap_page_size;
 290     size_t pretouch_bitmap_page_size = bitmap_page_size;
 291 
 292 #ifdef LINUX
 293     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 294     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 295     // them into huge one. Therefore, we need to pretouch with smaller pages.
 296     if (UseTransparentHugePages) {
 297       pretouch_heap_page_size = (size_t)os::vm_page_size();
 298       pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 299     }
 300 #endif
 301 
 302     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 303     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 304 
 305     log_info(gc, init)("Pretouch bitmap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 306                        _num_regions, pretouch_bitmap_page_size);
 307     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, pretouch_bitmap_page_size);
 308     _workers->run_task(&bcl);
 309 
 310     log_info(gc, init)("Pretouch heap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 311                        _num_regions, pretouch_heap_page_size);
 312     ShenandoahPretouchHeapTask hcl(pretouch_heap_page_size);
 313     _workers->run_task(&hcl);
 314   }
 315 
 316   //
 317   // Initialize the rest of GC subsystems
 318   //
 319 
 320   _liveness_cache = NEW_C_HEAP_ARRAY(jushort*, _max_workers, mtGC);
 321   for (uint worker = 0; worker < _max_workers; worker++) {
 322     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(jushort, _num_regions, mtGC);
 323     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(jushort));
 324   }
 325 
 326   // The call below uses stuff (the SATB* things) that are in G1, but probably
 327   // belong into a shared location.
 328   ShenandoahBarrierSet::satb_mark_queue_set().initialize(this,
 329                                                          SATB_Q_CBL_mon,
 330                                                          20 /* G1SATBProcessCompletedThreshold */,
 331                                                          60 /* G1SATBBufferEnqueueingThresholdPercent */);
 332 
 333   _monitoring_support = new ShenandoahMonitoringSupport(this);
 334   _phase_timings = new ShenandoahPhaseTimings();
 335   ShenandoahStringDedup::initialize();
 336   ShenandoahCodeRoots::initialize();
 337 
 338   if (ShenandoahAllocationTrace) {
 339     _alloc_tracker = new ShenandoahAllocTracker();
 340   }
 341 
 342   if (ShenandoahPacing) {
 343     _pacer = new ShenandoahPacer(this);
 344     _pacer->setup_for_idle();
 345   } else {
 346     _pacer = NULL;
 347   }
 348 
 349   _traversal_gc = heuristics()->can_do_traversal_gc() ?
 350                   new ShenandoahTraversalGC(this, _num_regions) :
 351                   NULL;
 352 
 353   _control_thread = new ShenandoahControlThread();
 354 
 355   log_info(gc, init)("Initialize Shenandoah heap with initial size " SIZE_FORMAT "%s",
 356                      byte_size_in_proper_unit(_initial_size), proper_unit_for_byte_size(_initial_size));
 357 
 358   log_info(gc, init)("Safepointing mechanism: %s",
 359                      SafepointMechanism::uses_thread_local_poll() ? "thread-local poll" :
 360                      (SafepointMechanism::uses_global_page_poll() ? "global-page poll" : "unknown"));
 361 
 362   return JNI_OK;
 363 }
 364 
 365 void ShenandoahHeap::initialize_heuristics() {
 366   if (ShenandoahGCHeuristics != NULL) {
 367     if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) {
 368       _heuristics = new ShenandoahAggressiveHeuristics();
 369     } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) {
 370       _heuristics = new ShenandoahStaticHeuristics();
 371     } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) {
 372       _heuristics = new ShenandoahAdaptiveHeuristics();
 373     } else if (strcmp(ShenandoahGCHeuristics, "passive") == 0) {
 374       _heuristics = new ShenandoahPassiveHeuristics();
 375     } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) {
 376       _heuristics = new ShenandoahCompactHeuristics();
 377     } else if (strcmp(ShenandoahGCHeuristics, "traversal") == 0) {
 378       _heuristics = new ShenandoahTraversalHeuristics();
 379     } else {
 380       vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option");
 381     }
 382 
 383     if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 384       vm_exit_during_initialization(
 385               err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 386                       _heuristics->name()));
 387     }
 388     if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 389       vm_exit_during_initialization(
 390               err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 391                       _heuristics->name()));
 392     }
 393 
 394     if (ShenandoahStoreValEnqueueBarrier && ShenandoahStoreValReadBarrier) {
 395       vm_exit_during_initialization("Cannot use both ShenandoahStoreValEnqueueBarrier and ShenandoahStoreValReadBarrier");
 396     }
 397     log_info(gc, init)("Shenandoah heuristics: %s",
 398                        _heuristics->name());
 399   } else {
 400       ShouldNotReachHere();
 401   }
 402 
 403 }
 404 
 405 #ifdef _MSC_VER
 406 #pragma warning( push )
 407 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 408 #endif
 409 
 410 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 411   CollectedHeap(),
 412   _initial_size(0),
 413   _used(0),
 414   _committed(0),
 415   _bytes_allocated_since_gc_start(0),
 416   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 417   _workers(NULL),
 418   _safepoint_workers(NULL),
 419   _heap_region_special(false),
 420   _num_regions(0),
 421   _regions(NULL),
 422   _update_refs_iterator(this),
 423   _control_thread(NULL),
 424   _shenandoah_policy(policy),
 425   _heuristics(NULL),
 426   _free_set(NULL),
 427   _scm(new ShenandoahConcurrentMark()),
 428   _traversal_gc(NULL),
 429   _full_gc(new ShenandoahMarkCompact()),
 430   _pacer(NULL),
 431   _verifier(NULL),
 432   _alloc_tracker(NULL),
 433   _phase_timings(NULL),
 434   _monitoring_support(NULL),
 435   _memory_pool(NULL),
 436   _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
 437   _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
 438   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
 439   _soft_ref_policy(),
 440   _ref_processor(NULL),
 441   _marking_context(NULL),
 442   _bitmap_size(0),
 443   _bitmap_regions_per_slice(0),
 444   _bitmap_bytes_per_slice(0),
 445   _bitmap_region_special(false),
 446   _aux_bitmap_region_special(false),
 447   _liveness_cache(NULL),
 448   _collection_set(NULL)
 449 {
 450   log_info(gc, init)("GC threads: " UINT32_FORMAT " parallel, " UINT32_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads);
 451   log_info(gc, init)("Reference processing: %s", ParallelRefProcEnabled ? "parallel" : "serial");
 452 
 453   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 454 
 455   _max_workers = MAX2(_max_workers, 1U);
 456   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
 457                             /* are_GC_task_threads */true,
 458                             /* are_ConcurrentGC_threads */false);
 459   if (_workers == NULL) {
 460     vm_exit_during_initialization("Failed necessary allocation.");
 461   } else {
 462     _workers->initialize_workers();
 463   }
 464 
 465   if (ShenandoahParallelSafepointThreads > 1) {
 466     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
 467                                                 ShenandoahParallelSafepointThreads,
 468                                                 false, false);
 469     _safepoint_workers->initialize_workers();
 470   }
 471 }
 472 
 473 #ifdef _MSC_VER
 474 #pragma warning( pop )
 475 #endif
 476 
 477 class ShenandoahResetBitmapTask : public AbstractGangTask {
 478 private:
 479   ShenandoahRegionIterator _regions;
 480 
 481 public:
 482   ShenandoahResetBitmapTask() :
 483     AbstractGangTask("Parallel Reset Bitmap Task") {}
 484 
 485   void work(uint worker_id) {
 486     ShenandoahHeapRegion* region = _regions.next();
 487     ShenandoahHeap* heap = ShenandoahHeap::heap();
 488     ShenandoahMarkingContext* const ctx = heap->marking_context();
 489     while (region != NULL) {
 490       if (heap->is_bitmap_slice_committed(region)) {
 491         ctx->clear_bitmap(region);
 492       }
 493       region = _regions.next();
 494     }
 495   }
 496 };
 497 
 498 void ShenandoahHeap::reset_mark_bitmap() {
 499   assert_gc_workers(_workers->active_workers());
 500   mark_incomplete_marking_context();
 501 
 502   ShenandoahResetBitmapTask task;
 503   _workers->run_task(&task);
 504 }
 505 
 506 void ShenandoahHeap::print_on(outputStream* st) const {
 507   st->print_cr("Shenandoah Heap");
 508   st->print_cr(" " SIZE_FORMAT "K total, " SIZE_FORMAT "K committed, " SIZE_FORMAT "K used",
 509                capacity() / K, committed() / K, used() / K);
 510   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"K regions",
 511                num_regions(), ShenandoahHeapRegion::region_size_bytes() / K);
 512 
 513   st->print("Status: ");
 514   if (has_forwarded_objects())               st->print("has forwarded objects, ");
 515   if (is_concurrent_mark_in_progress())      st->print("marking, ");
 516   if (is_evacuation_in_progress())           st->print("evacuating, ");
 517   if (is_update_refs_in_progress())          st->print("updating refs, ");
 518   if (is_concurrent_traversal_in_progress()) st->print("traversal, ");
 519   if (is_degenerated_gc_in_progress())       st->print("degenerated gc, ");
 520   if (is_full_gc_in_progress())              st->print("full gc, ");
 521   if (is_full_gc_move_in_progress())         st->print("full gc move, ");
 522 
 523   if (cancelled_gc()) {
 524     st->print("cancelled");
 525   } else {
 526     st->print("not cancelled");
 527   }
 528   st->cr();
 529 
 530   st->print_cr("Reserved region:");
 531   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 532                p2i(reserved_region().start()),
 533                p2i(reserved_region().end()));
 534 
 535   st->cr();
 536   MetaspaceUtils::print_on(st);
 537 
 538   if (Verbose) {
 539     print_heap_regions_on(st);
 540   }
 541 }
 542 
 543 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 544 public:
 545   void do_thread(Thread* thread) {
 546     assert(thread != NULL, "Sanity");
 547     assert(thread->is_Worker_thread(), "Only worker thread expected");
 548     ShenandoahThreadLocalData::initialize_gclab(thread);
 549   }
 550 };
 551 
 552 void ShenandoahHeap::post_initialize() {
 553   CollectedHeap::post_initialize();
 554   MutexLocker ml(Threads_lock);
 555 
 556   ShenandoahInitWorkerGCLABClosure init_gclabs;
 557   _workers->threads_do(&init_gclabs);
 558 
 559   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 560   // Now, we will let WorkGang to initialize gclab when new worker is created.
 561   _workers->set_initialize_gclab();
 562 
 563   _scm->initialize(_max_workers);
 564   _full_gc->initialize(_gc_timer);
 565 
 566   ref_processing_init();
 567 
 568   _heuristics->initialize();
 569 }
 570 
 571 size_t ShenandoahHeap::used() const {
 572   return OrderAccess::load_acquire(&_used);
 573 }
 574 
 575 size_t ShenandoahHeap::committed() const {
 576   OrderAccess::acquire();
 577   return _committed;
 578 }
 579 
 580 void ShenandoahHeap::increase_committed(size_t bytes) {
 581   assert_heaplock_or_safepoint();
 582   _committed += bytes;
 583 }
 584 
 585 void ShenandoahHeap::decrease_committed(size_t bytes) {
 586   assert_heaplock_or_safepoint();
 587   _committed -= bytes;
 588 }
 589 
 590 void ShenandoahHeap::increase_used(size_t bytes) {
 591   Atomic::add(bytes, &_used);
 592 }
 593 
 594 void ShenandoahHeap::set_used(size_t bytes) {
 595   OrderAccess::release_store_fence(&_used, bytes);
 596 }
 597 
 598 void ShenandoahHeap::decrease_used(size_t bytes) {
 599   assert(used() >= bytes, "never decrease heap size by more than we've left");
 600   Atomic::sub(bytes, &_used);
 601 }
 602 
 603 void ShenandoahHeap::increase_allocated(size_t bytes) {
 604   Atomic::add(bytes, &_bytes_allocated_since_gc_start);
 605 }
 606 
 607 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 608   size_t bytes = words * HeapWordSize;
 609   if (!waste) {
 610     increase_used(bytes);
 611   }
 612   increase_allocated(bytes);
 613   if (ShenandoahPacing) {
 614     control_thread()->pacing_notify_alloc(words);
 615     if (waste) {
 616       pacer()->claim_for_alloc(words, true);
 617     }
 618   }
 619 }
 620 
 621 size_t ShenandoahHeap::capacity() const {
 622   return num_regions() * ShenandoahHeapRegion::region_size_bytes();
 623 }
 624 
 625 size_t ShenandoahHeap::max_capacity() const {
 626   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 627 }
 628 
 629 size_t ShenandoahHeap::initial_capacity() const {
 630   return _initial_size;
 631 }
 632 
 633 bool ShenandoahHeap::is_in(const void* p) const {
 634   HeapWord* heap_base = (HeapWord*) base();
 635   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 636   return p >= heap_base && p < last_region_end;
 637 }
 638 
 639 void ShenandoahHeap::op_uncommit(double shrink_before) {
 640   assert (ShenandoahUncommit, "should be enabled");
 641 
 642   size_t count = 0;
 643   for (size_t i = 0; i < num_regions(); i++) {
 644     ShenandoahHeapRegion* r = get_region(i);
 645     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 646       ShenandoahHeapLocker locker(lock());
 647       if (r->is_empty_committed()) {
 648         r->make_uncommitted();
 649         count++;
 650       }
 651     }
 652     SpinPause(); // allow allocators to take the lock
 653   }
 654 
 655   if (count > 0) {
 656     log_info(gc)("Uncommitted " SIZE_FORMAT "M. Heap: " SIZE_FORMAT "M reserved, " SIZE_FORMAT "M committed, " SIZE_FORMAT "M used",
 657                  count * ShenandoahHeapRegion::region_size_bytes() / M, capacity() / M, committed() / M, used() / M);
 658     control_thread()->notify_heap_changed();
 659   }
 660 }
 661 
 662 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 663   // New object should fit the GCLAB size
 664   size_t min_size = MAX2(size, PLAB::min_size());
 665 
 666   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 667   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 668   new_size = MIN2(new_size, PLAB::max_size());
 669   new_size = MAX2(new_size, PLAB::min_size());
 670 
 671   // Record new heuristic value even if we take any shortcut. This captures
 672   // the case when moderately-sized objects always take a shortcut. At some point,
 673   // heuristics should catch up with them.
 674   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 675 
 676   if (new_size < size) {
 677     // New size still does not fit the object. Fall back to shared allocation.
 678     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 679     return NULL;
 680   }
 681 
 682   // Retire current GCLAB, and allocate a new one.
 683   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 684   gclab->retire();
 685 
 686   size_t actual_size = 0;
 687   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 688   if (gclab_buf == NULL) {
 689     return NULL;
 690   }
 691 
 692   assert (size <= actual_size, "allocation should fit");
 693 
 694   if (ZeroTLAB) {
 695     // ..and clear it.
 696     Copy::zero_to_words(gclab_buf, actual_size);
 697   } else {
 698     // ...and zap just allocated object.
 699 #ifdef ASSERT
 700     // Skip mangling the space corresponding to the object header to
 701     // ensure that the returned space is not considered parsable by
 702     // any concurrent GC thread.
 703     size_t hdr_size = oopDesc::header_size();
 704     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 705 #endif // ASSERT
 706   }
 707   gclab->set_buf(gclab_buf, actual_size);
 708   return gclab->allocate(size);
 709 }
 710 
 711 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 712                                             size_t requested_size,
 713                                             size_t* actual_size) {
 714   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 715   HeapWord* res = allocate_memory(req);
 716   if (res != NULL) {
 717     *actual_size = req.actual_size();
 718   } else {
 719     *actual_size = 0;
 720   }
 721   return res;
 722 }
 723 
 724 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 725                                              size_t word_size,
 726                                              size_t* actual_size) {
 727   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 728   HeapWord* res = allocate_memory(req);
 729   if (res != NULL) {
 730     *actual_size = req.actual_size();
 731   } else {
 732     *actual_size = 0;
 733   }
 734   return res;
 735 }
 736 
 737 ShenandoahHeap* ShenandoahHeap::heap() {
 738   CollectedHeap* heap = Universe::heap();
 739   assert(heap != NULL, "Unitialized access to ShenandoahHeap::heap()");
 740   assert(heap->kind() == CollectedHeap::Shenandoah, "not a shenandoah heap");
 741   return (ShenandoahHeap*) heap;
 742 }
 743 
 744 ShenandoahHeap* ShenandoahHeap::heap_no_check() {
 745   CollectedHeap* heap = Universe::heap();
 746   return (ShenandoahHeap*) heap;
 747 }
 748 
 749 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 750   ShenandoahAllocTrace trace_alloc(req.size(), req.type());
 751 
 752   intptr_t pacer_epoch = 0;
 753   bool in_new_region = false;
 754   HeapWord* result = NULL;
 755 
 756   if (req.is_mutator_alloc()) {
 757     if (ShenandoahPacing) {
 758       pacer()->pace_for_alloc(req.size());
 759       pacer_epoch = pacer()->epoch();
 760     }
 761 
 762     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 763       result = allocate_memory_under_lock(req, in_new_region);
 764     }
 765 
 766     // Allocation failed, block until control thread reacted, then retry allocation.
 767     //
 768     // It might happen that one of the threads requesting allocation would unblock
 769     // way later after GC happened, only to fail the second allocation, because
 770     // other threads have already depleted the free storage. In this case, a better
 771     // strategy is to try again, as long as GC makes progress.
 772     //
 773     // Then, we need to make sure the allocation was retried after at least one
 774     // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
 775 
 776     size_t tries = 0;
 777 
 778     while (result == NULL && _progress_last_gc.is_set()) {
 779       tries++;
 780       control_thread()->handle_alloc_failure(req.size());
 781       result = allocate_memory_under_lock(req, in_new_region);
 782     }
 783 
 784     while (result == NULL && tries <= ShenandoahFullGCThreshold) {
 785       tries++;
 786       control_thread()->handle_alloc_failure(req.size());
 787       result = allocate_memory_under_lock(req, in_new_region);
 788     }
 789 
 790   } else {
 791     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 792     result = allocate_memory_under_lock(req, in_new_region);
 793     // Do not call handle_alloc_failure() here, because we cannot block.
 794     // The allocation failure would be handled by the WB slowpath with handle_alloc_failure_evac().
 795   }
 796 
 797   if (in_new_region) {
 798     control_thread()->notify_heap_changed();
 799   }
 800 
 801   if (result != NULL) {
 802     size_t requested = req.size();
 803     size_t actual = req.actual_size();
 804 
 805     assert (req.is_lab_alloc() || (requested == actual),
 806             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 807             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 808 
 809     if (req.is_mutator_alloc()) {
 810       notify_mutator_alloc_words(actual, false);
 811 
 812       // If we requested more than we were granted, give the rest back to pacer.
 813       // This only matters if we are in the same pacing epoch: do not try to unpace
 814       // over the budget for the other phase.
 815       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 816         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 817       }
 818     } else {
 819       increase_used(actual*HeapWordSize);
 820     }
 821   }
 822 
 823   return result;
 824 }
 825 
 826 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 827   ShenandoahHeapLocker locker(lock());
 828   return _free_set->allocate(req, in_new_region);
 829 }
 830 
 831 class ShenandoahMemAllocator : public MemAllocator {
 832 private:
 833   MemAllocator& _initializer;
 834 public:
 835   ShenandoahMemAllocator(MemAllocator& initializer, Klass* klass, size_t word_size, Thread* thread) :
 836   MemAllocator(klass, word_size + ShenandoahBrooksPointer::word_size(), thread),
 837     _initializer(initializer) {}
 838 
 839 protected:
 840   virtual HeapWord* mem_allocate(Allocation& allocation) const {
 841     HeapWord* result = MemAllocator::mem_allocate(allocation);
 842     // Initialize brooks-pointer
 843     if (result != NULL) {
 844       result += ShenandoahBrooksPointer::word_size();
 845       ShenandoahBrooksPointer::initialize(oop(result));
 846       assert(! ShenandoahHeap::heap()->in_collection_set(result), "never allocate in targetted region");
 847     }
 848     return result;
 849   }
 850 
 851   virtual oop initialize(HeapWord* mem) const {
 852      return _initializer.initialize(mem);
 853   }
 854 };
 855 
 856 oop ShenandoahHeap::obj_allocate(Klass* klass, int size, TRAPS) {
 857   ObjAllocator initializer(klass, size, THREAD);
 858   ShenandoahMemAllocator allocator(initializer, klass, size, THREAD);
 859   return allocator.allocate();
 860 }
 861 
 862 oop ShenandoahHeap::array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS) {
 863   ObjArrayAllocator initializer(klass, size, length, do_zero, THREAD);
 864   ShenandoahMemAllocator allocator(initializer, klass, size, THREAD);
 865   return allocator.allocate();
 866 }
 867 
 868 oop ShenandoahHeap::class_allocate(Klass* klass, int size, TRAPS) {
 869   ClassAllocator initializer(klass, size, THREAD);
 870   ShenandoahMemAllocator allocator(initializer, klass, size, THREAD);
 871   return allocator.allocate();
 872 }
 873 
 874 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 875                                         bool*  gc_overhead_limit_was_exceeded) {
 876   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 877   return allocate_memory(req);
 878 }
 879 
 880 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 881                                                              size_t size,
 882                                                              Metaspace::MetadataType mdtype) {
 883   MetaWord* result;
 884 
 885   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 886   if (heuristics()->can_unload_classes()) {
 887     ShenandoahHeuristics* h = heuristics();
 888     h->record_metaspace_oom();
 889   }
 890 
 891   // Expand and retry allocation
 892   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 893   if (result != NULL) {
 894     return result;
 895   }
 896 
 897   // Start full GC
 898   collect(GCCause::_metadata_GC_clear_soft_refs);
 899 
 900   // Retry allocation
 901   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 902   if (result != NULL) {
 903     return result;
 904   }
 905 
 906   // Expand and retry allocation
 907   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 908   if (result != NULL) {
 909     return result;
 910   }
 911 
 912   // Out of memory
 913   return NULL;
 914 }
 915 
 916 void ShenandoahHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap) {
 917   HeapWord* obj = tlab_post_allocation_setup(start);
 918   CollectedHeap::fill_with_object(obj, end);
 919 }
 920 
 921 size_t ShenandoahHeap::min_dummy_object_size() const {
 922   return CollectedHeap::min_dummy_object_size() + ShenandoahBrooksPointer::word_size();
 923 }
 924 
 925 class ShenandoahEvacuateUpdateRootsClosure: public BasicOopIterateClosure {
 926 private:
 927   ShenandoahHeap* _heap;
 928   Thread* _thread;
 929 public:
 930   ShenandoahEvacuateUpdateRootsClosure() :
 931     _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
 932   }
 933 
 934 private:
 935   template <class T>
 936   void do_oop_work(T* p) {
 937     assert(_heap->is_evacuation_in_progress(), "Only do this when evacuation is in progress");
 938 
 939     T o = RawAccess<>::oop_load(p);
 940     if (! CompressedOops::is_null(o)) {
 941       oop obj = CompressedOops::decode_not_null(o);
 942       if (_heap->in_collection_set(obj)) {
 943         shenandoah_assert_marked(p, obj);
 944         oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 945         if (oopDesc::equals_raw(resolved, obj)) {
 946           resolved = _heap->evacuate_object(obj, _thread);
 947         }
 948         RawAccess<IS_NOT_NULL>::oop_store(p, resolved);
 949       }
 950     }
 951   }
 952 
 953 public:
 954   void do_oop(oop* p) {
 955     do_oop_work(p);
 956   }
 957   void do_oop(narrowOop* p) {
 958     do_oop_work(p);
 959   }
 960 };
 961 
 962 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
 963 private:
 964   ShenandoahHeap* const _heap;
 965   Thread* const _thread;
 966 public:
 967   ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
 968     _heap(heap), _thread(Thread::current()) {}
 969 
 970   void do_object(oop p) {
 971     shenandoah_assert_marked(NULL, p);
 972     if (oopDesc::equals_raw(p, ShenandoahBarrierSet::resolve_forwarded_not_null(p))) {
 973       _heap->evacuate_object(p, _thread);
 974     }
 975   }
 976 };
 977 
 978 class ShenandoahEvacuationTask : public AbstractGangTask {
 979 private:
 980   ShenandoahHeap* const _sh;
 981   ShenandoahCollectionSet* const _cs;
 982   bool _concurrent;
 983 public:
 984   ShenandoahEvacuationTask(ShenandoahHeap* sh,
 985                            ShenandoahCollectionSet* cs,
 986                            bool concurrent) :
 987     AbstractGangTask("Parallel Evacuation Task"),
 988     _sh(sh),
 989     _cs(cs),
 990     _concurrent(concurrent)
 991   {}
 992 
 993   void work(uint worker_id) {
 994     if (_concurrent) {
 995       ShenandoahConcurrentWorkerSession worker_session(worker_id);
 996       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 997       ShenandoahEvacOOMScope oom_evac_scope;
 998       do_work();
 999     } else {
1000       ShenandoahParallelWorkerSession worker_session(worker_id);
1001       ShenandoahEvacOOMScope oom_evac_scope;
1002       do_work();
1003     }
1004   }
1005 
1006 private:
1007   void do_work() {
1008     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
1009     ShenandoahHeapRegion* r;
1010     while ((r =_cs->claim_next()) != NULL) {
1011       assert(r->has_live(), "all-garbage regions are reclaimed early");
1012       _sh->marked_object_iterate(r, &cl);
1013 
1014       if (ShenandoahPacing) {
1015         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
1016       }
1017 
1018       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
1019         break;
1020       }
1021     }
1022   }
1023 };
1024 
1025 void ShenandoahHeap::trash_cset_regions() {
1026   ShenandoahHeapLocker locker(lock());
1027 
1028   ShenandoahCollectionSet* set = collection_set();
1029   ShenandoahHeapRegion* r;
1030   set->clear_current_index();
1031   while ((r = set->next()) != NULL) {
1032     r->make_trash();
1033   }
1034   collection_set()->clear();
1035 }
1036 
1037 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1038   st->print_cr("Heap Regions:");
1039   st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
1040   st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
1041   st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start (previous, next)");
1042   st->print_cr("SN=alloc sequence numbers (first mutator, last mutator, first gc, last gc)");
1043 
1044   for (size_t i = 0; i < num_regions(); i++) {
1045     get_region(i)->print_on(st);
1046   }
1047 }
1048 
1049 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1050   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1051 
1052   oop humongous_obj = oop(start->bottom() + ShenandoahBrooksPointer::word_size());
1053   size_t size = humongous_obj->size() + ShenandoahBrooksPointer::word_size();
1054   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1055   size_t index = start->region_number() + required_regions - 1;
1056 
1057   assert(!start->has_live(), "liveness must be zero");
1058 
1059   for(size_t i = 0; i < required_regions; i++) {
1060     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1061     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1062     ShenandoahHeapRegion* region = get_region(index --);
1063 
1064     assert(region->is_humongous(), "expect correct humongous start or continuation");
1065     assert(!region->is_cset(), "Humongous region should not be in collection set");
1066 
1067     region->make_trash_immediate();
1068   }
1069 }
1070 
1071 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1072 public:
1073   void do_thread(Thread* thread) {
1074     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1075     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1076     gclab->retire();
1077   }
1078 };
1079 
1080 void ShenandoahHeap::make_parsable(bool retire_tlabs) {
1081   if (UseTLAB) {
1082     CollectedHeap::ensure_parsability(retire_tlabs);
1083   }
1084   ShenandoahRetireGCLABClosure cl;
1085   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1086     cl.do_thread(t);
1087   }
1088   workers()->threads_do(&cl);
1089 }
1090 
1091 void ShenandoahHeap::resize_tlabs() {
1092   CollectedHeap::resize_all_tlabs();
1093 }
1094 
1095 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1096 private:
1097   ShenandoahRootEvacuator* _rp;
1098 
1099 public:
1100   ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1101     AbstractGangTask("Shenandoah evacuate and update roots"),
1102     _rp(rp) {}
1103 
1104   void work(uint worker_id) {
1105     ShenandoahParallelWorkerSession worker_session(worker_id);
1106     ShenandoahEvacOOMScope oom_evac_scope;
1107     ShenandoahEvacuateUpdateRootsClosure cl;
1108 
1109     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1110     _rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
1111   }
1112 };
1113 
1114 void ShenandoahHeap::evacuate_and_update_roots() {
1115 #if defined(COMPILER2) || INCLUDE_JVMCI
1116   DerivedPointerTable::clear();
1117 #endif
1118   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1119 
1120   {
1121     ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
1122     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1123     workers()->run_task(&roots_task);
1124   }
1125 
1126 #if defined(COMPILER2) || INCLUDE_JVMCI
1127   DerivedPointerTable::update_pointers();
1128 #endif
1129 }
1130 
1131 void ShenandoahHeap::roots_iterate(OopClosure* cl) {
1132   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1133 
1134   CodeBlobToOopClosure blobsCl(cl, false);
1135   CLDToOopClosure cldCl(cl, ClassLoaderData::_claim_strong);
1136 
1137   ShenandoahRootProcessor rp(this, 1, ShenandoahPhaseTimings::_num_phases);
1138   rp.process_all_roots(cl, NULL, &cldCl, &blobsCl, NULL, 0);
1139 }
1140 
1141 // Returns size in bytes
1142 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1143   if (ShenandoahElasticTLAB) {
1144     // With Elastic TLABs, return the max allowed size, and let the allocation path
1145     // figure out the safe size for current allocation.
1146     return ShenandoahHeapRegion::max_tlab_size_bytes();
1147   } else {
1148     return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1149   }
1150 }
1151 
1152 size_t ShenandoahHeap::max_tlab_size() const {
1153   // Returns size in words
1154   return ShenandoahHeapRegion::max_tlab_size_words();
1155 }
1156 
1157 class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
1158 public:
1159   void do_thread(Thread* thread) {
1160     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1161     gclab->retire();
1162     if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1163       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1164     }
1165   }
1166 };
1167 
1168 void ShenandoahHeap::retire_and_reset_gclabs() {
1169   ShenandoahRetireAndResetGCLABClosure cl;
1170   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1171     cl.do_thread(t);
1172   }
1173   workers()->threads_do(&cl);
1174 }
1175 
1176 void ShenandoahHeap::collect(GCCause::Cause cause) {
1177   control_thread()->request_gc(cause);
1178 }
1179 
1180 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1181   //assert(false, "Shouldn't need to do full collections");
1182 }
1183 
1184 CollectorPolicy* ShenandoahHeap::collector_policy() const {
1185   return _shenandoah_policy;
1186 }
1187 
1188 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1189   Space* sp = heap_region_containing(addr);
1190   if (sp != NULL) {
1191     return sp->block_start(addr);
1192   }
1193   return NULL;
1194 }
1195 
1196 size_t ShenandoahHeap::block_size(const HeapWord* addr) const {
1197   Space* sp = heap_region_containing(addr);
1198   assert(sp != NULL, "block_size of address outside of heap");
1199   return sp->block_size(addr);
1200 }
1201 
1202 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1203   Space* sp = heap_region_containing(addr);
1204   return sp->block_is_obj(addr);
1205 }
1206 
1207 jlong ShenandoahHeap::millis_since_last_gc() {
1208   double v = heuristics()->time_since_last_gc() * 1000;
1209   assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1210   return (jlong)v;
1211 }
1212 
1213 void ShenandoahHeap::prepare_for_verify() {
1214   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1215     make_parsable(false);
1216   }
1217 }
1218 
1219 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1220   workers()->print_worker_threads_on(st);
1221   if (ShenandoahStringDedup::is_enabled()) {
1222     ShenandoahStringDedup::print_worker_threads_on(st);
1223   }
1224 }
1225 
1226 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1227   workers()->threads_do(tcl);
1228   _safepoint_workers->threads_do(tcl);
1229   if (ShenandoahStringDedup::is_enabled()) {
1230     ShenandoahStringDedup::threads_do(tcl);
1231   }
1232 }
1233 
1234 void ShenandoahHeap::print_tracing_info() const {
1235   LogTarget(Info, gc, stats) lt;
1236   if (lt.is_enabled()) {
1237     ResourceMark rm;
1238     LogStream ls(lt);
1239 
1240     phase_timings()->print_on(&ls);
1241 
1242     ls.cr();
1243     ls.cr();
1244 
1245     shenandoah_policy()->print_gc_stats(&ls);
1246 
1247     ls.cr();
1248     ls.cr();
1249 
1250     if (ShenandoahPacing) {
1251       pacer()->print_on(&ls);
1252     }
1253 
1254     ls.cr();
1255     ls.cr();
1256 
1257     if (ShenandoahAllocationTrace) {
1258       assert(alloc_tracker() != NULL, "Must be");
1259       alloc_tracker()->print_on(&ls);
1260     } else {
1261       ls.print_cr("  Allocation tracing is disabled, use -XX:+ShenandoahAllocationTrace to enable.");
1262     }
1263   }
1264 }
1265 
1266 void ShenandoahHeap::verify(VerifyOption vo) {
1267   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1268     if (ShenandoahVerify) {
1269       verifier()->verify_generic(vo);
1270     } else {
1271       // TODO: Consider allocating verification bitmaps on demand,
1272       // and turn this on unconditionally.
1273     }
1274   }
1275 }
1276 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1277   return _free_set->capacity();
1278 }
1279 
1280 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1281 private:
1282   MarkBitMap* _bitmap;
1283   Stack<oop,mtGC>* _oop_stack;
1284 
1285   template <class T>
1286   void do_oop_work(T* p) {
1287     T o = RawAccess<>::oop_load(p);
1288     if (!CompressedOops::is_null(o)) {
1289       oop obj = CompressedOops::decode_not_null(o);
1290       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1291       assert(oopDesc::is_oop(obj), "must be a valid oop");
1292       if (!_bitmap->is_marked((HeapWord*) obj)) {
1293         _bitmap->mark((HeapWord*) obj);
1294         _oop_stack->push(obj);
1295       }
1296     }
1297   }
1298 public:
1299   ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1300     _bitmap(bitmap), _oop_stack(oop_stack) {}
1301   void do_oop(oop* p)       { do_oop_work(p); }
1302   void do_oop(narrowOop* p) { do_oop_work(p); }
1303 };
1304 
1305 /*
1306  * This is public API, used in preparation of object_iterate().
1307  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1308  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1309  * control, we call SH::make_tlabs_parsable().
1310  */
1311 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1312   // No-op.
1313 }
1314 
1315 /*
1316  * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1317  *
1318  * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1319  * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1320  * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1321  * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1322  * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1323  * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1324  * wiped the bitmap in preparation for next marking).
1325  *
1326  * For all those reasons, we implement object iteration as a single marking traversal, reporting
1327  * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1328  * is allowed to report dead objects, but is not required to do so.
1329  */
1330 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1331   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1332   if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1333     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1334     return;
1335   }
1336 
1337   // Reset bitmap
1338   _aux_bit_map.clear();
1339 
1340   Stack<oop,mtGC> oop_stack;
1341 
1342   // First, we process all GC roots. This populates the work stack with initial objects.
1343   ShenandoahRootProcessor rp(this, 1, ShenandoahPhaseTimings::_num_phases);
1344   ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1345   CLDToOopClosure clds(&oops, ClassLoaderData::_claim_none);
1346   CodeBlobToOopClosure blobs(&oops, false);
1347   rp.process_all_roots(&oops, &oops, &clds, &blobs, NULL, 0);
1348 
1349   // Work through the oop stack to traverse heap.
1350   while (! oop_stack.is_empty()) {
1351     oop obj = oop_stack.pop();
1352     assert(oopDesc::is_oop(obj), "must be a valid oop");
1353     cl->do_object(obj);
1354     obj->oop_iterate(&oops);
1355   }
1356 
1357   assert(oop_stack.is_empty(), "should be empty");
1358 
1359   if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1360     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1361   }
1362 }
1363 
1364 void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) {
1365   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1366   object_iterate(cl);
1367 }
1368 
1369 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1370   for (size_t i = 0; i < num_regions(); i++) {
1371     ShenandoahHeapRegion* current = get_region(i);
1372     blk->heap_region_do(current);
1373   }
1374 }
1375 
1376 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1377 private:
1378   ShenandoahHeap* const _heap;
1379   ShenandoahHeapRegionClosure* const _blk;
1380 
1381   DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
1382   volatile size_t _index;
1383   DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
1384 
1385 public:
1386   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1387           AbstractGangTask("Parallel Region Task"),
1388           _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1389 
1390   void work(uint worker_id) {
1391     size_t stride = ShenandoahParallelRegionStride;
1392 
1393     size_t max = _heap->num_regions();
1394     while (_index < max) {
1395       size_t cur = Atomic::add(stride, &_index) - stride;
1396       size_t start = cur;
1397       size_t end = MIN2(cur + stride, max);
1398       if (start >= max) break;
1399 
1400       for (size_t i = cur; i < end; i++) {
1401         ShenandoahHeapRegion* current = _heap->get_region(i);
1402         _blk->heap_region_do(current);
1403       }
1404     }
1405   }
1406 };
1407 
1408 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1409   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1410   if (num_regions() > ShenandoahParallelRegionStride) {
1411     ShenandoahParallelHeapRegionTask task(blk);
1412     workers()->run_task(&task);
1413   } else {
1414     heap_region_iterate(blk);
1415   }
1416 }
1417 
1418 class ShenandoahClearLivenessClosure : public ShenandoahHeapRegionClosure {
1419 private:
1420   ShenandoahMarkingContext* const _ctx;
1421 public:
1422   ShenandoahClearLivenessClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1423 
1424   void heap_region_do(ShenandoahHeapRegion* r) {
1425     if (r->is_active()) {
1426       r->clear_live_data();
1427       _ctx->capture_top_at_mark_start(r);
1428     } else {
1429       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->region_number());
1430       assert(_ctx->top_at_mark_start(r) == r->top(),
1431              "Region " SIZE_FORMAT " should already have correct TAMS", r->region_number());
1432     }
1433   }
1434 
1435   bool is_thread_safe() { return true; }
1436 };
1437 
1438 void ShenandoahHeap::op_init_mark() {
1439   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1440   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1441 
1442   assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1443   assert(!marking_context()->is_complete(), "should not be complete");
1444 
1445   if (ShenandoahVerify) {
1446     verifier()->verify_before_concmark();
1447   }
1448 
1449   if (VerifyBeforeGC) {
1450     Universe::verify();
1451   }
1452 
1453   set_concurrent_mark_in_progress(true);
1454   // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1455   {
1456     ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
1457     make_parsable(true);
1458   }
1459 
1460   {
1461     ShenandoahGCPhase phase(ShenandoahPhaseTimings::clear_liveness);
1462     ShenandoahClearLivenessClosure clc;
1463     parallel_heap_region_iterate(&clc);
1464   }
1465 
1466   // Make above changes visible to worker threads
1467   OrderAccess::fence();
1468 
1469   concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1470 
1471   if (UseTLAB) {
1472     ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
1473     resize_tlabs();
1474   }
1475 
1476   if (ShenandoahPacing) {
1477     pacer()->setup_for_mark();
1478   }
1479 }
1480 
1481 void ShenandoahHeap::op_mark() {
1482   concurrent_mark()->mark_from_roots();
1483 }
1484 
1485 class ShenandoahCompleteLivenessClosure : public ShenandoahHeapRegionClosure {
1486 private:
1487   ShenandoahMarkingContext* const _ctx;
1488 public:
1489   ShenandoahCompleteLivenessClosure() : _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
1490 
1491   void heap_region_do(ShenandoahHeapRegion* r) {
1492     if (r->is_active()) {
1493       HeapWord *tams = _ctx->top_at_mark_start(r);
1494       HeapWord *top = r->top();
1495       if (top > tams) {
1496         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1497       }
1498     } else {
1499       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->region_number());
1500       assert(_ctx->top_at_mark_start(r) == r->top(),
1501              "Region " SIZE_FORMAT " should have correct TAMS", r->region_number());
1502     }
1503   }
1504 
1505   bool is_thread_safe() { return true; }
1506 };
1507 
1508 void ShenandoahHeap::op_final_mark() {
1509   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1510 
1511   // It is critical that we
1512   // evacuate roots right after finishing marking, so that we don't
1513   // get unmarked objects in the roots.
1514 
1515   if (!cancelled_gc()) {
1516     concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1517 
1518     if (has_forwarded_objects()) {
1519       concurrent_mark()->update_roots(ShenandoahPhaseTimings::update_roots);
1520     }
1521 
1522     stop_concurrent_marking();
1523 
1524     {
1525       ShenandoahGCPhase phase(ShenandoahPhaseTimings::complete_liveness);
1526 
1527       // All allocations past TAMS are implicitly live, adjust the region data.
1528       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1529       ShenandoahCompleteLivenessClosure cl;
1530       parallel_heap_region_iterate(&cl);
1531     }
1532 
1533     {
1534       ShenandoahGCPhase prepare_evac(ShenandoahPhaseTimings::prepare_evac);
1535 
1536       make_parsable(true);
1537 
1538       trash_cset_regions();
1539 
1540       {
1541         ShenandoahHeapLocker locker(lock());
1542         _collection_set->clear();
1543         _free_set->clear();
1544 
1545         heuristics()->choose_collection_set(_collection_set);
1546 
1547         _free_set->rebuild();
1548       }
1549     }
1550 
1551     // If collection set has candidates, start evacuation.
1552     // Otherwise, bypass the rest of the cycle.
1553     if (!collection_set()->is_empty()) {
1554       ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1555 
1556       if (ShenandoahVerify) {
1557         verifier()->verify_before_evacuation();
1558       }
1559 
1560       set_evacuation_in_progress(true);
1561       // From here on, we need to update references.
1562       set_has_forwarded_objects(true);
1563 
1564       evacuate_and_update_roots();
1565 
1566       if (ShenandoahPacing) {
1567         pacer()->setup_for_evac();
1568       }
1569     } else {
1570       if (ShenandoahVerify) {
1571         verifier()->verify_after_concmark();
1572       }
1573 
1574       if (VerifyAfterGC) {
1575         Universe::verify();
1576       }
1577     }
1578 
1579   } else {
1580     concurrent_mark()->cancel();
1581     stop_concurrent_marking();
1582 
1583     if (process_references()) {
1584       // Abandon reference processing right away: pre-cleaning must have failed.
1585       ReferenceProcessor *rp = ref_processor();
1586       rp->disable_discovery();
1587       rp->abandon_partial_discovery();
1588       rp->verify_no_references_recorded();
1589     }
1590   }
1591 }
1592 
1593 void ShenandoahHeap::op_final_evac() {
1594   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1595 
1596   set_evacuation_in_progress(false);
1597 
1598   retire_and_reset_gclabs();
1599 
1600   if (ShenandoahVerify) {
1601     verifier()->verify_after_evacuation();
1602   }
1603 
1604   if (VerifyAfterGC) {
1605     Universe::verify();
1606   }
1607 }
1608 
1609 void ShenandoahHeap::op_conc_evac() {
1610   ShenandoahEvacuationTask task(this, _collection_set, true);
1611   workers()->run_task(&task);
1612 }
1613 
1614 void ShenandoahHeap::op_stw_evac() {
1615   ShenandoahEvacuationTask task(this, _collection_set, false);
1616   workers()->run_task(&task);
1617 }
1618 
1619 void ShenandoahHeap::op_updaterefs() {
1620   update_heap_references(true);
1621 }
1622 
1623 void ShenandoahHeap::op_cleanup() {
1624   free_set()->recycle_trash();
1625 }
1626 
1627 void ShenandoahHeap::op_reset() {
1628   reset_mark_bitmap();
1629 }
1630 
1631 void ShenandoahHeap::op_preclean() {
1632   concurrent_mark()->preclean_weak_refs();
1633 }
1634 
1635 void ShenandoahHeap::op_init_traversal() {
1636   traversal_gc()->init_traversal_collection();
1637 }
1638 
1639 void ShenandoahHeap::op_traversal() {
1640   traversal_gc()->concurrent_traversal_collection();
1641 }
1642 
1643 void ShenandoahHeap::op_final_traversal() {
1644   traversal_gc()->final_traversal_collection();
1645 }
1646 
1647 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1648   ShenandoahMetricsSnapshot metrics;
1649   metrics.snap_before();
1650 
1651   full_gc()->do_it(cause);
1652   if (UseTLAB) {
1653     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
1654     resize_all_tlabs();
1655   }
1656 
1657   metrics.snap_after();
1658   metrics.print();
1659 
1660   if (metrics.is_good_progress("Full GC")) {
1661     _progress_last_gc.set();
1662   } else {
1663     // Nothing to do. Tell the allocation path that we have failed to make
1664     // progress, and it can finally fail.
1665     _progress_last_gc.unset();
1666   }
1667 }
1668 
1669 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1670   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1671   // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1672   // some phase, we have to upgrade the Degenerate GC to Full GC.
1673 
1674   clear_cancelled_gc();
1675 
1676   ShenandoahMetricsSnapshot metrics;
1677   metrics.snap_before();
1678 
1679   switch (point) {
1680     case _degenerated_traversal:
1681       {
1682         // Drop the collection set. Note: this leaves some already forwarded objects
1683         // behind, which may be problematic, see comments for ShenandoahEvacAssist
1684         // workarounds in ShenandoahTraversalHeuristics.
1685 
1686         ShenandoahHeapLocker locker(lock());
1687         collection_set()->clear_current_index();
1688         for (size_t i = 0; i < collection_set()->count(); i++) {
1689           ShenandoahHeapRegion* r = collection_set()->next();
1690           r->make_regular_bypass();
1691         }
1692         collection_set()->clear();
1693       }
1694       op_final_traversal();
1695       op_cleanup();
1696       return;
1697 
1698     // The cases below form the Duff's-like device: it describes the actual GC cycle,
1699     // but enters it at different points, depending on which concurrent phase had
1700     // degenerated.
1701 
1702     case _degenerated_outside_cycle:
1703       // We have degenerated from outside the cycle, which means something is bad with
1704       // the heap, most probably heavy humongous fragmentation, or we are very low on free
1705       // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1706       // we can do the most aggressive degen cycle, which includes processing references and
1707       // class unloading, unless those features are explicitly disabled.
1708       //
1709       // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1710       // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1711       set_process_references(heuristics()->can_process_references());
1712       set_unload_classes(heuristics()->can_unload_classes());
1713 
1714       if (heuristics()->can_do_traversal_gc()) {
1715         // Not possible to degenerate from here, upgrade to Full GC right away.
1716         cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1717         op_degenerated_fail();
1718         return;
1719       }
1720 
1721       op_reset();
1722 
1723       op_init_mark();
1724       if (cancelled_gc()) {
1725         op_degenerated_fail();
1726         return;
1727       }
1728 
1729     case _degenerated_mark:
1730       op_final_mark();
1731       if (cancelled_gc()) {
1732         op_degenerated_fail();
1733         return;
1734       }
1735 
1736       op_cleanup();
1737 
1738     case _degenerated_evac:
1739       // If heuristics thinks we should do the cycle, this flag would be set,
1740       // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1741       if (is_evacuation_in_progress()) {
1742 
1743         // Degeneration under oom-evac protocol might have left some objects in
1744         // collection set un-evacuated. Restart evacuation from the beginning to
1745         // capture all objects. For all the objects that are already evacuated,
1746         // it would be a simple check, which is supposed to be fast. This is also
1747         // safe to do even without degeneration, as CSet iterator is at beginning
1748         // in preparation for evacuation anyway.
1749         collection_set()->clear_current_index();
1750 
1751         op_stw_evac();
1752         if (cancelled_gc()) {
1753           op_degenerated_fail();
1754           return;
1755         }
1756       }
1757 
1758       // If heuristics thinks we should do the cycle, this flag would be set,
1759       // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
1760       if (has_forwarded_objects()) {
1761         op_init_updaterefs();
1762         if (cancelled_gc()) {
1763           op_degenerated_fail();
1764           return;
1765         }
1766       }
1767 
1768     case _degenerated_updaterefs:
1769       if (has_forwarded_objects()) {
1770         op_final_updaterefs();
1771         if (cancelled_gc()) {
1772           op_degenerated_fail();
1773           return;
1774         }
1775       }
1776 
1777       op_cleanup();
1778       break;
1779 
1780     default:
1781       ShouldNotReachHere();
1782   }
1783 
1784   if (ShenandoahVerify) {
1785     verifier()->verify_after_degenerated();
1786   }
1787 
1788   if (VerifyAfterGC) {
1789     Universe::verify();
1790   }
1791 
1792   metrics.snap_after();
1793   metrics.print();
1794 
1795   // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
1796   // because that probably means the heap is overloaded and/or fragmented.
1797   if (!metrics.is_good_progress("Degenerated GC")) {
1798     _progress_last_gc.unset();
1799     cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1800     op_degenerated_futile();
1801   } else {
1802     _progress_last_gc.set();
1803   }
1804 }
1805 
1806 void ShenandoahHeap::op_degenerated_fail() {
1807   log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
1808   shenandoah_policy()->record_degenerated_upgrade_to_full();
1809   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
1810 }
1811 
1812 void ShenandoahHeap::op_degenerated_futile() {
1813   shenandoah_policy()->record_degenerated_upgrade_to_full();
1814   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
1815 }
1816 
1817 void ShenandoahHeap::stop_concurrent_marking() {
1818   assert(is_concurrent_mark_in_progress(), "How else could we get here?");
1819   if (!cancelled_gc()) {
1820     // If we needed to update refs, and concurrent marking has been cancelled,
1821     // we need to finish updating references.
1822     set_has_forwarded_objects(false);
1823     mark_complete_marking_context();
1824   }
1825   set_concurrent_mark_in_progress(false);
1826 }
1827 
1828 void ShenandoahHeap::force_satb_flush_all_threads() {
1829   if (!is_concurrent_mark_in_progress() && !is_concurrent_traversal_in_progress()) {
1830     // No need to flush SATBs
1831     return;
1832   }
1833 
1834   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1835     ShenandoahThreadLocalData::set_force_satb_flush(t, true);
1836   }
1837   // The threads are not "acquiring" their thread-local data, but it does not
1838   // hurt to "release" the updates here anyway.
1839   OrderAccess::fence();
1840 }
1841 
1842 void ShenandoahHeap::set_gc_state_all_threads(char state) {
1843   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1844     ShenandoahThreadLocalData::set_gc_state(t, state);
1845   }
1846 }
1847 
1848 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
1849   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
1850   _gc_state.set_cond(mask, value);
1851   set_gc_state_all_threads(_gc_state.raw_value());
1852 }
1853 
1854 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
1855   set_gc_state_mask(MARKING, in_progress);
1856   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1857 }
1858 
1859 void ShenandoahHeap::set_concurrent_traversal_in_progress(bool in_progress) {
1860    set_gc_state_mask(TRAVERSAL | HAS_FORWARDED, in_progress);
1861    ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1862 }
1863 
1864 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
1865   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
1866   set_gc_state_mask(EVACUATION, in_progress);
1867 }
1868 
1869 HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
1870   // Initialize Brooks pointer for the next object
1871   HeapWord* result = obj + ShenandoahBrooksPointer::word_size();
1872   ShenandoahBrooksPointer::initialize(oop(result));
1873   return result;
1874 }
1875 
1876 ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
1877   _mark_context(ShenandoahHeap::heap()->marking_context()) {
1878 }
1879 
1880 ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() :
1881   _mark_context(ShenandoahHeap::heap()->marking_context()) {
1882 }
1883 
1884 bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
1885   if (CompressedOops::is_null(obj)) {
1886     return false;
1887   }
1888   obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1889   shenandoah_assert_not_forwarded_if(NULL, obj, ShenandoahHeap::heap()->is_concurrent_mark_in_progress() || ShenandoahHeap::heap()->is_concurrent_traversal_in_progress());
1890   return _mark_context->is_marked(obj);
1891 }
1892 
1893 bool ShenandoahIsAliveClosure::do_object_b(oop obj) {
1894   if (CompressedOops::is_null(obj)) {
1895     return false;
1896   }
1897   shenandoah_assert_not_forwarded(NULL, obj);
1898   return _mark_context->is_marked(obj);
1899 }
1900 
1901 void ShenandoahHeap::ref_processing_init() {
1902   assert(_max_workers > 0, "Sanity");
1903 
1904   _ref_processor =
1905     new ReferenceProcessor(&_subject_to_discovery,  // is_subject_to_discovery
1906                            ParallelRefProcEnabled,  // MT processing
1907                            _max_workers,            // Degree of MT processing
1908                            true,                    // MT discovery
1909                            _max_workers,            // Degree of MT discovery
1910                            false,                   // Reference discovery is not atomic
1911                            NULL,                    // No closure, should be installed before use
1912                            true);                   // Scale worker threads
1913 
1914   shenandoah_assert_rp_isalive_not_installed();
1915 }
1916 
1917 GCTracer* ShenandoahHeap::tracer() {
1918   return shenandoah_policy()->tracer();
1919 }
1920 
1921 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
1922   return _free_set->used();
1923 }
1924 
1925 bool ShenandoahHeap::try_cancel_gc() {
1926   while (true) {
1927     jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
1928     if (prev == CANCELLABLE) return true;
1929     else if (prev == CANCELLED) return false;
1930     assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
1931     assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
1932     {
1933       // We need to provide a safepoint here, otherwise we might
1934       // spin forever if a SP is pending.
1935       ThreadBlockInVM sp(JavaThread::current());
1936       SpinPause();
1937     }
1938   }
1939 }
1940 
1941 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
1942   if (try_cancel_gc()) {
1943     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
1944     log_info(gc)("%s", msg.buffer());
1945     Events::log(Thread::current(), "%s", msg.buffer());
1946   }
1947 }
1948 
1949 uint ShenandoahHeap::max_workers() {
1950   return _max_workers;
1951 }
1952 
1953 void ShenandoahHeap::stop() {
1954   // The shutdown sequence should be able to terminate when GC is running.
1955 
1956   // Step 0. Notify policy to disable event recording.
1957   _shenandoah_policy->record_shutdown();
1958 
1959   // Step 1. Notify control thread that we are in shutdown.
1960   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
1961   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
1962   control_thread()->prepare_for_graceful_shutdown();
1963 
1964   // Step 2. Notify GC workers that we are cancelling GC.
1965   cancel_gc(GCCause::_shenandoah_stop_vm);
1966 
1967   // Step 3. Wait until GC worker exits normally.
1968   control_thread()->stop();
1969 
1970   // Step 4. Stop String Dedup thread if it is active
1971   if (ShenandoahStringDedup::is_enabled()) {
1972     ShenandoahStringDedup::stop();
1973   }
1974 }
1975 
1976 void ShenandoahHeap::unload_classes_and_cleanup_tables(bool full_gc) {
1977   assert(heuristics()->can_unload_classes(), "Class unloading should be enabled");
1978 
1979   ShenandoahGCPhase root_phase(full_gc ?
1980                                ShenandoahPhaseTimings::full_gc_purge :
1981                                ShenandoahPhaseTimings::purge);
1982 
1983   ShenandoahIsAliveSelector alive;
1984   BoolObjectClosure* is_alive = alive.is_alive_closure();
1985 
1986   bool purged_class;
1987 
1988   // Unload classes and purge SystemDictionary.
1989   {
1990     ShenandoahGCPhase phase(full_gc ?
1991                             ShenandoahPhaseTimings::full_gc_purge_class_unload :
1992                             ShenandoahPhaseTimings::purge_class_unload);
1993     purged_class = SystemDictionary::do_unloading(gc_timer());
1994   }
1995 
1996   {
1997     ShenandoahGCPhase phase(full_gc ?
1998                             ShenandoahPhaseTimings::full_gc_purge_par :
1999                             ShenandoahPhaseTimings::purge_par);
2000     uint active = _workers->active_workers();
2001     ParallelCleaningTask unlink_task(is_alive, active, purged_class, true);
2002     _workers->run_task(&unlink_task);
2003   }
2004 
2005   {
2006     ShenandoahGCPhase phase(full_gc ?
2007                       ShenandoahPhaseTimings::full_gc_purge_cldg :
2008                       ShenandoahPhaseTimings::purge_cldg);
2009     ClassLoaderDataGraph::purge();
2010   }
2011 }
2012 
2013 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2014   set_gc_state_mask(HAS_FORWARDED, cond);
2015 }
2016 
2017 void ShenandoahHeap::set_process_references(bool pr) {
2018   _process_references.set_cond(pr);
2019 }
2020 
2021 void ShenandoahHeap::set_unload_classes(bool uc) {
2022   _unload_classes.set_cond(uc);
2023 }
2024 
2025 bool ShenandoahHeap::process_references() const {
2026   return _process_references.is_set();
2027 }
2028 
2029 bool ShenandoahHeap::unload_classes() const {
2030   return _unload_classes.is_set();
2031 }
2032 
2033 address ShenandoahHeap::in_cset_fast_test_addr() {
2034   ShenandoahHeap* heap = ShenandoahHeap::heap();
2035   assert(heap->collection_set() != NULL, "Sanity");
2036   return (address) heap->collection_set()->biased_map_address();
2037 }
2038 
2039 address ShenandoahHeap::cancelled_gc_addr() {
2040   return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
2041 }
2042 
2043 address ShenandoahHeap::gc_state_addr() {
2044   return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
2045 }
2046 
2047 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
2048   return OrderAccess::load_acquire(&_bytes_allocated_since_gc_start);
2049 }
2050 
2051 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2052   OrderAccess::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
2053 }
2054 
2055 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2056   _degenerated_gc_in_progress.set_cond(in_progress);
2057 }
2058 
2059 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2060   _full_gc_in_progress.set_cond(in_progress);
2061 }
2062 
2063 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2064   assert (is_full_gc_in_progress(), "should be");
2065   _full_gc_move_in_progress.set_cond(in_progress);
2066 }
2067 
2068 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2069   set_gc_state_mask(UPDATEREFS, in_progress);
2070 }
2071 
2072 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2073   ShenandoahCodeRoots::add_nmethod(nm);
2074 }
2075 
2076 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2077   ShenandoahCodeRoots::remove_nmethod(nm);
2078 }
2079 
2080 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2081   o = ShenandoahBarrierSet::barrier_set()->write_barrier(o);
2082   ShenandoahHeapLocker locker(lock());
2083   heap_region_containing(o)->make_pinned();
2084   return o;
2085 }
2086 
2087 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2088   o = ShenandoahBarrierSet::barrier_set()->read_barrier(o);
2089   ShenandoahHeapLocker locker(lock());
2090   heap_region_containing(o)->make_unpinned();
2091 }
2092 
2093 GCTimer* ShenandoahHeap::gc_timer() const {
2094   return _gc_timer;
2095 }
2096 
2097 #ifdef ASSERT
2098 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2099   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2100 
2101   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2102     if (UseDynamicNumberOfGCThreads ||
2103         (FLAG_IS_DEFAULT(ParallelGCThreads) && ForceDynamicNumberOfGCThreads)) {
2104       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2105     } else {
2106       // Use ParallelGCThreads inside safepoints
2107       assert(nworkers == ParallelGCThreads, "Use ParalleGCThreads within safepoints");
2108     }
2109   } else {
2110     if (UseDynamicNumberOfGCThreads ||
2111         (FLAG_IS_DEFAULT(ConcGCThreads) && ForceDynamicNumberOfGCThreads)) {
2112       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2113     } else {
2114       // Use ConcGCThreads outside safepoints
2115       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2116     }
2117   }
2118 }
2119 #endif
2120 
2121 ShenandoahVerifier* ShenandoahHeap::verifier() {
2122   guarantee(ShenandoahVerify, "Should be enabled");
2123   assert (_verifier != NULL, "sanity");
2124   return _verifier;
2125 }
2126 
2127 template<class T>
2128 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2129 private:
2130   T cl;
2131   ShenandoahHeap* _heap;
2132   ShenandoahRegionIterator* _regions;
2133   bool _concurrent;
2134 public:
2135   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2136     AbstractGangTask("Concurrent Update References Task"),
2137     cl(T()),
2138     _heap(ShenandoahHeap::heap()),
2139     _regions(regions),
2140     _concurrent(concurrent) {
2141   }
2142 
2143   void work(uint worker_id) {
2144     if (_concurrent) {
2145       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2146       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2147       do_work();
2148     } else {
2149       ShenandoahParallelWorkerSession worker_session(worker_id);
2150       do_work();
2151     }
2152   }
2153 
2154 private:
2155   void do_work() {
2156     ShenandoahHeapRegion* r = _regions->next();
2157     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2158     while (r != NULL) {
2159       HeapWord* top_at_start_ur = r->concurrent_iteration_safe_limit();
2160       assert (top_at_start_ur >= r->bottom(), "sanity");
2161       if (r->is_active() && !r->is_cset()) {
2162         _heap->marked_object_oop_iterate(r, &cl, top_at_start_ur);
2163       }
2164       if (ShenandoahPacing) {
2165         _heap->pacer()->report_updaterefs(pointer_delta(top_at_start_ur, r->bottom()));
2166       }
2167       if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2168         return;
2169       }
2170       r = _regions->next();
2171     }
2172   }
2173 };
2174 
2175 void ShenandoahHeap::update_heap_references(bool concurrent) {
2176   ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2177   workers()->run_task(&task);
2178 }
2179 
2180 void ShenandoahHeap::op_init_updaterefs() {
2181   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2182 
2183   set_evacuation_in_progress(false);
2184 
2185   retire_and_reset_gclabs();
2186 
2187   if (ShenandoahVerify) {
2188     verifier()->verify_before_updaterefs();
2189   }
2190 
2191   set_update_refs_in_progress(true);
2192   make_parsable(true);
2193   for (uint i = 0; i < num_regions(); i++) {
2194     ShenandoahHeapRegion* r = get_region(i);
2195     r->set_concurrent_iteration_safe_limit(r->top());
2196   }
2197 
2198   // Reset iterator.
2199   _update_refs_iterator.reset();
2200 
2201   if (ShenandoahPacing) {
2202     pacer()->setup_for_updaterefs();
2203   }
2204 }
2205 
2206 void ShenandoahHeap::op_final_updaterefs() {
2207   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2208 
2209   // Check if there is left-over work, and finish it
2210   if (_update_refs_iterator.has_next()) {
2211     ShenandoahGCPhase final_work(ShenandoahPhaseTimings::final_update_refs_finish_work);
2212 
2213     // Finish updating references where we left off.
2214     clear_cancelled_gc();
2215     update_heap_references(false);
2216   }
2217 
2218   // Clear cancelled GC, if set. On cancellation path, the block before would handle
2219   // everything. On degenerated paths, cancelled gc would not be set anyway.
2220   if (cancelled_gc()) {
2221     clear_cancelled_gc();
2222   }
2223   assert(!cancelled_gc(), "Should have been done right before");
2224 
2225   concurrent_mark()->update_roots(is_degenerated_gc_in_progress() ?
2226                                  ShenandoahPhaseTimings::degen_gc_update_roots:
2227                                  ShenandoahPhaseTimings::final_update_refs_roots);
2228 
2229   ShenandoahGCPhase final_update_refs(ShenandoahPhaseTimings::final_update_refs_recycle);
2230 
2231   trash_cset_regions();
2232   set_has_forwarded_objects(false);
2233   set_update_refs_in_progress(false);
2234 
2235   if (ShenandoahVerify) {
2236     verifier()->verify_after_updaterefs();
2237   }
2238 
2239   if (VerifyAfterGC) {
2240     Universe::verify();
2241   }
2242 
2243   {
2244     ShenandoahHeapLocker locker(lock());
2245     _free_set->rebuild();
2246   }
2247 }
2248 
2249 #ifdef ASSERT
2250 void ShenandoahHeap::assert_heaplock_owned_by_current_thread() {
2251   _lock.assert_owned_by_current_thread();
2252 }
2253 
2254 void ShenandoahHeap::assert_heaplock_not_owned_by_current_thread() {
2255   _lock.assert_not_owned_by_current_thread();
2256 }
2257 
2258 void ShenandoahHeap::assert_heaplock_or_safepoint() {
2259   _lock.assert_owned_by_current_thread_or_safepoint();
2260 }
2261 #endif
2262 
2263 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2264   print_on(st);
2265   print_heap_regions_on(st);
2266 }
2267 
2268 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2269   size_t slice = r->region_number() / _bitmap_regions_per_slice;
2270 
2271   size_t regions_from = _bitmap_regions_per_slice * slice;
2272   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2273   for (size_t g = regions_from; g < regions_to; g++) {
2274     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2275     if (skip_self && g == r->region_number()) continue;
2276     if (get_region(g)->is_committed()) {
2277       return true;
2278     }
2279   }
2280   return false;
2281 }
2282 
2283 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2284   assert_heaplock_owned_by_current_thread();
2285 
2286   // Bitmaps in special regions do not need commits
2287   if (_bitmap_region_special) {
2288     return true;
2289   }
2290 
2291   if (is_bitmap_slice_committed(r, true)) {
2292     // Some other region from the group is already committed, meaning the bitmap
2293     // slice is already committed, we exit right away.
2294     return true;
2295   }
2296 
2297   // Commit the bitmap slice:
2298   size_t slice = r->region_number() / _bitmap_regions_per_slice;
2299   size_t off = _bitmap_bytes_per_slice * slice;
2300   size_t len = _bitmap_bytes_per_slice;
2301   if (!os::commit_memory((char*)_bitmap_region.start() + off, len, false)) {
2302     return false;
2303   }
2304   return true;
2305 }
2306 
2307 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2308   assert_heaplock_owned_by_current_thread();
2309 
2310   // Bitmaps in special regions do not need uncommits
2311   if (_bitmap_region_special) {
2312     return true;
2313   }
2314 
2315   if (is_bitmap_slice_committed(r, true)) {
2316     // Some other region from the group is still committed, meaning the bitmap
2317     // slice is should stay committed, exit right away.
2318     return true;
2319   }
2320 
2321   // Uncommit the bitmap slice:
2322   size_t slice = r->region_number() / _bitmap_regions_per_slice;
2323   size_t off = _bitmap_bytes_per_slice * slice;
2324   size_t len = _bitmap_bytes_per_slice;
2325   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2326     return false;
2327   }
2328   return true;
2329 }
2330 
2331 void ShenandoahHeap::safepoint_synchronize_begin() {
2332   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2333     SuspendibleThreadSet::synchronize();
2334   }
2335 }
2336 
2337 void ShenandoahHeap::safepoint_synchronize_end() {
2338   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2339     SuspendibleThreadSet::desynchronize();
2340   }
2341 }
2342 
2343 void ShenandoahHeap::vmop_entry_init_mark() {
2344   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2345   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2346   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark_gross);
2347 
2348   try_inject_alloc_failure();
2349   VM_ShenandoahInitMark op;
2350   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2351 }
2352 
2353 void ShenandoahHeap::vmop_entry_final_mark() {
2354   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2355   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2356   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark_gross);
2357 
2358   try_inject_alloc_failure();
2359   VM_ShenandoahFinalMarkStartEvac op;
2360   VMThread::execute(&op); // jump to entry_final_mark under safepoint
2361 }
2362 
2363 void ShenandoahHeap::vmop_entry_final_evac() {
2364   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2365   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2366   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac_gross);
2367 
2368   VM_ShenandoahFinalEvac op;
2369   VMThread::execute(&op); // jump to entry_final_evac under safepoint
2370 }
2371 
2372 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2373   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2374   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2375   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_gross);
2376 
2377   try_inject_alloc_failure();
2378   VM_ShenandoahInitUpdateRefs op;
2379   VMThread::execute(&op);
2380 }
2381 
2382 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2383   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2384   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2385   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_gross);
2386 
2387   try_inject_alloc_failure();
2388   VM_ShenandoahFinalUpdateRefs op;
2389   VMThread::execute(&op);
2390 }
2391 
2392 void ShenandoahHeap::vmop_entry_init_traversal() {
2393   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2394   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2395   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc_gross);
2396 
2397   try_inject_alloc_failure();
2398   VM_ShenandoahInitTraversalGC op;
2399   VMThread::execute(&op);
2400 }
2401 
2402 void ShenandoahHeap::vmop_entry_final_traversal() {
2403   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2404   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2405   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc_gross);
2406 
2407   try_inject_alloc_failure();
2408   VM_ShenandoahFinalTraversalGC op;
2409   VMThread::execute(&op);
2410 }
2411 
2412 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2413   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2414   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2415   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_gross);
2416 
2417   try_inject_alloc_failure();
2418   VM_ShenandoahFullGC op(cause);
2419   VMThread::execute(&op);
2420 }
2421 
2422 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2423   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2424   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2425   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_gross);
2426 
2427   VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2428   VMThread::execute(&degenerated_gc);
2429 }
2430 
2431 void ShenandoahHeap::entry_init_mark() {
2432   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2433   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark);
2434   const char* msg = init_mark_event_message();
2435   GCTraceTime(Info, gc) time(msg, gc_timer());
2436   EventMark em("%s", msg);
2437 
2438   ShenandoahWorkerScope scope(workers(),
2439                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2440                               "init marking");
2441 
2442   op_init_mark();
2443 }
2444 
2445 void ShenandoahHeap::entry_final_mark() {
2446   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2447   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark);
2448   const char* msg = final_mark_event_message();
2449   GCTraceTime(Info, gc) time(msg, gc_timer());
2450   EventMark em("%s", msg);
2451 
2452   ShenandoahWorkerScope scope(workers(),
2453                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2454                               "final marking");
2455 
2456   op_final_mark();
2457 }
2458 
2459 void ShenandoahHeap::entry_final_evac() {
2460   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2461   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac);
2462   static const char* msg = "Pause Final Evac";
2463   GCTraceTime(Info, gc) time(msg, gc_timer());
2464   EventMark em("%s", msg);
2465 
2466   op_final_evac();
2467 }
2468 
2469 void ShenandoahHeap::entry_init_updaterefs() {
2470   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2471   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs);
2472 
2473   static const char* msg = "Pause Init Update Refs";
2474   GCTraceTime(Info, gc) time(msg, gc_timer());
2475   EventMark em("%s", msg);
2476 
2477   // No workers used in this phase, no setup required
2478 
2479   op_init_updaterefs();
2480 }
2481 
2482 void ShenandoahHeap::entry_final_updaterefs() {
2483   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2484   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs);
2485 
2486   static const char* msg = "Pause Final Update Refs";
2487   GCTraceTime(Info, gc) time(msg, gc_timer());
2488   EventMark em("%s", msg);
2489 
2490   ShenandoahWorkerScope scope(workers(),
2491                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2492                               "final reference update");
2493 
2494   op_final_updaterefs();
2495 }
2496 
2497 void ShenandoahHeap::entry_init_traversal() {
2498   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2499   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc);
2500 
2501   static const char* msg = "Pause Init Traversal";
2502   GCTraceTime(Info, gc) time(msg, gc_timer());
2503   EventMark em("%s", msg);
2504 
2505   ShenandoahWorkerScope scope(workers(),
2506                               ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(),
2507                               "init traversal");
2508 
2509   op_init_traversal();
2510 }
2511 
2512 void ShenandoahHeap::entry_final_traversal() {
2513   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2514   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc);
2515 
2516   static const char* msg = "Pause Final Traversal";
2517   GCTraceTime(Info, gc) time(msg, gc_timer());
2518   EventMark em("%s", msg);
2519 
2520   ShenandoahWorkerScope scope(workers(),
2521                               ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(),
2522                               "final traversal");
2523 
2524   op_final_traversal();
2525 }
2526 
2527 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2528   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2529   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc);
2530 
2531   static const char* msg = "Pause Full";
2532   GCTraceTime(Info, gc) time(msg, gc_timer(), cause, true);
2533   EventMark em("%s", msg);
2534 
2535   ShenandoahWorkerScope scope(workers(),
2536                               ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2537                               "full gc");
2538 
2539   op_full(cause);
2540 }
2541 
2542 void ShenandoahHeap::entry_degenerated(int point) {
2543   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2544   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc);
2545 
2546   ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2547   const char* msg = degen_event_message(dpoint);
2548   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2549   EventMark em("%s", msg);
2550 
2551   ShenandoahWorkerScope scope(workers(),
2552                               ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2553                               "stw degenerated gc");
2554 
2555   set_degenerated_gc_in_progress(true);
2556   op_degenerated(dpoint);
2557   set_degenerated_gc_in_progress(false);
2558 }
2559 
2560 void ShenandoahHeap::entry_mark() {
2561   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2562 
2563   const char* msg = conc_mark_event_message();
2564   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2565   EventMark em("%s", msg);
2566 
2567   ShenandoahWorkerScope scope(workers(),
2568                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2569                               "concurrent marking");
2570 
2571   try_inject_alloc_failure();
2572   op_mark();
2573 }
2574 
2575 void ShenandoahHeap::entry_evac() {
2576   ShenandoahGCPhase conc_evac_phase(ShenandoahPhaseTimings::conc_evac);
2577   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2578 
2579   static const char* msg = "Concurrent evacuation";
2580   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2581   EventMark em("%s", msg);
2582 
2583   ShenandoahWorkerScope scope(workers(),
2584                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2585                               "concurrent evacuation");
2586 
2587   try_inject_alloc_failure();
2588   op_conc_evac();
2589 }
2590 
2591 void ShenandoahHeap::entry_updaterefs() {
2592   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2593 
2594   static const char* msg = "Concurrent update references";
2595   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2596   EventMark em("%s", msg);
2597 
2598   ShenandoahWorkerScope scope(workers(),
2599                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2600                               "concurrent reference update");
2601 
2602   try_inject_alloc_failure();
2603   op_updaterefs();
2604 }
2605 void ShenandoahHeap::entry_cleanup() {
2606   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
2607 
2608   static const char* msg = "Concurrent cleanup";
2609   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2610   EventMark em("%s", msg);
2611 
2612   // This phase does not use workers, no need for setup
2613 
2614   try_inject_alloc_failure();
2615   op_cleanup();
2616 }
2617 
2618 void ShenandoahHeap::entry_reset() {
2619   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_reset);
2620 
2621   static const char* msg = "Concurrent reset";
2622   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2623   EventMark em("%s", msg);
2624 
2625   ShenandoahWorkerScope scope(workers(),
2626                               ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2627                               "concurrent reset");
2628 
2629   try_inject_alloc_failure();
2630   op_reset();
2631 }
2632 
2633 void ShenandoahHeap::entry_preclean() {
2634   if (ShenandoahPreclean && process_references()) {
2635     static const char* msg = "Concurrent precleaning";
2636     GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2637     EventMark em("%s", msg);
2638 
2639     ShenandoahGCPhase conc_preclean(ShenandoahPhaseTimings::conc_preclean);
2640 
2641     ShenandoahWorkerScope scope(workers(),
2642                                 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2643                                 "concurrent preclean",
2644                                 /* check_workers = */ false);
2645 
2646     try_inject_alloc_failure();
2647     op_preclean();
2648   }
2649 }
2650 
2651 void ShenandoahHeap::entry_traversal() {
2652   static const char* msg = "Concurrent traversal";
2653   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2654   EventMark em("%s", msg);
2655 
2656   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2657 
2658   ShenandoahWorkerScope scope(workers(),
2659                               ShenandoahWorkerPolicy::calc_workers_for_conc_traversal(),
2660                               "concurrent traversal");
2661 
2662   try_inject_alloc_failure();
2663   op_traversal();
2664 }
2665 
2666 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2667   static const char *msg = "Concurrent uncommit";
2668   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2669   EventMark em("%s", msg);
2670 
2671   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_uncommit);
2672 
2673   op_uncommit(shrink_before);
2674 }
2675 
2676 void ShenandoahHeap::try_inject_alloc_failure() {
2677   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2678     _inject_alloc_failure.set();
2679     os::naked_short_sleep(1);
2680     if (cancelled_gc()) {
2681       log_info(gc)("Allocation failure was successfully injected");
2682     }
2683   }
2684 }
2685 
2686 bool ShenandoahHeap::should_inject_alloc_failure() {
2687   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2688 }
2689 
2690 void ShenandoahHeap::initialize_serviceability() {
2691   _memory_pool = new ShenandoahMemoryPool(this);
2692   _cycle_memory_manager.add_pool(_memory_pool);
2693   _stw_memory_manager.add_pool(_memory_pool);
2694 }
2695 
2696 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2697   GrowableArray<GCMemoryManager*> memory_managers(2);
2698   memory_managers.append(&_cycle_memory_manager);
2699   memory_managers.append(&_stw_memory_manager);
2700   return memory_managers;
2701 }
2702 
2703 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2704   GrowableArray<MemoryPool*> memory_pools(1);
2705   memory_pools.append(_memory_pool);
2706   return memory_pools;
2707 }
2708 
2709 MemoryUsage ShenandoahHeap::memory_usage() {
2710   return _memory_pool->get_memory_usage();
2711 }
2712 
2713 void ShenandoahHeap::enter_evacuation() {
2714   _oom_evac_handler.enter_evacuation();
2715 }
2716 
2717 void ShenandoahHeap::leave_evacuation() {
2718   _oom_evac_handler.leave_evacuation();
2719 }
2720 
2721 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2722   _heap(ShenandoahHeap::heap()),
2723   _index(0) {}
2724 
2725 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2726   _heap(heap),
2727   _index(0) {}
2728 
2729 void ShenandoahRegionIterator::reset() {
2730   _index = 0;
2731 }
2732 
2733 bool ShenandoahRegionIterator::has_next() const {
2734   return _index < _heap->num_regions();
2735 }
2736 
2737 char ShenandoahHeap::gc_state() const {
2738   return _gc_state.raw_value();
2739 }
2740 
2741 void ShenandoahHeap::deduplicate_string(oop str) {
2742   assert(java_lang_String::is_instance(str), "invariant");
2743 
2744   if (ShenandoahStringDedup::is_enabled()) {
2745     ShenandoahStringDedup::deduplicate(str);
2746   }
2747 }
2748 
2749 const char* ShenandoahHeap::init_mark_event_message() const {
2750   bool update_refs = has_forwarded_objects();
2751   bool proc_refs = process_references();
2752   bool unload_cls = unload_classes();
2753 
2754   if (update_refs && proc_refs && unload_cls) {
2755     return "Pause Init Mark (update refs) (process weakrefs) (unload classes)";
2756   } else if (update_refs && proc_refs) {
2757     return "Pause Init Mark (update refs) (process weakrefs)";
2758   } else if (update_refs && unload_cls) {
2759     return "Pause Init Mark (update refs) (unload classes)";
2760   } else if (proc_refs && unload_cls) {
2761     return "Pause Init Mark (process weakrefs) (unload classes)";
2762   } else if (update_refs) {
2763     return "Pause Init Mark (update refs)";
2764   } else if (proc_refs) {
2765     return "Pause Init Mark (process weakrefs)";
2766   } else if (unload_cls) {
2767     return "Pause Init Mark (unload classes)";
2768   } else {
2769     return "Pause Init Mark";
2770   }
2771 }
2772 
2773 const char* ShenandoahHeap::final_mark_event_message() const {
2774   bool update_refs = has_forwarded_objects();
2775   bool proc_refs = process_references();
2776   bool unload_cls = unload_classes();
2777 
2778   if (update_refs && proc_refs && unload_cls) {
2779     return "Pause Final Mark (update refs) (process weakrefs) (unload classes)";
2780   } else if (update_refs && proc_refs) {
2781     return "Pause Final Mark (update refs) (process weakrefs)";
2782   } else if (update_refs && unload_cls) {
2783     return "Pause Final Mark (update refs) (unload classes)";
2784   } else if (proc_refs && unload_cls) {
2785     return "Pause Final Mark (process weakrefs) (unload classes)";
2786   } else if (update_refs) {
2787     return "Pause Final Mark (update refs)";
2788   } else if (proc_refs) {
2789     return "Pause Final Mark (process weakrefs)";
2790   } else if (unload_cls) {
2791     return "Pause Final Mark (unload classes)";
2792   } else {
2793     return "Pause Final Mark";
2794   }
2795 }
2796 
2797 const char* ShenandoahHeap::conc_mark_event_message() const {
2798   bool update_refs = has_forwarded_objects();
2799   bool proc_refs = process_references();
2800   bool unload_cls = unload_classes();
2801 
2802   if (update_refs && proc_refs && unload_cls) {
2803     return "Concurrent marking (update refs) (process weakrefs) (unload classes)";
2804   } else if (update_refs && proc_refs) {
2805     return "Concurrent marking (update refs) (process weakrefs)";
2806   } else if (update_refs && unload_cls) {
2807     return "Concurrent marking (update refs) (unload classes)";
2808   } else if (proc_refs && unload_cls) {
2809     return "Concurrent marking (process weakrefs) (unload classes)";
2810   } else if (update_refs) {
2811     return "Concurrent marking (update refs)";
2812   } else if (proc_refs) {
2813     return "Concurrent marking (process weakrefs)";
2814   } else if (unload_cls) {
2815     return "Concurrent marking (unload classes)";
2816   } else {
2817     return "Concurrent marking";
2818   }
2819 }
2820 
2821 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
2822   switch (point) {
2823     case _degenerated_unset:
2824       return "Pause Degenerated GC (<UNSET>)";
2825     case _degenerated_traversal:
2826       return "Pause Degenerated GC (Traversal)";
2827     case _degenerated_outside_cycle:
2828       return "Pause Degenerated GC (Outside of Cycle)";
2829     case _degenerated_mark:
2830       return "Pause Degenerated GC (Mark)";
2831     case _degenerated_evac:
2832       return "Pause Degenerated GC (Evacuation)";
2833     case _degenerated_updaterefs:
2834       return "Pause Degenerated GC (Update Refs)";
2835     default:
2836       ShouldNotReachHere();
2837       return "ERROR";
2838   }
2839 }
2840 
2841 jushort* ShenandoahHeap::get_liveness_cache(uint worker_id) {
2842 #ifdef ASSERT
2843   assert(_liveness_cache != NULL, "sanity");
2844   assert(worker_id < _max_workers, "sanity");
2845   for (uint i = 0; i < num_regions(); i++) {
2846     assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
2847   }
2848 #endif
2849   return _liveness_cache[worker_id];
2850 }
2851 
2852 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
2853   assert(worker_id < _max_workers, "sanity");
2854   assert(_liveness_cache != NULL, "sanity");
2855   jushort* ld = _liveness_cache[worker_id];
2856   for (uint i = 0; i < num_regions(); i++) {
2857     ShenandoahHeapRegion* r = get_region(i);
2858     jushort live = ld[i];
2859     if (live > 0) {
2860       r->increase_live_data_gc_words(live);
2861       ld[i] = 0;
2862     }
2863   }
2864 }
2865 
2866 size_t ShenandoahHeap::obj_size(oop obj) const {
2867   return CollectedHeap::obj_size(obj) + ShenandoahBrooksPointer::word_size();
2868 }
2869 
2870 ptrdiff_t ShenandoahHeap::cell_header_size() const {
2871   return ShenandoahBrooksPointer::byte_size();
2872 }
2873 
2874 BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
2875   return ShenandoahHeap::heap()->has_forwarded_objects() ? reinterpret_cast<BoolObjectClosure*>(&_fwd_alive_cl)
2876                                                          : reinterpret_cast<BoolObjectClosure*>(&_alive_cl);
2877 }