1 /*
   2  * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "memory/allocation.hpp"
  26 #include "memory/universe.hpp"
  27 
  28 #include "gc/shared/gcArguments.hpp"
  29 #include "gc/shared/gcTimer.hpp"
  30 #include "gc/shared/gcTraceTime.inline.hpp"
  31 #include "gc/shared/locationPrinter.inline.hpp"
  32 #include "gc/shared/memAllocator.hpp"
  33 #include "gc/shared/plab.hpp"
  34 
  35 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
  36 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  37 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  38 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  39 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  40 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  41 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
  42 #include "gc/shenandoah/shenandoahControlThread.hpp"
  43 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  44 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  45 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  46 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
  47 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  48 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  49 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  50 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  51 #include "gc/shenandoah/shenandoahMetrics.hpp"
  52 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  53 #include "gc/shenandoah/shenandoahNormalMode.hpp"
  54 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  55 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  56 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  57 #include "gc/shenandoah/shenandoahPassiveMode.hpp"
  58 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  59 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  60 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
  61 #include "gc/shenandoah/shenandoahTraversalMode.hpp"
  62 #include "gc/shenandoah/shenandoahUtils.hpp"
  63 #include "gc/shenandoah/shenandoahVerifier.hpp"
  64 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  65 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  66 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  67 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  68 #if INCLUDE_JFR
  69 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  70 #endif
  71 
  72 #include "memory/metaspace.hpp"
  73 #include "oops/compressedOops.inline.hpp"
  74 #include "runtime/atomic.hpp"
  75 #include "runtime/globals.hpp"
  76 #include "runtime/interfaceSupport.inline.hpp"
  77 #include "runtime/orderAccess.hpp"
  78 #include "runtime/safepointMechanism.hpp"
  79 #include "runtime/vmThread.hpp"
  80 #include "services/mallocTracker.hpp"
  81 
  82 #ifdef ASSERT
  83 template <class T>
  84 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
  85   T o = RawAccess<>::oop_load(p);
  86   if (! CompressedOops::is_null(o)) {
  87     oop obj = CompressedOops::decode_not_null(o);
  88     shenandoah_assert_not_forwarded(p, obj);
  89   }
  90 }
  91 
  92 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
  93 void ShenandoahAssertToSpaceClosure::do_oop(oop* p)       { do_oop_work(p); }
  94 #endif
  95 
  96 class ShenandoahPretouchHeapTask : public AbstractGangTask {
  97 private:
  98   ShenandoahRegionIterator _regions;
  99   const size_t _page_size;
 100 public:
 101   ShenandoahPretouchHeapTask(size_t page_size) :
 102     AbstractGangTask("Shenandoah Pretouch Heap"),
 103     _page_size(page_size) {}
 104 
 105   virtual void work(uint worker_id) {
 106     ShenandoahHeapRegion* r = _regions.next();
 107     while (r != NULL) {
 108       os::pretouch_memory(r->bottom(), r->end(), _page_size);
 109       r = _regions.next();
 110     }
 111   }
 112 };
 113 
 114 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
 115 private:
 116   ShenandoahRegionIterator _regions;
 117   char* _bitmap_base;
 118   const size_t _bitmap_size;
 119   const size_t _page_size;
 120 public:
 121   ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
 122     AbstractGangTask("Shenandoah Pretouch Bitmap"),
 123     _bitmap_base(bitmap_base),
 124     _bitmap_size(bitmap_size),
 125     _page_size(page_size) {}
 126 
 127   virtual void work(uint worker_id) {
 128     ShenandoahHeapRegion* r = _regions.next();
 129     while (r != NULL) {
 130       size_t start = r->region_number()       * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 131       size_t end   = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 132       assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
 133 
 134       os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
 135 
 136       r = _regions.next();
 137     }
 138   }
 139 };
 140 
 141 jint ShenandoahHeap::initialize() {
 142   initialize_heuristics();
 143 
 144   //
 145   // Figure out heap sizing
 146   //
 147 
 148   size_t init_byte_size = InitialHeapSize;
 149   size_t min_byte_size  = MinHeapSize;
 150   size_t max_byte_size  = MaxHeapSize;
 151   size_t heap_alignment = HeapAlignment;
 152 
 153   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 154 
 155   if (ShenandoahAlwaysPreTouch) {
 156     // Enabled pre-touch means the entire heap is committed right away.
 157     init_byte_size = max_byte_size;
 158   }
 159 
 160   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 161   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 162 
 163   _num_regions = ShenandoahHeapRegion::region_count();
 164 
 165   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 166   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 167   assert(num_committed_regions <= _num_regions, "sanity");
 168   _initial_size = num_committed_regions * reg_size_bytes;
 169 
 170   size_t num_min_regions = min_byte_size / reg_size_bytes;
 171   num_min_regions = MIN2(num_min_regions, _num_regions);
 172   assert(num_min_regions <= _num_regions, "sanity");
 173   _minimum_size = num_min_regions * reg_size_bytes;
 174 
 175   _committed = _initial_size;
 176 
 177   size_t heap_page_size   = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 178   size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 179 
 180   //
 181   // Reserve and commit memory for heap
 182   //
 183 
 184   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 185   initialize_reserved_region(heap_rs);
 186   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 187   _heap_region_special = heap_rs.special();
 188 
 189   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 190          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 191 
 192 #if SHENANDOAH_OPTIMIZED_OBJTASK
 193   // The optimized ObjArrayChunkedTask takes some bits away from the full object bits.
 194   // Fail if we ever attempt to address more than we can.
 195   if ((uintptr_t)heap_rs.end() >= ObjArrayChunkedTask::max_addressable()) {
 196     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 197                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 198                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 199                 p2i(heap_rs.base()), p2i(heap_rs.end()), ObjArrayChunkedTask::max_addressable());
 200     vm_exit_during_initialization("Fatal Error", buf);
 201   }
 202 #endif
 203 
 204   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 205   if (!_heap_region_special) {
 206     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 207                               "Cannot commit heap memory");
 208   }
 209 
 210   //
 211   // Reserve and commit memory for bitmap(s)
 212   //
 213 
 214   _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
 215   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 216 
 217   size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
 218 
 219   guarantee(bitmap_bytes_per_region != 0,
 220             "Bitmap bytes per region should not be zero");
 221   guarantee(is_power_of_2(bitmap_bytes_per_region),
 222             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 223 
 224   if (bitmap_page_size > bitmap_bytes_per_region) {
 225     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 226     _bitmap_bytes_per_slice = bitmap_page_size;
 227   } else {
 228     _bitmap_regions_per_slice = 1;
 229     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 230   }
 231 
 232   guarantee(_bitmap_regions_per_slice >= 1,
 233             "Should have at least one region per slice: " SIZE_FORMAT,
 234             _bitmap_regions_per_slice);
 235 
 236   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 237             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 238             _bitmap_bytes_per_slice, bitmap_page_size);
 239 
 240   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 241   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 242   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 243   _bitmap_region_special = bitmap.special();
 244 
 245   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 246                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 247   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 248   if (!_bitmap_region_special) {
 249     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 250                               "Cannot commit bitmap memory");
 251   }
 252 
 253   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 254 
 255   if (ShenandoahVerify) {
 256     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 257     if (!verify_bitmap.special()) {
 258       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 259                                 "Cannot commit verification bitmap memory");
 260     }
 261     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 262     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 263     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 264     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 265   }
 266 
 267   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 268   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
 269   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 270   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 271   _aux_bitmap_region_special = aux_bitmap.special();
 272   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 273 
 274   //
 275   // Create regions and region sets
 276   //
 277 
 278   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 279   _free_set = new ShenandoahFreeSet(this, _num_regions);
 280   _collection_set = new ShenandoahCollectionSet(this, sh_rs.base(), sh_rs.size());
 281 
 282   {
 283     ShenandoahHeapLocker locker(lock());
 284 
 285     size_t size_words = ShenandoahHeapRegion::region_size_words();
 286 
 287     for (size_t i = 0; i < _num_regions; i++) {
 288       HeapWord* start = (HeapWord*)sh_rs.base() + size_words * i;
 289       bool is_committed = i < num_committed_regions;
 290       ShenandoahHeapRegion* r = new ShenandoahHeapRegion(this, start, size_words, i, is_committed);
 291 
 292       _marking_context->initialize_top_at_mark_start(r);
 293       _regions[i] = r;
 294       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 295     }
 296 
 297     // Initialize to complete
 298     _marking_context->mark_complete();
 299 
 300     _free_set->rebuild();
 301   }
 302 
 303   if (ShenandoahAlwaysPreTouch) {
 304     assert(!AlwaysPreTouch, "Should have been overridden");
 305 
 306     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 307     // before initialize() below zeroes it with initializing thread. For any given region,
 308     // we touch the region and the corresponding bitmaps from the same thread.
 309     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 310 
 311     size_t pretouch_heap_page_size = heap_page_size;
 312     size_t pretouch_bitmap_page_size = bitmap_page_size;
 313 
 314 #ifdef LINUX
 315     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 316     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 317     // them into huge one. Therefore, we need to pretouch with smaller pages.
 318     if (UseTransparentHugePages) {
 319       pretouch_heap_page_size = (size_t)os::vm_page_size();
 320       pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 321     }
 322 #endif
 323 
 324     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 325     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 326 
 327     log_info(gc, init)("Pretouch bitmap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 328                        _num_regions, pretouch_bitmap_page_size);
 329     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, pretouch_bitmap_page_size);
 330     _workers->run_task(&bcl);
 331 
 332     log_info(gc, init)("Pretouch heap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 333                        _num_regions, pretouch_heap_page_size);
 334     ShenandoahPretouchHeapTask hcl(pretouch_heap_page_size);
 335     _workers->run_task(&hcl);
 336   }
 337 
 338   //
 339   // Initialize the rest of GC subsystems
 340   //
 341 
 342   _liveness_cache = NEW_C_HEAP_ARRAY(jushort*, _max_workers, mtGC);
 343   for (uint worker = 0; worker < _max_workers; worker++) {
 344     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(jushort, _num_regions, mtGC);
 345     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(jushort));
 346   }
 347 
 348   // There should probably be Shenandoah-specific options for these,
 349   // just as there are G1-specific options.
 350   {
 351     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 352     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 353     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 354   }
 355 
 356   _monitoring_support = new ShenandoahMonitoringSupport(this);
 357   _phase_timings = new ShenandoahPhaseTimings();
 358   ShenandoahStringDedup::initialize();
 359   ShenandoahCodeRoots::initialize();
 360 
 361   if (ShenandoahAllocationTrace) {
 362     _alloc_tracker = new ShenandoahAllocTracker();
 363   }
 364 
 365   if (ShenandoahPacing) {
 366     _pacer = new ShenandoahPacer(this);
 367     _pacer->setup_for_idle();
 368   } else {
 369     _pacer = NULL;
 370   }
 371 
 372   _traversal_gc = strcmp(ShenandoahGCMode, "traversal") == 0 ?
 373                   new ShenandoahTraversalGC(this, _num_regions) :
 374                   NULL;
 375 
 376   _control_thread = new ShenandoahControlThread();
 377 
 378   log_info(gc, init)("Initialize Shenandoah heap: " SIZE_FORMAT "%s initial, " SIZE_FORMAT "%s min, " SIZE_FORMAT "%s max",
 379                      byte_size_in_proper_unit(_initial_size),  proper_unit_for_byte_size(_initial_size),
 380                      byte_size_in_proper_unit(_minimum_size),  proper_unit_for_byte_size(_minimum_size),
 381                      byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity())
 382   );
 383 
 384   log_info(gc, init)("Safepointing mechanism: %s",
 385                      SafepointMechanism::uses_thread_local_poll() ? "thread-local poll" :
 386                      (SafepointMechanism::uses_global_page_poll() ? "global-page poll" : "unknown"));
 387 
 388   return JNI_OK;
 389 }
 390 
 391 void ShenandoahHeap::initialize_heuristics() {
 392   if (ShenandoahGCMode != NULL) {
 393     if (strcmp(ShenandoahGCMode, "traversal") == 0) {
 394       _gc_mode = new ShenandoahTraversalMode();
 395     } else if (strcmp(ShenandoahGCMode, "normal") == 0) {
 396       _gc_mode = new ShenandoahNormalMode();
 397     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 398       _gc_mode = new ShenandoahPassiveMode();
 399     } else {
 400       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 401     }
 402   } else {
 403     ShouldNotReachHere();
 404   }
 405   _gc_mode->initialize_flags();
 406   _heuristics = _gc_mode->initialize_heuristics();
 407 
 408   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 409     vm_exit_during_initialization(
 410             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 411                     _heuristics->name()));
 412   }
 413   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 414     vm_exit_during_initialization(
 415             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 416                     _heuristics->name()));
 417   }
 418   log_info(gc, init)("Shenandoah heuristics: %s",
 419                      _heuristics->name());
 420 }
 421 
 422 #ifdef _MSC_VER
 423 #pragma warning( push )
 424 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 425 #endif
 426 
 427 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 428   CollectedHeap(),
 429   _initial_size(0),
 430   _used(0),
 431   _committed(0),
 432   _bytes_allocated_since_gc_start(0),
 433   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 434   _workers(NULL),
 435   _safepoint_workers(NULL),
 436   _heap_region_special(false),
 437   _num_regions(0),
 438   _regions(NULL),
 439   _update_refs_iterator(this),
 440   _control_thread(NULL),
 441   _shenandoah_policy(policy),
 442   _heuristics(NULL),
 443   _free_set(NULL),
 444   _scm(new ShenandoahConcurrentMark()),
 445   _traversal_gc(NULL),
 446   _full_gc(new ShenandoahMarkCompact()),
 447   _pacer(NULL),
 448   _verifier(NULL),
 449   _alloc_tracker(NULL),
 450   _phase_timings(NULL),
 451   _monitoring_support(NULL),
 452   _memory_pool(NULL),
 453   _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
 454   _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
 455   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
 456   _soft_ref_policy(),
 457   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 458   _ref_processor(NULL),
 459   _marking_context(NULL),
 460   _bitmap_size(0),
 461   _bitmap_regions_per_slice(0),
 462   _bitmap_bytes_per_slice(0),
 463   _bitmap_region_special(false),
 464   _aux_bitmap_region_special(false),
 465   _liveness_cache(NULL),
 466   _collection_set(NULL)
 467 {
 468   log_info(gc, init)("GC threads: " UINT32_FORMAT " parallel, " UINT32_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads);
 469   log_info(gc, init)("Reference processing: %s", ParallelRefProcEnabled ? "parallel" : "serial");
 470 
 471   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 472 
 473   _max_workers = MAX2(_max_workers, 1U);
 474   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
 475                             /* are_GC_task_threads */ true,
 476                             /* are_ConcurrentGC_threads */ true);
 477   if (_workers == NULL) {
 478     vm_exit_during_initialization("Failed necessary allocation.");
 479   } else {
 480     _workers->initialize_workers();
 481   }
 482 
 483   if (ShenandoahParallelSafepointThreads > 1) {
 484     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
 485                                                 ShenandoahParallelSafepointThreads,
 486                       /* are_GC_task_threads */ false,
 487                  /* are_ConcurrentGC_threads */ false);
 488     _safepoint_workers->initialize_workers();
 489   }
 490 }
 491 
 492 #ifdef _MSC_VER
 493 #pragma warning( pop )
 494 #endif
 495 
 496 class ShenandoahResetBitmapTask : public AbstractGangTask {
 497 private:
 498   ShenandoahRegionIterator _regions;
 499 
 500 public:
 501   ShenandoahResetBitmapTask() :
 502     AbstractGangTask("Parallel Reset Bitmap Task") {}
 503 
 504   void work(uint worker_id) {
 505     ShenandoahHeapRegion* region = _regions.next();
 506     ShenandoahHeap* heap = ShenandoahHeap::heap();
 507     ShenandoahMarkingContext* const ctx = heap->marking_context();
 508     while (region != NULL) {
 509       if (heap->is_bitmap_slice_committed(region)) {
 510         ctx->clear_bitmap(region);
 511       }
 512       region = _regions.next();
 513     }
 514   }
 515 };
 516 
 517 void ShenandoahHeap::reset_mark_bitmap() {
 518   assert_gc_workers(_workers->active_workers());
 519   mark_incomplete_marking_context();
 520 
 521   ShenandoahResetBitmapTask task;
 522   _workers->run_task(&task);
 523 }
 524 
 525 void ShenandoahHeap::print_on(outputStream* st) const {
 526   st->print_cr("Shenandoah Heap");
 527   st->print_cr(" " SIZE_FORMAT "%s total, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 528                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 529                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 530                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 531   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 532                num_regions(),
 533                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 534                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 535 
 536   st->print("Status: ");
 537   if (has_forwarded_objects())               st->print("has forwarded objects, ");
 538   if (is_concurrent_mark_in_progress())      st->print("marking, ");
 539   if (is_evacuation_in_progress())           st->print("evacuating, ");
 540   if (is_update_refs_in_progress())          st->print("updating refs, ");
 541   if (is_concurrent_traversal_in_progress()) st->print("traversal, ");
 542   if (is_degenerated_gc_in_progress())       st->print("degenerated gc, ");
 543   if (is_full_gc_in_progress())              st->print("full gc, ");
 544   if (is_full_gc_move_in_progress())         st->print("full gc move, ");
 545   if (is_concurrent_root_in_progress())      st->print("concurrent roots, ");
 546 
 547   if (cancelled_gc()) {
 548     st->print("cancelled");
 549   } else {
 550     st->print("not cancelled");
 551   }
 552   st->cr();
 553 
 554   st->print_cr("Reserved region:");
 555   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 556                p2i(reserved_region().start()),
 557                p2i(reserved_region().end()));
 558 
 559   ShenandoahCollectionSet* cset = collection_set();
 560   st->print_cr("Collection set:");
 561   if (cset != NULL) {
 562     st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
 563     st->print_cr(" - map (biased):  " PTR_FORMAT, p2i(cset->biased_map_address()));
 564   } else {
 565     st->print_cr(" (NULL)");
 566   }
 567 
 568   st->cr();
 569   MetaspaceUtils::print_on(st);
 570 
 571   if (Verbose) {
 572     print_heap_regions_on(st);
 573   }
 574 }
 575 
 576 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 577 public:
 578   void do_thread(Thread* thread) {
 579     assert(thread != NULL, "Sanity");
 580     assert(thread->is_Worker_thread(), "Only worker thread expected");
 581     ShenandoahThreadLocalData::initialize_gclab(thread);
 582   }
 583 };
 584 
 585 void ShenandoahHeap::post_initialize() {
 586   CollectedHeap::post_initialize();
 587   MutexLocker ml(Threads_lock);
 588 
 589   ShenandoahInitWorkerGCLABClosure init_gclabs;
 590   _workers->threads_do(&init_gclabs);
 591 
 592   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 593   // Now, we will let WorkGang to initialize gclab when new worker is created.
 594   _workers->set_initialize_gclab();
 595 
 596   _scm->initialize(_max_workers);
 597   _full_gc->initialize(_gc_timer);
 598 
 599   ref_processing_init();
 600 
 601   _heuristics->initialize();
 602 
 603   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 604 }
 605 
 606 size_t ShenandoahHeap::used() const {
 607   return Atomic::load_acquire(&_used);
 608 }
 609 
 610 size_t ShenandoahHeap::committed() const {
 611   OrderAccess::acquire();
 612   return _committed;
 613 }
 614 
 615 void ShenandoahHeap::increase_committed(size_t bytes) {
 616   assert_heaplock_or_safepoint();
 617   _committed += bytes;
 618 }
 619 
 620 void ShenandoahHeap::decrease_committed(size_t bytes) {
 621   assert_heaplock_or_safepoint();
 622   _committed -= bytes;
 623 }
 624 
 625 void ShenandoahHeap::increase_used(size_t bytes) {
 626   Atomic::add(&_used, bytes);
 627 }
 628 
 629 void ShenandoahHeap::set_used(size_t bytes) {
 630   Atomic::release_store_fence(&_used, bytes);
 631 }
 632 
 633 void ShenandoahHeap::decrease_used(size_t bytes) {
 634   assert(used() >= bytes, "never decrease heap size by more than we've left");
 635   Atomic::sub(&_used, bytes);
 636 }
 637 
 638 void ShenandoahHeap::increase_allocated(size_t bytes) {
 639   Atomic::add(&_bytes_allocated_since_gc_start, bytes);
 640 }
 641 
 642 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 643   size_t bytes = words * HeapWordSize;
 644   if (!waste) {
 645     increase_used(bytes);
 646   }
 647   increase_allocated(bytes);
 648   if (ShenandoahPacing) {
 649     control_thread()->pacing_notify_alloc(words);
 650     if (waste) {
 651       pacer()->claim_for_alloc(words, true);
 652     }
 653   }
 654 }
 655 
 656 size_t ShenandoahHeap::capacity() const {
 657   return committed();
 658 }
 659 
 660 size_t ShenandoahHeap::max_capacity() const {
 661   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 662 }
 663 
 664 size_t ShenandoahHeap::min_capacity() const {
 665   return _minimum_size;
 666 }
 667 
 668 size_t ShenandoahHeap::initial_capacity() const {
 669   return _initial_size;
 670 }
 671 
 672 bool ShenandoahHeap::is_in(const void* p) const {
 673   HeapWord* heap_base = (HeapWord*) base();
 674   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 675   return p >= heap_base && p < last_region_end;
 676 }
 677 
 678 void ShenandoahHeap::op_uncommit(double shrink_before) {
 679   assert (ShenandoahUncommit, "should be enabled");
 680 
 681   // Application allocates from the beginning of the heap, and GC allocates at
 682   // the end of it. It is more efficient to uncommit from the end, so that applications
 683   // could enjoy the near committed regions. GC allocations are much less frequent,
 684   // and therefore can accept the committing costs.
 685 
 686   size_t count = 0;
 687   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 688     ShenandoahHeapRegion* r = get_region(i - 1);
 689     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 690       ShenandoahHeapLocker locker(lock());
 691       if (r->is_empty_committed()) {
 692         // Do not uncommit below minimal capacity
 693         if (committed() < min_capacity() + ShenandoahHeapRegion::region_size_bytes()) {
 694           break;
 695         }
 696 
 697         r->make_uncommitted();
 698         count++;
 699       }
 700     }
 701     SpinPause(); // allow allocators to take the lock
 702   }
 703 
 704   if (count > 0) {
 705     control_thread()->notify_heap_changed();
 706   }
 707 }
 708 
 709 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 710   // New object should fit the GCLAB size
 711   size_t min_size = MAX2(size, PLAB::min_size());
 712 
 713   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 714   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 715   new_size = MIN2(new_size, PLAB::max_size());
 716   new_size = MAX2(new_size, PLAB::min_size());
 717 
 718   // Record new heuristic value even if we take any shortcut. This captures
 719   // the case when moderately-sized objects always take a shortcut. At some point,
 720   // heuristics should catch up with them.
 721   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 722 
 723   if (new_size < size) {
 724     // New size still does not fit the object. Fall back to shared allocation.
 725     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 726     return NULL;
 727   }
 728 
 729   // Retire current GCLAB, and allocate a new one.
 730   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 731   gclab->retire();
 732 
 733   size_t actual_size = 0;
 734   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 735   if (gclab_buf == NULL) {
 736     return NULL;
 737   }
 738 
 739   assert (size <= actual_size, "allocation should fit");
 740 
 741   if (ZeroTLAB) {
 742     // ..and clear it.
 743     Copy::zero_to_words(gclab_buf, actual_size);
 744   } else {
 745     // ...and zap just allocated object.
 746 #ifdef ASSERT
 747     // Skip mangling the space corresponding to the object header to
 748     // ensure that the returned space is not considered parsable by
 749     // any concurrent GC thread.
 750     size_t hdr_size = oopDesc::header_size();
 751     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 752 #endif // ASSERT
 753   }
 754   gclab->set_buf(gclab_buf, actual_size);
 755   return gclab->allocate(size);
 756 }
 757 
 758 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 759                                             size_t requested_size,
 760                                             size_t* actual_size) {
 761   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 762   HeapWord* res = allocate_memory(req);
 763   if (res != NULL) {
 764     *actual_size = req.actual_size();
 765   } else {
 766     *actual_size = 0;
 767   }
 768   return res;
 769 }
 770 
 771 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 772                                              size_t word_size,
 773                                              size_t* actual_size) {
 774   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 775   HeapWord* res = allocate_memory(req);
 776   if (res != NULL) {
 777     *actual_size = req.actual_size();
 778   } else {
 779     *actual_size = 0;
 780   }
 781   return res;
 782 }
 783 
 784 ShenandoahHeap* ShenandoahHeap::heap() {
 785   CollectedHeap* heap = Universe::heap();
 786   assert(heap != NULL, "Unitialized access to ShenandoahHeap::heap()");
 787   assert(heap->kind() == CollectedHeap::Shenandoah, "not a shenandoah heap");
 788   return (ShenandoahHeap*) heap;
 789 }
 790 
 791 ShenandoahHeap* ShenandoahHeap::heap_no_check() {
 792   CollectedHeap* heap = Universe::heap();
 793   return (ShenandoahHeap*) heap;
 794 }
 795 
 796 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 797   ShenandoahAllocTrace trace_alloc(req.size(), req.type());
 798 
 799   intptr_t pacer_epoch = 0;
 800   bool in_new_region = false;
 801   HeapWord* result = NULL;
 802 
 803   if (req.is_mutator_alloc()) {
 804     if (ShenandoahPacing) {
 805       pacer()->pace_for_alloc(req.size());
 806       pacer_epoch = pacer()->epoch();
 807     }
 808 
 809     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 810       result = allocate_memory_under_lock(req, in_new_region);
 811     }
 812 
 813     // Allocation failed, block until control thread reacted, then retry allocation.
 814     //
 815     // It might happen that one of the threads requesting allocation would unblock
 816     // way later after GC happened, only to fail the second allocation, because
 817     // other threads have already depleted the free storage. In this case, a better
 818     // strategy is to try again, as long as GC makes progress.
 819     //
 820     // Then, we need to make sure the allocation was retried after at least one
 821     // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
 822 
 823     size_t tries = 0;
 824 
 825     while (result == NULL && _progress_last_gc.is_set()) {
 826       tries++;
 827       control_thread()->handle_alloc_failure(req.size());
 828       result = allocate_memory_under_lock(req, in_new_region);
 829     }
 830 
 831     while (result == NULL && tries <= ShenandoahFullGCThreshold) {
 832       tries++;
 833       control_thread()->handle_alloc_failure(req.size());
 834       result = allocate_memory_under_lock(req, in_new_region);
 835     }
 836 
 837   } else {
 838     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 839     result = allocate_memory_under_lock(req, in_new_region);
 840     // Do not call handle_alloc_failure() here, because we cannot block.
 841     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
 842   }
 843 
 844   if (in_new_region) {
 845     control_thread()->notify_heap_changed();
 846   }
 847 
 848   if (result != NULL) {
 849     size_t requested = req.size();
 850     size_t actual = req.actual_size();
 851 
 852     assert (req.is_lab_alloc() || (requested == actual),
 853             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 854             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 855 
 856     if (req.is_mutator_alloc()) {
 857       notify_mutator_alloc_words(actual, false);
 858 
 859       // If we requested more than we were granted, give the rest back to pacer.
 860       // This only matters if we are in the same pacing epoch: do not try to unpace
 861       // over the budget for the other phase.
 862       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 863         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 864       }
 865     } else {
 866       increase_used(actual*HeapWordSize);
 867     }
 868   }
 869 
 870   return result;
 871 }
 872 
 873 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 874   ShenandoahHeapLocker locker(lock());
 875   return _free_set->allocate(req, in_new_region);
 876 }
 877 
 878 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 879                                         bool*  gc_overhead_limit_was_exceeded) {
 880   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 881   return allocate_memory(req);
 882 }
 883 
 884 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 885                                                              size_t size,
 886                                                              Metaspace::MetadataType mdtype) {
 887   MetaWord* result;
 888 
 889   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 890   if (heuristics()->can_unload_classes()) {
 891     ShenandoahHeuristics* h = heuristics();
 892     h->record_metaspace_oom();
 893   }
 894 
 895   // Expand and retry allocation
 896   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 897   if (result != NULL) {
 898     return result;
 899   }
 900 
 901   // Start full GC
 902   collect(GCCause::_metadata_GC_clear_soft_refs);
 903 
 904   // Retry allocation
 905   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 906   if (result != NULL) {
 907     return result;
 908   }
 909 
 910   // Expand and retry allocation
 911   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 912   if (result != NULL) {
 913     return result;
 914   }
 915 
 916   // Out of memory
 917   return NULL;
 918 }
 919 
 920 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
 921 private:
 922   ShenandoahHeap* const _heap;
 923   Thread* const _thread;
 924 public:
 925   ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
 926     _heap(heap), _thread(Thread::current()) {}
 927 
 928   void do_object(oop p) {
 929     shenandoah_assert_marked(NULL, p);
 930     if (!p->is_forwarded()) {
 931       _heap->evacuate_object(p, _thread);
 932     }
 933   }
 934 };
 935 
 936 class ShenandoahEvacuationTask : public AbstractGangTask {
 937 private:
 938   ShenandoahHeap* const _sh;
 939   ShenandoahCollectionSet* const _cs;
 940   bool _concurrent;
 941 public:
 942   ShenandoahEvacuationTask(ShenandoahHeap* sh,
 943                            ShenandoahCollectionSet* cs,
 944                            bool concurrent) :
 945     AbstractGangTask("Parallel Evacuation Task"),
 946     _sh(sh),
 947     _cs(cs),
 948     _concurrent(concurrent)
 949   {}
 950 
 951   void work(uint worker_id) {
 952     if (_concurrent) {
 953       ShenandoahConcurrentWorkerSession worker_session(worker_id);
 954       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 955       ShenandoahEvacOOMScope oom_evac_scope;
 956       do_work();
 957     } else {
 958       ShenandoahParallelWorkerSession worker_session(worker_id);
 959       ShenandoahEvacOOMScope oom_evac_scope;
 960       do_work();
 961     }
 962   }
 963 
 964 private:
 965   void do_work() {
 966     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
 967     ShenandoahHeapRegion* r;
 968     while ((r =_cs->claim_next()) != NULL) {
 969       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->region_number());
 970       _sh->marked_object_iterate(r, &cl);
 971 
 972       if (ShenandoahPacing) {
 973         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
 974       }
 975 
 976       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
 977         break;
 978       }
 979     }
 980   }
 981 };
 982 
 983 void ShenandoahHeap::trash_cset_regions() {
 984   ShenandoahHeapLocker locker(lock());
 985 
 986   ShenandoahCollectionSet* set = collection_set();
 987   ShenandoahHeapRegion* r;
 988   set->clear_current_index();
 989   while ((r = set->next()) != NULL) {
 990     r->make_trash();
 991   }
 992   collection_set()->clear();
 993 }
 994 
 995 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
 996   st->print_cr("Heap Regions:");
 997   st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
 998   st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
 999   st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start (previous, next)");
1000   st->print_cr("SN=alloc sequence numbers (first mutator, last mutator, first gc, last gc)");
1001 
1002   for (size_t i = 0; i < num_regions(); i++) {
1003     get_region(i)->print_on(st);
1004   }
1005 }
1006 
1007 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1008   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1009 
1010   oop humongous_obj = oop(start->bottom());
1011   size_t size = humongous_obj->size();
1012   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1013   size_t index = start->region_number() + required_regions - 1;
1014 
1015   assert(!start->has_live(), "liveness must be zero");
1016 
1017   for(size_t i = 0; i < required_regions; i++) {
1018     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1019     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1020     ShenandoahHeapRegion* region = get_region(index --);
1021 
1022     assert(region->is_humongous(), "expect correct humongous start or continuation");
1023     assert(!region->is_cset(), "Humongous region should not be in collection set");
1024 
1025     region->make_trash_immediate();
1026   }
1027 }
1028 
1029 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1030 public:
1031   void do_thread(Thread* thread) {
1032     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1033     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1034     gclab->retire();
1035   }
1036 };
1037 
1038 void ShenandoahHeap::make_parsable(bool retire_tlabs) {
1039   if (UseTLAB) {
1040     CollectedHeap::ensure_parsability(retire_tlabs);
1041   }
1042   ShenandoahRetireGCLABClosure cl;
1043   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1044     cl.do_thread(t);
1045   }
1046   workers()->threads_do(&cl);
1047 }
1048 
1049 void ShenandoahHeap::resize_tlabs() {
1050   CollectedHeap::resize_all_tlabs();
1051 }
1052 
1053 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1054 private:
1055   ShenandoahRootEvacuator* _rp;
1056 
1057 public:
1058   ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1059     AbstractGangTask("Shenandoah evacuate and update roots"),
1060     _rp(rp) {}
1061 
1062   void work(uint worker_id) {
1063     ShenandoahParallelWorkerSession worker_session(worker_id);
1064     ShenandoahEvacOOMScope oom_evac_scope;
1065     ShenandoahEvacuateUpdateRootsClosure cl;
1066     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1067     _rp->roots_do(worker_id, &cl);
1068   }
1069 };
1070 
1071 void ShenandoahHeap::evacuate_and_update_roots() {
1072 #if COMPILER2_OR_JVMCI
1073   DerivedPointerTable::clear();
1074 #endif
1075   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1076   {
1077     // Include concurrent roots if current cycle can not process those roots concurrently
1078     ShenandoahRootEvacuator rp(workers()->active_workers(),
1079                                ShenandoahPhaseTimings::init_evac,
1080                                !ShenandoahConcurrentRoots::should_do_concurrent_roots(),
1081                                !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
1082     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1083     workers()->run_task(&roots_task);
1084   }
1085 
1086 #if COMPILER2_OR_JVMCI
1087   DerivedPointerTable::update_pointers();
1088 #endif
1089 }
1090 
1091 // Returns size in bytes
1092 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1093   if (ShenandoahElasticTLAB) {
1094     // With Elastic TLABs, return the max allowed size, and let the allocation path
1095     // figure out the safe size for current allocation.
1096     return ShenandoahHeapRegion::max_tlab_size_bytes();
1097   } else {
1098     return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1099   }
1100 }
1101 
1102 size_t ShenandoahHeap::max_tlab_size() const {
1103   // Returns size in words
1104   return ShenandoahHeapRegion::max_tlab_size_words();
1105 }
1106 
1107 class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
1108 public:
1109   void do_thread(Thread* thread) {
1110     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1111     gclab->retire();
1112     if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1113       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1114     }
1115   }
1116 };
1117 
1118 void ShenandoahHeap::retire_and_reset_gclabs() {
1119   ShenandoahRetireAndResetGCLABClosure cl;
1120   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1121     cl.do_thread(t);
1122   }
1123   workers()->threads_do(&cl);
1124 }
1125 
1126 void ShenandoahHeap::collect(GCCause::Cause cause) {
1127   control_thread()->request_gc(cause);
1128 }
1129 
1130 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1131   //assert(false, "Shouldn't need to do full collections");
1132 }
1133 
1134 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1135   Space* sp = heap_region_containing(addr);
1136   if (sp != NULL) {
1137     return sp->block_start(addr);
1138   }
1139   return NULL;
1140 }
1141 
1142 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1143   Space* sp = heap_region_containing(addr);
1144   return sp->block_is_obj(addr);
1145 }
1146 
1147 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1148   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1149 }
1150 
1151 jlong ShenandoahHeap::millis_since_last_gc() {
1152   double v = heuristics()->time_since_last_gc() * 1000;
1153   assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1154   return (jlong)v;
1155 }
1156 
1157 void ShenandoahHeap::prepare_for_verify() {
1158   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1159     make_parsable(false);
1160   }
1161 }
1162 
1163 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1164   workers()->print_worker_threads_on(st);
1165   if (ShenandoahStringDedup::is_enabled()) {
1166     ShenandoahStringDedup::print_worker_threads_on(st);
1167   }
1168 }
1169 
1170 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1171   workers()->threads_do(tcl);
1172   if (_safepoint_workers != NULL) {
1173     _safepoint_workers->threads_do(tcl);
1174   }
1175   if (ShenandoahStringDedup::is_enabled()) {
1176     ShenandoahStringDedup::threads_do(tcl);
1177   }
1178 }
1179 
1180 void ShenandoahHeap::print_tracing_info() const {
1181   LogTarget(Info, gc, stats) lt;
1182   if (lt.is_enabled()) {
1183     ResourceMark rm;
1184     LogStream ls(lt);
1185 
1186     phase_timings()->print_on(&ls);
1187 
1188     ls.cr();
1189     ls.cr();
1190 
1191     shenandoah_policy()->print_gc_stats(&ls);
1192 
1193     ls.cr();
1194     ls.cr();
1195 
1196     if (ShenandoahPacing) {
1197       pacer()->print_on(&ls);
1198     }
1199 
1200     ls.cr();
1201     ls.cr();
1202 
1203     if (ShenandoahAllocationTrace) {
1204       assert(alloc_tracker() != NULL, "Must be");
1205       alloc_tracker()->print_on(&ls);
1206     } else {
1207       ls.print_cr("  Allocation tracing is disabled, use -XX:+ShenandoahAllocationTrace to enable.");
1208     }
1209   }
1210 }
1211 
1212 void ShenandoahHeap::verify(VerifyOption vo) {
1213   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1214     if (ShenandoahVerify) {
1215       verifier()->verify_generic(vo);
1216     } else {
1217       // TODO: Consider allocating verification bitmaps on demand,
1218       // and turn this on unconditionally.
1219     }
1220   }
1221 }
1222 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1223   return _free_set->capacity();
1224 }
1225 
1226 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1227 private:
1228   MarkBitMap* _bitmap;
1229   Stack<oop,mtGC>* _oop_stack;
1230 
1231   template <class T>
1232   void do_oop_work(T* p) {
1233     T o = RawAccess<>::oop_load(p);
1234     if (!CompressedOops::is_null(o)) {
1235       oop obj = CompressedOops::decode_not_null(o);
1236       oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
1237       if (fwd == NULL) {
1238         // There is an odd interaction with VM_HeapWalkOperation, see jvmtiTagMap.cpp.
1239         //
1240         // That operation walks the reachable objects on its own, storing the marking
1241         // wavefront in the object marks. When it is done, it calls the CollectedHeap
1242         // to iterate over all objects to clean up the mess. When it reaches here,
1243         // the Shenandoah fwdptr resolution code encounters the marked objects with
1244         // NULL forwardee. Trying to act on that would crash the VM. Or fail the
1245         // asserts, should we go for resolve_forwarded_pointer(obj).
1246         //
1247         // Therefore, we have to dodge it by doing the raw access to forwardee, and
1248         // assuming the object had no forwardee, if that thing is NULL.
1249       } else {
1250         obj = fwd;
1251       }
1252       assert(oopDesc::is_oop(obj), "must be a valid oop");
1253       if (!_bitmap->is_marked((HeapWord*) obj)) {
1254         _bitmap->mark((HeapWord*) obj);
1255         _oop_stack->push(obj);
1256       }
1257     }
1258   }
1259 public:
1260   ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1261     _bitmap(bitmap), _oop_stack(oop_stack) {}
1262   void do_oop(oop* p)       { do_oop_work(p); }
1263   void do_oop(narrowOop* p) { do_oop_work(p); }
1264 };
1265 
1266 /*
1267  * This is public API, used in preparation of object_iterate().
1268  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1269  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1270  * control, we call SH::make_tlabs_parsable().
1271  */
1272 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1273   // No-op.
1274 }
1275 
1276 /*
1277  * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1278  *
1279  * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1280  * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1281  * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1282  * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1283  * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1284  * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1285  * wiped the bitmap in preparation for next marking).
1286  *
1287  * For all those reasons, we implement object iteration as a single marking traversal, reporting
1288  * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1289  * is allowed to report dead objects, but is not required to do so.
1290  */
1291 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1292   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1293   if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1294     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1295     return;
1296   }
1297 
1298   // Reset bitmap
1299   _aux_bit_map.clear();
1300 
1301   Stack<oop,mtGC> oop_stack;
1302 
1303   // First, we process GC roots according to current GC cycle. This populates the work stack with initial objects.
1304   ShenandoahHeapIterationRootScanner rp;
1305   ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1306 
1307   // If we are unloading classes right now, we should not touch weak roots,
1308   // on the off-chance we would evacuate them and make them live accidentally.
1309   // In other cases, we have to scan all roots.
1310   if (is_evacuation_in_progress() && unload_classes()) {
1311     rp.strong_roots_do(&oops);
1312   } else {
1313     rp.roots_do(&oops);
1314   }
1315 
1316   // Work through the oop stack to traverse heap.
1317   while (! oop_stack.is_empty()) {
1318     oop obj = oop_stack.pop();
1319     assert(oopDesc::is_oop(obj), "must be a valid oop");
1320     cl->do_object(obj);
1321     obj->oop_iterate(&oops);
1322   }
1323 
1324   assert(oop_stack.is_empty(), "should be empty");
1325 
1326   if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1327     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1328   }
1329 }
1330 
1331 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1332 void ShenandoahHeap::keep_alive(oop obj) {
1333   if (is_concurrent_mark_in_progress()) {
1334     ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1335   } else {
1336     // Otherwise, it must be live, guaranteed by LRB
1337     shenandoah_assert_marked_if(NULL, obj, has_forwarded_objects());
1338   }
1339 }
1340 
1341 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1342   for (size_t i = 0; i < num_regions(); i++) {
1343     ShenandoahHeapRegion* current = get_region(i);
1344     blk->heap_region_do(current);
1345   }
1346 }
1347 
1348 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1349 private:
1350   ShenandoahHeap* const _heap;
1351   ShenandoahHeapRegionClosure* const _blk;
1352 
1353   DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));
1354   volatile size_t _index;
1355   DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
1356 
1357 public:
1358   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1359           AbstractGangTask("Parallel Region Task"),
1360           _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1361 
1362   void work(uint worker_id) {
1363     size_t stride = ShenandoahParallelRegionStride;
1364 
1365     size_t max = _heap->num_regions();
1366     while (_index < max) {
1367       size_t cur = Atomic::add(&_index, stride) - stride;
1368       size_t start = cur;
1369       size_t end = MIN2(cur + stride, max);
1370       if (start >= max) break;
1371 
1372       for (size_t i = cur; i < end; i++) {
1373         ShenandoahHeapRegion* current = _heap->get_region(i);
1374         _blk->heap_region_do(current);
1375       }
1376     }
1377   }
1378 };
1379 
1380 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1381   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1382   if (num_regions() > ShenandoahParallelRegionStride) {
1383     ShenandoahParallelHeapRegionTask task(blk);
1384     workers()->run_task(&task);
1385   } else {
1386     heap_region_iterate(blk);
1387   }
1388 }
1389 
1390 class ShenandoahClearLivenessClosure : public ShenandoahHeapRegionClosure {
1391 private:
1392   ShenandoahMarkingContext* const _ctx;
1393 public:
1394   ShenandoahClearLivenessClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1395 
1396   void heap_region_do(ShenandoahHeapRegion* r) {
1397     if (r->is_active()) {
1398       r->clear_live_data();
1399       _ctx->capture_top_at_mark_start(r);
1400     } else {
1401       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->region_number());
1402       assert(_ctx->top_at_mark_start(r) == r->top(),
1403              "Region " SIZE_FORMAT " should already have correct TAMS", r->region_number());
1404     }
1405   }
1406 
1407   bool is_thread_safe() { return true; }
1408 };
1409 
1410 void ShenandoahHeap::op_init_mark() {
1411   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1412   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1413 
1414   assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1415   assert(!marking_context()->is_complete(), "should not be complete");
1416 
1417   if (ShenandoahVerify) {
1418     verifier()->verify_before_concmark();
1419   }
1420 
1421   if (VerifyBeforeGC) {
1422     Universe::verify();
1423   }
1424 
1425   set_concurrent_mark_in_progress(true);
1426   // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1427   {
1428     ShenandoahGCPhase phase(ShenandoahPhaseTimings::make_parsable);
1429     make_parsable(true);
1430   }
1431 
1432   {
1433     ShenandoahGCPhase phase(ShenandoahPhaseTimings::clear_liveness);
1434     ShenandoahClearLivenessClosure clc;
1435     parallel_heap_region_iterate(&clc);
1436   }
1437 
1438   // Make above changes visible to worker threads
1439   OrderAccess::fence();
1440 
1441   concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1442 
1443   if (UseTLAB) {
1444     ShenandoahGCPhase phase(ShenandoahPhaseTimings::resize_tlabs);
1445     resize_tlabs();
1446   }
1447 
1448   if (ShenandoahPacing) {
1449     pacer()->setup_for_mark();
1450   }
1451 }
1452 
1453 void ShenandoahHeap::op_mark() {
1454   concurrent_mark()->mark_from_roots();
1455 }
1456 
1457 class ShenandoahCompleteLivenessClosure : public ShenandoahHeapRegionClosure {
1458 private:
1459   ShenandoahMarkingContext* const _ctx;
1460 public:
1461   ShenandoahCompleteLivenessClosure() : _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
1462 
1463   void heap_region_do(ShenandoahHeapRegion* r) {
1464     if (r->is_active()) {
1465       HeapWord *tams = _ctx->top_at_mark_start(r);
1466       HeapWord *top = r->top();
1467       if (top > tams) {
1468         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1469       }
1470     } else {
1471       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->region_number());
1472       assert(_ctx->top_at_mark_start(r) == r->top(),
1473              "Region " SIZE_FORMAT " should have correct TAMS", r->region_number());
1474     }
1475   }
1476 
1477   bool is_thread_safe() { return true; }
1478 };
1479 
1480 void ShenandoahHeap::op_final_mark() {
1481   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1482 
1483   // It is critical that we
1484   // evacuate roots right after finishing marking, so that we don't
1485   // get unmarked objects in the roots.
1486 
1487   if (!cancelled_gc()) {
1488     concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1489 
1490     // Marking is completed, deactivate SATB barrier
1491     set_concurrent_mark_in_progress(false);
1492     mark_complete_marking_context();
1493 
1494     parallel_cleaning(false /* full gc*/);
1495 
1496     if (has_forwarded_objects()) {
1497       // Degen may be caused by failed evacuation of roots
1498       if (is_degenerated_gc_in_progress()) {
1499         concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
1500       } else {
1501         concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::update_roots);
1502       }
1503       set_has_forwarded_objects(false);
1504    }
1505 
1506     if (ShenandoahVerify) {
1507       verifier()->verify_roots_no_forwarded();
1508     }
1509     // All allocations past TAMS are implicitly live, adjust the region data.
1510     // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1511     {
1512       ShenandoahGCPhase phase(ShenandoahPhaseTimings::complete_liveness);
1513       ShenandoahCompleteLivenessClosure cl;
1514       parallel_heap_region_iterate(&cl);
1515     }
1516 
1517     // Force the threads to reacquire their TLABs outside the collection set.
1518     {
1519       ShenandoahGCPhase phase(ShenandoahPhaseTimings::retire_tlabs);
1520       make_parsable(true);
1521     }
1522 
1523     // We are about to select the collection set, make sure it knows about
1524     // current pinning status. Also, this allows trashing more regions that
1525     // now have their pinning status dropped.
1526     {
1527       ShenandoahGCPhase phase(ShenandoahPhaseTimings::sync_pinned);
1528       sync_pinned_region_status();
1529     }
1530 
1531     // Trash the collection set left over from previous cycle, if any.
1532     {
1533       ShenandoahGCPhase phase(ShenandoahPhaseTimings::trash_cset);
1534       trash_cset_regions();
1535     }
1536 
1537     {
1538       ShenandoahGCPhase phase(ShenandoahPhaseTimings::prepare_evac);
1539 
1540       ShenandoahHeapLocker locker(lock());
1541       _collection_set->clear();
1542       _free_set->clear();
1543 
1544       heuristics()->choose_collection_set(_collection_set);
1545 
1546       _free_set->rebuild();
1547     }
1548 
1549     if (!is_degenerated_gc_in_progress()) {
1550       prepare_concurrent_roots();
1551       prepare_concurrent_unloading();
1552     }
1553 
1554     // If collection set has candidates, start evacuation.
1555     // Otherwise, bypass the rest of the cycle.
1556     if (!collection_set()->is_empty()) {
1557       ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1558 
1559       if (ShenandoahVerify) {
1560         verifier()->verify_before_evacuation();
1561       }
1562 
1563       set_evacuation_in_progress(true);
1564       // From here on, we need to update references.
1565       set_has_forwarded_objects(true);
1566 
1567       if (!is_degenerated_gc_in_progress()) {
1568         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1569           ShenandoahCodeRoots::arm_nmethods();
1570         }
1571         evacuate_and_update_roots();
1572       }
1573 
1574       if (ShenandoahPacing) {
1575         pacer()->setup_for_evac();
1576       }
1577 
1578       if (ShenandoahVerify) {
1579         ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::None;
1580         if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1581           types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots);
1582           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CLDGRoots);
1583         }
1584 
1585         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1586           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CodeRoots);
1587         }
1588         verifier()->verify_roots_no_forwarded_except(types);
1589         verifier()->verify_during_evacuation();
1590       }
1591     } else {
1592       if (ShenandoahVerify) {
1593         verifier()->verify_after_concmark();
1594       }
1595 
1596       if (VerifyAfterGC) {
1597         Universe::verify();
1598       }
1599     }
1600 
1601   } else {
1602     // If this cycle was updating references, we need to keep the has_forwarded_objects
1603     // flag on, for subsequent phases to deal with it.
1604     concurrent_mark()->cancel();
1605     set_concurrent_mark_in_progress(false);
1606 
1607     if (process_references()) {
1608       // Abandon reference processing right away: pre-cleaning must have failed.
1609       ReferenceProcessor *rp = ref_processor();
1610       rp->disable_discovery();
1611       rp->abandon_partial_discovery();
1612       rp->verify_no_references_recorded();
1613     }
1614   }
1615 }
1616 
1617 void ShenandoahHeap::op_final_evac() {
1618   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1619 
1620   set_evacuation_in_progress(false);
1621 
1622   {
1623     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac_retire_gclabs);
1624     retire_and_reset_gclabs();
1625   }
1626 
1627   if (ShenandoahVerify) {
1628     verifier()->verify_after_evacuation();
1629   }
1630 
1631   if (VerifyAfterGC) {
1632     Universe::verify();
1633   }
1634 }
1635 
1636 void ShenandoahHeap::op_conc_evac() {
1637   ShenandoahEvacuationTask task(this, _collection_set, true);
1638   workers()->run_task(&task);
1639 }
1640 
1641 void ShenandoahHeap::op_stw_evac() {
1642   ShenandoahEvacuationTask task(this, _collection_set, false);
1643   workers()->run_task(&task);
1644 }
1645 
1646 void ShenandoahHeap::op_updaterefs() {
1647   update_heap_references(true);
1648 }
1649 
1650 void ShenandoahHeap::op_cleanup() {
1651   free_set()->recycle_trash();
1652 }
1653 
1654 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
1655 private:
1656   ShenandoahVMRoots<true /*concurrent*/>        _vm_roots;
1657   ShenandoahWeakRoots<true /*concurrent*/>      _weak_roots;
1658   ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /*single threaded*/> _cld_roots;
1659 
1660 public:
1661   ShenandoahConcurrentRootsEvacUpdateTask() :
1662     AbstractGangTask("Shenandoah Evacuate/Update Concurrent Roots Task") {
1663   }
1664 
1665   void work(uint worker_id) {
1666     ShenandoahEvacOOMScope oom;
1667     {
1668       // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1669       // may race against OopStorage::release() calls.
1670       ShenandoahEvacUpdateOopStorageRootsClosure cl;
1671       _vm_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl);
1672       _weak_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl);
1673     }
1674 
1675     {
1676       ShenandoahEvacuateUpdateRootsClosure cl;
1677       CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1678       _cld_roots.cld_do(&clds);
1679     }
1680   }
1681 };
1682 
1683 void ShenandoahHeap::op_roots() {
1684   if (is_concurrent_root_in_progress()) {
1685     if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1686       _unloader.unload();
1687     }
1688 
1689     if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1690       ShenandoahConcurrentRootsEvacUpdateTask task;
1691       workers()->run_task(&task);
1692     }
1693   }
1694 
1695   set_concurrent_root_in_progress(false);
1696 }
1697 
1698 void ShenandoahHeap::op_reset() {
1699   reset_mark_bitmap();
1700 }
1701 
1702 void ShenandoahHeap::op_preclean() {
1703   concurrent_mark()->preclean_weak_refs();
1704 }
1705 
1706 void ShenandoahHeap::op_init_traversal() {
1707   traversal_gc()->init_traversal_collection();
1708 }
1709 
1710 void ShenandoahHeap::op_traversal() {
1711   traversal_gc()->concurrent_traversal_collection();
1712 }
1713 
1714 void ShenandoahHeap::op_final_traversal() {
1715   traversal_gc()->final_traversal_collection();
1716 }
1717 
1718 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1719   ShenandoahMetricsSnapshot metrics;
1720   metrics.snap_before();
1721 
1722   full_gc()->do_it(cause);
1723   if (UseTLAB) {
1724     ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
1725     resize_all_tlabs();
1726   }
1727 
1728   metrics.snap_after();
1729 
1730   if (metrics.is_good_progress()) {
1731     _progress_last_gc.set();
1732   } else {
1733     // Nothing to do. Tell the allocation path that we have failed to make
1734     // progress, and it can finally fail.
1735     _progress_last_gc.unset();
1736   }
1737 }
1738 
1739 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1740   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1741   // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1742   // some phase, we have to upgrade the Degenerate GC to Full GC.
1743 
1744   clear_cancelled_gc();
1745 
1746   ShenandoahMetricsSnapshot metrics;
1747   metrics.snap_before();
1748 
1749   switch (point) {
1750     case _degenerated_traversal:
1751       {
1752         // Drop the collection set. Note: this leaves some already forwarded objects
1753         // behind, which may be problematic, see comments for ShenandoahEvacAssist
1754         // workarounds in ShenandoahTraversalHeuristics.
1755 
1756         ShenandoahHeapLocker locker(lock());
1757         collection_set()->clear_current_index();
1758         for (size_t i = 0; i < collection_set()->count(); i++) {
1759           ShenandoahHeapRegion* r = collection_set()->next();
1760           r->make_regular_bypass();
1761         }
1762         collection_set()->clear();
1763       }
1764       op_final_traversal();
1765       op_cleanup();
1766       return;
1767 
1768     // The cases below form the Duff's-like device: it describes the actual GC cycle,
1769     // but enters it at different points, depending on which concurrent phase had
1770     // degenerated.
1771 
1772     case _degenerated_outside_cycle:
1773       // We have degenerated from outside the cycle, which means something is bad with
1774       // the heap, most probably heavy humongous fragmentation, or we are very low on free
1775       // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1776       // we can do the most aggressive degen cycle, which includes processing references and
1777       // class unloading, unless those features are explicitly disabled.
1778       //
1779       // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1780       // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1781       set_process_references(heuristics()->can_process_references());
1782       set_unload_classes(heuristics()->can_unload_classes());
1783 
1784       if (is_traversal_mode()) {
1785         // Not possible to degenerate from here, upgrade to Full GC right away.
1786         cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1787         op_degenerated_fail();
1788         return;
1789       }
1790 
1791       op_reset();
1792 
1793       op_init_mark();
1794       if (cancelled_gc()) {
1795         op_degenerated_fail();
1796         return;
1797       }
1798 
1799     case _degenerated_mark:
1800       op_final_mark();
1801       if (cancelled_gc()) {
1802         op_degenerated_fail();
1803         return;
1804       }
1805 
1806       op_cleanup();
1807 
1808     case _degenerated_evac:
1809       // If heuristics thinks we should do the cycle, this flag would be set,
1810       // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1811       if (is_evacuation_in_progress()) {
1812 
1813         // Degeneration under oom-evac protocol might have left some objects in
1814         // collection set un-evacuated. Restart evacuation from the beginning to
1815         // capture all objects. For all the objects that are already evacuated,
1816         // it would be a simple check, which is supposed to be fast. This is also
1817         // safe to do even without degeneration, as CSet iterator is at beginning
1818         // in preparation for evacuation anyway.
1819         //
1820         // Before doing that, we need to make sure we never had any cset-pinned
1821         // regions. This may happen if allocation failure happened when evacuating
1822         // the about-to-be-pinned object, oom-evac protocol left the object in
1823         // the collection set, and then the pin reached the cset region. If we continue
1824         // the cycle here, we would trash the cset and alive objects in it. To avoid
1825         // it, we fail degeneration right away and slide into Full GC to recover.
1826 
1827         {
1828           sync_pinned_region_status();
1829           collection_set()->clear_current_index();
1830 
1831           ShenandoahHeapRegion* r;
1832           while ((r = collection_set()->next()) != NULL) {
1833             if (r->is_pinned()) {
1834               cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1835               op_degenerated_fail();
1836               return;
1837             }
1838           }
1839 
1840           collection_set()->clear_current_index();
1841         }
1842 
1843         op_stw_evac();
1844         if (cancelled_gc()) {
1845           op_degenerated_fail();
1846           return;
1847         }
1848       }
1849 
1850       // If heuristics thinks we should do the cycle, this flag would be set,
1851       // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
1852       if (has_forwarded_objects()) {
1853         op_init_updaterefs();
1854         if (cancelled_gc()) {
1855           op_degenerated_fail();
1856           return;
1857         }
1858       }
1859 
1860     case _degenerated_updaterefs:
1861       if (has_forwarded_objects()) {
1862         op_final_updaterefs();
1863         if (cancelled_gc()) {
1864           op_degenerated_fail();
1865           return;
1866         }
1867       }
1868 
1869       op_cleanup();
1870       break;
1871 
1872     default:
1873       ShouldNotReachHere();
1874   }
1875 
1876   if (ShenandoahVerify) {
1877     verifier()->verify_after_degenerated();
1878   }
1879 
1880   if (VerifyAfterGC) {
1881     Universe::verify();
1882   }
1883 
1884   metrics.snap_after();
1885 
1886   // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
1887   // because that probably means the heap is overloaded and/or fragmented.
1888   if (!metrics.is_good_progress()) {
1889     _progress_last_gc.unset();
1890     cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1891     op_degenerated_futile();
1892   } else {
1893     _progress_last_gc.set();
1894   }
1895 }
1896 
1897 void ShenandoahHeap::op_degenerated_fail() {
1898   log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
1899   shenandoah_policy()->record_degenerated_upgrade_to_full();
1900   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
1901 }
1902 
1903 void ShenandoahHeap::op_degenerated_futile() {
1904   shenandoah_policy()->record_degenerated_upgrade_to_full();
1905   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
1906 }
1907 
1908 void ShenandoahHeap::force_satb_flush_all_threads() {
1909   if (!is_concurrent_mark_in_progress() && !is_concurrent_traversal_in_progress()) {
1910     // No need to flush SATBs
1911     return;
1912   }
1913 
1914   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1915     ShenandoahThreadLocalData::set_force_satb_flush(t, true);
1916   }
1917   // The threads are not "acquiring" their thread-local data, but it does not
1918   // hurt to "release" the updates here anyway.
1919   OrderAccess::fence();
1920 }
1921 
1922 void ShenandoahHeap::set_gc_state_all_threads(char state) {
1923   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1924     ShenandoahThreadLocalData::set_gc_state(t, state);
1925   }
1926 }
1927 
1928 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
1929   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
1930   _gc_state.set_cond(mask, value);
1931   set_gc_state_all_threads(_gc_state.raw_value());
1932 }
1933 
1934 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
1935   if (has_forwarded_objects()) {
1936     set_gc_state_mask(MARKING | UPDATEREFS, in_progress);
1937   } else {
1938     set_gc_state_mask(MARKING, in_progress);
1939   }
1940   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1941 }
1942 
1943 void ShenandoahHeap::set_concurrent_traversal_in_progress(bool in_progress) {
1944    set_gc_state_mask(TRAVERSAL, in_progress);
1945    ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1946 }
1947 
1948 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
1949   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
1950   set_gc_state_mask(EVACUATION, in_progress);
1951 }
1952 
1953 void ShenandoahHeap::set_concurrent_root_in_progress(bool in_progress) {
1954   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
1955   if (in_progress) {
1956     _concurrent_root_in_progress.set();
1957   } else {
1958     _concurrent_root_in_progress.unset();
1959   }
1960 }
1961 
1962 void ShenandoahHeap::ref_processing_init() {
1963   assert(_max_workers > 0, "Sanity");
1964 
1965   _ref_processor =
1966     new ReferenceProcessor(&_subject_to_discovery,  // is_subject_to_discovery
1967                            ParallelRefProcEnabled,  // MT processing
1968                            _max_workers,            // Degree of MT processing
1969                            true,                    // MT discovery
1970                            _max_workers,            // Degree of MT discovery
1971                            false,                   // Reference discovery is not atomic
1972                            NULL,                    // No closure, should be installed before use
1973                            true);                   // Scale worker threads
1974 
1975   shenandoah_assert_rp_isalive_not_installed();
1976 }
1977 
1978 GCTracer* ShenandoahHeap::tracer() {
1979   return shenandoah_policy()->tracer();
1980 }
1981 
1982 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
1983   return _free_set->used();
1984 }
1985 
1986 bool ShenandoahHeap::try_cancel_gc() {
1987   while (true) {
1988     jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
1989     if (prev == CANCELLABLE) return true;
1990     else if (prev == CANCELLED) return false;
1991     assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
1992     assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
1993     if (Thread::current()->is_Java_thread()) {
1994       // We need to provide a safepoint here, otherwise we might
1995       // spin forever if a SP is pending.
1996       ThreadBlockInVM sp(JavaThread::current());
1997       SpinPause();
1998     }
1999   }
2000 }
2001 
2002 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2003   if (try_cancel_gc()) {
2004     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2005     log_info(gc)("%s", msg.buffer());
2006     Events::log(Thread::current(), "%s", msg.buffer());
2007   }
2008 }
2009 
2010 uint ShenandoahHeap::max_workers() {
2011   return _max_workers;
2012 }
2013 
2014 void ShenandoahHeap::stop() {
2015   // The shutdown sequence should be able to terminate when GC is running.
2016 
2017   // Step 0. Notify policy to disable event recording.
2018   _shenandoah_policy->record_shutdown();
2019 
2020   // Step 1. Notify control thread that we are in shutdown.
2021   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2022   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2023   control_thread()->prepare_for_graceful_shutdown();
2024 
2025   // Step 2. Notify GC workers that we are cancelling GC.
2026   cancel_gc(GCCause::_shenandoah_stop_vm);
2027 
2028   // Step 3. Wait until GC worker exits normally.
2029   control_thread()->stop();
2030 
2031   // Step 4. Stop String Dedup thread if it is active
2032   if (ShenandoahStringDedup::is_enabled()) {
2033     ShenandoahStringDedup::stop();
2034   }
2035 }
2036 
2037 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
2038   if (!unload_classes()) return;
2039   bool purged_class;
2040 
2041   // Unload classes and purge SystemDictionary.
2042   {
2043     ShenandoahGCPhase phase(full_gc ?
2044                             ShenandoahPhaseTimings::full_gc_purge_class_unload :
2045                             ShenandoahPhaseTimings::purge_class_unload);
2046     purged_class = SystemDictionary::do_unloading(gc_timer());
2047   }
2048 
2049   {
2050     ShenandoahGCPhase phase(full_gc ?
2051                             ShenandoahPhaseTimings::full_gc_purge_par :
2052                             ShenandoahPhaseTimings::purge_par);
2053     ShenandoahIsAliveSelector is_alive;
2054     uint num_workers = _workers->active_workers();
2055     ShenandoahClassUnloadingTask unlink_task(is_alive.is_alive_closure(), num_workers, purged_class);
2056     _workers->run_task(&unlink_task);
2057   }
2058 
2059   {
2060     ShenandoahGCPhase phase(full_gc ?
2061                             ShenandoahPhaseTimings::full_gc_purge_cldg :
2062                             ShenandoahPhaseTimings::purge_cldg);
2063     ClassLoaderDataGraph::purge();
2064   }
2065   // Resize and verify metaspace
2066   MetaspaceGC::compute_new_size();
2067   MetaspaceUtils::verify_metrics();
2068 }
2069 
2070 // Weak roots are either pre-evacuated (final mark) or updated (final updaterefs),
2071 // so they should not have forwarded oops.
2072 // However, we do need to "null" dead oops in the roots, if can not be done
2073 // in concurrent cycles.
2074 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
2075   ShenandoahGCPhase root_phase(full_gc ?
2076                                ShenandoahPhaseTimings::full_gc_purge :
2077                                ShenandoahPhaseTimings::purge);
2078   uint num_workers = _workers->active_workers();
2079   ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
2080                                                ShenandoahPhaseTimings::full_gc_purge_par :
2081                                                ShenandoahPhaseTimings::purge_par;
2082   // Cleanup weak roots
2083   ShenandoahGCPhase phase(timing_phase);
2084   if (has_forwarded_objects()) {
2085     if (is_traversal_mode()) {
2086       ShenandoahForwardedIsAliveClosure is_alive;
2087       ShenandoahTraversalUpdateRefsClosure keep_alive;
2088       ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahTraversalUpdateRefsClosure>
2089         cleaning_task(&is_alive, &keep_alive, num_workers);
2090       _workers->run_task(&cleaning_task);
2091     } else {
2092       ShenandoahForwardedIsAliveClosure is_alive;
2093       ShenandoahUpdateRefsClosure keep_alive;
2094       ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
2095         cleaning_task(&is_alive, &keep_alive, num_workers);
2096       _workers->run_task(&cleaning_task);
2097     }
2098   } else {
2099     ShenandoahIsAliveClosure is_alive;
2100 #ifdef ASSERT
2101   ShenandoahAssertNotForwardedClosure verify_cl;
2102   ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
2103     cleaning_task(&is_alive, &verify_cl, num_workers);
2104 #else
2105   ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
2106     cleaning_task(&is_alive, &do_nothing_cl, num_workers);
2107 #endif
2108     _workers->run_task(&cleaning_task);
2109   }
2110 }
2111 
2112 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
2113   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2114   stw_process_weak_roots(full_gc);
2115   if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2116     stw_unload_classes(full_gc);
2117   }
2118 }
2119 
2120 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2121   if (is_traversal_mode()) {
2122     set_gc_state_mask(HAS_FORWARDED | UPDATEREFS, cond);
2123   } else {
2124     set_gc_state_mask(HAS_FORWARDED, cond);
2125   }
2126 
2127 }
2128 
2129 void ShenandoahHeap::set_process_references(bool pr) {
2130   _process_references.set_cond(pr);
2131 }
2132 
2133 void ShenandoahHeap::set_unload_classes(bool uc) {
2134   _unload_classes.set_cond(uc);
2135 }
2136 
2137 bool ShenandoahHeap::process_references() const {
2138   return _process_references.is_set();
2139 }
2140 
2141 bool ShenandoahHeap::unload_classes() const {
2142   return _unload_classes.is_set();
2143 }
2144 
2145 address ShenandoahHeap::in_cset_fast_test_addr() {
2146   ShenandoahHeap* heap = ShenandoahHeap::heap();
2147   assert(heap->collection_set() != NULL, "Sanity");
2148   return (address) heap->collection_set()->biased_map_address();
2149 }
2150 
2151 address ShenandoahHeap::cancelled_gc_addr() {
2152   return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
2153 }
2154 
2155 address ShenandoahHeap::gc_state_addr() {
2156   return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
2157 }
2158 
2159 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
2160   return Atomic::load_acquire(&_bytes_allocated_since_gc_start);
2161 }
2162 
2163 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2164   Atomic::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
2165 }
2166 
2167 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2168   _degenerated_gc_in_progress.set_cond(in_progress);
2169 }
2170 
2171 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2172   _full_gc_in_progress.set_cond(in_progress);
2173 }
2174 
2175 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2176   assert (is_full_gc_in_progress(), "should be");
2177   _full_gc_move_in_progress.set_cond(in_progress);
2178 }
2179 
2180 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2181   set_gc_state_mask(UPDATEREFS, in_progress);
2182 }
2183 
2184 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2185   ShenandoahCodeRoots::register_nmethod(nm);
2186 }
2187 
2188 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2189   ShenandoahCodeRoots::unregister_nmethod(nm);
2190 }
2191 
2192 void ShenandoahHeap::flush_nmethod(nmethod* nm) {
2193   ShenandoahCodeRoots::flush_nmethod(nm);
2194 }
2195 
2196 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2197   heap_region_containing(o)->record_pin();
2198   return o;
2199 }
2200 
2201 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2202   heap_region_containing(o)->record_unpin();
2203 }
2204 
2205 void ShenandoahHeap::sync_pinned_region_status() {
2206   ShenandoahHeapLocker locker(lock());
2207 
2208   for (size_t i = 0; i < num_regions(); i++) {
2209     ShenandoahHeapRegion *r = get_region(i);
2210     if (r->is_active()) {
2211       if (r->is_pinned()) {
2212         if (r->pin_count() == 0) {
2213           r->make_unpinned();
2214         }
2215       } else {
2216         if (r->pin_count() > 0) {
2217           r->make_pinned();
2218         }
2219       }
2220     }
2221   }
2222 
2223   assert_pinned_region_status();
2224 }
2225 
2226 #ifdef ASSERT
2227 void ShenandoahHeap::assert_pinned_region_status() {
2228   for (size_t i = 0; i < num_regions(); i++) {
2229     ShenandoahHeapRegion* r = get_region(i);
2230     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2231            "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2232   }
2233 }
2234 #endif
2235 
2236 GCTimer* ShenandoahHeap::gc_timer() const {
2237   return _gc_timer;
2238 }
2239 
2240 void ShenandoahHeap::prepare_concurrent_roots() {
2241   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2242   if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
2243     set_concurrent_root_in_progress(true);
2244   }
2245 }
2246 
2247 void ShenandoahHeap::prepare_concurrent_unloading() {
2248   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2249   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2250     _unloader.prepare();
2251   }
2252 }
2253 
2254 void ShenandoahHeap::finish_concurrent_unloading() {
2255   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2256   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2257     _unloader.finish();
2258   }
2259 }
2260 
2261 #ifdef ASSERT
2262 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2263   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2264 
2265   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2266     if (UseDynamicNumberOfGCThreads ||
2267         (FLAG_IS_DEFAULT(ParallelGCThreads) && ForceDynamicNumberOfGCThreads)) {
2268       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2269     } else {
2270       // Use ParallelGCThreads inside safepoints
2271       assert(nworkers == ParallelGCThreads, "Use ParalleGCThreads within safepoints");
2272     }
2273   } else {
2274     if (UseDynamicNumberOfGCThreads ||
2275         (FLAG_IS_DEFAULT(ConcGCThreads) && ForceDynamicNumberOfGCThreads)) {
2276       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2277     } else {
2278       // Use ConcGCThreads outside safepoints
2279       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2280     }
2281   }
2282 }
2283 #endif
2284 
2285 ShenandoahVerifier* ShenandoahHeap::verifier() {
2286   guarantee(ShenandoahVerify, "Should be enabled");
2287   assert (_verifier != NULL, "sanity");
2288   return _verifier;
2289 }
2290 
2291 template<class T>
2292 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2293 private:
2294   T cl;
2295   ShenandoahHeap* _heap;
2296   ShenandoahRegionIterator* _regions;
2297   bool _concurrent;
2298 public:
2299   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2300     AbstractGangTask("Concurrent Update References Task"),
2301     cl(T()),
2302     _heap(ShenandoahHeap::heap()),
2303     _regions(regions),
2304     _concurrent(concurrent) {
2305   }
2306 
2307   void work(uint worker_id) {
2308     if (_concurrent) {
2309       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2310       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2311       do_work();
2312     } else {
2313       ShenandoahParallelWorkerSession worker_session(worker_id);
2314       do_work();
2315     }
2316   }
2317 
2318 private:
2319   void do_work() {
2320     ShenandoahHeapRegion* r = _regions->next();
2321     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2322     while (r != NULL) {
2323       HeapWord* top_at_start_ur = r->concurrent_iteration_safe_limit();
2324       assert (top_at_start_ur >= r->bottom(), "sanity");
2325       if (r->is_active() && !r->is_cset()) {
2326         _heap->marked_object_oop_iterate(r, &cl, top_at_start_ur);
2327       }
2328       if (ShenandoahPacing) {
2329         _heap->pacer()->report_updaterefs(pointer_delta(top_at_start_ur, r->bottom()));
2330       }
2331       if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2332         return;
2333       }
2334       r = _regions->next();
2335     }
2336   }
2337 };
2338 
2339 void ShenandoahHeap::update_heap_references(bool concurrent) {
2340   ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2341   workers()->run_task(&task);
2342 }
2343 
2344 void ShenandoahHeap::op_init_updaterefs() {
2345   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2346 
2347   set_evacuation_in_progress(false);
2348 
2349   {
2350     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
2351     retire_and_reset_gclabs();
2352   }
2353 
2354   if (ShenandoahVerify) {
2355     if (!is_degenerated_gc_in_progress()) {
2356       verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::ThreadRoots);
2357     }
2358     verifier()->verify_before_updaterefs();
2359   }
2360 
2361   set_update_refs_in_progress(true);
2362 
2363   {
2364     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_prepare);
2365 
2366     make_parsable(true);
2367     for (uint i = 0; i < num_regions(); i++) {
2368       ShenandoahHeapRegion* r = get_region(i);
2369       r->set_concurrent_iteration_safe_limit(r->top());
2370     }
2371 
2372     // Reset iterator.
2373     _update_refs_iterator.reset();
2374   }
2375 
2376   if (ShenandoahPacing) {
2377     pacer()->setup_for_updaterefs();
2378   }
2379 }
2380 
2381 void ShenandoahHeap::op_final_updaterefs() {
2382   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2383 
2384   finish_concurrent_unloading();
2385 
2386   // Check if there is left-over work, and finish it
2387   if (_update_refs_iterator.has_next()) {
2388     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work);
2389 
2390     // Finish updating references where we left off.
2391     clear_cancelled_gc();
2392     update_heap_references(false);
2393   }
2394 
2395   // Clear cancelled GC, if set. On cancellation path, the block before would handle
2396   // everything. On degenerated paths, cancelled gc would not be set anyway.
2397   if (cancelled_gc()) {
2398     clear_cancelled_gc();
2399   }
2400   assert(!cancelled_gc(), "Should have been done right before");
2401 
2402   if (ShenandoahVerify && !is_degenerated_gc_in_progress()) {
2403     verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2404   }
2405 
2406   if (is_degenerated_gc_in_progress()) {
2407     concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
2408   } else {
2409     concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots);
2410   }
2411 
2412   // Has to be done before cset is clear
2413   if (ShenandoahVerify) {
2414     verifier()->verify_roots_in_to_space();
2415   }
2416 
2417   // Drop unnecessary "pinned" state from regions that does not have CP marks
2418   // anymore, as this would allow trashing them below.
2419   {
2420     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_sync_pinned);
2421     sync_pinned_region_status();
2422   }
2423 
2424   {
2425     ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
2426     trash_cset_regions();
2427   }
2428 
2429   set_has_forwarded_objects(false);
2430   set_update_refs_in_progress(false);
2431 
2432   if (ShenandoahVerify) {
2433     verifier()->verify_after_updaterefs();
2434   }
2435 
2436   if (VerifyAfterGC) {
2437     Universe::verify();
2438   }
2439 
2440   {
2441     ShenandoahHeapLocker locker(lock());
2442     _free_set->rebuild();
2443   }
2444 }
2445 
2446 #ifdef ASSERT
2447 void ShenandoahHeap::assert_heaplock_owned_by_current_thread() {
2448   _lock.assert_owned_by_current_thread();
2449 }
2450 
2451 void ShenandoahHeap::assert_heaplock_not_owned_by_current_thread() {
2452   _lock.assert_not_owned_by_current_thread();
2453 }
2454 
2455 void ShenandoahHeap::assert_heaplock_or_safepoint() {
2456   _lock.assert_owned_by_current_thread_or_safepoint();
2457 }
2458 #endif
2459 
2460 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2461   print_on(st);
2462   print_heap_regions_on(st);
2463 }
2464 
2465 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2466   size_t slice = r->region_number() / _bitmap_regions_per_slice;
2467 
2468   size_t regions_from = _bitmap_regions_per_slice * slice;
2469   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2470   for (size_t g = regions_from; g < regions_to; g++) {
2471     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2472     if (skip_self && g == r->region_number()) continue;
2473     if (get_region(g)->is_committed()) {
2474       return true;
2475     }
2476   }
2477   return false;
2478 }
2479 
2480 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2481   assert_heaplock_owned_by_current_thread();
2482 
2483   // Bitmaps in special regions do not need commits
2484   if (_bitmap_region_special) {
2485     return true;
2486   }
2487 
2488   if (is_bitmap_slice_committed(r, true)) {
2489     // Some other region from the group is already committed, meaning the bitmap
2490     // slice is already committed, we exit right away.
2491     return true;
2492   }
2493 
2494   // Commit the bitmap slice:
2495   size_t slice = r->region_number() / _bitmap_regions_per_slice;
2496   size_t off = _bitmap_bytes_per_slice * slice;
2497   size_t len = _bitmap_bytes_per_slice;
2498   if (!os::commit_memory((char*)_bitmap_region.start() + off, len, false)) {
2499     return false;
2500   }
2501   return true;
2502 }
2503 
2504 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2505   assert_heaplock_owned_by_current_thread();
2506 
2507   // Bitmaps in special regions do not need uncommits
2508   if (_bitmap_region_special) {
2509     return true;
2510   }
2511 
2512   if (is_bitmap_slice_committed(r, true)) {
2513     // Some other region from the group is still committed, meaning the bitmap
2514     // slice is should stay committed, exit right away.
2515     return true;
2516   }
2517 
2518   // Uncommit the bitmap slice:
2519   size_t slice = r->region_number() / _bitmap_regions_per_slice;
2520   size_t off = _bitmap_bytes_per_slice * slice;
2521   size_t len = _bitmap_bytes_per_slice;
2522   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2523     return false;
2524   }
2525   return true;
2526 }
2527 
2528 void ShenandoahHeap::safepoint_synchronize_begin() {
2529   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2530     SuspendibleThreadSet::synchronize();
2531   }
2532 }
2533 
2534 void ShenandoahHeap::safepoint_synchronize_end() {
2535   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2536     SuspendibleThreadSet::desynchronize();
2537   }
2538 }
2539 
2540 void ShenandoahHeap::vmop_entry_init_mark() {
2541   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2542   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2543   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark_gross);
2544 
2545   try_inject_alloc_failure();
2546   VM_ShenandoahInitMark op;
2547   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2548 }
2549 
2550 void ShenandoahHeap::vmop_entry_final_mark() {
2551   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2552   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2553   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark_gross);
2554 
2555   try_inject_alloc_failure();
2556   VM_ShenandoahFinalMarkStartEvac op;
2557   VMThread::execute(&op); // jump to entry_final_mark under safepoint
2558 }
2559 
2560 void ShenandoahHeap::vmop_entry_final_evac() {
2561   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2562   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2563   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac_gross);
2564 
2565   VM_ShenandoahFinalEvac op;
2566   VMThread::execute(&op); // jump to entry_final_evac under safepoint
2567 }
2568 
2569 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2570   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2571   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2572   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_gross);
2573 
2574   try_inject_alloc_failure();
2575   VM_ShenandoahInitUpdateRefs op;
2576   VMThread::execute(&op);
2577 }
2578 
2579 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2580   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2581   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2582   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_gross);
2583 
2584   try_inject_alloc_failure();
2585   VM_ShenandoahFinalUpdateRefs op;
2586   VMThread::execute(&op);
2587 }
2588 
2589 void ShenandoahHeap::vmop_entry_init_traversal() {
2590   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2591   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2592   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc_gross);
2593 
2594   try_inject_alloc_failure();
2595   VM_ShenandoahInitTraversalGC op;
2596   VMThread::execute(&op);
2597 }
2598 
2599 void ShenandoahHeap::vmop_entry_final_traversal() {
2600   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2601   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2602   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc_gross);
2603 
2604   try_inject_alloc_failure();
2605   VM_ShenandoahFinalTraversalGC op;
2606   VMThread::execute(&op);
2607 }
2608 
2609 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2610   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2611   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2612   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_gross);
2613 
2614   try_inject_alloc_failure();
2615   VM_ShenandoahFullGC op(cause);
2616   VMThread::execute(&op);
2617 }
2618 
2619 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2620   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2621   ShenandoahGCPhase total(ShenandoahPhaseTimings::total_pause_gross);
2622   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_gross);
2623 
2624   VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2625   VMThread::execute(&degenerated_gc);
2626 }
2627 
2628 void ShenandoahHeap::entry_init_mark() {
2629   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2630   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark);
2631   const char* msg = init_mark_event_message();
2632   GCTraceTime(Info, gc) time(msg, gc_timer());
2633   EventMark em("%s", msg);
2634 
2635   ShenandoahWorkerScope scope(workers(),
2636                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2637                               "init marking");
2638 
2639   op_init_mark();
2640 }
2641 
2642 void ShenandoahHeap::entry_final_mark() {
2643   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2644   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark);
2645   const char* msg = final_mark_event_message();
2646   GCTraceTime(Info, gc) time(msg, gc_timer());
2647   EventMark em("%s", msg);
2648 
2649   ShenandoahWorkerScope scope(workers(),
2650                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2651                               "final marking");
2652 
2653   op_final_mark();
2654 }
2655 
2656 void ShenandoahHeap::entry_final_evac() {
2657   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2658   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac);
2659   static const char* msg = "Pause Final Evac";
2660   GCTraceTime(Info, gc) time(msg, gc_timer());
2661   EventMark em("%s", msg);
2662 
2663   op_final_evac();
2664 }
2665 
2666 void ShenandoahHeap::entry_init_updaterefs() {
2667   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2668   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs);
2669 
2670   static const char* msg = "Pause Init Update Refs";
2671   GCTraceTime(Info, gc) time(msg, gc_timer());
2672   EventMark em("%s", msg);
2673 
2674   // No workers used in this phase, no setup required
2675 
2676   op_init_updaterefs();
2677 }
2678 
2679 void ShenandoahHeap::entry_final_updaterefs() {
2680   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2681   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs);
2682 
2683   static const char* msg = "Pause Final Update Refs";
2684   GCTraceTime(Info, gc) time(msg, gc_timer());
2685   EventMark em("%s", msg);
2686 
2687   ShenandoahWorkerScope scope(workers(),
2688                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2689                               "final reference update");
2690 
2691   op_final_updaterefs();
2692 }
2693 
2694 void ShenandoahHeap::entry_init_traversal() {
2695   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2696   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_traversal_gc);
2697 
2698   static const char* msg = "Pause Init Traversal";
2699   GCTraceTime(Info, gc) time(msg, gc_timer());
2700   EventMark em("%s", msg);
2701 
2702   ShenandoahWorkerScope scope(workers(),
2703                               ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(),
2704                               "init traversal");
2705 
2706   op_init_traversal();
2707 }
2708 
2709 void ShenandoahHeap::entry_final_traversal() {
2710   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2711   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_traversal_gc);
2712 
2713   static const char* msg = "Pause Final Traversal";
2714   GCTraceTime(Info, gc) time(msg, gc_timer());
2715   EventMark em("%s", msg);
2716 
2717   ShenandoahWorkerScope scope(workers(),
2718                               ShenandoahWorkerPolicy::calc_workers_for_stw_traversal(),
2719                               "final traversal");
2720 
2721   op_final_traversal();
2722 }
2723 
2724 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2725   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2726   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc);
2727 
2728   static const char* msg = "Pause Full";
2729   GCTraceTime(Info, gc) time(msg, gc_timer(), cause, true);
2730   EventMark em("%s", msg);
2731 
2732   ShenandoahWorkerScope scope(workers(),
2733                               ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2734                               "full gc");
2735 
2736   op_full(cause);
2737 }
2738 
2739 void ShenandoahHeap::entry_degenerated(int point) {
2740   ShenandoahGCPhase total_phase(ShenandoahPhaseTimings::total_pause);
2741   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc);
2742 
2743   ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2744   const char* msg = degen_event_message(dpoint);
2745   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2746   EventMark em("%s", msg);
2747 
2748   ShenandoahWorkerScope scope(workers(),
2749                               ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2750                               "stw degenerated gc");
2751 
2752   set_degenerated_gc_in_progress(true);
2753   op_degenerated(dpoint);
2754   set_degenerated_gc_in_progress(false);
2755 }
2756 
2757 void ShenandoahHeap::entry_mark() {
2758   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2759 
2760   const char* msg = conc_mark_event_message();
2761   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2762   EventMark em("%s", msg);
2763 
2764   ShenandoahWorkerScope scope(workers(),
2765                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2766                               "concurrent marking");
2767 
2768   try_inject_alloc_failure();
2769   op_mark();
2770 }
2771 
2772 void ShenandoahHeap::entry_evac() {
2773   ShenandoahGCPhase conc_evac_phase(ShenandoahPhaseTimings::conc_evac);
2774   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2775 
2776   static const char* msg = "Concurrent evacuation";
2777   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2778   EventMark em("%s", msg);
2779 
2780   ShenandoahWorkerScope scope(workers(),
2781                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2782                               "concurrent evacuation");
2783 
2784   try_inject_alloc_failure();
2785   op_conc_evac();
2786 }
2787 
2788 void ShenandoahHeap::entry_updaterefs() {
2789   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2790 
2791   static const char* msg = "Concurrent update references";
2792   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2793   EventMark em("%s", msg);
2794 
2795   ShenandoahWorkerScope scope(workers(),
2796                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2797                               "concurrent reference update");
2798 
2799   try_inject_alloc_failure();
2800   op_updaterefs();
2801 }
2802 
2803 void ShenandoahHeap::entry_roots() {
2804   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_roots);
2805 
2806   static const char* msg = "Concurrent roots processing";
2807   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2808   EventMark em("%s", msg);
2809 
2810   ShenandoahWorkerScope scope(workers(),
2811                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2812                               "concurrent root processing");
2813 
2814   try_inject_alloc_failure();
2815   op_roots();
2816 }
2817 
2818 void ShenandoahHeap::entry_cleanup() {
2819   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
2820 
2821   static const char* msg = "Concurrent cleanup";
2822   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2823   EventMark em("%s", msg);
2824 
2825   // This phase does not use workers, no need for setup
2826 
2827   try_inject_alloc_failure();
2828   op_cleanup();
2829 }
2830 
2831 void ShenandoahHeap::entry_reset() {
2832   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_reset);
2833 
2834   static const char* msg = "Concurrent reset";
2835   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2836   EventMark em("%s", msg);
2837 
2838   ShenandoahWorkerScope scope(workers(),
2839                               ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2840                               "concurrent reset");
2841 
2842   try_inject_alloc_failure();
2843   op_reset();
2844 }
2845 
2846 void ShenandoahHeap::entry_preclean() {
2847   if (ShenandoahPreclean && process_references()) {
2848     static const char* msg = "Concurrent precleaning";
2849     GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2850     EventMark em("%s", msg);
2851 
2852     ShenandoahGCPhase conc_preclean(ShenandoahPhaseTimings::conc_preclean);
2853 
2854     ShenandoahWorkerScope scope(workers(),
2855                                 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2856                                 "concurrent preclean",
2857                                 /* check_workers = */ false);
2858 
2859     try_inject_alloc_failure();
2860     op_preclean();
2861   }
2862 }
2863 
2864 void ShenandoahHeap::entry_traversal() {
2865   static const char* msg = "Concurrent traversal";
2866   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2867   EventMark em("%s", msg);
2868 
2869   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2870 
2871   ShenandoahWorkerScope scope(workers(),
2872                               ShenandoahWorkerPolicy::calc_workers_for_conc_traversal(),
2873                               "concurrent traversal");
2874 
2875   try_inject_alloc_failure();
2876   op_traversal();
2877 }
2878 
2879 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2880   static const char *msg = "Concurrent uncommit";
2881   GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2882   EventMark em("%s", msg);
2883 
2884   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_uncommit);
2885 
2886   op_uncommit(shrink_before);
2887 }
2888 
2889 void ShenandoahHeap::try_inject_alloc_failure() {
2890   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2891     _inject_alloc_failure.set();
2892     os::naked_short_sleep(1);
2893     if (cancelled_gc()) {
2894       log_info(gc)("Allocation failure was successfully injected");
2895     }
2896   }
2897 }
2898 
2899 bool ShenandoahHeap::should_inject_alloc_failure() {
2900   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2901 }
2902 
2903 void ShenandoahHeap::initialize_serviceability() {
2904   _memory_pool = new ShenandoahMemoryPool(this);
2905   _cycle_memory_manager.add_pool(_memory_pool);
2906   _stw_memory_manager.add_pool(_memory_pool);
2907 }
2908 
2909 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2910   GrowableArray<GCMemoryManager*> memory_managers(2);
2911   memory_managers.append(&_cycle_memory_manager);
2912   memory_managers.append(&_stw_memory_manager);
2913   return memory_managers;
2914 }
2915 
2916 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2917   GrowableArray<MemoryPool*> memory_pools(1);
2918   memory_pools.append(_memory_pool);
2919   return memory_pools;
2920 }
2921 
2922 MemoryUsage ShenandoahHeap::memory_usage() {
2923   return _memory_pool->get_memory_usage();
2924 }
2925 
2926 void ShenandoahHeap::enter_evacuation() {
2927   _oom_evac_handler.enter_evacuation();
2928 }
2929 
2930 void ShenandoahHeap::leave_evacuation() {
2931   _oom_evac_handler.leave_evacuation();
2932 }
2933 
2934 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2935   _heap(ShenandoahHeap::heap()),
2936   _index(0) {}
2937 
2938 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2939   _heap(heap),
2940   _index(0) {}
2941 
2942 void ShenandoahRegionIterator::reset() {
2943   _index = 0;
2944 }
2945 
2946 bool ShenandoahRegionIterator::has_next() const {
2947   return _index < _heap->num_regions();
2948 }
2949 
2950 char ShenandoahHeap::gc_state() const {
2951   return _gc_state.raw_value();
2952 }
2953 
2954 void ShenandoahHeap::deduplicate_string(oop str) {
2955   assert(java_lang_String::is_instance(str), "invariant");
2956 
2957   if (ShenandoahStringDedup::is_enabled()) {
2958     ShenandoahStringDedup::deduplicate(str);
2959   }
2960 }
2961 
2962 const char* ShenandoahHeap::init_mark_event_message() const {
2963   bool update_refs = has_forwarded_objects();
2964   bool proc_refs = process_references();
2965   bool unload_cls = unload_classes();
2966 
2967   if (update_refs && proc_refs && unload_cls) {
2968     return "Pause Init Mark (update refs) (process weakrefs) (unload classes)";
2969   } else if (update_refs && proc_refs) {
2970     return "Pause Init Mark (update refs) (process weakrefs)";
2971   } else if (update_refs && unload_cls) {
2972     return "Pause Init Mark (update refs) (unload classes)";
2973   } else if (proc_refs && unload_cls) {
2974     return "Pause Init Mark (process weakrefs) (unload classes)";
2975   } else if (update_refs) {
2976     return "Pause Init Mark (update refs)";
2977   } else if (proc_refs) {
2978     return "Pause Init Mark (process weakrefs)";
2979   } else if (unload_cls) {
2980     return "Pause Init Mark (unload classes)";
2981   } else {
2982     return "Pause Init Mark";
2983   }
2984 }
2985 
2986 const char* ShenandoahHeap::final_mark_event_message() const {
2987   bool update_refs = has_forwarded_objects();
2988   bool proc_refs = process_references();
2989   bool unload_cls = unload_classes();
2990 
2991   if (update_refs && proc_refs && unload_cls) {
2992     return "Pause Final Mark (update refs) (process weakrefs) (unload classes)";
2993   } else if (update_refs && proc_refs) {
2994     return "Pause Final Mark (update refs) (process weakrefs)";
2995   } else if (update_refs && unload_cls) {
2996     return "Pause Final Mark (update refs) (unload classes)";
2997   } else if (proc_refs && unload_cls) {
2998     return "Pause Final Mark (process weakrefs) (unload classes)";
2999   } else if (update_refs) {
3000     return "Pause Final Mark (update refs)";
3001   } else if (proc_refs) {
3002     return "Pause Final Mark (process weakrefs)";
3003   } else if (unload_cls) {
3004     return "Pause Final Mark (unload classes)";
3005   } else {
3006     return "Pause Final Mark";
3007   }
3008 }
3009 
3010 const char* ShenandoahHeap::conc_mark_event_message() const {
3011   bool update_refs = has_forwarded_objects();
3012   bool proc_refs = process_references();
3013   bool unload_cls = unload_classes();
3014 
3015   if (update_refs && proc_refs && unload_cls) {
3016     return "Concurrent marking (update refs) (process weakrefs) (unload classes)";
3017   } else if (update_refs && proc_refs) {
3018     return "Concurrent marking (update refs) (process weakrefs)";
3019   } else if (update_refs && unload_cls) {
3020     return "Concurrent marking (update refs) (unload classes)";
3021   } else if (proc_refs && unload_cls) {
3022     return "Concurrent marking (process weakrefs) (unload classes)";
3023   } else if (update_refs) {
3024     return "Concurrent marking (update refs)";
3025   } else if (proc_refs) {
3026     return "Concurrent marking (process weakrefs)";
3027   } else if (unload_cls) {
3028     return "Concurrent marking (unload classes)";
3029   } else {
3030     return "Concurrent marking";
3031   }
3032 }
3033 
3034 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
3035   switch (point) {
3036     case _degenerated_unset:
3037       return "Pause Degenerated GC (<UNSET>)";
3038     case _degenerated_traversal:
3039       return "Pause Degenerated GC (Traversal)";
3040     case _degenerated_outside_cycle:
3041       return "Pause Degenerated GC (Outside of Cycle)";
3042     case _degenerated_mark:
3043       return "Pause Degenerated GC (Mark)";
3044     case _degenerated_evac:
3045       return "Pause Degenerated GC (Evacuation)";
3046     case _degenerated_updaterefs:
3047       return "Pause Degenerated GC (Update Refs)";
3048     default:
3049       ShouldNotReachHere();
3050       return "ERROR";
3051   }
3052 }
3053 
3054 jushort* ShenandoahHeap::get_liveness_cache(uint worker_id) {
3055 #ifdef ASSERT
3056   assert(_liveness_cache != NULL, "sanity");
3057   assert(worker_id < _max_workers, "sanity");
3058   for (uint i = 0; i < num_regions(); i++) {
3059     assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
3060   }
3061 #endif
3062   return _liveness_cache[worker_id];
3063 }
3064 
3065 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
3066   assert(worker_id < _max_workers, "sanity");
3067   assert(_liveness_cache != NULL, "sanity");
3068   jushort* ld = _liveness_cache[worker_id];
3069   for (uint i = 0; i < num_regions(); i++) {
3070     ShenandoahHeapRegion* r = get_region(i);
3071     jushort live = ld[i];
3072     if (live > 0) {
3073       r->increase_live_data_gc_words(live);
3074       ld[i] = 0;
3075     }
3076   }
3077 }