1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "memory/allocation.hpp"
  27 #include "memory/universe.hpp"
  28 
  29 #include "gc/shared/gcArguments.hpp"
  30 #include "gc/shared/gcTimer.hpp"
  31 #include "gc/shared/gcTraceTime.inline.hpp"
  32 #include "gc/shared/locationPrinter.inline.hpp"
  33 #include "gc/shared/memAllocator.hpp"
  34 #include "gc/shared/oopStorageSet.hpp"
  35 #include "gc/shared/plab.hpp"
  36 
  37 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  38 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  39 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  40 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  41 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  42 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
  43 #include "gc/shenandoah/shenandoahControlThread.hpp"
  44 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  45 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  46 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  47 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  48 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  49 #include "gc/shenandoah/shenandoahIUMode.hpp"
  50 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  51 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  52 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  53 #include "gc/shenandoah/shenandoahMetrics.hpp"
  54 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  55 #include "gc/shenandoah/shenandoahNormalMode.hpp"
  56 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  57 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  58 #include "gc/shenandoah/shenandoahPadding.hpp"
  59 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  60 #include "gc/shenandoah/shenandoahPassiveMode.hpp"
  61 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  62 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  63 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
  64 #include "gc/shenandoah/shenandoahUtils.hpp"
  65 #include "gc/shenandoah/shenandoahVerifier.hpp"
  66 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  67 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  68 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  69 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  70 #if INCLUDE_JFR
  71 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
  72 #endif
  73 
  74 #include "memory/metaspace.hpp"
  75 #include "oops/compressedOops.inline.hpp"
  76 #include "runtime/atomic.hpp"
  77 #include "runtime/globals.hpp"
  78 #include "runtime/interfaceSupport.inline.hpp"
  79 #include "runtime/orderAccess.hpp"
  80 #include "runtime/safepointMechanism.hpp"
  81 #include "runtime/vmThread.hpp"
  82 #include "services/mallocTracker.hpp"
  83 #include "utilities/powerOfTwo.hpp"
  84 
  85 ShenandoahHeap* ShenandoahHeap::_heap = NULL;
  86 
  87 #ifdef ASSERT
  88 template <class T>
  89 void ShenandoahAssertToSpaceClosure::do_oop_work(T* p) {
  90   T o = RawAccess<>::oop_load(p);
  91   if (! CompressedOops::is_null(o)) {
  92     oop obj = CompressedOops::decode_not_null(o);
  93     shenandoah_assert_not_forwarded(p, obj);
  94   }
  95 }
  96 
  97 void ShenandoahAssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_work(p); }
  98 void ShenandoahAssertToSpaceClosure::do_oop(oop* p)       { do_oop_work(p); }
  99 #endif
 100 
 101 class ShenandoahPretouchHeapTask : public AbstractGangTask {
 102 private:
 103   ShenandoahRegionIterator _regions;
 104   const size_t _page_size;
 105 public:
 106   ShenandoahPretouchHeapTask(size_t page_size) :
 107     AbstractGangTask("Shenandoah Pretouch Heap"),
 108     _page_size(page_size) {}
 109 
 110   virtual void work(uint worker_id) {
 111     ShenandoahHeapRegion* r = _regions.next();
 112     while (r != NULL) {
 113       os::pretouch_memory(r->bottom(), r->end(), _page_size);
 114       r = _regions.next();
 115     }
 116   }
 117 };
 118 
 119 class ShenandoahPretouchBitmapTask : public AbstractGangTask {
 120 private:
 121   ShenandoahRegionIterator _regions;
 122   char* _bitmap_base;
 123   const size_t _bitmap_size;
 124   const size_t _page_size;
 125 public:
 126   ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
 127     AbstractGangTask("Shenandoah Pretouch Bitmap"),
 128     _bitmap_base(bitmap_base),
 129     _bitmap_size(bitmap_size),
 130     _page_size(page_size) {}
 131 
 132   virtual void work(uint worker_id) {
 133     ShenandoahHeapRegion* r = _regions.next();
 134     while (r != NULL) {
 135       size_t start = r->index()       * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 136       size_t end   = (r->index() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 137       assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
 138 
 139       os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
 140 
 141       r = _regions.next();
 142     }
 143   }
 144 };
 145 
 146 jint ShenandoahHeap::initialize() {
 147   //
 148   // Figure out heap sizing
 149   //
 150 
 151   size_t init_byte_size = InitialHeapSize;
 152   size_t min_byte_size  = MinHeapSize;
 153   size_t max_byte_size  = MaxHeapSize;
 154   size_t heap_alignment = HeapAlignment;
 155 
 156   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 157 
 158   if (ShenandoahAlwaysPreTouch) {
 159     // Enabled pre-touch means the entire heap is committed right away.
 160     init_byte_size = max_byte_size;
 161   }
 162 
 163   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 164   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 165 
 166   _num_regions = ShenandoahHeapRegion::region_count();
 167 
 168   // Now we know the number of regions, initialize the heuristics.
 169   initialize_heuristics();
 170 
 171   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 172   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 173   assert(num_committed_regions <= _num_regions, "sanity");
 174   _initial_size = num_committed_regions * reg_size_bytes;
 175 
 176   size_t num_min_regions = min_byte_size / reg_size_bytes;
 177   num_min_regions = MIN2(num_min_regions, _num_regions);
 178   assert(num_min_regions <= _num_regions, "sanity");
 179   _minimum_size = num_min_regions * reg_size_bytes;
 180 
 181   _committed = _initial_size;
 182 
 183   size_t heap_page_size   = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 184   size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 185   size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
 186 
 187   //
 188   // Reserve and commit memory for heap
 189   //
 190 
 191   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 192   initialize_reserved_region(heap_rs);
 193   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 194   _heap_region_special = heap_rs.special();
 195 
 196   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 197          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 198 
 199 #if SHENANDOAH_OPTIMIZED_OBJTASK
 200   // The optimized ObjArrayChunkedTask takes some bits away from the full object bits.
 201   // Fail if we ever attempt to address more than we can.
 202   if ((uintptr_t)heap_rs.end() >= ObjArrayChunkedTask::max_addressable()) {
 203     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 204                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 205                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 206                 p2i(heap_rs.base()), p2i(heap_rs.end()), ObjArrayChunkedTask::max_addressable());
 207     vm_exit_during_initialization("Fatal Error", buf);
 208   }
 209 #endif
 210 
 211   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 212   if (!_heap_region_special) {
 213     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 214                               "Cannot commit heap memory");
 215   }
 216 
 217   //
 218   // Reserve and commit memory for bitmap(s)
 219   //
 220 
 221   _bitmap_size = MarkBitMap::compute_size(heap_rs.size());
 222   _bitmap_size = align_up(_bitmap_size, bitmap_page_size);
 223 
 224   size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
 225 
 226   guarantee(bitmap_bytes_per_region != 0,
 227             "Bitmap bytes per region should not be zero");
 228   guarantee(is_power_of_2(bitmap_bytes_per_region),
 229             "Bitmap bytes per region should be power of two: " SIZE_FORMAT, bitmap_bytes_per_region);
 230 
 231   if (bitmap_page_size > bitmap_bytes_per_region) {
 232     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 233     _bitmap_bytes_per_slice = bitmap_page_size;
 234   } else {
 235     _bitmap_regions_per_slice = 1;
 236     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 237   }
 238 
 239   guarantee(_bitmap_regions_per_slice >= 1,
 240             "Should have at least one region per slice: " SIZE_FORMAT,
 241             _bitmap_regions_per_slice);
 242 
 243   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 244             "Bitmap slices should be page-granular: bps = " SIZE_FORMAT ", page size = " SIZE_FORMAT,
 245             _bitmap_bytes_per_slice, bitmap_page_size);
 246 
 247   ReservedSpace bitmap(_bitmap_size, bitmap_page_size);
 248   MemTracker::record_virtual_memory_type(bitmap.base(), mtGC);
 249   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 250   _bitmap_region_special = bitmap.special();
 251 
 252   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 253                               align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 254   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 255   if (!_bitmap_region_special) {
 256     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 257                               "Cannot commit bitmap memory");
 258   }
 259 
 260   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 261 
 262   if (ShenandoahVerify) {
 263     ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size);
 264     if (!verify_bitmap.special()) {
 265       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 266                                 "Cannot commit verification bitmap memory");
 267     }
 268     MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
 269     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 270     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 271     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 272   }
 273 
 274   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 275   ReservedSpace aux_bitmap(_bitmap_size, bitmap_page_size);
 276   MemTracker::record_virtual_memory_type(aux_bitmap.base(), mtGC);
 277   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 278   _aux_bitmap_region_special = aux_bitmap.special();
 279   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 280 
 281   //
 282   // Create regions and region sets
 283   //
 284   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 285   size_t region_storage_size = align_up(region_align * _num_regions, region_page_size);
 286   region_storage_size = align_up(region_storage_size, os::vm_allocation_granularity());
 287 
 288   ReservedSpace region_storage(region_storage_size, region_page_size);
 289   MemTracker::record_virtual_memory_type(region_storage.base(), mtGC);
 290   if (!region_storage.special()) {
 291     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 292                               "Cannot commit region memory");
 293   }
 294 
 295   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 296   _free_set = new ShenandoahFreeSet(this, _num_regions);
 297   _collection_set = new ShenandoahCollectionSet(this, sh_rs.base(), sh_rs.size());
 298 
 299   {
 300     ShenandoahHeapLocker locker(lock());
 301 
 302     for (size_t i = 0; i < _num_regions; i++) {
 303       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 304       bool is_committed = i < num_committed_regions;
 305       void* loc = region_storage.base() + i * region_align;
 306 
 307       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 308       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 309 
 310       _marking_context->initialize_top_at_mark_start(r);
 311       _regions[i] = r;
 312       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 313     }
 314 
 315     // Initialize to complete
 316     _marking_context->mark_complete();
 317 
 318     _free_set->rebuild();
 319   }
 320 
 321   if (ShenandoahAlwaysPreTouch) {
 322     assert(!AlwaysPreTouch, "Should have been overridden");
 323 
 324     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 325     // before initialize() below zeroes it with initializing thread. For any given region,
 326     // we touch the region and the corresponding bitmaps from the same thread.
 327     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 328 
 329     size_t pretouch_heap_page_size = heap_page_size;
 330     size_t pretouch_bitmap_page_size = bitmap_page_size;
 331 
 332 #ifdef LINUX
 333     // UseTransparentHugePages would madvise that backing memory can be coalesced into huge
 334     // pages. But, the kernel needs to know that every small page is used, in order to coalesce
 335     // them into huge one. Therefore, we need to pretouch with smaller pages.
 336     if (UseTransparentHugePages) {
 337       pretouch_heap_page_size = (size_t)os::vm_page_size();
 338       pretouch_bitmap_page_size = (size_t)os::vm_page_size();
 339     }
 340 #endif
 341 
 342     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 343     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 344 
 345     log_info(gc, init)("Pretouch bitmap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 346                        _num_regions, pretouch_bitmap_page_size);
 347     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, pretouch_bitmap_page_size);
 348     _workers->run_task(&bcl);
 349 
 350     log_info(gc, init)("Pretouch heap: " SIZE_FORMAT " regions, " SIZE_FORMAT " bytes page",
 351                        _num_regions, pretouch_heap_page_size);
 352     ShenandoahPretouchHeapTask hcl(pretouch_heap_page_size);
 353     _workers->run_task(&hcl);
 354   }
 355 
 356   //
 357   // Initialize the rest of GC subsystems
 358   //
 359 
 360   _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
 361   for (uint worker = 0; worker < _max_workers; worker++) {
 362     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 363     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 364   }
 365 
 366   // There should probably be Shenandoah-specific options for these,
 367   // just as there are G1-specific options.
 368   {
 369     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 370     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 371     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 372   }
 373 
 374   _monitoring_support = new ShenandoahMonitoringSupport(this);
 375   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 376   ShenandoahStringDedup::initialize();
 377   ShenandoahCodeRoots::initialize();
 378 
 379   if (ShenandoahPacing) {
 380     _pacer = new ShenandoahPacer(this);
 381     _pacer->setup_for_idle();
 382   } else {
 383     _pacer = NULL;
 384   }
 385 
 386   _control_thread = new ShenandoahControlThread();
 387 
 388   log_info(gc, init)("Initialize Shenandoah heap: " SIZE_FORMAT "%s initial, " SIZE_FORMAT "%s min, " SIZE_FORMAT "%s max",
 389                      byte_size_in_proper_unit(_initial_size),  proper_unit_for_byte_size(_initial_size),
 390                      byte_size_in_proper_unit(_minimum_size),  proper_unit_for_byte_size(_minimum_size),
 391                      byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity())
 392   );
 393 
 394   log_info(gc, init)("Safepointing mechanism: thread-local poll");
 395 
 396   return JNI_OK;
 397 }
 398 
 399 void ShenandoahHeap::initialize_heuristics() {
 400   if (ShenandoahGCMode != NULL) {
 401     if (strcmp(ShenandoahGCMode, "normal") == 0) {
 402       _gc_mode = new ShenandoahNormalMode();
 403     } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
 404       _gc_mode = new ShenandoahIUMode();
 405     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 406       _gc_mode = new ShenandoahPassiveMode();
 407     } else {
 408       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 409     }
 410   } else {
 411     ShouldNotReachHere();
 412   }
 413   _gc_mode->initialize_flags();
 414   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 415     vm_exit_during_initialization(
 416             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 417                     _gc_mode->name()));
 418   }
 419   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 420     vm_exit_during_initialization(
 421             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 422                     _gc_mode->name()));
 423   }
 424   log_info(gc, init)("Shenandoah GC mode: %s",
 425                      _gc_mode->name());
 426 
 427   _heuristics = _gc_mode->initialize_heuristics();
 428 
 429   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 430     vm_exit_during_initialization(
 431             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 432                     _heuristics->name()));
 433   }
 434   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 435     vm_exit_during_initialization(
 436             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 437                     _heuristics->name()));
 438   }
 439   log_info(gc, init)("Shenandoah heuristics: %s",
 440                      _heuristics->name());
 441 }
 442 
 443 #ifdef _MSC_VER
 444 #pragma warning( push )
 445 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 446 #endif
 447 
 448 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 449   CollectedHeap(),
 450   _initial_size(0),
 451   _used(0),
 452   _committed(0),
 453   _bytes_allocated_since_gc_start(0),
 454   _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
 455   _workers(NULL),
 456   _safepoint_workers(NULL),
 457   _heap_region_special(false),
 458   _num_regions(0),
 459   _regions(NULL),
 460   _update_refs_iterator(this),
 461   _control_thread(NULL),
 462   _shenandoah_policy(policy),
 463   _heuristics(NULL),
 464   _free_set(NULL),
 465   _scm(new ShenandoahConcurrentMark()),
 466   _full_gc(new ShenandoahMarkCompact()),
 467   _pacer(NULL),
 468   _verifier(NULL),
 469   _phase_timings(NULL),
 470   _monitoring_support(NULL),
 471   _memory_pool(NULL),
 472   _stw_memory_manager("Shenandoah Pauses", "end of GC pause"),
 473   _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"),
 474   _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
 475   _soft_ref_policy(),
 476   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 477   _ref_processor(NULL),
 478   _marking_context(NULL),
 479   _bitmap_size(0),
 480   _bitmap_regions_per_slice(0),
 481   _bitmap_bytes_per_slice(0),
 482   _bitmap_region_special(false),
 483   _aux_bitmap_region_special(false),
 484   _liveness_cache(NULL),
 485   _collection_set(NULL)
 486 {
 487   _heap = this;
 488 
 489   log_info(gc, init)("GC threads: " UINT32_FORMAT " parallel, " UINT32_FORMAT " concurrent", ParallelGCThreads, ConcGCThreads);
 490 
 491   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this));
 492 
 493   _max_workers = MAX2(_max_workers, 1U);
 494   _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
 495                             /* are_GC_task_threads */ true,
 496                             /* are_ConcurrentGC_threads */ true);
 497   if (_workers == NULL) {
 498     vm_exit_during_initialization("Failed necessary allocation.");
 499   } else {
 500     _workers->initialize_workers();
 501   }
 502 
 503   if (ParallelGCThreads > 1) {
 504     _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread",
 505                                                 ParallelGCThreads,
 506                       /* are_GC_task_threads */ false,
 507                  /* are_ConcurrentGC_threads */ false);
 508     _safepoint_workers->initialize_workers();
 509   }
 510 }
 511 
 512 #ifdef _MSC_VER
 513 #pragma warning( pop )
 514 #endif
 515 
 516 class ShenandoahResetBitmapTask : public AbstractGangTask {
 517 private:
 518   ShenandoahRegionIterator _regions;
 519 
 520 public:
 521   ShenandoahResetBitmapTask() :
 522     AbstractGangTask("Parallel Reset Bitmap Task") {}
 523 
 524   void work(uint worker_id) {
 525     ShenandoahHeapRegion* region = _regions.next();
 526     ShenandoahHeap* heap = ShenandoahHeap::heap();
 527     ShenandoahMarkingContext* const ctx = heap->marking_context();
 528     while (region != NULL) {
 529       if (heap->is_bitmap_slice_committed(region)) {
 530         ctx->clear_bitmap(region);
 531       }
 532       region = _regions.next();
 533     }
 534   }
 535 };
 536 
 537 void ShenandoahHeap::reset_mark_bitmap() {
 538   assert_gc_workers(_workers->active_workers());
 539   mark_incomplete_marking_context();
 540 
 541   ShenandoahResetBitmapTask task;
 542   _workers->run_task(&task);
 543 }
 544 
 545 void ShenandoahHeap::print_on(outputStream* st) const {
 546   st->print_cr("Shenandoah Heap");
 547   st->print_cr(" " SIZE_FORMAT "%s total, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used",
 548                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 549                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 550                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 551   st->print_cr(" " SIZE_FORMAT " x " SIZE_FORMAT"%s regions",
 552                num_regions(),
 553                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 554                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 555 
 556   st->print("Status: ");
 557   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 558   if (is_concurrent_mark_in_progress())        st->print("marking, ");
 559   if (is_evacuation_in_progress())             st->print("evacuating, ");
 560   if (is_update_refs_in_progress())            st->print("updating refs, ");
 561   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 562   if (is_full_gc_in_progress())                st->print("full gc, ");
 563   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 564   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 565   if (is_concurrent_strong_root_in_progress() &&
 566       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 567 
 568   if (cancelled_gc()) {
 569     st->print("cancelled");
 570   } else {
 571     st->print("not cancelled");
 572   }
 573   st->cr();
 574 
 575   st->print_cr("Reserved region:");
 576   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 577                p2i(reserved_region().start()),
 578                p2i(reserved_region().end()));
 579 
 580   ShenandoahCollectionSet* cset = collection_set();
 581   st->print_cr("Collection set:");
 582   if (cset != NULL) {
 583     st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
 584     st->print_cr(" - map (biased):  " PTR_FORMAT, p2i(cset->biased_map_address()));
 585   } else {
 586     st->print_cr(" (NULL)");
 587   }
 588 
 589   st->cr();
 590   MetaspaceUtils::print_on(st);
 591 
 592   if (Verbose) {
 593     print_heap_regions_on(st);
 594   }
 595 }
 596 
 597 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 598 public:
 599   void do_thread(Thread* thread) {
 600     assert(thread != NULL, "Sanity");
 601     assert(thread->is_Worker_thread(), "Only worker thread expected");
 602     ShenandoahThreadLocalData::initialize_gclab(thread);
 603   }
 604 };
 605 
 606 void ShenandoahHeap::post_initialize() {
 607   CollectedHeap::post_initialize();
 608   MutexLocker ml(Threads_lock);
 609 
 610   ShenandoahInitWorkerGCLABClosure init_gclabs;
 611   _workers->threads_do(&init_gclabs);
 612 
 613   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 614   // Now, we will let WorkGang to initialize gclab when new worker is created.
 615   _workers->set_initialize_gclab();
 616 
 617   _scm->initialize(_max_workers);
 618   _full_gc->initialize(_gc_timer);
 619 
 620   ref_processing_init();
 621 
 622   _heuristics->initialize();
 623 
 624   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
 625 }
 626 
 627 size_t ShenandoahHeap::used() const {
 628   return Atomic::load_acquire(&_used);
 629 }
 630 
 631 size_t ShenandoahHeap::committed() const {
 632   OrderAccess::acquire();
 633   return _committed;
 634 }
 635 
 636 void ShenandoahHeap::increase_committed(size_t bytes) {
 637   shenandoah_assert_heaplocked_or_safepoint();
 638   _committed += bytes;
 639 }
 640 
 641 void ShenandoahHeap::decrease_committed(size_t bytes) {
 642   shenandoah_assert_heaplocked_or_safepoint();
 643   _committed -= bytes;
 644 }
 645 
 646 void ShenandoahHeap::increase_used(size_t bytes) {
 647   Atomic::add(&_used, bytes);
 648 }
 649 
 650 void ShenandoahHeap::set_used(size_t bytes) {
 651   Atomic::release_store_fence(&_used, bytes);
 652 }
 653 
 654 void ShenandoahHeap::decrease_used(size_t bytes) {
 655   assert(used() >= bytes, "never decrease heap size by more than we've left");
 656   Atomic::sub(&_used, bytes);
 657 }
 658 
 659 void ShenandoahHeap::increase_allocated(size_t bytes) {
 660   Atomic::add(&_bytes_allocated_since_gc_start, bytes);
 661 }
 662 
 663 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
 664   size_t bytes = words * HeapWordSize;
 665   if (!waste) {
 666     increase_used(bytes);
 667   }
 668   increase_allocated(bytes);
 669   if (ShenandoahPacing) {
 670     control_thread()->pacing_notify_alloc(words);
 671     if (waste) {
 672       pacer()->claim_for_alloc(words, true);
 673     }
 674   }
 675 }
 676 
 677 size_t ShenandoahHeap::capacity() const {
 678   return committed();
 679 }
 680 
 681 size_t ShenandoahHeap::max_capacity() const {
 682   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 683 }
 684 
 685 size_t ShenandoahHeap::min_capacity() const {
 686   return _minimum_size;
 687 }
 688 
 689 size_t ShenandoahHeap::initial_capacity() const {
 690   return _initial_size;
 691 }
 692 
 693 bool ShenandoahHeap::is_in(const void* p) const {
 694   HeapWord* heap_base = (HeapWord*) base();
 695   HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions();
 696   return p >= heap_base && p < last_region_end;
 697 }
 698 
 699 void ShenandoahHeap::op_uncommit(double shrink_before) {
 700   assert (ShenandoahUncommit, "should be enabled");
 701 
 702   // Application allocates from the beginning of the heap, and GC allocates at
 703   // the end of it. It is more efficient to uncommit from the end, so that applications
 704   // could enjoy the near committed regions. GC allocations are much less frequent,
 705   // and therefore can accept the committing costs.
 706 
 707   size_t count = 0;
 708   for (size_t i = num_regions(); i > 0; i--) { // care about size_t underflow
 709     ShenandoahHeapRegion* r = get_region(i - 1);
 710     if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
 711       ShenandoahHeapLocker locker(lock());
 712       if (r->is_empty_committed()) {
 713         // Do not uncommit below minimal capacity
 714         if (committed() < min_capacity() + ShenandoahHeapRegion::region_size_bytes()) {
 715           break;
 716         }
 717 
 718         r->make_uncommitted();
 719         count++;
 720       }
 721     }
 722     SpinPause(); // allow allocators to take the lock
 723   }
 724 
 725   if (count > 0) {
 726     control_thread()->notify_heap_changed();
 727   }
 728 }
 729 
 730 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 731   // New object should fit the GCLAB size
 732   size_t min_size = MAX2(size, PLAB::min_size());
 733 
 734   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 735   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 736   new_size = MIN2(new_size, PLAB::max_size());
 737   new_size = MAX2(new_size, PLAB::min_size());
 738 
 739   // Record new heuristic value even if we take any shortcut. This captures
 740   // the case when moderately-sized objects always take a shortcut. At some point,
 741   // heuristics should catch up with them.
 742   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 743 
 744   if (new_size < size) {
 745     // New size still does not fit the object. Fall back to shared allocation.
 746     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 747     return NULL;
 748   }
 749 
 750   // Retire current GCLAB, and allocate a new one.
 751   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 752   gclab->retire();
 753 
 754   size_t actual_size = 0;
 755   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 756   if (gclab_buf == NULL) {
 757     return NULL;
 758   }
 759 
 760   assert (size <= actual_size, "allocation should fit");
 761 
 762   if (ZeroTLAB) {
 763     // ..and clear it.
 764     Copy::zero_to_words(gclab_buf, actual_size);
 765   } else {
 766     // ...and zap just allocated object.
 767 #ifdef ASSERT
 768     // Skip mangling the space corresponding to the object header to
 769     // ensure that the returned space is not considered parsable by
 770     // any concurrent GC thread.
 771     size_t hdr_size = oopDesc::header_size();
 772     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 773 #endif // ASSERT
 774   }
 775   gclab->set_buf(gclab_buf, actual_size);
 776   return gclab->allocate(size);
 777 }
 778 
 779 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 780                                             size_t requested_size,
 781                                             size_t* actual_size) {
 782   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 783   HeapWord* res = allocate_memory(req);
 784   if (res != NULL) {
 785     *actual_size = req.actual_size();
 786   } else {
 787     *actual_size = 0;
 788   }
 789   return res;
 790 }
 791 
 792 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 793                                              size_t word_size,
 794                                              size_t* actual_size) {
 795   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 796   HeapWord* res = allocate_memory(req);
 797   if (res != NULL) {
 798     *actual_size = req.actual_size();
 799   } else {
 800     *actual_size = 0;
 801   }
 802   return res;
 803 }
 804 
 805 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 806   intptr_t pacer_epoch = 0;
 807   bool in_new_region = false;
 808   HeapWord* result = NULL;
 809 
 810   if (req.is_mutator_alloc()) {
 811     if (ShenandoahPacing) {
 812       pacer()->pace_for_alloc(req.size());
 813       pacer_epoch = pacer()->epoch();
 814     }
 815 
 816     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 817       result = allocate_memory_under_lock(req, in_new_region);
 818     }
 819 
 820     // Allocation failed, block until control thread reacted, then retry allocation.
 821     //
 822     // It might happen that one of the threads requesting allocation would unblock
 823     // way later after GC happened, only to fail the second allocation, because
 824     // other threads have already depleted the free storage. In this case, a better
 825     // strategy is to try again, as long as GC makes progress.
 826     //
 827     // Then, we need to make sure the allocation was retried after at least one
 828     // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
 829 
 830     size_t tries = 0;
 831 
 832     while (result == NULL && _progress_last_gc.is_set()) {
 833       tries++;
 834       control_thread()->handle_alloc_failure(req);
 835       result = allocate_memory_under_lock(req, in_new_region);
 836     }
 837 
 838     while (result == NULL && tries <= ShenandoahFullGCThreshold) {
 839       tries++;
 840       control_thread()->handle_alloc_failure(req);
 841       result = allocate_memory_under_lock(req, in_new_region);
 842     }
 843 
 844   } else {
 845     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
 846     result = allocate_memory_under_lock(req, in_new_region);
 847     // Do not call handle_alloc_failure() here, because we cannot block.
 848     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
 849   }
 850 
 851   if (in_new_region) {
 852     control_thread()->notify_heap_changed();
 853   }
 854 
 855   if (result != NULL) {
 856     size_t requested = req.size();
 857     size_t actual = req.actual_size();
 858 
 859     assert (req.is_lab_alloc() || (requested == actual),
 860             "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT,
 861             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
 862 
 863     if (req.is_mutator_alloc()) {
 864       notify_mutator_alloc_words(actual, false);
 865 
 866       // If we requested more than we were granted, give the rest back to pacer.
 867       // This only matters if we are in the same pacing epoch: do not try to unpace
 868       // over the budget for the other phase.
 869       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
 870         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
 871       }
 872     } else {
 873       increase_used(actual*HeapWordSize);
 874     }
 875   }
 876 
 877   return result;
 878 }
 879 
 880 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
 881   ShenandoahHeapLocker locker(lock());
 882   return _free_set->allocate(req, in_new_region);
 883 }
 884 
 885 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
 886                                         bool*  gc_overhead_limit_was_exceeded) {
 887   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
 888   return allocate_memory(req);
 889 }
 890 
 891 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 892                                                              size_t size,
 893                                                              Metaspace::MetadataType mdtype) {
 894   MetaWord* result;
 895 
 896   // Inform metaspace OOM to GC heuristics if class unloading is possible.
 897   if (heuristics()->can_unload_classes()) {
 898     ShenandoahHeuristics* h = heuristics();
 899     h->record_metaspace_oom();
 900   }
 901 
 902   // Expand and retry allocation
 903   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 904   if (result != NULL) {
 905     return result;
 906   }
 907 
 908   // Start full GC
 909   collect(GCCause::_metadata_GC_clear_soft_refs);
 910 
 911   // Retry allocation
 912   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
 913   if (result != NULL) {
 914     return result;
 915   }
 916 
 917   // Expand and retry allocation
 918   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
 919   if (result != NULL) {
 920     return result;
 921   }
 922 
 923   // Out of memory
 924   return NULL;
 925 }
 926 
 927 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
 928 private:
 929   ShenandoahHeap* const _heap;
 930   Thread* const _thread;
 931 public:
 932   ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
 933     _heap(heap), _thread(Thread::current()) {}
 934 
 935   void do_object(oop p) {
 936     shenandoah_assert_marked(NULL, p);
 937     if (!p->is_forwarded()) {
 938       _heap->evacuate_object(p, _thread);
 939     }
 940   }
 941 };
 942 
 943 class ShenandoahEvacuationTask : public AbstractGangTask {
 944 private:
 945   ShenandoahHeap* const _sh;
 946   ShenandoahCollectionSet* const _cs;
 947   bool _concurrent;
 948 public:
 949   ShenandoahEvacuationTask(ShenandoahHeap* sh,
 950                            ShenandoahCollectionSet* cs,
 951                            bool concurrent) :
 952     AbstractGangTask("Parallel Evacuation Task"),
 953     _sh(sh),
 954     _cs(cs),
 955     _concurrent(concurrent)
 956   {}
 957 
 958   void work(uint worker_id) {
 959     if (_concurrent) {
 960       ShenandoahConcurrentWorkerSession worker_session(worker_id);
 961       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
 962       ShenandoahEvacOOMScope oom_evac_scope;
 963       do_work();
 964     } else {
 965       ShenandoahParallelWorkerSession worker_session(worker_id);
 966       ShenandoahEvacOOMScope oom_evac_scope;
 967       do_work();
 968     }
 969   }
 970 
 971 private:
 972   void do_work() {
 973     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
 974     ShenandoahHeapRegion* r;
 975     while ((r =_cs->claim_next()) != NULL) {
 976       assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index());
 977       _sh->marked_object_iterate(r, &cl);
 978 
 979       if (ShenandoahPacing) {
 980         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
 981       }
 982 
 983       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
 984         break;
 985       }
 986     }
 987   }
 988 };
 989 
 990 void ShenandoahHeap::trash_cset_regions() {
 991   ShenandoahHeapLocker locker(lock());
 992 
 993   ShenandoahCollectionSet* set = collection_set();
 994   ShenandoahHeapRegion* r;
 995   set->clear_current_index();
 996   while ((r = set->next()) != NULL) {
 997     r->make_trash();
 998   }
 999   collection_set()->clear();
1000 }
1001 
1002 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1003   st->print_cr("Heap Regions:");
1004   st->print_cr("EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HC=humongous continuation, CS=collection set, T=trash, P=pinned");
1005   st->print_cr("BTE=bottom/top/end, U=used, T=TLAB allocs, G=GCLAB allocs, S=shared allocs, L=live data");
1006   st->print_cr("R=root, CP=critical pins, TAMS=top-at-mark-start, UWM=update watermark");
1007   st->print_cr("SN=alloc sequence number");
1008 
1009   for (size_t i = 0; i < num_regions(); i++) {
1010     get_region(i)->print_on(st);
1011   }
1012 }
1013 
1014 void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1015   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1016 
1017   oop humongous_obj = oop(start->bottom());
1018   size_t size = humongous_obj->size();
1019   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1020   size_t index = start->index() + required_regions - 1;
1021 
1022   assert(!start->has_live(), "liveness must be zero");
1023 
1024   for(size_t i = 0; i < required_regions; i++) {
1025     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1026     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1027     ShenandoahHeapRegion* region = get_region(index --);
1028 
1029     assert(region->is_humongous(), "expect correct humongous start or continuation");
1030     assert(!region->is_cset(), "Humongous region should not be in collection set");
1031 
1032     region->make_trash_immediate();
1033   }
1034 }
1035 
1036 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1037 public:
1038   void do_thread(Thread* thread) {
1039     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1040     assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name());
1041     gclab->retire();
1042   }
1043 };
1044 
1045 void ShenandoahHeap::make_parsable(bool retire_tlabs) {
1046   if (UseTLAB) {
1047     CollectedHeap::ensure_parsability(retire_tlabs);
1048   }
1049   ShenandoahRetireGCLABClosure cl;
1050   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1051     cl.do_thread(t);
1052   }
1053   workers()->threads_do(&cl);
1054 }
1055 
1056 void ShenandoahHeap::resize_tlabs() {
1057   CollectedHeap::resize_all_tlabs();
1058 }
1059 
1060 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1061 private:
1062   ShenandoahRootEvacuator* _rp;
1063 
1064 public:
1065   ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1066     AbstractGangTask("Shenandoah evacuate and update roots"),
1067     _rp(rp) {}
1068 
1069   void work(uint worker_id) {
1070     ShenandoahParallelWorkerSession worker_session(worker_id);
1071     ShenandoahEvacOOMScope oom_evac_scope;
1072     ShenandoahEvacuateUpdateRootsClosure<> cl;
1073     MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1074     _rp->roots_do(worker_id, &cl);
1075   }
1076 };
1077 
1078 void ShenandoahHeap::evacuate_and_update_roots() {
1079 #if COMPILER2_OR_JVMCI
1080   DerivedPointerTable::clear();
1081 #endif
1082   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1083   {
1084     // Include concurrent roots if current cycle can not process those roots concurrently
1085     ShenandoahRootEvacuator rp(workers()->active_workers(),
1086                                ShenandoahPhaseTimings::init_evac,
1087                                !ShenandoahConcurrentRoots::should_do_concurrent_roots(),
1088                                !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
1089     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1090     workers()->run_task(&roots_task);
1091   }
1092 
1093 #if COMPILER2_OR_JVMCI
1094   DerivedPointerTable::update_pointers();
1095 #endif
1096 }
1097 
1098 // Returns size in bytes
1099 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1100   if (ShenandoahElasticTLAB) {
1101     // With Elastic TLABs, return the max allowed size, and let the allocation path
1102     // figure out the safe size for current allocation.
1103     return ShenandoahHeapRegion::max_tlab_size_bytes();
1104   } else {
1105     return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1106   }
1107 }
1108 
1109 size_t ShenandoahHeap::max_tlab_size() const {
1110   // Returns size in words
1111   return ShenandoahHeapRegion::max_tlab_size_words();
1112 }
1113 
1114 class ShenandoahRetireAndResetGCLABClosure : public ThreadClosure {
1115 public:
1116   void do_thread(Thread* thread) {
1117     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1118     gclab->retire();
1119     if (ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1120       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1121     }
1122   }
1123 };
1124 
1125 void ShenandoahHeap::retire_and_reset_gclabs() {
1126   ShenandoahRetireAndResetGCLABClosure cl;
1127   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1128     cl.do_thread(t);
1129   }
1130   workers()->threads_do(&cl);
1131 }
1132 
1133 void ShenandoahHeap::collect(GCCause::Cause cause) {
1134   control_thread()->request_gc(cause);
1135 }
1136 
1137 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1138   //assert(false, "Shouldn't need to do full collections");
1139 }
1140 
1141 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1142   ShenandoahHeapRegion* r = heap_region_containing(addr);
1143   if (r != NULL) {
1144     return r->block_start(addr);
1145   }
1146   return NULL;
1147 }
1148 
1149 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1150   ShenandoahHeapRegion* r = heap_region_containing(addr);
1151   return r->block_is_obj(addr);
1152 }
1153 
1154 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1155   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1156 }
1157 
1158 jlong ShenandoahHeap::millis_since_last_gc() {
1159   double v = heuristics()->time_since_last_gc() * 1000;
1160   assert(0 <= v && v <= max_jlong, "value should fit: %f", v);
1161   return (jlong)v;
1162 }
1163 
1164 void ShenandoahHeap::prepare_for_verify() {
1165   if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1166     make_parsable(false);
1167   }
1168 }
1169 
1170 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1171   workers()->print_worker_threads_on(st);
1172   if (ShenandoahStringDedup::is_enabled()) {
1173     ShenandoahStringDedup::print_worker_threads_on(st);
1174   }
1175 }
1176 
1177 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1178   workers()->threads_do(tcl);
1179   if (_safepoint_workers != NULL) {
1180     _safepoint_workers->threads_do(tcl);
1181   }
1182   if (ShenandoahStringDedup::is_enabled()) {
1183     ShenandoahStringDedup::threads_do(tcl);
1184   }
1185 }
1186 
1187 void ShenandoahHeap::print_tracing_info() const {
1188   LogTarget(Info, gc, stats) lt;
1189   if (lt.is_enabled()) {
1190     ResourceMark rm;
1191     LogStream ls(lt);
1192 
1193     phase_timings()->print_global_on(&ls);
1194 
1195     ls.cr();
1196     ls.cr();
1197 
1198     shenandoah_policy()->print_gc_stats(&ls);
1199 
1200     ls.cr();
1201     ls.cr();
1202 
1203     if (ShenandoahPacing) {
1204       pacer()->print_on(&ls);
1205     }
1206 
1207     ls.cr();
1208     ls.cr();
1209   }
1210 }
1211 
1212 void ShenandoahHeap::verify(VerifyOption vo) {
1213   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1214     if (ShenandoahVerify) {
1215       verifier()->verify_generic(vo);
1216     } else {
1217       // TODO: Consider allocating verification bitmaps on demand,
1218       // and turn this on unconditionally.
1219     }
1220   }
1221 }
1222 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1223   return _free_set->capacity();
1224 }
1225 
1226 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1227 private:
1228   MarkBitMap* _bitmap;
1229   Stack<oop,mtGC>* _oop_stack;
1230   ShenandoahHeap* const _heap;
1231   ShenandoahMarkingContext* const _marking_context;
1232 
1233   template <class T>
1234   void do_oop_work(T* p) {
1235     T o = RawAccess<>::oop_load(p);
1236     if (!CompressedOops::is_null(o)) {
1237       oop obj = CompressedOops::decode_not_null(o);
1238       if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1239         // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1240         return;
1241       }
1242       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1243 
1244       assert(oopDesc::is_oop(obj), "must be a valid oop");
1245       if (!_bitmap->is_marked(obj)) {
1246         _bitmap->mark(obj);
1247         _oop_stack->push(obj);
1248       }
1249     }
1250   }
1251 public:
1252   ObjectIterateScanRootClosure(MarkBitMap* bitmap, Stack<oop,mtGC>* oop_stack) :
1253     _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1254     _marking_context(_heap->marking_context()) {}
1255   void do_oop(oop* p)       { do_oop_work(p); }
1256   void do_oop(narrowOop* p) { do_oop_work(p); }
1257 };
1258 
1259 /*
1260  * This is public API, used in preparation of object_iterate().
1261  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1262  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1263  * control, we call SH::make_tlabs_parsable().
1264  */
1265 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1266   // No-op.
1267 }
1268 
1269 /*
1270  * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1271  *
1272  * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1273  * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1274  * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1275  * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1276  * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1277  * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1278  * wiped the bitmap in preparation for next marking).
1279  *
1280  * For all those reasons, we implement object iteration as a single marking traversal, reporting
1281  * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1282  * is allowed to report dead objects, but is not required to do so.
1283  */
1284 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1285   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1286   if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1287     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1288     return;
1289   }
1290 
1291   // Reset bitmap
1292   _aux_bit_map.clear();
1293 
1294   Stack<oop,mtGC> oop_stack;
1295 
1296   // First, we process GC roots according to current GC cycle. This populates the work stack with initial objects.
1297   ShenandoahHeapIterationRootScanner rp;
1298   ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1299 
1300   rp.roots_do(&oops);
1301 
1302   // Work through the oop stack to traverse heap.
1303   while (! oop_stack.is_empty()) {
1304     oop obj = oop_stack.pop();
1305     assert(oopDesc::is_oop(obj), "must be a valid oop");
1306     cl->do_object(obj);
1307     obj->oop_iterate(&oops);
1308   }
1309 
1310   assert(oop_stack.is_empty(), "should be empty");
1311 
1312   if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1313     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1314   }
1315 }
1316 
1317 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1318 void ShenandoahHeap::keep_alive(oop obj) {
1319   if (is_concurrent_mark_in_progress()) {
1320     ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1321   }
1322 }
1323 
1324 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1325   for (size_t i = 0; i < num_regions(); i++) {
1326     ShenandoahHeapRegion* current = get_region(i);
1327     blk->heap_region_do(current);
1328   }
1329 }
1330 
1331 class ShenandoahParallelHeapRegionTask : public AbstractGangTask {
1332 private:
1333   ShenandoahHeap* const _heap;
1334   ShenandoahHeapRegionClosure* const _blk;
1335 
1336   shenandoah_padding(0);
1337   volatile size_t _index;
1338   shenandoah_padding(1);
1339 
1340 public:
1341   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk) :
1342           AbstractGangTask("Parallel Region Task"),
1343           _heap(ShenandoahHeap::heap()), _blk(blk), _index(0) {}
1344 
1345   void work(uint worker_id) {
1346     size_t stride = ShenandoahParallelRegionStride;
1347 
1348     size_t max = _heap->num_regions();
1349     while (_index < max) {
1350       size_t cur = Atomic::fetch_and_add(&_index, stride);
1351       size_t start = cur;
1352       size_t end = MIN2(cur + stride, max);
1353       if (start >= max) break;
1354 
1355       for (size_t i = cur; i < end; i++) {
1356         ShenandoahHeapRegion* current = _heap->get_region(i);
1357         _blk->heap_region_do(current);
1358       }
1359     }
1360   }
1361 };
1362 
1363 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1364   assert(blk->is_thread_safe(), "Only thread-safe closures here");
1365   if (num_regions() > ShenandoahParallelRegionStride) {
1366     ShenandoahParallelHeapRegionTask task(blk);
1367     workers()->run_task(&task);
1368   } else {
1369     heap_region_iterate(blk);
1370   }
1371 }
1372 
1373 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1374 private:
1375   ShenandoahMarkingContext* const _ctx;
1376 public:
1377   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1378 
1379   void heap_region_do(ShenandoahHeapRegion* r) {
1380     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1381     if (r->is_active()) {
1382       // Check if region needs updating its TAMS. We have updated it already during concurrent
1383       // reset, so it is very likely we don't need to do another write here.
1384       if (_ctx->top_at_mark_start(r) != r->top()) {
1385         _ctx->capture_top_at_mark_start(r);
1386       }
1387     } else {
1388       assert(_ctx->top_at_mark_start(r) == r->top(),
1389              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
1390     }
1391   }
1392 
1393   bool is_thread_safe() { return true; }
1394 };
1395 
1396 void ShenandoahHeap::op_init_mark() {
1397   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1398   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
1399 
1400   assert(marking_context()->is_bitmap_clear(), "need clear marking bitmap");
1401   assert(!marking_context()->is_complete(), "should not be complete");
1402   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1403 
1404   if (ShenandoahVerify) {
1405     verifier()->verify_before_concmark();
1406   }
1407 
1408   if (VerifyBeforeGC) {
1409     Universe::verify();
1410   }
1411 
1412   set_concurrent_mark_in_progress(true);
1413   // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1414   {
1415     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::make_parsable);
1416     make_parsable(true);
1417   }
1418 
1419   {
1420     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::init_update_region_states);
1421     ShenandoahInitMarkUpdateRegionStateClosure cl;
1422     parallel_heap_region_iterate(&cl);
1423   }
1424 
1425   // Make above changes visible to worker threads
1426   OrderAccess::fence();
1427 
1428   concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots);
1429 
1430   if (UseTLAB) {
1431     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::resize_tlabs);
1432     resize_tlabs();
1433   }
1434 
1435   if (ShenandoahPacing) {
1436     pacer()->setup_for_mark();
1437   }
1438 
1439   // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
1440   // we need to make sure that all its metadata are marked. alternative is to remark
1441   // thread roots at final mark pause, but it can be potential latency killer.
1442   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1443     ShenandoahCodeRoots::arm_nmethods();
1444   }
1445 }
1446 
1447 void ShenandoahHeap::op_mark() {
1448   concurrent_mark()->mark_from_roots();
1449 }
1450 
1451 class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1452 private:
1453   ShenandoahMarkingContext* const _ctx;
1454   ShenandoahHeapLock* const _lock;
1455 
1456 public:
1457   ShenandoahFinalMarkUpdateRegionStateClosure() :
1458     _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {}
1459 
1460   void heap_region_do(ShenandoahHeapRegion* r) {
1461     if (r->is_active()) {
1462       // All allocations past TAMS are implicitly live, adjust the region data.
1463       // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
1464       HeapWord *tams = _ctx->top_at_mark_start(r);
1465       HeapWord *top = r->top();
1466       if (top > tams) {
1467         r->increase_live_data_alloc_words(pointer_delta(top, tams));
1468       }
1469 
1470       // We are about to select the collection set, make sure it knows about
1471       // current pinning status. Also, this allows trashing more regions that
1472       // now have their pinning status dropped.
1473       if (r->is_pinned()) {
1474         if (r->pin_count() == 0) {
1475           ShenandoahHeapLocker locker(_lock);
1476           r->make_unpinned();
1477         }
1478       } else {
1479         if (r->pin_count() > 0) {
1480           ShenandoahHeapLocker locker(_lock);
1481           r->make_pinned();
1482         }
1483       }
1484 
1485       // Remember limit for updating refs. It's guaranteed that we get no
1486       // from-space-refs written from here on.
1487       r->set_update_watermark(r->top());
1488     } else {
1489       assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
1490       assert(_ctx->top_at_mark_start(r) == r->top(),
1491              "Region " SIZE_FORMAT " should have correct TAMS", r->index());
1492     }
1493   }
1494 
1495   bool is_thread_safe() { return true; }
1496 };
1497 
1498 void ShenandoahHeap::op_final_mark() {
1499   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1500   assert(!has_forwarded_objects(), "No forwarded objects on this path");
1501 
1502   // It is critical that we
1503   // evacuate roots right after finishing marking, so that we don't
1504   // get unmarked objects in the roots.
1505 
1506   if (!cancelled_gc()) {
1507     concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
1508 
1509     // Marking is completed, deactivate SATB barrier
1510     set_concurrent_mark_in_progress(false);
1511     mark_complete_marking_context();
1512 
1513     parallel_cleaning(false /* full gc*/);
1514 
1515     if (ShenandoahVerify) {
1516       verifier()->verify_roots_no_forwarded();
1517     }
1518 
1519     {
1520       ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_update_region_states);
1521       ShenandoahFinalMarkUpdateRegionStateClosure cl;
1522       parallel_heap_region_iterate(&cl);
1523 
1524       assert_pinned_region_status();
1525     }
1526 
1527     // Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
1528     // This is needed for two reasons. Strong one: new allocations would be with new freeset,
1529     // which would be outside the collection set, so no cset writes would happen there.
1530     // Weaker one: new allocations would happen past update watermark, and so less work would
1531     // be needed for reference updates (would update the large filler instead).
1532     {
1533       ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::retire_tlabs);
1534       make_parsable(true);
1535     }
1536 
1537     {
1538       ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::choose_cset);
1539       ShenandoahHeapLocker locker(lock());
1540       _collection_set->clear();
1541       heuristics()->choose_collection_set(_collection_set);
1542     }
1543 
1544     {
1545       ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_rebuild_freeset);
1546       ShenandoahHeapLocker locker(lock());
1547       _free_set->rebuild();
1548     }
1549 
1550     if (!is_degenerated_gc_in_progress()) {
1551       prepare_concurrent_roots();
1552       prepare_concurrent_unloading();
1553     }
1554 
1555     // If collection set has candidates, start evacuation.
1556     // Otherwise, bypass the rest of the cycle.
1557     if (!collection_set()->is_empty()) {
1558       ShenandoahGCSubPhase init_evac(ShenandoahPhaseTimings::init_evac);
1559 
1560       if (ShenandoahVerify) {
1561         verifier()->verify_before_evacuation();
1562       }
1563 
1564       set_evacuation_in_progress(true);
1565       // From here on, we need to update references.
1566       set_has_forwarded_objects(true);
1567 
1568       if (!is_degenerated_gc_in_progress()) {
1569         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1570           ShenandoahCodeRoots::arm_nmethods();
1571         }
1572         evacuate_and_update_roots();
1573       }
1574 
1575       if (ShenandoahPacing) {
1576         pacer()->setup_for_evac();
1577       }
1578 
1579       if (ShenandoahVerify) {
1580         ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::None;
1581         if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1582           types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::WeakRoots);
1583           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CLDGRoots);
1584           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::StringDedupRoots);
1585         }
1586 
1587         if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1588           types = ShenandoahRootVerifier::combine(types, ShenandoahRootVerifier::CodeRoots);
1589         }
1590         verifier()->verify_roots_no_forwarded_except(types);
1591         verifier()->verify_during_evacuation();
1592       }
1593     } else {
1594       if (ShenandoahVerify) {
1595         verifier()->verify_after_concmark();
1596       }
1597 
1598       if (VerifyAfterGC) {
1599         Universe::verify();
1600       }
1601     }
1602 
1603   } else {
1604     // If this cycle was updating references, we need to keep the has_forwarded_objects
1605     // flag on, for subsequent phases to deal with it.
1606     concurrent_mark()->cancel();
1607     set_concurrent_mark_in_progress(false);
1608 
1609     if (process_references()) {
1610       // Abandon reference processing right away: pre-cleaning must have failed.
1611       ReferenceProcessor *rp = ref_processor();
1612       rp->disable_discovery();
1613       rp->abandon_partial_discovery();
1614       rp->verify_no_references_recorded();
1615     }
1616   }
1617 }
1618 
1619 void ShenandoahHeap::op_conc_evac() {
1620   ShenandoahEvacuationTask task(this, _collection_set, true);
1621   workers()->run_task(&task);
1622 }
1623 
1624 void ShenandoahHeap::op_stw_evac() {
1625   ShenandoahEvacuationTask task(this, _collection_set, false);
1626   workers()->run_task(&task);
1627 }
1628 
1629 void ShenandoahHeap::op_updaterefs() {
1630   update_heap_references(true);
1631 }
1632 
1633 void ShenandoahHeap::op_cleanup_early() {
1634   free_set()->recycle_trash();
1635 }
1636 
1637 void ShenandoahHeap::op_cleanup_complete() {
1638   free_set()->recycle_trash();
1639 }
1640 
1641 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
1642 private:
1643   ShenandoahVMRoots<true /*concurrent*/>        _vm_roots;
1644   ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /*single threaded*/> _cld_roots;
1645   ShenandoahConcurrentStringDedupRoots          _dedup_roots;
1646 
1647 public:
1648   ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1649     AbstractGangTask("Shenandoah Evacuate/Update Concurrent Strong Roots Task"),
1650     _vm_roots(phase),
1651     _cld_roots(phase) {}
1652 
1653   void work(uint worker_id) {
1654     ShenandoahEvacOOMScope oom;
1655     {
1656       // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1657       // may race against OopStorage::release() calls.
1658       ShenandoahEvacUpdateOopStorageRootsClosure cl;
1659       _vm_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl, worker_id);
1660     }
1661 
1662     {
1663       ShenandoahEvacuateUpdateRootsClosure<> cl;
1664       CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1665       _cld_roots.cld_do(&clds, worker_id);
1666     }
1667 
1668     {
1669       ShenandoahForwardedIsAliveClosure is_alive;
1670       ShenandoahEvacuateUpdateRootsClosure<MO_RELEASE> keep_alive;
1671       _dedup_roots.oops_do(&is_alive, &keep_alive, worker_id);
1672     }
1673   }
1674 };
1675 
1676 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
1677 private:
1678   ShenandoahHeap* const _heap;
1679   ShenandoahMarkingContext* const _mark_context;
1680   bool  _evac_in_progress;
1681   Thread* const _thread;
1682   size_t  _dead_counter;
1683 
1684 public:
1685   ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
1686   void do_oop(oop* p);
1687   void do_oop(narrowOop* p);
1688 
1689   size_t dead_counter() const;
1690   void reset_dead_counter();
1691 };
1692 
1693 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
1694   _heap(ShenandoahHeap::heap()),
1695   _mark_context(ShenandoahHeap::heap()->marking_context()),
1696   _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
1697   _thread(Thread::current()),
1698   _dead_counter(0) {
1699 }
1700 
1701 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
1702   const oop obj = RawAccess<>::oop_load(p);
1703   if (!CompressedOops::is_null(obj)) {
1704     if (!_mark_context->is_marked(obj)) {
1705       shenandoah_assert_correct(p, obj);
1706       oop old = Atomic::cmpxchg(p, obj, oop(NULL));
1707       if (obj == old) {
1708         _dead_counter ++;
1709       }
1710     } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
1711       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
1712       if (resolved == obj) {
1713         resolved = _heap->evacuate_object(obj, _thread);
1714       }
1715       Atomic::cmpxchg(p, obj, resolved);
1716       assert(_heap->cancelled_gc() ||
1717              _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
1718              "Sanity");
1719     }
1720   }
1721 }
1722 
1723 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
1724   ShouldNotReachHere();
1725 }
1726 
1727 size_t ShenandoahEvacUpdateCleanupOopStorageRootsClosure::dead_counter() const {
1728   return _dead_counter;
1729 }
1730 
1731 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::reset_dead_counter() {
1732   _dead_counter = 0;
1733 }
1734 
1735 class ShenandoahIsCLDAliveClosure : public CLDClosure {
1736 public:
1737   void do_cld(ClassLoaderData* cld) {
1738     cld->is_alive();
1739   }
1740 };
1741 
1742 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
1743 public:
1744   void do_nmethod(nmethod* n) {
1745     n->is_unloading();
1746   }
1747 };
1748 
1749 // This task not only evacuates/updates marked weak roots, but also "NULL"
1750 // dead weak roots.
1751 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public AbstractGangTask {
1752 private:
1753   ShenandoahWeakRoot<true /*concurrent*/>  _jni_roots;
1754   ShenandoahWeakRoot<true /*concurrent*/>  _string_table_roots;
1755   ShenandoahWeakRoot<true /*concurrent*/>  _resolved_method_table_roots;
1756   ShenandoahWeakRoot<true /*concurrent*/>  _vm_roots;
1757 
1758   // Roots related to concurrent class unloading
1759   ShenandoahClassLoaderDataRoots<true /* concurrent */, false /* single thread*/>
1760                                            _cld_roots;
1761   ShenandoahConcurrentNMethodIterator      _nmethod_itr;
1762   bool                                     _concurrent_class_unloading;
1763 
1764 public:
1765   ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1766     AbstractGangTask("Shenandoah Concurrent Weak Root Task"),
1767     _jni_roots(OopStorageSet::jni_weak(), phase, ShenandoahPhaseTimings::JNIWeakRoots),
1768     _string_table_roots(OopStorageSet::string_table_weak(), phase, ShenandoahPhaseTimings::StringTableRoots),
1769     _resolved_method_table_roots(OopStorageSet::resolved_method_table_weak(), phase, ShenandoahPhaseTimings::ResolvedMethodTableRoots),
1770     _vm_roots(OopStorageSet::vm_weak(), phase, ShenandoahPhaseTimings::VMWeakRoots),
1771     _cld_roots(phase),
1772     _nmethod_itr(ShenandoahCodeRoots::table()),
1773     _concurrent_class_unloading(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1774     StringTable::reset_dead_counter();
1775     ResolvedMethodTable::reset_dead_counter();
1776     if (_concurrent_class_unloading) {
1777       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1778       _nmethod_itr.nmethods_do_begin();
1779     }
1780   }
1781 
1782   ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
1783     StringTable::finish_dead_counter();
1784     ResolvedMethodTable::finish_dead_counter();
1785     if (_concurrent_class_unloading) {
1786       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1787       _nmethod_itr.nmethods_do_end();
1788     }
1789   }
1790 
1791   void work(uint worker_id) {
1792     {
1793       ShenandoahEvacOOMScope oom;
1794       // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
1795       // may race against OopStorage::release() calls.
1796       ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
1797       _jni_roots.oops_do(&cl, worker_id);
1798       _vm_roots.oops_do(&cl, worker_id);
1799 
1800       cl.reset_dead_counter();
1801       _string_table_roots.oops_do(&cl, worker_id);
1802       StringTable::inc_dead_counter(cl.dead_counter());
1803 
1804       cl.reset_dead_counter();
1805       _resolved_method_table_roots.oops_do(&cl, worker_id);
1806       ResolvedMethodTable::inc_dead_counter(cl.dead_counter());
1807     }
1808 
1809     // If we are going to perform concurrent class unloading later on, we need to
1810     // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
1811     // can cleanup immediate garbage sooner.
1812     if (_concurrent_class_unloading) {
1813       // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either NULL the
1814       // CLD's holder or evacuate it.
1815       ShenandoahIsCLDAliveClosure is_cld_alive;
1816       _cld_roots.cld_do(&is_cld_alive, worker_id);
1817 
1818       // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
1819       // The closure calls nmethod->is_unloading(). The is_unloading
1820       // state is cached, therefore, during concurrent class unloading phase,
1821       // we will not touch the metadata of unloading nmethods
1822       ShenandoahIsNMethodAliveClosure is_nmethod_alive;
1823       _nmethod_itr.nmethods_do(&is_nmethod_alive);
1824     }
1825   }
1826 };
1827 
1828 void ShenandoahHeap::op_weak_roots() {
1829   if (is_concurrent_weak_root_in_progress()) {
1830     // Concurrent weak root processing
1831     ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots);
1832     workers()->run_task(&task);
1833     if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
1834       set_concurrent_weak_root_in_progress(false);
1835     }
1836   }
1837 }
1838 
1839 void ShenandoahHeap::op_class_unloading() {
1840   assert (is_concurrent_weak_root_in_progress() &&
1841           ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
1842           "Checked by caller");
1843   _unloader.unload();
1844   set_concurrent_weak_root_in_progress(false);
1845 }
1846 
1847 void ShenandoahHeap::op_strong_roots() {
1848   assert(is_concurrent_strong_root_in_progress(), "Checked by caller");
1849   ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1850   workers()->run_task(&task);
1851   set_concurrent_strong_root_in_progress(false);
1852 }
1853 
1854 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
1855 private:
1856   ShenandoahMarkingContext* const _ctx;
1857 public:
1858   ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
1859 
1860   void heap_region_do(ShenandoahHeapRegion* r) {
1861     if (r->is_active()) {
1862       // Reset live data and set TAMS optimistically. We would recheck these under the pause
1863       // anyway to capture any updates that happened since now.
1864       r->clear_live_data();
1865       _ctx->capture_top_at_mark_start(r);
1866     }
1867   }
1868 
1869   bool is_thread_safe() { return true; }
1870 };
1871 
1872 void ShenandoahHeap::op_reset() {
1873   if (ShenandoahPacing) {
1874     pacer()->setup_for_reset();
1875   }
1876   reset_mark_bitmap();
1877 
1878   ShenandoahResetUpdateRegionStateClosure cl;
1879   parallel_heap_region_iterate(&cl);
1880 }
1881 
1882 void ShenandoahHeap::op_preclean() {
1883   if (ShenandoahPacing) {
1884     pacer()->setup_for_preclean();
1885   }
1886   concurrent_mark()->preclean_weak_refs();
1887 }
1888 
1889 void ShenandoahHeap::op_full(GCCause::Cause cause) {
1890   ShenandoahMetricsSnapshot metrics;
1891   metrics.snap_before();
1892 
1893   full_gc()->do_it(cause);
1894   if (UseTLAB) {
1895     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
1896     resize_all_tlabs();
1897   }
1898 
1899   metrics.snap_after();
1900 
1901   if (metrics.is_good_progress()) {
1902     _progress_last_gc.set();
1903   } else {
1904     // Nothing to do. Tell the allocation path that we have failed to make
1905     // progress, and it can finally fail.
1906     _progress_last_gc.unset();
1907   }
1908 }
1909 
1910 void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
1911   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
1912   // GC failure via cancelled_concgc() flag. So, if we detect the failure after
1913   // some phase, we have to upgrade the Degenerate GC to Full GC.
1914 
1915   clear_cancelled_gc();
1916 
1917   ShenandoahMetricsSnapshot metrics;
1918   metrics.snap_before();
1919 
1920   switch (point) {
1921     // The cases below form the Duff's-like device: it describes the actual GC cycle,
1922     // but enters it at different points, depending on which concurrent phase had
1923     // degenerated.
1924 
1925     case _degenerated_outside_cycle:
1926       // We have degenerated from outside the cycle, which means something is bad with
1927       // the heap, most probably heavy humongous fragmentation, or we are very low on free
1928       // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
1929       // we can do the most aggressive degen cycle, which includes processing references and
1930       // class unloading, unless those features are explicitly disabled.
1931       //
1932       // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
1933       // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
1934       set_process_references(heuristics()->can_process_references());
1935       set_unload_classes(heuristics()->can_unload_classes());
1936 
1937       op_reset();
1938 
1939       op_init_mark();
1940       if (cancelled_gc()) {
1941         op_degenerated_fail();
1942         return;
1943       }
1944 
1945     case _degenerated_mark:
1946       op_final_mark();
1947       if (cancelled_gc()) {
1948         op_degenerated_fail();
1949         return;
1950       }
1951 
1952       if (!has_forwarded_objects() && ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) {
1953         // Disarm nmethods that armed for concurrent mark. On normal cycle, it would
1954         // be disarmed while conc-roots phase is running.
1955         // TODO: Call op_conc_roots() here instead
1956         ShenandoahCodeRoots::disarm_nmethods();
1957       }
1958 
1959       op_cleanup_early();
1960 
1961     case _degenerated_evac:
1962       // If heuristics thinks we should do the cycle, this flag would be set,
1963       // and we can do evacuation. Otherwise, it would be the shortcut cycle.
1964       if (is_evacuation_in_progress()) {
1965 
1966         // Degeneration under oom-evac protocol might have left some objects in
1967         // collection set un-evacuated. Restart evacuation from the beginning to
1968         // capture all objects. For all the objects that are already evacuated,
1969         // it would be a simple check, which is supposed to be fast. This is also
1970         // safe to do even without degeneration, as CSet iterator is at beginning
1971         // in preparation for evacuation anyway.
1972         //
1973         // Before doing that, we need to make sure we never had any cset-pinned
1974         // regions. This may happen if allocation failure happened when evacuating
1975         // the about-to-be-pinned object, oom-evac protocol left the object in
1976         // the collection set, and then the pin reached the cset region. If we continue
1977         // the cycle here, we would trash the cset and alive objects in it. To avoid
1978         // it, we fail degeneration right away and slide into Full GC to recover.
1979 
1980         {
1981           sync_pinned_region_status();
1982           collection_set()->clear_current_index();
1983 
1984           ShenandoahHeapRegion* r;
1985           while ((r = collection_set()->next()) != NULL) {
1986             if (r->is_pinned()) {
1987               cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
1988               op_degenerated_fail();
1989               return;
1990             }
1991           }
1992 
1993           collection_set()->clear_current_index();
1994         }
1995 
1996         op_stw_evac();
1997         if (cancelled_gc()) {
1998           op_degenerated_fail();
1999           return;
2000         }
2001       }
2002 
2003       // If heuristics thinks we should do the cycle, this flag would be set,
2004       // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
2005       if (has_forwarded_objects()) {
2006         op_init_updaterefs();
2007         if (cancelled_gc()) {
2008           op_degenerated_fail();
2009           return;
2010         }
2011       }
2012 
2013     case _degenerated_updaterefs:
2014       if (has_forwarded_objects()) {
2015         op_final_updaterefs();
2016         if (cancelled_gc()) {
2017           op_degenerated_fail();
2018           return;
2019         }
2020       }
2021 
2022       op_cleanup_complete();
2023       break;
2024 
2025     default:
2026       ShouldNotReachHere();
2027   }
2028 
2029   if (ShenandoahVerify) {
2030     verifier()->verify_after_degenerated();
2031   }
2032 
2033   if (VerifyAfterGC) {
2034     Universe::verify();
2035   }
2036 
2037   metrics.snap_after();
2038 
2039   // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
2040   // because that probably means the heap is overloaded and/or fragmented.
2041   if (!metrics.is_good_progress()) {
2042     _progress_last_gc.unset();
2043     cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
2044     op_degenerated_futile();
2045   } else {
2046     _progress_last_gc.set();
2047   }
2048 }
2049 
2050 void ShenandoahHeap::op_degenerated_fail() {
2051   log_info(gc)("Cannot finish degeneration, upgrading to Full GC");
2052   shenandoah_policy()->record_degenerated_upgrade_to_full();
2053   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2054 }
2055 
2056 void ShenandoahHeap::op_degenerated_futile() {
2057   shenandoah_policy()->record_degenerated_upgrade_to_full();
2058   op_full(GCCause::_shenandoah_upgrade_to_full_gc);
2059 }
2060 
2061 void ShenandoahHeap::force_satb_flush_all_threads() {
2062   if (!is_concurrent_mark_in_progress()) {
2063     // No need to flush SATBs
2064     return;
2065   }
2066 
2067   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2068     ShenandoahThreadLocalData::set_force_satb_flush(t, true);
2069   }
2070   // The threads are not "acquiring" their thread-local data, but it does not
2071   // hurt to "release" the updates here anyway.
2072   OrderAccess::fence();
2073 }
2074 
2075 void ShenandoahHeap::set_gc_state_all_threads(char state) {
2076   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
2077     ShenandoahThreadLocalData::set_gc_state(t, state);
2078   }
2079 }
2080 
2081 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
2082   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
2083   _gc_state.set_cond(mask, value);
2084   set_gc_state_all_threads(_gc_state.raw_value());
2085 }
2086 
2087 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
2088   if (has_forwarded_objects()) {
2089     set_gc_state_mask(MARKING | UPDATEREFS, in_progress);
2090   } else {
2091     set_gc_state_mask(MARKING, in_progress);
2092   }
2093   ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
2094 }
2095 
2096 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2097   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2098   set_gc_state_mask(EVACUATION, in_progress);
2099 }
2100 
2101 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
2102   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2103   if (in_progress) {
2104     _concurrent_strong_root_in_progress.set();
2105   } else {
2106     _concurrent_strong_root_in_progress.unset();
2107   }
2108 }
2109 
2110 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool in_progress) {
2111   assert(ShenandoahConcurrentRoots::can_do_concurrent_roots(), "Why set the flag?");
2112   if (in_progress) {
2113     _concurrent_weak_root_in_progress.set();
2114   } else {
2115     _concurrent_weak_root_in_progress.unset();
2116   }
2117 }
2118 
2119 void ShenandoahHeap::ref_processing_init() {
2120   assert(_max_workers > 0, "Sanity");
2121 
2122   bool mt_processing = ParallelRefProcEnabled && (ParallelGCThreads > 1);
2123   bool mt_discovery = _max_workers > 1;
2124 
2125   _ref_processor =
2126     new ReferenceProcessor(&_subject_to_discovery,  // is_subject_to_discovery
2127                            mt_processing,           // MT processing
2128                            _max_workers,            // Degree of MT processing
2129                            mt_discovery,            // MT discovery
2130                            _max_workers,            // Degree of MT discovery
2131                            false,                   // Reference discovery is not atomic
2132                            NULL,                    // No closure, should be installed before use
2133                            true);                   // Scale worker threads
2134 
2135   log_info(gc, init)("Reference processing: %s discovery, %s processing",
2136           mt_discovery ? "parallel" : "serial",
2137           mt_processing ? "parallel" : "serial");
2138 
2139   shenandoah_assert_rp_isalive_not_installed();
2140 }
2141 
2142 GCTracer* ShenandoahHeap::tracer() {
2143   return shenandoah_policy()->tracer();
2144 }
2145 
2146 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2147   return _free_set->used();
2148 }
2149 
2150 bool ShenandoahHeap::try_cancel_gc() {
2151   while (true) {
2152     jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
2153     if (prev == CANCELLABLE) return true;
2154     else if (prev == CANCELLED) return false;
2155     assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
2156     assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
2157     if (Thread::current()->is_Java_thread()) {
2158       // We need to provide a safepoint here, otherwise we might
2159       // spin forever if a SP is pending.
2160       ThreadBlockInVM sp(JavaThread::current());
2161       SpinPause();
2162     }
2163   }
2164 }
2165 
2166 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2167   if (try_cancel_gc()) {
2168     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2169     log_info(gc)("%s", msg.buffer());
2170     Events::log(Thread::current(), "%s", msg.buffer());
2171   }
2172 }
2173 
2174 uint ShenandoahHeap::max_workers() {
2175   return _max_workers;
2176 }
2177 
2178 void ShenandoahHeap::stop() {
2179   // The shutdown sequence should be able to terminate when GC is running.
2180 
2181   // Step 0. Notify policy to disable event recording.
2182   _shenandoah_policy->record_shutdown();
2183 
2184   // Step 1. Notify control thread that we are in shutdown.
2185   // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2186   // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2187   control_thread()->prepare_for_graceful_shutdown();
2188 
2189   // Step 2. Notify GC workers that we are cancelling GC.
2190   cancel_gc(GCCause::_shenandoah_stop_vm);
2191 
2192   // Step 3. Wait until GC worker exits normally.
2193   control_thread()->stop();
2194 
2195   // Step 4. Stop String Dedup thread if it is active
2196   if (ShenandoahStringDedup::is_enabled()) {
2197     ShenandoahStringDedup::stop();
2198   }
2199 }
2200 
2201 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
2202   if (!unload_classes()) return;
2203 
2204   // Unload classes and purge SystemDictionary.
2205   {
2206     ShenandoahGCSubPhase phase(full_gc ?
2207                                ShenandoahPhaseTimings::full_gc_purge_class_unload :
2208                                ShenandoahPhaseTimings::purge_class_unload);
2209     bool purged_class = SystemDictionary::do_unloading(gc_timer());
2210 
2211     ShenandoahIsAliveSelector is_alive;
2212     uint num_workers = _workers->active_workers();
2213     ShenandoahClassUnloadingTask unlink_task(is_alive.is_alive_closure(), num_workers, purged_class);
2214     _workers->run_task(&unlink_task);
2215   }
2216 
2217   {
2218     ShenandoahGCSubPhase phase(full_gc ?
2219                                ShenandoahPhaseTimings::full_gc_purge_cldg :
2220                                ShenandoahPhaseTimings::purge_cldg);
2221     ClassLoaderDataGraph::purge();
2222   }
2223   // Resize and verify metaspace
2224   MetaspaceGC::compute_new_size();
2225   MetaspaceUtils::verify_metrics();
2226 }
2227 
2228 // Weak roots are either pre-evacuated (final mark) or updated (final updaterefs),
2229 // so they should not have forwarded oops.
2230 // However, we do need to "null" dead oops in the roots, if can not be done
2231 // in concurrent cycles.
2232 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
2233   ShenandoahGCSubPhase root_phase(full_gc ?
2234                                   ShenandoahPhaseTimings::full_gc_purge :
2235                                   ShenandoahPhaseTimings::purge);
2236   uint num_workers = _workers->active_workers();
2237   ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
2238                                                ShenandoahPhaseTimings::full_gc_purge_weak_par :
2239                                                ShenandoahPhaseTimings::purge_weak_par;
2240   ShenandoahGCSubPhase phase(timing_phase);
2241   ShenandoahGCWorkerPhase worker_phase(timing_phase);
2242 
2243   // Cleanup weak roots
2244   if (has_forwarded_objects()) {
2245     ShenandoahForwardedIsAliveClosure is_alive;
2246     ShenandoahUpdateRefsClosure keep_alive;
2247     ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
2248       cleaning_task(timing_phase, &is_alive, &keep_alive, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2249     _workers->run_task(&cleaning_task);
2250   } else {
2251     ShenandoahIsAliveClosure is_alive;
2252 #ifdef ASSERT
2253     ShenandoahAssertNotForwardedClosure verify_cl;
2254     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
2255       cleaning_task(timing_phase, &is_alive, &verify_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2256 #else
2257     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
2258       cleaning_task(timing_phase, &is_alive, &do_nothing_cl, num_workers, !ShenandoahConcurrentRoots::should_do_concurrent_class_unloading());
2259 #endif
2260     _workers->run_task(&cleaning_task);
2261   }
2262 }
2263 
2264 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
2265   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2266   stw_process_weak_roots(full_gc);
2267   if (!ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2268     stw_unload_classes(full_gc);
2269   }
2270 }
2271 
2272 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2273   set_gc_state_mask(HAS_FORWARDED, cond);
2274 }
2275 
2276 void ShenandoahHeap::set_process_references(bool pr) {
2277   _process_references.set_cond(pr);
2278 }
2279 
2280 void ShenandoahHeap::set_unload_classes(bool uc) {
2281   _unload_classes.set_cond(uc);
2282 }
2283 
2284 bool ShenandoahHeap::process_references() const {
2285   return _process_references.is_set();
2286 }
2287 
2288 bool ShenandoahHeap::unload_classes() const {
2289   return _unload_classes.is_set();
2290 }
2291 
2292 address ShenandoahHeap::in_cset_fast_test_addr() {
2293   ShenandoahHeap* heap = ShenandoahHeap::heap();
2294   assert(heap->collection_set() != NULL, "Sanity");
2295   return (address) heap->collection_set()->biased_map_address();
2296 }
2297 
2298 address ShenandoahHeap::cancelled_gc_addr() {
2299   return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of();
2300 }
2301 
2302 address ShenandoahHeap::gc_state_addr() {
2303   return (address) ShenandoahHeap::heap()->_gc_state.addr_of();
2304 }
2305 
2306 size_t ShenandoahHeap::bytes_allocated_since_gc_start() {
2307   return Atomic::load_acquire(&_bytes_allocated_since_gc_start);
2308 }
2309 
2310 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2311   Atomic::release_store_fence(&_bytes_allocated_since_gc_start, (size_t)0);
2312 }
2313 
2314 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2315   _degenerated_gc_in_progress.set_cond(in_progress);
2316 }
2317 
2318 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2319   _full_gc_in_progress.set_cond(in_progress);
2320 }
2321 
2322 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2323   assert (is_full_gc_in_progress(), "should be");
2324   _full_gc_move_in_progress.set_cond(in_progress);
2325 }
2326 
2327 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2328   set_gc_state_mask(UPDATEREFS, in_progress);
2329 }
2330 
2331 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2332   ShenandoahCodeRoots::register_nmethod(nm);
2333 }
2334 
2335 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2336   ShenandoahCodeRoots::unregister_nmethod(nm);
2337 }
2338 
2339 void ShenandoahHeap::flush_nmethod(nmethod* nm) {
2340   ShenandoahCodeRoots::flush_nmethod(nm);
2341 }
2342 
2343 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2344   heap_region_containing(o)->record_pin();
2345   return o;
2346 }
2347 
2348 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2349   heap_region_containing(o)->record_unpin();
2350 }
2351 
2352 void ShenandoahHeap::sync_pinned_region_status() {
2353   ShenandoahHeapLocker locker(lock());
2354 
2355   for (size_t i = 0; i < num_regions(); i++) {
2356     ShenandoahHeapRegion *r = get_region(i);
2357     if (r->is_active()) {
2358       if (r->is_pinned()) {
2359         if (r->pin_count() == 0) {
2360           r->make_unpinned();
2361         }
2362       } else {
2363         if (r->pin_count() > 0) {
2364           r->make_pinned();
2365         }
2366       }
2367     }
2368   }
2369 
2370   assert_pinned_region_status();
2371 }
2372 
2373 #ifdef ASSERT
2374 void ShenandoahHeap::assert_pinned_region_status() {
2375   for (size_t i = 0; i < num_regions(); i++) {
2376     ShenandoahHeapRegion* r = get_region(i);
2377     assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2378            "Region " SIZE_FORMAT " pinning status is inconsistent", i);
2379   }
2380 }
2381 #endif
2382 
2383 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2384   return _gc_timer;
2385 }
2386 
2387 void ShenandoahHeap::prepare_concurrent_roots() {
2388   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2389   if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
2390     set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2391     set_concurrent_weak_root_in_progress(true);
2392   }
2393 }
2394 
2395 void ShenandoahHeap::prepare_concurrent_unloading() {
2396   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2397   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2398     _unloader.prepare();
2399   }
2400 }
2401 
2402 void ShenandoahHeap::finish_concurrent_unloading() {
2403   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2404   if (ShenandoahConcurrentRoots::should_do_concurrent_class_unloading()) {
2405     _unloader.finish();
2406   }
2407 }
2408 
2409 #ifdef ASSERT
2410 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2411   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2412 
2413   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2414     if (UseDynamicNumberOfGCThreads) {
2415       assert(nworkers <= ParallelGCThreads, "Cannot use more than it has");
2416     } else {
2417       // Use ParallelGCThreads inside safepoints
2418       assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads within safepoints");
2419     }
2420   } else {
2421     if (UseDynamicNumberOfGCThreads) {
2422       assert(nworkers <= ConcGCThreads, "Cannot use more than it has");
2423     } else {
2424       // Use ConcGCThreads outside safepoints
2425       assert(nworkers == ConcGCThreads, "Use ConcGCThreads outside safepoints");
2426     }
2427   }
2428 }
2429 #endif
2430 
2431 ShenandoahVerifier* ShenandoahHeap::verifier() {
2432   guarantee(ShenandoahVerify, "Should be enabled");
2433   assert (_verifier != NULL, "sanity");
2434   return _verifier;
2435 }
2436 
2437 template<class T>
2438 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2439 private:
2440   T cl;
2441   ShenandoahHeap* _heap;
2442   ShenandoahRegionIterator* _regions;
2443   bool _concurrent;
2444 public:
2445   ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool concurrent) :
2446     AbstractGangTask("Concurrent Update References Task"),
2447     cl(T()),
2448     _heap(ShenandoahHeap::heap()),
2449     _regions(regions),
2450     _concurrent(concurrent) {
2451   }
2452 
2453   void work(uint worker_id) {
2454     if (_concurrent) {
2455       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2456       ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
2457       do_work();
2458     } else {
2459       ShenandoahParallelWorkerSession worker_session(worker_id);
2460       do_work();
2461     }
2462   }
2463 
2464 private:
2465   void do_work() {
2466     ShenandoahHeapRegion* r = _regions->next();
2467     ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
2468     while (r != NULL) {
2469       HeapWord* update_watermark = r->get_update_watermark();
2470       assert (update_watermark >= r->bottom(), "sanity");
2471       if (r->is_active() && !r->is_cset()) {
2472         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2473       }
2474       if (ShenandoahPacing) {
2475         _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom()));
2476       }
2477       if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2478         return;
2479       }
2480       r = _regions->next();
2481     }
2482   }
2483 };
2484 
2485 void ShenandoahHeap::update_heap_references(bool concurrent) {
2486   ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2487   workers()->run_task(&task);
2488 }
2489 
2490 void ShenandoahHeap::op_init_updaterefs() {
2491   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2492 
2493   set_evacuation_in_progress(false);
2494 
2495   {
2496     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
2497     retire_and_reset_gclabs();
2498   }
2499 
2500   if (ShenandoahVerify) {
2501     if (!is_degenerated_gc_in_progress()) {
2502       verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2503     }
2504     verifier()->verify_before_updaterefs();
2505   }
2506 
2507   set_update_refs_in_progress(true);
2508 
2509   _update_refs_iterator.reset();
2510 
2511   if (ShenandoahPacing) {
2512     pacer()->setup_for_updaterefs();
2513   }
2514 }
2515 
2516 class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
2517 private:
2518   ShenandoahHeapLock* const _lock;
2519 
2520 public:
2521   ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {}
2522 
2523   void heap_region_do(ShenandoahHeapRegion* r) {
2524     // Drop unnecessary "pinned" state from regions that does not have CP marks
2525     // anymore, as this would allow trashing them.
2526 
2527     if (r->is_active()) {
2528       if (r->is_pinned()) {
2529         if (r->pin_count() == 0) {
2530           ShenandoahHeapLocker locker(_lock);
2531           r->make_unpinned();
2532         }
2533       } else {
2534         if (r->pin_count() > 0) {
2535           ShenandoahHeapLocker locker(_lock);
2536           r->make_pinned();
2537         }
2538       }
2539     }
2540   }
2541 
2542   bool is_thread_safe() { return true; }
2543 };
2544 
2545 void ShenandoahHeap::op_final_updaterefs() {
2546   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2547 
2548   finish_concurrent_unloading();
2549 
2550   // Check if there is left-over work, and finish it
2551   if (_update_refs_iterator.has_next()) {
2552     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work);
2553 
2554     // Finish updating references where we left off.
2555     clear_cancelled_gc();
2556     update_heap_references(false);
2557   }
2558 
2559   // Clear cancelled GC, if set. On cancellation path, the block before would handle
2560   // everything. On degenerated paths, cancelled gc would not be set anyway.
2561   if (cancelled_gc()) {
2562     clear_cancelled_gc();
2563   }
2564   assert(!cancelled_gc(), "Should have been done right before");
2565 
2566   if (ShenandoahVerify && !is_degenerated_gc_in_progress()) {
2567     verifier()->verify_roots_in_to_space_except(ShenandoahRootVerifier::ThreadRoots);
2568   }
2569 
2570   if (is_degenerated_gc_in_progress()) {
2571     concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
2572   } else {
2573     concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::final_update_refs_roots);
2574   }
2575 
2576   // Has to be done before cset is clear
2577   if (ShenandoahVerify) {
2578     verifier()->verify_roots_in_to_space();
2579   }
2580 
2581   {
2582     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_update_refs_update_region_states);
2583     ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl;
2584     parallel_heap_region_iterate(&cl);
2585 
2586     assert_pinned_region_status();
2587   }
2588 
2589   {
2590     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
2591     trash_cset_regions();
2592   }
2593 
2594   set_has_forwarded_objects(false);
2595   set_update_refs_in_progress(false);
2596 
2597   if (ShenandoahVerify) {
2598     verifier()->verify_after_updaterefs();
2599   }
2600 
2601   if (VerifyAfterGC) {
2602     Universe::verify();
2603   }
2604 
2605   {
2606     ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::final_update_refs_rebuild_freeset);
2607     ShenandoahHeapLocker locker(lock());
2608     _free_set->rebuild();
2609   }
2610 }
2611 
2612 void ShenandoahHeap::print_extended_on(outputStream *st) const {
2613   print_on(st);
2614   print_heap_regions_on(st);
2615 }
2616 
2617 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2618   size_t slice = r->index() / _bitmap_regions_per_slice;
2619 
2620   size_t regions_from = _bitmap_regions_per_slice * slice;
2621   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2622   for (size_t g = regions_from; g < regions_to; g++) {
2623     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2624     if (skip_self && g == r->index()) continue;
2625     if (get_region(g)->is_committed()) {
2626       return true;
2627     }
2628   }
2629   return false;
2630 }
2631 
2632 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2633   shenandoah_assert_heaplocked();
2634 
2635   // Bitmaps in special regions do not need commits
2636   if (_bitmap_region_special) {
2637     return true;
2638   }
2639 
2640   if (is_bitmap_slice_committed(r, true)) {
2641     // Some other region from the group is already committed, meaning the bitmap
2642     // slice is already committed, we exit right away.
2643     return true;
2644   }
2645 
2646   // Commit the bitmap slice:
2647   size_t slice = r->index() / _bitmap_regions_per_slice;
2648   size_t off = _bitmap_bytes_per_slice * slice;
2649   size_t len = _bitmap_bytes_per_slice;
2650   if (!os::commit_memory((char*)_bitmap_region.start() + off, len, false)) {
2651     return false;
2652   }
2653   return true;
2654 }
2655 
2656 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2657   shenandoah_assert_heaplocked();
2658 
2659   // Bitmaps in special regions do not need uncommits
2660   if (_bitmap_region_special) {
2661     return true;
2662   }
2663 
2664   if (is_bitmap_slice_committed(r, true)) {
2665     // Some other region from the group is still committed, meaning the bitmap
2666     // slice is should stay committed, exit right away.
2667     return true;
2668   }
2669 
2670   // Uncommit the bitmap slice:
2671   size_t slice = r->index() / _bitmap_regions_per_slice;
2672   size_t off = _bitmap_bytes_per_slice * slice;
2673   size_t len = _bitmap_bytes_per_slice;
2674   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2675     return false;
2676   }
2677   return true;
2678 }
2679 
2680 void ShenandoahHeap::safepoint_synchronize_begin() {
2681   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2682     SuspendibleThreadSet::synchronize();
2683   }
2684 }
2685 
2686 void ShenandoahHeap::safepoint_synchronize_end() {
2687   if (ShenandoahSuspendibleWorkers || UseStringDeduplication) {
2688     SuspendibleThreadSet::desynchronize();
2689   }
2690 }
2691 
2692 void ShenandoahHeap::vmop_entry_init_mark() {
2693   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2694   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark_gross);
2695 
2696   try_inject_alloc_failure();
2697   VM_ShenandoahInitMark op;
2698   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
2699 }
2700 
2701 void ShenandoahHeap::vmop_entry_final_mark() {
2702   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2703   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark_gross);
2704 
2705   try_inject_alloc_failure();
2706   VM_ShenandoahFinalMarkStartEvac op;
2707   VMThread::execute(&op); // jump to entry_final_mark under safepoint
2708 }
2709 
2710 void ShenandoahHeap::vmop_entry_init_updaterefs() {
2711   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2712   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_gross);
2713 
2714   try_inject_alloc_failure();
2715   VM_ShenandoahInitUpdateRefs op;
2716   VMThread::execute(&op);
2717 }
2718 
2719 void ShenandoahHeap::vmop_entry_final_updaterefs() {
2720   TraceCollectorStats tcs(monitoring_support()->stw_collection_counters());
2721   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_gross);
2722 
2723   try_inject_alloc_failure();
2724   VM_ShenandoahFinalUpdateRefs op;
2725   VMThread::execute(&op);
2726 }
2727 
2728 void ShenandoahHeap::vmop_entry_full(GCCause::Cause cause) {
2729   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2730   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_gross);
2731 
2732   try_inject_alloc_failure();
2733   VM_ShenandoahFullGC op(cause);
2734   VMThread::execute(&op);
2735 }
2736 
2737 void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) {
2738   TraceCollectorStats tcs(monitoring_support()->full_stw_collection_counters());
2739   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_gross);
2740 
2741   VM_ShenandoahDegeneratedGC degenerated_gc((int)point);
2742   VMThread::execute(&degenerated_gc);
2743 }
2744 
2745 void ShenandoahHeap::entry_init_mark() {
2746   const char* msg = init_mark_event_message();
2747   ShenandoahPausePhase gc_phase(msg);
2748   EventMark em("%s", msg);
2749 
2750   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_mark);
2751 
2752   ShenandoahWorkerScope scope(workers(),
2753                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
2754                               "init marking");
2755 
2756   op_init_mark();
2757 }
2758 
2759 void ShenandoahHeap::entry_final_mark() {
2760   const char* msg = final_mark_event_message();
2761   ShenandoahPausePhase gc_phase(msg);
2762   EventMark em("%s", msg);
2763 
2764   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_mark);
2765 
2766   ShenandoahWorkerScope scope(workers(),
2767                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
2768                               "final marking");
2769 
2770   op_final_mark();
2771 }
2772 
2773 void ShenandoahHeap::entry_init_updaterefs() {
2774   static const char* msg = "Pause Init Update Refs";
2775   ShenandoahPausePhase gc_phase(msg);
2776   EventMark em("%s", msg);
2777 
2778   ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs);
2779 
2780   // No workers used in this phase, no setup required
2781 
2782   op_init_updaterefs();
2783 }
2784 
2785 void ShenandoahHeap::entry_final_updaterefs() {
2786   static const char* msg = "Pause Final Update Refs";
2787   ShenandoahPausePhase gc_phase(msg);
2788   EventMark em("%s", msg);
2789 
2790   ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs);
2791 
2792   ShenandoahWorkerScope scope(workers(),
2793                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
2794                               "final reference update");
2795 
2796   op_final_updaterefs();
2797 }
2798 
2799 void ShenandoahHeap::entry_full(GCCause::Cause cause) {
2800   static const char* msg = "Pause Full";
2801   ShenandoahPausePhase gc_phase(msg, true /* log_heap_usage */);
2802   EventMark em("%s", msg);
2803 
2804   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc);
2805 
2806   ShenandoahWorkerScope scope(workers(),
2807                               ShenandoahWorkerPolicy::calc_workers_for_fullgc(),
2808                               "full gc");
2809 
2810   op_full(cause);
2811 }
2812 
2813 void ShenandoahHeap::entry_degenerated(int point) {
2814   ShenandoahDegenPoint dpoint = (ShenandoahDegenPoint)point;
2815   const char* msg = degen_event_message(dpoint);
2816   ShenandoahPausePhase gc_phase(msg, true /* log_heap_usage */);
2817   EventMark em("%s", msg);
2818 
2819   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc);
2820 
2821   ShenandoahWorkerScope scope(workers(),
2822                               ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
2823                               "stw degenerated gc");
2824 
2825   set_degenerated_gc_in_progress(true);
2826   op_degenerated(dpoint);
2827   set_degenerated_gc_in_progress(false);
2828 }
2829 
2830 void ShenandoahHeap::entry_mark() {
2831   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2832 
2833   const char* msg = conc_mark_event_message();
2834   ShenandoahConcurrentPhase gc_phase(msg);
2835   EventMark em("%s", msg);
2836 
2837   ShenandoahGCPhase conc_mark_phase(ShenandoahPhaseTimings::conc_mark);
2838 
2839   ShenandoahWorkerScope scope(workers(),
2840                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
2841                               "concurrent marking");
2842 
2843   try_inject_alloc_failure();
2844   op_mark();
2845 }
2846 
2847 void ShenandoahHeap::entry_evac() {
2848   TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters());
2849 
2850   static const char* msg = "Concurrent evacuation";
2851   ShenandoahConcurrentPhase gc_phase(msg);
2852   EventMark em("%s", msg);
2853 
2854   ShenandoahGCPhase conc_evac_phase(ShenandoahPhaseTimings::conc_evac);
2855 
2856   ShenandoahWorkerScope scope(workers(),
2857                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
2858                               "concurrent evacuation");
2859 
2860   try_inject_alloc_failure();
2861   op_conc_evac();
2862 }
2863 
2864 void ShenandoahHeap::entry_updaterefs() {
2865   static const char* msg = "Concurrent update references";
2866   ShenandoahConcurrentPhase gc_phase(msg);
2867   EventMark em("%s", msg);
2868 
2869   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2870 
2871   ShenandoahWorkerScope scope(workers(),
2872                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2873                               "concurrent reference update");
2874 
2875   try_inject_alloc_failure();
2876   op_updaterefs();
2877 }
2878 
2879 void ShenandoahHeap::entry_weak_roots() {
2880   static const char* msg = "Concurrent weak roots";
2881   ShenandoahConcurrentPhase gc_phase(msg);
2882   EventMark em("%s", msg);
2883 
2884   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_weak_roots);
2885   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots);
2886 
2887   ShenandoahWorkerScope scope(workers(),
2888                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2889                               "concurrent weak root");
2890 
2891   try_inject_alloc_failure();
2892   op_weak_roots();
2893 }
2894 
2895 void ShenandoahHeap::entry_class_unloading() {
2896   static const char* msg = "Concurrent class unloading";
2897   ShenandoahConcurrentPhase gc_phase(msg);
2898   EventMark em("%s", msg);
2899 
2900   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_class_unloading);
2901 
2902   ShenandoahWorkerScope scope(workers(),
2903                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2904                               "concurrent class unloading");
2905 
2906   try_inject_alloc_failure();
2907   op_class_unloading();
2908 }
2909 
2910 void ShenandoahHeap::entry_strong_roots() {
2911   static const char* msg = "Concurrent strong roots";
2912   ShenandoahConcurrentPhase gc_phase(msg);
2913   EventMark em("%s", msg);
2914 
2915   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_strong_roots);
2916   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
2917 
2918   ShenandoahWorkerScope scope(workers(),
2919                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2920                               "concurrent strong root");
2921 
2922   try_inject_alloc_failure();
2923   op_strong_roots();
2924 }
2925 
2926 void ShenandoahHeap::entry_cleanup_early() {
2927   static const char* msg = "Concurrent cleanup";
2928   ShenandoahConcurrentPhase gc_phase(msg,  true /* log_heap_usage */);
2929   EventMark em("%s", msg);
2930 
2931   ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::conc_cleanup_early);
2932 
2933   // This phase does not use workers, no need for setup
2934 
2935   try_inject_alloc_failure();
2936   op_cleanup_early();
2937 }
2938 
2939 void ShenandoahHeap::entry_cleanup_complete() {
2940   static const char* msg = "Concurrent cleanup";
2941   ShenandoahConcurrentPhase gc_phase(msg,  true /* log_heap_usage */);
2942   EventMark em("%s", msg);
2943 
2944   ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::conc_cleanup_complete);
2945 
2946   // This phase does not use workers, no need for setup
2947 
2948   try_inject_alloc_failure();
2949   op_cleanup_complete();
2950 }
2951 
2952 void ShenandoahHeap::entry_reset() {
2953   static const char* msg = "Concurrent reset";
2954   ShenandoahConcurrentPhase gc_phase(msg);
2955   EventMark em("%s", msg);
2956 
2957   ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::conc_reset);
2958 
2959   ShenandoahWorkerScope scope(workers(),
2960                               ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
2961                               "concurrent reset");
2962 
2963   try_inject_alloc_failure();
2964   op_reset();
2965 }
2966 
2967 void ShenandoahHeap::entry_preclean() {
2968   if (ShenandoahPreclean && process_references()) {
2969     static const char* msg = "Concurrent precleaning";
2970     ShenandoahConcurrentPhase gc_phase(msg);
2971     EventMark em("%s", msg);
2972 
2973     ShenandoahGCSubPhase conc_preclean(ShenandoahPhaseTimings::conc_preclean);
2974 
2975     ShenandoahWorkerScope scope(workers(),
2976                                 ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
2977                                 "concurrent preclean",
2978                                 /* check_workers = */ false);
2979 
2980     try_inject_alloc_failure();
2981     op_preclean();
2982   }
2983 }
2984 
2985 void ShenandoahHeap::entry_uncommit(double shrink_before) {
2986   static const char *msg = "Concurrent uncommit";
2987   ShenandoahConcurrentPhase gc_phase(msg, true /* log_heap_usage */);
2988   EventMark em("%s", msg);
2989 
2990   ShenandoahGCSubPhase phase(ShenandoahPhaseTimings::conc_uncommit);
2991 
2992   op_uncommit(shrink_before);
2993 }
2994 
2995 void ShenandoahHeap::try_inject_alloc_failure() {
2996   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2997     _inject_alloc_failure.set();
2998     os::naked_short_sleep(1);
2999     if (cancelled_gc()) {
3000       log_info(gc)("Allocation failure was successfully injected");
3001     }
3002   }
3003 }
3004 
3005 bool ShenandoahHeap::should_inject_alloc_failure() {
3006   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
3007 }
3008 
3009 void ShenandoahHeap::initialize_serviceability() {
3010   _memory_pool = new ShenandoahMemoryPool(this);
3011   _cycle_memory_manager.add_pool(_memory_pool);
3012   _stw_memory_manager.add_pool(_memory_pool);
3013 }
3014 
3015 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
3016   GrowableArray<GCMemoryManager*> memory_managers(2);
3017   memory_managers.append(&_cycle_memory_manager);
3018   memory_managers.append(&_stw_memory_manager);
3019   return memory_managers;
3020 }
3021 
3022 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
3023   GrowableArray<MemoryPool*> memory_pools(1);
3024   memory_pools.append(_memory_pool);
3025   return memory_pools;
3026 }
3027 
3028 MemoryUsage ShenandoahHeap::memory_usage() {
3029   return _memory_pool->get_memory_usage();
3030 }
3031 
3032 void ShenandoahHeap::enter_evacuation() {
3033   _oom_evac_handler.enter_evacuation();
3034 }
3035 
3036 void ShenandoahHeap::leave_evacuation() {
3037   _oom_evac_handler.leave_evacuation();
3038 }
3039 
3040 ShenandoahRegionIterator::ShenandoahRegionIterator() :
3041   _heap(ShenandoahHeap::heap()),
3042   _index(0) {}
3043 
3044 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
3045   _heap(heap),
3046   _index(0) {}
3047 
3048 void ShenandoahRegionIterator::reset() {
3049   _index = 0;
3050 }
3051 
3052 bool ShenandoahRegionIterator::has_next() const {
3053   return _index < _heap->num_regions();
3054 }
3055 
3056 char ShenandoahHeap::gc_state() const {
3057   return _gc_state.raw_value();
3058 }
3059 
3060 void ShenandoahHeap::deduplicate_string(oop str) {
3061   assert(java_lang_String::is_instance(str), "invariant");
3062 
3063   if (ShenandoahStringDedup::is_enabled()) {
3064     ShenandoahStringDedup::deduplicate(str);
3065   }
3066 }
3067 
3068 const char* ShenandoahHeap::init_mark_event_message() const {
3069   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3070 
3071   bool proc_refs = process_references();
3072   bool unload_cls = unload_classes();
3073 
3074   if (proc_refs && unload_cls) {
3075     return "Pause Init Mark (process weakrefs) (unload classes)";
3076   } else if (proc_refs) {
3077     return "Pause Init Mark (process weakrefs)";
3078   } else if (unload_cls) {
3079     return "Pause Init Mark (unload classes)";
3080   } else {
3081     return "Pause Init Mark";
3082   }
3083 }
3084 
3085 const char* ShenandoahHeap::final_mark_event_message() const {
3086   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3087 
3088   bool proc_refs = process_references();
3089   bool unload_cls = unload_classes();
3090 
3091   if (proc_refs && unload_cls) {
3092     return "Pause Final Mark (process weakrefs) (unload classes)";
3093   } else if (proc_refs) {
3094     return "Pause Final Mark (process weakrefs)";
3095   } else if (unload_cls) {
3096     return "Pause Final Mark (unload classes)";
3097   } else {
3098     return "Pause Final Mark";
3099   }
3100 }
3101 
3102 const char* ShenandoahHeap::conc_mark_event_message() const {
3103   assert(!has_forwarded_objects(), "Should not have forwarded objects here");
3104 
3105   bool proc_refs = process_references();
3106   bool unload_cls = unload_classes();
3107 
3108   if (proc_refs && unload_cls) {
3109     return "Concurrent marking (process weakrefs) (unload classes)";
3110   } else if (proc_refs) {
3111     return "Concurrent marking (process weakrefs)";
3112   } else if (unload_cls) {
3113     return "Concurrent marking (unload classes)";
3114   } else {
3115     return "Concurrent marking";
3116   }
3117 }
3118 
3119 const char* ShenandoahHeap::degen_event_message(ShenandoahDegenPoint point) const {
3120   switch (point) {
3121     case _degenerated_unset:
3122       return "Pause Degenerated GC (<UNSET>)";
3123     case _degenerated_outside_cycle:
3124       return "Pause Degenerated GC (Outside of Cycle)";
3125     case _degenerated_mark:
3126       return "Pause Degenerated GC (Mark)";
3127     case _degenerated_evac:
3128       return "Pause Degenerated GC (Evacuation)";
3129     case _degenerated_updaterefs:
3130       return "Pause Degenerated GC (Update Refs)";
3131     default:
3132       ShouldNotReachHere();
3133       return "ERROR";
3134   }
3135 }
3136 
3137 ShenandoahLiveData* ShenandoahHeap::get_liveness_cache(uint worker_id) {
3138 #ifdef ASSERT
3139   assert(_liveness_cache != NULL, "sanity");
3140   assert(worker_id < _max_workers, "sanity");
3141   for (uint i = 0; i < num_regions(); i++) {
3142     assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
3143   }
3144 #endif
3145   return _liveness_cache[worker_id];
3146 }
3147 
3148 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
3149   assert(worker_id < _max_workers, "sanity");
3150   assert(_liveness_cache != NULL, "sanity");
3151   ShenandoahLiveData* ld = _liveness_cache[worker_id];
3152   for (uint i = 0; i < num_regions(); i++) {
3153     ShenandoahLiveData live = ld[i];
3154     if (live > 0) {
3155       ShenandoahHeapRegion* r = get_region(i);
3156       r->increase_live_data_gc_words(live);
3157       ld[i] = 0;
3158     }
3159   }
3160 }